From 34ac80e530bc72e5a4f36936d05db59f08909a3a Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 13 Oct 2026 14:05:24 +0800 Subject: [PATCH 01/36] Make inference sampling independent of worker scheduling --- tests/test_inference_sampling.py | 360 ++++++++++++++++++ traincheck/infer_engine.py | 70 +++- traincheck/inference_sampling.py | 54 +++ .../invariant/consistency_transient_vars.py | 17 +- traincheck/invariant/contain_relation.py | 14 +- traincheck/invariant/precondition.py | 7 +- 6 files changed, 485 insertions(+), 37 deletions(-) create mode 100644 tests/test_inference_sampling.py create mode 100644 traincheck/inference_sampling.py diff --git a/tests/test_inference_sampling.py b/tests/test_inference_sampling.py new file mode 100644 index 00000000..bfd87977 --- /dev/null +++ b/tests/test_inference_sampling.py @@ -0,0 +1,360 @@ +"""Regression coverage for task-local sampling, including actual relation paths.""" + +import importlib +import json +import multiprocessing +import os +import random +import subprocess +import sys +import threading +import unittest +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from unittest.mock import patch + +from traincheck.infer_engine import ( + InferEngine, + _find_precondition_task, + _generate_hypothesis_task, + _genhypo_worker_task, + _hypothesis_sampling_key, + _precond_worker_task, +) +from traincheck.inference_sampling import sampling_rng, sampling_task +from traincheck.invariant import relation_pool +from traincheck.invariant.base_cls import ( + APIParam, + Example, + ExampleList, + Hypothesis, + Invariant, + UnconditionalPrecondition, +) +from traincheck.invariant.contain_relation import ( + PARENT_GROUP_NAME, + APIContainRelation, + cache_events_scanner, + prune_func_call, +) +from traincheck.parallel import config_snapshot, worker_init +from traincheck.trace.trace_pandas import TracePandas +from traincheck.trace.types import FuncCallEvent + +PARENT = "torch.nn.modules.activation.ReLU.forward" +CHILD = "torch.nn.modules.dropout.Dropout.forward" + + +def nested_call_trace(prefix, calls=1005): + """Complete synthetic events, not training: >1000 calls exercises both samplers.""" + rows = [] + for index in range(calls): + events = [ + (PARENT, "parent", "pre"), + (CHILD, "child", "pre"), + (CHILD, "child", "post"), + (PARENT, "parent", "post"), + ] + for offset, (function, name, kind) in enumerate(events): + row = { + "time": index * 10 + offset + 1, + "func_call_id": f"{prefix}-{name}-{index}", + "function": function, + "type": f"function_call ({kind})", + "process_id": 1, + "thread_id": 1, + "is_bound_method": True, + "obj_id": 1, + "meta_vars": {}, + "args": {"0": {"object": {}}, "1": {"float": 1.0}}, + "kwargs": {}, + } + if kind == "post": + row["return_values"] = {"float": 1.0} + rows.append(row) + return TracePandas(rows) + + +def hypothesis_evidence(hypotheses): + """Compare exact retained call IDs/order as well as relation and parameters.""" + results = [] + for hypothesis in hypotheses: + + def examples(items): + return [ + { + name: [row["func_call_id"] for row in rows] + for name, rows in sorted(example.trace_groups.items()) + } + for example in items.examples + ] + + results.append( + { + "key": _hypothesis_sampling_key(hypothesis), + "positive": examples(hypothesis.positive_examples), + "negative": examples(hypothesis.negative_examples), + } + ) + return sorted(results, key=lambda item: item["key"]) + + +def simple_hypothesis(positive_count=3, negative_count=0): + def examples(count, prefix): + # Direct construction deliberately reaches the >5000 precondition branch; + # add_example's separate reservoir would otherwise cap the input first. + return ExampleList.from_iterable_of_examples( + [ + Example( + {PARENT_GROUP_NAME: [{"id": f"{prefix}-{i}", "function": PARENT}]} + ) + for i in range(count) + ], + {PARENT_GROUP_NAME}, + ) + + return Hypothesis( + Invariant( + APIContainRelation, [APIParam(PARENT), APIParam(CHILD)], None, "fixture" + ), + examples(positive_count, "positive"), + examples(negative_count, "negative"), + ) + + +class SamplingScopeTests(unittest.TestCase): + def test_reproducible_keys_and_untouched_global_rng(self): + before = random.getstate() + self.assertIs(sampling_rng(), random) + with sampling_task("generate", 0, "relation"): + first = [sampling_rng().random() for _ in range(20)] + self.assertEqual(random.getstate(), before) + random.random() + after_noise = random.getstate() + with sampling_task("generate", 0, "relation"): + second = [sampling_rng().random() for _ in range(20)] + with sampling_task("generate", 1, "relation"): + other = [sampling_rng().random() for _ in range(20)] + self.assertEqual(first, second) + self.assertNotEqual(first, other) + self.assertEqual(random.getstate(), after_noise) + + def test_nested_scopes_restore_on_frozen_exception(self): + @dataclass(frozen=True) + class FrozenError(Exception): + reason: str + + with sampling_task("outer") as expected_rng: + expected = [expected_rng.random() for _ in range(2)] + error = FrozenError("expected") + before = random.getstate() + with sampling_task("outer") as outer: + first = sampling_rng().random() + try: + with sampling_task("inner"): + sampling_rng().random() + raise error + except FrozenError as caught: + self.assertIs(caught, error) + self.assertIs(sampling_rng(), outer) + second = sampling_rng().random() + self.assertEqual([first, second], expected) + self.assertEqual(random.getstate(), before) + self.assertIs(sampling_rng(), random) + + def test_context_is_thread_local(self): + errors = [] + values = [] + + def worker(): + try: + self.assertIs(sampling_rng(), random) + with sampling_task("worker"): + values.append(sampling_rng().random()) + except BaseException as error: + errors.append(error) + + with sampling_task("parent") as parent: + thread = threading.Thread(target=worker) + thread.start() + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertIs(sampling_rng(), parent) + self.assertEqual(errors, []) + self.assertEqual(len(values), 1) + + def test_seed_does_not_depend_on_python_hash_seed(self): + script = ( + "import json; from traincheck.inference_sampling import sampling_task,sampling_rng; " + "\nwith sampling_task('generate', 4, {'b': 2, 'a': 1}): " + "print(json.dumps([sampling_rng().random() for _ in range(8)]))" + ) + root = str(Path(__file__).resolve().parents[1]) + outputs = [] + for seed in ("1", "98765"): + env = dict(os.environ, PYTHONHASHSEED=seed, PYTHONPATH=root) + outputs.append( + subprocess.check_output( + [sys.executable, "-c", script], env=env, text=True + ) + ) + self.assertEqual(outputs[0], outputs[1]) + + def test_actual_pruning_is_nontrivial_and_task_deterministic(self): + def retained(identity): + sizes, types, kept = [], [], [] + with sampling_task("pruning", identity): + for index in range(1500): + if not prune_func_call(1500, FuncCallEvent, sizes, types): + kept.append(index) + sizes.append(1) + types.append(FuncCallEvent) + return kept + + first = retained("same") + for _ in range(101): + random.random() + self.assertEqual(first, retained("same")) + self.assertNotEqual(first, retained("different")) + self.assertGreater(len(first), 10) + self.assertLess(len(first), 1500) + + +class InferenceTaskTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.traces = [nested_call_trace("trace0"), nested_call_trace("trace1")] + + def setUp(self): + cache_events_scanner.clear() + + def test_hypothesis_key_ignores_mutable_results_but_tracks_semantics(self): + hypothesis = simple_hypothesis() + key = _hypothesis_sampling_key(hypothesis) + hypothesis.invariant.num_positive_examples = 900 + hypothesis.invariant.num_negative_examples = 8 + hypothesis.invariant.text_description = "different display text" + hypothesis.invariant.precondition = object() + self.assertEqual(_hypothesis_sampling_key(hypothesis), key) + hypothesis.valid_only_for_stages = True + hypothesis.valid_stages = {"training", "evaluation"} + staged = _hypothesis_sampling_key(hypothesis) + self.assertNotEqual(staged, key) + hypothesis.valid_stages = set(["evaluation", "training"]) + self.assertEqual(_hypothesis_sampling_key(hypothesis), staged) + hypothesis.invariant.params.reverse() + self.assertNotEqual(_hypothesis_sampling_key(hypothesis), staged) + + def test_real_relation_retained_examples_repeat_despite_ambient_rng(self): + before = random.getstate() + hypotheses = _generate_hypothesis_task(0, APIContainRelation, self.traces[0]) + self.assertEqual(random.getstate(), before) + first = hypothesis_evidence(hypotheses) + self.assertEqual(len(first), 1) + self.assertGreater(len(first[0]["positive"]), 10) + self.assertLess(len(first[0]["positive"]), 120) + for _ in range(77): + random.random() + second = hypothesis_evidence( + _generate_hypothesis_task(0, APIContainRelation, self.traces[0]) + ) + self.assertEqual(first, second) + + def test_engine_serial_dispatches_each_relation_through_shared_task(self): + disabled = [ + relation for relation in relation_pool if relation is not APIContainRelation + ] + engine = InferEngine(self.traces, disabled_relations=disabled, num_workers=1) + with patch( + "traincheck.infer_engine._generate_hypothesis_task", + wraps=_generate_hypothesis_task, + ) as task: + hypotheses = engine.generate_hypothesis() + self.assertEqual(task.call_count, 2) + self.assertEqual([call.args[0] for call in task.call_args_list], [0, 1]) + self.assertEqual(len(hypotheses), 1) + self.assertEqual(next(iter(hypotheses.values())), [0, 1]) + + def test_actual_relation_matches_process_counts_orders_and_start_methods(self): + expected = { + index: hypothesis_evidence( + _generate_hypothesis_task(index, APIContainRelation, trace) + ) + for index, trace in enumerate(self.traces) + } + methods = [ + method + for method in ("spawn", "fork") + if method in multiprocessing.get_all_start_methods() + ] + self.assertTrue(methods) + for method in methods: + for workers in (1, 2, 4): + with self.subTest(start_method=method, workers=workers): + # Repeated tasks consume distinct worker histories. Reversing + # dispatch changes assignment without changing semantic keys. + tasks = [(index, APIContainRelation) for index in (1, 0, 1, 0)] + with ProcessPoolExecutor( + max_workers=workers, + mp_context=multiprocessing.get_context(method), + initializer=worker_init, + initargs=(self.traces, config_snapshot()), + ) as pool: + futures = [ + pool.submit(_genhypo_worker_task, task) for task in tasks + ] + for future in futures: + index, relation, hypotheses = future.result(timeout=90) + self.assertIs(relation, APIContainRelation) + self.assertEqual( + hypothesis_evidence(hypotheses), expected[index] + ) + + def test_precondition_over_5000_preserves_legacy_samples_without_global_seed(self): + precondition = importlib.import_module("traincheck.invariant.precondition") + hypothesis = simple_hypothesis(5007, 5011) + captured = [] + + def solve(positive, negative, traces, keys_to_skip): + captured.append( + ( + [item[0]["id"] for item in positive], + [item[0]["id"] for item in negative], + ) + ) + return [UnconditionalPrecondition()] + + legacy_rng = random.Random(42) + expected = ( + legacy_rng.sample([f"positive-{i}" for i in range(5007)], 5000), + legacy_rng.sample([f"negative-{i}" for i in range(5011)], 5000), + ) + before = random.getstate() + with patch.object( + precondition, "find_precondition_from_single_group", side_effect=solve + ): + result = _find_precondition_task( + hypothesis, [], _hypothesis_sampling_key(hypothesis) + ) + self.assertIsNotNone(result) + self.assertEqual(captured, [expected]) + self.assertEqual(random.getstate(), before) + + def test_precondition_worker_dispatch_matches_serial_key(self): + hypothesis = simple_hypothesis() + key = _hypothesis_sampling_key(hypothesis) + worker_init([], config_snapshot()) + + def result(*args): + return [sampling_rng().random() for _ in range(6)] + + with patch("traincheck.infer_engine.find_precondition", side_effect=result): + expected = _find_precondition_task(hypothesis, [], key) + random.random() + actual = _precond_worker_task((key, hypothesis)) + self.assertEqual(actual, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/infer_engine.py b/traincheck/infer_engine.py index 09ff0411..f5958efb 100644 --- a/traincheck/infer_engine.py +++ b/traincheck/infer_engine.py @@ -3,7 +3,6 @@ import json import logging import os -import random import time from concurrent.futures import ProcessPoolExecutor, as_completed @@ -11,6 +10,7 @@ import traincheck.config.config as config import traincheck.utils as _tc_utils +from traincheck.inference_sampling import sampling_task from traincheck.invariant import ( FailedHypothesis, Hypothesis, @@ -27,8 +27,35 @@ logger = logging.getLogger(__name__) -# set random seed -random.seed(0) + +def _generate_hypothesis_task(trace_idx, relation, trace): + """Use the same logical task RNG in serial and process-based generation.""" + with sampling_task( + "generate", trace_idx, relation.__module__, relation.__qualname__ + ): + return relation.generate_hypothesis(trace) + + +def _hypothesis_sampling_key(hypothesis): + """Identify a hypothesis before dispatch, excluding mutable results/counts.""" + relation = hypothesis.invariant.relation + identity = { + "relation": [relation.__module__, relation.__qualname__], + "params": [param.to_dict() for param in hypothesis.invariant.params], + "valid_only_for_stages": hypothesis.valid_only_for_stages, + "valid_stages": sorted(hypothesis.valid_stages), + } + # Normalize JSON object keys and MD_NONE with the invariant-output encoder, + # then sort keys for insertion-order independence. + normalized = json.loads(json.dumps(identity, cls=MDNONEJSONEncoder)) + return json.dumps( + normalized, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + + +def _find_precondition_task(hypothesis, traces, key): + with sampling_task("precondition", key): + return find_precondition(hypothesis, traces) # === parallel inference worker tasks === @@ -45,16 +72,18 @@ def _genhypo_worker_task(task: tuple): """ trace_idx, relation = task traces = get_worker_traces() - return trace_idx, relation, relation.generate_hypothesis(traces[trace_idx]) + hypotheses = _generate_hypothesis_task(trace_idx, relation, traces[trace_idx]) + return trace_idx, relation, hypotheses -def _precond_worker_task(hypothesis: Hypothesis): +def _precond_worker_task(task): """Run precondition inference for a single hypothesis in a worker process. Returns just the precondition (picklable); the parent applies it to its own copy of the hypothesis. """ - return find_precondition(hypothesis, get_worker_traces()) + key, hypothesis = task + return _find_precondition_task(hypothesis, get_worker_traces(), key) class InferEngine: @@ -152,7 +181,7 @@ def _generate_hypothesis_serial( hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], ) -> None: - """Serial hypothesis generation (preserves the original behavior exactly).""" + """Serial generation with the same per-task sampling as the worker path.""" n_traces = len(self.traces) rel_bar_fmt = "{desc} {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]" for trace_idx, trace in enumerate(self.traces): @@ -169,7 +198,9 @@ def _generate_hypothesis_serial( f"Generating hypotheses for relation: {relation.__name__}" ) t0 = time.time() - inferred_hypos = relation.generate_hypothesis(trace) + inferred_hypos = _generate_hypothesis_task( + trace_idx, relation, trace + ) elapsed = time.time() - t0 tqdm.write( f" {relation.__name__}: {len(inferred_hypos)} hypotheses ({elapsed:.1f}s)" @@ -197,7 +228,8 @@ def _generate_hypothesis_parallel( is CPU-bound and pure w.r.t. its (per-worker) trace, so we fan them out across processes. Workers only PRODUCE hypothesis lists; the merge into shared state stays in the parent. Results are merged in trace-then-relation order (identical - to the serial path) regardless of completion order, so the output is unchanged. + to the serial path) regardless of completion order. Each task also has + its own stable RNG. """ n_traces = len(self.traces) tasks = [ @@ -255,13 +287,15 @@ def collect_examples(self, hypotheses: dict[Hypothesis, list[int]]): f"\nCollecting examples for {len(cross_trace_hypos)} cross-trace hypotheses" ) for hypo, trace_idxs in cross_trace_hypos: + key = _hypothesis_sampling_key(hypo) for trace_idx, trace in enumerate(self.traces): if trace_idx in trace_idxs: continue logger.info( f"Collecting examples for hypothesis: {hypo} on trace {trace_idx + 1}/{len(self.traces)}" ) - hypo.invariant.relation.collect_examples(trace, hypo) + with sampling_task("collect", trace_idx, key): + hypo.invariant.relation.collect_examples(trace, hypo) def prune_incorrect_hypos(self, hypotheses: dict[Hypothesis, list[int]]): """Prune incorrect hypotheses based on the collected examples""" @@ -283,6 +317,8 @@ def infer_precondition(self, hypotheses: dict[Hypothesis, list[int]]): """TODO: move the precondition inference driving code into Hypothesis.get_invariant()""" logger.info("============= INFERING PRECONDITIONS =============") all_hypotheses = list(hypotheses.keys()) + # Build keys in the parent, before result mutation or worker pickling. + tasks = [(_hypothesis_sampling_key(hypo), hypo) for hypo in all_hypotheses] total = len(all_hypotheses) print(f"\nInferring preconditions for {total} hypotheses") @@ -301,17 +337,19 @@ def _record_result(hypothesis: Hypothesis, precondition): invariants.append(hypothesis.get_invariant(self.all_stages)) if self.num_workers <= 1: - # Serial path (preserves the original behavior exactly). + # Serial path uses the same task scopes as the workers. _tc_utils._suppress_inner_progress = True try: with tqdm(total=total, bar_format=bar_fmt, unit="hypo") as pbar: pbar.set_description("0 done · 0 failed") - for hypo_idx, hypothesis in enumerate(all_hypotheses): + for hypo_idx, (key, hypothesis) in enumerate(tasks): logger.info( f"Inferring precondition for hypothesis {hypo_idx + 1}/{total}: " f"{hypothesis.invariant.text_description}" ) - precondition = find_precondition(hypothesis, self.traces) + precondition = _find_precondition_task( + hypothesis, self.traces, key + ) _record_result(hypothesis, precondition) pbar.set_description( f"{len(invariants)} done · {len(failed_hypos)} failed" @@ -334,8 +372,10 @@ def _record_result(hypothesis: Hypothesis, precondition): initargs=(self.traces, config_snapshot()), ) as executor: future_to_hypo = { - executor.submit(_precond_worker_task, hypothesis): hypothesis - for hypothesis in all_hypotheses + executor.submit( + _precond_worker_task, (key, hypothesis) + ): hypothesis + for key, hypothesis in tasks } for future in as_completed(future_to_hypo): hypothesis = future_to_hypo[future] diff --git a/traincheck/inference_sampling.py b/traincheck/inference_sampling.py new file mode 100644 index 00000000..a4e26949 --- /dev/null +++ b/traincheck/inference_sampling.py @@ -0,0 +1,54 @@ +"""Task-local randomness for inference, independent of process scheduling. + +The engine supplies stable semantic task keys in both its serial and process +paths. Direct relation callers outside an engine task keep the legacy stdlib +random API, including control via random.seed(). Sampling rates and reservoirs +are unchanged; the chosen examples can differ from the old global RNG stream. +""" + +import hashlib +import json +import random +from contextvars import ContextVar + +_TASK_RNG: ContextVar[random.Random | None] = ContextVar( + "inference_task_rng", default=None +) + + +class sampling_task: + """Give one logical task a fresh reproducible RNG without touching global state. + + Keys must be JSON values with stable ordering, never Python hash(), object + addresses, worker IDs, or arrival order. Trace indexes refer to the engine's + ordered input list. A versioned digest keeps seeds stable across processes + and PYTHONHASHSEED values. ContextVar restores enclosing scopes on exceptions + and isolates concurrently executing threads. + """ + + def __init__(self, *identity): + self.identity = identity + + def __enter__(self): + payload = json.dumps( + ["traincheck-inference-sampling-v1", 0, *self.identity], + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + rng = random.Random(int.from_bytes(hashlib.sha256(payload).digest(), "big")) + self.token = _TASK_RNG.set(rng) + return rng + + def __exit__(self, exc_type, exc_value, traceback): + # A class context manager also preserves frozen exception objects; the + # generator-based contextlib helper may assign exception.__traceback__. + _TASK_RNG.reset(self.token) + return False + + +def sampling_rng(): + """Return the task's RNG, or the legacy module for standalone relation calls.""" + rng = _TASK_RNG.get() + return random if rng is None else rng diff --git a/traincheck/invariant/consistency_transient_vars.py b/traincheck/invariant/consistency_transient_vars.py index f02e65bb..eb9196ed 100644 --- a/traincheck/invariant/consistency_transient_vars.py +++ b/traincheck/invariant/consistency_transient_vars.py @@ -5,6 +5,7 @@ import pandas as pd from traincheck.config.config import ANALYSIS_SKIP_FUNC_NAMES +from traincheck.inference_sampling import sampling_rng from traincheck.instrumentor.tracer import TraceLineType from traincheck.invariant.base_cls import ( _NOT_SET, @@ -262,11 +263,9 @@ def get_events_of_funcs_with_tensors( } # sampling 1000 if more than 1000 - import random - all_func_call_ids = { func_name: ( - random.sample(func_call_ids, 1000) + sampling_rng().sample(func_call_ids, 1000) if len(func_call_ids) > 1000 else func_call_ids ) @@ -452,10 +451,8 @@ def collect_examples(trace, hypothesis): func_call_ids = trace.get_func_call_ids(func_name) # down sample to 1000 - import random - if len(func_call_ids) > 1000: - func_call_ids = random.sample(func_call_ids, 1000) + func_call_ids = sampling_rng().sample(func_call_ids, 1000) for func_call_id in tqdm( func_call_ids, desc=f"Adding examples for {inv.text_description}" @@ -867,10 +864,8 @@ def collect_examples(trace, hypothesis): func_name = api_param.api_full_name func_call_ids = trace.get_func_call_ids(func_name) - import random - if len(func_call_ids) > 1000: - func_call_ids = random.sample(func_call_ids, 1000) + func_call_ids = sampling_rng().sample(func_call_ids, 1000) for func_call_id in tqdm( func_call_ids, desc=f"Checking invariant {inv.text_description}" @@ -1360,10 +1355,8 @@ def collect_examples(trace, hypothesis): # get all function calls for the function func_call_ids = trace.get_func_call_ids(func_name) - import random - if len(func_call_ids) > 1000: - func_call_ids = random.sample(func_call_ids, 1000) + func_call_ids = sampling_rng().sample(func_call_ids, 1000) for func_call_id in tqdm( func_call_ids, desc=f"Checking invariant {inv.text_description}" diff --git a/traincheck/invariant/contain_relation.py b/traincheck/invariant/contain_relation.py index 84fa76d2..cdc82829 100644 --- a/traincheck/invariant/contain_relation.py +++ b/traincheck/invariant/contain_relation.py @@ -1,12 +1,12 @@ import copy import logging -import random import time from typing import Type import numpy as np from traincheck.config.config import ANALYSIS_SKIP_FUNC_NAMES +from traincheck.inference_sampling import sampling_rng from traincheck.instrumentor.tracer import TraceLineType from traincheck.invariant.base_cls import ( _NOT_SET, @@ -169,10 +169,10 @@ def prune_func_call( - whether the function call should be pruned or not The pruning logic is as follows: - - If the number of function calls total present in the trace is less than 1000, no pruning is done - - Pruning is probabilistic, with initial probability of (1 - 1000/total_func_calls) - - If `list_num_events_scanned` indicates that the last 10 function calls have the same number of events scanned, the pruning probability is increased by 100 times for the next function call - - We use random.random() to determine whether the function call should be pruned or not + - Initially keep all calls when there are at most 1000; otherwise keep 1000/total. + - After more than 10 scans, equal event counts in the last 10 reduce the keep + probability by 100, even below 1000 total calls. + - We use sampling_rng().random() to determine whether the function call should be pruned or not """ logger = logging.getLogger(__name__) @@ -197,7 +197,7 @@ def prune_func_call( f"Same number of events observed in the last 10 attempts: {list_num_events_scanned[0]}, increasing the pruning probability to: {1 - not_pruning_prob}" ) - is_skipping = random.random() > not_pruning_prob + is_skipping = sampling_rng().random() > not_pruning_prob if is_skipping: logger.debug( f"Skipping the function call due to sampling with a pruning probability of: {1 - not_pruning_prob}" @@ -650,7 +650,7 @@ def _infer( # down sample the function calls, keep the first 10, last 10, and randomly sample 100 from the rest parent_func_call_ids = ( parent_func_call_ids[:10] - + random.sample(parent_func_call_ids[10:-10], 100) # type: ignore + + sampling_rng().sample(parent_func_call_ids[10:-10], 100) # type: ignore + parent_func_call_ids[-10:] ) diff --git a/traincheck/invariant/precondition.py b/traincheck/invariant/precondition.py index f86ffb95..45c17f5e 100644 --- a/traincheck/invariant/precondition.py +++ b/traincheck/invariant/precondition.py @@ -306,17 +306,18 @@ def find_precondition( import random - random.seed(42) + # Preserve this group's existing sample while leaving the enclosing task RNG alone. + group_rng = random.Random(42) if len(positive_examples) > 5000: logger.warning( f"Too many positive examples found for group {group_name}, downsampling to 100000" ) - positive_examples = random.sample(positive_examples, 5000) + positive_examples = group_rng.sample(positive_examples, 5000) if len(negative_examples) > 5000: logger.warning( f"Too many negative examples found for group {group_name}, downsampling to 100000" ) - negative_examples = random.sample(negative_examples, 5000) + negative_examples = group_rng.sample(negative_examples, 5000) grouped_preconditions[group_name] = Preconditions( find_precondition_from_single_group( From 6c1a5d414137c96013acd2fc90e0e90088919de1 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:05:47 +0800 Subject: [PATCH 02/36] Fix bounded reservoir construction and population-aware merges --- tests/test_example_reservoir.py | 528 +++++++++++++++++++++++ traincheck/infer_engine.py | 40 +- traincheck/invariant/base_cls.py | 138 ++++-- traincheck/invariant/contain_relation.py | 51 ++- traincheck/parallel.py | 14 +- 5 files changed, 705 insertions(+), 66 deletions(-) create mode 100644 tests/test_example_reservoir.py diff --git a/tests/test_example_reservoir.py b/tests/test_example_reservoir.py new file mode 100644 index 00000000..d632ab0a --- /dev/null +++ b/tests/test_example_reservoir.py @@ -0,0 +1,528 @@ +"""Reservoir invariants, weighted population merges, and actual merge call sites.""" + +import multiprocessing +import random +import unittest +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from functools import partial +from unittest.mock import patch + +import traincheck.config.config as config +from traincheck.infer_engine import InferEngine +from traincheck.invariant.base_cls import ( + APIParam, + Example, + ExampleList, + GroupedPreconditions, + Hypothesis, + Invariant, + Preconditions, + UnconditionalPrecondition, +) +from traincheck.invariant.contain_relation import PARENT_GROUP_NAME, APIContainRelation +from traincheck.parallel import config_snapshot, worker_init + +GROUP = PARENT_GROUP_NAME + + +def example(value, group=GROUP): + return Example({group: [{"id": value}]}) + + +def values(reservoir): + return [item.get_group(GROUP)[0]["id"] for item in reservoir.examples] + + +def reservoir(items, cap, seed=42, groups=None): + with ( + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", cap), + patch.object(config, "EXAMPLE_SAMPLING_SEED", seed), + ): + result = ExampleList({GROUP} if groups is None else groups) + for item in items: + result.add_example(example(item)) + return result + + +def snapshot(reservoir): + return ( + reservoir.num_seen, + reservoir.max_size, + set(reservoir.group_names), + list(reservoir.examples), + reservoir._rng.getstate(), + ) + + +def hypothesis(prefix, positive, negative, cap): + precondition = GroupedPreconditions( + {GROUP: Preconditions([UnconditionalPrecondition()])} + ) + invariant = Invariant( + APIContainRelation, + [ + APIParam("torch.nn.modules.activation.ReLU.forward"), + APIParam("torch.nn.modules.dropout.Dropout.forward"), + ], + precondition, + "ReLU.forward() contains Dropout.forward()", + ) + return Hypothesis( + invariant, + reservoir([f"{prefix}-positive-{i}" for i in range(positive)], cap), + reservoir([f"{prefix}-negative-{i}" for i in range(negative)], cap), + ) + + +def worker_reservoir_result(count): + result = ExampleList({GROUP}) + for index in range(count): + result.add_example(example(index)) + return ( + result.max_size, + config.EXAMPLE_SAMPLING_SEED, + result.num_seen, + values(result), + ) + + +@dataclass +class ReservoirFixtureTrace: + """A picklable trace carrying populations, without any trace parser work.""" + + prefix: str + positive_count: int + negative_count: int + + def is_stage_annotated(self): + return False + + +class ReservoirFixtureRelation(APIContainRelation): + @classmethod + def generate_hypothesis(cls, trace): + # Read config directly here: patching construction inside the fixture + # would mask a missing config_snapshot/worker_init propagation field. + precondition = GroupedPreconditions( + {GROUP: Preconditions([UnconditionalPrecondition()])} + ) + invariant = Invariant( + cls, + [APIParam("fixture.parent"), APIParam("fixture.child")], + precondition, + "fixture.parent contains fixture.child", + ) + positive, negative = ExampleList({GROUP}), ExampleList({GROUP}) + for kind, count, examples in [ + ("positive", trace.positive_count, positive), + ("negative", trace.negative_count, negative), + ]: + for index in range(count): + examples.add_example(example(f"{trace.prefix}-{kind}-{index}")) + return [Hypothesis(invariant, positive, negative)] + + +class ExampleReservoirConstructionTests(unittest.TestCase): + def test_from_iterable_streams_through_capped_add_and_counts_every_item(self): + applied = [] + original = ExampleList.add_example + + def observed_add(result, item): + original(result, item) + applied.append(result.num_seen) + self.assertLessEqual(len(result), 3) + + def items(): + for index in range(37): + self.assertEqual( + len(applied), index, "input must be consumed incrementally" + ) + yield example(index) + + with ( + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 3), + patch.object(ExampleList, "add_example", observed_add), + ): + result = ExampleList.from_iterable_of_examples(items(), {GROUP}) + self.assertEqual(len(result), 3) + self.assertEqual(result.num_seen, 37) + self.assertEqual(applied, list(range(1, 38))) + + def test_unlimited_zero_and_empty_iterables(self): + for cap, expected in [(None, list(range(12))), (0, []), (20, list(range(12)))]: + with ( + self.subTest(cap=cap), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", cap), + ): + result = ExampleList.from_iterable_of_examples( + (example(i) for i in range(12)), {GROUP} + ) + self.assertEqual(values(result), expected) + self.assertEqual(result.num_seen, 12) + empty = ExampleList.from_iterable_of_examples(iter(()), {GROUP}) + self.assertEqual(len(empty), 0) + self.assertEqual(empty.num_seen, 0) + + def test_invalid_caps_fail_at_construction(self): + for cap in [-1, True, False, 1.5, "3"]: + with ( + self.subTest(cap=cap), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", cap), + ): + with self.assertRaises(ValueError): + ExampleList({GROUP}) + + def test_from_iterable_validates_group_membership(self): + with self.assertRaises((AssertionError, ValueError)): + ExampleList.from_iterable_of_examples([example(1, "different")], {GROUP}) + + +class ExampleReservoirMergeTests(unittest.TestCase): + def assert_rejected_without_mutation(self, left, right): + prior_left, prior_right = snapshot(left), snapshot(right) + rng = random.Random(761) + rng_before = rng.getstate() + with self.assertRaises(ValueError): + left.merge(right, rng=rng) + self.assertEqual(snapshot(left), prior_left) + self.assertEqual(snapshot(right), prior_right) + self.assertEqual(rng.getstate(), rng_before) + + def test_exact_unlimited_and_under_cap_order_without_rng_consumption(self): + for cap in [None, 10]: + with self.subTest(cap=cap): + left = reservoir(["a", "a", "b"], cap) + right = reservoir(["c", "a"], cap) + before = snapshot(right) + rng = random.Random(54) + rng_before = rng.getstate() + self.assertIsNone(left.merge(right, rng=rng)) + self.assertEqual(values(left), ["a", "a", "b", "c", "a"]) + self.assertEqual(left.num_seen, 5) + self.assertEqual(snapshot(right), before) + self.assertEqual(rng.getstate(), rng_before) + + def test_empty_destination_adopts_groups_and_empty_source_preserves_order(self): + left = reservoir([], 3, groups=set()) + right = reservoir([1, 2], 3) + left.merge(right, rng=random.Random(1)) + self.assertEqual(left.group_names, {GROUP}) + self.assertEqual(values(left), [1, 2]) + self.assertEqual(left.num_seen, 2) + + uninitialized = reservoir([], 3, groups=set()) + rng = random.Random(3) + before = rng.getstate() + uninitialized.merge(reservoir([], 3), rng=rng) + self.assertEqual(uninitialized.group_names, {GROUP}) + self.assertEqual(uninitialized.num_seen, 0) + self.assertEqual(rng.getstate(), before) + left.merge(reservoir([], 3), rng=random.Random(2)) + self.assertEqual(values(left), [1, 2]) + self.assertEqual(left.num_seen, 2) + + def test_zero_capacity_accumulates_counts_and_preserves_no_items(self): + left, right = reservoir(range(20), 0), reservoir(range(7), 0) + left.merge(right, rng=random.Random(7)) + left.add_example(example("next")) + self.assertEqual(left.num_seen, 28) + self.assertEqual(len(left), 0) + + def test_unequal_populations_are_counted_and_source_remains_unchanged(self): + left, right = reservoir(range(10), 4, 11), reservoir(range(10, 100), 4, 19) + prior_right = snapshot(right) + left.merge(right, rng=random.Random(37)) + self.assertEqual(left.num_seen, 100) + self.assertEqual(len(left), 4) + self.assertTrue(set(values(left)).issubset(range(100))) + self.assertEqual(snapshot(right), prior_right) + for value in range(100, 117): + left.add_example(example(value)) + self.assertEqual(left.num_seen, 117) + self.assertEqual(len(left), 4) + + def test_duplicate_examples_are_a_multiset_not_a_set(self): + left, right = reservoir(["same"] * 7, 3), reservoir(["same"] * 11, 3) + left.merge(right, rng=random.Random(42)) + self.assertEqual(left.num_seen, 18) + self.assertEqual(values(left), ["same", "same", "same"]) + + def test_different_caps_are_allowed_when_source_has_enough_information(self): + left, right = reservoir(range(10), 5), reservoir([10, 11], 2) + left.merge(right, rng=random.Random(8)) + self.assertEqual((len(left), left.num_seen), (5, 12)) + left, right = reservoir(range(7), 2), reservoir(range(7, 17), None) + left.merge(right, rng=random.Random(9)) + self.assertEqual((len(left), left.num_seen), (2, 17)) + + def test_rejects_self_group_mismatch_and_incompatible_sampled_sources(self): + left = reservoir(range(3), 3) + self.assert_rejected_without_mutation(left, left) + right = reservoir([], 3, groups={"other"}) + self.assert_rejected_without_mutation(left, right) + self.assert_rejected_without_mutation( + reservoir(range(7), 5), reservoir(range(9), 2) + ) + self.assert_rejected_without_mutation( + reservoir(range(3), None), reservoir(range(9), 2) + ) + + def test_rejects_underfilled_and_malformed_reservoir_states(self): + for corrupt_left in [False, True]: + left, right = reservoir(range(9), 3), reservoir(range(12), 3) + (left if corrupt_left else right).examples.pop() + self.assert_rejected_without_mutation(left, right) + for malformed in [-1, True, 2.5]: + left, right = reservoir(range(3), 3), reservoir(range(3), 3) + right.num_seen = malformed + self.assert_rejected_without_mutation(left, right) + + def test_weighted_merge_and_continued_add_have_expected_marginals(self): + # Fixed seed enumeration is deterministic, not a fresh statistical draw. + # Concatenating retained samples gives 'left' ~75%, not the required30%. + before_add = after_add = 0 + trials = 2000 + for seed in range(trials): + left = reservoir(["left"], 3, 2 * seed + 1) + right = reservoir(range(9), 3, 2 * seed + 2) + left.merge(right, rng=random.Random(100000 + seed)) + before_add += "left" in values(left) + for index in range(10): + left.add_example(example(f"new-{index}")) + after_add += "left" in values(left) + self.assertAlmostEqual(before_add / trials, 3 / 10, delta=0.04) + self.assertAlmostEqual(after_add / trials, 3 / 20, delta=0.04) + + def test_nested_merge_tree_preserves_marginals_with_shared_or_distinct_source_seeds( + self, + ): + # An independent sequential merge RNG must be threaded through the entire + # uneven tree. Replayed per-list merge streams can over-select 'e' (~27%). + trials = 4000 + for same_source_seed in [True, False]: + merged, continued = Counter(), Counter() + for seed in range(trials): + source_seeds = ( + [seed] * 5 + if same_source_seed + else [31 * seed + i for i in range(5)] + ) + a, b, c, d, e = [ + reservoir([name], 1, value) + for name, value in zip("abcde", source_seeds) + ] + rng = random.Random(200000 + seed) + a.merge(b, rng=rng) + a.merge(c, rng=rng) + d.merge(e, rng=rng) + a.merge(d, rng=rng) + self.assertEqual(a.num_seen, 5) + merged.update(values(a)) + a.add_example(example("f")) + self.assertEqual(a.num_seen, 6) + continued.update(values(a)) + with self.subTest(same_source_seed=same_source_seed): + for name in "abcde": + self.assertAlmostEqual(merged[name] / trials, 1 / 5, delta=0.025) + for name in "abcdef": + self.assertAlmostEqual(continued[name] / trials, 1 / 6, delta=0.025) + + def test_sampled_unequal_sources_then_add_preserve_every_occurrence_marginal(self): + # Both sources have already sampled, using the same construction seed. + # Continuing the destination's old RNG replays draws used by the longer + # source and biases some occurrence marginals to ~14% or ~20%, versus + # 2/12. Checking just aggregate source weights would hide this effect. + merged, continued = Counter(), Counter() + trials = 6000 + for seed in range(trials): + left = reservoir(range(3), 2, seed) + right = reservoir(range(3, 10), 2, seed) + left.merge(right, rng=random.Random(300000 + seed)) + self.assertEqual((left.num_seen, len(left)), (10, 2)) + merged.update(values(left)) + for index in (10, 11): + left.add_example(example(index)) + self.assertEqual((left.num_seen, len(left)), (12, 2)) + continued.update(values(left)) + for occurrence in range(10): + self.assertAlmostEqual(merged[occurrence] / trials, 2 / 10, delta=0.02) + for occurrence in range(12): + self.assertAlmostEqual(continued[occurrence] / trials, 2 / 12, delta=0.02) + + +class ReservoirIntegrationTests(unittest.TestCase): + def test_actual_generate_hypothesis_matches_workers_1_2_4_fork_and_spawn(self): + traces = [ + ReservoirFixtureTrace("a", 17, 11), + ReservoirFixtureTrace("b", 31, 19), + ReservoirFixtureTrace("c", 53, 29), + ] + + def run(workers): + generated = InferEngine(traces, num_workers=workers).generate_hypothesis() + self.assertEqual(len(generated), 1) + item, trace_indexes = next(iter(generated.items())) + self.assertEqual(trace_indexes, [0, 1, 2]) + result = [] + for examples, count in [ + (item.positive_examples, 101), + (item.negative_examples, 59), + ]: + self.assertEqual((examples.max_size, len(examples)), (5, 5)) + self.assertEqual(examples.num_seen, count) + # Include the continuation state: pickling must preserve it and + # the parent must consume the same merge stream in both paths. + result.append((values(examples), count, examples._rng.getstate())) + return result + + with ( + patch("traincheck.infer_engine.relation_pool", [ReservoirFixtureRelation]), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 5), + patch.object(config, "EXAMPLE_SAMPLING_SEED", 907), + ): + expected = run(1) + for method in [ + value + for value in ("spawn", "fork") + if value in multiprocessing.get_all_start_methods() + ]: + executor = partial( + ProcessPoolExecutor, mp_context=multiprocessing.get_context(method) + ) + for workers in (1, 2, 4): + with ( + self.subTest(method=method, workers=workers), + patch("traincheck.infer_engine.ProcessPoolExecutor", executor), + ): + self.assertEqual(run(workers), expected) + + def test_actual_engine_cross_trace_merge_keeps_caps_and_total_counts(self): + first, second = hypothesis("a", 17, 8, 3), hypothesis("b", 29, 11, 3) + engine, trace_indexes, lookup = InferEngine([]), {}, {} + rng = random.Random(712) + engine._merge_hypotheses([first], 0, trace_indexes, lookup, rng) + engine._merge_hypotheses([second], 1, trace_indexes, lookup, rng) + self.assertEqual(len(trace_indexes), 1) + self.assertEqual(next(iter(trace_indexes.values())), [0, 1]) + self.assertEqual( + (len(first.positive_examples), first.positive_examples.num_seen), (3, 46) + ) + self.assertEqual( + (len(first.negative_examples), first.negative_examples.num_seen), (3, 19) + ) + + def test_actual_stage_merge_keeps_caps_and_total_counts(self): + first, second = hypothesis("a", 17, 8, 3), hypothesis("b", 29, 11, 3) + + class Stages: + def is_stage_annotated(self): + return True + + def get_traces_for_stage(self): + return {"training": "training", "evaluation": "evaluation"} + + def get_all_stages(self): + return {"training", "evaluation"} + + by_stage = {"training": first, "evaluation": second} + + def inferred(stage): + item = by_stage[stage] + return [item.invariant], [], [item] + + with patch.object(APIContainRelation, "_infer", side_effect=inferred): + invariants, failures, successful = APIContainRelation.infer(Stages(), True) + self.assertEqual((len(invariants), len(failures), len(successful)), (1, 0, 1)) + self.assertEqual( + ( + len(successful[0].positive_examples), + successful[0].positive_examples.num_seen, + ), + (3, 46), + ) + self.assertEqual( + ( + len(successful[0].negative_examples), + successful[0].negative_examples.num_seen, + ), + (3, 19), + ) + + def test_sampled_generalization_does_not_treat_retained_union_as_population(self): + from traincheck.invariant.contain_relation import _merge_hypotheses + + hypotheses = [hypothesis("a", 20, 9, 3), hypothesis("b", 40, 15, 3)] + with patch( + "traincheck.invariant.contain_relation.calc_likelihood", + side_effect=AssertionError("sampled union must not be generalized"), + ): + result = _merge_hypotheses(hypotheses) + self.assertEqual( + [id(item) for item in result], [id(item) for item in hypotheses] + ) + + def test_complete_generalization_deduplicates_known_union_then_applies_cap(self): + from traincheck.invariant.contain_relation import _merge_hypotheses + + first, second = hypothesis("a", 0, 0, 10), hypothesis("b", 0, 0, 10) + first.positive_examples = reservoir(range(10), 10) + second.positive_examples = reservoir(range(10, 20), 10) + first.negative_examples = reservoir(range(10, 20), 10) + second.negative_examples = reservoir(range(10), 10) + expected = values(reservoir(range(20), 10)) + # Repeating one complete input must not double-count the known union. + for inputs in ([first, second], [first, second, first]): + with ( + self.subTest(inputs=len(inputs)), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 10), + patch.object(config, "EXAMPLE_SAMPLING_SEED", 42), + ): + merged = _merge_hypotheses(inputs) + self.assertEqual(len(merged), 1) + positive, negative = ( + merged[0].positive_examples, + merged[0].negative_examples, + ) + self.assertEqual((len(positive), positive.num_seen), (10, 20)) + self.assertEqual((len(negative), negative.num_seen), (0, 0)) + self.assertEqual(values(positive), expected) + self.assertEqual(negative.group_names, {GROUP}) + self.assertEqual(first.positive_examples.num_seen, 10) + self.assertEqual(second.positive_examples.num_seen, 10) + + def test_worker_snapshot_carries_reservoir_settings_under_fork_and_spawn(self): + with ( + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 7), + patch.object(config, "EXAMPLE_SAMPLING_SEED", 907), + ): + settings = config_snapshot() + expected = worker_reservoir_result(29) + self.assertEqual(settings["MAX_EXAMPLE_LIST_SIZE"], 7) + self.assertEqual(settings["EXAMPLE_SAMPLING_SEED"], 907) + for method in [ + value + for value in ("spawn", "fork") + if value in multiprocessing.get_all_start_methods() + ]: + with ( + self.subTest(method=method), + ProcessPoolExecutor( + max_workers=2, + mp_context=multiprocessing.get_context(method), + initializer=worker_init, + initargs=([], settings), + ) as executor, + ): + futures = [ + executor.submit(worker_reservoir_result, 29) for _ in range(2) + ] + self.assertEqual( + [future.result(timeout=60) for future in futures], + [expected, expected], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/infer_engine.py b/traincheck/infer_engine.py index 09ff0411..ffc27963 100644 --- a/traincheck/infer_engine.py +++ b/traincheck/infer_engine.py @@ -6,6 +6,7 @@ import random import time from concurrent.futures import ProcessPoolExecutor, as_completed +from random import Random from tqdm import tqdm @@ -92,14 +93,19 @@ def generate_hypothesis(self) -> dict[Hypothesis, list[int]]: active_relations = [ r for r in relation_pool if r not in self.disabled_relations ] + # One stream for the fixed parent merge order, independent of reservoir + # construction and stage merging. Never reset it per hypothesis. + merge_rng = Random( + f"traincheck-example-merge-v1:cross-trace:{config.EXAMPLE_SAMPLING_SEED!r}" + ) if self.num_workers <= 1: self._generate_hypothesis_serial( - active_relations, hypotheses_and_trace_idxs, hypo_lookup + active_relations, hypotheses_and_trace_idxs, hypo_lookup, merge_rng ) else: self._generate_hypothesis_parallel( - active_relations, hypotheses_and_trace_idxs, hypo_lookup + active_relations, hypotheses_and_trace_idxs, hypo_lookup, merge_rng ) total = len(hypotheses_and_trace_idxs) @@ -113,6 +119,7 @@ def _merge_hypotheses( trace_idx: int, hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], + merge_rng: Random, ) -> None: """Merge one trace's inferred hypotheses into the running collection. @@ -126,33 +133,21 @@ def _merge_hypotheses( else: hypotheses_and_trace_idxs[hypo].append(trace_idx) original_hypo = hypo_lookup[hypo] - orig_num_pos_exps = len(original_hypo.positive_examples) - orig_num_neg_exps = len(original_hypo.negative_examples) - original_hypo.positive_examples.examples.extend( - hypo.positive_examples.examples + original_hypo.positive_examples.merge( + hypo.positive_examples, rng=merge_rng ) - original_hypo.negative_examples.examples.extend( - hypo.negative_examples.examples + original_hypo.negative_examples.merge( + hypo.negative_examples, rng=merge_rng ) - assert len( - hypo_lookup[hypo].positive_examples - ) == orig_num_pos_exps + len( - hypo.positive_examples - ), f"Expected {orig_num_pos_exps} + {len(hypo.positive_examples)} positive examples, got {len(hypo_lookup[hypo].positive_examples)}" - assert len( - hypo_lookup[hypo].negative_examples - ) == orig_num_neg_exps + len( - hypo.negative_examples - ), f"Expected {orig_num_neg_exps} + {len(hypo.negative_examples)} negative examples, got {len(hypo_lookup[hypo].negative_examples)}" - def _generate_hypothesis_serial( self, active_relations: list, hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], + merge_rng: Random, ) -> None: - """Serial hypothesis generation (preserves the original behavior exactly).""" + """Generate hypotheses serially, merging in trace-then-relation order.""" n_traces = len(self.traces) rel_bar_fmt = "{desc} {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]" for trace_idx, trace in enumerate(self.traces): @@ -183,6 +178,7 @@ def _generate_hypothesis_serial( trace_idx, hypotheses_and_trace_idxs, hypo_lookup, + merge_rng, ) def _generate_hypothesis_parallel( @@ -190,6 +186,7 @@ def _generate_hypothesis_parallel( active_relations: list, hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], + merge_rng: Random, ) -> None: """Parallel hypothesis generation. @@ -197,7 +194,7 @@ def _generate_hypothesis_parallel( is CPU-bound and pure w.r.t. its (per-worker) trace, so we fan them out across processes. Workers only PRODUCE hypothesis lists; the merge into shared state stays in the parent. Results are merged in trace-then-relation order (identical - to the serial path) regardless of completion order, so the output is unchanged. + to the serial path) regardless of completion order. """ n_traces = len(self.traces) tasks = [ @@ -241,6 +238,7 @@ def _generate_hypothesis_parallel( trace_idx, hypotheses_and_trace_idxs, hypo_lookup, + merge_rng, ) def collect_examples(self, hypotheses: dict[Hypothesis, list[int]]): diff --git a/traincheck/invariant/base_cls.py b/traincheck/invariant/base_cls.py index 588e7260..59ddd2c3 100644 --- a/traincheck/invariant/base_cls.py +++ b/traincheck/invariant/base_cls.py @@ -182,7 +182,7 @@ def __init__( and param.default != inspect.Parameter.empty ): self.arguments[param_name] = var_to_serializable(param.default) - + def __getstate__(self) -> dict: # ``signature`` is a cached ``inspect.Signature`` derived from # ``func_name``. It is not picklable -- its ``_parameters`` is a @@ -1677,18 +1677,47 @@ def __eq__(self, value: object) -> bool: class ExampleList: + """A bounded sample of example occurrences, including duplicate values. + + ``num_seen`` counts the underlying stream, while ``len(self)`` counts the + retained examples. Different lists retain the historical equal-seed RNG + behavior, so their samples can be correlated; merging does not promise a + uniform joint sample across lists. + """ + def __init__(self, group_names: set[str]): self.group_names = group_names self.examples: list[Example] = [] - # Bound memory: some relations produce an unbounded number of examples - # (see config.MAX_EXAMPLE_LIST_SIZE). Once at capacity we keep a uniform - # random subset via reservoir sampling; num_seen tracks the true number - # of examples added so the retained subset stays uniform. The RNG is - # seeded so results are deterministic and identical across the serial - # and parallel code paths. - self.max_size = config.MAX_EXAMPLE_LIST_SIZE - self.num_seen = 0 - self._rng = random.Random(config.EXAMPLE_SAMPLING_SEED) + self.max_size = config.MAX_EXAMPLE_LIST_SIZE + self._validate_cap(self.max_size) + self.num_seen = 0 + self._rng = random.Random(config.EXAMPLE_SAMPLING_SEED) + + @staticmethod + def _validate_cap(cap: int | None) -> None: + # Reject bool and int subclasses rather than accepting isinstance(int). + if cap is not None and (type(cap) is not int or cap < 0): # noqa: E721 + raise ValueError( + "Example reservoir capacity must be a nonnegative integer or None" + ) + + def _validate_state(self) -> None: + self._validate_cap(self.max_size) + # A population count must be a builtin integer. + if type(self.num_seen) is not int or self.num_seen < 0: # noqa: E721 + raise ValueError( + "Example reservoir population must be a nonnegative integer" + ) + expected = ( + self.num_seen + if self.max_size is None + else min(self.max_size, self.num_seen) + ) + if len(self.examples) != expected: + raise ValueError( + "Example reservoir is underfilled or has inconsistent population accounting: " + f"retained {len(self.examples)}, expected {expected} for {self.num_seen} seen" + ) def add_example(self, example: Example): if len(self.group_names) == 0: @@ -1698,17 +1727,80 @@ def add_example(self, example: Example): assert ( example.get_group_names() == self.group_names ), f"Example groups do not match the expected group names, expected: {self.group_names}, got: {set(example.trace_groups.keys())}" - #self.examples.append(example) - self.num_seen += 1 - if self.max_size is None or len(self.examples) < self.max_size: - self.examples.append(example) - else: - # reservoir sampling: replace an existing example with probability - # max_size / num_seen, keeping self.examples a uniform random sample - j = self._rng.randrange(self.num_seen) - if j < self.max_size: + self.num_seen += 1 + if self.max_size is None or len(self.examples) < self.max_size: + self.examples.append(example) + else: + # Algorithm R retains each occurrence with probability cap / num_seen. + j = self._rng.randrange(self.num_seen) + if j < self.max_size: self.examples[j] = example + def merge(self, other: ExampleList, *, rng: random.Random) -> None: + """Append another stream's reservoir without exceeding this list's cap. + + The caller's RNG must be independent of both input selections. Share one + RNG sequentially within a merge task; do not replay its state in nested + merge operations. This preserves each occurrence's marginal inclusion + probability, even when the input reservoirs are correlated. A uniform + joint sample additionally requires independent input reservoirs. + + A sampled source cannot be expanded beyond its retained capacity or + merged into an uncapped destination: discarded examples cannot be + recovered. Invalid inputs are rejected before changing either list or + consuming the supplied RNG. Complete streams preserve concatenation + order whenever they fit in the destination. + """ + if self is other: + raise ValueError("Cannot merge an example reservoir into itself") + self._validate_state() + other._validate_state() + adopt_groups = not self.group_names and self.num_seen == 0 + source_uninitialized = not other.group_names and other.num_seen == 0 + if ( + not adopt_groups + and not source_uninitialized + and self.group_names != other.group_names + ): + raise ValueError( + "Cannot merge example reservoirs with different group names" + ) + + total = self.num_seen + other.num_seen + target = total if self.max_size is None else min(self.max_size, total) + for source in (self, other): + if len(source.examples) < min(target, source.num_seen): + raise ValueError( + "Source reservoir retains too few examples for the destination capacity" + ) + if other.num_seen == 0: + if adopt_groups: + self.group_names = other.group_names.copy() + return + + if target == total: + # Both inputs are complete: no sampling or RNG consumption is needed. + examples = self.examples + other.examples + continuation_rng = self._rng + else: + # Sampling population positions gives an exact hypergeometric split + # in O(cap) space, without expanding either represented population. + take_self = sum( + index < self.num_seen for index in rng.sample(range(total), target) + ) + examples = rng.sample(self.examples, take_self) + rng.sample( + other.examples, target - take_self + ) + # Further additions must not replay RNG draws used by a longer source + # with the same initial reservoir seed. + continuation_rng = random.Random(rng.getrandbits(256)) + + self.examples = examples + self.num_seen = total + self._rng = continuation_rng + if adopt_groups: + self.group_names = other.group_names.copy() + def get_group_from_examples(self, group_name: str) -> list[list[dict]]: return [example.get_group(group_name) for example in self.examples] @@ -1725,14 +1817,10 @@ def __len__(self): def from_iterable_of_examples( input: Iterable[Example], preset_group_names: set[str] ) -> ExampleList: - examples = [] + example_list = ExampleList(preset_group_names) for exp in input: assert preset_group_names == exp.get_group_names() - examples.append(exp) - - example_list = ExampleList(preset_group_names) - example_list.examples = examples - example_list.num_seen = len(examples) + example_list.add_example(exp) return example_list diff --git a/traincheck/invariant/contain_relation.py b/traincheck/invariant/contain_relation.py index 84fa76d2..e1a9de13 100644 --- a/traincheck/invariant/contain_relation.py +++ b/traincheck/invariant/contain_relation.py @@ -2,10 +2,12 @@ import logging import random import time +from random import Random from typing import Type import numpy as np +import traincheck.config.config as config from traincheck.config.config import ANALYSIS_SKIP_FUNC_NAMES from traincheck.instrumentor.tracer import TraceLineType from traincheck.invariant.base_cls import ( @@ -209,6 +211,15 @@ def _merge_hypotheses(hypotheses: list[Hypothesis]) -> list[Hypothesis]: assert len(hypotheses) > 0, "Expected at least one hypotheses to be merged" logger = logging.getLogger(__name__) + if any( + examples.num_seen > len(examples) + for hypothesis in hypotheses + for examples in (hypothesis.positive_examples, hypothesis.negative_examples) + ): + # This optional generalization uses set unions and differences. Reservoirs + # cannot recover the discarded values or their overlap, so keep the + # original hypotheses instead of treating retained samples as full sets. + return hypotheses pos_exp_group_names = hypotheses[0].positive_examples.get_group_names() neg_exp_group_names = hypotheses[0].negative_examples.get_group_names() logger.debug( @@ -221,9 +232,10 @@ def _merge_hypotheses(hypotheses: list[Hypothesis]) -> list[Hypothesis]: all_likelihoods = [hypo.calc_likelihood() for hypo in hypotheses] # calculate the number of positive and negative examples after merge - all_positive_examples = set() + # Insertion-ordered sets keep constructor sampling independent of hash order. + all_positive_examples: dict[Example, None] = {} all_positive_parent_examples = set() - all_negative_examples = set() + all_negative_examples: dict[Example, None] = {} for idx in range(len(hypotheses)): @@ -234,16 +246,22 @@ def _merge_hypotheses(hypotheses: list[Hypothesis]) -> list[Hypothesis]: Example({PARENT_GROUP_NAME: pre_record}) for pre_record in pre_records ] all_positive_parent_examples.update(exps_with_only_parent) - all_positive_examples.update(hypotheses[idx].positive_examples.examples) - all_negative_examples.update(hypotheses[idx].negative_examples.examples) + all_positive_examples.update( + (example, None) for example in hypotheses[idx].positive_examples.examples + ) + all_negative_examples.update( + (example, None) for example in hypotheses[idx].negative_examples.examples + ) # let's remove those negative examples that are present in the positive examples of other hypotheses - if used_dynamic_analysis: - all_negative_examples = all_negative_examples.difference(all_positive_examples) - else: - all_negative_examples = all_negative_examples.difference( - all_positive_parent_examples - ) + positive_comparison = ( + all_positive_examples if used_dynamic_analysis else all_positive_parent_examples + ) + all_negative_examples = { + example: None + for example in all_negative_examples + if example not in positive_comparison + } # calculate the likelihood of the merged hypotheses now merged_likelihood = calc_likelihood( @@ -575,15 +593,20 @@ def infer( merged_invariants = [] merged_successful_hypotheses: list[Hypothesis] = [] all_stages = trace.get_all_stages() + # Reuse one independent stream for this stage-merge task. Cross-trace + # merging uses a different domain, so nested merges do not replay draws. + merge_rng = Random( + f"traincheck-example-merge-v1:stage:{config.EXAMPLE_SAMPLING_SEED!r}" + ) for invariant in invariants_by_stage: supported_stages = set(invariants_by_stage[invariant]) hypothesis = invariants_to_hypos[invariant][0] for other_hypothesis in invariants_to_hypos[invariant][1:]: - hypothesis.positive_examples.examples.extend( - other_hypothesis.positive_examples.examples + hypothesis.positive_examples.merge( + other_hypothesis.positive_examples, rng=merge_rng ) - hypothesis.negative_examples.examples.extend( - other_hypothesis.negative_examples.examples + hypothesis.negative_examples.merge( + other_hypothesis.negative_examples, rng=merge_rng ) hypothesis.add_stage_info(supported_stages, all_stages) merged_invariants.append(hypothesis.get_invariant(all_stages)) diff --git a/traincheck/parallel.py b/traincheck/parallel.py index 4d8bcb45..b1c35dac 100644 --- a/traincheck/parallel.py +++ b/traincheck/parallel.py @@ -20,16 +20,18 @@ def config_snapshot() -> dict: - """CLI-overridable config values that must be propagated to workers. + """Runtime inference config values that must be propagated to workers. - These are set in the entry points' main() at runtime; workers started with - the 'spawn' method begin from a fresh interpreter and would otherwise see - only the module defaults (harmless under 'fork'). Add any future - CLI-overridable config here. + These include CLI sampling options and programmatic reservoir settings. + Workers started with 'spawn' begin from a fresh interpreter and would + otherwise see only the module defaults. Add future runtime-overridable + inference configuration here. """ return { "ENABLE_PRECOND_SAMPLING": config.ENABLE_PRECOND_SAMPLING, "PRECOND_SAMPLING_THRESHOLD": config.PRECOND_SAMPLING_THRESHOLD, + "MAX_EXAMPLE_LIST_SIZE": config.MAX_EXAMPLE_LIST_SIZE, + "EXAMPLE_SAMPLING_SEED": config.EXAMPLE_SAMPLING_SEED, } @@ -37,7 +39,7 @@ def worker_init(traces: list, snapshot: dict) -> None: """Initializer run once per worker process. Stores the traces in a module global (one copy per worker, not per task) and - restores CLI-overridable config values (see config_snapshot). Also suppresses + restores runtime inference config values (see config_snapshot). Also suppresses the per-invariant inner progress bars, otherwise every worker would fight over the terminal. """ From 1eb17b7c43b616b8ddc11bfce411848557c7b967 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:08:47 +0800 Subject: [PATCH 03/36] Group context-manager events for invariant inference initialization --- tests/fixtures/__init__.py | 0 .../legacy_trace_pandas.provenance.json | 13 + tests/fixtures/legacy_trace_pandas.py | 244 +++++++++++++++ tests/test_context_index.py | 287 ++++++++++++++++++ traincheck/trace/trace_pandas.py | 154 +++++++--- 5 files changed, 650 insertions(+), 48 deletions(-) create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/legacy_trace_pandas.provenance.json create mode 100644 tests/fixtures/legacy_trace_pandas.py create mode 100644 tests/test_context_index.py diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/legacy_trace_pandas.provenance.json b/tests/fixtures/legacy_trace_pandas.provenance.json new file mode 100644 index 00000000..00498a4d --- /dev/null +++ b/tests/fixtures/legacy_trace_pandas.provenance.json @@ -0,0 +1,13 @@ +{ + "repository": "https://github.com/OrderLab/TrainCheck", + "commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "source_path": "traincheck/trace/trace_pandas.py", + "source_sha256": "e1bb62eae653f95c5498dc921d991af013e8b5717398f94cfbc718812bb600ad", + "method_sha256": { + "_index_context_manager_meta_vars": "6b7e5646260ee0ce7377c7cf19edb4774bdf69825e15cf10ff3c29cd4555c13c", + "get_pre_func_call_record": "bf89fcd11b98e08a3029ee6ff3b2a4b6df980aa49bfe6c13bcdbafcb10e8d74c", + "get_post_func_call_record": "a36cd5bc217f8e85f0f30775c24480f0060b6f3496e8fb2a2cb45a4214d46c07", + "get_traces_for_stage": "583c5adfd122d86027f427beea7c7823783e14b37f078d01740f55cdd72f04f3" + }, + "hash_convention": "Original indented method source including its final newline. No source transformations." +} diff --git a/tests/fixtures/legacy_trace_pandas.py b/tests/fixtures/legacy_trace_pandas.py new file mode 100644 index 00000000..c7256089 --- /dev/null +++ b/tests/fixtures/legacy_trace_pandas.py @@ -0,0 +1,244 @@ +"""Frozen legacy query methods used as a differential oracle. + +Source: OrderLab/TrainCheck commit 9eb4f320d3db34d372145261704b805b88e39bda, +traincheck/trace/trace_pandas.py (SHA256 + e1bb62eae653f95c5498dc921d991af013e8b5717398f94cfbc718812bb600ad). +Only four exact original methods are retained. The production constructor and +supporting types are shared so tests isolate these query/index changes. Defining +this class as TracePandas intentionally makes the original stage factory create +legacy instances too. See legacy_trace_pandas.provenance.json for method hashes. +""" + +import logging + +import pandas as pd + +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.instrumentor.types import PTID +from traincheck.trace.trace_pandas import STAGE_KEY +from traincheck.trace.trace_pandas import TracePandas as ProductionTracePandas +from traincheck.trace.types import ContextManagerState, Liveness +from traincheck.trace.utils import ( + BindedFuncInput, + bind_args_kwargs_to_signature, + load_signature_from_class_method_name, +) +from traincheck.utils import safe_isnan + +logger = logging.getLogger(__name__) + + +class TracePandas(ProductionTracePandas): + def _index_context_manager_meta_vars(self): + """Identify context manager entry and exit events, and add them to the meta_vars.""" + # 1. Find all trace records that are related to __enter__ and __exit__ functions + if "function" not in self.events.columns: + self.context_manager_states = None + return + + if ( + hasattr(self, "context_manager_states") + and self.context_manager_states is not None + ): + return self.context_manager_states + + all_context_managers: dict[PTID, list[ContextManagerState]] = {} + + context_manager_names = self.events[ + (self.events["function"].str.contains("__enter__", na=False)) + & ( + ~self.events["function"].str.contains( + "torch.autograd.grad_mode", na=False + ) + ) # HACK: ignore all autograd related context managers for now as it doesn't have an __init__ event + & ( + ~self.events["function"].str.contains( + "torch.autograd.profiler.record_function", na=False + ) + ) + ]["function"].unique() + + context_manager_init_pre_records = [] + for context_manager_name in context_manager_names: + context_manager_name = context_manager_name.removesuffix(".__enter__") + context_manager_init_pre_records.extend( + self.events[ + (self.events["function"] == f"{context_manager_name}.__init__") + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ].to_dict(orient="records") + ) + + for context_manager_init_pre_record in context_manager_init_pre_records: + init_pre_record = context_manager_init_pre_record + process_id = init_pre_record["process_id"] + thread_id = init_pre_record["thread_id"] + + context_manager_name = init_pre_record["function"].removesuffix(".__init__") + + # find nearest enter and exit events with the same obj_id + obj_id = init_pre_record["obj_id"] + try: + enter_post_record = ( + self.events[ + (self.events["type"] == TraceLineType.FUNC_CALL_POST) + & ( + self.events["function"] + == f"{context_manager_name}.__enter__" + ) + & (self.events["obj_id"] == obj_id) + & (self.events["time"] > init_pre_record["time"]) + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] + .iloc[0] + .to_dict() + ) + + exit_pre_record = ( + self.events[ + (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + & ( + self.events["function"] + == f"{context_manager_name}.__exit__" + ) + & (self.events["obj_id"] == obj_id) + & (self.events["time"] > enter_post_record["time"]) + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] + .iloc[0] + .to_dict() + ) + except IndexError: + # sometimes this happens. an enter might not have a corresponding exit, vice versa. e.g. torch.serialization._opener.__enter__ and torch.serialization._open_zipfile_writer_file.__exit__ + logger.warning( + f"Context manager {context_manager_name} not used properly. Skipping." + ) + continue + + assert ( + enter_post_record["time"] < exit_pre_record["time"] + ), f"enter not before exit: {enter_post_record.to_dict()} {exit_pre_record.to_dict()}" + + start_time, end_time = ( + enter_post_record["time"], + exit_pre_record["time"], + ) + + args = init_pre_record["args"] + kwargs = init_pre_record["kwargs"] + if not safe_isnan(args): + signature = load_signature_from_class_method_name( + init_pre_record["function"] + ) + + binded_args_and_kwargs = bind_args_kwargs_to_signature( + args, kwargs, signature + ) + else: + # create an empty BindedFuncInput if args is NaN, as it indicates + # that we did not record the args and kwargs for this function call + binded_args_and_kwargs = BindedFuncInput({}) + + ptid = PTID(process_id, thread_id) + if ptid not in all_context_managers: + all_context_managers[ptid] = [] + + logger.debug(f"Adding context manager: {context_manager_name} {ptid}") + all_context_managers[ptid].append( + ContextManagerState( + name=init_pre_record["function"].removesuffix(".__init__"), + ptid=ptid, + liveness=Liveness(start_time, end_time), + input=binded_args_and_kwargs, + ) + ) + + logger.info(f"Found {len(all_context_managers)} context managers.") + + self.context_manager_states = all_context_managers + + def get_pre_func_call_record(self, func_call_id: str) -> dict: + """Get the pre call record of a function given its func_call_id.""" + # TODO: check if this is correct + pre_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ] + + assert ( + len(pre_records) == 1 + ), f"{len(pre_records)} pre call events found for {func_call_id}, expected 1" + + pre_record = pre_records.iloc[0].to_dict() + + return pre_record + + def get_post_func_call_record(self, func_call_id: str) -> dict | None: + """Get the post call record of a function given its func_call_id. + Returns None if the post call event is not found and the pre-call event is the outermost function call. (see the doc of `_rm_incomplete_trailing_func_calls`) + """ + # TODO: check if this is correct + post_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) + ) + ] + + assert ( + len(post_records) <= 1 + ), f"{post_records.shape[0]} post call events found for {func_call_id}, expected 1" + + if len(post_records) == 0: + logger.warning(f"No post call event found for {func_call_id}") + # check if the pre-call event is the outermost function call + pre_record = self.get_pre_func_call_record(func_call_id) + + outermost_func_call_pre = ( + self.events[ + (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + & (self.events["process_id"] == pre_record["process_id"]) + & (self.events["thread_id"] == pre_record["thread_id"]) + ] + .iloc[0] + .to_dict() + ) + + if pre_record["func_call_id"] == outermost_func_call_pre["func_call_id"]: + return None + else: + raise ValueError( + f"No post call event found for {func_call_id}, but it is not the outermost function call." + ) + + post_record = post_records.iloc[ + 0 + ].to_dict() # order of events doesn't matter as the result here is a single row + + return post_record + + def get_traces_for_stage(self) -> dict[str, "TracePandas"]: # type: ignore + """Get the traces split by stages.""" + + if not self.is_stage_annotated(): + raise ValueError("Trace is not annotated with stages.") + + if self.stage_traces: + return self.stage_traces + + traces = {} + for stage in self.events[STAGE_KEY].unique(): + traces[stage] = TracePandas( + self.events[self.events[STAGE_KEY] == stage], + truncate_incomplete_func_calls=False, + ) + + self.stage_traces = traces + + return traces diff --git a/tests/test_context_index.py b/tests/test_context_index.py new file mode 100644 index 00000000..a6390d8e --- /dev/null +++ b/tests/test_context_index.py @@ -0,0 +1,287 @@ +"""Differential context-index tests against frozen legacy query methods.""" + +import inspect +import logging +import random +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as NEW +from tests.fixtures import legacy_trace_pandas as OLD + +PRE = OLD.TraceLineType.FUNC_CALL_PRE +POST = OLD.TraceLineType.FUNC_CALL_POST + + +def event( + time, + method, + *, + name="fixtures.Ctx", + obj=1, + pid=10, + tid=20, + kind=None, + args=float("nan"), + kwargs=None, + stage="train", +): + return { + "time": time, + "function": f"{name}.{method}", + "type": kind or (POST if method == "__enter__" else PRE), + "obj_id": obj, + "process_id": pid, + "thread_id": tid, + "args": args, + "kwargs": kwargs or {}, + "meta_vars.stage": stage, + } + + +def cycle(start, **kwargs): + return [ + event(start, "__init__", **kwargs), + event(start + 1, "__enter__", **kwargs), + event(start + 2, "__exit__", **kwargs), + ] + + +def run(module, frame): + trace = module.TracePandas( + frame.copy(deep=True), truncate_incomplete_func_calls=False + ) + return trace, encode(trace) + + +def encode(trace): + states = trace.context_manager_states + if states is None: + return None + return [ + (tuple(key), [state.to_dict() for state in value]) + for key, value in states.items() + ] + + +class ContextIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.WARNING) + self.addCleanup(logging.disable, previous) + + def equivalent(self, rows, **frame_options): + frame = ( + rows + if isinstance(rows, pd.DataFrame) + else pd.DataFrame(rows, **frame_options) + ) + original, expected = run(OLD, frame) + optimized, actual = run(NEW, frame) + self.assertEqual(expected, actual) + # Native scalar types remain suitable for the same JSON serialization. + for _, states in actual or []: + for state in states: + self.assertNotIsInstance(state["start_time"], np.generic) + self.assertNotIsInstance(state["end_time"], np.generic) + return original, optimized + + def test_nested_reused_ids_processes_threads_and_class_order(self): + rows = cycle(0, name="fixtures.B", obj=1) + rows += cycle(5, name="fixtures.A", obj=1) + rows += cycle(10, name="fixtures.B", obj=1) + rows += cycle(1, name="fixtures.B", obj=2) + rows += cycle(0, name="fixtures.B", obj=1, pid=11) + rows += cycle(0, name="fixtures.B", obj=1, tid=21) + old, new = self.equivalent(rows) + self.assertEqual( + [s.name for s in new.context_manager_states[OLD.PTID(10, 20)]], + ["fixtures.B", "fixtures.B", "fixtures.B", "fixtures.A"], + ) + for when in [0, 1, 2, 6, 11, 12, 100]: + self.assertEqual( + [s.to_dict() for s in old.query_active_context_managers(when, 10, 20)], + [s.to_dict() for s in new.query_active_context_managers(when, 10, 20)], + ) + + def test_first_row_not_nearest_timestamp(self): + rows = [ + event(0, "__init__"), + event(8, "__enter__"), + event(2, "__enter__"), + event(15, "__exit__"), + event(9, "__exit__"), + ] + _, new = self.equivalent(rows) + state = next(iter(new.context_manager_states.values()))[0] + self.assertEqual((state.liveness.start_time, state.liveness.end_time), (8, 15)) + + def test_equal_timestamps_strictly_later(self): + rows = [ + event(5, "__init__"), + event(5, "__enter__"), + event(6, "__enter__"), + event(6, "__exit__"), + event(7, "__exit__"), + ] + _, new = self.equivalent(rows) + self.assertEqual( + next(iter(new.context_manager_states.values()))[0].liveness.start_time, 6 + ) + + def test_null_keys_do_not_match_each_other(self): + for field in ["obj_id", "process_id", "thread_id"]: + for missing in [None, float("nan"), pd.NA]: + with self.subTest(field=field, missing=missing): + rows = cycle(0) + for row in rows: + row[field] = missing + rows += cycle(5, obj=2) + self.equivalent(rows) + + def test_md_none_key_matches_original_custom_equality(self): + from traincheck.trace.types import MD_NONE + + for field in ["obj_id", "process_id", "thread_id"]: + rows = cycle(0) + for row in rows: + row[field] = MD_NONE() + self.equivalent(rows) + + def test_mixed_md_none_and_null_keys(self): + from traincheck.trace.types import MD_NONE + + rows = [ + event(0, "__init__", obj=None), + event(1, "__enter__", obj=MD_NONE()), + event(2, "__exit__", obj=MD_NONE()), + ] + self.equivalent(rows) + + def test_nullable_numeric_keys(self): + frame = pd.DataFrame(cycle(0) + cycle(5, obj=None) + cycle(10, obj=2)) + for col in ["obj_id", "process_id", "thread_id"]: + frame[col] = frame[col].astype("Int64") + self.equivalent(frame) + + def test_missing_pairs_and_wrong_event_types(self): + rows = [ + event(0, "__init__"), + event(1, "__enter__", kind=PRE), + event(2, "__exit__"), + ] + rows += [event(4, "__init__", obj=2), event(5, "__enter__", obj=2)] + rows += [event(6, "__enter__", obj=3), event(7, "__exit__", obj=3)] + rows += cycle(10, obj=4) + self.equivalent(rows) + + def test_infinite_missing_and_nullable_timestamps(self): + rows = [ + event(float("nan"), "__init__", obj=2), + event(0, "__init__"), + event(float("nan"), "__enter__"), + event(1, "__enter__"), + event(float("inf"), "__exit__"), + ] + self.equivalent(rows) + frame = pd.DataFrame(rows) + frame["time"] = frame["time"].astype("Float64") + for module in [OLD, NEW]: + with self.assertRaises(TypeError): + run(module, frame) + + def test_large_integer_timestamps_preserve_precision(self): + for dtype in ["int64", "uint64", "Int64", "UInt64", "float64", "object"]: + for start in [2**53, 2**60, 2**63 - 4]: + with self.subTest(dtype=dtype, start=start): + frame = pd.DataFrame(cycle(start)) + frame["time"] = frame["time"].astype(dtype) + self.equivalent(frame) + for start in [2**53, 2**60, 2**63 - 8]: + # Object dtype retains a float init query against exact integers. + frame = pd.DataFrame(cycle(start)) + frame["time"] = pd.Series( + [float(start), start + 1, start + 2], dtype=object + ) + self.equivalent(frame) + + def test_duplicate_dataframe_indexes_and_mixed_key_types(self): + rows = cycle(0, obj="1") + cycle(5, obj=1) + cycle(10, obj=True) + self.equivalent(pd.DataFrame(rows, index=[4] * len(rows))) + + def test_no_context_or_no_function(self): + self.equivalent(pd.DataFrame({"time": [0]})) + self.equivalent([event(0, "unrelated")]) + self.equivalent([event(0, "__enter__")]) + + def test_ignore_autograd_contexts(self): + rows = cycle(0, name="torch.autograd.grad_mode.enable_grad") + rows += cycle(3, name="torch.autograd.profiler.record_function") + rows += cycle(6) + _, new = self.equivalent(rows) + self.assertEqual(len(next(iter(new.context_manager_states.values()))), 1) + + def test_stage_split_rebuild_and_cache(self): + rows = cycle(0, stage="init") + cycle(3, stage="train") + rows += [ + event(10, "__init__", obj=2, stage="init"), + event(11, "__enter__", obj=2, stage="train"), + event(12, "__exit__", obj=2, stage="train"), + ] + old, new = self.equivalent(rows) + expected = old.get_traces_for_stage() + actual = new.get_traces_for_stage() + self.assertEqual(list(expected), list(actual)) + for stage in expected: + self.assertIs(type(expected[stage]), OLD.TracePandas) + self.assertIs(type(actual[stage]), NEW.TracePandas) + self.assertEqual(encode(expected[stage]), encode(actual[stage])) + self.assertIs(new.get_traces_for_stage(), actual) + cached = new.context_manager_states + self.assertIs(new._index_context_manager_meta_vars(), cached) + + def test_bindings_and_signature_lookup_remain_per_init(self): + rows = cycle(0, args={"0": {"object": {}}, "1": {"int": 7}}) + rows += cycle(5, args={"0": {"object": {}}, "1": {"int": 9}}) + signatures = [ + inspect.Signature( + [ + inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD), + inspect.Parameter(name, inspect.Parameter.POSITIONAL_OR_KEYWORD), + ] + ) + for name in ["first", "second"] + ] + outputs = [] + for module in [OLD, NEW]: + with patch.object( + module, "load_signature_from_class_method_name", side_effect=signatures + ) as loader: + outputs.append(run(module, pd.DataFrame(rows))[1]) + self.assertEqual(loader.call_count, 2) + self.assertEqual(*outputs) + + def test_randomized_row_order_and_reused_identity_oracle(self): + for seed in range(80): + rng = random.Random(seed) + rows = [] + for _ in range(20): + rows.extend( + cycle( + rng.randrange(20), + obj=rng.randrange(3), + pid=rng.randrange(2), + tid=rng.randrange(2), + name=rng.choice(["fixtures.A", "fixtures.B"]), + ) + ) + rng.shuffle(rows) + with self.subTest(seed=seed): + self.equivalent(rows) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 8bb89ca7..1cc0eb6b 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -1,5 +1,6 @@ import logging import re +from numbers import Real from typing import Any import pandas as pd @@ -15,6 +16,7 @@ ContextManagerState, FuncCallEvent, FuncCallExceptionEvent, + IncompleteFuncCallEvent, Liveness, VarChangeEvent, VarInstId, @@ -65,7 +67,7 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self.function_pool = None self.cache_id_2_func_call_event: dict[ - str, FuncCallEvent | FuncCallExceptionEvent + str, FuncCallEvent | FuncCallExceptionEvent | IncompleteFuncCallEvent ] = {} self.all_func_call_ids: dict[str, list[str]] = {} @@ -312,15 +314,91 @@ def _index_context_manager_meta_vars(self): ) ]["function"].unique() - context_manager_init_pre_records = [] - for context_manager_name in context_manager_names: - context_manager_name = context_manager_name.removesuffix(".__enter__") - context_manager_init_pre_records.extend( - self.events[ - (self.events["function"] == f"{context_manager_name}.__init__") - & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) - ].to_dict(orient="records") + # Group relevant events once, retaining their original row order. Context + # names and init records keep the old class-first encounter order. + init_functions = [ + f"{name.removesuffix('.__enter__')}.__init__" + for name in context_manager_names + ] + if not init_functions: + self.context_manager_states = all_context_managers + return + + init_records = self.events[ + self.events["function"].isin(init_functions) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ] + init_by_function = { + name: records.to_dict(orient="records") + for name, records in init_records.groupby("function", sort=False) + } + context_manager_init_pre_records = [ + record + for name in init_functions + for record in init_by_function.get(name, []) + ] + if not context_manager_init_pre_records: + self.context_manager_states = all_context_managers + return + + context_names = [ + name.removesuffix(".__enter__") for name in context_manager_names + ] + group_columns = ["function", "type", "process_id", "thread_id", "obj_id"] + matching_events = self.events.loc[ + ( + self.events["function"].isin( + [f"{name}.__enter__" for name in context_names] + ) + & (self.events["type"] == TraceLineType.FUNC_CALL_POST) ) + | ( + self.events["function"].isin( + [f"{name}.__exit__" for name in context_names] + ) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ), + group_columns + ["time"], + ] + event_times = {} + for key, records in matching_events.groupby( + group_columns, sort=False, dropna=True, observed=True + ): + times = records["time"] + # Binary search is equivalent to the original first matching row + # only when this group's numeric times are sorted and non-null. + # Nullable extension arrays may promote UInt64 search values to + # floats and lose precision above 2**53, so keep their comparison. + ordered = ( + not isinstance(times.dtype, pd.api.extensions.ExtensionDtype) + and times.dtype.kind in "biuf" + and not times.isna().any() + and times.is_monotonic_increasing + ) + event_times[key] = (times, ordered) + + def first_time_after(function, event_type, process_id, thread_id, obj_id, time): + if not isinstance(time, Real): + # Keep pandas validation for nonnumeric/null scalar thresholds, + # including malformed nullable timestamps, even with no group. + self.events["time"] > time + if any(pd.isna(key) for key in (process_id, thread_id, obj_id)): + # pandas equality never matches missing identity values, even + # when a custom non-null key compares equal to None. + raise IndexError("No matching context event") + group = event_times.get( + (function, event_type, process_id, thread_id, obj_id) + ) + if group is None: + raise IndexError("No matching context event") + times, ordered = group + if ordered and isinstance(time, Real) and not pd.isna(time): + position = times.searchsorted(time, side="right") + return times.iloc[position : position + 1].tolist()[0] + # Unsorted times must retain row-order selection, not nearest time. + # This also preserves pandas comparison behavior for nullable and + # nonnumeric timestamp dtypes. + return times[times > time].iloc[:1].tolist()[0] for context_manager_init_pre_record in context_manager_init_pre_records: init_pre_record = context_manager_init_pre_record @@ -329,55 +407,35 @@ def _index_context_manager_meta_vars(self): context_manager_name = init_pre_record["function"].removesuffix(".__init__") - # find nearest enter and exit events with the same obj_id + # Select the first later event in row order with the same identity. obj_id = init_pre_record["obj_id"] try: - enter_post_record = ( - self.events[ - (self.events["type"] == TraceLineType.FUNC_CALL_POST) - & ( - self.events["function"] - == f"{context_manager_name}.__enter__" - ) - & (self.events["obj_id"] == obj_id) - & (self.events["time"] > init_pre_record["time"]) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - .iloc[0] - .to_dict() + start_time = first_time_after( + f"{context_manager_name}.__enter__", + TraceLineType.FUNC_CALL_POST, + process_id, + thread_id, + obj_id, + init_pre_record["time"], ) - - exit_pre_record = ( - self.events[ - (self.events["type"] == TraceLineType.FUNC_CALL_PRE) - & ( - self.events["function"] - == f"{context_manager_name}.__exit__" - ) - & (self.events["obj_id"] == obj_id) - & (self.events["time"] > enter_post_record["time"]) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - .iloc[0] - .to_dict() + end_time = first_time_after( + f"{context_manager_name}.__exit__", + TraceLineType.FUNC_CALL_PRE, + process_id, + thread_id, + obj_id, + start_time, ) except IndexError: - # sometimes this happens. an enter might not have a corresponding exit, vice versa. e.g. torch.serialization._opener.__enter__ and torch.serialization._open_zipfile_writer_file.__exit__ + # Some traces have an unmatched entry or exit event. logger.warning( f"Context manager {context_manager_name} not used properly. Skipping." ) continue assert ( - enter_post_record["time"] < exit_pre_record["time"] - ), f"enter not before exit: {enter_post_record.to_dict()} {exit_pre_record.to_dict()}" - - start_time, end_time = ( - enter_post_record["time"], - exit_pre_record["time"], - ) + start_time < end_time + ), f"enter not before exit: {start_time} {end_time}" args = init_pre_record["args"] kwargs = init_pre_record["kwargs"] @@ -558,7 +616,7 @@ def get_column_dtype(self, column_name: str) -> type: return self.column_dtypes_cached[column_name] filtered_values = self.events[column_name].dropna() - #filtered_values = filtered_values[filtered_values != MD_NONE()] + # filtered_values = filtered_values[filtered_values != MD_NONE()] if filtered_values.dtype == object: filtered_values = filtered_values[filtered_values != MD_NONE()] From d144015c662f5834196a4e9d9d354a72e31f4f04 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:10:00 +0800 Subject: [PATCH 04/36] Index pre and post call records by ID during inference --- .../trace-pandas-indexes-20261025.json | 319 ++++++++++++++++++ docs/measurements/trace-pandas-indexes.md | 38 +++ tests/test_call_index.py | 251 ++++++++++++++ tools/benchmarks/trace_pandas_indexes.py | 141 ++++++++ traincheck/trace/trace_pandas.py | 110 +++++- 5 files changed, 845 insertions(+), 14 deletions(-) create mode 100644 docs/measurements/trace-pandas-indexes-20261025.json create mode 100644 docs/measurements/trace-pandas-indexes.md create mode 100644 tests/test_call_index.py create mode 100644 tools/benchmarks/trace_pandas_indexes.py diff --git a/docs/measurements/trace-pandas-indexes-20261025.json b/docs/measurements/trace-pandas-indexes-20261025.json new file mode 100644 index 00000000..94cf3a94 --- /dev/null +++ b/docs/measurements/trace-pandas-indexes-20261025.json @@ -0,0 +1,319 @@ +{ + "measurement_date": "2026-10-25", + "source": { + "repository": "https://github.com/OrderLab/TrainCheck", + "base_commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "unchanged_prerequisites": [ + "deterministic inference sampling", + "example reservoir cap fixes" + ], + "trace_pandas_sha256": { + "legacy": "e1bb62eae653f95c5498dc921d991af013e8b5717398f94cfbc718812bb600ad", + "context_index": "13d5d1273595f8183d0511ae288a998cb57e3ea66dc232e4bfec503739de825c", + "context_and_call_index": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5" + }, + "pr_copies": { + "variants": { + "context_index": { + "measured_source_sha256": "13d5d1273595f8183d0511ae288a998cb57e3ea66dc232e4bfec503739de825c", + "pr_source_sha256": "52c296917bba877b1e87aebb54cfee1e05813e2ae32f85744d5ddfe27383d5e0", + "equivalent_ast_after_normalizing_only_documented_typing_change": true + }, + "context_and_call_index": { + "measured_source_sha256": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5", + "pr_source_sha256": "58aebdba9955834f4d6b152527e8ba2f9166f6e62f43a098c48b115b0bbc48f5", + "equivalent_ast_after_normalizing_only_documented_typing_change": true + } + }, + "changes_from_measured_sources": [ + "Both PR copies widen the inherited call-event cache annotation to include IncompleteFuncCallEvent, matching the base Trace contract, and import that existing type.", + "Final PR copy adds narrow E721 suppressions to preserve intentional exact-string checks; only comments/parentheses changed in lookup logic.", + "Existing explicit None-or-cache annotations are retained for the new lazy index." + ], + "no_frozen_source_edits": true, + "local_validation_tools": { + "ruff": "0.15.7", + "mypy": "1.19.1", + "black": "26.3.1", + "isort": "6.0.1" + }, + "pinned_repository_hooks_need_final_worktree_verification": true + } + }, + "saved_trace": { + "events": 573494, + "model_parameters": 6163712, + "training_steps": 10, + "archive_sha256": "62022da5a997c16ffe49aa44486a495b651c8758199f29732a216c7cf257559d", + "raw_inputs_unchanged": true, + "collection_pytorch": "2.16.0.dev20261012+cu130" + }, + "runtime": { + "platform": "Linux", + "python": "3.10.16", + "pandas": "2.3.3", + "numpy": "1.26.4", + "torch": "2.2.2+cu121", + "workers": 1, + "native_threads": 1 + }, + "preparation": { + "runs": { + "legacy": { + "context_index_seconds": 159.43910144269466, + "load_seconds": 90.64373006299138, + "rows": 573494, + "stage_rows": { + "training": 573494 + }, + "stage_split_seconds": 80.21087664365768, + "total_seconds": 170.85460681840777, + "sampled_peak_pss_bytes": 2409772032, + "context_states": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 79.72032228112221, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 79.71877916157246, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + } + ] + }, + "context_index": { + "context_index_seconds": 0.8803581297397614, + "load_seconds": 10.837436098605394, + "rows": 573494, + "stage_rows": { + "training": 573494 + }, + "stage_split_seconds": 0.8764883801341057, + "total_seconds": 11.713924750685692, + "sampled_peak_pss_bytes": 2411297792, + "context_states": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 0.4527508243918419, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 0.42760730534791946, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + } + ] + } + }, + "comparison": { + "same_number_of_calls": true, + "calls": [ + { + "call": 0, + "rows": 573494, + "contexts": 711, + "equal": true + }, + { + "call": 1, + "rows": 573494, + "contexts": 711, + "equal": true + } + ], + "all_equal": true + }, + "paired_repetitions": 1, + "timing_scope": "Load/init plus stage splitting, excluding process startup; includes context-state serialization. Context-index timers exclude serialization. PSS peaks are sampled." + }, + "large_trace_diagnostic": { + "runs": { + "optimized-infer-profile": { + "wall_seconds": 546.4971170388162, + "exit_code": 1, + "stop_reason": null, + "source_unchanged": true, + "traces_unchanged": true, + "error": "ValueError: Too many positional arguments for function torch._prims_common.is_contiguous, expecting 1 (['a']) but got 2", + "lookups": [ + { + "function": "get_pre_func_call_record", + "primitive_calls": 6891, + "calls": 6891, + "self_seconds": 0.087277701, + "cumulative_seconds": 187.91447304200003 + }, + { + "function": "get_post_func_call_record", + "primitive_calls": 13498, + "calls": 13498, + "self_seconds": 0.18074669200000001, + "cumulative_seconds": 289.47232359500003 + } + ] + }, + "call-index-infer-profile": { + "wall_seconds": 71.98188185319304, + "exit_code": 1, + "stop_reason": null, + "source_unchanged": true, + "traces_unchanged": true, + "error": "ValueError: Too many positional arguments for function torch._prims_common.is_contiguous, expecting 1 (['a']) but got 2", + "lookups": [ + { + "function": "get_pre_func_call_record", + "primitive_calls": 6891, + "calls": 6891, + "self_seconds": 0.020590995, + "cumulative_seconds": 1.034124108 + }, + { + "function": "get_post_func_call_record", + "primitive_calls": 13498, + "calls": 13498, + "self_seconds": 0.038829666000000006, + "cumulative_seconds": 3.888496781 + } + ] + } + }, + "same_error": true, + "completed_inference": false + }, + "diagnostic_scope": { + "completed_inference": false, + "profiled": true, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "same_error_is_framework_version_mismatch": true, + "eligible_containment_apis_in_replay": 35, + "eligible_containment_apis_in_collection_environment_log": 41, + "time_to_error_is_not_completed_inference_throughput": true + }, + "correctness": { + "differential_tests": 32, + "context_randomized_traces": 80, + "independent_extra_call_query_comparisons": 640, + "small_replay_run_count": 9, + "all_full_json_comparisons_equal": true, + "comparison": "Structural JSON multisets; object-key and file-line order ignored, multiplicity and array order retained.", + "runs": [ + { + "case": "mnist-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "base" + }, + { + "case": "mnist-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "optimized" + }, + { + "case": "transformer-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "optimized" + }, + { + "case": "transformer-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "base" + }, + { + "case": "dcgan-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "base" + }, + { + "case": "dcgan-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "optimized" + }, + { + "case": "mnist-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "context_and_call_index" + }, + { + "case": "transformer-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "context_and_call_index" + }, + { + "case": "dcgan-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "context_and_call_index" + } + ], + "outputs_by_workload": { + "mnist-fixed-seed42": { + "invariants": { + "canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84", + "distinct_records": 193, + "records": 193 + }, + "failed_hypotheses": { + "canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820", + "distinct_records": 21, + "records": 21 + } + }, + "transformer-fixed-seed42": { + "invariants": { + "canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09", + "distinct_records": 518, + "records": 518 + }, + "failed_hypotheses": { + "canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7", + "distinct_records": 10, + "records": 10 + } + }, + "dcgan-fixed-seed42": { + "invariants": { + "canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a", + "distinct_records": 512, + "records": 512 + }, + "failed_hypotheses": { + "canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204", + "distinct_records": 86, + "records": 86 + } + } + } + }, + "limitations": [ + "Event row order, call IDs, and event types must remain stable after initialization. Frame replacement or a length change rebuilds the call index; same-size in-place changes or reordering are unsupported.", + "The large trace uses a different PyTorch version; both diagnostic variants end at the same signature error. No complete large-trace inference or online-checker overhead result is claimed." + ] +} diff --git a/docs/measurements/trace-pandas-indexes.md b/docs/measurements/trace-pandas-indexes.md new file mode 100644 index 00000000..03e5d3b0 --- /dev/null +++ b/docs/measurements/trace-pandas-indexes.md @@ -0,0 +1,38 @@ +# TracePandas indexing measurements + +The context index removes two full-frame scans per context instance. The call-ID index removes repeated full-frame scans for pre/post lookups, including lookups made before containment sampling. The optimizations preserve selection coverage and record-validation behavior. + +The adjacent `trace-pandas-indexes-20261025.json` records sanitized measurements, source/input hashes, environment versions, state equality, profiler call counts, and nine successful small-workload replay results. The complete saved Llama trace contains 573,494 events from ten training steps of a 6,163,712-parameter model. + +| Measurement | Before | After | +|---|---:|---:| +| Preparation, original → context index | 170.855 s | 11.714 s | +| Context-index work within preparation | 159.439 s | 0.880 s | +| Pre-record profiling, context → combined | 187.914 s / 6,891 calls | 1.034 s / 6,891 calls | +| Post-record profiling, context → combined | 289.472 s / 13,498 calls | 3.888 s / 13,498 calls | +| Diagnostic wall time to the same error | 546.50 s | 71.98 s | + +Preparation is one paired measurement on the same Linux host using one native thread. Total preparation includes state serialization and excludes process startup; context-index timers exclude serialization. Both preparation variants produce the same 711 ordered states in the full trace and stage trace. The profiler's cumulative times overlap across callers and must not be added indiscriminately. + +The large-trace diagnostics both fail at `torch._prims_common.is_contiguous` argument binding: replay PyTorch 2.2.2 expects one argument while the PyTorch 2.16.0.dev20261012 trace contains two. These results demonstrate reduced search cost, not completed large-trace inference throughput or online-checker overhead. The replay environment also resolves fewer eligible containment APIs than the collection environment (35 versus 41), so a final reference run needs compatible framework dependencies. + +Nine small-workload runs complete successfully: original/context/combined variants for MNIST, small Transformer, and DCGAN. Full invariant and failed-hypothesis JSON match under structural multiset comparison, retaining multiplicity and array order. Per-workload counts are 193/21, 518/10, and 512/86. + +## Reproduce local correctness and synthetic measurements + +From a checkout with TrainCheck dependencies installed: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m unittest tests.test_context_index tests.test_call_index +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m tools.benchmarks.trace_pandas_indexes --output measurements.json +``` + +The synthetic benchmark constructs its inputs and needs no external trace or cluster. It alternates variant order, checks exact results, and reports every repetition and the median. Its context case times construction plus state serialization; its call case times construction and all lookups, including the first lazy-index build. Its sizes/repetition count are configurable; these synthetic timings are separate from the saved-trace measurements above. + +The oracle fixture contains only the exact original context/pre/post/stage methods from commit `9eb4f320d3db34d372145261704b805b88e39bda`; supporting types and the constructor come from the package under test. Provenance includes the original file and method hashes. The fixture's `TracePandas` binding ensures stage splitting creates legacy instances, and the tests assert this explicitly. + +The published source also corrects the inherited call-event cache annotation to include `IncompleteFuncCallEvent`; narrow lint comments preserve exact-string guards. These typing/comment changes do not alter lookup behavior. + +## Cache contract + +Event rows, their order, call IDs, and event types must remain stable after initialization. Frame replacement or a length change rebuilds the call index. Same-size in-place edits or row reordering can leave stale positions and require a new `TracePandas` instance. The tested inference path does not perform those mutations. This change does not introduce general cache invalidation for arbitrary dataframe mutations. diff --git a/tests/test_call_index.py b/tests/test_call_index.py new file mode 100644 index 00000000..acf8f030 --- /dev/null +++ b/tests/test_call_index.py @@ -0,0 +1,251 @@ +"""Call-record index differential tests against frozen legacy query methods.""" + +import copy +import logging +import unittest + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as NEW +from tests.fixtures import legacy_trace_pandas as OLD + +PRE = OLD.TraceLineType.FUNC_CALL_PRE +POST = OLD.TraceLineType.FUNC_CALL_POST +EXCEPTION = OLD.TraceLineType.FUNC_CALL_POST_EXCEPTION + + +def record(call_id, when, kind, *, pid=1, tid=2, stage="train", **extra): + return { + "func_call_id": call_id, + "function": "fixtures.work", + "time": when, + "type": kind, + "process_id": pid, + "thread_id": tid, + "args": {"0": {"int": 7}}, + "kwargs": {}, + "return_values": {"int": 9}, + "exception": "fixture exception", + "meta_vars.stage": stage, + **extra, + } + + +def pair(call_id="call", start=0, **extra): + return [ + record(call_id, start, PRE, **extra), + record(call_id, start + 1, POST, **extra), + ] + + +def make(module, frame, truncate=False): + if not isinstance(frame, pd.DataFrame): + frame = pd.DataFrame(copy.deepcopy(frame)) + else: + frame = frame.copy(deep=True) + return module.TracePandas(frame, truncate_incomplete_func_calls=truncate) + + +def outcome(trace, method, call_id): + try: + return ("ok", getattr(trace, method)(call_id)) + except Exception as exc: + return (type(exc).__name__, str(exc)) + + +class StringAlias: + def __eq__(self, other): + return other == "alias" + + def __repr__(self): + return "StringAlias()" + + +class CallIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.WARNING) + self.addCleanup(logging.disable, previous) + + def equivalent(self, rows, call_ids, *, truncate=False): + old, new = [make(module, rows, truncate) for module in [OLD, NEW]] + for call_id in call_ids: + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + with self.subTest(call_id=repr(call_id), method=method): + expected = outcome(old, method, call_id) + actual = outcome(new, method, call_id) + self.assertEqual(expected, actual) + return old, new + + def test_exact_pre_post_records_and_row_order(self): + rows = pair("z", 10) + pair("a", 0, pid=9, tid=8) + _, new = self.equivalent(rows, ["a", "z", "missing"]) + self.assertEqual(new.get_pre_func_call_record("z"), rows[0]) + self.assertEqual(new.get_post_func_call_record("a"), rows[3]) + self.assertIs(type(new.get_pre_func_call_record("z")["time"]), int) + + def test_exception_post_record(self): + rows = [record("call", 0, PRE), record("call", 1, EXCEPTION)] + old, new = self.equivalent(rows, ["call"]) + self.assertEqual( + old.query_func_call_event("call").__dict__, + new.query_func_call_event("call").__dict__, + ) + + def test_duplicate_pre_count_assertions(self): + rows = [record("bad", 0, PRE), record("bad", 1, PRE), record("bad", 2, POST)] + old, new = self.equivalent(rows, ["bad"]) + self.assertEqual( + outcome(new, "get_pre_func_call_record", "bad"), + ("AssertionError", "2 pre call events found for bad, expected 1"), + ) + + def test_duplicate_post_and_exception_count_assertions(self): + rows = [ + record("bad", 0, PRE), + record("bad", 1, POST), + record("bad", 2, EXCEPTION), + ] + self.equivalent(rows, ["bad"]) + + def test_unqueried_duplicates_do_not_raise_early(self): + rows = pair("good") + [record("bad", 4, PRE), record("bad", 5, PRE)] + _, new = self.equivalent(rows, ["good"]) + self.assertIsNotNone(new._func_call_positions) + self.assertEqual(len(new._func_call_positions["bad"][0]), 2) + + def test_missing_outermost_and_nonoutermost_posts_per_thread(self): + rows = [ + record("outer", 100, PRE), + record("inner", 0, PRE), + record("other_thread", 2, PRE, tid=3), + ] + old, new = self.equivalent(rows, ["outer", "inner", "other_thread", "absent"]) + self.assertIsNone(new.get_post_func_call_record("outer")) + self.assertIsNone(new.get_post_func_call_record("other_thread")) + self.assertEqual( + outcome(new, "get_post_func_call_record", "inner"), + ( + "ValueError", + "No post call event found for inner, but it is not the outermost function call.", + ), + ) + + def test_duplicate_dataframe_labels_use_positions(self): + rows = pair("a") + pair("b", 4) + self.equivalent(pd.DataFrame(rows, index=[17, 17, 17, 17]), ["a", "b"]) + + def test_null_ids_never_use_dictionary_equality(self): + rows = pair("good") + for value in [None, float("nan"), pd.NA]: + rows += pair(value, 5) + _, new = self.equivalent(rows, ["good", None, float("nan"), pd.NA]) + self.assertIsNotNone(new._func_call_positions) + + def test_numeric_ids_and_custom_equality_fall_back(self): + for rows, ids in [ + (pair(7), [7, "7"]), + (pair(StringAlias()), ["alias"]), + (pair(np.str_("numpy")), ["numpy"]), + (pair("good") + pair(7, 5), ["good", 7]), + ]: + old, new = self.equivalent(rows, ids) + self.assertIsNone(new._func_call_positions) + + def test_pandas_string_and_unsupported_categorical_dtypes(self): + for dtype in ["string", "category"]: + frame = pd.DataFrame(pair("a") + pair("b", 4)) + for column in ["func_call_id", "type"]: + frame[column] = frame[column].astype(dtype) + _, new = self.equivalent(frame, ["a", "b"]) + if dtype == "category": + self.assertIsNone(new._func_call_positions) + else: + self.assertIsNotNone(new._func_call_positions) + + def test_null_event_types_are_ignored(self): + rows = pair("good") + rows += [record("extra", 3, value) for value in [None, float("nan"), pd.NA]] + self.equivalent(rows, ["good", "extra"]) + + def test_stage_split_indexes_are_independent(self): + rows = pair("init", stage="init") + pair("train", 5, stage="train") + rows += [ + record("cross", 10, PRE, stage="init"), + record("cross", 11, POST, stage="train"), + ] + old, new = self.equivalent(rows, ["init", "train", "cross"]) + old_stages = old.get_traces_for_stage() + new_stages = new.get_traces_for_stage() + for stage in old_stages: + self.assertIs(type(old_stages[stage]), OLD.TracePandas) + self.assertIs(type(new_stages[stage]), NEW.TracePandas) + for call_id in ["init", "train", "cross"]: + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + self.assertEqual( + outcome(old_stages[stage], method, call_id), + outcome(new_stages[stage], method, call_id), + ) + self.assertIsNot( + new._func_call_positions, new_stages["train"]._func_call_positions + ) + self.assertIsNot( + new_stages["init"]._func_call_positions, + new_stages["train"]._func_call_positions, + ) + + def test_top_level_return_dicts_are_fresh(self): + for module in [OLD, NEW]: + trace = make(module, pair()) + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + first = getattr(trace, method)("call") + second = getattr(trace, method)("call") + self.assertIsNot(first, second) + first["function"] = "mutated" + first["new_key"] = True + self.assertEqual(getattr(trace, method)("call"), second) + + def test_index_is_lazy_reused_and_rebuilt_on_frame_replacement(self): + old, new = [make(module, pair()) for module in [OLD, NEW]] + self.assertIsNone(new._func_call_positions) + new.get_pre_func_call_record("call") + cached = new._func_call_positions + new.get_post_func_call_record("call") + self.assertIs(new._func_call_positions, cached) + for trace in [old, new]: + trace.events = pd.DataFrame(pair("replacement", 9)) + for call_id in ["call", "replacement"]: + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + self.assertEqual( + outcome(old, method, call_id), outcome(new, method, call_id) + ) + self.assertIsNot(new._func_call_positions, cached) + + def test_in_place_row_count_change_rebuilds(self): + old, new = [make(module, pair()) for module in [OLD, NEW]] + new.get_pre_func_call_record("call") + cached = new._func_call_positions + for trace in [old, new]: + trace.events.loc[len(trace.events)] = record("call", 4, PRE) + self.assertEqual( + outcome(old, "get_pre_func_call_record", "call"), + outcome(new, "get_pre_func_call_record", "call"), + ) + self.assertIsNot(new._func_call_positions, cached) + + def test_index_built_after_incomplete_trailing_call_truncation(self): + rows = [ + record("outer", 0, PRE), + record("inner", 1, PRE), + record("inner", 2, POST), + record("outer", 10, POST), + record("trailing", 11, PRE), + ] + old, new = self.equivalent(rows, ["outer", "inner", "trailing"], truncate=True) + self.assertEqual(len(new.events), 4) + self.assertIs(new._func_call_positions_source, new.events) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/benchmarks/trace_pandas_indexes.py b/tools/benchmarks/trace_pandas_indexes.py new file mode 100644 index 00000000..a9ff2fbe --- /dev/null +++ b/tools/benchmarks/trace_pandas_indexes.py @@ -0,0 +1,141 @@ +"""Reproduce bounded synthetic TracePandas indexing measurements from a checkout. + +Run: python -m tools.benchmarks.trace_pandas_indexes --output measurements.json +No training, cluster access, or saved trace files are required. Each lookup run +includes its first lazy-index construction. The legacy oracle preserves the four +original query methods; supporting runtime types are identical on both sides. +""" + +import argparse +import hashlib +import inspect +import json +import logging +import platform +import statistics +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as candidate +from tests.fixtures import legacy_trace_pandas as legacy +from tests.test_call_index import pair +from tests.test_context_index import cycle, encode, event + + +def compare(frame, operation, repetitions): + seconds = {"legacy": [], "candidate": []} + expected = None + for repetition in range(repetitions): + variants = [("legacy", legacy), ("candidate", candidate)] + if repetition % 2: + variants.reverse() + for name, module in variants: + events = frame.copy(deep=True) + started = time.perf_counter() + value = operation(module, events) + seconds[name].append(time.perf_counter() - started) + if expected is None: + expected = value + assert value == expected, f"{name} result differs from the legacy oracle" + medians = {name: statistics.median(values) for name, values in seconds.items()} + return { + "elapsed_seconds": seconds, + "median_seconds": medians, + "speedup": medians["legacy"] / medians["candidate"], + "exact_results_equal": True, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["all", "context", "call"], default="all") + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--contexts", type=int, default=500) + parser.add_argument("--noise", type=int, default=50_000) + parser.add_argument("--calls", type=int, default=20_000) + parser.add_argument( + "--queries", type=int, default=500, help="Pre/post pairs queried" + ) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + for name in ["repetitions", "contexts", "calls", "queries"]: + if getattr(args, name) < 1: + parser.error(f"--{name} must be positive") + if args.noise < 0: + parser.error("--noise must be nonnegative") + logging.disable(logging.WARNING) + results = {} + if args.mode in ("all", "context"): + rows = [] + for index in range(args.contexts): + rows.extend( + cycle(index * 3, obj=index % 20, name=f"fixtures.Ctx{index % 4}") + ) + rows.extend( + event(args.contexts * 3 + index, "math_call") for index in range(args.noise) + ) + frame = pd.DataFrame(rows) + + def prepare(module, events): + return encode( + module.TracePandas(events, truncate_incomplete_func_calls=False) + ) + + results["context"] = { + "rows": len(frame), + "contexts": args.contexts, + "timed_work": "constructor plus context-state serialization", + **compare(frame, prepare, args.repetitions), + } + if args.mode in ("all", "call"): + frame = pd.DataFrame( + [ + record + for index in range(args.calls) + for record in pair(f"call-{index:08}", start=index * 2) + ] + ) + wanted = [ + f"call-{(index * 117) % args.calls:08}" for index in range(args.queries) + ] + + def lookup(module, events): + trace = module.TracePandas(events, truncate_incomplete_func_calls=False) + return [ + ( + trace.get_pre_func_call_record(call_id), + trace.get_post_func_call_record(call_id), + ) + for call_id in wanted + ] + + results["call"] = { + "rows": len(frame), + "record_queries": args.queries * 2, + "timed_work": "constructor plus record lookups, including lazy-index construction", + **compare(frame, lookup, args.repetitions), + } + output = { + "description": "Synthetic fixtures, alternating variant order; no training or inference", + "repetitions": args.repetitions, + "runtime": { + "python": platform.python_version(), + "pandas": pd.__version__, + "numpy": np.__version__, + }, + "candidate_source_sha256": hashlib.sha256( + Path(inspect.getfile(candidate)).read_bytes() + ).hexdigest(), + "results": results, + } + rendered = json.dumps(output, indent=2) + "\n" + if args.output: + args.output.write_text(rendered) + print(rendered, end="") + + +if __name__ == "__main__": + main() diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 1cc0eb6b..2ab42924 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -103,6 +103,14 @@ def __init__(self, events, truncate_incomplete_func_calls=True): "Done truncating incomplete trailing function calls from the trace." ) + # Build call-record indexes lazily, after incomplete-call truncation. + # Like the other trace caches, these assume stable event values after + # initialization. Replacing the frame or changing its length rebuilds + # this index; same-shape in-place cell edits require a new TracePandas. + self._func_call_positions_source: pd.DataFrame | None = None + self._func_call_positions_length = -1 + self._func_call_positions: dict[str, tuple[list[int], list[int]]] | None = None + self.column_dtypes_cached = {} self.stage_traces: dict[str, "TracePandas"] = {} @@ -1065,13 +1073,83 @@ def query_var_changes_within_func_call( (start_time, end_time), process_id=pre_record["process_id"] ) + def _get_func_call_record_positions( + self, func_call_id: str, *, post: bool + ) -> list[int] | None: + """Return indexed positions, or None to use the original pandas filter. + + Index only ordinary strings/nulls: pandas equality for arbitrary objects + and extension dtypes can differ from Python dictionary equality. + """ + if type(func_call_id) is not str: # noqa: E721 - exclude custom equality + return None + + if ( + self._func_call_positions_source is not self.events + or self._func_call_positions_length != len(self.events) + ): + ids = self.events["func_call_id"] + event_types = self.events["type"] + positions: dict[str, tuple[list[int], list[int]]] | None = None + if all( + isinstance(column, pd.Series) + and ( + pd.api.types.is_object_dtype(column.dtype) + or isinstance(column.dtype, pd.StringDtype) + ) + for column in (ids, event_types) + ): + positions = {} + for position, (call_id, event_type) in enumerate(zip(ids, event_types)): + # None, NaN, and pd.NA cannot equal an ordinary string in + # the original pandas filter. All other nonstrings use it. + if any( + type(value) is not str # noqa: E721 - exclude custom equality + and value is not None + and value is not pd.NA + and not (isinstance(value, float) and pd.isna(value)) + for value in (call_id, event_type) + ): + positions = None + break + if ( + type(call_id) is not str # noqa: E721 - exact scalar types + or type(event_type) is not str # noqa: E721 + ): + continue + if event_type == TraceLineType.FUNC_CALL_PRE: + side = 0 + elif event_type in ( + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ): + side = 1 + else: + continue + if call_id not in positions: + positions[call_id] = ([], []) + positions[call_id][side].append(position) + + self._func_call_positions = positions + self._func_call_positions_source = self.events + self._func_call_positions_length = len(self.events) + + if self._func_call_positions is None: + return None + records = self._func_call_positions.get(func_call_id) + return records[int(post)] if records is not None else [] + def get_pre_func_call_record(self, func_call_id: str) -> dict: """Get the pre call record of a function given its func_call_id.""" # TODO: check if this is correct - pre_records = self.events[ - (self.events["func_call_id"] == func_call_id) - & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) - ] + positions = self._get_func_call_record_positions(func_call_id, post=False) + if positions is None: + pre_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ] + else: + pre_records = self.events.iloc[positions] assert ( len(pre_records) == 1 @@ -1086,17 +1164,21 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: Returns None if the post call event is not found and the pre-call event is the outermost function call. (see the doc of `_rm_incomplete_trailing_func_calls`) """ # TODO: check if this is correct - post_records = self.events[ - (self.events["func_call_id"] == func_call_id) - & ( - self.events["type"].isin( - [ - TraceLineType.FUNC_CALL_POST, - TraceLineType.FUNC_CALL_POST_EXCEPTION, - ] + positions = self._get_func_call_record_positions(func_call_id, post=True) + if positions is None: + post_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) ) - ) - ] + ] + else: + post_records = self.events.iloc[positions] assert ( len(post_records) <= 1 From 7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:12:24 +0800 Subject: [PATCH 05/36] Fix recorder re-entry and atomic trace buffer delivery --- docs/experiments/trace-integrity-results.json | 125 ++++++ docs/experiments/trace-integrity.md | 43 ++ tests/test_trace_integrity.py | 272 ++++++++++++ traincheck/instrumentor/dumper.py | 27 +- traincheck/instrumentor/tracer.py | 417 ++++++++++-------- 5 files changed, 684 insertions(+), 200 deletions(-) create mode 100644 docs/experiments/trace-integrity-results.json create mode 100644 docs/experiments/trace-integrity.md create mode 100644 tests/test_trace_integrity.py diff --git a/docs/experiments/trace-integrity-results.json b/docs/experiments/trace-integrity-results.json new file mode 100644 index 00000000..d44c4508 --- /dev/null +++ b/docs/experiments/trace-integrity-results.json @@ -0,0 +1,125 @@ +{ + "measurement_date": "2026-10-13", + "traincheck_base": "9eb4f320d3db34d372145261704b805b88e39bda", + "benchmark_source": "ae6601978217bc0f06b9a78479cf00f0bc7242c9", + "controlled_reentry": { + "buggy": { + "application_calls": 100, + "events": 400, + "metadata_helper_events": 200, + "application_results_match": true, + "recursion_errors": 0, + "scope": "Controlled injected conversion helper, not a natural workload frequency" + }, + "fixed": { + "application_calls": 100, + "events": 200, + "metadata_helper_events": 0, + "application_results_match": true, + "recursion_errors": 0, + "scope": "Controlled injected conversion helper, not a natural workload frequency" + } + }, + "controlled_buffer_interleavings": { + "scope": "Controlled forced interleavings, not a natural workload frequency or performance benchmark", + "trials_per_mode": 50, + "records_preloaded_per_trial": 100, + "results": { + "buggy": { + "duplicate": { + "records": 10000, + "duplicates": 5000, + "missing": 0 + }, + "loss": { + "records": 5000, + "duplicates": 0, + "missing": 50 + } + }, + "fixed": { + "duplicate": { + "records": 5000, + "duplicates": 0, + "missing": 0 + }, + "loss": { + "records": 5050, + "duplicates": 0, + "missing": 0 + } + } + } + }, + "natural_workload_matrix": { + "runs_completed": 45, + "instrumented_runs": 36, + "conditions": [ + "baseline", + "buggy", + "guard", + "buffer", + "fixed" + ], + "seeds": [ + 42, + 43, + 44 + ], + "training_steps": 5, + "batch_size": 2, + "native_threads": 1, + "runtime": { + "python": "3.10.16", + "torch": "2.2.2+cu121", + "gpu": "RTX A2000 12 GB" + }, + "scope": "Adapted small model workloads with ordinary local tensors; not full published benchmark reproductions. All instrumented variants share the same dynamic-type-name compatibility patch.", + "instrumentation": "Default API coverage, use_full_instr=False, sampling interval one, no model/proxy tracking; training bodies only.", + "workloads": [ + { + "name": "mnist", + "parameters": 1199882, + "instrumented_runs": 12, + "records_per_run": 654, + "raw_bytes_per_run": 472863, + "instrumented_variants_and_seeds_equal": true + }, + { + "name": "transformer", + "parameters": 23522, + "instrumented_runs": 12, + "records_per_run": 2006, + "raw_bytes_per_run": 1564970, + "instrumented_variants_and_seeds_equal": true + }, + { + "name": "dcgan", + "parameters": 190672, + "instrumented_runs": 12, + "records_per_run": 4596, + "raw_bytes_per_run": 3401128, + "instrumented_variants_and_seeds_equal": true + } + ], + "total_instrumented_records": 87072, + "total_instrumented_raw_bytes": 65267532, + "all_recorded_losses_and_final_model_state_hashes_match_baseline": true, + "observed_duplicate_records": 0, + "observed_recursion_error_events": 0, + "natural_trace_size_reduction_observed": false + }, + "evidence_artifact_sha256": { + "matrix.csv": "f196211cc478ec532338de06247895cfa5bcd379770fd66583c5b80189127cfe", + "matrix-summary.json": "a6a18173e3a0550500160b0e0b3169553ee714bb0ce175048cc5998f19c2ba06", + "controlled-races.json": "6b0ab42d2d8f1ff37557627bd8b261610f7d84adf65f88e2025739dfd875c4aa", + "finite-reentry-buggy.json": "25f5dfeab0f68c080682f21f0fd6e0314e7c4254a2a9e538b9081aad3a382475", + "finite-reentry-fixed.json": "18470fb8d6c929296feecf3485242775dd63331e89c0669244d7ed763bdf6595" + }, + "limitations": [ + "Injected helpers and forced interleavings establish possible failure severity, not natural occurrence rates.", + "All 45 ordinary-tensor runs completed; none of the 36 instrumented runs showed trace inflation or a trace-size reduction from these fixes.", + "No general performance improvement or invariant-checking overhead claim is made.", + "Historical incomplete and completed TorchTitan traces are not comparable percentage-size-reduction measurements." + ] +} diff --git a/docs/experiments/trace-integrity.md b/docs/experiments/trace-integrity.md new file mode 100644 index 00000000..68a1dfc6 --- /dev/null +++ b/docs/experiments/trace-integrity.md @@ -0,0 +1,43 @@ +# Recorder re-entry and buffer integrity + +Recorder-owned argument conversion, tensor metadata inspection, and JSON encoding can call instrumented APIs again. The wrapper previously did not honor the conversion helper's local suppression flag, so recorder work could generate more records or recurse. Independently, the producer and periodic flusher accessed the same buffer without synchronization: overlapping flushes could duplicate a batch, and a producer append could be cleared before delivery. + +The recorder now uses an exception-safe, thread-local suppression depth around its own work. It releases suppression before executing application code, preserving nested application calls and generator execution. A class context manager preserves frozen framework exceptions. The buffer snapshots mutable values synchronously, then locks append, queue handoff, and clear as one ordered operation. Buffers still belong to individual producer threads for trace routing. + +## Controlled impact + +These tests deliberately inject a conversion helper or force a thread interleaving; they do not estimate how often an ordinary training job triggers either defect. + +| Controlled trigger | Buggy | Fixed | +|---|---:|---:| +| 100 application calls with a wrapped serialization helper | 400 records, including 200 recorder-owned records | 200 records, zero recorder-owned records | +| Two overlapping flushes; 50 trials × 100 records | 10,000 emitted, including 5,000 duplicates | 5,000 emitted, zero duplicates | +| Append during paused flush; 50 trials, 5,050 expected records | 50 records missing | Zero records missing | + +The finite re-entry case preserves application return values and raises no `RecursionError`: self-tracing can inflate a trace without an obvious recursion failure. The buffer race can cause loss as well as duplication; valid pre/post pairing alone cannot detect a completely missing pair. + +## Ordinary workload control + +All 45 runs completed: MNIST CNN, a small Transformer, and reduced-width DCGAN × five conditions × seeds 42/43/44. Each run used five training steps, batch size two, one GPU, and one native CPU thread. Conditions were no instrumentation, the buggy snapshot, guard only, buffer lock only, and both fixes. + +None of the 36 instrumented runs triggered either defect. Each instrumented condition had the same record count and raw byte size at every seed: + +| Workload | Records per run | Raw bytes per run | Buggy → fixed size change | +|---|---:|---:|---:| +| MNIST CNN | 654 | 472,863 | 0% | +| Small Transformer | 2,006 | 1,564,970 | 0% | +| DCGAN | 4,596 | 3,401,128 | 0% | + +All recorded losses and canonical final model-state hashes matched their corresponding uninstrumented baselines. These are adapted micro-workloads with ordinary local tensors on PyTorch 2.2.2+cu121, not full published benchmark reproductions or DTensor coverage. The observations support these correctness repairs; they do not establish a general performance improvement or invariant-checking overhead result. + +## Validation and provenance + +Run the eleven portable regression tests from the checkout: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m unittest tests.test_trace_integrity -v +``` + +The suite covers recorder re-entry, JSON emission re-entry, real tensor snapshots, nested application calls, generators, original and frozen exceptions, thread-local suppression, overlapping flushes, and concurrent producer ordering. + +[Sanitized results](trace-integrity-results.json) retain controlled counts, workload settings, base/source revisions, and evidence-file hashes. Measurements used TrainCheck base `9eb4f320d3db34d372145261704b805b88e39bda` and adapted benchmark source `ae6601978217bc0f06b9a78479cf00f0bc7242c9`; all instrumented conditions shared the same dynamic-type-name compatibility patch. No private paths, raw training traces, or checkpoint data are included. diff --git a/tests/test_trace_integrity.py b/tests/test_trace_integrity.py new file mode 100644 index 00000000..26d9e0b2 --- /dev/null +++ b/tests/test_trace_integrity.py @@ -0,0 +1,272 @@ +"""Regression coverage for tracer self-reentry and concurrent buffer flushing.""" + +import copy +import json +import threading +import unittest +from queue import Queue +from unittest.mock import patch + +from traincheck.instrumentor import dumper, tracer + + +def wrapped(fn, name="outer", scan_proxy=False): + def call(*args, **kwargs): + return tracer.function_wrapper( + fn, + name, + False, + scan_proxy, + False, + True, + None, + True, + None, + False, + {}, + False, + *args, + **kwargs, + ) + + return call + + +class TracingTests(unittest.TestCase): + def setUp(self): + self.records = [] + self.patches = [ + patch.object(tracer.config, "DISABLE_WRAPPER", False), + patch.object( + tracer, + "dump_trace_API", + side_effect=lambda r: self.records.append(copy.deepcopy(r)), + ), + ] + for p in self.patches: + p.start() + self.addCleanup(p.stop) + + def test_serializer_reentry_does_not_trace_itself(self): + helper = wrapped(lambda x: x, "serializer_helper") + with patch.object( + tracer, "var_to_serializable", side_effect=lambda x: helper(x) + ): + self.assertEqual(wrapped(lambda x: x + 1)(3), 4) + self.assertEqual([r["function"] for r in self.records], ["outer", "outer"]) + + def test_encoding_reentry_and_nested_guard_restoration(self): + helper = wrapped(lambda: "encoded", "encoder_helper") + + def emit(record): + helper() + self.records.append(copy.deepcopy(record)) + + with ( + patch.object(tracer, "dump_trace_API", side_effect=emit), + patch.object(tracer, "var_to_serializable", side_effect=copy.deepcopy), + ): + wrapped(lambda x: x)(1) + with tracer.suppress_tracing(): + tracer.to_dict_return_value(1) + self.assertTrue(tracer.tracing_suppressed()) + self.assertFalse(tracer.tracing_suppressed()) + self.assertEqual([r["function"] for r in self.records], ["outer", "outer"]) + + def test_nested_user_calls_and_mutable_snapshots(self): + inner = wrapped(lambda x: x.append(2), "inner") + + def outer(x): + inner(x) + return x + + with patch.object(tracer, "var_to_serializable", side_effect=copy.deepcopy): + self.assertEqual(wrapped(outer)([1]), [1, 2]) + self.assertEqual( + [r["function"] for r in self.records], ["outer", "inner", "inner", "outer"] + ) + self.assertEqual(self.records[0]["args"][0], [1]) + self.assertEqual(self.records[-1]["return_values"], [1, 2]) + self.assertEqual( + self.records[0]["func_call_id"], self.records[-1]["func_call_id"] + ) + self.assertEqual( + self.records[1]["func_call_id"], self.records[2]["func_call_id"] + ) + + def test_real_tensor_conversion_preserves_pre_post_shape(self): + import torch + + value = torch.ones(2) + + def resize(tensor): + tensor.resize_(3) + return tensor + + self.assertIs(wrapped(resize)(value), value) + before = next(iter(self.records[0]["args"][0].values())) + after = next(iter(self.records[-1]["return_values"].values())) + self.assertEqual(before["shape"], (2,)) + self.assertEqual(after["shape"], (3,)) + self.assertEqual(self.records[0]["meta_vars"], self.records[-1]["meta_vars"]) + + def test_generator_application_calls_remain_visible(self): + inner = wrapped(lambda: 7, "inner") + + def values(): + yield inner() + + with patch.object(tracer, "var_to_serializable", side_effect=lambda x: repr(x)): + self.assertEqual(wrapped(lambda x: list(x), scan_proxy=True)(values()), [7]) + self.assertEqual( + [r["function"] for r in self.records], ["inner", "inner", "outer", "outer"] + ) + self.assertFalse(tracer.tracing_suppressed()) + + def test_conversion_exception_restores_guard(self): + with patch.object( + tracer, "var_to_serializable", side_effect=ValueError("conversion") + ): + with self.assertRaisesRegex(ValueError, "conversion"): + tracer.to_dict_args_kwargs((1,), {}) + with patch.object(tracer, "var_to_serializable", side_effect=copy.deepcopy): + wrapped(lambda x: x)(1) + self.assertEqual(len(self.records), 2) + + def test_frozen_framework_exception_preserved(self): + from dataclasses import dataclass + + @dataclass(frozen=True, slots=True) + class FrameworkSignal(Exception): + reason: str + + error = FrameworkSignal("cache bypass") + + def fail(): + raise error + + try: + wrapped(fail)() + except FrameworkSignal as caught: + self.assertIs(caught, error) + else: + self.fail("Original exception was swallowed") + self.assertFalse(tracer.tracing_suppressed()) + + def test_original_exception_preserved(self): + error = ValueError("original") + + def fail(): + raise error + + with self.assertRaises(ValueError) as caught: + wrapped(fail)() + self.assertIs(caught.exception, error) + self.assertEqual( + self.records[-1]["type"], tracer.TraceLineType.FUNC_CALL_POST_EXCEPTION + ) + self.assertEqual( + self.records[0]["func_call_id"], self.records[-1]["func_call_id"] + ) + + def test_conversion_suppression_is_thread_local(self): + entered, release = threading.Event(), threading.Event() + + def convert(x): + if x == "blocked": + entered.set() + if not release.wait(5): + raise TimeoutError("test release") + return x + + with patch.object(tracer, "var_to_serializable", side_effect=convert): + thread = threading.Thread( + target=tracer.to_dict_args_kwargs, args=(("blocked",), {}) + ) + thread.start() + try: + self.assertTrue(entered.wait(5)) + self.assertEqual(wrapped(lambda x: x)(3), 3) + finally: + release.set() + thread.join(5) + self.assertEqual(len(self.records), 2) + + +class BufferTests(unittest.TestCase): + def make_buffer(self, queue): + # Drive the periodic thread's flush explicitly, so scheduling is controlled. + with patch.object(threading.Thread, "start"): + return dumper.TraceBuffer( + lambda: queue, buffer_size=100, flush_interval=3600 + ) + + def test_concurrent_flush_exactly_once(self): + entered, release, second_done = ( + threading.Event(), + threading.Event(), + threading.Event(), + ) + + class BlockingQueue(Queue): + def put(self, item): + if not entered.is_set(): + entered.set() + if not release.wait(5): + raise TimeoutError("test release") + super().put(item) + + queue = BlockingQueue() + buffer = self.make_buffer(queue) + buffer.add_trace({"id": 1}) + first = threading.Thread(target=buffer._flush) + + def flush_again(): + buffer._flush() + second_done.set() + + second = threading.Thread(target=flush_again) + first.start() + try: + self.assertTrue(entered.wait(5)) + second.start() + second_done.wait(0.1) + finally: + release.set() + first.join(5) + second.join(5) + rows = [] + while not queue.empty(): + rows.extend(json.loads(row) for row in queue.get_nowait().splitlines()) + self.assertEqual(rows, [{"id": 1}]) + + def test_concurrent_producer_and_flush_order_and_snapshot(self): + queue = Queue() + buffer = self.make_buffer(queue) + done = threading.Event() + + def flush(): + while not done.is_set(): + buffer._flush() + + thread = threading.Thread(target=flush) + thread.start() + try: + item = {"id": 0} + for i in range(2000): + item["id"] = i + buffer.add_trace(item) + finally: + done.set() + thread.join(5) + buffer._flush() + rows = [] + while not queue.empty(): + rows.extend( + json.loads(row)["id"] for row in queue.get_nowait().splitlines() + ) + self.assertEqual(rows, list(range(2000))) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index b808e986..1bf7120f 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -181,7 +181,7 @@ def get_trace_VAR_dumper_queue(): class TraceBuffer: """Buffer traces and flush them to the trace log file periodically or when the buffer is full. - Caveat: **This class must be instantiated PER THREAD*, as the buffer is not thread-safe. + Caveat: **This class must be instantiated PER THREAD*, to preserve per-thread trace routing. Also, the queue must be created at the same time as the buffer (as did in the __init__ method). Previously, the queue was dynamic getted from the queue_getter method but it caused a mysterious bug, where a trace file contains traces of different thread ids and process ids. @@ -194,25 +194,32 @@ def __init__( self.buffer_size = buffer_size self.flush_interval = flush_interval self.buffer = [] + self._lock = threading.Lock() self.last_flush_time = time.time() self.flush_thread = threading.Thread(target=self._flush_periodically) self.flush_thread.daemon = True self.flush_thread.start() def add_trace(self, trace): - self.buffer.append( - serialize(trace) - ) # TODO: serialization step cannot be buffered rn as trace dicts might get modified - if ( - len(self.buffer) >= self.buffer_size - or (time.time() - self.last_flush_time) >= self.flush_interval - ): - self._flush() + # Snapshot mutable values in the producer, before returning to the caller. + serialized = serialize(trace) + with self._lock: + self.buffer.append(serialized) + if ( + len(self.buffer) >= self.buffer_size + or (time.time() - self.last_flush_time) >= self.flush_interval + ): + self._flush_locked() def _flush(self): + with self._lock: + self._flush_locked() + + def _flush_locked(self): if not self.buffer: return - # serialize all traces in the buffer + # Keep enqueue and clear atomic with respect to the producer and timer. + # Enqueue under the lock also preserves batch ordering. self.queue.put("\n".join(self.buffer)) self.buffer.clear() self.last_flush_time = time.time() diff --git a/traincheck/instrumentor/tracer.py b/traincheck/instrumentor/tracer.py index 1f9d4f1e..33c8f05e 100644 --- a/traincheck/instrumentor/tracer.py +++ b/traincheck/instrumentor/tracer.py @@ -1,6 +1,5 @@ import functools import importlib -from importlib.util import find_spec import inspect import os import pathlib @@ -9,6 +8,8 @@ import time import traceback import types +from contextlib import ContextDecorator +from importlib.util import find_spec from typing import Callable, Optional import torch @@ -46,8 +47,6 @@ IS_INSTRUMENTING = False -DISABLE_WRAPPER = False - # for prompt generation tasks using the transformers library (see traincheck/developer/instr_stage_annotation.py:annotate_answer_start_token_ids) GENERATE_START_TOKEN_ID: None | int = None GENERATE_START_TOKEN_ID_INCLUDE_START_TOKEN = False @@ -58,6 +57,41 @@ THREAD_DATA = threading.local() +class suppress_tracing(ContextDecorator): + """Suppress recorder-owned calls independently per thread, exception-safely. + + Use a class rather than a generator context manager: Python 3.11+ contextlib + assigns exception.__traceback__, which breaks frozen framework exceptions. + Never hold this scope over the original application call. + """ + + def _recreate_cm(self): + # Decorated conversions may nest or run concurrently on different threads. + return type(self)() + + def __enter__(self): + self.depth = getattr(THREAD_DATA, "suppression_depth", 0) + THREAD_DATA.suppression_depth = self.depth + 1 + return self + + def __exit__(self, exc_type, exc_value, traceback): + THREAD_DATA.suppression_depth = self.depth + return False + + +class application_tracing(suppress_tracing): + """Restore application tracing while advancing a user-supplied generator.""" + + def __enter__(self): + super().__enter__() + THREAD_DATA.suppression_depth = 0 + return self + + +def tracing_suppressed(): + return getattr(THREAD_DATA, "suppression_depth", 0) > 0 + + class TraceLineType: FUNC_CALL_PRE = "function_call (pre)" FUNC_CALL_POST = "function_call (post)" @@ -103,9 +137,8 @@ def get_owner_class(func): return owner +@suppress_tracing() def to_dict_args_kwargs(args, kwargs, dump_args_config=None) -> dict: - global DISABLE_WRAPPER - DISABLE_WRAPPER = True if dump_args_config is None: result = { "args": {i: var_to_serializable(arg) for i, arg in enumerate(args)}, @@ -125,20 +158,17 @@ def to_dict_args_kwargs(args, kwargs, dump_args_config=None) -> dict: result["args"] = args_dicts result["kwargs"] = kwargs_dicts - DISABLE_WRAPPER = False return result +@suppress_tracing() def to_dict_return_value(result) -> dict | list[dict]: - global DISABLE_WRAPPER - DISABLE_WRAPPER = True result_dict: dict | list[dict] if isinstance(result, tuple): result_dict = [var_to_serializable(r) for r in result] else: result_dict = var_to_serializable(result) - DISABLE_WRAPPER = False return result_dict @@ -167,7 +197,7 @@ def function_wrapper( global PROCESS_ID - if config.DISABLE_WRAPPER: + if config.DISABLE_WRAPPER or tracing_suppressed(): # TODO: all meta vars update should be done outside the function_wrapper (e.g. step increment) by applying a separate wrapper return original_function(*args, **kwargs) @@ -179,79 +209,81 @@ def function_wrapper( # tracing here and just delegate to the original function. return original_function(*args, **kwargs) - if COLLECT_OVERHEAD_METRICS: - ENTER_PERF_TIME = time.perf_counter() - - func_call_id = get_unique_id() - process_id, thread_id = get_process_thread_id() - - pre_meta_vars = get_meta_vars() - if IS_INSTRUMENTING: # during instrumentation, skip the dumping to avoid infinite recursion # and interference with the import system return original_function(*args, **kwargs) - pre_record = { - "func_call_id": func_call_id, - "thread_id": thread_id, - "process_id": process_id, - "meta_vars": pre_meta_vars, - "type": TraceLineType.FUNC_CALL_PRE, - "function": original_function_name, - "is_bound_method": is_bound_method, - "obj_id": None if not is_bound_method else id(args[0]), - } - - if dump_stack_trace: - pre_record["stack_trace"] = traceback.format_stack() - - if scan_proxy_in_args: - proxy_in_args = [] - - def find_proxy_in_args(args): - for i, arg in enumerate(args): - if is_proxied(arg) or is_proxyparameter(arg): - proxy_in_args.append(arg) - elif type(arg) in [list, tuple]: - find_proxy_in_args(arg) - elif isinstance(arg, types.GeneratorType) and not isinstance( - arg, tuple - ): - arg_list = list(arg) - args[i] = iter(arg_list) - find_proxy_in_args(arg_list) - - args = list(args) # type: ignore[assignment] - find_proxy_in_args(args) - args = tuple(args) - - if proxy_in_args: - if "proxy_obj_names" not in pre_record: - pre_record["proxy_obj_names"] = [] - for proxy in proxy_in_args: - if is_proxyparameter(proxy): - pre_record["proxy_obj_names"].append( - [proxy.__dict__["var_name"], "Parameter"] - ) - else: - pre_record["proxy_obj_names"].append( - [proxy.__dict__["var_name"], type(proxy._obj).__name__] - ) - if dump_args: - dict_args_kwargs = to_dict_args_kwargs(args, kwargs, dump_args_config) - pre_record["args"] = dict_args_kwargs["args"] - pre_record["kwargs"] = dict_args_kwargs["kwargs"] - dump_trace_API(pre_record) - - if trigger_var_dump: - """Mimicking the behavior the observer wrapper: pre-observe""" - get_global_registry().dump_modified( - dump_loc=original_function_name, dump_config=var_dump_config - ) + with suppress_tracing(): + if COLLECT_OVERHEAD_METRICS: + ENTER_PERF_TIME = time.perf_counter() + + func_call_id = get_unique_id() + process_id, thread_id = get_process_thread_id() + + pre_meta_vars = get_meta_vars() + + pre_record = { + "func_call_id": func_call_id, + "thread_id": thread_id, + "process_id": process_id, + "meta_vars": pre_meta_vars, + "type": TraceLineType.FUNC_CALL_PRE, + "function": original_function_name, + "is_bound_method": is_bound_method, + "obj_id": None if not is_bound_method else id(args[0]), + } - if need_unproxy_args_kwargs: - args, kwargs = unproxy_args_kwargs(args, kwargs) + if dump_stack_trace: + pre_record["stack_trace"] = traceback.format_stack() + + if scan_proxy_in_args: + proxy_in_args = [] + + def find_proxy_in_args(args): + for i, arg in enumerate(args): + if is_proxied(arg) or is_proxyparameter(arg): + proxy_in_args.append(arg) + elif type(arg) in [list, tuple]: + find_proxy_in_args(arg) + elif isinstance(arg, types.GeneratorType) and not isinstance( + arg, tuple + ): + with application_tracing(): + arg_list = list(arg) + args[i] = iter(arg_list) + find_proxy_in_args(arg_list) + + args = list(args) # type: ignore[assignment] + find_proxy_in_args(args) + args = tuple(args) + + if proxy_in_args: + if "proxy_obj_names" not in pre_record: + pre_record["proxy_obj_names"] = [] + for proxy in proxy_in_args: + if is_proxyparameter(proxy): + pre_record["proxy_obj_names"].append( + [proxy.__dict__["var_name"], "Parameter"] + ) + else: + pre_record["proxy_obj_names"].append( + [proxy.__dict__["var_name"], type(proxy._obj).__name__] + ) + if dump_args: + dict_args_kwargs = to_dict_args_kwargs(args, kwargs, dump_args_config) + pre_record["args"] = dict_args_kwargs["args"] + pre_record["kwargs"] = dict_args_kwargs["kwargs"] + dump_trace_API(pre_record) + + if trigger_var_dump: + """Mimicking the behavior the observer wrapper: pre-observe""" + get_global_registry().dump_modified( + dump_loc=original_function_name, dump_config=var_dump_config + ) + + if need_unproxy_args_kwargs: + args, kwargs = unproxy_args_kwargs(args, kwargs) try: if COLLECT_OVERHEAD_METRICS: @@ -260,134 +292,138 @@ def find_proxy_in_args(args): if COLLECT_OVERHEAD_METRICS: ORIG_EXIT_PERF_TIME = time.perf_counter() except Exception as e: - if COLLECT_OVERHEAD_METRICS: - ORIG_EXIT_PERF_TIME = time.perf_counter() + with suppress_tracing(): + if COLLECT_OVERHEAD_METRICS: + ORIG_EXIT_PERF_TIME = time.perf_counter() + + if trigger_var_dump: + """Mimicking the behavior the observer wrapper: post-observe""" + get_global_registry().dump_modified( + dump_loc=original_function_name, dump_config=var_dump_config + ) + + dump_trace_API( + { + "func_call_id": func_call_id, + "thread_id": thread_id, + "process_id": process_id, + "meta_vars": pre_meta_vars, + "type": TraceLineType.FUNC_CALL_POST_EXCEPTION, + "function": original_function_name, + "exception": typename(e, is_runtime=True), + "exception_msg": str(e), + "is_bound_method": is_bound_method, + "obj_id": None if not is_bound_method else id(args[0]), + }, + ) + + if COLLECT_OVERHEAD_METRICS: + EXIT_PERF_TIME = time.perf_counter() + print( + f"WRAPPER TIME: {original_function_name},{ORIG_EXIT_PERF_TIME - ORIG_ENTER_PERF_TIME},{EXIT_PERF_TIME - ENTER_PERF_TIME}" + ) + raise e + with suppress_tracing(): if trigger_var_dump: - """Mimicking the behavior the observer wrapper: post-observe""" get_global_registry().dump_modified( dump_loc=original_function_name, dump_config=var_dump_config ) - dump_trace_API( - { - "func_call_id": func_call_id, - "thread_id": thread_id, - "process_id": process_id, - "meta_vars": pre_meta_vars, - "type": TraceLineType.FUNC_CALL_POST_EXCEPTION, - "function": original_function_name, - "exception": typename(e, is_runtime=True), - "exception_msg": str(e), - "is_bound_method": is_bound_method, - "obj_id": None if not is_bound_method else id(args[0]), - }, - ) + post_record = { + "func_call_id": func_call_id, + "thread_id": thread_id, + "process_id": process_id, + "meta_vars": pre_meta_vars, + "type": TraceLineType.FUNC_CALL_POST, + "function": original_function_name, + "is_bound_method": is_bound_method, + "obj_id": None if not is_bound_method else id(args[0]), + } + + result_to_dump = result + + # if the current function name is transformers.generate, then we will dump the response tokens only, let's see. + # a concrete name: "transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.generate" + # we want a pattern that abstracts the specific model name + pattern = "transformers.models.*.generate" + # find matches in the pattern + import re + + if ( + GENERATE_START_TOKEN_ID is not None + and re.match(pattern, original_function_name) + and isinstance(result, torch.Tensor) + ): + print(f"Found match for {original_function_name}") + # the first dimension is the batch size, and each corresponds to a separate response, let's try to match the batch size with the start token ids first + response_starting_indices = [] + for i in range(result.size(0)): + # try to find the match of the start token ids in the response + response = result[i] + # Find all indices where the start_token_id matches + matches = (response == GENERATE_START_TOKEN_ID).nonzero(as_tuple=True)[ + 0 + ] + indexes = matches.tolist() + if len(indexes) == 0: + # No occurrences found + print( + f"start_token_id ({GENERATE_START_TOKEN_ID}) not found in response {i}" + ) + start_index = -1 # Handle case where token is not found + elif len(indexes) > 1: + # Multiple occurrences found, raise an error + raise ValueError( + f"Multiple occurrences of start_token_id ({GENERATE_START_TOKEN_ID}) found in response {i}: {matches.tolist()}" + ) + else: + # Single occurrence found, get the index + start_index = indexes[0] + if not GENERATE_START_TOKEN_ID_INCLUDE_START_TOKEN: + start_index += 1 + + response_starting_indices.append(start_index) + + # compute the length of each response + response_lengths = [] + for i in range(result.size(0)): + response = result[i] + start_index = response_starting_indices[i] + if start_index == -1: + response_lengths.append(0) + else: + response_lengths.append(response.size(0) - start_index) + + result_to_dump = result.detach() + setattr( + result_to_dump, + "_TRAINCHECK_RESPONSE_STARTING_INDICES", + response_starting_indices, + ) + setattr(result_to_dump, "_TRAINCHECK_RESPONSE_LENGTHS", response_lengths) + + print(response_starting_indices) + print(response_lengths) + if dump_ret: + post_record["return_values"] = to_dict_return_value(result_to_dump) + dump_trace_API(post_record) if COLLECT_OVERHEAD_METRICS: EXIT_PERF_TIME = time.perf_counter() print( f"WRAPPER TIME: {original_function_name},{ORIG_EXIT_PERF_TIME - ORIG_ENTER_PERF_TIME},{EXIT_PERF_TIME - ENTER_PERF_TIME}" ) - raise e - - if trigger_var_dump: - get_global_registry().dump_modified( - dump_loc=original_function_name, dump_config=var_dump_config - ) - - post_record = { - "func_call_id": func_call_id, - "thread_id": thread_id, - "process_id": process_id, - "meta_vars": pre_meta_vars, - "type": TraceLineType.FUNC_CALL_POST, - "function": original_function_name, - "is_bound_method": is_bound_method, - "obj_id": None if not is_bound_method else id(args[0]), - } - - result_to_dump = result - - # if the current function name is transformers.generate, then we will dump the response tokens only, let's see. - # a concrete name: "transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.generate" - # we want a pattern that abstracts the specific model name - pattern = "transformers.models.*.generate" - # find matches in the pattern - import re - - if ( - GENERATE_START_TOKEN_ID is not None - and re.match(pattern, original_function_name) - and isinstance(result, torch.Tensor) - ): - print(f"Found match for {original_function_name}") - # the first dimension is the batch size, and each corresponds to a separate response, let's try to match the batch size with the start token ids first - response_starting_indices = [] - for i in range(result.size(0)): - # try to find the match of the start token ids in the response - response = result[i] - # Find all indices where the start_token_id matches - matches = (response == GENERATE_START_TOKEN_ID).nonzero(as_tuple=True)[0] - indexes = matches.tolist() - if len(indexes) == 0: - # No occurrences found - print( - f"start_token_id ({GENERATE_START_TOKEN_ID}) not found in response {i}" - ) - start_index = -1 # Handle case where token is not found - elif len(indexes) > 1: - # Multiple occurrences found, raise an error - raise ValueError( - f"Multiple occurrences of start_token_id ({GENERATE_START_TOKEN_ID}) found in response {i}: {matches.tolist()}" - ) - else: - # Single occurrence found, get the index - start_index = indexes[0] - if not GENERATE_START_TOKEN_ID_INCLUDE_START_TOKEN: - start_index += 1 - - response_starting_indices.append(start_index) - - # compute the length of each response - response_lengths = [] - for i in range(result.size(0)): - response = result[i] - start_index = response_starting_indices[i] - if start_index == -1: - response_lengths.append(0) - else: - response_lengths.append(response.size(0) - start_index) - - result_to_dump = result.detach() - setattr( - result_to_dump, - "_TRAINCHECK_RESPONSE_STARTING_INDICES", - response_starting_indices, - ) - setattr(result_to_dump, "_TRAINCHECK_RESPONSE_LENGTHS", response_lengths) - - print(response_starting_indices) - print(response_lengths) - if dump_ret: - post_record["return_values"] = to_dict_return_value(result_to_dump) - dump_trace_API(post_record) - - if COLLECT_OVERHEAD_METRICS: - EXIT_PERF_TIME = time.perf_counter() - print( - f"WRAPPER TIME: {original_function_name},{ORIG_EXIT_PERF_TIME - ORIG_ENTER_PERF_TIME},{EXIT_PERF_TIME - ENTER_PERF_TIME}" - ) - return result + return result def core_wrapper_proxy(original_function, *args, **kwargs): """Core wrapper that only handles unproxying for built-in functions.""" - global DISABLE_WRAPPER - if DISABLE_WRAPPER: + if tracing_suppressed(): return original_function(*args, **kwargs) - args, kwargs = unproxy_args_kwargs(args, kwargs) + with suppress_tracing(): + args, kwargs = unproxy_args_kwargs(args, kwargs) return original_function(*args, **kwargs) @@ -539,7 +575,7 @@ def get_module_path_from_file_path(file_path: str, root_module: str) -> str | No if spec is None or spec.origin is None: # raise error raise ImportError(f"Cannot locate root module {root_module!r}") - + # get the path to the root module root_module_path = pathlib.PurePath(spec.origin).parent @@ -552,13 +588,14 @@ def get_module_path_from_file_path(file_path: str, root_module: str) -> str | No except ValueError: # file_path is not under root module raise ImportError(f"File path is not under root module {root_module!r}") - + # strip off .py - module_name = str(relative_path.with_suffix('')) + module_name = str(relative_path.with_suffix("")) # replace / with . return f"{root_module}.{module_name.replace('/', '.')}" + class Instrumentor: def __init__( self, From 504e04d4de1a4168aabb5436f1b236edef1dadd7 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:15:31 +0800 Subject: [PATCH 06/36] Cache process and thread identities during precondition inference --- tests/test_identity_cache.py | 194 +++++++++++++++++++++++++++++++ traincheck/trace/trace_pandas.py | 32 ++++- 2 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 tests/test_identity_cache.py diff --git a/tests/test_identity_cache.py b/tests/test_identity_cache.py new file mode 100644 index 00000000..4093d74a --- /dev/null +++ b/tests/test_identity_cache.py @@ -0,0 +1,194 @@ +"""Identity lookup equivalence during precondition inference.""" + +import copy +import json +import logging +import pickle +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from traincheck.invariant.precondition import find_precondition_from_single_group +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyIdentityTrace(TracePandas): + def get_process_ids(self): + if "process_id" not in self.events.columns: + return [] + return self.events["process_id"].dropna().unique().tolist() + + def get_thread_ids(self): + if "thread_id" not in self.events.columns: + return [] + return self.events["thread_id"].dropna().unique().tolist() + + +def make(frame, cls=TracePandas): + return cls(frame.copy(deep=True), truncate_incomplete_func_calls=False) + + +def frame(process_ids=(1, 2, 1), thread_ids=(4, 3, 4)): + return pd.DataFrame( + { + "time": range(len(process_ids)), + "process_id": process_ids, + "thread_id": thread_ids, + } + ) + + +class IdentityCacheTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def assert_equivalent(self, data): + old, new = make(data, LegacyIdentityTrace), make(data) + for _ in range(2): + for method in ("get_process_ids", "get_thread_ids"): + expected, actual = getattr(old, method)(), getattr(new, method)() + self.assertEqual(expected, actual) + self.assertEqual(list(map(type, expected)), list(map(type, actual))) + return old, new + + def test_values_types_and_encounter_order(self): + for values in ( + pd.Series([8, 2, 8, -1], dtype="int64"), + pd.Series([2**63 + 7, 2**63 + 2, None, 2**63 + 7], dtype="UInt64"), + pd.Series([8, pd.NA, 2, 8], dtype="Int64"), + pd.Series([8.5, np.nan, 2.5, 8.5]), + pd.Series(["z", None, "a", "z"], dtype="string"), + pd.Series(["z", None, "a", "z"], dtype="category"), + pd.Series([True, 1, False, 0], dtype=object), + pd.Series([None, np.nan, pd.NA, None], dtype=object), + ): + with self.subTest(dtype=str(values.dtype), values=repr(values.tolist())): + self.assert_equivalent(frame(values, values)) + + def test_randomized_identity_columns(self): + rng = np.random.default_rng(42) + for _ in range(80): + values = rng.integers(-8, 8, size=50).astype(float) + values[rng.random(50) < 0.2] = np.nan + self.assert_equivalent(frame(values, values[::-1])) + + def test_empty_and_missing_columns(self): + for data in (frame([], []), pd.DataFrame({"time": [0]})): + self.assert_equivalent(data) + + def test_returned_lists_are_independent(self): + trace = make(frame()) + result = trace.get_process_ids() + result[:] = [999] + self.assertEqual(trace.get_process_ids(), [1, 2]) + result = trace.get_thread_ids() + result.clear() + self.assertEqual(trace.get_thread_ids(), [4, 3]) + + def test_frame_replacement_rebuilds_same_size_cache(self): + trace = make(frame()) + trace.get_process_ids() + trace.get_thread_ids() + trace.events = frame((9, 7, 9), (5, 6, 5)) + self.assertEqual(trace.get_process_ids(), [9, 7]) + self.assertEqual(trace.get_thread_ids(), [5, 6]) + + def test_same_frame_length_change_rebuilds_cache(self): + trace = make(frame()) + trace.get_process_ids() + trace.get_thread_ids() + trace.events.loc[3] = [3, 9, 7] + self.assertEqual(trace.get_process_ids(), [1, 2, 9]) + self.assertEqual(trace.get_thread_ids(), [4, 3, 7]) + trace.events.drop(index=[1, 3], inplace=True) + self.assertEqual(trace.get_process_ids(), [1]) + self.assertEqual(trace.get_thread_ids(), [4]) + + def test_stage_traces_and_instances_have_separate_caches(self): + data = frame() + data["meta_vars.stage"] = ["init", "train", "init"] + trace = make(data) + trace.get_process_ids() + stages = trace.get_traces_for_stage() + self.assertEqual(stages["init"].get_process_ids(), [1]) + self.assertEqual(stages["train"].get_process_ids(), [2]) + self.assertEqual(make(frame((9,), (7,))).get_process_ids(), [9]) + + def test_pickled_initialized_cache(self): + trace = make(frame()) + trace.get_process_ids() + restored = pickle.loads(pickle.dumps(trace)) + self.assertIs(restored._identity_values_source, restored.events) + self.assertEqual(restored.get_process_ids(), [1, 2]) + self.assertEqual(restored.get_thread_ids(), [4, 3]) + + def test_meta_vars_membership_preserves_independent_id_checks(self): + old, new = self.assert_equivalent(frame((1, 2), (3, 4))) + for pid, tid in ((1, 3), (1, 4), (2, 3), (9, 3), (1, 9)): + self.assertEqual( + old.get_meta_vars(0, pid, tid), new.get_meta_vars(0, pid, tid) + ) + self.assertEqual(new.get_meta_vars(0, 1, 4), {}) + self.assertIsNone(new.get_meta_vars(0, 9, 3)) + + def test_repeated_meta_vars_queries_only_scan_each_column_once(self): + trace = make(frame()) + original = pd.Series.unique + scanned = [] + + def unique(series): + scanned.append(series.name) + return original(series) + + with patch.object(pd.Series, "unique", unique): + for _ in range(100): + self.assertEqual(trace.get_meta_vars(0, 1, 4), {}) + self.assertEqual(scanned, ["process_id", "thread_id"]) + + def test_real_precondition_inference_matches_legacy(self): + old, new = self.assert_equivalent(frame()) + positive = [ + [ + { + "time": i, + "process_id": 1, + "thread_id": 4, + "function": "fixtures.step", + "meta_vars.phase": "train", + } + ] + for i in range(12) + ] + negative = [ + [ + { + "time": i, + "process_id": 2, + "thread_id": 3, + "function": "fixtures.step", + "meta_vars.phase": "eval", + } + ] + for i in range(8) + ] + results = [] + for trace in (old, new): + pos, neg = copy.deepcopy(positive), copy.deepcopy(negative) + clauses = find_precondition_from_single_group(pos, neg, [trace]) + results.append( + ( + sorted(json.dumps(c.to_dict(), sort_keys=True) for c in clauses), + pos, + neg, + ) + ) + self.assertTrue(results[0][0]) + self.assertEqual(*results) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 2ab42924..e7e20cfe 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -111,6 +111,12 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self._func_call_positions_length = -1 self._func_call_positions: dict[str, tuple[list[int], list[int]]] | None = None + # Inference treats event identities as immutable after initialization. + # Keep the frame alive so object identity cannot be reused across frames. + self._identity_values_source: pd.DataFrame | None = None + self._identity_values_length = -1 + self._identity_values: dict[str, tuple] = {} + self.column_dtypes_cached = {} self.stage_traces: dict[str, "TracePandas"] = {} @@ -573,6 +579,28 @@ def get_end_time(self, process_id=None, thread_id=None) -> float: return self.events["time"].max() + def _get_identity_values(self, column: str) -> list: + """Cache stable identity columns; return a fresh list for every caller. + + As with call-record indexes, frame replacement or length changes rebuild + the cache. Same-size in-place identity edits, row reordering, or dtype + changes require a new TracePandas. + Keep pandas' dropna/unique conversion to preserve dtype and encounter + order instead of replacing it with Python set semantics. + """ + if ( + self._identity_values_source is not self.events + or self._identity_values_length != len(self.events) + ): + self._identity_values = {} + self._identity_values_source = self.events + self._identity_values_length = len(self.events) + if column not in self._identity_values: + self._identity_values[column] = tuple( + self.events[column].dropna().unique().tolist() + ) + return list(self._identity_values[column]) + def get_process_ids(self) -> list[int]: """Find all process ids from the trace.""" if "process_id" not in self.events.columns: @@ -580,7 +608,7 @@ def get_process_ids(self) -> list[int]: "process_id column not found in the events, no process related invariants will be extracted." ) return [] - return self.events["process_id"].dropna().unique().tolist() + return self._get_identity_values("process_id") def get_thread_ids(self) -> list[int]: """Find all thread ids from the trace.""" @@ -589,7 +617,7 @@ def get_thread_ids(self) -> list[int]: "thread_id column not found in the events, no thread related invariants will be extracted." ) return [] - return self.events["thread_id"].dropna().unique().tolist() + return self._get_identity_values("thread_id") def get_func_call_ids(self, func_name: str = "") -> list[str]: """Find all function call ids from the trace.""" From 9ac61d1140cf8679c24442a174ec12f4de069980 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:16:20 +0800 Subject: [PATCH 07/36] Preserve TorchScript builtin identity through tracing wrappers --- .../torchscript-builtin-validation.json | 208 ++++++++++++ .../torchscript-builtin-validation.md | 38 +++ tests/test_torchscript_compat.py | 305 ++++++++++++++++++ traincheck/instrumentor/torchscript_compat.py | 27 ++ traincheck/instrumentor/tracer.py | 2 + 5 files changed, 580 insertions(+) create mode 100644 docs/experiments/torchscript-builtin-validation.json create mode 100644 docs/experiments/torchscript-builtin-validation.md create mode 100644 tests/test_torchscript_compat.py create mode 100644 traincheck/instrumentor/torchscript_compat.py diff --git a/docs/experiments/torchscript-builtin-validation.json b/docs/experiments/torchscript-builtin-validation.json new file mode 100644 index 00000000..13020e1c --- /dev/null +++ b/docs/experiments/torchscript-builtin-validation.json @@ -0,0 +1,208 @@ +{ + "historical_cpu_study": { + "linux": { + "controls": [ + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 0, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "baseline", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-unprimed", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-warm", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-unprimed", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "integrated-fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + } + ], + "device": "cpu", + "helper_test_log_sha256": "feabb4add9e2ca3374e538560634d1ec97dc4f4ce89aab12f3d726e36ea9b2f6", + "helper_unit_tests_passed": 4, + "native_threads": 1, + "python": "3.10.16", + "registry_already_initialized_after_import": true, + "semantic_trace_checks_passed": 13, + "summary_sha256": "68c1888b7df4a9a0248e6d218ea5e82d2daeb90c3efcf50b852faaf7e3bfa7c2", + "torch": "2.2.2+cu121" + }, + "macos": { + "controls": [ + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 0, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "baseline", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-unprimed", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-warm", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-unprimed", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "integrated-fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + } + ], + "device": "cpu", + "helper_test_log_sha256": "d094b1bd2c8318c96097f80c50da282add5b2972970a77046de8db6c8935f6ca", + "helper_unit_tests_passed": 4, + "native_threads": 1, + "python": "3.10.16", + "registry_already_initialized_after_import": true, + "semantic_trace_checks_passed": 13, + "summary_sha256": "4e8775e72f31a344f6e82f59f6b3009fddebc9f71c9463f6f5a009b5885d8c43", + "torch": "2.2.2" + } + }, + "historical_source_sha256": { + "bias_gelu_fixture.py": "c6d7503996e5fa2872114aa7ba8b3b37151f03e8f7cc2302a76e6c451fb70059", + "builtin_compat_candidate.py": "744506a0c035396aea443f39394a700ef7908bc7d36b8117ba76fae97c92e845", + "candidate-src/traincheck/instrumentor/torchscript_compat.py": "744506a0c035396aea443f39394a700ef7908bc7d36b8117ba76fae97c92e845", + "candidate-src/traincheck/instrumentor/tracer.py": "a8e39a9dea2c4940543174c2bca97146665f0f08068bdf350b3f16802d4e79bd", + "probe.py": "58b0ce8f8a1fb65b96cd5a756bb82ef5ff8d315b3673081ea58d49ebffb0b29c", + "run.py": "f4c549ea1dd21c47f4e98f35df4126b23895391eacb2f89c182aaf7f2ccf8d12", + "test_candidate.py": "6f5fbbab5053e2303a53209ec25d71c1c240e9c06b9038af844adf762f636eff", + "torchscript-builtin-identity.patch": "8ae809f16bf6464d5a152d9612e37bd81edf04ef6680fee7e3042b42c8f026a5" + }, + "limits": [ + "Registry was already initialized after imports; no genuinely cold-registry result is claimed.", + "Scripted operators bypass Python recorder hooks; successful graphs emit no inner tanh API records.", + "Registered wrappers remain alive for the process lifetime; repeated creation retains additional objects.", + "Full Megatron startup, checkpoint-shape compatibility, and other PyTorch versions/operators are not established.", + "The converted portable checkout test ran locally on macOS; Linux evidence comes from the prior equivalent focused study." + ], + "linux_saved_evidence_archive_sha256": "81f5ede08024c3e099ad61b7489ffa420e4da490c743cded9d9236fd6453a339", + "portable_checkout_validation": { + "fresh_probe_processes": 6, + "fresh_process_regression_methods": 1, + "helper_test_methods": 4, + "log_sha256": "4e23c888ea4641c17ac45c1a413ba0a9b74afbbda23c87e364458b283a4b2438", + "platform": "macOS", + "python": "3.10.16", + "semantic_trace_checks": 13, + "source_includes_trace_integrity_fix": true, + "test_methods_passed": 5, + "tested_base_commit": "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4", + "tested_base_pr": 142, + "timeout_seconds_per_probe": 90, + "torch": "2.2.2" + }, + "recorded_date": "2026-10-25", + "schema_version": 1, + "source": { + "base_dumper_sha256": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "base_tracer_sha256": "bce6546cfa13a263150dff891f2f8d7d4af8c5ccb28dc2faedcedc691c5857da", + "candidate_dumper_sha256": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "candidate_tracer_sha256": "ad57320f0a3764c02c73cf4bfa623f1ef73b9835dfa6cb1365dc2988551de428", + "helper_implementation_ast_unchanged_except_docstring": true, + "helper_sha256": "0d0c348fa02d5f93bb74e874829e5fb4c57365b82ce973ae8ff1f21a42c2d64e", + "integration_patch_sha256": "b374501d22408ebace2d1e2e69c03d2a98abcf3ebfa2e4e056b62adf27874397", + "portable_test_sha256": "653be1032409579030ebd3ec5694cc3e43f34df8e4f5750a76a9ffc64dd0a2e2", + "validated_helper_before_docstring_sha256": "744506a0c035396aea443f39394a700ef7908bc7d36b8117ba76fae97c92e845" + } +} diff --git a/docs/experiments/torchscript-builtin-validation.md b/docs/experiments/torchscript-builtin-validation.md new file mode 100644 index 00000000..71c9f4b3 --- /dev/null +++ b/docs/experiments/torchscript-builtin-validation.md @@ -0,0 +1,38 @@ +# TorchScript builtin identity validation + +TorchScript records builtin operators by function ID. Replacing `torch.tanh` with a TrainCheck wrapper loses the `aten::tanh` lookup. TorchScript attempts to inspect Python source, follows the wrapper metadata to the C builtin, and raises `TypeError: ... expected, got builtin_function_or_method`. + +The compatibility helper looks up the original mapping before the wrapper is published and registers the wrapper with that same operator. It guesses no operator names. Missing private APIs and unregistered Python functions are unchanged. Registered wrappers are retained because the registry stores only IDs; otherwise a later unrelated callable could reuse a stale ID. + +## Controls and results + +The 2026-10-25 CPU experiment uses a real Python source file containing a representative tanh-based bias-GELU expression. Each control runs in a fresh interpreter, with a 90-second timeout and one native/inter-op thread. + +| Control | Compilation | Eager tanh records | Scripted operator | +|---|---|---:|---| +| Unwrapped baseline | Pass | 0 | `aten::tanh` | +| Wrapper without compatibility, no explicit priming | TypeError | 2, one pair | Unregistered wrapper | +| Wrapper without compatibility, explicitly primed | TypeError | 2, one pair | Unregistered wrapper | +| Wrapper plus helper, no explicit priming | Pass | 2, one pair | `aten::tanh` | +| Wrapper plus helper, explicitly primed | Pass | 2, one pair | `aten::tanh` | +| Helper integrated into the actual wrapper | Pass | 2, one pair | `aten::tanh` | + +The original study passed all 13 semantic/trace checks and four helper unit tests on macOS with PyTorch 2.2.2 and Linux with PyTorch 2.2.2+cu121; both used Python 3.10.16 and CPU tensors. Candidate eager outputs and both gradients exactly matched the unwrapped baseline within each platform. Successful scripted outputs and both gradients also exactly matched that platform's scripted baseline. Every eager pair retained arguments, return snapshots, and a shared call ID. + +The portable checkout test was additionally run on macOS against the trace-integrity source plus this integration: all five test methods passed, comprising the four helper contracts and six fresh-process controls with the same 13 semantic/trace checks. The helper implementation is unchanged from the Linux-tested version apart from its module docstring and formatting. Linux's earlier study is separate evidence; the converted checkout test was not rerun there. + +Run the portable tests from the repository root: + +```bash +python -m unittest discover -s tests -p test_torchscript_compat.py -v +``` + +Compact source hashes, control outcomes, and validation details are recorded in [torchscript-builtin-validation.json](torchscript-builtin-validation.json). + +## Limits + +TrainCheck imports had already initialized the registry in both tested environments. “No explicit priming” therefore does not prove a genuinely uninitialized registry path. + +Scripted graphs call `aten::tanh` directly and produce no inner Python tanh records. Eager tracing remains intact; complete compiled-graph tracing is outside this change. Full instrument-before-import Megatron startup and checkpoint-shape compatibility were not validated by this focused fixture. + +Registered wrappers remain alive for the process lifetime; repeatedly creating new wrappers retains additional objects. The helper depends on feature-detected private APIs and does not establish compatibility with every future PyTorch version or wrapped operator. diff --git a/tests/test_torchscript_compat.py b/tests/test_torchscript_compat.py new file mode 100644 index 00000000..05f2b8bb --- /dev/null +++ b/tests/test_torchscript_compat.py @@ -0,0 +1,305 @@ +"""Builtin-registration contracts and fresh-process bias-GELU regressions.""" + +import argparse +import copy +import gc +import json +import os +import subprocess +import sys +import tempfile +import unittest +import weakref +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import torchscript_compat +from traincheck.instrumentor.torchscript_compat import ( + _REGISTERED_WRAPPERS, + preserve_torchscript_builtin, +) + + +def bias_gelu(bias: torch.Tensor, value: torch.Tensor) -> torch.Tensor: + """Real-source-file fixture for the tanh-based fused bias-GELU expression.""" + x = bias + value + return ( + x * 0.5 * (1.0 + torch.tanh(0.7978845608028654 * x * (1.0 + 0.044715 * x * x))) + ) + + +def evaluate(function): + bias = torch.linspace(-0.7, 0.9, 8, dtype=torch.float64, requires_grad=True) + value = ( + torch.linspace(-2.3, 2.1, 24, dtype=torch.float64) + .reshape(3, 8) + .requires_grad_() + ) + output = function(bias, value) + grad_bias, grad_value = torch.autograd.grad(output.square().sum(), (bias, value)) + return { + "output": output.detach().tolist(), + "grad_bias": grad_bias.tolist(), + "grad_value": grad_value.tolist(), + } + + +def probe(mode): + """Run one control in its own interpreter so JIT caches cannot leak.""" + from traincheck.instrumentor import tracer + + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + records = [] + registry = torch.jit._builtins + original = torch.tanh + result = { + "mode": mode, + "torch": torch.__version__, + "initial_registry_initialized": registry._builtin_table is not None, + } + if mode.endswith("warm"): + result["original_operator_before_wrapping"] = registry._find_builtin(original) + + with ( + patch.object(tracer.config, "DISABLE_WRAPPER", False), + patch.object( + tracer, + "dump_trace_API", + side_effect=lambda record: records.append(copy.deepcopy(record)), + ), + ): + if mode != "baseline": + if mode == "integrated-fixed-warm": + wrapped = tracer.wrapper( + original, + False, + False, + False, + dump_args=True, + dump_ret=True, + handle_proxy=False, + ) + else: + # Disable only registration to reproduce the former wrapper; + # the helper controls then apply the production helper directly. + with patch.object( + tracer, "preserve_torchscript_builtin", return_value=None + ): + wrapped = tracer.wrapper( + original, + False, + False, + False, + dump_args=True, + dump_ret=True, + handle_proxy=False, + ) + if mode.startswith("fixed"): + result["candidate"] = preserve_torchscript_builtin( + original, wrapped, torch + ) + torch.tanh = wrapped + try: + result["eager"] = evaluate(bias_gelu) + result["eager_records"] = copy.deepcopy(records) + records.clear() + scripted = torch.jit.script(bias_gelu) + result["graph_has_aten_tanh"] = "aten::tanh" in str(scripted.graph) + result["scripted"] = evaluate(scripted) + result["scripted_records"] = copy.deepcopy(records) + result["script_compilation"] = "passed" + except Exception as error: + result["script_compilation"] = "failed" + result["exception_type"] = type(error).__name__ + result["exception"] = str(error) + finally: + result["current_operator_after_compilation"] = registry._find_builtin( + torch.tanh + ) + torch.tanh = original + return result + + +class BuiltinCompatibilityTests(unittest.TestCase): + def fake_torch(self, operator): + table, events = {}, [] + + def find(function): + events.append(("find", function)) + return operator + + def register(function, mapped_operator): + events.append(("register", function, mapped_operator)) + table[id(function)] = mapped_operator + + builtins = SimpleNamespace(_find_builtin=find, _register_builtin=register) + return SimpleNamespace(jit=SimpleNamespace(_builtins=builtins)), table, events + + def test_copies_exact_original_mapping(self): + fake, table, events = self.fake_torch("aten::tanh") + + def original(): + pass + + def wrapped(): + pass + + result = preserve_torchscript_builtin(original, wrapped, fake) + self.assertEqual(result, {"status": "preserved", "operator": "aten::tanh"}) + self.assertEqual( + events, [("find", original), ("register", wrapped, "aten::tanh")] + ) + self.assertEqual(table[id(wrapped)], "aten::tanh") + + def test_unregistered_function_is_unchanged(self): + fake, table, events = self.fake_torch(None) + original = lambda: None + wrapped = lambda: None + self.assertEqual( + preserve_torchscript_builtin(original, wrapped, fake)["status"], + "not-registered", + ) + self.assertEqual(table, {}) + self.assertEqual(events, [("find", original)]) + self.assertNotIn(wrapped, _REGISTERED_WRAPPERS) + + def test_unavailable_private_api_is_noop(self): + original = lambda: None + wrapped = lambda: None + missing_register = SimpleNamespace( + jit=SimpleNamespace( + _builtins=SimpleNamespace(_find_builtin=lambda f: "aten::tanh") + ) + ) + for fake in [SimpleNamespace(), missing_register]: + self.assertEqual( + preserve_torchscript_builtin(original, wrapped, fake)["status"], + "unsupported-private-api", + ) + self.assertNotIn(wrapped, _REGISTERED_WRAPPERS) + + def test_registered_wrapper_cannot_leave_stale_reusable_id(self): + fake, table, events = self.fake_torch("aten::tanh") + original = lambda: None + wrapped = lambda: None + weak = weakref.ref(wrapped) + address = id(wrapped) + preserve_torchscript_builtin(original, wrapped, fake) + events.clear() + del wrapped + gc.collect() + self.assertIsNotNone(weak()) + self.assertEqual(table[address], "aten::tanh") + self.assertEqual(_REGISTERED_WRAPPERS[weak()], "aten::tanh") + + +class TorchScriptWrapperTests(unittest.TestCase): + def test_fresh_process_bias_gelu_semantics_and_eager_traces(self): + registry = getattr(getattr(torch, "jit", None), "_builtins", None) + if not all( + callable(getattr(registry, name, None)) + for name in ["_find_builtin", "_register_builtin"] + ): + self.skipTest("PyTorch lacks the private builtin-registration APIs") + + source = Path(torchscript_compat.__file__).resolve().parents[2] + environment = os.environ | { + "PYTHONPATH": str(source), + "PYTHONDONTWRITEBYTECODE": "1", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "CUDA_VISIBLE_DEVICES": "", + } + modes = [ + "baseline", + "buggy-unprimed", + "buggy-warm", + "fixed-unprimed", + "fixed-warm", + "integrated-fixed-warm", + ] + results = {} + with tempfile.TemporaryDirectory(prefix="traincheck-torchscript-") as temporary: + for mode in modes: + output = Path(temporary) / (mode + ".json") + completed = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + "--probe", + mode, + "--output", + str(output), + ], + cwd=source, + env=environment, + capture_output=True, + text=True, + timeout=90, + ) + self.assertEqual( + completed.returncode, 0, completed.stdout + completed.stderr + ) + results[mode] = json.loads(output.read_text()) + + baseline = results["baseline"] + checks = 0 + for mode in modes[:-1]: + result = results[mode] + with self.subTest(mode=mode): + self.assertEqual(result["eager"], baseline["eager"]) + checks += 1 + if mode.startswith("buggy"): + self.assertEqual(result["script_compilation"], "failed") + self.assertEqual(result["exception_type"], "TypeError") + self.assertIn("builtin_function_or_method", result["exception"]) + self.assertIsNone(result["current_operator_after_compilation"]) + else: + self.assertEqual(result["script_compilation"], "passed") + self.assertTrue(result["graph_has_aten_tanh"]) + self.assertEqual(result["scripted"], baseline["scripted"]) + self.assertEqual( + result["current_operator_after_compilation"], "aten::tanh" + ) + self.assertEqual(result["scripted_records"], []) + checks += 1 + if mode != "baseline": + self.assert_paired_eager_record(result) + checks += 1 + integrated = results["integrated-fixed-warm"] + self.assertEqual(integrated["script_compilation"], "passed") + self.assertTrue(integrated["graph_has_aten_tanh"]) + self.assertEqual(integrated["scripted"], baseline["scripted"]) + self.assertEqual(integrated["eager"], baseline["eager"]) + self.assertEqual(integrated["scripted_records"], []) + self.assertEqual(integrated["current_operator_after_compilation"], "aten::tanh") + self.assert_paired_eager_record(integrated) + checks += 1 + self.assertEqual(checks, 13) + + def assert_paired_eager_record(self, result): + records = result["eager_records"] + self.assertEqual(len(records), 2) + self.assertEqual(records[0]["type"], "function_call (pre)") + self.assertEqual(records[1]["type"], "function_call (post)") + self.assertEqual(records[0]["func_call_id"], records[1]["func_call_id"]) + self.assertEqual(records[0]["function"], records[1]["function"]) + self.assertIn("tanh", records[0]["function"]) + self.assertTrue(records[0]["args"]) + self.assertTrue(records[1]["return_values"]) + + +if __name__ == "__main__": + if "--probe" in sys.argv: + parser = argparse.ArgumentParser() + parser.add_argument("--probe", required=True) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + arguments.output.write_text(json.dumps(probe(arguments.probe)) + "\n") + else: + unittest.main() diff --git a/traincheck/instrumentor/torchscript_compat.py b/traincheck/instrumentor/torchscript_compat.py new file mode 100644 index 00000000..03087d84 --- /dev/null +++ b/traincheck/instrumentor/torchscript_compat.py @@ -0,0 +1,27 @@ +"""Preserve TorchScript operator identity for instrumented builtin functions.""" + +# TorchScript indexes by id only. Keep registered wrappers alive, like PyTorch's +# own _builtin_ops list, so a later unrelated object cannot reuse a stale ID. +_REGISTERED_WRAPPERS = {} + + +def preserve_torchscript_builtin(original, wrapped, torch_module): + """Call before publishing wrapped into its module, while original is visible. + + TorchScript compiles the existing operator directly and does not run Python + recorder hooks inside its graph. Eager calls still execute the wrapper. + Private API feature detection leaves unsupported versions unchanged. + """ + jit = getattr(torch_module, "jit", None) + builtins = getattr(jit, "_builtins", None) + find_builtin = getattr(builtins, "_find_builtin", None) + register_builtin = getattr(builtins, "_register_builtin", None) + if not callable(find_builtin) or not callable(register_builtin): + return {"status": "unsupported-private-api", "operator": None} + # _find_builtin also initializes the lazy table before the first replacement. + operator = find_builtin(original) + if operator is None: + return {"status": "not-registered", "operator": None} + register_builtin(wrapped, operator) + _REGISTERED_WRAPPERS[wrapped] = operator + return {"status": "preserved", "operator": operator} diff --git a/traincheck/instrumentor/tracer.py b/traincheck/instrumentor/tracer.py index 33c8f05e..dc1cbc7c 100644 --- a/traincheck/instrumentor/tracer.py +++ b/traincheck/instrumentor/tracer.py @@ -39,6 +39,7 @@ funcs_to_be_replaced, is_funcs_to_be_unproxied, ) +from traincheck.instrumentor.torchscript_compat import preserve_torchscript_builtin from traincheck.utils import get_timestamp_ns, get_unique_id, typename _instancemethod_t = type(torch._C._distributed_c10d.ProcessGroup.broadcast) @@ -486,6 +487,7 @@ def wrapped(*args, **kwargs): wrapped._traincheck_original_function = original_function wrapped._traincheck_instrumented = True wrapped._traincheck_dump_disabled = disable_dump + preserve_torchscript_builtin(original_function, wrapped, torch) return wrapped From fa277435dbff801f3b51ef3660f4eb003b1e5f2c Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:25:11 +0800 Subject: [PATCH 08/36] Document identity cache measurements and validation --- .../inference-identity-cache-20261025.json | 126 +++++++++ docs/measurements/inference-identity-cache.md | 65 +++++ tools/benchmarks/identity_cache.py | 246 ++++++++++++++++++ 3 files changed, 437 insertions(+) create mode 100644 docs/measurements/inference-identity-cache-20261025.json create mode 100644 docs/measurements/inference-identity-cache.md create mode 100644 tools/benchmarks/identity_cache.py diff --git a/docs/measurements/inference-identity-cache-20261025.json b/docs/measurements/inference-identity-cache-20261025.json new file mode 100644 index 00000000..fa3df296 --- /dev/null +++ b/docs/measurements/inference-identity-cache-20261025.json @@ -0,0 +1,126 @@ +{ + "measurement_date": "2026-10-25", + "scope": "construction plus repeated identity-checked get_meta_vars calls", + "settings": { + "rows": 573494, + "calls": 1000, + "repetitions": 3 + }, + "runtime": { + "python": "3.10.16", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "host_class": "local Darwin arm64 workstation", + "logical_cpu_count": 12, + "pandas": "2.2.3", + "numpy": "1.26.4", + "native_thread_settings": { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1" + }, + "candidate_source_sha256": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "candidate_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7", + "candidate_source_matches_commit": true, + "legacy_methods_source_sha256": "22a7976d0fc8558b5a3dcc77e9e527572ec1f1e36b15d5e244a4a0b7ec6f44e7" + }, + "input": { + "sha256_pandas_row_hashes": "ce7af99430be7c9494ef8ee397049b77847897dd965c1082a3cda9e88a159f8b", + "query_sha256": "cd7c7c854fec82747a9704c95dc1e897d6f2dca5664b1b9167a8bd845f227384", + "dtypes": { + "time": "int64", + "process_id": "float64", + "thread_id": "float64" + }, + "dataframe_deep_bytes": 13763984, + "context_manager_count": 0, + "process_count": 8, + "thread_count": 16 + }, + "medians": { + "legacy": { + "construction_seconds": 8.187499770428985e-05, + "queries_seconds": 9.489749166001275, + "total_seconds": 9.489831040998979 + }, + "candidate": { + "construction_seconds": 5.3582996770273894e-05, + "queries_seconds": 0.01196087500284193, + "total_seconds": 0.01200404100382002 + } + }, + "trials": [ + { + "repetition": 1, + "position": 1, + "variant": "legacy", + "total_seconds": 9.2562226250011 + }, + { + "repetition": 1, + "position": 2, + "variant": "candidate", + "total_seconds": 0.01413512499857461 + }, + { + "repetition": 2, + "position": 1, + "variant": "candidate", + "total_seconds": 0.010777290997793898 + }, + { + "repetition": 2, + "position": 2, + "variant": "legacy", + "total_seconds": 9.489831040998979 + }, + { + "repetition": 3, + "position": 1, + "variant": "legacy", + "total_seconds": 9.548994832999597 + }, + { + "repetition": 3, + "position": 2, + "variant": "candidate", + "total_seconds": 0.01200404100382002 + } + ], + "correctness": { + "all_results_equal_expected": true, + "result_sha256": "481e48248fde7cbd3011bd60ecb24d4b0ccaa470a6e5a1986acf27bcc3fa9570", + "input_unchanged": true + }, + "validation": { + "local_combined_tests_passed": 43, + "matching_runtime_linux_identity_tests_passed": 11, + "independent_differential_comparisons": 78, + "independent_dtype_cases": 13, + "pickle_protocols": 6, + "spawn_worker_tasks_passed": 2, + "matching_linux_runtime": { + "python": "3.11.4", + "pytorch": "2.16.0.dev20261012+cpu", + "numpy": "2.1.0", + "pandas": "2.3.3" + } + }, + "full_inference": { + "status": "paired_unprofiled_replay_pending", + "profile_attempts": "Both attempts terminated with SIGSEGV at scheduled diagnostic timers; no completed profile/output. Candidate stopped before the identity-cache phase.", + "performance_comparison_valid": false + }, + "limitations": [ + "Synthetic identity-lookup microbenchmark, not completed inference.", + "No context-manager work or trace-file loading; input generation excluded.", + "No RSS/PSS, peak-memory, or multiprocessing memory-scaling measurement.", + "Cache stores O(P+T) values and returns fresh O(P+T) lists.", + "Stable identity values, dtypes, and row order required within one frame." + ], + "source_evidence_sha256": { + "benchmark-full.json": "7c3b84db5645226b4d841b929185edad33c075c7868ad9951607a57b2852febe", + "review-results.json": "19e4c7bf403a8a3732e8dbf0fda3ab4f6902b99c0e997dc490d9383ffc406376", + "combined-tests.log": "6334a444075d2bd539f965998354922ea2d940d74536d61790e39e840bcc345c", + "matching-runtime-linux-tests.log": "56b695709855bb8a17cd0fbf58e1bf3e7d17ed228719d5cc2350d3718f8e3604" + } +} diff --git a/docs/measurements/inference-identity-cache.md b/docs/measurements/inference-identity-cache.md new file mode 100644 index 00000000..09c9919e --- /dev/null +++ b/docs/measurements/inference-identity-cache.md @@ -0,0 +1,65 @@ +# Cache process and thread identities during precondition inference + +`get_meta_vars` checks process and thread membership for each request. Previously, +each check repeated `dropna().unique().tolist()` over the full event column. +The change caches those exact pandas results lazily per trace and returns a fresh +list to each caller. Null filtering, scalar types, encounter order, and independent +process/thread membership remain unchanged. Frame replacement or length changes +invalidate both identity caches. + +## Isolated measurement + +On a deterministic 573,494-row synthetic trace, the median time for construction +plus 1,000 `get_meta_vars` calls fell from **9.489831 s to 0.012004 s** across three +alternating repetitions. Each trial starts with a fresh trace and includes the +first cache fill. The baseline restores only the two original identity methods; +both variants use the same constructor and other methods. + +| Repetition | Legacy | Cache | +|---|---:|---:| +| 1 (legacy first) | 9.256223 s | 0.014135 s | +| 2 (cache first) | 9.489831 s | 0.010777 s | +| 3 (legacy first) | 9.548995 s | 0.012004 s | + +The shared input has eight process IDs, sixteen thread IDs, deterministic missing +values, and no context managers. All six ordered outputs match: 750 empty +dictionaries and 250 `None` values; input hashes remain unchanged. Input generation, +garbage collection, and result hashing are excluded. The run used a macOS arm64 +workstation, Python 3.10.16, pandas 2.2.3, NumPy 1.26.4, and native thread settings +of one. No CPU affinity or exclusive machine access was imposed. + +This is an identity-lookup microbenchmark, not completed invariant inference or +training overhead. Paired full-inference results remain pending. Earlier profiling +attempts terminated with SIGSEGV at scheduled diagnostic timers and produced no +completed profile/output; the candidate stopped before the cache phase. Those +attempts provide no valid performance comparison. + +## Correctness and contract + +All 43 local context/call/identity regression tests and all 11 identity tests in the +matching Linux runtime pass. Independent review adds 78 differential comparisons +across 13 dtype/value cases, six pickle protocols, and two spawned-worker tasks. +The checks cover nulls, integer precision, scalar types, encounter order, missing +columns, independent process/thread membership, fresh return lists, stage isolation, +frame/length invalidation, serialization, and actual precondition inference. + +Identity values, dtypes, and row order must remain stable within a frame. Same-size +in-place edits, column replacement, and reordering require a new `TracePandas`. +The cache stores O(P + T) values; returned lists and membership checks still cost +O(P + T). Its source reference is not a copied DataFrame, but can retain the old +frame until invalidation. No RSS/PSS, peak-memory, or multiprocessing memory-scaling +improvement is claimed. + +From the checkout, reproduce the isolated measurement with: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m tools.benchmarks.identity_cache --rows 573494 --calls 1000 --repetitions 3 + +python -m unittest tests.test_context_index tests.test_call_index tests.test_identity_cache +``` + +Use `--rows 1000 --calls 20` for a smoke test. The script needs no external trace; +it records the checkout commit and source hash and checks output equality. +[Compact evidence and provenance](inference-identity-cache-20261025.json) retain the +measured source commit, runtime, inputs, per-trial timings, and limitations. diff --git a/tools/benchmarks/identity_cache.py b/tools/benchmarks/identity_cache.py new file mode 100644 index 00000000..3a82e1d2 --- /dev/null +++ b/tools/benchmarks/identity_cache.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Deterministic identity-lookup microbenchmark, with no external trace input. + +Run against the candidate checkout, for example: + python -m tools.benchmarks.identity_cache > results.json +Small smoke test: + python -m tools.benchmarks.identity_cache --rows 1000 --calls 20 + +The legacy subclass changes only the two identity methods back to their +pre-cache implementation. Each trial constructs a fresh trace over the same +read-only DataFrame, then makes the same ordered get_meta_vars calls. The first +lookup/cache build is included. Three repetitions alternate execution order. +Input generation, correctness hashing, and garbage collection are not timed. + +This measures identity scans, not full inference. The synthetic trace has no +context managers. No RSS/PSS, peak memory, or multiprocessing scaling is measured. +The frame-memory estimate is not whole-process memory. Cache tuples add O(P+T) +stored values; each return still allocates a fresh O(P+T) list. The extra source +reference retains a frame until invalidation and is not a DataFrame copy. +""" + +import argparse +import gc +import hashlib +import inspect +import json +import logging +import os +import platform +import statistics +import subprocess +import sys +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as trace_module +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyIdentityTrace(TracePandas): + """Original lookup expressions; all other behavior uses the same checkout.""" + + def get_process_ids(self): + if "process_id" not in self.events.columns: + return [] + return self.events["process_id"].dropna().unique().tolist() + + def get_thread_ids(self): + if "thread_id" not in self.events.columns: + return [] + return self.events["thread_id"].dropna().unique().tolist() + + +def synthetic_frame(rows): + sequence = np.arange(rows, dtype=np.int64) + process_ids = ((sequence // 16) % 8 + 100).astype(np.float64) + thread_ids = (sequence % 16 + 1000).astype(np.float64) + # NaNs exercise the same dropna work as mixed event traces. They never + # remove every occurrence of an identity at the minimum supported size. + process_ids[sequence % 97 == 0] = np.nan + thread_ids[sequence % 193 == 0] = np.nan + return pd.DataFrame( + {"time": sequence, "process_id": process_ids, "thread_id": thread_ids} + ) + + +def fixed_queries(calls): + # Six valid, one unknown-process, and one unknown-thread call per cycle. + # Process/thread presence remains independent, matching get_meta_vars. + return [ + ( + i, + -1 if i % 8 == 6 else 100 + i % 8, + -1 if i % 8 == 7 else 1000 + (i * 3) % 16, + ) + for i in range(calls) + ] + + +def digest(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def trial(cls, data, queries): + gc.collect() + begin = time.perf_counter() + trace = cls(data, truncate_incomplete_func_calls=False) + constructed = time.perf_counter() + results = [trace.get_meta_vars(*query) for query in queries] + finished = time.perf_counter() + record = { + "construction_seconds": constructed - begin, + "queries_seconds": finished - constructed, + "total_seconds": finished - begin, + "result_sha256": digest(results), + "none_results": sum(value is None for value in results), + "empty_dict_results": sum(value == {} for value in results), + } + if cls is TracePandas: + assert trace._identity_values_source is trace.events + record["cached_identity_counts"] = { + key: len(values) for key, values in trace._identity_values.items() + } + record["cache_tuple_containers_shallow_bytes"] = sum( + sys.getsizeof(values) for values in trace._identity_values.values() + ) + return record + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=573494) + parser.add_argument("--calls", type=int, default=1000) + parser.add_argument("--repetitions", type=int, default=3) + args = parser.parse_args() + if args.rows < 256 or args.calls < 8 or args.repetitions < 1: + parser.error("require rows >= 256, calls >= 8, repetitions >= 1") + if not hasattr(TracePandas, "_get_identity_values"): + parser.error("import a candidate checkout containing the identity cache") + + logging.disable(logging.CRITICAL) + data = synthetic_frame(args.rows) + queries = fixed_queries(args.calls) + input_hash = hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + expected = [None if i % 8 in (6, 7) else {} for i in range(args.calls)] + expected_hash = digest(expected) + measurements = [] + variants = {"legacy": LegacyIdentityTrace, "candidate": TracePandas} + for repetition in range(args.repetitions): + order = ["legacy", "candidate"] + if repetition % 2: + order.reverse() + for position, variant in enumerate(order): + record = trial(variants[variant], data, queries) + assert record["result_sha256"] == expected_hash, record + record.update( + repetition=repetition + 1, position=position + 1, variant=variant + ) + measurements.append(record) + + # Catch accidental mutation of the shared input by either implementation. + assert ( + hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + == input_hash + ) + medians = { + variant: { + metric: statistics.median( + record[metric] + for record in measurements + if record["variant"] == variant + ) + for metric in ("construction_seconds", "queries_seconds", "total_seconds") + } + for variant in variants + } + report = { + "scope": "construction plus repeated identity-checked get_meta_vars calls", + "settings": vars(args), + "input": { + "sha256_pandas_row_hashes": input_hash, + "query_sha256": digest(queries), + "dtypes": {key: str(value) for key, value in data.dtypes.items()}, + "dataframe_deep_bytes": int(data.memory_usage(index=True, deep=True).sum()), + "context_manager_count": 0, + "process_count": 8, + "thread_count": 16, + }, + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "host_class": f"local {platform.system()} {platform.machine()} workstation", + "logical_cpu_count": os.cpu_count(), + "pandas": pd.__version__, + "numpy": np.__version__, + "native_thread_settings": { + name: os.environ.get(name) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + ) + }, + "candidate_source_sha256": hashlib.sha256( + Path(trace_module.__file__).read_bytes() + ).hexdigest(), + "candidate_commit": subprocess.check_output( + [ + "git", + "-C", + str(Path(trace_module.__file__).resolve().parents[2]), + "rev-parse", + "HEAD", + ], + text=True, + ).strip(), + "candidate_source_matches_commit": subprocess.run( + [ + "git", + "-C", + str(Path(trace_module.__file__).resolve().parents[2]), + "diff", + "--quiet", + "HEAD", + "--", + "traincheck/trace/trace_pandas.py", + ], + check=False, + ).returncode + == 0, + "legacy_methods_source_sha256": hashlib.sha256( + inspect.getsource(LegacyIdentityTrace).encode() + ).hexdigest(), + }, + "measurements": measurements, + "medians": medians, + "median_total_speedup": ( + medians["legacy"]["total_seconds"] / medians["candidate"]["total_seconds"] + ), + "correctness": { + "all_results_equal_expected": True, + "result_sha256": expected_hash, + "input_unchanged": True, + }, + "limitations": [ + "Synthetic identity-lookup microbenchmark, not completed inference.", + "No context-manager work or trace-file loading; input generation excluded.", + "No RSS/PSS, peak-memory, or multiprocessing memory-scaling measurement.", + "Cache stores O(P+T) values and returns fresh O(P+T) lists.", + "Stable identity values, dtypes, and row order required within one frame.", + ], + } + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() From 1b5aedffbeefd4cdd9d30036dbb0a833521526cf Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:43:35 +0800 Subject: [PATCH 09/36] Isolate pruning state between independent precondition solves --- ...precondition-state-isolation-20261025.json | 130 +++++++++++++++ .../precondition-state-isolation.md | 59 +++++++ tests/test_precondition_state_isolation.py | 152 ++++++++++++++++++ traincheck/invariant/precondition.py | 6 +- 4 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 docs/measurements/precondition-state-isolation-20261025.json create mode 100644 docs/measurements/precondition-state-isolation.md create mode 100644 tests/test_precondition_state_isolation.py diff --git a/docs/measurements/precondition-state-isolation-20261025.json b/docs/measurements/precondition-state-isolation-20261025.json new file mode 100644 index 00000000..25924a80 --- /dev/null +++ b/docs/measurements/precondition-state-isolation-20261025.json @@ -0,0 +1,130 @@ +{ + "measurement_date": "2026-10-25", + "base_commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "base_source_sha256": "b32c83db3546567b3a5bc4d8b6a589a862c39b5fb59bd284b885d72e47d47a68", + "fixed_source_sha256": "af6a2d33b7aff2f09260144dd73a828910357e520be259995c5a2d529b719f7a", + "fixture": { + "common_record": { + "time": 0, + "process_id": 1, + "thread_id": 2, + "function": "fixture.step" + }, + "A": { + "positive": { + "meta_vars.phase": "train", + "meta_vars.signal": true + }, + "negative": { + "meta_vars.phase": "train", + "meta_vars.signal": false + } + }, + "B": { + "positive": { + "meta_vars.phase": "train" + }, + "negative": { + "meta_vars.phase": "eval" + } + }, + "expected_A_clause": { + "prop_name": "meta_vars.signal", + "type": "constant", + "values": [ + true + ] + }, + "expected_B_clause": { + "prop_name": "meta_vars.phase", + "type": "constant", + "values": [ + "train" + ] + } + }, + "fresh_process_sequences": [ + { + "order": [ + "B" + ], + "before_counts": [ + 1 + ], + "after_counts": [ + 1 + ], + "python_random_state_unchanged_before_and_after": true + }, + { + "order": [ + "A", + "B" + ], + "before_counts": [ + 1, + 0 + ], + "after_counts": [ + 1, + 1 + ], + "python_random_state_unchanged_before_and_after": true + }, + { + "order": [ + "B", + "A" + ], + "before_counts": [ + 1, + 1 + ], + "after_counts": [ + 1, + 1 + ], + "python_random_state_unchanged_before_and_after": true + }, + { + "order": [ + "B", + "B" + ], + "before_counts": [ + 1, + 1 + ], + "after_counts": [ + 1, + 1 + ], + "python_random_state_unchanged_before_and_after": true + } + ], + "regression_tests": { + "total": 6, + "before_failures": 5, + "before_passes": 1, + "after_passes": 6 + }, + "coverage": [ + "independent and reversed order", + "repeated independent solves", + "explicit set updated and reused", + "recursive sibling sharing with distinct root sets", + "fresh versus warm process histories" + ], + "limits": [ + "No multi-worker-count scheduling matrix measured.", + "No global inference ordering or scheduling determinism claim.", + "No unchanged large-trace output claim; fixing leakage may recover additional preconditions and change inference outputs.", + "Independent correctness fix; not part of the identity-cache performance comparison." + ], + "evidence_sha256": { + "before.json": "cb4fe5ac7cdb5ebe1544dc61ec06ed18de2a435cba2fad2d33fbbfb4ac022119", + "after.json": "62beead0bab542c5b7961588ed99d8c0977696169caacbca77d845c0b74e1383", + "tests-before.log": "970f9696404be2409515ab16d0a44d452e3a82d4d85acfb42844ef3b8f23b0d5", + "tests-after.log": "bebcffb3ddb091ef39bac8cfea9085bbe09e97363ba67c01acd50c3821e1096d" + } +} diff --git a/docs/measurements/precondition-state-isolation.md b/docs/measurements/precondition-state-isolation.md new file mode 100644 index 00000000..4e431546 --- /dev/null +++ b/docs/measurements/precondition-state-isolation.md @@ -0,0 +1,59 @@ +# Isolate pruning state between independent precondition solves + +`find_precondition_from_single_group` used a mutable default set for +`_pruned_clauses`. A clause rejected as uninformative for one solve remained in +that set and was removed from later independent solves in the same process. +This can suppress a valid precondition based on prior process history, even +without sampling or RNG changes. + +The fix uses a `None` default and allocates a fresh set only when no set is +provided. Recursive calls still receive the same explicit set, and callers that +supply a set retain its update/reuse behavior. No recursion or pruning rule changes. + +## Minimal reproduction + +Each synthetic example has one record with the same process/thread/function. +Solve A has phase `train` in both positive and negative examples; its signal is +`True` for positives and `False` for negatives. A correctly learns +`signal == True` and prunes `phase == train` as irrelevant to A. + +Independent solve B has positive phase `train` and negative phase `eval`. +It needs precisely the phase clause that A pruned. Before the fix, running B +after A loses that clause and returns no precondition. After the fix, B returns +`phase == train`. + +| Sequence in a fresh process | Before: preconditions per call | After | +|---|---|---| +| B | 1 | 1 | +| A, B | 1, **0** | 1, **1** | +| B, A | 1, 1 | 1, 1 | +| B, B | 1, 1 | 1, 1 | + +All examples use the actual `TracePandas`, clause construction, pruning, and +verification code. Python RNG state stays unchanged in every sequence. These are +fresh-process versus warm-process histories, not a multi-worker-count matrix. +Because workers retain Python module state across tasks, this defect can make a +solve depend on which earlier tasks ran in that process, independently of RNG. + +## Validation and scope + +Six focused regression tests had five failures before the fix and all pass +afterward. They cover independent/reversed/repeated solves, fresh versus warm +processes, caller-supplied sets, and recursion. The recursive fixture splits two +positive groups into two valid preconditions; siblings share one set, while a +new root solve receives a different set. Repository-pinned lint, type, and format +hooks pass. + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m unittest tests.test_precondition_state_isolation -v +``` + +This is an independent correctness fix. Recovering previously suppressed clauses +can add preconditions or otherwise change inference outputs. No unchanged +large-trace output, global order independence, scheduling determinism, or +performance improvement is claimed. The identity-cache performance comparison +uses its own unchanged candidates. + +[Compact reproduction evidence](precondition-state-isolation-20261025.json) +records the fixture, before/after counts, source hashes, test results, and limits. diff --git a/tests/test_precondition_state_isolation.py b/tests/test_precondition_state_isolation.py new file mode 100644 index 00000000..1da6f370 --- /dev/null +++ b/tests/test_precondition_state_isolation.py @@ -0,0 +1,152 @@ +"""Keep pruning state local to one precondition solve, including its children.""" + +import copy +import json +import logging +import subprocess +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +import pandas as pd + +import traincheck.invariant.precondition as precondition +from traincheck.invariant.base_cls import PT, Precondition, PreconditionClause +from traincheck.trace.trace_pandas import TracePandas + + +def example(phase, signal=None): + record = { + "time": 0, + "process_id": 1, + "thread_id": 2, + "function": "fixture.step", + "meta_vars.phase": phase, + } + if signal is not None: + record["meta_vars.signal"] = signal + return [record] + + +def trace(): + return TracePandas( + pd.DataFrame({"time": [0, 1], "process_id": [1, 1], "thread_id": [2, 2]}), + truncate_incomplete_func_calls=False, + ) + + +def solve(name, **kwargs): + fixtures = { + "A": ([example("train", True)], [example("train", False)]), + "B": ([example("train")], [example("eval")]), + } + positive, negative = copy.deepcopy(fixtures[name]) + return precondition.find_precondition_from_single_group( + positive, negative, [trace()], **kwargs + ) + + +def constant(prop, dtype, value): + return PreconditionClause(prop, dtype, PT.CONSTANT, None, {value}) + + +class PreconditionStateIsolationTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def assert_phase_condition(self, result): + self.assertEqual( + result, [Precondition([constant("meta_vars.phase", str, "train")])] + ) + + def test_prior_independent_solve_does_not_prune_required_clause(self): + self.assertEqual( + solve("A"), + [Precondition([constant("meta_vars.signal", bool, True)])], + ) + self.assert_phase_condition(solve("B")) + + def test_reversed_independent_order_preserves_later_result(self): + self.assert_phase_condition(solve("B")) + solve("A") + self.assert_phase_condition(solve("B")) + + def test_repeated_independent_solve(self): + for _ in range(3): + self.assert_phase_condition(solve("B")) + + def test_explicit_empty_set_is_updated_and_reused(self): + state = set() + solve("A", _pruned_clauses=state) + self.assertIn(constant("meta_vars.phase", str, "train"), state) + # Explicit state still means to suppress these clauses in this solve. + self.assertEqual(solve("B", _pruned_clauses=state), []) + self.assert_phase_condition(solve("B")) + + def test_recursive_siblings_share_state_but_new_roots_do_not(self): + original = precondition.find_precondition_from_single_group + root_children = [] + for _ in range(2): + children = [] + + def observe(*args, **kwargs): + children.append( + ( + kwargs["_pruned_clauses"], + kwargs["_skip_pruning"], + kwargs["_current_depth"], + ) + ) + return original(*args, **kwargs) + + positive = [example("train"), example("eval")] + negative = [example("test")] + with patch.object( + precondition, "find_precondition_from_single_group", side_effect=observe + ): + result = original(positive, negative, [trace()]) + self.assertEqual(len(result), 2) + self.assertTrue( + all( + any(condition.verify(item) for condition in result) + for item in positive + ) + ) + self.assertTrue( + all(not condition.verify(negative[0]) for condition in result) + ) + self.assertEqual(len(children), 2) + self.assertIs(children[0][0], children[1][0]) + self.assertTrue(all(skip and depth == 1 for _, skip, depth in children)) + self.assertIn(constant("process_id", int, 1), children[0][0]) + root_children.append(children) + self.assertIsNot(root_children[0][0][0], root_children[1][0][0]) + + def test_fresh_and_warm_processes_find_same_clause(self): + outputs = [] + for order in (("B",), ("A", "B"), ("B", "A", "B")): + script = ( + "import json\n" + "from tests.test_precondition_state_isolation import solve\n" + f"results = [solve(name) for name in {order!r}]\n" + "print(json.dumps([item.to_dict() for item in results[-1]], sort_keys=True))\n" + ) + child = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=True, + ) + outputs.append(json.loads(child.stdout)) + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[0], outputs[2]) + self.assertEqual(len(outputs[0]), 1) + self.assertEqual(outputs[0][0]["clauses"][0]["prop_name"], "meta_vars.phase") + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/invariant/precondition.py b/traincheck/invariant/precondition.py index f86ffb95..7fe71573 100644 --- a/traincheck/invariant/precondition.py +++ b/traincheck/invariant/precondition.py @@ -424,7 +424,7 @@ def find_precondition_from_single_group( negative_examples: list[list[dict]], traces: list[Trace], keys_to_skip: list[str] = [], - _pruned_clauses: set[PreconditionClause] = set(), + _pruned_clauses: set[PreconditionClause] | None = None, _skip_pruning: bool = False, _current_depth: int = 0, ) -> list[Precondition]: @@ -444,6 +444,10 @@ def find_precondition_from_single_group( To implement the invariant split OP. We need to determine how this verification / pruning process should be done, because now all the `Precondition` objects have to be violated in the negative examples. """ + # Recursive children share pruning state; independent solves start fresh. + if _pruned_clauses is None: + _pruned_clauses = set() + logger.debug( f"Calling precondition inference with \n# positive examples: {len(positive_examples)}, \n# negative examples: {len(negative_examples)}, at depth {_current_depth}" ) From 99d16905296d83495570fdcafa91370b242d2528 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 15:57:10 +0800 Subject: [PATCH 10/36] Document complete identity-cache inference comparison --- .../inference-identity-cache-20261025.json | 14 +- ...ce-identity-cache-full-trace-20261025.json | 247 ++++++++++++++++++ docs/measurements/inference-identity-cache.md | 68 ++++- 3 files changed, 319 insertions(+), 10 deletions(-) create mode 100644 docs/measurements/inference-identity-cache-full-trace-20261025.json diff --git a/docs/measurements/inference-identity-cache-20261025.json b/docs/measurements/inference-identity-cache-20261025.json index fa3df296..690fa7fa 100644 --- a/docs/measurements/inference-identity-cache-20261025.json +++ b/docs/measurements/inference-identity-cache-20261025.json @@ -106,9 +106,17 @@ } }, "full_inference": { - "status": "paired_unprofiled_replay_pending", - "profile_attempts": "Both attempts terminated with SIGSEGV at scheduled diagnostic timers; no completed profile/output. Candidate stopped before the identity-cache phase.", - "performance_comparison_valid": false + "status": "completed_unprofiled_pair", + "evidence": "inference-identity-cache-full-trace-20261025.json", + "baseline_wall_seconds": 1385.656381815672, + "candidate_wall_seconds": 98.55111291632056, + "wall_time_ratio": 14.060281419573906, + "invariant_records": 835, + "failed_hypothesis_records": 105, + "complete_json_multisets_equal": true, + "performance_comparison_valid": true, + "scope": "One sequential saved-trace CPU pair with six relations enabled; no repeated-run distribution or training-overhead claim.", + "profile_attempts": "Two earlier periodic-faulthandler plus cProfile attempts exited SIGSEGV without completed outputs and are excluded." }, "limitations": [ "Synthetic identity-lookup microbenchmark, not completed inference.", diff --git a/docs/measurements/inference-identity-cache-full-trace-20261025.json b/docs/measurements/inference-identity-cache-full-trace-20261025.json new file mode 100644 index 00000000..99c1b289 --- /dev/null +++ b/docs/measurements/inference-identity-cache-full-trace-20261025.json @@ -0,0 +1,247 @@ +{ + "measurement_date": "2026-10-25", + "scope": "One baseline-first sequential unprofiled pair on the same saved training trace; no repeats, affinity, or exclusive host reservation.", + "workload": { + "model_parameters": 6163712, + "training_steps": 10, + "sequence_length": 256, + "batch_size": 1, + "seed": 42, + "events": 573494, + "raw_trace_bytes": 286335289, + "training_run": "20261012T163111Z-88c4d357", + "trace_files": { + "trace_API_1088885_23438542501568.log": { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + "trace_API_1088885_23456244348224.log": { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + } + }, + "settings": { + "backend": "pandas", + "workers": 1, + "native_threads": 1, + "python_hash_seed": 0, + "gpu": false, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "timeout_seconds": 1800, + "sampled_child_session_pss_limit_gib": 12, + "memory_sampling_seconds": 1, + "profile_enabled": false, + "periodic_traceback_dumps": false, + "eligible_api_count": 41 + }, + "runtime": { + "host": "labmach", + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "torch_git_version": "8f9045d72f71e7aa9e16b6d7a055ec5bc1f720b5", + "pandas": "2.3.3", + "numpy": "2.1.0", + "original_trace_runtime_difference": "Original training used Python 3.12.3 and a CUDA 13.0 Torch build at the same Torch source revision. Original eligible API name set was not logged; only its count of 41 is available." + }, + "sources": { + "baseline": "Frozen call-index snapshot underlying PR141; before publication-only typing/comment formatting changes.", + "candidate_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7", + "trace_pandas_sha256": { + "baseline": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5", + "candidate": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a" + }, + "provenance_manifest_sha256": { + "baseline": "7e8e267085ece31844a820752004af6a1c5bb293a3a2733990b039d72fcd872a", + "candidate": "b48d705bd9dccbdf7bce17e8704e7ae5554a7d6239cfd545d7b5ffc66a888b09" + }, + "candidate_production_delta": "Identity caching plus publication-only import/type-annotation changes, comments/docstrings and formatting; all other production behavior unchanged. Independent pruning-state fix PR145 excluded from both variants." + }, + "comparison": { + "complete_pair": true, + "scope": "one sequential unprofiled run per source; no replicate/median claim", + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "baseline": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291", + "distinct_records": 105, + "records": 105 + }, + "invariants": { + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65", + "distinct_records": 835, + "records": 835 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2283671552, + "rss_bytes": 2291527680, + "uss_bytes": 2280935424 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 619.58, + "precondition_and_output_sort_seconds": 751.403 + }, + "markers": { + "collection_start": "2026-10-25T03:37:57.781000", + "generation_start": "2026-10-25T03:27:38.201000", + "inference_end": "2026-10-25T03:50:29.185000", + "precondition_start": "2026-10-25T03:37:57.782000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 1371.0323550701141, + "load_seconds": 10.503035306930542 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 1385.656381815672 + }, + "candidate": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291", + "distinct_records": 105, + "records": 105 + }, + "invariants": { + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65", + "distinct_records": 835, + "records": 835 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286718976, + "rss_bytes": 2294530048, + "uss_bytes": 2283991040 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 72.759, + "precondition_and_output_sort_seconds": 10.913 + }, + "markers": { + "collection_start": "2026-10-25T03:51:57.205000", + "generation_start": "2026-10-25T03:50:44.446000", + "inference_end": "2026-10-25T03:52:08.118000", + "precondition_start": "2026-10-25T03:51:57.205000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 83.71693992614746, + "load_seconds": 10.698827981948853 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 98.55111291632056 + }, + "invariants": { + "comparison": "structural_json_multiset", + "equal": true, + "reference": { + "records": 835, + "distinct_records": 835, + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65" + }, + "candidate": { + "records": 835, + "distinct_records": 835, + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65" + }, + "missing_records": 0, + "extra_records": 0, + "missing_examples": [], + "extra_examples": [] + }, + "failed_hypotheses": { + "comparison": "structural_json_multiset", + "equal": true, + "reference": { + "records": 105, + "distinct_records": 105, + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291" + }, + "candidate": { + "records": 105, + "distinct_records": 105, + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291" + }, + "missing_records": 0, + "extra_records": 0, + "missing_examples": [], + "extra_examples": [] + }, + "wall_time_ratio_baseline_over_candidate": 14.060281419573906 + }, + "context_index": { + "baseline": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 0.46406279876828194, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 0.4610072821378708, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + } + ], + "candidate": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 0.4815337397158146, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 0.46630632504820824, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + } + ] + }, + "verification": { + "complete_structural_json_multisets_equal": true, + "duplicate_multiplicity_and_array_order_preserved": true, + "json_object_key_order_and_record_line_order_ignored": true, + "both_full_and_stage_context_files_byte_identical": true, + "all_source_and_input_hashes_unchanged": true + }, + "timer_scope": { + "wall": "Child startup through exit and monitor polling, including imports and JSON serialization; initial/final integrity hashing excluded.", + "load": "Read/concatenate/TracePandas construction, including harness context serialization.", + "inference": "Engine construction, hypothesis generation/pruning, collection, final preconditions and sort; ends before invariant JSON serialization.", + "marker_intervals": "Millisecond log timestamp resolution; generation/pruning and final preconditions/sort are parts of inference, not additional wall time." + }, + "memory_scope": "Peak concurrent child-session samples, excluding monitor; PSS depends on shared pages. No memory improvement or multiprocessing scaling claim.", + "evidence_archive": { + "name": "matching-runtime-unprofiled-pair-evidence.tar.gz", + "bytes": 314914, + "sha256": "d78a815e1516131b28342c7d7e9251bc195f9708bab247c9f2d184558ec26f9e" + }, + "limitations": [ + "One sequential pair, no distribution or confidence interval.", + "Offline saved-trace CPU inference, not training, distributed scaling, online checking overhead, or new trace collection.", + "Six enabled relations; FunctionLeadRelation and FunctionCoverRelation disabled identically.", + "This preserves existing inference behavior; it does not certify inferred invariants as ground truth.", + "Earlier periodic-faulthandler plus cProfile attempts crashed and are excluded; this successful pair alone does not isolate the crash cause." + ] +} diff --git a/docs/measurements/inference-identity-cache.md b/docs/measurements/inference-identity-cache.md index 09c9919e..7cf62e09 100644 --- a/docs/measurements/inference-identity-cache.md +++ b/docs/measurements/inference-identity-cache.md @@ -7,6 +7,64 @@ list to each caller. Null filtering, scalar types, encounter order, and independ process/thread membership remain unchanged. Frame replacement or length changes invalidate both identity caches. +## Complete saved-trace inference + +On labmach, one sequential unprofiled pair reduced total wall time from +**1,385.656 s to 98.551 s (14.06x)**. Both runs completed successfully. All **835 +invariant records and 105 saved failed-hypothesis records** match as complete +structural JSON multisets, preserving duplicate occurrences and array order. +Object key order and record line order are ignored. No records were missing or +extra. Both full/stage context files are byte-identical (711 states each), and +all source and input hashes remained unchanged. + +| Measurement | Baseline | Identity cache | +|---|---:|---:| +| Total child wall time | 1,385.656 s | 98.551 s | +| Trace loading/construction | 10.503 s | 10.699 s | +| Reported inference | 1,371.032 s | 83.717 s | +| Generation and pruning (part of inference) | 619.580 s | 72.759 s | +| Final preconditions and sort (part of inference) | 751.403 s | 10.913 s | +| Peak sampled child-session PSS | 2.127 GiB | 2.130 GiB | + +The workload is the two complete trace files from the saved 6,163,712-parameter, +10-step reference training run: 573,494 events and 286,335,289 raw bytes, sequence +length 256, batch size 1, seed 42. No training was rerun. Both variants use the +context/call indexes from PR #141; this comparison isolates the identity cache, +with only publication-time imports/type annotations, comments/docstrings, and +formatting also differing. +The independent pruning-state correctness fix in PR #145 is excluded from both +variants and evaluated separately. + +Both runs used pandas, one worker, native thread settings of one, no GPU, +`PYTHONHASHSEED=0`, and the same six enabled relations. `FunctionLeadRelation` +and `FunctionCoverRelation` were disabled. Both had a 1,800-second cutoff and a +12-GiB sampled child-session PSS limit. The runtime was Python 3.11.4, Torch +2.16.0.dev20261012+cpu, pandas 2.3.3, and NumPy 2.1.0. Torch's source revision +matches the original CUDA wheel, but the original training used Python 3.12.3 +and CUDA 13.0. The eligible API count is 41 in both logs; the original name set +was not recorded, so exact original-environment coverage is not established. + +This is one baseline-first pair without CPU affinity or exclusive host access; +it is not a repeated-run speedup estimate. Total wall time includes child startup, +imports, JSON serialization, and monitor polling (about one-second resolution), +but excludes initial/final integrity hashing. The load timer includes harness +context serialization. The inference timer ends before invariant JSON writes; +its two large phases above are log-marker intervals, not additional time. Memory +is sampled every second for the concurrent child session, excludes the monitor, +and depends on shared pages; no memory reduction is established. + +The result concerns offline inference over this saved trace and enabled relation +set. It does not establish training overhead, distributed scaling, or correctness +of the inferred invariants against ground truth. Earlier attempts combining +cProfile and periodic traceback dumps exited with SIGSEGV and are excluded. Both +were disabled for this timed pair; its success alone does not identify the crash +cause. + +[Full-trace measurements and provenance](inference-identity-cache-full-trace-20261025.json) +include phase times, output digests, trace/source hashes, runtime, and archive +identity. The retained evidence archive contains both complete output sets, +context files, stdout logs, memory samples, and invocation/provenance manifests. + ## Isolated measurement On a deterministic 573,494-row synthetic trace, the median time for construction @@ -28,11 +86,8 @@ garbage collection, and result hashing are excluded. The run used a macOS arm64 workstation, Python 3.10.16, pandas 2.2.3, NumPy 1.26.4, and native thread settings of one. No CPU affinity or exclusive machine access was imposed. -This is an identity-lookup microbenchmark, not completed invariant inference or -training overhead. Paired full-inference results remain pending. Earlier profiling -attempts terminated with SIGSEGV at scheduled diagnostic timers and produced no -completed profile/output; the candidate stopped before the cache phase. Those -attempts provide no valid performance comparison. +This isolates identity lookup. The separate complete inference comparison above +measures the saved training trace. ## Correctness and contract @@ -47,8 +102,7 @@ Identity values, dtypes, and row order must remain stable within a frame. Same-s in-place edits, column replacement, and reordering require a new `TracePandas`. The cache stores O(P + T) values; returned lists and membership checks still cost O(P + T). Its source reference is not a copied DataFrame, but can retain the old -frame until invalidation. No RSS/PSS, peak-memory, or multiprocessing memory-scaling -improvement is claimed. +frame until invalidation. No memory or multiprocessing memory-scaling improvement is claimed. From the checkout, reproduce the isolated measurement with: From 64af67bbebf24c4f98d10977f0654637892ae14b Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 16:10:41 +0800 Subject: [PATCH 11/36] Document pruning-state replay output changes --- ...ition-state-isolation-replay-20261025.json | 296 ++++++++++++++++++ .../precondition-state-isolation.md | 41 +++ 2 files changed, 337 insertions(+) create mode 100644 docs/measurements/precondition-state-isolation-replay-20261025.json diff --git a/docs/measurements/precondition-state-isolation-replay-20261025.json b/docs/measurements/precondition-state-isolation-replay-20261025.json new file mode 100644 index 00000000..9b85db11 --- /dev/null +++ b/docs/measurements/precondition-state-isolation-replay-20261025.json @@ -0,0 +1,296 @@ +{ + "measurement_date": "2026-10-25", + "scope": "One completed configured replay pair; observed output differences only, not ground truth or a speed comparison.", + "source_composition": { + "cache_base_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7", + "state_fix_commit": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "application": "Only the None-default and fresh-set initialization hunks applied to cache source; prior RNG isolation retained. Standalone PR145 source/base unchanged.", + "independent_audit": { + "production_python_files_compared": 76, + "production_changed": [ + "traincheck/invariant/precondition.py" + ], + "exactly_none_default_and_fresh_set_initialization": true, + "prior_local_rng_fix_preserved": true, + "all_combined_python_source_hashes_match_manifest": true, + "new_regression_tests_match_PR145": true, + "base_archive_sha256": "c1c6e2a0dc8af8664e4e3f9a421837bee8c966a5bbe5233ab89ed68491058884", + "combined_archive_sha256": "2e22746a5f2b002c267269a9680873dcbe4f673d9988c7faefd4399e1323454a", + "combined_precondition_source_sha256": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "standalone_PR145_source_sha256": "af6a2d33b7aff2f09260144dd73a828910357e520be259995c5a2d529b719f7a" + } + }, + "configuration": { + "trace_rows": 573494, + "rank_count": 1, + "trace_file_count": 2, + "workers": 1, + "native_threads": 1, + "profile_enabled": false, + "periodic_traceback_dumps": false, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ] + }, + "runtime": { + "native_threads": 1, + "packages": { + "numpy": "2.1.0", + "pandas": "2.3.3", + "torch": "2.16.0.dev20261012+cpu" + }, + "profile_enabled": false, + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "workers": 1 + }, + "focused_tests": { + "passed": 17, + "identity_cache_tests": 11, + "state_isolation_tests": 6, + "source_unchanged": true, + "log_sha256": "1453d313f13a7cc4c3e43491f54cd604ecc30449c80422a137f2f8252c8a6d52" + }, + "outputs": { + "cache_only": { + "emitted": { + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65", + "distinct_records": 835, + "records": 835 + }, + "failed": { + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291", + "distinct_records": 105, + "records": 105 + } + }, + "cache_plus_state_isolation": { + "emitted": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "failed": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + } + } + }, + "hypothesis_identity_method": "all invariant fields except precondition, using outer FailedHypothesis example counts for failures; dictionary keys canonicalized, array order preserved", + "hypothesis_identity_counts": { + "after": 940, + "before": 940, + "only_after": 0, + "only_before": 0, + "shared": 940 + }, + "status_transitions": { + "emitted_to_emitted": 835, + "failed_to_emitted": 1, + "failed_to_failed": 104 + }, + "existing_emitted_preconditions": { + "different_even_after_order_normalization": 2, + "only_clause_disjunction_or_constant_value_order_changed": 0, + "strictly_identical": 833 + }, + "strict_structural_multisets": { + "emitted": { + "added_records": 3, + "equal": false, + "removed_records": 2, + "unchanged_records": 833 + }, + "failed": { + "added_records": 0, + "equal": false, + "removed_records": 1, + "unchanged_records": 104 + } + }, + "changed_records": [ + { + "relation": "APIContainRelation", + "apis": [ + "torch.cuda.random.get_rng_state", + "torch.cuda.current_device" + ], + "hypothesis_core_sha256": "670eebef31bf6194128618f1094c29bc334ad493c5f689f442174209287e4f5f", + "before_status": "emitted", + "after_status": "emitted", + "before_clause_count": 2, + "after_clause_count": 3, + "existing_clauses_preserved": true, + "new_constraint_targets": [ + { + "field": "thread_id", + "path": "None", + "type": "constant" + } + ], + "contains_concrete_trace_thread_id": true + }, + { + "relation": "APIContainRelation", + "apis": [ + "torch.nn.modules.normalization.RMSNorm.forward", + "torch.utils.checkpoint._internal_assert" + ], + "hypothesis_core_sha256": "10cbe781752237a942f768e17cae45ee45e56ebfab3c0c2c2ee13bdf8144d2ea", + "before_status": "emitted", + "after_status": "emitted", + "before_clause_count": 1, + "after_clause_count": 7, + "existing_clauses_preserved": true, + "new_constraint_targets": [ + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": "None", + "type": "exist" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "cache_enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "device_type" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "dtype" + ], + "type": "constant" + }, + { + "field": "thread_id", + "path": "None", + "type": "constant" + } + ], + "contains_concrete_trace_thread_id": true + }, + { + "relation": "APIContainRelation", + "apis": [ + "torch.nn.modules.linear.Linear.forward", + "torch.utils.checkpoint._internal_assert" + ], + "hypothesis_core_sha256": "bad3f84294dde635e3ea302d3ee6e21cd85658685621629c45e9cda388199a6c", + "before_status": "failed", + "after_status": "emitted", + "before_clause_count": null, + "after_clause_count": 6, + "existing_clauses_preserved": false, + "new_constraint_targets": [ + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": "None", + "type": "exist" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "cache_enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "dtype" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "device_type" + ], + "type": "constant" + }, + { + "field": "thread_id", + "path": "None", + "type": "constant" + } + ], + "contains_concrete_trace_thread_id": true + } + ], + "portability_observation": "All three changed/new predicates include a concrete trace-specific thread_id constraint; literal IDs are omitted. Applicability to future checking runs is unvalidated.", + "validation": { + "both_runs_exit_zero": true, + "both_sources_and_traces_unchanged": true, + "same_trace_hashes": true, + "same_harness_and_runtime": true, + "context_rows_states_and_order_hashes_unchanged": true, + "run_source_hashes_match_audited_manifest": true + }, + "input_traces": [ + { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + ], + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "evidence_sha256": { + "cache_only": { + "invariants.jsonl": "313f9d56d25a0a015fde67135db558f0df891a07d825e425aa47f5cc9a9d11ab", + "invariants.jsonl.failed": "3f3e1b8ecf3193dde9ef727b07a95fb8d26d6feb67faa78807cac3f93f45f8d1", + "invocation.json": "4b650c6a1725f5294b10fa6122198486a4b6c0e9fae5a1530baa16b1a000a5e9", + "provenance.json": "b48d705bd9dccbdf7bce17e8704e7ae5554a7d6239cfd545d7b5ffc66a888b09", + "result.json": "663d9f99601e1725355796a7d62a6c269c0390f4ca40d7f1b2030ef38f39a85c", + "runtime.json": "b5e7bc216ea115fa35f940dc3b1b97604e23098e70ff4145523847b0e236e8c3" + }, + "cache_plus_state_isolation": { + "invariants.jsonl": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0", + "invariants.jsonl.failed": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2", + "invocation.json": "5cdd44c2575906509285c987aa33710362c79ebdae71b10f7334f7df3b6cbbd3", + "provenance.json": "040a61bd84007f029d636449da38b3df2ce967260948c43d361a61628d7f8623", + "result.json": "732dda3e56e54b1a9d50a198522d5144c37142caa014ed1287fd0446a5f38001", + "runtime.json": "0f3b11a9f79ae2d827d003c17b57881276caafa2d0232a750751ceadeef5e9b3" + }, + "independent_reviews": { + "combined-source-review.json": "84e490ad710ccea93d300d4923f3625f0fa677212b3aef3b5ab20d35e9f49bfc", + "combined-output-review.json": "4f1810a5cc627a49556e913b950fc53ef65f0fb32ee67567288405754e2336e1", + "combined-structure-details.json": "9b6056356655c02bb361ad2b496c5543323b984f088822c4faf7c222d80128a8", + "compare_replay_outputs.py": "2749417cde5ed5f65314dc3ec4b83350f7cda32532fc9044637fea788e358367" + }, + "source_manifest": "33569e886e5ec1f5a4e5b985744652ad7b62a331256775fe2a87b6a0e3fa19e9" + }, + "limitations": [ + "One configured single-worker replay pair, not a worker-count or scheduling matrix.", + "FunctionLeadRelation and FunctionCoverRelation were disabled.", + "Structural comparisons are not ground-truth invariant validation.", + "Order normalization is limited to disjunctions, conjunctions, and constant-value membership; it is not general logical-equivalence proving.", + "Concrete thread-ID predicates may not apply across future runs; no generalization claim.", + "No speed or memory improvement claim." + ] +} diff --git a/docs/measurements/precondition-state-isolation.md b/docs/measurements/precondition-state-isolation.md index 4e431546..ca6e4c74 100644 --- a/docs/measurements/precondition-state-isolation.md +++ b/docs/measurements/precondition-state-isolation.md @@ -35,6 +35,47 @@ fresh-process versus warm-process histories, not a multi-worker-count matrix. Because workers retain Python module state across tasks, this defect can make a solve depend on which earlier tasks ran in that process, independently of RNG. +## Observed changes in a complete configured replay + +A separate correctness replay applied only this PR's two production hunks to the +identity-cache candidate (`504e04d`), preserving that candidate's earlier RNG fix. +The standalone source and base of this PR are unchanged. Independent inspection +of all 76 production Python files confirmed that only `precondition.py` changed, +exactly as intended. All 17 combined tests passed (11 cache tests and six state +isolation tests). + +Both unprofiled runs completed on the same 573,494-row, single-rank trace, with +the same Linux runtime (Python 3.11.4, PyTorch 2.16.0.dev20261012+cpu, pandas 2.3.3, +NumPy 2.1.0), one worker, and one native thread. `FunctionLeadRelation` and +`FunctionCoverRelation` were disabled. Input/source hashes and ordered context +states remained unchanged. + +| Output | Cache only | Cache + state isolation | +|---|---:|---:| +| Emitted invariants | 835 | 836 | +| Failed hypotheses | 105 | 104 | + +All 940 hypothesis identities match after excluding preconditions and accounting +for the failed-record wrapper's example-count fields. One formerly failed +`APIContainRelation` is now emitted; none moved from emitted to failed. Of the +835 previously emitted records, 833 remain structurally identical and **two gain +additional constraints**. These two differences persist after normalizing clause, +disjunction, and constant-value order; they are not ordering noise. + +| API containment predicate | Observed change | +|---|---| +| `Linear.forward` calls `checkpoint._internal_assert` | Previously failed; now emitted with six autocast/context/thread clauses. | +| `RMSNorm.forward` calls `checkpoint._internal_assert` | Retains its step clause and gains six autocast/context/thread clauses (one to seven). | +| `get_rng_state` calls `current_device` | Retains both CUDA-device clauses and gains a thread clause (two to three). | + +The two existing predicates become more restrictive through added conjuncts. +All three changed/new predicates include concrete trace-specific thread IDs. +Their applicability to future checking runs is unvalidated; these results do not +establish generalization or ground-truth correctness. This single replay pair is +not a speed or memory comparison. [Compact replay evidence](precondition-state-isolation-replay-20261025.json) +records source composition, checks, output hashes, and the observed structures +without publishing literal thread IDs or raw traces. + ## Validation and scope Six focused regression tests had five failures before the fix and all pass From 24e7cb0f7a573ff6225a388eff189d9e4af5e8b5 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:34:03 +0800 Subject: [PATCH 12/36] Index function-call time windows without changing pandas grouping --- tests/test_time_range_index.py | 325 +++++++++++++++++++++++++++++++ traincheck/trace/trace_pandas.py | 158 +++++++++++++-- 2 files changed, 466 insertions(+), 17 deletions(-) create mode 100644 tests/test_time_range_index.py diff --git a/tests/test_time_range_index.py b/tests/test_time_range_index.py new file mode 100644 index 00000000..7ea110e9 --- /dev/null +++ b/tests/test_time_range_index.py @@ -0,0 +1,325 @@ +"""Differential tests against the unchanged pandas filtering path.""" + +import copy +import logging +import math +import pickle +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from traincheck.instrumentor.tracer import TraceLineType as T +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyRangeTrace(TracePandas): + def _get_func_call_time_positions(self, *args): + return None + + +def frame(times=(1, 2, 3, 4), ids=("z", "z", "a", "a")): + n = len(times) + return pd.DataFrame( + { + "time": times, + "type": [T.FUNC_CALL_PRE, T.FUNC_CALL_POST] * (n // 2), + "process_id": pd.Series([1] * n, dtype="int64"), + "thread_id": pd.Series([2] * n, dtype="int64"), + "func_call_id": ids, + "function": ["fixture.step"] * n, + "args": [{} for _ in range(n)], + "kwargs": [{} for _ in range(n)], + "exception": ["fixture failure"] * n, + } + ) + + +def make(data, cls=TracePandas): + return cls(data.copy(deep=True), truncate_incomplete_func_calls=False) + + +def value(v): + if isinstance(v, dict): + return {k: value(x) for k, x in v.items()} + if isinstance(v, list): + return [value(x) for x in v] + if isinstance(v, float) and math.isnan(v): + return (type(v).__name__, "nan") + return (type(v).__name__, v) + + +def outcome(trace, bounds, process=1, thread=2): + try: + events = trace.query_func_call_events_within_time(bounds, process, thread) + return ("result", [(type(e).__name__, value(e.__dict__)) for e in events]) + except Exception as error: + return ("exception", type(error).__name__, str(error)) + + +class IntLike(int): + pass + + +class EqualType: + def __init__(self, text): + self.text = text + + def __eq__(self, other): + return self.text == other + + def __hash__(self): + return hash(self.text) + + +class TimeRangeIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def compare(self, data, queries): + old, new = make(data, LegacyRangeTrace), make(data) + for query in queries: + with self.subTest(query=query, dtypes=str(data.dtypes)): + self.assertEqual(outcome(old, *query), outcome(new, *query)) + return old, new + + def test_strict_bounds_ties_reverse_and_empty(self): + data = frame((1, 2, 2, 3)) + _, new = self.compare( + data, + [ + ((a, b),) + for a, b in [(0, 4), (1, 3), (1, 2), (2, 3), (2, 2), (4, 0), (9, 10)] + ], + ) + self.assertIsNotNone(new._func_time_positions) + self.assertEqual( + [ + e.pre_record["func_call_id"] + for e in new.query_func_call_events_within_time((0, 4), 1, 2) + ], + ["a", "z"], + ) + + def test_empty_native_frame_and_extreme_identity_keys(self): + _, empty = self.compare(frame().iloc[:0], [((0, 5),)]) + self.assertEqual(empty._get_func_call_time_positions((0, 5), 1, 2), []) + for pid, tid in [(-(2**63), 2**63 - 1), (2**63 - 1, -(2**63))]: + data = frame() + data["process_id"] = pd.Series([pid] * 4, dtype="int64") + data["thread_id"] = pd.Series([tid] * 4, dtype="int64") + _, trace = self.compare(data, [((0, 5), pid, tid)]) + self.assertEqual( + trace._get_func_call_time_positions((0, 5), pid, tid), [0, 1, 2, 3] + ) + + def test_float_nan_inf_signed_zero(self): + data = frame( + (-math.inf, -0.0, 0.0, math.inf, math.nan, math.nan), + ("z", "z", "a", "a", "missing", "missing"), + ) + self.compare( + data, + [ + ((a, b),) + for a, b in [ + (-math.inf, math.inf), + (-1.0, 1.0), + (-0.0, 0.0), + (math.inf, -math.inf), + (math.nan, 9.0), + ] + ], + ) + + def test_integer_extremes_and_precision_fallback(self): + for times in [ + (2**53, 2**53 + 1, 2**53 + 2, 2**53 + 3), + (-(2**63), -(2**63) + 1, 2**63 - 2, 2**63 - 1), + ]: + data = frame(times) + old, new = self.compare( + data, + [ + ((times[0], times[-1]),), + ((float(times[0]), float(times[-1])),), + ((-(2**63) - 1, 2**63),), + ], + ) + self.assertIsNone( + new._get_func_call_time_positions( + (float(times[0]), float(times[-1])), 1, 2 + ) + ) + data = frame(tuple(float(2**53 + i) for i in range(4))) + self.compare( + data, [((2**53 + 1, 2**53 + 3),), ((float(2**53), float(2**53 + 4)),)] + ) + + def test_numeric_query_types_fallback(self): + data = frame() + for process, thread in [ + (True, 2), + (1.0, 2.0), + (np.int64(1), np.int64(2)), + (IntLike(1), 2), + (2**64, 2), + (None, 2), + ]: + _, new = self.compare(data, [((0, 5), process, thread)]) + self.assertIsNone( + new._get_func_call_time_positions((0, 5), process, thread) + ) + for bounds in [ + (np.int64(0), np.int64(5)), + (IntLike(0), 5), + [0, 5], + (0,), + None, + ("a", "b"), + ]: + self.compare(data, [(bounds,)]) + + def test_other_processes_threads_and_missing_identity(self): + data = pd.concat([frame(), frame()], ignore_index=True) + data.loc[4:, "process_id"] = 3 + data.loc[4:, "thread_id"] = 4 + self.compare( + data, [((0, 5), pid, tid) for pid, tid in [(1, 2), (3, 4), (1, 4), (99, 2)]] + ) + + def test_unsorted_input_duplicate_labels_preserves_row_order(self): + data = frame((3, 4, 1, 2)) + data.index = [4, 4, 1, 1] + self.compare(data, [((0, 5),), ((0, 3),)]) + backwards = data.iloc[[1, 0, 2, 3]].copy() + self.assertEqual( + self.compare(backwards, [((0, 5),)])[1].query_func_call_events_within_time( + (0, 0), 1, 2 + ), + [], + ) + self.assertEqual(outcome(make(backwards), (0, 5))[1], "AssertionError") + + def test_duplicate_incomplete_and_exception_records(self): + for data in [ + frame().iloc[:3], + pd.concat([frame(), frame().iloc[:1]], ignore_index=True), + ]: + self.compare(data, [((0, 5),), ((0, 3),)]) + data = frame() + data.loc[1, "type"] = T.FUNC_CALL_POST_EXCEPTION + self.compare(data, [((0, 5),), ((0, 3),)]) + + def test_null_mixed_and_categorical_call_ids(self): + for ids in [ + pd.Series(["z", "z", None, None]), + pd.Series([1, 1, "a", "a"], dtype=object), + pd.Categorical(["z", "z", "a", "a"], categories=["z", "a", "unused"]), + pd.Series(["z", "z", "a", "a"], dtype="string"), + ]: + data = frame() + data["func_call_id"] = ids + self.compare(data, [((0, 5),), ((9, 10),)]) + + def test_time_and_identity_dtype_fallback(self): + for column in ["time", "process_id", "thread_id"]: + for dtype in ["object", "Int64", "Float64", "category", "uint64", "int32"]: + data = frame() + data[column] = data[column].astype(dtype) + _, new = self.compare(data, [((0, 5),)]) + self.assertIsNone(new._get_func_call_time_positions((0, 5), 1, 2)) + data = frame() + data[column] = data[column].astype(float) + data.loc[0, column] = np.nan + self.compare(data, [((0, 5),), ((0.0, 5.0),)]) + + def test_type_nulls_extensions_and_custom_comparison(self): + for dtype in ["string", "category"]: + data = frame() + data["type"] = data["type"].astype(dtype) + _, new = self.compare(data, [((0, 5),)]) + self.assertIsNone(new._get_func_call_time_positions((0, 5), 1, 2)) + for kind in [None, np.nan, pd.NA, EqualType(T.FUNC_CALL_PRE)]: + data = frame() + data.at[0, "type"] = kind + self.compare(data, [((0, 5),), ((9, 10),)]) + + def test_non_function_events_are_excluded(self): + data = pd.concat([frame(), frame()], ignore_index=True) + data.loc[4:, "type"] = "unrelated" + self.compare(data, [((0, 5),)]) + + def test_missing_and_duplicate_columns_keep_exceptions(self): + for col in ["type", "process_id", "thread_id", "func_call_id"]: + self.compare(frame().drop(columns=col), [((0, 5),)]) + data = pd.concat([frame(), frame()[["type"]]], axis=1) + self.compare(data, [((0, 5),)]) + + def test_cache_rebuild_on_replacement_and_length_change(self): + old, new = self.compare(frame(), [((0, 5),)]) + for trace in [old, new]: + trace.events = frame((10, 11, 12, 13)) + self.assertEqual(outcome(old, (9, 14)), outcome(new, (9, 14))) + for trace in [old, new]: + trace.events.drop(index=[2, 3], inplace=True) + self.assertEqual(outcome(old, (9, 14)), outcome(new, (9, 14))) + self.assertEqual(new._func_time_length, 2) + + def test_fresh_events_and_record_dicts(self): + trace = make(frame()) + first = trace.query_func_call_events_within_time((0, 5), 1, 2) + second = trace.query_func_call_events_within_time((0, 5), 1, 2) + self.assertIsNot(first[0], second[0]) + self.assertIsNot(first[0].pre_record, second[0].pre_record) + first[0].pre_record["function"] = "mutated" + self.assertEqual(second[0].pre_record["function"], "fixture.step") + self.assertEqual( + trace.query_func_call_events_within_time((0, 5), 1, 2)[0].pre_record[ + "function" + ], + "fixture.step", + ) + + def test_repeated_query_avoids_type_isin_rescan(self): + trace = make(frame()) + with patch.object( + pd.Series, "isin", side_effect=AssertionError("unexpected full scan") + ): + for _ in range(10): + self.assertEqual( + len(trace.query_func_call_events_within_time((0, 5), 1, 2)), 2 + ) + + def test_random_windows_and_permuted_rows(self): + rng = np.random.default_rng(91) + data = pd.concat( + [ + frame( + (i * 4 + 1, i * 4 + 2, i * 4 + 3, i * 4 + 4), + (f"{i}z", f"{i}z", f"{i}a", f"{i}a"), + ) + for i in range(20) + ], + ignore_index=True, + ) + for _ in range(10): + permuted = data.iloc[rng.permutation(len(data))].copy() + self.compare( + permuted, + [((int(a), int(b)),) for a, b in rng.integers(-2, 83, size=(20, 2))], + ) + + def test_pickled_index_retains_frame_and_results(self): + trace = make(frame()) + expected = outcome(trace, (0, 5)) + restored = pickle.loads(pickle.dumps(trace)) + self.assertIs(restored._func_time_source, restored.events) + self.assertEqual(outcome(restored, (0, 5)), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index e7e20cfe..49c65828 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -1,8 +1,11 @@ import logging +import math import re +from bisect import bisect_left, bisect_right from numbers import Real from typing import Any +import numpy as np import pandas as pd from traincheck.config import config @@ -117,6 +120,15 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self._identity_values_length = -1 self._identity_values: dict[str, tuple] = {} + # As with the other caches, time/type/PID/TID values, dtypes, and row + # order must remain stable. Same-size in-place edits require a new trace. + # Frame replacement or length changes invalidate this positional index. + self._func_time_source: pd.DataFrame | None = None + self._func_time_length = -1 + self._func_time_positions: ( + dict[tuple[int, int], tuple[list, list[int]]] | None + ) = None + self.column_dtypes_cached = {} self.stage_traces: dict[str, "TracePandas"] = {} @@ -1240,6 +1252,113 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: return post_record + def _get_func_call_time_positions( + self, time_range, process_id, thread_id + ) -> list[int] | None: + """Return exact positional matches, or None to retain pandas semantics. + + The fast path deliberately excludes extension/object numeric columns and + mixed scalar comparisons: pandas may coerce those differently from Python + ordering (notably integer/float comparisons above 2**53). + """ + columns = ("type", "time", "process_id", "thread_id") + if any( + name not in self.events or not isinstance(self.events[name], pd.Series) + for name in columns + ): + return None + if ( + type(time_range) is not tuple # noqa: E721 - exclude custom indexing + or len(time_range) != 2 + or any( + type(value) is not int # noqa: E721 - exact numeric comparison + for value in (process_id, thread_id) + ) + or any(not -(2**63) <= value < 2**63 for value in (process_id, thread_id)) + or any( + not isinstance(self.events[name].dtype, np.dtype) + or self.events[name].dtype != "int64" + for name in columns[2:] + ) + ): + return None + time_dtype = self.events["time"].dtype + if not isinstance(time_dtype, np.dtype): + return None + if time_dtype == "int64": + if any( + type(bound) is not int # noqa: E721 - exclude numeric subclasses + or not -(2**63) <= bound < 2**63 + for bound in time_range + ): + return None + elif time_dtype == "float64": + if any( + type(bound) is not float # noqa: E721 - preserve pandas coercion + or math.isnan(bound) + for bound in time_range + ): + return None + else: + return None + + if self._func_time_source is not self.events or self._func_time_length != len( + self.events + ): + event_types = self.events["type"] + positions = None + # A custom __eq__ or extension dtype must still execute in pandas. + if ( + isinstance(event_types.dtype, np.dtype) + and pd.api.types.is_object_dtype(event_types.dtype) + and all( + type(value) is str # noqa: E721 - exclude custom equality + or value is None + or value is pd.NA + or (type(value) is float and math.isnan(value)) # noqa: E721 + for value in event_types + ) + ): + rows: dict[tuple[int, int], list[tuple[Any, int]]] = {} + types = event_types.tolist() + times = self.events["time"].tolist() + processes = self.events["process_id"].tolist() + threads = self.events["thread_id"].tolist() + for position, (kind, time, process, thread) in enumerate( + zip(types, times, processes, threads) + ): + if type(kind) is not str or kind not in ( # noqa: E721 + TraceLineType.FUNC_CALL_PRE, + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ): + continue + if time_dtype == "float64" and math.isnan(time): + continue + rows.setdefault((process, thread), []).append((time, position)) + positions = {} + for identity, records in rows.items(): + records.sort() + positions[identity] = ( + [time for time, _ in records], + [position for _, position in records], + ) + self._func_time_positions = positions + self._func_time_source = self.events + self._func_time_length = len(self.events) + + if self._func_time_positions is None: + return None + indexed = self._func_time_positions.get((process_id, thread_id)) + if indexed is None: + return [] + times, row_positions = indexed + left = bisect_right(times, time_range[0]) + right = bisect_left(times, time_range[1]) + # pandas groupby preserves input row order within each call. Time order + # cannot replace that order, even when traces arrive out of order. + return sorted(row_positions[left:right]) + def query_func_call_events_within_time( self, time_range: tuple[int | float, int | float], @@ -1250,25 +1369,30 @@ def query_func_call_events_within_time( # TODO: check if this is correct func_call_events: list[FuncCallEvent | FuncCallExceptionEvent] = [] - func_call_records = self.events[ - ( - self.events["type"].isin( - [ - TraceLineType.FUNC_CALL_PRE, - TraceLineType.FUNC_CALL_POST, - TraceLineType.FUNC_CALL_POST_EXCEPTION, - ] + positions = self._get_func_call_time_positions( + time_range, process_id, thread_id + ) + if positions is not None: + func_call_records = self.events.iloc[positions] + else: + func_call_records = self.events[ + ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_PRE, + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) ) - ) - & ( - self.events["time"].between( - time_range[0], time_range[1], inclusive="neither" + & ( + self.events["time"].between( + time_range[0], time_range[1], inclusive="neither" + ) ) - ) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] # group function calls by func_call_id func_call_groups = func_call_records.groupby("func_call_id") From 39152aded8b97ed83988e3292bb89c6e3adcf750 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:36:13 +0800 Subject: [PATCH 13/36] Add alternating time-window query benchmark with build costs --- tools/benchmarks/time_range_index.py | 179 +++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tools/benchmarks/time_range_index.py diff --git a/tools/benchmarks/time_range_index.py b/tools/benchmarks/time_range_index.py new file mode 100644 index 00000000..427a61d7 --- /dev/null +++ b/tools/benchmarks/time_range_index.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Compare exact pandas filtering with the time-range index on synthetic calls. + +Run: python -m tools.benchmarks.time_range_index > results.json +Each trial uses a fresh TracePandas. Construction and the first lazy index build +are included, along with fresh event/record construction for every query. Input +creation, result hashing, and garbage collection are outside the timed region. +Repetitions alternate legacy/candidate order. This isolates window queries; it +is not full inference and does not measure peak process memory. +""" + +import argparse +import gc +import hashlib +import json +import logging +import os +import platform +import statistics +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as trace_module +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyRangeTrace(TracePandas): + def _get_func_call_time_positions(self, *args): + return None + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=200000) + parser.add_argument("--queries", type=int, default=1000) + parser.add_argument("--repetitions", type=int, default=3) + args = parser.parse_args() + if args.rows < 4 or args.rows % 2 or args.queries < 2 or args.repetitions < 1: + parser.error("require even rows >= 4, queries >= 2, repetitions >= 1") + logging.disable(logging.CRITICAL) + row = np.arange(args.rows, dtype=np.int64) + call = row // 2 + data = pd.DataFrame( + { + "time": row.astype(np.float64), + "type": np.where( + row % 2 == 0, TraceLineType.FUNC_CALL_PRE, TraceLineType.FUNC_CALL_POST + ), + "process_id": 100 + call % 4, + "thread_id": 1000 + call % 8, + "func_call_id": [f"call-{i}" for i in call], + "function": "fixture.step", + } + ) + queries = [] + for i in range(args.queries): + index = (i * 97) % (args.rows // 2) + queries.append( + ( + (float(index * 2) - 0.5, float(index * 2) + 1.5), + 100 + index % 4, + 1000 + index % 8, + ) + ) + + def input_hash(): + return hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + + before = input_hash() + records = [] + for repetition in range(args.repetitions): + variants = [("legacy", LegacyRangeTrace), ("candidate", TracePandas)] + if repetition % 2: + variants.reverse() + for order, (name, cls) in enumerate(variants): + gc.collect() + start = time.perf_counter() + trace = cls(data, truncate_incomplete_func_calls=False) + constructed = time.perf_counter() + results = [trace.query_func_call_events_within_time(*queries[0])] + first = time.perf_counter() + results.extend( + trace.query_func_call_events_within_time(*query) + for query in queries[1:] + ) + end = time.perf_counter() + assert all(len(events) == 1 for events in results) + summary = [ + [ + { + "class": type(event).__name__, + "pre": event.pre_record, + "post": event.post_record, + } + for event in events + ] + for events in results + ] + record = { + "repetition": repetition + 1, + "order": order + 1, + "variant": name, + "construction_seconds": constructed - start, + "first_query_seconds": first - constructed, + "remaining_queries_seconds": end - first, + "total_seconds": end - start, + "result_sha256": digest(summary), + } + if name == "candidate": + assert trace._func_time_positions is not None + record["indexed_rows"] = sum( + len(positions) + for _, positions in trace._func_time_positions.values() + ) + record["identity_groups"] = len(trace._func_time_positions) + records.append(record) + assert input_hash() == before + assert len({record["result_sha256"] for record in records}) == 1 + medians = { + name: { + metric: statistics.median( + record[metric] for record in records if record["variant"] == name + ) + for metric in [ + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", + ] + } + for name in ["legacy", "candidate"] + } + print( + json.dumps( + { + "scope": "synthetic function-call window queries, including construction and lazy build; not full inference or peak memory", + "settings": vars(args), + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "pandas": pd.__version__, + "numpy": np.__version__, + "native_threads": { + name: os.environ.get(name) + for name in [ + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + ] + }, + }, + "source_sha256": hashlib.sha256( + Path(trace_module.__file__).read_bytes() + ).hexdigest(), + "input_sha256": before, + "input_unchanged": True, + "all_ordered_results_equal": True, + "measurements": records, + "medians": medians, + "median_total_ratio": medians["legacy"]["total_seconds"] + / medians["candidate"]["total_seconds"], + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() From 2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:41:58 +0800 Subject: [PATCH 14/36] fix: validate dynamic type-name metadata before fallback --- .../dynamic-type-names-results.json | 25 +++++ docs/experiments/dynamic-type-names.md | 43 ++++++++ tests/test_dynamic_typename.py | 100 ++++++++++++++++++ traincheck/utils.py | 2 +- 4 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 docs/experiments/dynamic-type-names-results.json create mode 100644 docs/experiments/dynamic-type-names.md create mode 100644 tests/test_dynamic_typename.py diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json new file mode 100644 index 00000000..b77bb34a --- /dev/null +++ b/docs/experiments/dynamic-type-names-results.json @@ -0,0 +1,25 @@ +{ + "baseline_commit": "9ac61d1140cf8679c24442a174ec12f4de069980", + "python": "3.10.16", + "torch": "2.2.2", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "method": "Identical seven-test suite, candidate module globals, exact parent typename function extracted by AST versus candidate function.", + "before": { + "tests": 7, + "failures": 3, + "errors": 0, + "passed": 4, + "failed_tests": [ + "test_dynamic_typename.DynamicTypenameTests.test_dynamic_callable_metadata_uses_class_name", + "test_dynamic_typename.DynamicTypenameTests.test_non_string_name_values_fall_back_without_truth_testing", + "test_dynamic_typename.DynamicTypenameTests.test_real_pytorch_dynamic_ops_wrapper" + ] + }, + "after": { + "tests": 7, + "failures": 0, + "errors": 0, + "passed": 7, + "failed_tests": [] + } +} diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md new file mode 100644 index 00000000..3b93cd47 --- /dev/null +++ b/docs/experiments/dynamic-type-names.md @@ -0,0 +1,43 @@ +# Dynamic type-name metadata + +## Problem and change + +PyTorch `torch._inductor.virtualized.ops` returns callable values for dynamic +`__qualname__` and `__name__` attributes. TrainCheck rejected the first value but +accepted the second, then asserted that its type name was a string. A fresh +MNIST trace-collection attempt failed before training reached its first step. + +Validate `__name__` as a nonempty string before using it. Otherwise, use the +existing reported-class fallback. This does not call dynamic metadata, truth-test +non-string values, change valid name precedence, or replace the reported +`__class__` with the object's actual type. + +## Validation + +An identical seven-test suite was run against the exact parent `typename` +function extracted from commit `9ac61d1140cf8679c24442a174ec12f4de069980` and against +the changed function, with identical module globals. Python 3.10.16 / PyTorch +2.2.2 on macOS arm64: + +| Result | Parent | Changed | +| --- | ---: | ---: | +| Tests passed | 4 | 7 | +| Tests failed | 3 | 0 | +| Test errors | 0 | 0 | + +The three previously failing tests cover callable dynamic metadata, non-string +names (including an unsafe truth-value implementation), and the real PyTorch +`OpsWrapper`. The remaining tests preserve qualified/simple names, reported-class +proxies, functions, descriptors, tensors, and parameters. Both instrumentation +and runtime modes are exercised for dynamic metadata. + +Reproduce the candidate suite with: + +```sh +python -m unittest tests.test_dynamic_typename -v +``` + +The motivating Linux training run used an existing broader temporary workaround. +Its successful collection is not evidence of an end-to-end run of this narrower +production change; the direct real-object regression test is the evidence here. +This is a compatibility fix, with no performance improvement claim. diff --git a/tests/test_dynamic_typename.py b/tests/test_dynamic_typename.py new file mode 100644 index 00000000..d5f96194 --- /dev/null +++ b/tests/test_dynamic_typename.py @@ -0,0 +1,100 @@ +"""Runtime type names must tolerate dynamic metadata without calling it.""" + +import unittest + +import torch + +from traincheck.utils import typename + + +class DynamicMetadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name): + if name in {"__qualname__", "__name__"}: + return lambda: None + raise AttributeError(name) + + +class UnsafeTruthValue: + def __bool__(self): + raise AssertionError("Metadata truth value must not be evaluated") + + +class VisibleType: + pass + + +class ReportedClass: + __module__ = "fixtures.dynamic" + + @property + def __class__(self): + return VisibleType + + +class DynamicTypenameTests(unittest.TestCase): + def test_dynamic_callable_metadata_uses_class_name(self): + for runtime in [False, True]: + self.assertEqual( + typename(DynamicMetadata(), is_runtime=runtime), + "fixtures.dynamic.DynamicMetadata", + ) + + def test_non_string_name_values_fall_back_without_truth_testing(self): + for value in [object(), 17, ["name"], UnsafeTruthValue()]: + + class Metadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name): + if name == "__name__": + return value + raise AttributeError(name) + + for runtime in [False, True]: + self.assertEqual( + typename(Metadata(), is_runtime=runtime), + "fixtures.dynamic.Metadata", + ) + + def test_valid_string_names_still_take_precedence(self): + class Metadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name): + if name == "__qualname__": + return "Outer.call" + if name == "__name__": + raise AssertionError("Qualified name should take precedence") + raise AttributeError(name) + + self.assertEqual(typename(Metadata()), "fixtures.dynamic.Outer.call") + + def test_valid_unqualified_name_is_preserved(self): + value = DynamicMetadata() + value.__name__ = "exported" + self.assertEqual(typename(value), "fixtures.dynamic.exported") + + def test_reported_class_fallback_is_preserved(self): + self.assertEqual(typename(ReportedClass()), "fixtures.dynamic.VisibleType") + + def test_standard_functions_descriptors_tensors_and_parameters(self): + self.assertEqual(typename(len), "len") + self.assertEqual(typename(str.upper), "str.upper") + tensor = torch.zeros(1) + self.assertEqual(typename(tensor), tensor.type()) + self.assertEqual(typename(torch.nn.Parameter(tensor)), "torch.nn.Parameter") + + def test_real_pytorch_dynamic_ops_wrapper(self): + from torch._inductor.virtualized import ops + + self.assertEqual(typename(ops), "torch._inductor.virtualized.OpsWrapper") + self.assertEqual( + typename(ops, is_runtime=True), + "torch._inductor.virtualized.OpsWrapper", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/utils.py b/traincheck/utils.py index 332cfe04..f62971fd 100644 --- a/traincheck/utils.py +++ b/traincheck/utils.py @@ -68,7 +68,7 @@ def typename(o, is_runtime=False): if not last_name: last_name = safe_getattr(o, "__name__", "") is_class_name_qualname = False - if not last_name: + if not isinstance(last_name, str) or not last_name: last_name = safe_getattr(o, "__class__", type(o)).__name__ is_class_name_qualname = False # Handle the typename logic for method descriptors. The `is_class_name_qualname` flag is used as a heuristic to determine if `o` is likely a method descriptor. From 74b3adf51c594d32e700bb18d45f584b78708f54 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:52:36 +0800 Subject: [PATCH 15/36] Add fresh-process invariant validation harness --- .../fresh-run-invariant-validation.md | 77 +++ tests/test_fresh_validation_coverage.py | 278 +++++++++ tools/fresh_validation/ADAPTERS.md | 29 + tools/fresh_validation/benchmark_workload.py | 528 ++++++++++++++++++ tools/fresh_validation/check_coverage.py | 394 +++++++++++++ tools/fresh_validation/compare_study.py | 195 +++++++ tools/fresh_validation/fault_control.py | 193 +++++++ tools/fresh_validation/prepare_source.py | 104 ++++ tools/fresh_validation/run_phase.py | 134 +++++ tools/fresh_validation/run_study.py | 186 ++++++ 10 files changed, 2118 insertions(+) create mode 100644 docs/measurements/fresh-run-invariant-validation.md create mode 100644 tests/test_fresh_validation_coverage.py create mode 100644 tools/fresh_validation/ADAPTERS.md create mode 100644 tools/fresh_validation/benchmark_workload.py create mode 100644 tools/fresh_validation/check_coverage.py create mode 100644 tools/fresh_validation/compare_study.py create mode 100644 tools/fresh_validation/fault_control.py create mode 100644 tools/fresh_validation/prepare_source.py create mode 100644 tools/fresh_validation/run_phase.py create mode 100644 tools/fresh_validation/run_study.py diff --git a/docs/measurements/fresh-run-invariant-validation.md b/docs/measurements/fresh-run-invariant-validation.md new file mode 100644 index 00000000..b7eda530 --- /dev/null +++ b/docs/measurements/fresh-run-invariant-validation.md @@ -0,0 +1,77 @@ +# Fresh-process invariant validation + +This harness learns rules from one small training run, then checks the frozen +rules against separate OS processes: a same-seed control and a longer run with a +different initialization/training seed. Final measurements are pending. Earlier +MNIST pilot runs are diagnostics and are not final validation results. + +The initial workloads are benchmark-derived MNIST and a small Transformer. +`benchmark_workload.py` extracts the named model definitions from a supplied +TrainCheck-Benchmarks checkout; it never executes the benchmark's top-level CLI +or downloads data. MNIST requires cached real IDX data. The Transformer uses the +checked-in WikiText prefix. See `tools/fresh_validation/ADAPTERS.md` for the original +adaptations. The fixed `data_steps=20` prepares the same corpus/vocabulary/model +shape for every run: reference/control train 10 steps with seed 42; holdout and its +uninstrumented baseline train 20 steps with seed 43. The first half of holdout inputs +overlaps reference inputs; this is not disjoint-dataset validation. + +`prepare_source.py` creates an isolated snapshot from signed source revisions: +identity/cache stack 99d1690, tracing integrity 7d6c683, TorchScript 9ac61d1, +precondition-state fix 1b5aedf (only its two production hunks), and dynamic type-name +fix 2f034ae. The existing deterministic sampling/reservoir changes are retained. +The harness PR contains no production-code change. Earlier failed setup and +historical-compatibility-patch pilots remain separate from this final recipe. + +Example from a checkout containing those commits, on Linux with an existing +compatible CPU PyTorch runtime and benchmark/data files: + +```sh +python tools/fresh_validation/prepare_source.py --repo "$PWD" --output /path/to/new-source +python tools/fresh_validation/run_study.py \ + --source /path/to/new-source --python /path/to/runtime/bin/python \ + --benchmark-root /path/to/TrainCheck-Benchmarks --data-root /path/to/MNIST-cache \ + --output /path/to/new-study --workload mnist +# Use --workload transformer and omit --data-root for the language-model adapter. +``` + +Each phase starts a new process, with native threads limited to one, a 600-second +timeout, an 8 GiB process-group RSS limit, and a 256 MiB API-trace limit. Inference is +single-worker and disables FunctionLeadRelation/FunctionCoverRelation. These +bounds and observed phase times are operational controls, not performance data. + +Coverage calls the real `inv.check(trace, check_relation_first=False)`. It counts +precondition group verifications separately from actual relation decisions using +AST-located line observations of the installed checker. API opportunities are +calls; DistinctArgument opportunities are within-step argument pairs. The checker +stops at the first violation per invariant, so unvisited remainders are explicit. +A `triggered` flag or zero violations alone does not establish active checking. +Missing tensor values, dormant rules, checker errors, unsupported coverage, and +requested-but-skipped proxy-variable checks are distinguished. Direct checking +without observers must produce exactly the same CheckerResult structures. + +Raw process/thread IDs and frozen predicates are preserved. ID-bearing clause +kinds/values and activation are reported; their mere presence does not establish +that an ID caused dormancy. A workload with no ID-bearing rules leaves the +original TorchTitan rule set's portability unresolved. + +A post-hoc sensitivity control selects a healthy, actually evaluated output-shape +rule and changes one recorded tensor's first shape dimension, retaining its type, +IDs and precondition inputs. Every other semantic field and every other raw line +must remain identical. The frozen checker must activate and violate the selected +rule. This is trace-level detector-path validation, not a reproduced training bug +or a general defect-coverage result. + +`compare_study.py` verifies source/input immutability, fresh producer PIDs, exact +same-seed control losses and canonical parameter hashes, instrumented-versus-plain +holdout losses/hashes, observer/direct equality, and the targeted fault response. +Its `study_completed` status certifies these evidence gates. It does not assert +that every invariant passed the healthy runs; those violations are retained for +analysis. Per-family reports distinguish all active decisions from rules whose +requested checks were fully executed. + +Focused counter tests: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ + python -m unittest tests.test_fresh_validation_coverage -v +``` diff --git a/tests/test_fresh_validation_coverage.py b/tests/test_fresh_validation_coverage.py new file mode 100644 index 00000000..a0519e42 --- /dev/null +++ b/tests/test_fresh_validation_coverage.py @@ -0,0 +1,278 @@ +"""Small real-checker counter oracles, including dormant and early-exit cases.""" + +import copy +import logging +import unittest + +import pandas as pd + +from tools.fresh_validation.check_coverage import observe +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.invariant.base_cls import ( + PT, + APIParam, + GroupedPreconditions, + InputOutputParam, + Invariant, + Precondition, + PreconditionClause, + Preconditions, + UnconditionalPrecondition, + VarTypeParam, +) +from traincheck.invariant.consistency_transient_vars import ( + ConsistentInputOutputRelation, + ConsistentOutputRelation, + ThresholdRelation, +) +from traincheck.invariant.contain_relation import ( + APIContainRelation, + cache_events_scanner, +) +from traincheck.invariant.DistinctArgumentRelation import ( + EXP_GROUP_NAME, + DistinctArgumentRelation, +) +from traincheck.trace.trace_pandas import TracePandas + + +def frame(outputs, *, steps=None, inputs=None): + rows = [] + for i, output in enumerate(outputs): + tensor = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float32"}} + common = { + "func_call_id": f"call-{i}", + "function": "torch.fake", + "process_id": 1, + "thread_id": 2, + "is_bound_method": False, + "obj_id": None, + "meta_vars.step": steps[i] if steps else i, + "args": {"0": copy.deepcopy(tensor if inputs is None else inputs[i])}, + "kwargs": {}, + "return_values": copy.deepcopy(output), + } + rows += [ + {**common, "type": TraceLineType.FUNC_CALL_PRE, "time": i * 3}, + {**common, "type": TraceLineType.FUNC_CALL_POST, "time": i * 3 + 1}, + ] + return TracePandas(pd.DataFrame(rows), truncate_incomplete_func_calls=False) + + +def precondition(group="pre_event", enabled=True): + return GroupedPreconditions( + {group: Preconditions([UnconditionalPrecondition()] if enabled else [])} + ) + + +def output_rule(enabled=True): + return Invariant( + ConsistentOutputRelation, + [ + APIParam("torch.fake"), + VarTypeParam("torch.Tensor", "dtype", const_value="torch.float32"), + ], + precondition(enabled=enabled), + "fixture output dtype", + ) + + +class CoverageTests(unittest.TestCase): + def setUp(self): + logging.disable(logging.CRITICAL) + self.good = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float32"}} + + def assert_same_result(self, make_trace, inv): + cache_events_scanner.clear() + observed = observe(make_trace(), inv) + cache_events_scanner.clear() + plain = observe(make_trace(), inv, enabled=False) + self.assertEqual(observed.get("checker_result"), plain.get("checker_result")) + self.assertIsNone(plain["active"]) + self.assertIsNone(plain["dormant"]) + self.assertEqual(observed["status"], "completed", observed) + return observed + + def test_healthy_real_decisions(self): + got = self.assert_same_result(lambda: frame([self.good] * 3), output_rule()) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_passes"], + got["decisions_completed"], + ), + (3, 3, 3), + ) + self.assertTrue(got["active"]) + + def test_all_preconditions_false_are_dormant(self): + got = self.assert_same_result( + lambda: frame([self.good] * 3), output_rule(False) + ) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_failures"], + got["decisions_completed"], + ), + (3, 3, 0), + ) + self.assertTrue(got["dormant"]) + + def test_missing_io_value_passes_precondition_but_runs_no_predicate(self): + inv = Invariant( + ConsistentInputOutputRelation, + [ + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, True), + APIParam("torch.fake"), + InputOutputParam(None, 0, "torch.Tensor", ("missing",), None, False), + ], + precondition(), + "missing value", + ) + got = self.assert_same_result(lambda: frame([self.good]), inv) + self.assertEqual(got["precondition_group_passes"], 1) + self.assertEqual(got["decisions_completed"], 0) + self.assertTrue(got["dormant"]) + + def test_distinct_single_call_has_zero_pairs_despite_triggered(self): + inv = Invariant( + DistinctArgumentRelation, + [APIParam("torch.fake")], + precondition(EXP_GROUP_NAME), + "distinct", + ) + got = self.assert_same_result(lambda: frame([self.good]), inv) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_passes"], + got["decisions_completed"], + ), + (0, 1, 0), + ) + self.assertTrue(got["checker_triggered"]) + self.assertTrue(got["dormant"]) + + def test_first_failure_leaves_unvisited_remainder(self): + wrong = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float64"}} + got = self.assert_same_result( + lambda: frame([wrong, self.good, self.good]), output_rule() + ) + self.assertEqual( + ( + got["opportunities"], + got["decisions_completed"], + got["unvisited_calls_after_stop"], + ), + (3, 1, 2), + ) + self.assertFalse(got["check_passed"]) + + def test_missing_tensor_is_completed_output_violation(self): + got = self.assert_same_result(lambda: frame([{"int": 1}]), output_rule()) + self.assertEqual(got["decisions_completed"], 1) + self.assertFalse(got["check_passed"]) + + def test_distinct_pair_count_and_first_violation(self): + inv = Invariant( + DistinctArgumentRelation, + [APIParam("torch.fake")], + precondition(EXP_GROUP_NAME), + "distinct", + ) + got = self.assert_same_result( + lambda: frame([self.good] * 3, steps=[1, 1, 1]), inv + ) + self.assertEqual((got["opportunities"], got["decisions_completed"]), (3, 1)) + self.assertFalse(got["check_passed"]) + self.assertIsNone(got["unvisited_calls_after_stop"]) + self.assertEqual(got["unvisited_pairs_after_stop"], 2) + + def test_io_actual_comparison_and_failure(self): + inv = Invariant( + ConsistentInputOutputRelation, + [ + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, True), + APIParam("torch.fake"), + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, False), + ], + precondition(), + "shape agreement", + ) + for shape, passed in (([2], True), ([9], False)): + got = self.assert_same_result( + lambda: frame([{"torch.FloatTensor": {"shape": shape}}]), inv + ) + self.assertEqual( + (got["decisions_completed"], got["check_passed"]), (1, passed) + ) + + def test_containment_counts_parent_decisions_even_when_empty(self): + cache_events_scanner.clear() + inv = Invariant( + APIContainRelation, + [APIParam("torch.fake"), APIParam("torch.child")], + precondition("parent_func_call_pre"), + "requires child", + ) + got = self.assert_same_result(lambda: frame([self.good] * 3), inv) + self.assertEqual((got["opportunities"], got["decisions_completed"]), (3, 1)) + self.assertFalse(got["check_passed"]) + + def test_requested_variable_precondition_skipped_without_proxy_data(self): + conditions = precondition("parent_func_call_pre") + conditions.grouped_preconditions["var_events"] = Preconditions( + [ + Precondition( + [ + PreconditionClause( + "meta_vars.phase", str, PT.CONSTANT, None, {"train"} + ) + ] + ) + ] + ) + inv = Invariant( + APIContainRelation, + [APIParam("torch.fake"), VarTypeParam("torch.Tensor", "shape")], + conditions, + "requires variable checking", + ) + + def bound_trace(): + trace = frame([self.good]) + trace.events["is_bound_method"] = True + trace.events["obj_id"] = 123 + return trace + + got = self.assert_same_result(bound_trace, inv) + self.assertTrue(got["variable_precondition_requested"]) + self.assertTrue(got["variable_check_skipped"]) + self.assertFalse(got["fully_validated_active"]) + + def test_threshold_comparison_after_successful_extraction(self): + # torch.clamp has a real signature; max=3 is the checker threshold. + def make_trace(): + trace = frame([{"torch.FloatTensor": {"max": 2}}]) + trace.events["function"] = "torch.clamp" + trace.events["kwargs"] = [{"max": {"int": 3}}] * 2 + return trace + + inv = Invariant( + ThresholdRelation, + [ + InputOutputParam("max", None, "int", None, "torch.clamp", True), + APIParam("torch.clamp"), + InputOutputParam(None, 0, "torch.Tensor", ("max",), None, False), + ], + precondition(), + "max threshold", + ) + got = self.assert_same_result(make_trace, inv) + self.assertEqual(got["decisions_completed"], 1) + self.assertTrue(got["check_passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/fresh_validation/ADAPTERS.md b/tools/fresh_validation/ADAPTERS.md new file mode 100644 index 00000000..c3f84528 --- /dev/null +++ b/tools/fresh_validation/ADAPTERS.md @@ -0,0 +1,29 @@ +# Bounded trace-impact workloads + +`benchmark_workload.py` reads TrainCheck-Benchmarks without modifying it. Run each invocation in a fresh process with the chosen TrainCheck source tree in `PYTHONPATH`: + +```sh +PYTHONPATH=/path/to/traincheck-variant python benchmark_workload.py \ + --benchmark-root /path/to/clean/TrainCheck-Benchmarks \ + --workload mnist --steps 5 --seed 42 --batch-size 2 \ + --device cuda --data-root /path/to/MNIST-cache \ + --output /path/to/new/result --mode instrumented --save-checkpoint +``` + +Use `--mode baseline` for uninstrumented execution. That mode does not import TrainCheck. Modes share source, batch preparation, model initialization, explicit training RNG reset, CUDA synchronization, and training math. The output directory must not already contain a run. No datasets or packages are downloaded. + +These are **model micro-workloads adapted from the benchmark**, not full end-to-end benchmark reproductions: + +| Name | Exact benchmark definitions | Training/data adaptation | +| --- | --- | --- | +| `mnist` | `instr-overhead/overhead-e2e/mnist/main.py:Net` | Original forward, NLL, Adadelta(lr=1.0). First cached MNIST training samples if `--data-root` is supplied; otherwise explicitly synthetic uniform images and random labels. Original normalization preserved. No epoch scheduler or evaluation. | +| `dcgan` | `instr-overhead/overhead-e2e/dcgan/main.py:Generator,Discriminator,weights_init` | Original discriminator real/fake BCE then generator BCE updates and Adam hyperparameters. ngf/ndf reduced from 64 to 8. Synthetic uniform RGB images and precomputed Gaussian noise; not torchvision FakeData's exact generation. No image visualization. | +| `transformer` | `false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py:PositionalEncoding,TransformerModel` | Two 32-dimensional encoder layers, 64-dimensional feedforward, 16-token sequences, vocabulary from checked-in WikiText training prefix. NLL/backward/clipped manual SGD. This repo's main loop is RNN-specific; this is an explicit Transformer training adaptation. | + +`adaptation.json` identifies exact source paths/SHA256, extracted class source SHA256, dataset hashes and adaptation details, version, model configuration, parameter count, and initial state shapes/hash. Full class/function source spans are saved in `benchmark_definitions.py`; top-level source CLIs are never executed. Original benchmark files retain their original license/copyright. + +Instrumented mode calls the public `Instrumentor` programmatically over `torch`, with API-only tracing, normal API coverage (`use_full_instr=False`), no selected API list, no proxy/model tracking, interval 1, and no warmup. It enables tracing only during training bodies via explicit `start_step`; initialization, dataset preparation, metric extraction, timing synchronization, and checkpoint writes are excluded. In both modes the training RNG is reset to `seed + 1` after setup. `TC_ORIGIN_AUDIT=1` enables a separately supplied `origin_audit.install()` before instrumentation, for a distinct diagnostic run. + +`steps.jsonl` preserves completed step timing/losses even if a later step fails. `metrics.json` includes completion, error, canonical initial/final model-state hashes and state shapes. `--save-checkpoint` writes the final model state dictionaries after successful training; it does not save optimizer/RNG state and therefore is not by itself a resume-equivalence evaluation. Canonical state hashes cover sorted names, dtypes, shapes, and tensor bytes, avoiding `torch.save` archive metadata differences. + +For fresh validation, `--data-steps 20` fixes corpus preparation and model vocabulary across runs that execute different numbers of training steps. `adaptation.json` records both the preparation horizon and actual steps, together with the producer PID, runtime and imported tracing-module path. diff --git a/tools/fresh_validation/benchmark_workload.py b/tools/fresh_validation/benchmark_workload.py new file mode 100644 index 00000000..90355957 --- /dev/null +++ b/tools/fresh_validation/benchmark_workload.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +"""Bounded, source-identified model micro-workloads for the trace bug study. + +Run each invocation in a fresh process with the desired TrainCheck variant on +PYTHONPATH. The benchmark checkout is only read. These are explicit adaptations, +not claims to reproduce the complete benchmark's data or end-to-end training. +""" + +import argparse +import ast +import gzip +import hashlib +import json +import os +import random +import struct +import subprocess +import time +from pathlib import Path + +SOURCES = { + "mnist": "instr-overhead/overhead-e2e/mnist/main.py", + "dcgan": "instr-overhead/overhead-e2e/dcgan/main.py", + "transformer": "false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py", +} + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def definitions(source, names): + """Extract exact source spans, never execute the benchmark's top-level CLI.""" + text = source.read_text() + nodes = { + node.name: node + for node in ast.parse(text).body + if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + } + missing = set(names) - nodes.keys() + if missing: + raise ValueError("Missing benchmark definitions: " + str(sorted(missing))) + return ( + "\n\n".join(ast.get_source_segment(text, nodes[name]) for name in names) + "\n" + ) + + +def mnist_raw(data_root, count): + """Read local torchvision-style IDX cache without network or torchvision.""" + candidates = [data_root / "MNIST" / "raw", data_root / "raw", data_root] + for root in candidates: + image = next( + ( + root / name + for name in ("train-images-idx3-ubyte", "train-images-idx3-ubyte.gz") + if (root / name).is_file() + ), + None, + ) + labels = next( + ( + root / name + for name in ("train-labels-idx1-ubyte", "train-labels-idx1-ubyte.gz") + if (root / name).is_file() + ), + None, + ) + if image and labels: + break + else: + raise FileNotFoundError( + "--data-root must contain the cached MNIST training IDX images and labels" + ) + + def prefix(path, length): + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rb") as handle: + return handle.read(length) + + images = prefix(image, 16 + count * 28 * 28) + targets = prefix(labels, 8 + count) + magic, available, rows, cols = struct.unpack(">IIII", images[:16]) + label_magic, available_labels = struct.unpack(">II", targets[:8]) + if (magic, rows, cols, label_magic) != (2051, 28, 28, 2049): + raise ValueError("Unexpected MNIST IDX header") + if ( + min(available, available_labels) < count + or len(images) != 16 + count * 784 + or len(targets) != 8 + count + ): + raise ValueError("MNIST cache has fewer requested samples than needed") + return ( + images[16:], + targets[8:], + { + "kind": "cached_mnist_training_prefix", + "count": count, + "files": [{"path": str(p), "sha256": sha256(p)} for p in (image, labels)], + "ordering": "first samples in IDX order; no shuffle", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + }, + ) + + +def text_prefix(path, count): + words = [] + with path.open() as handle: + for line in handle: + words.extend(line.split() + [""]) + if len(words) >= count: + break + if len(words) < count: + raise ValueError("WikiText source is shorter than the bounded workload") + words = words[:count] + vocab = {} + ids = [vocab.setdefault(word, len(vocab)) for word in words] + return ( + ids, + len(vocab), + { + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": str(path), + "sha256": sha256(path), + "token_count": len(ids), + "vocabulary_size": len(vocab), + "prefix_sha256": hashlib.sha256("\n".join(words).encode()).hexdigest(), + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + }, + ) + + +def prepare(args, torch): + import math + + import torch.nn as nn + import torch.nn.functional as F + + source = args.benchmark_root / SOURCES[args.workload] + symbols = { + "mnist": ["Net"], + "dcgan": ["weights_init", "Generator", "Discriminator"], + "transformer": ["PositionalEncoding", "TransformerModel"], + }[args.workload] + extracted = definitions(source, symbols) + (args.output / "benchmark_definitions.py").write_text(extracted) + ns = { + "__name__": "trace_impact_benchmark", + "torch": torch, + "nn": nn, + "F": F, + "math": math, + "nc": 3, + "nz": 100, + "ngf": 8, + "ndf": 8, + } + exec(compile(extracted, str(args.output / "benchmark_definitions.py"), "exec"), ns) + manifest = { + "workload_kind": "bounded_model_micro_workload", + "benchmark_source": str(source), + "benchmark_source_sha256": sha256(source), + "extracted_symbols": symbols, + "extracted_definitions_sha256": hashlib.sha256(extracted.encode()).hexdigest(), + "steps": args.steps, + "seed": args.seed, + "batch_size": args.batch_size, + "device": args.device, + "cpu_threads": args.cpu_threads, + "precision": "float32", + "deterministic_algorithms": True, + "training_only_trace": True, + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing", + ], + } + try: + manifest["benchmark_git_head"] = subprocess.check_output( + ["git", "-C", str(args.benchmark_root), "rev-parse", "HEAD"], text=True + ).strip() + except (OSError, subprocess.CalledProcessError): + manifest["benchmark_git_head"] = None + batch = args.batch_size + count = args.steps * batch + device = torch.device(args.device) + if args.workload == "mnist": + model = ns["Net"]().to(device) + optimizer = torch.optim.Adadelta(model.parameters(), lr=1.0) + if args.data_root: + images, labels, provenance = mnist_raw(args.data_root, count) + x = ( + torch.tensor(list(images), dtype=torch.float32).reshape( + count, 1, 28, 28 + ) + / 255 + ) + y = torch.tensor(list(labels), dtype=torch.int64) + else: + x = torch.rand(count, 1, 28, 28) + y = torch.randint(0, 10, (count,)) + provenance = { + "kind": "synthetic_uniform_images_and_random_labels", + "count": count, + "is_original_mnist": False, + "normalization": "(uniform[0,1] - 0.1307) / 0.3081", + } + x = ((x - 0.1307) / 0.3081).to(device) + y = y.to(device) + model.train() + + def step(index): + optimizer.zero_grad() + output = model(x[index * batch : (index + 1) * batch]) + loss = F.nll_loss(output, y[index * batch : (index + 1) * batch]) + loss.backward() + optimizer.step() + return {"loss": loss} + + models = {"model": model} + manifest.update( + data=provenance, + model_config={"architecture": "exact Net class"}, + update="Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", + ) + elif args.workload == "dcgan": + generator = ns["Generator"](1).to(device) + generator.apply(ns["weights_init"]) + discriminator = ns["Discriminator"](1).to(device) + discriminator.apply(ns["weights_init"]) + optimizer_d = torch.optim.Adam( + discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999) + ) + optimizer_g = torch.optim.Adam( + generator.parameters(), lr=0.0002, betas=(0.5, 0.999) + ) + criterion = nn.BCELoss() + images = torch.rand(count, 3, 64, 64).to(device) + # FakeData/ToTensor in the benchmark also supplies [0,1] images, but its + # per-index generation is not identical to these explicitly synthetic inputs. + noise_batches = torch.randn(args.steps, batch, 100, 1, 1).to(device) + generator.train() + discriminator.train() + + def step(index): + discriminator.zero_grad() + real = images[index * batch : (index + 1) * batch] + label = torch.full((batch,), 1, dtype=real.dtype, device=device) + output = discriminator(real) + error_real = criterion(output, label) + error_real.backward() + fake = generator(noise_batches[index]) + label.fill_(0) + output = discriminator(fake.detach()) + error_fake = criterion(output, label) + error_fake.backward() + loss_d = error_real + error_fake + optimizer_d.step() + generator.zero_grad() + label.fill_(1) + output = discriminator(fake) + loss_g = criterion(output, label) + loss_g.backward() + optimizer_g.step() + return {"loss_discriminator": loss_d, "loss_generator": loss_g} + + models = {"generator": generator, "discriminator": discriminator} + manifest.update( + data={ + "kind": "synthetic_uniform_rgb_images_and_fixed_gaussian_noise", + "count": count, + "is_original_dataset": False, + "noise_precomputed_before_trace": True, + }, + model_config={ + "ngpu": 1, + "nz": 100, + "ngf": 8, + "ndf": 8, + "nc": 3, + "image_size": 64, + }, + update="Benchmark BCE discriminator real/fake updates then generator update; Adam lr=0.0002 betas=(0.5,0.999). Channels reduced from defaults 64 to 8. Image visualization and logging reductions omitted.", + ) + else: + seq = 16 + corpus = source.parent / "data/wikitext-2/train.txt" + ids, vocab, provenance = text_prefix(corpus, (args.steps * seq + 1) * batch) + data = ( + torch.tensor(ids, dtype=torch.int64) + .view(batch, -1) + .t() + .contiguous() + .to(device) + ) + model = ns["TransformerModel"](vocab, 32, 2, 64, 2, 0.2).to(device) + model.train() + + def step(index): + start = index * seq + inputs = data[start : start + seq] + targets = data[start + 1 : start + seq + 1].reshape(-1) + model.zero_grad() + output = model(inputs) + loss = F.nll_loss(output.reshape(-1, vocab), targets) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25) + for parameter in model.parameters(): + if parameter.grad is not None: + parameter.data.add_(parameter.grad, alpha=-1.0) + return {"loss": loss} + + models = {"model": model} + manifest.update( + data=provenance, + model_config={ + "ntoken": vocab, + "ninp": 32, + "nhead": 2, + "nhid": 64, + "nlayers": 2, + "dropout_argument": 0.2, + "sequence_length": seq, + "lr": 1.0, + "clip": 0.25, + }, + update="Adapted language-model forward/NLL/backward/clipped manual SGD. Model classes copied exactly; the benchmark main.py training path is RNN-specific and is not executed. Transformer dimensions/vocabulary reduced.", + ) + return models, step, manifest + + +def model_digest(models): + """Hash tensor contents, names, dtype and shape; independent of torch.save metadata.""" + digest = hashlib.sha256() + shapes = {} + for model_name, model in sorted(models.items()): + for name, value in sorted(model.state_dict().items()): + key = model_name + "." + name + tensor = value.detach().cpu().contiguous() + metadata = {"shape": list(tensor.shape), "dtype": str(tensor.dtype)} + shapes[key] = metadata + digest.update( + key.encode() + + b"\0" + + json.dumps(metadata, sort_keys=True).encode() + + b"\0" + ) + digest.update(tensor.numpy().tobytes()) + return digest.hexdigest(), shapes + + +def run(args): + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ["TRAINCHECK_OUTPUT_DIR"] = str(args.output / "traces") + import torch + + torch.set_num_threads(args.cpu_threads) + torch.set_num_interop_threads(1) + random.seed(args.seed) + torch.manual_seed(args.seed) + torch.use_deterministic_algorithms(True) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(args.seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + prepare_args = argparse.Namespace(**vars(args)) + prepare_args.steps = args.data_steps or args.steps + if prepare_args.steps < args.steps: + raise ValueError("data_steps must cover every training step") + models, step, manifest = prepare(prepare_args, torch) + manifest.update( + steps=args.steps, + prepared_data_steps=prepare_args.steps, + process_id=os.getpid(), + source_root=os.environ.get("PYTHONPATH"), + ) + initial_hash, shapes = model_digest(models) + manifest["adapter_sha256"] = sha256(Path(__file__)) + manifest["mode"] = args.mode + manifest["torch_version"] = str(torch.__version__) + manifest["parameter_count"] = sum( + p.numel() for model in models.values() for p in model.parameters() + ) + manifest["initial_model_state_sha256"] = initial_hash + manifest["state_shapes"] = shapes + manifest["instrumentation"] = { + "target": "torch", + "models_to_track": [], + "use_full_instr": False, + "scan_proxy_in_args": False, + "funcs_to_instr": None, + "sampling_interval": 1, + "warm_up_steps": 0, + } + manifest["python_version"] = __import__("sys").version + write_json(args.output / "adaptation.json", manifest) + tc_config = None + start_step = None + if args.mode == "instrumented": + (args.output / "traces").mkdir() + import traincheck.instrumentor.tracer as tracer_module + from traincheck.config import config as tc_config + from traincheck.instrumentor.control import start_step + from traincheck.instrumentor.tracer import Instrumentor + + manifest["tracer_module"] = str(Path(tracer_module.__file__).resolve()) + tc_config.DISABLE_WRAPPER = True + tc_config.MODEL_TRACKER_STYLE = None + tc_config.INSTR_DESCRIPTORS = False + tc_config.INSTRUMENTATION_POLICY.update(interval=1, warm_up=0) + if os.environ.get("TC_ORIGIN_AUDIT"): + from origin_audit import install + + install() + count = Instrumentor( + torch, + scan_proxy_in_args=False, + use_full_instr=False, + funcs_to_instr=None, + API_dump_stack_trace=False, + sampling_interval=1, + warm_up_steps=0, + ).instrument() + manifest["instrumented_function_count"] = count + write_json(args.output / "adaptation.json", manifest) + # Imports/instrumentation are setup, and must not alter training RNG state. + random.seed(args.seed + 1) + torch.manual_seed(args.seed + 1) + if args.device == "cuda": + torch.cuda.manual_seed_all(args.seed + 1) + manifest["training_rng_seed"] = args.seed + 1 + write_json(args.output / "adaptation.json", manifest) + records = [] + success = False + error = None + try: + for index in range(args.steps): + if args.device == "cuda": + torch.cuda.synchronize() + started = time.perf_counter() + if start_step: + start_step() + try: + losses = step(index) + finally: + if tc_config is not None: + tc_config.DISABLE_WRAPPER = True + if args.device == "cuda": + torch.cuda.synchronize() + elapsed = time.perf_counter() - started + record = { + "step": index + 1, + "seconds": elapsed, + **{name: float(loss.detach().item()) for name, loss in losses.items()}, + } + records.append(record) + with (args.output / "steps.jsonl").open("a") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + print(json.dumps(record, sort_keys=True), flush=True) + success = True + except BaseException as exc: + error = {"type": type(exc).__name__, "message": str(exc)} + raise + finally: + if tc_config is not None: + tc_config.DISABLE_WRAPPER = True + final_hash, final_shapes = model_digest(models) + result = { + "success": success, + "error": error, + "completed_steps": len(records), + "steps": records, + "initial_model_state_sha256": initial_hash, + "final_model_state_sha256": final_hash, + "state_shapes_unchanged": shapes == final_shapes, + "state_shapes": final_shapes, + "timing": "synchronized training-body wall seconds; includes start_step and wrapper-disable assignment; excludes loss extraction/data preparation/checkpoint", + } + write_json(args.output / "metrics.json", result) + if success and args.save_checkpoint: + torch.save( + {name: model.state_dict() for name, model in models.items()}, + args.output / "checkpoint.pt", + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--benchmark-root", type=Path, required=True) + parser.add_argument("--workload", choices=sorted(SOURCES), required=True) + parser.add_argument("--steps", type=int, default=5) + parser.add_argument( + "--data-steps", + type=int, + help="Fixed corpus/model preparation horizon shared by reference and holdout", + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mode", choices=("baseline", "instrumented"), required=True) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--cpu-threads", type=int, default=1) + parser.add_argument( + "--data-root", type=Path, help="Optional local MNIST IDX cache; never downloads" + ) + parser.add_argument("--save-checkpoint", action="store_true") + args = parser.parse_args() + if args.steps < 1 or args.batch_size < 1 or args.cpu_threads < 1: + parser.error("steps, batch-size, cpu-threads must be positive") + if args.data_root and args.workload != "mnist": + parser.error("--data-root currently applies only to MNIST") + args.benchmark_root = args.benchmark_root.resolve() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + if (args.output / "adaptation.json").exists() or ( + args.output / "steps.jsonl" + ).exists(): + parser.error("Output already contains a run; choose a new directory") + run(args) + + +if __name__ == "__main__": + main() diff --git a/tools/fresh_validation/check_coverage.py b/tools/fresh_validation/check_coverage.py new file mode 100644 index 00000000..14773529 --- /dev/null +++ b/tools/fresh_validation/check_coverage.py @@ -0,0 +1,394 @@ +"""Observe the real offline checker; never replace its predicate or learned rules. + +Counters have explicit units. API candidates are calls, DistinctArgument candidates +are within-step argument pairs, and precondition counters are group-verification +calls. Each checker stops at its first violation. No wall-time claims are made. +""" + +import argparse +import ast +import hashlib +import importlib.metadata +import inspect +import json +import logging +import math +import os +import sys +from collections import Counter +from pathlib import Path +from unittest.mock import patch + +from traincheck.invariant import read_inv_file +from traincheck.invariant.base_cls import GroupedPreconditions +from traincheck.invariant.DistinctArgumentRelation import ( + get_event_data_per_function_per_step, +) +from traincheck.trace.trace_pandas import read_trace_file_Pandas +from traincheck.trace.types import ( + FuncCallExceptionEvent, + IncompleteFuncCallEvent, + MDNONEJSONEncoder, +) + +SUPPORTED = { + "APIContainRelation", + "ConsistentOutputRelation", + "ConsistentInputOutputRelation", + "ThresholdRelation", + "DistinctArgumentRelation", +} + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observer_sites(function, family): + """Locate exact predicate ASTs in the installed method, failing closed on drift.""" + lines, first = inspect.getsourcelines(function) + # Dedent code by its method indentation. A zero-indented line inside a + # multiline diagnostic string defeats textwrap.dedent's common prefix. + indent = len(lines[0]) - len(lines[0].lstrip()) + source = "".join( + line[indent:] if line.startswith(" " * indent) else line for line in lines + ) + tree = ast.parse(source) + starts, ends, visits = {}, {}, set() + predicates = { + "APIContainRelation": lambda s: "skip_var_unchanged_check" in s + and "found_expected_child_event" in s, + "ConsistentInputOutputRelation": lambda s: s == "input_value != output_value", + "ConsistentOutputRelation": lambda s: s == "len(returned_tensors) == 0", + "ThresholdRelation": lambda s: s + in ("output_value >= threshold_value", "output_value <= threshold_value"), + "DistinctArgumentRelation": lambda s: s + == "is_arguments_list_same(event1['args'], event2['args'])", + } + for node in ast.walk(tree): + if isinstance(node, ast.If) and predicates[family](ast.unparse(node.test)): + line = first + node.lineno - 1 + starts[line] = ast.unparse(node.test) + ends[line] = first + node.test.end_lineno - 1 + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id in {"func_call_event", "parent_event_type"} + for t in node.targets + ): + visits.add(first + node.lineno - 1) + expected = 2 if family in {"ThresholdRelation", "DistinctArgumentRelation"} else 1 + if len(starts) != expected: + raise ValueError(f"Unrecognized checker source for {family}: {starts}") + return { + "starts": starts, + "ends": ends, + "visits": visits, + "method_source_sha256": hashlib.sha256(source.encode()).hexdigest(), + } + + +def candidate_counts(trace, inv): + family = inv.relation.__name__ + api = ( + inv.params[1] + if family in {"ConsistentInputOutputRelation", "ThresholdRelation"} + else inv.params[0] + ) + calls = trace.get_func_call_ids(api.api_full_name) + if family == "DistinctArgumentRelation": + _, groups = get_event_data_per_function_per_step(trace, {api.api_full_name}) + pairs = sum( + math.comb(sum(map(len, records.values())), 2) + for records in groups.get(api.api_full_name, {}).values() + ) + return { + "candidate_calls": len(calls), + "opportunities": pairs, + "opportunity_unit": "within_step_argument_pair", + } + if family == "APIContainRelation": + eligible = len(calls) + else: + eligible = sum( + not isinstance( + trace.query_func_call_event(call), + (FuncCallExceptionEvent, IncompleteFuncCallEvent), + ) + for call in calls + ) + return { + "candidate_calls": len(calls), + "opportunities": eligible, + "opportunity_unit": ( + "parent_call" + if family == "APIContainRelation" + else "complete_function_call" + ), + "incomplete_or_exception_calls": len(calls) - eligible, + } + + +def id_clauses(value): + found = [] + if isinstance(value, dict): + if value.get("prop_name") in { + "process_id", + "thread_id", + "meta_vars.process_id", + "meta_vars.thread_id", + }: + found.append(value) + for child in value.values(): + found.extend(id_clauses(child)) + elif isinstance(value, list): + for child in value: + found.extend(id_clauses(child)) + return found + + +def observe(trace, inv, enabled=True): + """Run one unchanged inv.check(False), counting only observed completed decisions.""" + family = inv.relation.__name__ + report = { + "relation": family, + "description": inv.text_description, + "identity_clauses": id_clauses(inv.precondition.to_dict()), + "observer_supported": family in SUPPORTED, + } + if not enabled: + # Direct oracle: no opportunity pre-scan, source parsing, or monkeypatches. + try: + result = inv.check(trace, check_relation_first=False) + result.calc_and_set_time_precentage( + trace.get_start_time(), trace.get_end_time() + ) + report.update( + status="completed", + checker_result=result.to_dict(), + check_passed=result.check_passed, + checker_triggered=result.triggered, + ) + except Exception as exc: + report.update( + status="checker_error", + error={"type": type(exc).__name__, "message": str(exc)}, + check_passed=None, + ) + report.update(coverage_observed=False, active=None, dormant=None) + return report + if family not in SUPPORTED: + report.update( + status="unsupported_coverage", check_passed=None, decisions_completed=None + ) + return report + report.update(candidate_counts(trace, inv)) + sites = observer_sites(inv.relation.static_check_all, family) + report["method_source_sha256"] = sites["method_source_sha256"] + counts = Counter( + decisions_started=0, + decisions_completed=0, + visited_calls=0, + precondition_group_attempts=0, + precondition_group_passes=0, + precondition_group_failures=0, + precondition_group_errors=0, + ) + groups = {} + decision_records = [] + variable_precondition_requested = ( + family == "APIContainRelation" + and "var_events" in inv.precondition.get_group_names() + and len(inv.precondition.get_group("var_events")) > 0 + and not inv.precondition.is_group_unconditional("var_events") + ) + variable_check_skipped = False + method_code = inv.relation.static_check_all.__code__ + pending = None + original_verify = GroupedPreconditions.verify + + def verify(precondition, records, group_name, trace_for_metavars=None): + group = groups.setdefault( + group_name, {"attempts": 0, "passes": 0, "failures": 0, "errors": 0} + ) + counts["precondition_group_attempts"] += 1 + group["attempts"] += 1 + try: + result = original_verify( + precondition, records, group_name, trace_for_metavars + ) + except Exception: + counts["precondition_group_errors"] += 1 + group["errors"] += 1 + raise + label = "passes" if result else "failures" + group[label] += 1 + counts["precondition_group_" + label] += 1 + return result + + def trace_event(frame, event, arg): + nonlocal pending, variable_check_skipped + if frame.f_code is not method_code: + return None + if event == "exception": + # An extraction error can be caught by the real method; it is not a decision. + pending = None + if pending is not None: + done = event == "return" + if event == "line": + if family == "ConsistentOutputRelation": + done = frame.f_lineno in sites["visits"] + else: + done = not pending["start"] <= frame.f_lineno <= pending["end"] + if done: + counts["decisions_completed"] += 1 + decision_records.append(pending["record"]) + pending = None + if event == "line": + if frame.f_lineno in sites["visits"]: + counts["visited_calls"] += 1 + if frame.f_lineno in sites["starts"] and pending is None: + if variable_precondition_requested and frame.f_locals.get( + "skip_var_unchanged_check" + ): + variable_check_skipped = True + counts["decisions_started"] += 1 + call = frame.f_locals.get( + "func_call_id", frame.f_locals.get("parent_func_call_id") + ) + record = {"func_call_id": call} + if family == "DistinctArgumentRelation": + record = { + "pair_func_call_ids": [ + frame.f_locals[e].get("func_call_id") + for e in ("event1", "event2") + ] + } + pending = { + "start": frame.f_lineno, + "end": sites["ends"][frame.f_lineno], + "record": record, + } + return trace_event + + previous_trace = sys.gettrace() + if enabled and previous_trace is not None: + raise RuntimeError("A pre-existing Python trace observer would be overwritten") + try: + with ( + patch.object(GroupedPreconditions, "verify", verify) + if enabled + else patch.object(GroupedPreconditions, "verify", original_verify) + ): + if enabled: + sys.settrace(trace_event) + result = inv.check(trace, check_relation_first=False) + result.calc_and_set_time_precentage( + trace.get_start_time(), trace.get_end_time() + ) + report.update( + status="completed", + checker_result=result.to_dict(), + check_passed=result.check_passed, + checker_triggered=result.triggered, + ) + except Exception as exc: + report.update( + status="checker_error", + error={"type": type(exc).__name__, "message": str(exc)}, + check_passed=None, + ) + finally: + if enabled: + sys.settrace(previous_trace) + report.update(counts) + report["precondition_groups"] = groups + report["decision_records"] = decision_records + report["decisions_not_completed"] = ( + counts["decisions_started"] - counts["decisions_completed"] + ) + stopped = report["status"] != "completed" or report["check_passed"] is False + if family == "DistinctArgumentRelation": + report["visited_calls"] = None + report["unvisited_calls_after_stop"] = None + report["unvisited_pairs_after_stop"] = ( + max(0, report["opportunities"] - counts["decisions_started"]) + if stopped + else 0 + ) + else: + report["unvisited_calls_after_stop"] = ( + max(0, report["candidate_calls"] - counts["visited_calls"]) + if stopped + else 0 + ) + report["active"] = counts["decisions_completed"] > 0 + report["dormant"] = report["status"] == "completed" and not report["active"] + report["coverage_observed"] = True + report["variable_precondition_requested"] = variable_precondition_requested + report["variable_check_skipped"] = variable_check_skipped + report["fully_validated_active"] = ( + report["active"] + and not variable_check_skipped + and report["status"] == "completed" + ) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--traces", nargs="+", type=Path, required=True) + parser.add_argument("--invariants", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--observer-off", action="store_true") + args = parser.parse_args() + logging.basicConfig(level=logging.ERROR) + import traincheck.utils + + traincheck.utils._suppress_inner_progress = True + before = {str(p): sha256(p) for p in [*args.traces, args.invariants]} + trace = read_trace_file_Pandas(list(map(str, args.traces))) + invariants = read_inv_file(str(args.invariants)) + results = [] + for index, inv in enumerate(invariants): + result = observe(trace, inv, enabled=not args.observer_off) + result["invariant_index"] = index + results.append(result) + summary = Counter() + for result in results: + summary[result["status"]] += 1 + summary["active_invariants"] += bool(result.get("active")) + summary["dormant_invariants"] += bool(result.get("dormant")) + summary["violated_invariants"] += result.get("check_passed") is False + summary["identity_bearing_invariants"] += bool(result.get("identity_clauses")) + summary["identity_bearing_active"] += bool( + result.get("identity_clauses") + ) and bool(result.get("active")) + import traincheck.trace.trace_pandas as trace_module + + output = { + "pid": os.getpid(), + "check_relation_first": False, + "observer_enabled": not args.observer_off, + "runtime": { + "python": sys.version, + "executable": sys.executable, + "trace_module": str(Path(trace_module.__file__).resolve()), + "packages": { + p: importlib.metadata.version(p) for p in ("torch", "pandas", "numpy") + }, + }, + "harness_sha256": sha256(Path(__file__)), + "raw_process_ids": trace.get_process_ids(), + "raw_thread_ids": trace.get_thread_ids(), + "trace_rows": len(trace.events), + "input_sha256": before, + "inputs_unchanged": all(sha256(Path(p)) == h for p, h in before.items()), + "summary": dict(summary), + "per_invariant": results, + } + args.output.write_text(json.dumps(output, cls=MDNONEJSONEncoder, indent=2) + "\n") + print(json.dumps(summary), flush=True) + return int(bool(summary["checker_error"] or summary["unsupported_coverage"])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/fresh_validation/compare_study.py b/tools/fresh_validation/compare_study.py new file mode 100644 index 00000000..fae6b6de --- /dev/null +++ b/tools/fresh_validation/compare_study.py @@ -0,0 +1,195 @@ +"""Verify the paired study's actual outputs; do not label dormant rules as passes.""" + +import argparse +import hashlib +import json +import math +from collections import Counter +from pathlib import Path + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def compare(root): + def read(name): + return json.loads((root / name).read_text()) + + phases = { + p.parent.name: json.loads(p.read_text()) + for p in root.glob("*/phase-result.json") + } + assert phases and all( + p["exit_code"] == 0 and p["stop_reason"] is None and p["source_unchanged"] + for p in phases.values() + ), phases + metrics = { + role: read(role + "/metrics.json") + for role in ("reference", "control", "holdout", "baseline-holdout") + } + adaptations = {role: read(role + "/adaptation.json") for role in metrics} + assert all(m["success"] and m["state_shapes_unchanged"] for m in metrics.values()) + workload_keys = ( + "benchmark_source_sha256", + "extracted_definitions_sha256", + "adapter_sha256", + "data", + "prepared_data_steps", + "model_config", + "batch_size", + "device", + "cpu_threads", + "precision", + "deterministic_algorithms", + "torch_version", + "python_version", + ) + for role in metrics: + for key in workload_keys: + assert adaptations[role][key] == adaptations["reference"][key], (role, key) + invocations = [read(name + "/invocation.json") for name in phases] + assert all( + i["source_sha256"] == invocations[0]["source_sha256"] for i in invocations + ) + assert all( + i["harness_sha256"] == invocations[0]["harness_sha256"] for i in invocations + ) + + assert len({adaptations[r]["process_id"] for r in metrics}) == 4 + for role in metrics: + assert metrics[role]["completed_steps"] == ( + 10 if role in ("reference", "control") else 20 + ) + if role != "baseline-holdout": + assert adaptations[role]["tracer_module"] == str( + Path(adaptations[role]["source_root"]) + / "traincheck/instrumentor/tracer.py" + ) + if adaptations[role]["data"]["kind"].startswith("synthetic"): + raise AssertionError("Synthetic data is outside this study protocol") + + def losses(metric): + rows = [ + {k: v for k, v in step.items() if k.startswith("loss")} + for step in metric["steps"] + ] + assert all(math.isfinite(v) for row in rows for v in row.values()) + return rows + + paired = {} + for name, left, right in ( + ("same_seed_fresh_control", "reference", "control"), + ("holdout_instrumentation_comparison", "holdout", "baseline-holdout"), + ): + m1, m2 = metrics[left], metrics[right] + checks = { + "loss_trajectory_equal": losses(m1) == losses(m2), + "initial_state_sha256_equal": m1["initial_model_state_sha256"] + == m2["initial_model_state_sha256"], + "final_state_sha256_equal": m1["final_model_state_sha256"] + == m2["final_model_state_sha256"], + "shapes_equal": m1["state_shapes"] == m2["state_shapes"], + } + assert all(checks.values()), (name, checks) + paired[name] = checks + + coverage = { + role: read("check-" + role + "/coverage.json") + for role in ("reference", "control", "holdout", "holdout-direct", "fault") + } + assert all(c["inputs_unchanged"] for c in coverage.values()) + frozen_hash = hashlib.sha256( + (root / "inference/invariants.jsonl").read_bytes() + ).hexdigest() + for checked in coverage.values(): + assert ( + checked["input_sha256"][str(root / "inference/invariants.jsonl")] + == frozen_hash + ) + observed = coverage["holdout"]["per_invariant"] + direct = coverage["holdout-direct"]["per_invariant"] + assert len(observed) == len(direct) + assert all( + canonical(a["checker_result"]) == canonical(b["checker_result"]) + for a, b in zip(observed, direct) + ) + assert all(item["active"] is None and item["dormant"] is None for item in direct) + + mutation = read("fault/mutation.json") + selected = mutation["invariant_index"] + healthy = observed[selected] + faulty = coverage["fault"]["per_invariant"][selected] + assert healthy["check_passed"] and healthy["active"] + assert faulty["status"] == "completed" and not faulty["check_passed"] + assert faulty["decisions_completed"] > 0 and faulty["precondition_group_passes"] > 0 + assert ( + mutation["ids_unchanged"] + and mutation["precondition_inputs_unchanged"] + and len(mutation["edits"]) == 1 + ) + assert ( + coverage["fault"]["raw_process_ids"] == coverage["holdout"]["raw_process_ids"] + ) + assert coverage["fault"]["raw_thread_ids"] == coverage["holdout"]["raw_thread_ids"] + families = {} + for role in ("reference", "control", "holdout", "fault"): + family_counts = {} + for item in coverage[role]["per_invariant"]: + assert item["status"] == "completed", item + counts = family_counts.setdefault(item["relation"], Counter()) + counts.update( + invariants=1, + active=int(item["active"]), + dormant=int(item["dormant"]), + violations=int(not item["check_passed"]), + opportunities=item["opportunities"], + precondition_group_passes=item["precondition_group_passes"], + actual_decisions=item["decisions_completed"], + proxy_checks_skipped=int(item["variable_check_skipped"]), + fully_validated_active=int(item["fully_validated_active"]), + ) + families[role] = {name: dict(counts) for name, counts in family_counts.items()} + output = { + "status": "study_completed", + "invariants_sha256": frozen_hash, + "training_comparisons": paired, + "benchmark_data_runtime_source_and_harness_matched": True, + "observer_matches_direct_checker": True, + "per_run_summary": { + r: coverage[r]["summary"] + for r in ("reference", "control", "holdout", "fault") + }, + "per_relation": families, + "raw_ids": { + r: { + "process": coverage[r]["raw_process_ids"], + "thread": coverage[r]["raw_thread_ids"], + } + for r in ("reference", "control", "holdout") + }, + "fault": { + "kind": mutation["kind"], + "invariant_index": selected, + "healthy_decisions": healthy["decisions_completed"], + "fault_decisions": faulty["decisions_completed"], + "fault_precondition_group_passes": faulty["precondition_group_passes"], + "violation": True, + "edit": mutation["edits"][0], + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant.", + ], + } + (root / "comparison.json").write_text(json.dumps(output, indent=2) + "\n") + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + print(json.dumps(compare(parser.parse_args().root.resolve()), indent=2)) diff --git a/tools/fresh_validation/fault_control.py b/tools/fresh_validation/fault_control.py new file mode 100644 index 00000000..0da4fe66 --- /dev/null +++ b/tools/fresh_validation/fault_control.py @@ -0,0 +1,193 @@ +"""Post-hoc, one-field trace sensitivity control for a live frozen shape invariant. + +This does not inject a training bug. It changes one recorded tensor shape after +healthy training, preserves IDs and precondition inputs, and checks the same rules. +""" + +import argparse +import copy +import hashlib +import json +from pathlib import Path + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def differences(before, after, path=()): + if type(before) is not type(after): + return [path] + if isinstance(before, dict): + if before.keys() != after.keys(): + return [path] + return [p for k in before for p in differences(before[k], after[k], (*path, k))] + if isinstance(before, list): + if len(before) != len(after): + return [path] + return [ + p + for i, (a, b) in enumerate(zip(before, after)) + for p in differences(a, b, (*path, i)) + ] + if before == after or ( + isinstance(before, float) and before != before and after != after + ): + return [] + return [path] + + +def mutate(traces, invariants, healthy, output): + rules = [json.loads(line) for line in invariants.read_text().splitlines()] + coverage = json.loads(healthy.read_text()) + assert coverage["inputs_unchanged"] and coverage["observer_enabled"] + assert coverage["input_sha256"][str(invariants)] == digest(invariants) + chosen = None + call_id = None + post_events = {} + for trace in traces: + assert coverage["input_sha256"][str(trace)] == digest(trace) + for line in trace.read_bytes().splitlines(): + event = json.loads(line) + if event.get("type") == "function_call (post)": + post_events.setdefault(event["func_call_id"], []).append(event) + for item in coverage["per_invariant"]: + rule = rules[item["invariant_index"]] + if not ( + rule["relation"] == "ConsistentOutputRelation" + and rule["params"][1]["attr_name"] == "shape" + and item["active"] + and item["check_passed"] + and item["decisions_completed"] > 0 + ): + continue + for decision in item["decision_records"]: + events = post_events.get(decision["func_call_id"], []) + if len(events) != 1: + continue + values = events[0]["return_values"] + tensors = values if isinstance(values, list) else [values] + eligible = any( + "Tensor" in name + and isinstance(attributes, dict) + and isinstance(attributes.get("shape"), list) + and attributes["shape"] + and isinstance(attributes["shape"][0], int) + and not isinstance(attributes["shape"][0], bool) + for tensor in tensors + for name, attributes in tensor.items() + ) + if eligible: + chosen, call_id = item, decision["func_call_id"] + break + if chosen is not None: + break + if chosen is None: + raise ValueError( + "No healthy, actually evaluated output-shape rule with a nonempty integer shape" + ) + rule = rules[chosen["invariant_index"]] + assert call_id is not None + output.mkdir(parents=True, exist_ok=False) + (output / "traces").mkdir() + edits, fingerprints = [], {} + for trace in traces: + assert coverage["input_sha256"][str(trace)] == digest(trace) + raw_lines = trace.read_bytes().splitlines(keepends=True) + transformed = list(raw_lines) + for index, raw in enumerate(raw_lines): + event = json.loads(raw) + if ( + event.get("func_call_id") != call_id + or event.get("type") != "function_call (post)" + ): + continue + before = copy.deepcopy(event) + values = event["return_values"] + tensor_values = values if isinstance(values, list) else [values] + for tensor_index, value in enumerate(tensor_values): + for typename, attributes in value.items(): + if ( + "Tensor" not in typename + or not isinstance(attributes, dict) + or "shape" not in attributes + ): + continue + old = copy.deepcopy(attributes["shape"]) + if ( + not isinstance(old, list) + or not old + or not isinstance(old[0], int) + or isinstance(old[0], bool) + ): + continue + attributes["shape"][0] += 1 + diff = differences(before, event) + assert len(diff) == 1 + assert diff[0][:1] == ("return_values",) + assert ( + before["process_id"] == event["process_id"] + and before["thread_id"] == event["thread_id"] + ) + edits.append( + { + "file": trace.name, + "zero_based_line": index, + "path": list(diff[0]), + "before_shape": old, + "after_shape": attributes["shape"], + "raw_process_id": event["process_id"], + "raw_thread_id": event["thread_id"], + } + ) + transformed[index] = ( + json.dumps(event, separators=(",", ":")).encode() + b"\n" + ) + break + if edits: + break + target = output / "traces" / trace.name + target.write_bytes(b"".join(transformed)) + assert all( + a == b + for i, (a, b) in enumerate(zip(raw_lines, transformed)) + if not any( + edit["file"] == trace.name and edit["zero_based_line"] == i + for edit in edits + ) + ) + fingerprints[trace.name] = { + "original_sha256": digest(trace), + "fault_sha256": digest(target), + } + assert len(edits) == 1, edits + manifest = { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy shape rule with a mutable nonempty integer shape", + "invariant_index": chosen["invariant_index"], + "invariant": rule, + "func_call_id": call_id, + "edits": edits, + "traces": fingerprints, + "invariants_sha256": digest(invariants), + "healthy_coverage_sha256": digest(healthy), + "precondition_inputs_unchanged": True, + "ids_unchanged": True, + "invariants_not_retrained_or_rewritten": True, + } + (output / "mutation.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--traces", nargs="+", type=Path, required=True) + parser.add_argument("--invariants", type=Path, required=True) + parser.add_argument("--healthy-coverage", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print( + json.dumps( + mutate(args.traces, args.invariants, args.healthy_coverage, args.output) + ) + ) diff --git a/tools/fresh_validation/prepare_source.py b/tools/fresh_validation/prepare_source.py new file mode 100644 index 00000000..a2d498ba --- /dev/null +++ b/tools/fresh_validation/prepare_source.py @@ -0,0 +1,104 @@ +"""Build the explicitly composed, disposable validation source without changing Git.""" + +import argparse +import hashlib +import io +import json +import subprocess +import tarfile +from pathlib import Path + +BASE = "99d16905296d83495570fdcafa91370b242d2528" +INTEGRITY = "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4" +TORCHSCRIPT = "9ac61d1140cf8679c24442a174ec12f4de069980" +STATE = "1b5aedffbeefd4cdd9d30036dbb0a833521526cf" +TYPENAME = "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c" + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def build(repo, destination): + def git(*args): + return subprocess.check_output(["git", "-C", str(repo), *args]) + + destination.mkdir(parents=True, exist_ok=False) + with tarfile.open( + fileobj=io.BytesIO(git("archive", BASE, "traincheck", "tests")) + ) as tar: + for member in tar: + if member.isfile(): + path = destination / member.name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(tar.extractfile(member).read()) + old_hashes = { + str(p.relative_to(destination)): digest(p.read_bytes()) + for p in destination.rglob("*.py") + } + # These instrumentor files have no changes on the inference/cache stack. + for path in ( + "traincheck/instrumentor/tracer.py", + "traincheck/instrumentor/dumper.py", + ): + assert git("show", f"{INTEGRITY}^:{path}") == (destination / path).read_bytes() + assert ( + git("show", f"{TYPENAME}^:traincheck/utils.py") + == (destination / "traincheck/utils.py").read_bytes() + ) + replacements = { + "traincheck/instrumentor/tracer.py": TORCHSCRIPT, + "traincheck/instrumentor/dumper.py": INTEGRITY, + "traincheck/instrumentor/torchscript_compat.py": TORCHSCRIPT, + "tests/test_trace_integrity.py": INTEGRITY, + "tests/test_torchscript_compat.py": TORCHSCRIPT, + "tests/test_precondition_state_isolation.py": STATE, + "traincheck/utils.py": TYPENAME, + "tests/test_dynamic_typename.py": TYPENAME, + } + for path, commit in replacements.items(): + (destination / path).write_bytes(git("show", f"{commit}:{path}")) + path = destination / "traincheck/invariant/precondition.py" + source = path.read_text() + old = "_pruned_clauses: set[PreconditionClause] = set()," + new = "_pruned_clauses: set[PreconditionClause] | None = None," + assert source.count(old) == 1 + source = source.replace(old, new) + # Copy only the two approved state hunks, preserving the base RNG isolation. + state_source = git("show", f"{STATE}:traincheck/invariant/precondition.py").decode() + comment = " # Recursive children share pruning state; independent solves start fresh.\n" + insertion = ( + comment + " if _pruned_clauses is None:\n _pruned_clauses = set()\n" + ) + assert insertion in state_source + anchor = state_source.split(insertion)[1][:100] + assert source.count(anchor) == 1 + source = source.replace(anchor, insertion + anchor, 1) + assert "group_rng = random.Random(42)" in source + path.write_text(source) + hashes = { + str(p.relative_to(destination)): digest(p.read_bytes()) + for p in destination.rglob("*.py") + } + changed = sorted(p for p, h in hashes.items() if h != old_hashes.get(p)) + manifest = { + "base": BASE, + "trace_integrity": INTEGRITY, + "torchscript": TORCHSCRIPT, + "state_fix_hunks_from": STATE, + "changed_python_files": changed, + "dynamic_typename_fix": TYPENAME, + "source_sha256": hashes, + } + (destination / "source-manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n" + ) + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(build(args.repo.resolve(), args.output.resolve()), indent=2)) diff --git a/tools/fresh_validation/run_phase.py b/tools/fresh_validation/run_phase.py new file mode 100644 index 00000000..3ab1e64f --- /dev/null +++ b/tools/fresh_validation/run_phase.py @@ -0,0 +1,134 @@ +"""Run one bounded correctness phase in a fresh OS process (Linux only).""" + +import argparse +import hashlib +import json +import os +import signal +import subprocess +import time +from pathlib import Path + + +def group_memory(group): + total, processes = 0, [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + stat = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if int(stat[2]) != group: + continue + rss = next( + int(line.split()[1]) * 1024 + for line in (entry / "status").read_text().splitlines() + if line.startswith("VmRSS:") + ) + total += rss + processes.append(int(entry.name)) + except (FileNotFoundError, ProcessLookupError, StopIteration): + continue + return total, processes + + +def run(args): + args.output.mkdir(parents=True, exist_ok=False) + source_hashes = { + str(p.relative_to(args.source)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in args.source.rglob("*.py") + } + env = os.environ.copy() + env.update( + PYTHONPATH=str(args.source.resolve()), + PYTHONHASHSEED="0", + PYTHONUNBUFFERED="1", + PYTHONDONTWRITEBYTECODE="1", + CUDA_VISIBLE_DEVICES="", + OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", + OPENBLAS_NUM_THREADS="1", + NUMEXPR_NUM_THREADS="1", + VECLIB_MAXIMUM_THREADS="1", + ) + command = args.command[1:] if args.command[0] == "--" else args.command + started, peak, reason = time.monotonic(), 0, None + with (args.output / "stdout.log").open("w") as log: + proc = subprocess.Popen( + ["nice", "-n", "10", *command], + cwd=args.output, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + invocation = { + "command": command, + "pid": proc.pid, + "native_threads": 1, + "timeout_seconds": args.timeout, + "memory_limit_gib": args.memory_gib, + "trace_limit_mib": args.trace_mib, + "source_sha256": source_hashes, + "harness_sha256": { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in Path(__file__).resolve().parent.glob("*.py") + }, + } + (args.output / "invocation.json").write_text( + json.dumps(invocation, indent=2) + "\n" + ) + try: + while proc.poll() is None: + memory, _ = group_memory(proc.pid) + peak = max(peak, memory) + if memory > args.memory_gib * 1024**3: + reason = "process_group_rss_budget" + if time.monotonic() - started > args.timeout: + reason = "timeout" + if ( + sum(p.stat().st_size for p in args.output.rglob("trace_API*.log")) + > args.trace_mib * 1024**2 + ): + reason = "trace_size_budget" + if reason: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + break + time.sleep(0.25) + except BaseException: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=10) + raise + unchanged = all( + hashlib.sha256((args.source / p).read_bytes()).hexdigest() == h + for p, h in source_hashes.items() + ) + result = { + "exit_code": proc.returncode, + "stop_reason": reason, + "source_unchanged": unchanged, + "pid": proc.pid, + "elapsed_seconds_operational_only": time.monotonic() - started, + "peak_process_group_rss_bytes_operational_only": peak, + } + (args.output / "phase-result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + return int(proc.returncode != 0 or reason is not None or not unchanged) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=600) + parser.add_argument("--memory-gib", type=float, default=8) + parser.add_argument("--trace-mib", type=float, default=256) + parser.add_argument("command", nargs=argparse.REMAINDER) + options = parser.parse_args() + if not options.command: + parser.error("A command is required") + raise SystemExit(run(options)) diff --git a/tools/fresh_validation/run_study.py b/tools/fresh_validation/run_study.py new file mode 100644 index 00000000..e6c1c7d8 --- /dev/null +++ b/tools/fresh_validation/run_study.py @@ -0,0 +1,186 @@ +"""Collect reference/control/holdout traces and check frozen same-workload rules. + +Every training, inference, and checking phase starts a separate bounded process. +Only the reference is used to infer rules. This is a correctness/coverage study; +the supervisor's time and memory observations are operational limits only. +""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--benchmark-root", type=Path, required=True) + parser.add_argument("--data-root", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--workload", choices=("mnist", "transformer"), required=True) + args = parser.parse_args() + if args.workload == "mnist" and args.data_root is None: + parser.error("MNIST requires an existing real IDX data cache") + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=False) + harness = Path(__file__).resolve().parent + hashes = { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in harness.glob("*.py") + } + (args.output / "study-plan.json").write_text( + json.dumps( + { + "workload": args.workload, + "reference_steps": 10, + "holdout_steps": 20, + "reference_seed": 42, + "holdout_seed": 43, + "data_steps": 20, + "reference_inputs_overlap_first_half_of_holdout": True, + "disabled_relations": ["FunctionLeadRelation", "FunctionCoverRelation"], + "harness_sha256": hashes, + }, + indent=2, + ) + + "\n" + ) + + def phase(name, command): + subprocess.run( + [ + str(args.python), + str(harness / "run_phase.py"), + "--source", + str(args.source.resolve()), + "--output", + str(args.output / name), + "--", + str(args.python), + *map(str, command), + ], + check=True, + ) + + for role, steps, seed, mode in ( + ("reference", 10, 42, "instrumented"), + ("control", 10, 42, "instrumented"), + ("holdout", 20, 43, "instrumented"), + ("baseline-holdout", 20, 43, "baseline"), + ): + command = [ + harness / "benchmark_workload.py", + "--benchmark-root", + args.benchmark_root.resolve(), + "--workload", + args.workload, + "--steps", + steps, + "--data-steps", + 20, + "--seed", + seed, + "--batch-size", + 2, + "--device", + "cpu", + "--mode", + mode, + "--output", + args.output / role, + ] + if args.data_root is not None: + command.extend(["--data-root", args.data_root.resolve()]) + phase(role, command) + invariants = args.output / "inference/invariants.jsonl" + traces = sorted((args.output / "reference/traces").glob("trace_API*.log")) + if not traces: + raise RuntimeError("No reference API traces") + phase( + "inference", + [ + "-m", + "traincheck.infer_engine", + "-t", + *traces, + "-o", + invariants, + "--backend", + "pandas", + "-j", + 1, + "--disable-relation", + "FunctionLeadRelation", + "FunctionCoverRelation", + ], + ) + for role in ("reference", "control", "holdout"): + traces = sorted((args.output / role / "traces").glob("trace_API*.log")) + phase( + "check-" + role, + [ + harness / "check_coverage.py", + "--traces", + *traces, + "--invariants", + invariants, + "--output", + args.output / ("check-" + role) / "coverage.json", + ], + ) + traces = sorted((args.output / "holdout/traces").glob("trace_API*.log")) + phase( + "check-holdout-direct", + [ + harness / "check_coverage.py", + "--traces", + *traces, + "--invariants", + invariants, + "--observer-off", + "--output", + args.output / "check-holdout-direct/coverage.json", + ], + ) + + phase( + "prepare-fault", + [ + harness / "fault_control.py", + "--traces", + *traces, + "--invariants", + invariants, + "--healthy-coverage", + args.output / "check-holdout/coverage.json", + "--output", + args.output / "fault", + ], + ) + faulty_traces = sorted((args.output / "fault/traces").glob("trace_API*.log")) + phase( + "check-fault", + [ + harness / "check_coverage.py", + "--traces", + *faulty_traces, + "--invariants", + invariants, + "--output", + args.output / "check-fault/coverage.json", + ], + ) + subprocess.run( + [ + str(args.python), + str(harness / "compare_study.py"), + "--root", + str(args.output), + ], + check=True, + ) + + +if __name__ == "__main__": + main() From e673266115b4e809a2392f2edaaf0dbd96ae1fe6 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:01:26 +0800 Subject: [PATCH 16/36] Document dynamic type-name validation on matching Linux runtime --- .../dynamic-type-names-results.json | 27 +++++++++++++++++++ docs/experiments/dynamic-type-names.md | 15 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json index b77bb34a..b785b380 100644 --- a/docs/experiments/dynamic-type-names-results.json +++ b/docs/experiments/dynamic-type-names-results.json @@ -21,5 +21,32 @@ "errors": 0, "passed": 7, "failed_tests": [] + }, + "linux_followup": { + "fix_commit": "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c", + "utils_sha256": "854c11519e7aa6d0cbbfd79694d2ad4895fb1aaf72e9efbb0068048598b70750", + "runtime": { + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "torch": "2.16.0.dev20261012+cpu" + }, + "ops_wrapper_names": { + "instrumentation": "torch._inductor.virtualized.OpsWrapper", + "runtime": "torch._inductor.virtualized.OpsWrapper" + }, + "tests": { + "v3-integration-tests": { + "tests_passed": 29, + "exit_code": 0, + "source_unchanged": true, + "stdout_sha256": "be7f9c929651de7f09f383a8f3b0176ec8d3285f0ad053f4d49e6e837a40309b" + }, + "v3-coverage-tests": { + "tests_passed": 11, + "exit_code": 0, + "source_unchanged": true, + "stdout_sha256": "a33575d7feb6117b55c84c6919d21911e1f76065db033ebbdd9270671c536239" + } + }, + "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results." } } diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md index 3b93cd47..cfe34ca9 100644 --- a/docs/experiments/dynamic-type-names.md +++ b/docs/experiments/dynamic-type-names.md @@ -41,3 +41,18 @@ The motivating Linux training run used an existing broader temporary workaround. Its successful collection is not evidence of an end-to-end run of this narrower production change; the direct real-object regression test is the evidence here. This is a compatibility fix, with no performance improvement claim. + +## Matching Linux runtime follow-up + +The exact signed utility change was also tested on Python 3.11.4 and PyTorch +2.16.0.dev20261012+cpu. All seven type-name regressions passed within a **29-test +integration suite** (also covering trace integrity, TorchScript, and precondition +state isolation). A separate 11-test checker-observer suite passed. The real +`OpsWrapper` returned `torch._inductor.virtualized.OpsWrapper` in both modes. + +The disposable integration source combined the signed tracing, TorchScript, +cache, sampling, state-isolation, and type-name changes. Its `utils.py` matched +commit `2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c` exactly, and all source hashes +were unchanged after testing. Compact evidence includes runtime versions and +log/source hashes. These preflight results do not establish training overhead +or cross-workload invariant generalization. From 927707d319e36337979a287e9d184073547cf309 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:01:49 +0800 Subject: [PATCH 17/36] Document audited time-window inference measurements and tradeoffs --- .../inference-time-range-index-20261025.json | 872 ++++++++++++++++++ .../inference-time-range-index.md | 157 ++++ 2 files changed, 1029 insertions(+) create mode 100644 docs/measurements/inference-time-range-index-20261025.json create mode 100644 docs/measurements/inference-time-range-index.md diff --git a/docs/measurements/inference-time-range-index-20261025.json b/docs/measurements/inference-time-range-index-20261025.json new file mode 100644 index 00000000..2ae789d6 --- /dev/null +++ b/docs/measurements/inference-time-range-index-20261025.json @@ -0,0 +1,872 @@ +{ + "scope": "Three alternating sequential unprofiled CPU saved-trace inference pairs; both variants include the common pruning-state correction.", + "date_utc": "2026-10-25", + "started_utc": "2026-10-25T10:44:10.532938+00:00", + "ended_utc": "2026-10-25T10:53:12.729963+00:00", + "source": { + "base_branch_commit": "99d16905296d83495570fdcafa91370b242d2528", + "candidate_production_commit": "24e7cb0f7a573ff6225a388eff189d9e4af5e8b5", + "candidate_benchmark_commit": "39152aded8b97ed83988e3292bb89c6e3adcf750", + "common_state_fix_commit": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "only_production_difference": [ + "traincheck/trace/trace_pandas.py" + ], + "production_sha256": { + "baseline": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "c78bda77cfe83be10b6b385b55563fa6ca72ecc116f8bcc1557544d2e5d2df98", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/tracer.py": "5476b155823b4287f9cb01e498f70b99921063a052512a9f0d5d5b76974850f4", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "c36764ec7b9d7224c82071abf2f83d5ee9952e6868774dff62d851da9d0270e1" + }, + "candidate": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "c78bda77cfe83be10b6b385b55563fa6ca72ecc116f8bcc1557544d2e5d2df98", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/tracer.py": "5476b155823b4287f9cb01e498f70b99921063a052512a9f0d5d5b76974850f4", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "c36764ec7b9d7224c82071abf2f83d5ee9952e6868774dff62d851da9d0270e1" + } + }, + "all_source_files_unchanged_after_all_runs": true + }, + "runtime": { + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "packages": { + "numpy": "2.1.0", + "pandas": "2.3.3", + "torch": "2.16.0.dev20261012+cpu" + }, + "torch_git_revision": "8f9045d72f71e7aa9e16b6d7a055ec5bc1f720b5", + "original_training_runtime_difference": "Original training used Python 3.12.3 and CUDA 13.0; inference uses Python 3.11.4 and CPU torch from the same source revision." + }, + "workload": { + "saved_model_parameters": 6163712, + "training_steps": 10, + "sequence_length": 256, + "batch_size": 1, + "seed": 42, + "trace_events": 573494, + "raw_bytes": 286335289, + "traces": { + "trace_API_1088885_23438542501568.log": { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + "trace_API_1088885_23456244348224.log": { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + }, + "all_traces_unchanged_after_all_runs": true, + "enabled_relation_count": 6, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "eligible_api_count": 41, + "api_coverage_limit": "Original name set was not recorded; matching count alone does not establish exact original-environment API coverage." + }, + "settings": { + "backend": "pandas", + "workers": 1, + "native_threads": 1, + "CUDA_VISIBLE_DEVICES": "", + "PYTHONHASHSEED": "0", + "timeout_seconds": 1800, + "sampled_child_session_pss_budget_gib": 12, + "memory_sample_interval_seconds": 1, + "cprofile": false, + "periodic_traceback_dumps": false, + "fatal_faulthandler": true, + "fresh_process_and_trace_per_run": true, + "lazy_index_build_included": true, + "cpu_affinity": null, + "exclusive_host": false, + "run_order": [ + [ + "baseline", + "candidate" + ], + [ + "candidate", + "baseline" + ], + [ + "baseline", + "candidate" + ] + ] + }, + "runs": [ + { + "pair": 1, + "order": 1, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2327513088, + "rss_bytes": 2335973376, + "uss_bytes": 2324221952 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 75.47, + "precondition_and_output_sort_seconds": 12.565 + }, + "markers": { + "collection_start": "2026-10-25T06:45:38.025000", + "generation_start": "2026-10-25T06:44:22.555000", + "inference_end": "2026-10-25T06:45:50.590000", + "precondition_start": "2026-10-25T06:45:38.025000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 88.07923460006714, + "load_seconds": 10.64183259010315 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 102.65519127622247 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 1.29, + 1.24, + 1.16 + ], + "host_cpu_percent_sample": 4.6, + "available_memory_bytes": 52341108736 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 1.93, + 1.5, + 1.27 + ], + "host_cpu_percent_sample": 3.5, + "available_memory_bytes": 52397568000 + } + } + }, + { + "pair": 1, + "order": 2, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328130560, + "rss_bytes": 2336595968, + "uss_bytes": 2324832256 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 49.225, + "precondition_and_output_sort_seconds": 12.592 + }, + "markers": { + "collection_start": "2026-10-25T06:46:55.427000", + "generation_start": "2026-10-25T06:46:06.202000", + "inference_end": "2026-10-25T06:47:08.019000", + "precondition_start": "2026-10-25T06:46:55.427000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.86267161369324, + "load_seconds": 10.646003484725952 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.7478895187378 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 1.93, + 1.5, + 1.27 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52395216896 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.14, + 1.64, + 1.34 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52372901888 + } + } + }, + { + "pair": 2, + "order": 1, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328266752, + "rss_bytes": 2336739328, + "uss_bytes": 2324983808 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 49.337, + "precondition_and_output_sort_seconds": 12.469 + }, + "markers": { + "collection_start": "2026-10-25T06:48:13.227000", + "generation_start": "2026-10-25T06:47:23.890000", + "inference_end": "2026-10-25T06:48:25.697000", + "precondition_start": "2026-10-25T06:48:13.228000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.853256940841675, + "load_seconds": 10.614852905273438 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.78805723041296 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.14, + 1.64, + 1.34 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52375351296 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.0, + 1.73, + 1.4 + ], + "host_cpu_percent_sample": 3.7, + "available_memory_bytes": 52377493504 + } + } + }, + { + "pair": 2, + "order": 2, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286166016, + "rss_bytes": 2294603776, + "uss_bytes": 2282872832 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 73.854, + "precondition_and_output_sort_seconds": 12.484 + }, + "markers": { + "collection_start": "2026-10-25T06:49:55.626000", + "generation_start": "2026-10-25T06:48:41.772000", + "inference_end": "2026-10-25T06:50:08.110000", + "precondition_start": "2026-10-25T06:49:55.626000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 86.3845362663269, + "load_seconds": 10.706615447998047 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 101.79567445069551 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.0, + 1.73, + 1.4 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52377260032 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.19, + 1.88, + 1.49 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52357947392 + } + } + }, + { + "pair": 3, + "order": 1, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286102528, + "rss_bytes": 2294554624, + "uss_bytes": 2282823680 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 73.811, + "precondition_and_output_sort_seconds": 12.432 + }, + "markers": { + "collection_start": "2026-10-25T06:51:38.197000", + "generation_start": "2026-10-25T06:50:24.386000", + "inference_end": "2026-10-25T06:51:50.629000", + "precondition_start": "2026-10-25T06:51:38.197000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 86.28747224807739, + "load_seconds": 10.534409046173096 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 101.61026244610548 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.19, + 1.88, + 1.49 + ], + "host_cpu_percent_sample": 3.4, + "available_memory_bytes": 52357136384 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.25, + 1.98, + 1.57 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52370046976 + } + } + }, + { + "pair": 3, + "order": 2, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328313856, + "rss_bytes": 2336751616, + "uss_bytes": 2325057536 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 49.004, + "precondition_and_output_sort_seconds": 12.445 + }, + "markers": { + "collection_start": "2026-10-25T06:52:56.197000", + "generation_start": "2026-10-25T06:52:07.193000", + "inference_end": "2026-10-25T06:53:08.643000", + "precondition_start": "2026-10-25T06:52:56.198000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.49606490135193, + "load_seconds": 10.764497995376587 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.6580608561635 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.25, + 1.98, + 1.57 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52370046976 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.26, + 2.03, + 1.62 + ], + "host_cpu_percent_sample": 3.5, + "available_memory_bytes": 52363665408 + } + } + } + ], + "medians": { + "baseline": { + "wall_seconds": 101.79567445069551, + "wall_min_seconds": 101.61026244610548, + "wall_max_seconds": 102.65519127622247, + "peak_pss_bytes": 2286166016, + "load_seconds": 10.64183259010315, + "inference_seconds": 86.3845362663269, + "generation_and_pruning_seconds": 73.854, + "precondition_and_output_sort_seconds": 12.484 + }, + "candidate": { + "wall_seconds": 76.7478895187378, + "wall_min_seconds": 76.6580608561635, + "wall_max_seconds": 76.78805723041296, + "peak_pss_bytes": 2328266752, + "load_seconds": 10.646003484725952, + "inference_seconds": 61.853256940841675, + "generation_and_pruning_seconds": 49.225, + "precondition_and_output_sort_seconds": 12.469 + } + }, + "ratio_of_median_wall_times": 1.3263644784113884, + "median_wall_reduction_percent": 24.605942312499295, + "median_peak_pss_increase_bytes": 42100736, + "equivalence": { + "all_outputs_equal_corrected_reference": true, + "comparison": "Complete structural JSON multisets; duplicate multiplicity and array order preserved, object key and record line order ignored.", + "outputs": { + "invariants": { + "records": 836, + "distinct_records": 836, + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb" + }, + "failed_hypotheses": { + "records": 104, + "distinct_records": 104, + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798" + } + }, + "all_six_output_files_also_byte_identical": true, + "raw_output_sha256": { + "invariants.jsonl": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0", + "invariants.jsonl.failed": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2" + }, + "all_full_and_stage_context_files_equal_reference": true, + "context_sha256": { + "contexts-0.json": "bea2a909d42bc621be2fea66ad7de4e7e871b7f40903f546934e3d7e4d9c1207", + "contexts-1.json": "bea2a909d42bc621be2fea66ad7de4e7e871b7f40903f546934e3d7e4d9c1207" + } + }, + "fast_path_probe": { + "rows": 573494, + "dtypes": { + "type": "object", + "time": "int64", + "process_id": "int64", + "thread_id": "int64", + "func_call_id": "object" + }, + "sampled_queries": 24, + "all_sampled_queries_hit": true, + "indexed_rows": 573494, + "identity_groups": 2, + "limit": "Sampled real-call queries in a separate probe, not a measured hit count for every inference query." + }, + "validation": { + "local_combined_tests": 61, + "matching_runtime_combined_tests": 67, + "matching_runtime_tests_note": "Includes six common pruning-state tests. First packaging attempt lacked tests.fixtures; completed test-support-only rerun passed without production changes.", + "matching_runtime_test_log_sha256": "84cfeba80e7c2032b2e7a28baebc4c7bd2fc2e778f27bf6eb16362b4f857b5d3", + "independent_differential_audit": { + "parent_commit": "99d16905296d83495570fdcafa91370b242d2528", + "parent_source_sha256": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "candidate_source_sha256": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "pandas": "2.3.3", + "numpy": "2.1.0", + "cases": 144, + "comparisons": 6480, + "differences": [] + }, + "independent_evidence_audit": { + "passed": true, + "archive_files_verified": 295, + "all_outputs_and_contexts_equal_corrected_reference": true, + "only_signed_production_delta_confirmed": true, + "raw_trace_hash_verification_scope": "Before/after hashes computed remotely and reviewed; raw input is not in evidence archive and was not rehashed locally." + } + }, + "synthetic_benchmark": { + "scope": "synthetic function-call window queries, including construction and lazy build; not full inference or peak memory", + "settings": { + "rows": 200000, + "queries": 1000, + "repetitions": 3 + }, + "runtime": { + "python": "3.11.4", + "platform": "Linux-6.1.0-1027-oem-x86_64-with-glibc2.35", + "pandas": "2.3.3", + "numpy": "2.1.0", + "native_threads": { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1" + } + }, + "source_sha256": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "input_sha256": "4d2a33e4e4c692eaa5339751b3db19fbb8a2335cb1c9ebd26ba073fbbf407c1f", + "input_unchanged": true, + "all_ordered_results_equal": true, + "measurements": [ + { + "repetition": 1, + "order": 1, + "variant": "legacy", + "construction_seconds": 0.04303494095802307, + "first_query_seconds": 0.006073214113712311, + "remaining_queries_seconds": 2.613212414085865, + "total_seconds": 2.6623205691576004, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 1, + "order": 2, + "variant": "candidate", + "construction_seconds": 0.0409860722720623, + "first_query_seconds": 0.07652264088392258, + "remaining_queries_seconds": 0.21955957636237144, + "total_seconds": 0.3370682895183563, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + }, + { + "repetition": 2, + "order": 1, + "variant": "candidate", + "construction_seconds": 0.044426146894693375, + "first_query_seconds": 0.07581401616334915, + "remaining_queries_seconds": 0.21997691318392754, + "total_seconds": 0.34021707624197006, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + }, + { + "repetition": 2, + "order": 2, + "variant": "legacy", + "construction_seconds": 0.04476165026426315, + "first_query_seconds": 0.003517802804708481, + "remaining_queries_seconds": 2.71007227152586, + "total_seconds": 2.7583517245948315, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 3, + "order": 1, + "variant": "legacy", + "construction_seconds": 0.04182035103440285, + "first_query_seconds": 0.0035678334534168243, + "remaining_queries_seconds": 2.596189245581627, + "total_seconds": 2.6415774300694466, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 3, + "order": 2, + "variant": "candidate", + "construction_seconds": 0.04158874601125717, + "first_query_seconds": 0.0771775059401989, + "remaining_queries_seconds": 0.22500132396817207, + "total_seconds": 0.34376757591962814, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + } + ], + "medians": { + "legacy": { + "construction_seconds": 0.04303494095802307, + "first_query_seconds": 0.0035678334534168243, + "remaining_queries_seconds": 2.613212414085865, + "total_seconds": 2.6623205691576004 + }, + "candidate": { + "construction_seconds": 0.04158874601125717, + "first_query_seconds": 0.07652264088392258, + "remaining_queries_seconds": 0.21997691318392754, + "total_seconds": 0.34021707624197006 + } + }, + "median_total_ratio": 7.825358440456698 + }, + "provenance": { + "evidence_archive_name": "timing-evidence.tar.gz", + "evidence_archive_bytes": 1705821, + "evidence_archive_sha256": "7f45cee3ca5bf8f4b8b1fa2f29ee54234fafd1fe81b57fd0badb7cb38fa81c36", + "evidence_archive_access": "Retained study artifact; raw trace/output data are not published in this repository.", + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "helper_sha256": "1fc8409ec68d23da26223424a41dcaeff252ede84723b4256fd7dc5797f8e994", + "controller_sha256": "416db6554b02299ccc44ef73ed4eba7735bfe925a0e68a708701ae0f6baf1271", + "paired_runs_json_sha256": "5304104c25281c369b00ff217fb2989a0be712dc9f4cfe08ac4d54c2b3af9b27", + "snapshot_manifest_sha256": "2f85ca6e20061565b6b6cf02ad2760f3c8abb70978ce76eb5d1ed69cd032d750", + "final_integrity_validation_sha256": "0bf446f2fe7387e4fdcc9e30f0c4e9012162251b340158e844d8e0712f1a2913", + "independent_audit_json_sha256": "6ca49fd92ede07e153c81c373d3242fa49c379204e3dc2e8fc3d1497057cfdd8", + "fast_path_probe_json_sha256": "dbc0be5aacdab32979a46e21a11c55fa6f4bcdf49a25c4d369113925a22006c7", + "process_check_utc": "2026-10-25T10:54:52.654222+00:00", + "remaining_study_processes": [] + }, + "limits": [ + "Three pairs on one saved trace and six enabled relations, with unrelated background load and no exclusive host or CPU affinity.", + "Measures offline inference, not training overhead or distributed scalability; output equality is not independent invariant ground truth.", + "Wall time includes child startup/imports/output writes and monitor polling, excludes integrity hashing; load includes harness context serialization.", + "Reported inference ends before invariant JSON writes; marker durations are subphases, not additive extra time.", + "One-second sampled concurrent child-session PSS excludes the monitor and depends on shared pages; the 12-GiB budget is a monitor threshold, not an OS hard limit.", + "The per-frame index stores O(N) values/positions and adds first-query build cost; same-size in-place edits to cached fields or row order require a new TracePandas.", + "This incremental speedup must not be multiplied by the previous single-pair identity-cache measurement as an independently measured combined speedup." + ] +} diff --git a/docs/measurements/inference-time-range-index.md b/docs/measurements/inference-time-range-index.md new file mode 100644 index 00000000..5212bdb8 --- /dev/null +++ b/docs/measurements/inference-time-range-index.md @@ -0,0 +1,157 @@ +# Index function-call time windows + +Repeated calls to `query_func_call_events_within_time` previously filtered every +trace row by type, time, process, and thread before grouping the selected calls. +The new lazy index uses binary search within each process/thread pair, restores +the selected rows to their original order, and retains the existing pandas +`groupby`, record construction, and assertions. A conservative fast path handles +ordinary int64 identities with int64 or float64 time; all unsupported dtypes, +query scalar types, and comparison cases use the original pandas mask. + +## Complete saved-trace inference + +Three alternating sequential pairs on labmach reduced median total wall time from +**101.796 s to 76.748 s (1.326x, 24.61% less time)**. Both variants include the +identity cache and the independent pruning-state correction from PR #145. The +only production difference between the measured snapshots is the time-window +index in `trace_pandas.py`. + +| Pair | Execution order | Baseline wall | Candidate wall | Baseline peak PSS | Candidate peak PSS | +|---|---|---:|---:|---:|---:| +| 1 | Baseline, candidate | 102.655 s | 76.748 s | 2.167665 GiB | 2.168241 GiB | +| 2 | Candidate, baseline | 101.796 s | 76.788 s | 2.129158 GiB | 2.168367 GiB | +| 3 | Baseline, candidate | 101.610 s | 76.658 s | 2.129099 GiB | 2.168411 GiB | + +All six runs completed and produced the same **836 invariant records and 104 +saved failed-hypothesis records** as the corrected reference. Complete structural +JSON multisets match, preserving duplicate multiplicity and array order. The raw +output files also match byte for byte across the six runs. Full/stage context +files match the corrected reference byte for byte, and all source/input hashes +remained unchanged. Independent review verified all 295 archived files, source +snapshots, complete outputs, contexts, and calculations. Trace integrity checks +ran before and after inference on the remote host; the raw 286-MB input is not +in the downloaded archive and was not independently rehashed locally. This is +an incremental measurement over the corrected cached +baseline; it should not be multiplied by the earlier single-pair identity-cache +speedup as an independently measured combined result. + +| Median measurement | Baseline | Time-window index | +|---|---:|---:| +| Total child wall time | 101.796 s | 76.748 s | +| Trace loading/construction | 10.642 s | 10.646 s | +| Reported inference | 86.385 s | 61.853 s | +| Generation and pruning (part of inference) | 73.854 s | 49.225 s | +| Final preconditions and sort (part of inference) | 12.484 s | 12.469 s | +| Peak sampled child-session PSS | 2.129 GiB | 2.168 GiB | + +The median sampled memory peak increased by **40.15 MiB**. The first baseline +peak was higher than the other two and close to its paired candidate peak; +per-run values above retain that variation. Each trial uses a fresh process and +trace, including the first lazy index construction. The index stores +O(N) timestamp/position values and requires temporary build storage. It is a +runtime/memory tradeoff, and its first-query cost matters for short-lived traces. + +The input is both complete trace files from the saved 6,163,712-parameter, +10-step reference run: 573,494 events and 286,335,289 raw bytes, sequence length +256, batch size 1, seed 42. No training was rerun. A separate probe confirmed that +all 24 sampled real-call queries hit the new fast path, with int64 time/PID/TID +columns and built-in integer bounds/identities. This is sampled eligibility +validation, not a hit count for every query in the timed inference runs. + +Both variants use pandas, one worker, native thread settings of one, no GPU, +`PYTHONHASHSEED=0`, and six enabled relations; `FunctionLeadRelation` and +`FunctionCoverRelation` remain disabled. Every run has a 1,800-second cutoff and +12-GiB sampled child-session PSS budget. The runtime is Python 3.11.4, +Torch 2.16.0.dev20261012+cpu, pandas 2.3.3, and NumPy 2.1.0. Torch's source revision +matches the original CUDA wheel, but training used Python 3.12.3 and CUDA 13.0. +The eligible API count is 41; the original name set was not retained, so matching +the count does not establish exact original-environment API coverage. + +The host was shared, with no CPU affinity or exclusive reservation. Other owned +experiment jobs were paused during the six timed runs; unrelated background +work was left untouched. Before-run one-minute load averages ranged from 1.29 +to 2.25, and sampled whole-host CPU utilization from 3.3% to 4.6%. These snapshots +do not bound interference during each run. The result covers one saved workload +and relation set, not training overhead, distributed scaling, or invariant +ground-truth correctness. + +Total wall time includes child startup, imports, output writes, and monitor +polling (about one-second resolution), but excludes integrity hashing. The load +timer includes harness context serialization. The inference timer ends before +invariant JSON writes. Its two phases in the table are log-marker intervals, +not additional time. Memory is sampled once per second for the concurrent child +session, excludes the monitor, and depends on shared pages. The memory budget is +a monitor threshold, not an OS hard limit. cProfile and periodic traceback dumps +were disabled; fatal-error traceback handling remained enabled. + +[Compact measurements and provenance](inference-time-range-index-20261025.json) +retain every run's timing, memory peak, background-load snapshots, source/input +hashes, output digests, and archive identity. Raw traces and output records are +retained as study artifacts rather than published in this repository. + +## Correctness and cache contract + +The 18 new differential tests compare the indexed path with the unchanged pandas +fallback. Together with context/call/identity tests, 61 tests pass locally. The +matching Linux runtime passes those tests plus six common pruning-state tests +(67 total). An independent oracle extracted from the exact parent method checks +144 dtype/offset cases and 6,480 queries with no differences in each of two +pandas/NumPy environments: 2.2.3/1.26.4 and 2.3.3/2.1.0. + +The tests cover strict boundaries, tied/reversed bounds, signed integer extremes, +mixed integer/float precision above 2**53, NaNs/infinities/signed zero, malformed +bounds and scalar subclasses, null/extension/categorical dtypes, empty frames, +unsorted rows and duplicate labels, categorical/null call IDs, incomplete or +exception records, fresh returned objects, and cache invalidation. Empty +selections still run the original grouping logic, including its assertions. + +Time, event type, process/thread identity values and dtypes, and row order must +remain stable within a cached frame. Replacement or length changes invalidate +the cache; same-size in-place changes require a new `TracePandas`. The cache holds +a source-frame reference rather than copying the DataFrame, but can retain an old +frame until invalidation. Existing pandas group ordering, category/dropna +behavior, within-group row order, assertion messages, and fresh record/event +objects remain unchanged. + +## Reproduction + +The portable synthetic benchmark needs no saved trace: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m tools.benchmarks.time_range_index --rows 200000 --queries 1000 --repetitions 3 + +python -m unittest tests.test_context_index tests.test_call_index \ + tests.test_identity_cache tests.test_time_range_index +``` + +On the matching Linux runtime, its median construction-plus-query time fell from +**2.662321 s to 0.340217 s (7.825x)**. Ordered outputs match in all six trials, and +the input remains unchanged. Candidate first-query median was **0.076523 s** +versus **0.003568 s** for legacy filtering, exposing the lazy build cost. Input +generation, garbage collection, and result hashing are excluded; fresh record +construction is included. This synthetic result is separate from full inference. + +For the saved-trace measurement, reconstruct the baseline from commit +`99d16905296d83495570fdcafa91370b242d2528` plus only the pruning-state correction +hunks from `1b5aedffbeefd4cdd9d30036dbb0a833521526cf`. The candidate adds the production +change at `24e7cb0f7a573ff6225a388eff189d9e4af5e8b5`; the benchmark was checkpointed +at `39152aded8b97ed83988e3292bb89c6e3adcf750`. Match the recorded per-file hashes. +With the retained `inference_replay_fatal_only.py` and `inference_scaling.py` +harness files, invoke each fresh variant as follows, using the two trace files +identified by digest in the evidence JSON: + +```sh +"$INFERENCE_PYTHON" "$STUDY/inference_replay_fatal_only.py" \ + --mode infer --source "$SOURCE_SNAPSHOT" --traces "$TRACE_A" "$TRACE_B" \ + --output "$NEW_OUTPUT_DIRECTORY" --timeout 1800 --memory-limit-gib 12 +``` + +Run baseline/candidate, candidate/baseline, baseline/candidate sequentially, with +no `--profile` flag. The harness fixes workers/native threads/hash seed and CPU +mode, verifies source/input hashes, and saves phase/memory/context evidence. The +controller compares complete outputs. `run_pairs.py`, exact invocations, both +immutable source snapshots, full outputs, and the verified harness/helper hashes are +retained in the evidence archive. Reproducing this full-trace result requires +access to the saved trace and retained harness; the synthetic benchmark and +regression tests are self-contained in the checkout. From cad62cc98ef5fb7b313325443caa33f715ed141e Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:05:16 +0800 Subject: [PATCH 18/36] Validate a live dtype rule when no shape control is available --- .../fresh-run-invariant-validation.md | 9 +- tests/test_fresh_validation_fault.py | 81 +++++++++++++ tools/fresh_validation/compare_study.py | 8 ++ tools/fresh_validation/fault_control.py | 111 ++++++++++-------- 4 files changed, 158 insertions(+), 51 deletions(-) create mode 100644 tests/test_fresh_validation_fault.py diff --git a/docs/measurements/fresh-run-invariant-validation.md b/docs/measurements/fresh-run-invariant-validation.md index b7eda530..ff4e6314 100644 --- a/docs/measurements/fresh-run-invariant-validation.md +++ b/docs/measurements/fresh-run-invariant-validation.md @@ -55,11 +55,16 @@ that an ID caused dormancy. A workload with no ID-bearing rules leaves the original TorchTitan rule set's portability unresolved. A post-hoc sensitivity control selects a healthy, actually evaluated output-shape -rule and changes one recorded tensor's first shape dimension, retaining its type, +rule, or an output-dtype rule when no eligible shape rule exists. It changes one +recorded shape dimension or float32/float64 dtype value, retaining its JSON type, IDs and precondition inputs. Every other semantic field and every other raw line must remain identical. The frozen checker must activate and violate the selected rule. This is trace-level detector-path validation, not a reproduced training bug -or a general defect-coverage result. +or a general defect-coverage result. Dtype mutation changes recorded metadata only; +the tensor type-name key remains unchanged, so it does not represent a complete +alternative tensor execution. The original 74b3adf MNIST run completed its healthy +phases but stopped at the shape-only selector; that diagnostic remains separate +from fresh runs with the revised selector. `compare_study.py` verifies source/input immutability, fresh producer PIDs, exact same-seed control losses and canonical parameter hashes, instrumented-versus-plain diff --git a/tests/test_fresh_validation_fault.py b/tests/test_fresh_validation_fault.py new file mode 100644 index 00000000..a308506d --- /dev/null +++ b/tests/test_fresh_validation_fault.py @@ -0,0 +1,81 @@ +"""The fallback must activate the real frozen checker, without other edits.""" + +import json +import tempfile +import unittest +from pathlib import Path + +import pandas as pd + +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import observe +from tools.fresh_validation.fault_control import differences, digest, mutate +from traincheck.trace.trace_pandas import TracePandas + + +class FaultControlTests(unittest.TestCase): + def run_mutation(self, attribute, shape): + rule = output_rule() + if attribute == "shape": + rule.params[1].attr_name = "shape" + rule.params[1].const_value = tuple(shape) + good = {"torch.FloatTensor": {"shape": shape, "dtype": "torch.float32"}} + traced = frame([good, good]) + healthy = observe(traced, rule) + healthy["invariant_index"] = 0 + before = traced.events.to_dict("records") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace, invariants, coverage = ( + root / x for x in ("trace.log", "rules.jsonl", "coverage.json") + ) + trace.write_text("".join(json.dumps(row) + "\n" for row in before)) + invariants.write_text(json.dumps(rule.to_dict()) + "\n") + coverage.write_text( + json.dumps( + { + "inputs_unchanged": True, + "observer_enabled": True, + "input_sha256": { + str(p): digest(p) for p in (trace, invariants) + }, + "per_invariant": [healthy], + } + ) + ) + manifest = mutate([trace], invariants, coverage, root / "fault") + after = [ + json.loads(line) + for line in (root / "fault/traces/trace.log").read_text().splitlines() + ] + self.assertEqual(len(differences(before, after)), 1) + self.assertEqual(before[0], after[0]) + self.assertEqual(before[2:], after[2:]) + self.assertEqual(manifest["attribute"], attribute) + self.assertEqual( + manifest["edits"][0]["path"][-1], 0 if attribute == "shape" else "dtype" + ) + checked = observe( + TracePandas(pd.DataFrame(after), truncate_incomplete_func_calls=False), + rule, + ) + self.assertTrue(healthy["check_passed"]) + self.assertEqual(checked["status"], "completed") + self.assertFalse(checked["check_passed"]) + self.assertEqual(checked["precondition_group_passes"], 1) + self.assertEqual(checked["decisions_completed"], 1) + self.assertEqual( + checked["decision_records"][0]["func_call_id"], manifest["func_call_id"] + ) + self.assertTrue( + any( + row["func_call_id"] == manifest["func_call_id"] + for row in checked["checker_result"]["trace"] + ) + ) + + def test_no_shape_rule_uses_live_dtype_and_detects_exact_edited_call(self): + self.run_mutation("dtype", []) + + def test_mutable_shape_keeps_original_one_dimension_control(self): + self.run_mutation("shape", [2, 3]) diff --git a/tools/fresh_validation/compare_study.py b/tools/fresh_validation/compare_study.py index fae6b6de..b79fccba 100644 --- a/tools/fresh_validation/compare_study.py +++ b/tools/fresh_validation/compare_study.py @@ -123,6 +123,14 @@ def losses(metric): assert healthy["check_passed"] and healthy["active"] assert faulty["status"] == "completed" and not faulty["check_passed"] assert faulty["decisions_completed"] > 0 and faulty["precondition_group_passes"] > 0 + assert any( + d.get("func_call_id") == mutation["func_call_id"] + for d in faulty["decision_records"] + ) + assert any( + event.get("func_call_id") == mutation["func_call_id"] + for event in faulty["checker_result"]["trace"] + ) assert ( mutation["ids_unchanged"] and mutation["precondition_inputs_unchanged"] diff --git a/tools/fresh_validation/fault_control.py b/tools/fresh_validation/fault_control.py index 0da4fe66..f86dab36 100644 --- a/tools/fresh_validation/fault_control.py +++ b/tools/fresh_validation/fault_control.py @@ -1,6 +1,6 @@ -"""Post-hoc, one-field trace sensitivity control for a live frozen shape invariant. +"""Post-hoc, one-field trace sensitivity control for a live frozen output invariant. -This does not inject a training bug. It changes one recorded tensor shape after +This does not inject a training bug. It changes one recorded tensor shape or dtype after healthy training, preserves IDs and precondition inputs, and checks the same rules. """ @@ -37,6 +37,24 @@ def differences(before, after, path=()): return [path] +def replacement(attributes, attribute): + """Return one valid same-type unequal metadata value, or no eligible edit.""" + if not isinstance(attributes, dict): + return None + value = attributes.get(attribute) + if attribute == "shape": + if ( + isinstance(value, list) + and value + and isinstance(value[0], int) + and not isinstance(value[0], bool) + ): + return [value[0] + 1, *value[1:]] + elif attribute == "dtype" and value in ("torch.float32", "torch.float64"): + return "torch.float64" if value == "torch.float32" else "torch.float32" + return None + + def mutate(traces, invariants, healthy, output): rules = [json.loads(line) for line in invariants.read_text().splitlines()] coverage = json.loads(healthy.read_text()) @@ -51,41 +69,40 @@ def mutate(traces, invariants, healthy, output): event = json.loads(line) if event.get("type") == "function_call (post)": post_events.setdefault(event["func_call_id"], []).append(event) - for item in coverage["per_invariant"]: - rule = rules[item["invariant_index"]] - if not ( - rule["relation"] == "ConsistentOutputRelation" - and rule["params"][1]["attr_name"] == "shape" - and item["active"] - and item["check_passed"] - and item["decisions_completed"] > 0 - ): - continue - for decision in item["decision_records"]: - events = post_events.get(decision["func_call_id"], []) - if len(events) != 1: + # Prefer shape; dtype is a prospective fallback when no live mutable shape + # rule exists. Selection remains post hoc and does not change learned rules. + for attribute in ("shape", "dtype"): + for item in coverage["per_invariant"]: + rule = rules[item["invariant_index"]] + if not ( + rule["relation"] == "ConsistentOutputRelation" + and rule["params"][1]["attr_name"] == attribute + and item["active"] + and item["check_passed"] + and item["decisions_completed"] > 0 + ): continue - values = events[0]["return_values"] - tensors = values if isinstance(values, list) else [values] - eligible = any( - "Tensor" in name - and isinstance(attributes, dict) - and isinstance(attributes.get("shape"), list) - and attributes["shape"] - and isinstance(attributes["shape"][0], int) - and not isinstance(attributes["shape"][0], bool) - for tensor in tensors - for name, attributes in tensor.items() - ) - if eligible: - chosen, call_id = item, decision["func_call_id"] + for decision in item["decision_records"]: + events = post_events.get(decision["func_call_id"], []) + if len(events) != 1: + continue + values = events[0]["return_values"] + tensors = values if isinstance(values, list) else [values] + eligible = any( + "Tensor" in name and replacement(attributes, attribute) is not None + for tensor in tensors + if isinstance(tensor, dict) + for name, attributes in tensor.items() + ) + if eligible: + chosen, call_id = item, decision["func_call_id"] + break + if chosen is not None: break if chosen is not None: break if chosen is None: - raise ValueError( - "No healthy, actually evaluated output-shape rule with a nonempty integer shape" - ) + raise ValueError("No healthy, actually evaluated mutable shape or dtype rule") rule = rules[chosen["invariant_index"]] assert call_id is not None output.mkdir(parents=True, exist_ok=False) @@ -105,23 +122,17 @@ def mutate(traces, invariants, healthy, output): before = copy.deepcopy(event) values = event["return_values"] tensor_values = values if isinstance(values, list) else [values] - for tensor_index, value in enumerate(tensor_values): + for value in tensor_values: + if not isinstance(value, dict): + continue for typename, attributes in value.items(): - if ( - "Tensor" not in typename - or not isinstance(attributes, dict) - or "shape" not in attributes - ): + if "Tensor" not in typename: continue - old = copy.deepcopy(attributes["shape"]) - if ( - not isinstance(old, list) - or not old - or not isinstance(old[0], int) - or isinstance(old[0], bool) - ): + new_value = replacement(attributes, attribute) + if new_value is None: continue - attributes["shape"][0] += 1 + old = copy.deepcopy(attributes[attribute]) + attributes[attribute] = new_value diff = differences(before, event) assert len(diff) == 1 assert diff[0][:1] == ("return_values",) @@ -134,8 +145,9 @@ def mutate(traces, invariants, healthy, output): "file": trace.name, "zero_based_line": index, "path": list(diff[0]), - "before_shape": old, - "after_shape": attributes["shape"], + "attribute": attribute, + "before_value": old, + "after_value": attributes[attribute], "raw_process_id": event["process_id"], "raw_thread_id": event["thread_id"], } @@ -163,7 +175,8 @@ def mutate(traces, invariants, healthy, output): assert len(edits) == 1, edits manifest = { "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", - "selection": "first actually evaluated healthy shape rule with a mutable nonempty integer shape", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": attribute, "invariant_index": chosen["invariant_index"], "invariant": rule, "func_call_id": call_id, From 2333682cb2f3f48d338ff27a6d70682f0f4e4702 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:09:35 +0800 Subject: [PATCH 19/36] Record successful collection with the narrow type-name fix --- .../experiments/dynamic-type-names-results.json | 17 ++++++++++++++++- docs/experiments/dynamic-type-names.md | 17 +++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json index b785b380..ab69fc52 100644 --- a/docs/experiments/dynamic-type-names-results.json +++ b/docs/experiments/dynamic-type-names-results.json @@ -47,6 +47,21 @@ "stdout_sha256": "a33575d7feb6117b55c84c6919d21911e1f76065db033ebbdd9270671c536239" } }, - "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results." + "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results.", + "collection_smoke": { + "workload": "Bounded benchmark MNIST Net / cached real MNIST / batch2 / CPUfloat32 / seed42", + "completed_steps": 10, + "success": true, + "source_unchanged": true, + "state_shapes_unchanged": true, + "source_stack": "PR144 plus PR142/143/145/146 production fixes; immutable integration-v3, signed74b harness", + "evidence_sha256": { + "phase-result.json": "03a50887287fb8e98398735e738412edd6c356bcc21dc39ee15929d5ffcf324c", + "metrics.json": "d99a6fd45dc534964835937a8a623ad98309052a3771e9750c3f8dfb0bfe0713", + "adaptation.json": "8a533afe8ea099fbb255124ba589f2edf068f1c17d5d3774d73c0ff88a63fa8e", + "invocation.json": "8ecea3da01502bff67024e8aa303f3d7e6ab72e7a7d1191b3aef53d61868fc75" + }, + "scope": "Collection completed with exact narrow PR146 utility in composed stack. Separate later shape-only fault-selector failure does not affect this collection result. No fix-isolated training timing claim." + } } } diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md index cfe34ca9..2a7ab3e6 100644 --- a/docs/experiments/dynamic-type-names.md +++ b/docs/experiments/dynamic-type-names.md @@ -37,10 +37,10 @@ Reproduce the candidate suite with: python -m unittest tests.test_dynamic_typename -v ``` -The motivating Linux training run used an existing broader temporary workaround. -Its successful collection is not evidence of an end-to-end run of this narrower -production change; the direct real-object regression test is the evidence here. -This is a compatibility fix, with no performance improvement claim. +The first successful Linux training pilot used an existing broader temporary +workaround. That pilot is not evidence for this narrower production change; the +follow-up below uses the exact signed fix. This is a compatibility fix, with no +performance improvement claim. ## Matching Linux runtime follow-up @@ -56,3 +56,12 @@ commit `2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c` exactly, and all source hashes were unchanged after testing. Compact evidence includes runtime versions and log/source hashes. These preflight results do not establish training overhead or cross-workload invariant generalization. + +The composed source also completed a fresh **10-step MNIST trace collection** +with cached real MNIST, batch size 2, CPU float32, and seed 42. Its source hashes +and model-state shapes stayed unchanged. This supports collection compatibility +in the identified stack; it does not isolate training behavior to this one fix. +The study later stopped at a shape-only negative-control selector because no +shape invariant existed, after collection and healthy checks had succeeded. +That harness issue and its preserved evidence are separate from the type-name +fix. Evidence hashes for the successful collection are included in the JSON. From 27e67aafc1a526ed8e69e0a8e59ff30b8ff827cb Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:20:53 +0800 Subject: [PATCH 20/36] Document fresh-run validation and object-identity false alarms --- ...fresh-run-invariant-validation-mnist.jsonl | 204 ++ ...run-invariant-validation-transformer.jsonl | 564 +++++ .../data/fresh-run-invariant-validation.json | 1894 +++++++++++++++++ .../fresh-run-invariant-validation.md | 78 +- 4 files changed, 2732 insertions(+), 8 deletions(-) create mode 100644 docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl create mode 100644 docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl create mode 100644 docs/measurements/data/fresh-run-invariant-validation.json diff --git a/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl b/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl new file mode 100644 index 00000000..9fbcea68 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl @@ -0,0 +1,204 @@ +{"index":0,"relation":"APIContainRelation","description":"Adadelta.step() always calls Adadelta._init_group()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":1,"relation":"APIContainRelation","description":"Adadelta.step() always calls Optimizer._accelerator_graph_capture_health_check()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":2,"relation":"APIContainRelation","description":"Adadelta.step() always calls Optimizer._optimizer_step_code()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":3,"relation":"APIContainRelation","description":"Adadelta.step() always calls TorchBindOpOverload.__call__()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":4,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.is_complex()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":5,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.zeros()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":6,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.zeros_like()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":7,"relation":"APIContainRelation","description":"Adadelta.step() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":8,"relation":"APIContainRelation","description":"Adadelta.step() always calls _foreach_utils._get_fused_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":9,"relation":"APIContainRelation","description":"Adadelta.step() always calls _ops._contains_fake_script_object()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":10,"relation":"APIContainRelation","description":"Adadelta.step() always calls _ops._must_dispatch_in_python()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":11,"relation":"APIContainRelation","description":"Adadelta.step() always calls accelerator.current_accelerator()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":12,"relation":"APIContainRelation","description":"Adadelta.step() always calls adadelta._single_tensor_adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":13,"relation":"APIContainRelation","description":"Adadelta.step() always calls adadelta.adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":14,"relation":"APIContainRelation","description":"Adadelta.step() always calls compiler.is_compiling()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":15,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._default_to_fused_or_foreach()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":16,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._get_scalar_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":17,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._to_scalar()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":18,"relation":"APIContainRelation","description":"Adadelta.step() always calls torch.is_grad_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":19,"relation":"APIContainRelation","description":"Conv2d.forward() always calls Conv2d._conv_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":20,"relation":"APIContainRelation","description":"Conv2d.forward() always calls _VariableFunctionsClass.conv2d()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":21,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":22,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":23,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":24,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":25,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":26,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":27,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":28,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":29,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":30,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":31,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":32,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":33,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":34,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":35,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":36,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":37,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":38,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":39,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":40,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":41,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":42,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with ndim=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":43,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":44,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":45,"relation":"DistinctArgumentRelation","description":"Conv2d.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":46,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":47,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":48,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":49,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":50,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":51,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":52,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":53,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":54,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":55,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":56,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":57,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":58,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":59,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":60,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":61,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":62,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":63,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":64,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":65,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":66,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":67,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":68,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":69,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":70,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].itemsize consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":71,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":72,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":73,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].ndim consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":74,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":75,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":76,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":77,"relation":"APIContainRelation","description":"Dropout.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":78,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":79,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":80,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":81,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":82,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":83,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":84,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":85,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":86,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":87,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":88,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":89,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":90,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":91,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":92,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":93,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":94,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":95,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":96,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":97,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":98,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":99,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":100,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":101,"relation":"DistinctArgumentRelation","description":"Dropout.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":102,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":103,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":104,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":105,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":106,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":107,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":108,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":109,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":110,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":111,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":112,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":113,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":114,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":115,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":116,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":117,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":118,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":119,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":120,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":121,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":122,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":123,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":124,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":125,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":126,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":127,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":128,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":129,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":130,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":131,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":132,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":133,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":134,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":135,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.3 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":136,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.3 consistent with output[0].shape.3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":137,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":138,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":139,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":140,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":141,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":142,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":143,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":144,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":145,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":146,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":147,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":148,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":149,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":150,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":151,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":152,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":153,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":154,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":155,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":156,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":157,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":158,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with ndim=2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":159,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":160,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":161,"relation":"DistinctArgumentRelation","description":"Linear.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":162,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":163,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":164,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":165,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":166,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":167,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":168,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":169,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":170,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":171,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":172,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":173,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":174,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":175,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":176,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":177,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":178,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":179,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":180,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":181,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":182,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":183,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":184,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":185,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":186,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":187,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":188,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":189,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":190,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":191,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":192,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":193,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls TorchBindOpOverload.__call__()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":194,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls _ops._contains_fake_script_object()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":195,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls _ops._must_dispatch_in_python()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":196,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls compiler.is_exporting()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":197,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _VariableFunctionsClass.is_complex()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":198,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":199,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _foreach_utils._get_fused_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":200,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls adadelta._single_tensor_adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":201,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls compiler.is_compiling()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":202,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls optimizer._default_to_fused_or_foreach()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":203,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls optimizer._to_scalar()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} diff --git a/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl b/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl new file mode 100644 index 00000000..b5594e51 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl @@ -0,0 +1,564 @@ +{"index":0,"relation":"APIContainRelation","description":"Dropout.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[70,"parent_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"parent_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"parent_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"parent_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":1,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":2,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":3,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":4,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":5,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":6,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":7,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":8,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":9,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":10,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":11,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":12,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":13,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":14,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":15,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":16,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":17,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":18,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":19,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":20,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":21,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":22,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":23,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":24,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":25,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":26,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":27,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":28,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":29,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":30,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":31,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":32,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":33,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":34,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":35,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":36,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":37,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":38,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":39,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":40,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":41,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":42,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":43,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":44,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":45,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":46,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":47,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":48,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":49,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":50,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":51,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":52,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":53,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":54,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":55,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":56,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":57,"relation":"APIContainRelation","description":"Embedding.forward() always calls _VariableFunctionsClass.embedding()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":58,"relation":"APIContainRelation","description":"Embedding.forward() always calls functional.embedding()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":59,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":60,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":61,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":62,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":63,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":64,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":65,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":66,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":67,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":68,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":69,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":70,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":71,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":72,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":73,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":74,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":75,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":76,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":77,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":78,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":79,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":80,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":81,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":82,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":83,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":84,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":85,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":86,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":87,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":88,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":89,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":90,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":91,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":92,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":93,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":94,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":95,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":96,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":97,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":98,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":99,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":100,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":101,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":102,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":103,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":104,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":105,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":106,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":107,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":108,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":109,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":110,"relation":"APIContainRelation","description":"LayerNorm.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":111,"relation":"APIContainRelation","description":"LayerNorm.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":112,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":113,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":114,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":115,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":116,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":117,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":118,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":119,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":120,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":121,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":122,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":123,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":124,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":125,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":126,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":127,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":128,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":129,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":130,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":131,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":132,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":133,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":134,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":135,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":136,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":137,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":138,"relation":"DistinctArgumentRelation","description":"LayerNorm.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"control":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"holdout":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true],"fault":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true]}} +{"index":139,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":140,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":141,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":142,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":143,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":144,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":145,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":146,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":147,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":148,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":149,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":150,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":151,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":152,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":153,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":154,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":155,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":156,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":157,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":158,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":159,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":160,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":161,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":162,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":163,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":164,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":165,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":166,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":167,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":168,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":169,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":170,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":171,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":172,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":173,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":174,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":175,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":176,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":177,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":178,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":179,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":180,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":181,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":182,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":183,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":184,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":185,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":186,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":187,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":188,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":189,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":190,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":191,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":192,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":193,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":194,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":195,"relation":"DistinctArgumentRelation","description":"Linear.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"control":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"holdout":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true],"fault":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true]}} +{"index":196,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":197,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":198,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":199,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":200,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":201,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":202,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":203,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":204,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":205,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":206,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":207,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":208,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":209,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":210,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":211,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":212,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":213,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":214,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":215,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":216,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":217,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":218,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":219,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":220,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":221,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":222,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":223,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":224,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":225,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":226,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":227,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":228,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":229,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":230,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":231,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":232,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":233,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":234,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":235,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":236,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":237,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":238,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":239,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":240,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":241,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":242,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":243,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":244,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":245,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":246,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":247,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":248,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":249,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":250,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":251,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":252,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":253,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":254,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":255,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":256,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":257,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":258,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":259,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":260,"relation":"DistinctArgumentRelation","description":"MultiheadAttention.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":261,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":262,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":263,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":264,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":265,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":266,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":267,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":268,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":269,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":270,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":271,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":272,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":273,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":274,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":275,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":276,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":277,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":278,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":279,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":280,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":281,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":282,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":283,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":284,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":285,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":286,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":287,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":288,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":289,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":290,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":291,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":292,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":293,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":294,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":295,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":296,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":297,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":298,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":299,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":300,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":301,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":302,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":303,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":304,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":305,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":306,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":307,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":308,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":309,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":310,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":311,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":312,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":313,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":314,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":315,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":316,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":317,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":318,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":319,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":320,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":321,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":322,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":323,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":324,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":325,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":326,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":327,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":328,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":329,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":330,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":331,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":332,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":333,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":334,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":335,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Dropout.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":336,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls LayerNorm.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":337,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Linear.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":338,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls ModuleList._get_abs_string_index()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":339,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls MultiheadAttention.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":340,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":341,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer._ff_block()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":342,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer._sa_block()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":343,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":344,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.full()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":345,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":346,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":347,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.relu()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":348,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.triu()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":349,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":350,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":351,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":352,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":353,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":354,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":355,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":356,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":357,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._detect_is_causal_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":358,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._generate_square_subsequent_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":359,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._get_seq_len()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":360,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":361,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":362,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":363,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":364,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":365,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":366,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":367,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":368,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":369,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":370,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":371,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":372,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":373,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":374,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":375,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":376,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":377,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":378,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":379,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":380,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":381,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":382,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":383,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":384,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":385,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":386,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":387,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":388,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":389,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":390,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":391,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":392,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":393,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":394,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":395,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":396,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":397,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":398,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":399,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":400,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":401,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":402,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":403,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":404,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":405,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":406,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":407,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":408,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":409,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":410,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":411,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":412,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":413,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":414,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":415,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":416,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":417,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":418,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":419,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":420,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":421,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":422,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":423,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":424,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":425,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":426,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":427,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":428,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":429,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":430,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":431,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":432,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Dropout.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":433,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls LayerNorm.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":434,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Linear.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":435,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls MultiheadAttention.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":436,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":437,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls TransformerEncoderLayer._ff_block()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":438,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls TransformerEncoderLayer._sa_block()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":439,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":440,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":441,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.relu()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":442,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":443,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":444,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":445,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":446,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":447,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":448,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":449,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":450,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":451,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":452,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":453,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":454,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":455,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":456,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":457,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":458,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":459,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":460,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":461,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":462,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":463,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":464,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":465,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":466,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":467,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":468,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":469,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":470,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":471,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":472,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":473,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":474,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":475,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":476,"relation":"DistinctArgumentRelation","description":"TransformerEncoderLayer.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":477,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":478,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":479,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":480,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":481,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":482,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":483,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":484,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":485,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":486,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":487,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":488,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":489,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":490,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":491,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":492,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":493,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":494,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":495,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":496,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":497,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":498,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":499,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":500,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":501,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":502,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":503,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":504,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":505,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":506,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":507,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":508,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":509,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":510,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":511,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":512,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":513,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":514,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":515,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":516,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":517,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":518,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":519,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":520,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":521,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":522,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":523,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module._named_members()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",1,1,0,0,1,1,0,1,9,null,true,false,false,false,false,true],"holdout":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true],"fault":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":524,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module.named_modules()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":525,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module.named_parameters()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",1,1,0,0,1,1,0,1,9,null,true,false,false,false,false,true],"holdout":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true],"fault":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":526,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass._foreach_mul_()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":527,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass._foreach_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":528,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass.clamp()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":529,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass.stack()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":530,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._device_has_foreach_support()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":531,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":532,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._group_tensors_by_device_and_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":533,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._has_foreach_support()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":534,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls clip_grad._clip_grads_with_norm_()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":535,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls clip_grad._get_total_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":536,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls torch.is_grad_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":537,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":538,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":539,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":540,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad_fn=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":541,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":542,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":543,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":544,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_leaf=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":545,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":546,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":547,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":548,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":549,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":550,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":551,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":552,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":553,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":554,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":555,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":556,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":557,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":558,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":559,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with nbytes=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":560,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with ndim=0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":561,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with requires_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":562,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":563,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with shape=()","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} diff --git a/docs/measurements/data/fresh-run-invariant-validation.json b/docs/measurements/data/fresh-run-invariant-validation.json new file mode 100644 index 00000000..95340944 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation.json @@ -0,0 +1,1894 @@ +{ + "study_date": "2026-10-25", + "harness_commit": "cad62cc98ef5fb7b313325443caa33f715ed141e", + "source_composition": { + "base": "99d16905296d83495570fdcafa91370b242d2528", + "trace_integrity": "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4", + "torchscript": "9ac61d1140cf8679c24442a174ec12f4de069980", + "state_fix_hunks_from": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "changed_python_files": [ + "tests/test_dynamic_typename.py", + "tests/test_precondition_state_isolation.py", + "tests/test_torchscript_compat.py", + "tests/test_trace_integrity.py", + "traincheck/instrumentor/dumper.py", + "traincheck/instrumentor/torchscript_compat.py", + "traincheck/instrumentor/tracer.py", + "traincheck/invariant/precondition.py", + "traincheck/utils.py" + ], + "dynamic_typename_fix": "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c", + "source_sha256": { + "tests/test_utils.py": "cc63afa128223d1a7c518becf2c215ffd0ab91891a01b23e0bbeba2e9726e56c", + "tests/test_dynamic_policy.py": "ac82536add642da8179acd54b07d86cff81b5856d90fc41896d18a57334ccdec", + "tests/test_inference_sampling.py": "41a632102208e53091615d21714b7dc1d6840465fd550efa33caf8eeca3fe361", + "tests/test_torchscript_compat.py": "653be1032409579030ebd3ec5694cc3e43f34df8e4f5750a76a9ffc64dd0a2e2", + "tests/test_trace_integrity.py": "0d12f81b39c4772e9d6743605c4a627078b4a9f5e85b254d694ad999d3496f08", + "tests/test_loop_injection.py": "c9fda263c5b014f2a56030d070f72144e5aea3be50c431d2a67a910e5978e11d", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/test_display_names.py": "bf85fee31e1bb8384468df76bf30d4209aa8e14d0b9b822a2af9860ee6c624cd", + "tests/test_dynamic_typename.py": "fc86e92919f07a82f5f06e2d9ebcd52431ec2ea175ceb5d1c2417aab45924ba8", + "tests/test_call_index.py": "979c4b4489b5ca8c2468db7ee5d7b39bb9fe8b3c193f7b43f1627ada17c7b541", + "tests/test_example_reservoir.py": "4805845c77e5bdeaf48793dd8b8cc1ed42126e9c2ea5154c073e9fa2cc075b90", + "tests/test_violation_summary.py": "fa5bd46bbc5592867eb7d018a494b81aa5d59f451fd282a147ed56aefd7e3619", + "tests/test_precondition_state_isolation.py": "d56e2e0e3ca5d410ccef5cb1de350a21f37a6be679efd302b523aae1f7a1d919", + "tests/test_identity_cache.py": "733575f48f690cfc2913169bff2a8656787fe94f37e4658850e29d2f4656ac50", + "tests/test_context_index.py": "7702f13e695f430e2aa1e302e3719ce9d5e8bd74a2d65cac5b7fbfa2178159dc", + "tests/test_policy_injection.py": "9409b8f693da8b05e556f658486346422c6e9b853f1a991cad28572dfd977848", + "tests/test_variable_tracking_policy.py": "059178ac644d6d9305dd10ffcea75dc99e5fd5019b02f09f37a8a93728f5d989", + "tests/fixtures/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/fixtures/legacy_trace_pandas.py": "342bef3c1de26d77cc8b1e34f94dc470f45d929e038e458ac00eff11eebc1860", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/utils.py": "854c11519e7aa6d0cbbfd79694d2ad4895fb1aaf72e9efbb0068048598b70750", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/precondition.py": "dfd02e6c68a6c3f2d06d6a9beaa109dcf99ef099efeda75252fd8c559fdbb5da", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_pandas.py": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/dumper.py": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/instrumentor/torchscript_compat.py": "0d0c348fa02d5f93bb74e874829e5fb4c57365b82ce973ae8ff1f21a42c2d64e", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/tracer.py": "ad57320f0a3764c02c73cf4bfa623f1ef73b9835dfa6cb1365dc2988551de428", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0", + "device": "cpu", + "native_threads": 1 + }, + "protocol": { + "reference_seed": 42, + "reference_steps": 10, + "control_seed": 42, + "control_steps": 10, + "holdout_seed": 43, + "holdout_steps": 20, + "batch_size": 2, + "prepared_data_steps": 20, + "check_relation_first": false, + "inference_workers": 1, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "each_phase_fresh_OS_process": true, + "holdout_inputs_overlap_reference": true, + "API_instrumentation": true, + "proxy_scanning": false + }, + "tests": { + "local_counter_and_mutation": 13, + "matching_linux_counter_and_mutation": 13, + "matching_linux_composed_production_stack": 29 + }, + "archives_sha256": { + "harness-cad62cc-complete.tar.gz": "b9ed7f3401363e73d8b69c9b12f183429365df18c28a61c1969e3be90285cb5a", + "integration-v3-source.tar.gz": "4b19005ae6ecad19e48b531185a7552cdfd0d08b6d23c907460ac33adb6498c3", + "final-mnist-74b-diagnostic.tar.gz": "0612f10428e1f22ebc831e6e63f990a91c917aa6128044e43a86c3790ba61195", + "final-mnist-cad62cc-evidence.tar.gz": "6a9a6866c4ac78d6050527fd6dac1f7d9586427b2cd607cbe182e86bd8135e6d", + "final-transformer-cad62cc-evidence.tar.gz": "f3b5d6b4c9065362d3d59cfe2339363700137a2aec1cbe865217a3e5f6120acc" + }, + "workloads": { + "mnist": { + "comparison": { + "status": "study_completed", + "invariants_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "training_comparisons": { + "same_seed_fresh_control": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + }, + "holdout_instrumentation_comparison": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + } + }, + "benchmark_data_runtime_source_and_harness_matched": true, + "observer_matches_direct_checker": true, + "per_run_summary": { + "reference": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "control": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "holdout": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "fault": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 3, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + } + }, + "per_relation": { + "reference": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 360, + "precondition_group_passes": 333, + "actual_decisions": 333, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 1420, + "precondition_group_passes": 1420, + "actual_decisions": 1420, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 30, + "precondition_group_passes": 3, + "actual_decisions": 30, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 1940, + "precondition_group_passes": 1940, + "actual_decisions": 1880, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "control": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 360, + "precondition_group_passes": 333, + "actual_decisions": 333, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 1420, + "precondition_group_passes": 1420, + "actual_decisions": 1420, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 30, + "precondition_group_passes": 3, + "actual_decisions": 30, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 1940, + "precondition_group_passes": 1940, + "actual_decisions": 1880, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "holdout": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 720, + "precondition_group_passes": 663, + "actual_decisions": 663, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 2840, + "precondition_group_passes": 2840, + "actual_decisions": 2840, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 60, + "precondition_group_passes": 3, + "actual_decisions": 60, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 3880, + "precondition_group_passes": 3880, + "actual_decisions": 3760, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "fault": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 720, + "precondition_group_passes": 663, + "actual_decisions": 663, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 1, + "opportunities": 2840, + "precondition_group_passes": 2801, + "actual_decisions": 2801, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 60, + "precondition_group_passes": 3, + "actual_decisions": 60, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 2, + "opportunities": 3880, + "precondition_group_passes": 3802, + "actual_decisions": 3682, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + } + }, + "raw_ids": { + "reference": { + "process": [ + 1347106 + ], + "thread": [ + 140498110309440 + ] + }, + "control": { + "process": [ + 1347133 + ], + "thread": [ + 140011360134208 + ] + }, + "holdout": { + "process": [ + 1347155 + ], + "thread": [ + 140714782577728 + ] + } + }, + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "invariant_index": 21, + "healthy_decisions": 40, + "fault_decisions": 1, + "fault_precondition_group_passes": 1, + "violation": true, + "edit": { + "file": "trace_API_1347155_140714782577728.log", + "zero_based_line": 15, + "path": [ + "return_values", + "torch.FloatTensor", + "dtype" + ], + "attribute": "dtype", + "before_value": "torch.float32", + "after_value": "torch.float64", + "raw_process_id": 1347155, + "raw_thread_id": 140714782577728 + } + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant." + ] + }, + "model": { + "model_config": { + "architecture": "exact Net class" + }, + "parameter_count": 1199882, + "precision": "float32", + "extracted_symbols": [ + "Net" + ], + "benchmark_source_sha256": "4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7", + "extracted_definitions_sha256": "f839142a57ce449054e05ffa79d746e5282e2599e101810e6ccf6e7a9dd33ce5", + "adapter_sha256": "876f28f728ef924fa97afb7be7786c8001df9e549e1d724fc6ba60c7fa6bf2f9", + "instrumentation": { + "funcs_to_instr": null, + "models_to_track": [], + "sampling_interval": 1, + "scan_proxy_in_args": false, + "target": "torch", + "use_full_instr": false, + "warm_up_steps": 0 + }, + "instrumented_function_count": 20998 + }, + "data": { + "count": 40, + "files": [ + { + "path": "train-images-idx3-ubyte", + "sha256": "ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db" + }, + { + "path": "train-labels-idx1-ubyte", + "sha256": "65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5" + } + ], + "kind": "cached_mnist_training_prefix", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + "ordering": "first samples in IDX order; no shuffle" + }, + "trace_rows": { + "reference": 1584, + "control": 1584, + "holdout": 3104, + "fault": 3104 + }, + "failed_inference_hypotheses": 20, + "artifact_sha256": { + "baseline-holdout/adaptation.json": "e513882d9150e0f71fd1389dc3a2c21901a3098c44475d207856fb0f0c0020d3", + "baseline-holdout/metrics.json": "b6fed469dd317b29f34b26a0be43035ce046483277b4b2ec0786deed72a4a445", + "baseline-holdout/phase-result.json": "57a7f3f7877aec96c5a7581b1bc3934e3aca82521719c35646b95033d56b0f34", + "check-control/coverage.json": "f07b558285dbc44eafccaea64f6f812f9b3ed3f52a8155b22e0c8fdc649a4a6b", + "check-control/phase-result.json": "3f19692915fe7560759742d1bc0e50d40a74f7c0eed628e36621e83b06dcdf3b", + "check-fault/coverage.json": "d3cc7954b4b07506c319bfb36ef8b60205392a02ceaaa8b4e5a17b2a8077097e", + "check-fault/phase-result.json": "d382c3a28c73aec5594cb5e998b58c1a32f858ba11b6126fd6479919acb3fbc4", + "check-holdout/coverage.json": "5dc166605d095aeaada421e5e201774f0c91d83e58cf097b1f5ac11f2cb33c2d", + "check-holdout/phase-result.json": "87586fb7106d5425b88dce9d696c661773b29bcf5daa123fefb57d17176f6cf7", + "check-holdout-direct/coverage.json": "29d316bf84d3de6dec6d2441c8a4f52e6a03a953e1f16182cf1255ae31a5fde0", + "check-holdout-direct/phase-result.json": "4c978a879f1af1e88595bbb7126ed8da69dc566598fe0a8958aa149d9cb24523", + "check-reference/coverage.json": "b30b981c7a88357ab883d3cde688e88d40c784204b474418d10d42a8643f99b8", + "check-reference/phase-result.json": "de96730dcb4d33a6760189b98824f0e4d5728bb9488984f7a2d69e36920c8d15", + "comparison.json": "e9feb79f1954ac3f5dc7a75e703a6bd00b29f4b59a6519d268f801e8a9a0697e", + "control/adaptation.json": "5abdf5a1f238cbd0e8a83ed88a908698e8b8cdb460a466f043f558950e7532af", + "control/metrics.json": "61e93a78e4612d6201decf311dde5b726d62405c5772f541c3ced16102226788", + "control/phase-result.json": "8c7a42816ea67cf150fe037715e14a64437f7519c803460183e8a26909f26c3c", + "fault/mutation.json": "c4b7db0e8d68453fb5cb89611573db5aec1773ebaca0ada67f866938e0da1a0a", + "holdout/adaptation.json": "2d4c4079c071a0500d8f08e734096b767c7da1998ebd667f4ace6b1c97d14f6d", + "holdout/metrics.json": "30e4e104d56b2a9bb5218e3b891dbab483362a435b1f3f3455fb3aba33bd6218", + "holdout/phase-result.json": "b00787f9a5b65f2b8b724ef6e38b71b126ec0bb0c19897806e8ae9fd24e4c754", + "inference/phase-result.json": "cec09cc23d492a1a5fa910ee69412d210b0cc47ca5b6dfe6ac878570f6043ad3", + "prepare-fault/phase-result.json": "46590662108731e2fac6096653c72c9e2dc6ddb51355c82bae69d5119993dd56", + "reference/adaptation.json": "8204e4744995631ee4982033a3949d1b59c99fa58cdf624031c250610dac8603", + "reference/metrics.json": "31a6c6e58a0ead22b57ad69584672e956491dc54872f14b8a20c08968df444b0", + "reference/phase-result.json": "d511dfe65964b24f114852e0ad8acfe36a016dc43aa4833e8c537f82fe23cb02", + "study-plan.json": "bd5d1b4ec4ffa8ef68813ef0f781de6b3cfa9790b323e4d98447cc8de501af87" + }, + "invariants_jsonl_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "trace_sha256": { + "control/traces/trace_API_1347133_140011360134208.log": "e68a120d1713eeec0fb916a7938ee8b2ee2088ff16f9e2c41e89f1291cbee8d2", + "fault/traces/trace_API_1347155_140714782577728.log": "953fd8ed2ae5c7b9b8489a83b114c1b6f63a1fb210fe328ce63c65e8507f00fa", + "holdout/traces/trace_API_1347155_140714782577728.log": "764533de616105fecaf8f5a5b7c2bb9bf84b9a14c3cffb33fa0f88a92d047c44", + "reference/traces/trace_API_1347106_140498110309440.log": "c4800bbf95b3ef0fa367e107ca621beaba9768dc6a6c07fcc4e9439246da20be" + }, + "training_metrics": { + "reference": { + "completed_steps": 10, + "initial_model_state_sha256": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "final_model_state_sha256": "1b6002ccebfac00518fc53119b3166628a01d87a03a53b791b2759594281dc84", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.2311902046203613 + }, + { + "loss": 3.296325445175171 + }, + { + "loss": 2.6148438453674316 + }, + { + "loss": 2.3794608116149902 + }, + { + "loss": 2.291707992553711 + }, + { + "loss": 2.423546552658081 + }, + { + "loss": 2.289013385772705 + }, + { + "loss": 2.39404559135437 + }, + { + "loss": 2.370988368988037 + }, + { + "loss": 2.282020092010498 + } + ] + }, + "control": { + "completed_steps": 10, + "initial_model_state_sha256": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "final_model_state_sha256": "1b6002ccebfac00518fc53119b3166628a01d87a03a53b791b2759594281dc84", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.2311902046203613 + }, + { + "loss": 3.296325445175171 + }, + { + "loss": 2.6148438453674316 + }, + { + "loss": 2.3794608116149902 + }, + { + "loss": 2.291707992553711 + }, + { + "loss": 2.423546552658081 + }, + { + "loss": 2.289013385772705 + }, + { + "loss": 2.39404559135437 + }, + { + "loss": 2.370988368988037 + }, + { + "loss": 2.282020092010498 + } + ] + }, + "holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "5ce184958e1b3340cb58e0cc5f54150a9e7a3c8c84b6d5bce9a095f4c6f1256b", + "final_model_state_sha256": "5a1c83cc4671c529a5c7525303db31a3e9196de4619a49ef101a294c27686f3e", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.237604856491089 + }, + { + "loss": 3.1580514907836914 + }, + { + "loss": 2.471099853515625 + }, + { + "loss": 2.4436941146850586 + }, + { + "loss": 2.2179641723632812 + }, + { + "loss": 2.353369951248169 + }, + { + "loss": 2.487359046936035 + }, + { + "loss": 2.25258731842041 + }, + { + "loss": 2.375113010406494 + }, + { + "loss": 2.3781189918518066 + }, + { + "loss": 2.244692802429199 + }, + { + "loss": 2.16099214553833 + }, + { + "loss": 1.955580711364746 + }, + { + "loss": 1.7661290168762207 + }, + { + "loss": 2.8482470512390137 + }, + { + "loss": 2.6151835918426514 + }, + { + "loss": 2.337063789367676 + }, + { + "loss": 2.2885587215423584 + }, + { + "loss": 2.2511115074157715 + }, + { + "loss": 2.382375717163086 + } + ] + }, + "baseline-holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "5ce184958e1b3340cb58e0cc5f54150a9e7a3c8c84b6d5bce9a095f4c6f1256b", + "final_model_state_sha256": "5a1c83cc4671c529a5c7525303db31a3e9196de4619a49ef101a294c27686f3e", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.237604856491089 + }, + { + "loss": 3.1580514907836914 + }, + { + "loss": 2.471099853515625 + }, + { + "loss": 2.4436941146850586 + }, + { + "loss": 2.2179641723632812 + }, + { + "loss": 2.353369951248169 + }, + { + "loss": 2.487359046936035 + }, + { + "loss": 2.25258731842041 + }, + { + "loss": 2.375113010406494 + }, + { + "loss": 2.3781189918518066 + }, + { + "loss": 2.244692802429199 + }, + { + "loss": 2.16099214553833 + }, + { + "loss": 1.955580711364746 + }, + { + "loss": 1.7661290168762207 + }, + { + "loss": 2.8482470512390137 + }, + { + "loss": 2.6151835918426514 + }, + { + "loss": 2.337063789367676 + }, + { + "loss": 2.2885587215423584 + }, + { + "loss": 2.2511115074157715 + }, + { + "loss": 2.382375717163086 + } + ] + } + }, + "coverage_columns": [ + "opportunities", + "opportunity_unit", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "visited_calls", + "unvisited_calls_after_stop", + "unvisited_pairs_after_stop", + "active", + "dormant", + "check_passed", + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active" + ], + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": "dtype", + "invariant_index": 21, + "invariant": { + "text_description": "Conv2d.forward() consistently returns tensors with dtype=torch.float32", + "relation": "ConsistentOutputRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.conv.Conv2d.forward" + }, + { + "param_type": "VarTypeParam", + "var_type": "torch.Tensor", + "attr_name": "dtype", + "const_value": "torch.float32" + } + ], + "precondition": { + "pre_event": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 20, + "num_negative_examples": 0 + }, + "func_call_id": "9d110c7f11d845f9ad86b2a2e542b000_23960437373776510", + "edits": [ + { + "file": "trace_API_1347155_140714782577728.log", + "zero_based_line": 15, + "path": [ + "return_values", + "torch.FloatTensor", + "dtype" + ], + "attribute": "dtype", + "before_value": "torch.float32", + "after_value": "torch.float64", + "raw_process_id": 1347155, + "raw_thread_id": 140714782577728 + } + ], + "traces": { + "trace_API_1347155_140714782577728.log": { + "original_sha256": "764533de616105fecaf8f5a5b7c2bb9bf84b9a14c3cffb33fa0f88a92d047c44", + "fault_sha256": "953fd8ed2ae5c7b9b8489a83b114c1b6f63a1fb210fe328ce63c65e8507f00fa" + } + }, + "invariants_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "healthy_coverage_sha256": "5dc166605d095aeaada421e5e201774f0c91d83e58cf097b1f5ac11f2cb33c2d", + "precondition_inputs_unchanged": true, + "ids_unchanged": true, + "invariants_not_retrained_or_rewritten": true + }, + "per_invariant_ledger": { + "file": "fresh-run-invariant-validation-mnist.jsonl", + "sha256": "59a8cf816bd4474ed4505d23959d52a7d050ef01ea222471f8fc3f1020d27bbd", + "format": "One rule per line; run arrays use coverage_columns in this document; null means not applicable/unobserved, never zero." + } + }, + "transformer": { + "comparison": { + "status": "study_completed", + "invariants_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "training_comparisons": { + "same_seed_fresh_control": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + }, + "holdout_instrumentation_comparison": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + } + }, + "benchmark_data_runtime_source_and_harness_matched": true, + "observer_matches_direct_checker": true, + "per_run_summary": { + "reference": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "control": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "holdout": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "fault": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 4, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + } + }, + "per_relation": { + "reference": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 0, + "opportunities": 1100, + "precondition_group_passes": 1100, + "actual_decisions": 1100, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 6010, + "precondition_group_passes": 6010, + "actual_decisions": 6010, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 8450, + "precondition_group_passes": 8450, + "actual_decisions": 8450, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 220, + "precondition_group_passes": 4, + "actual_decisions": 220, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "control": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 1100, + "precondition_group_passes": 1082, + "actual_decisions": 1082, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 6010, + "precondition_group_passes": 6010, + "actual_decisions": 6010, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 8450, + "precondition_group_passes": 8450, + "actual_decisions": 8450, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 220, + "precondition_group_passes": 4, + "actual_decisions": 220, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "holdout": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 2200, + "precondition_group_passes": 2162, + "actual_decisions": 2162, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 12020, + "precondition_group_passes": 12020, + "actual_decisions": 12020, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 16900, + "precondition_group_passes": 16900, + "actual_decisions": 16900, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 440, + "precondition_group_passes": 4, + "actual_decisions": 440, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "fault": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 2200, + "precondition_group_passes": 2162, + "actual_decisions": 2162, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 1, + "opportunities": 12020, + "precondition_group_passes": 12001, + "actual_decisions": 12001, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 1, + "opportunities": 16900, + "precondition_group_passes": 16881, + "actual_decisions": 16881, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 440, + "precondition_group_passes": 4, + "actual_decisions": 440, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + } + }, + "raw_ids": { + "reference": { + "process": [ + 1347485 + ], + "thread": [ + 140490739766336 + ] + }, + "control": { + "process": [ + 1347544 + ], + "thread": [ + 140262258558016 + ] + }, + "holdout": { + "process": [ + 1347647 + ], + "thread": [ + 139745137321024 + ] + } + }, + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "invariant_index": 84, + "healthy_decisions": 20, + "fault_decisions": 1, + "fault_precondition_group_passes": 1, + "violation": true, + "edit": { + "file": "trace_API_1347647_139745137321024.log", + "zero_based_line": 75, + "path": [ + "return_values", + "torch.FloatTensor", + "shape", + 0 + ], + "attribute": "shape", + "before_value": [ + 16, + 2, + 32 + ], + "after_value": [ + 17, + 2, + 32 + ], + "raw_process_id": 1347647, + "raw_thread_id": 139745137321024 + } + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant." + ] + }, + "model": { + "model_config": { + "clip": 0.25, + "dropout_argument": 0.2, + "lr": 1.0, + "nhead": 2, + "nhid": 64, + "ninp": 32, + "nlayers": 2, + "ntoken": 293, + "sequence_length": 16 + }, + "parameter_count": 36197, + "precision": "float32", + "extracted_symbols": [ + "PositionalEncoding", + "TransformerModel" + ], + "benchmark_source_sha256": "e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c", + "extracted_definitions_sha256": "1f2db95454690be6a91aaa141e3a5a246308f9abca11444c9113e4adfb78b95d", + "adapter_sha256": "876f28f728ef924fa97afb7be7786c8001df9e549e1d724fc6ba60c7fa6bf2f9", + "instrumentation": { + "funcs_to_instr": null, + "models_to_track": [], + "sampling_interval": 1, + "scan_proxy_in_args": false, + "target": "torch", + "use_full_instr": false, + "warm_up_steps": 0 + }, + "instrumented_function_count": 20998 + }, + "data": { + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": "train.txt", + "prefix_sha256": "457b31f0b69fdfbf48c919d7fa9a7b073c2b03b8adfa51de3543e109b0e01903", + "sha256": "9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f", + "token_count": 642, + "vocabulary_size": 293 + }, + "trace_rows": { + "reference": 4446, + "control": 4446, + "holdout": 8886, + "fault": 8886 + }, + "failed_inference_hypotheses": 11, + "artifact_sha256": { + "baseline-holdout/adaptation.json": "d73ea9cff0d15217963372d4c6b18eba13e0eaa4e03d45aa627ff9c753597a32", + "baseline-holdout/metrics.json": "f9d111863e26ff2023405c41cf862513d8ea167464ad4f7c10989abb257d669d", + "baseline-holdout/phase-result.json": "02e5f2576d7223c059e159f624d6e5030f3d08c9bab8391ee2e9bd051d1af4ef", + "check-control/coverage.json": "2fcdad5051bf4679d3663a0aaf86048a236e971a80517f33890a7f56b17f64c0", + "check-control/phase-result.json": "58d12a56e1be8fb59638f8ab13cb6257958be989bda2e0028d900a28dc3e64c6", + "check-fault/coverage.json": "a89b68a24e46228872553b8ee6fed51984e172e52651583be2ac4f23330d6d6d", + "check-fault/phase-result.json": "e063ef35ebd7eea7accff6e046e5fc2ca7650e4c94e3c3c2af7f69f8c99b114e", + "check-holdout/coverage.json": "079382f905a5809e6d66e2f1fd4ca4ca65423b1056eb69bdea1fa93a488f00f0", + "check-holdout/phase-result.json": "4807c3c7edd8e49d87150c29a3bd8aeee71992d21e08718c68976da3967c6d89", + "check-holdout-direct/coverage.json": "c060b0f9ef7da9002198017625d56876950518c5f4072d39d75e3982cc5019dd", + "check-holdout-direct/phase-result.json": "86455b01be33c9798433c7be940fc8bad767789e95852a0014fdabdbce1b34a9", + "check-reference/coverage.json": "771e09f0692dee2230d8e073d515945ebc74292b61baa04d830facb049284af2", + "check-reference/phase-result.json": "fde3c128ccc040852965261555e0d65b543a9360268f46a443cc2c8382a06eb7", + "comparison.json": "ce41f5fa74255424fb7c87344fbf3ba540629004f6e37662e2b5509444fed201", + "control/adaptation.json": "f1f9d3cebb92dc9a53a1810fd7ac2b9784f826359010fe8669209f17a2d6afef", + "control/metrics.json": "c91ee13d398cad807b7a62841f47e82d37492d49c23b961dcdb28a68ca7503f9", + "control/phase-result.json": "4d8a83a3b84ed1cc11c745392b8c3cdd5d34b395a99a4fb732c7449582de372a", + "fault/mutation.json": "cadfcc8f26752d903f5d05170cb38ff320b66f431e46518b190c8e3f0d7cd444", + "holdout/adaptation.json": "d9162c4392cf9a64111ed97779344ceda32f74e02bcf4c1274b2e14e2d7ec6da", + "holdout/metrics.json": "17a4d6ff99936f05e49552b26569ae8309955ff7088d69c3130327bed2b61d6c", + "holdout/phase-result.json": "2bb9f6cc8ffda717bb89688697da7889ce5855965c9e7f3b5087000b4c06e8d8", + "inference/phase-result.json": "15c4fbd12e3cf9609dd9be61491fd8a9f04e5b9813f55eed0656766b1a538d7c", + "prepare-fault/phase-result.json": "9b90cb87a59c3e12fb99dd819276f8b85231c431d0566a18ba88559c51b30cfb", + "reference/adaptation.json": "6078c8beafec0759c2742020cf94899ded01b5ebcbfb633d34ed6bb982d79c5f", + "reference/metrics.json": "710ef8a517ffd905f03efb88a59e35f2c7353b20d512d75e48f331738cd87b57", + "reference/phase-result.json": "25adbbf0eb426c585ced6ef4605d2d227bcda0b5c7079b9ceafa4908d1a89780", + "study-plan.json": "9d065aae89605f2fe63b299f15164f07275980c775bfe993c7baa12a6a0a9534" + }, + "invariants_jsonl_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "trace_sha256": { + "control/traces/trace_API_1347544_140262258558016.log": "440ad458b125167c19146e5043ab355e236067dae7df623a8abd992e16b3df96", + "fault/traces/trace_API_1347647_139745137321024.log": "860ee3203f82a34b9e119cb975231ec57a6a98c548e81dae02809f422dc0dc55", + "holdout/traces/trace_API_1347647_139745137321024.log": "6f5985e8fb93c8e2783d28813457581e2f491592d62c0fa4c581ca2ce558f73d", + "reference/traces/trace_API_1347485_140490739766336.log": "4184c9abeacc8c3c49a583038f507a3b0c3fb7218733230f34530df54003f60a" + }, + "training_metrics": { + "reference": { + "completed_steps": 10, + "initial_model_state_sha256": "83911ab0a0000c4a6fd71aa5bed8ca44de3241c5cd123367e70181c6eee5903c", + "final_model_state_sha256": "5034413e68786dd722b7681035bd6efb1c7d64b3c28ab450e458f5d80853fd76", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.8035569190979 + }, + { + "loss": 5.7292656898498535 + }, + { + "loss": 5.618026256561279 + }, + { + "loss": 5.672450065612793 + }, + { + "loss": 5.592149257659912 + }, + { + "loss": 5.601678848266602 + }, + { + "loss": 5.614868640899658 + }, + { + "loss": 5.65100622177124 + }, + { + "loss": 5.449929237365723 + }, + { + "loss": 5.456549167633057 + } + ] + }, + "control": { + "completed_steps": 10, + "initial_model_state_sha256": "83911ab0a0000c4a6fd71aa5bed8ca44de3241c5cd123367e70181c6eee5903c", + "final_model_state_sha256": "5034413e68786dd722b7681035bd6efb1c7d64b3c28ab450e458f5d80853fd76", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.8035569190979 + }, + { + "loss": 5.7292656898498535 + }, + { + "loss": 5.618026256561279 + }, + { + "loss": 5.672450065612793 + }, + { + "loss": 5.592149257659912 + }, + { + "loss": 5.601678848266602 + }, + { + "loss": 5.614868640899658 + }, + { + "loss": 5.65100622177124 + }, + { + "loss": 5.449929237365723 + }, + { + "loss": 5.456549167633057 + } + ] + }, + "holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "31d5c753005bcf762ac2c1af38c3e71693cde10d193d80a85729ec342e69a572", + "final_model_state_sha256": "7e7a7529ef3c7d351ca0b0b3beccaf39c8eb8df71f2c8e8f9f9f5d936f9431c5", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.669175624847412 + }, + { + "loss": 5.593283653259277 + }, + { + "loss": 5.664262771606445 + }, + { + "loss": 5.635302543640137 + }, + { + "loss": 5.59808874130249 + }, + { + "loss": 5.639740467071533 + }, + { + "loss": 5.552767753601074 + }, + { + "loss": 5.676934719085693 + }, + { + "loss": 5.419702529907227 + }, + { + "loss": 5.414367198944092 + }, + { + "loss": 5.355993747711182 + }, + { + "loss": 5.5031280517578125 + }, + { + "loss": 5.387535572052002 + }, + { + "loss": 5.472236633300781 + }, + { + "loss": 5.2465410232543945 + }, + { + "loss": 5.450842380523682 + }, + { + "loss": 5.42155647277832 + }, + { + "loss": 5.213186264038086 + }, + { + "loss": 5.500979423522949 + }, + { + "loss": 5.30380392074585 + } + ] + }, + "baseline-holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "31d5c753005bcf762ac2c1af38c3e71693cde10d193d80a85729ec342e69a572", + "final_model_state_sha256": "7e7a7529ef3c7d351ca0b0b3beccaf39c8eb8df71f2c8e8f9f9f5d936f9431c5", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.669175624847412 + }, + { + "loss": 5.593283653259277 + }, + { + "loss": 5.664262771606445 + }, + { + "loss": 5.635302543640137 + }, + { + "loss": 5.59808874130249 + }, + { + "loss": 5.639740467071533 + }, + { + "loss": 5.552767753601074 + }, + { + "loss": 5.676934719085693 + }, + { + "loss": 5.419702529907227 + }, + { + "loss": 5.414367198944092 + }, + { + "loss": 5.355993747711182 + }, + { + "loss": 5.5031280517578125 + }, + { + "loss": 5.387535572052002 + }, + { + "loss": 5.472236633300781 + }, + { + "loss": 5.2465410232543945 + }, + { + "loss": 5.450842380523682 + }, + { + "loss": 5.42155647277832 + }, + { + "loss": 5.213186264038086 + }, + { + "loss": 5.500979423522949 + }, + { + "loss": 5.30380392074585 + } + ] + } + }, + "coverage_columns": [ + "opportunities", + "opportunity_unit", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "visited_calls", + "unvisited_calls_after_stop", + "unvisited_pairs_after_stop", + "active", + "dormant", + "check_passed", + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active" + ], + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": "shape", + "invariant_index": 84, + "invariant": { + "text_description": "Embedding.forward() consistently returns tensors with shape=(16, 2, 32)", + "relation": "ConsistentOutputRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.sparse.Embedding.forward" + }, + { + "param_type": "VarTypeParam", + "var_type": "torch.Tensor", + "attr_name": "shape", + "const_value": [ + 16, + 2, + 32 + ] + } + ], + "precondition": { + "pre_event": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "func_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502411558452", + "edits": [ + { + "file": "trace_API_1347647_139745137321024.log", + "zero_based_line": 75, + "path": [ + "return_values", + "torch.FloatTensor", + "shape", + 0 + ], + "attribute": "shape", + "before_value": [ + 16, + 2, + 32 + ], + "after_value": [ + 17, + 2, + 32 + ], + "raw_process_id": 1347647, + "raw_thread_id": 139745137321024 + } + ], + "traces": { + "trace_API_1347647_139745137321024.log": { + "original_sha256": "6f5985e8fb93c8e2783d28813457581e2f491592d62c0fa4c581ca2ce558f73d", + "fault_sha256": "860ee3203f82a34b9e119cb975231ec57a6a98c548e81dae02809f422dc0dc55" + } + }, + "invariants_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "healthy_coverage_sha256": "079382f905a5809e6d66e2f1fd4ca4ca65423b1056eb69bdea1fa93a488f00f0", + "precondition_inputs_unchanged": true, + "ids_unchanged": true, + "invariants_not_retrained_or_rewritten": true + }, + "healthy_false_alarm_evidence": [ + { + "index": 523, + "rule": { + "text_description": "clip_grad.clip_grad_norm_() always calls Module._named_members()", + "relation": "APIContainRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.utils.clip_grad.clip_grad_norm_" + }, + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.module.Module._named_members", + "arguments": { + "args": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + }, + "self": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + }, + "get_members_fn": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "func_name": "torch.nn.modules.module.Module._named_members" + } + } + ], + "precondition": { + "parent_func_call_pre": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "evidence": { + "reference": { + "parent_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884548567", + "child_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884819012", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": {}, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + }, + "control": { + "parent_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769087879", + "child_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769353813", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 140261014914224 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140261014914224, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + }, + "holdout": { + "parent_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637904174", + "child_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502638163694", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 139743891407568 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 139743891407568, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + } + } + }, + { + "index": 525, + "rule": { + "text_description": "clip_grad.clip_grad_norm_() always calls Module.named_parameters()", + "relation": "APIContainRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.utils.clip_grad.clip_grad_norm_" + }, + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.module.Module.named_parameters", + "arguments": { + "args": { + "recurse": { + "bool": true + }, + "self": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + } + }, + "func_name": "torch.nn.modules.module.Module.named_parameters" + } + } + ], + "precondition": { + "parent_func_call_pre": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "evidence": { + "reference": { + "parent_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884548567", + "child_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884631538", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": {}, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + }, + "control": { + "parent_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769087879", + "child_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769166145", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 140261014914224 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140261014914224, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + }, + "holdout": { + "parent_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637904174", + "child_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637977940", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 139743891407568 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 139743891407568, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + } + } + } + ], + "per_invariant_ledger": { + "file": "fresh-run-invariant-validation-transformer.jsonl", + "sha256": "2e7ec6dbc35408bb1a39bff7be96943e00a64cf9d464c6091d54f275d06534f9", + "format": "One rule per line; run arrays use coverage_columns in this document; null means not applicable/unobserved, never zero." + }, + "literal_object_ID_argument_false_alarm_rules": 2 + } + }, + "limitations": [ + "One reference/control/holdout per workload; no population generalization estimate.", + "Fresh initialization and training RNG with longer horizon; not disjoint-dataset validation.", + "Offline checking; no online, distributed, performance, or inference-memory claim.", + "Faults alter recorded metadata only, not real training execution.", + "Active means at least one actual predicate decision; it does not mean every opportunity was evaluated.", + "Checker stops at first violation per rule; violation counts are failing invariants, not all bad events.", + "No raw process/thread clauses were inferred here; original TorchTitan literal-thread-ID rules remain untested.", + "The 74b3adf MNIST attempt stopped after healthy phases because its shape-only fault selector had no eligible rule; preserved separately." + ], + "metric_semantics": { + "identity_bearing_invariants": "Frozen checker counter means raw process/thread-ID PRECONDITION clauses only; excludes object IDs in API argument selectors.", + "identity_bearing_active": "Active rules among that same narrow PID/TID-precondition group.", + "fully_validated_active": "At least one completed decision and no requested proxy-check skip; does not mean all opportunities were evaluated." + } +} diff --git a/docs/measurements/fresh-run-invariant-validation.md b/docs/measurements/fresh-run-invariant-validation.md index ff4e6314..80b02f75 100644 --- a/docs/measurements/fresh-run-invariant-validation.md +++ b/docs/measurements/fresh-run-invariant-validation.md @@ -1,9 +1,65 @@ # Fresh-process invariant validation -This harness learns rules from one small training run, then checks the frozen -rules against separate OS processes: a same-seed control and a longer run with a -different initialization/training seed. Final measurements are pending. Earlier -MNIST pilot runs are diagnostics and are not final validation results. +Fresh-process checking works for the tested MNIST rules, but exposes two object-ID +false alarms in the small Transformer rules. This is a reproducible offline +validation harness and a bounded result, not a generalization or performance claim. +No production code changes are included. + +| Workload | Learned rules | Reference active / violations | Fresh same-seed control active / violations | Longer changed-seed holdout active / violations | Dormant rules | Requested proxy checks skipped | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| MNIST, 1,199,882 parameters | 204 | 204 / 0 | 204 / 0 | 204 / 0 | 0 | 0 | +| Transformer, 36,197 parameters | 564 | 564 / 0 | 564 / 2 | 564 / 2 | 0 | 0 | + +Every active rule completed at least one actual relation decision. That does not +mean every candidate opportunity was evaluated: for example, MNIST holdout IO +rules have 3,880 opportunities, 3,880 precondition passes, and 3,760 comparisons +because missing values skip 120 comparisons. Rules stop on their first violation. +There were no checker errors or unsupported-coverage results. The direct checker +without observers produced identical CheckerResult structures on both holdouts. + +For both workloads, the same-seed fresh control's complete loss trajectory and +initial/final canonical parameter hashes exactly match the reference. The +instrumented 20-step holdout likewise exactly matches its uninstrumented baseline. +These comparisons apply to these CPU runs and are not a claim of universal +instrumentation transparency. + +The two Transformer alarms are unconditional APIContain rules asserting that +`clip_grad_norm_` contains `Module.named_parameters` / `Module._named_members` with +particular arguments. Both child calls are present in the failed parent's strict +time interval, on the same process/thread. Their recorded model arguments differ +only in `_TRAINCHECK_src_mask_ID`: the reference's memory-object ID was embedded +literally into both learned child selectors. Fresh processes have different IDs. +Thus both rules fail even on the same-seed control despite identical losses and +parameters. The evidence retains the frozen selectors and raw child arguments; +no IDs or learned rules were rewritten. This is an object-identity portability +failure, distinct from thread-ID precondition portability. + +The post-hoc single-field controls activate a passing frozen rule and violate it +at the edited call. MNIST changes one recorded output dtype from `torch.float32` +to `torch.float64`: one output rule plus two IO rules fail (3 new violations). +Transformer changes one output shape from `[16, 2, 32]` to `[17, 2, 32]`: two rules +newly fail, in addition to the 2 existing healthy alarms (4 total). These are +trace-metadata sensitivity controls, not injected training bugs. They establish +neither broad defect coverage nor the consistency of an alternative tensor execution. + +The [compact evidence](data/fresh-run-invariant-validation.json) records source +commits/hashes, data hashes, actual runtime, paired parameter hashes/losses, raw +process/thread IDs, fault edits, family totals, and explicit failed-attempt history. +The [MNIST ledger](data/fresh-run-invariant-validation-mnist.jsonl) and +[Transformer ledger](data/fresh-run-invariant-validation-transformer.jsonl) retain +every rule's opportunities, group-level precondition counts, completed decisions, +visited/unvisited counts, violations, and zeros for all four checked traces. +Their run arrays use the evidence JSON's `coverage_columns`; null means not +applicable/unobserved. `fully_validated_active` means an active rule without a +requested proxy-check skip; it does not promise all candidates were checked. + +Final runs use signed harness `cad62cc98ef5fb7b313325443caa33f715ed141e`, Python +3.11.4, PyTorch `2.16.0.dev20261012+cpu`, pandas 2.3.3, and NumPy 2.1.0. Thirteen +focused counter/mutation tests pass locally and on that Linux runtime; 29 composed +production-stack tests also pass there. Both complete study commands exited 0, +with all input/source immutability and comparison gates satisfied. The +`study_completed` status means those gates passed; it does not erase the two +healthy Transformer violations. The initial workloads are benchmark-derived MNIST and a small Transformer. `benchmark_workload.py` extracts the named model definitions from a supplied @@ -34,8 +90,10 @@ python tools/fresh_validation/run_study.py \ # Use --workload transformer and omit --data-root for the language-model adapter. ``` -Each phase starts a new process, with native threads limited to one, a 600-second -timeout, an 8 GiB process-group RSS limit, and a 256 MiB API-trace limit. Inference is +Each phase starts a new process, with native threads limited to one. A monitor +polls every 250 ms and stops the process group after a 600-second timeout, +8 GiB process-group RSS threshold, or 256 MiB API-trace threshold. These are +sampled monitor thresholds, not OS hard resource limits. Inference is single-worker and disables FunctionLeadRelation/FunctionCoverRelation. These bounds and observed phase times are operational controls, not performance data. @@ -52,7 +110,11 @@ without observers must produce exactly the same CheckerResult structures. Raw process/thread IDs and frozen predicates are preserved. ID-bearing clause kinds/values and activation are reported; their mere presence does not establish that an ID caused dormancy. A workload with no ID-bearing rules leaves the -original TorchTitan rule set's portability unresolved. +original TorchTitan rule set's portability unresolved. Neither new workload has +any process/thread-ID-bearing inferred clause. The frozen `identity_bearing_*` +counters measure only PID/TID precondition clauses, not object IDs in API arguments. In particular, these results do not +validate the original 836-rule corpus's three literal-thread-ID APIContain rules +or its fourth nonliteral thread-inequality rule. A post-hoc sensitivity control selects a healthy, actually evaluated output-shape rule, or an output-dtype rule when no eligible shape rule exists. It changes one @@ -78,5 +140,5 @@ Focused counter tests: ```sh OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ - python -m unittest tests.test_fresh_validation_coverage -v + python -m unittest tests.test_fresh_validation_coverage tests.test_fresh_validation_fault -v ``` From aec748ea46b35b6b43652985f00820afa813175c Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:23:10 +0800 Subject: [PATCH 21/36] test: type pruning-state regression contracts --- .../precondition-state-isolation.md | 18 ++++ tests/test_precondition_state_isolation.py | 84 ++++++++++++------- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/docs/measurements/precondition-state-isolation.md b/docs/measurements/precondition-state-isolation.md index ca6e4c74..500393ac 100644 --- a/docs/measurements/precondition-state-isolation.md +++ b/docs/measurements/precondition-state-isolation.md @@ -98,3 +98,21 @@ uses its own unchanged candidates. [Compact reproduction evidence](precondition-state-isolation-20261025.json) records the fixture, before/after counts, source hashes, test results, and limits. + +## Engineering contract checks + +The regression fixture now has explicit record, example, solve, and recursion +observer types. The observer requires the actual recursive call parameters rather +than accepting arbitrary keyword arguments, and fresh-process comparisons check +the entire expected serialized precondition. The production fix is unchanged. + +Pinned Mypy 1.9 reports **14 errors before and zero after** on the fixture with +`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics`, +`--ignore-missing-imports`, and `--follow-imports=silent`. The last option limits +diagnostics to this fixture; it does not certify strict typing of imported legacy +modules. All six regressions pass on Python 3.10.16 / PyTorch 2.2.2 / pandas 2.2.3. +The configured hooks also pass on the production function and fixture; the hooks +exclude tests from most checks, so the explicit fixture check is necessary. + +These checks add no new performance measurement. The earlier replay results above +remain tied to their recorded source revisions. diff --git a/tests/test_precondition_state_isolation.py b/tests/test_precondition_state_isolation.py index 1da6f370..186ec7ea 100644 --- a/tests/test_precondition_state_isolation.py +++ b/tests/test_precondition_state_isolation.py @@ -7,17 +7,23 @@ import sys import unittest from pathlib import Path +from typing import Literal, TypeAlias from unittest.mock import patch import pandas as pd import traincheck.invariant.precondition as precondition from traincheck.invariant.base_cls import PT, Precondition, PreconditionClause +from traincheck.trace import Trace from traincheck.trace.trace_pandas import TracePandas +Record: TypeAlias = dict[str, object] +Example: TypeAlias = list[Record] +ObservedChild: TypeAlias = tuple[set[PreconditionClause], bool, int] -def example(phase, signal=None): - record = { + +def example(phase: str, signal: bool | None = None) -> Example: + record: Record = { "time": 0, "process_id": 1, "thread_id": 2, @@ -29,78 +35,91 @@ def example(phase, signal=None): return [record] -def trace(): +def trace() -> TracePandas: return TracePandas( pd.DataFrame({"time": [0, 1], "process_id": [1, 1], "thread_id": [2, 2]}), truncate_incomplete_func_calls=False, ) -def solve(name, **kwargs): - fixtures = { +def solve( + name: Literal["A", "B"], *, _pruned_clauses: set[PreconditionClause] | None = None +) -> list[Precondition]: + fixtures: dict[str, tuple[list[Example], list[Example]]] = { "A": ([example("train", True)], [example("train", False)]), "B": ([example("train")], [example("eval")]), } positive, negative = copy.deepcopy(fixtures[name]) return precondition.find_precondition_from_single_group( - positive, negative, [trace()], **kwargs + positive, negative, [trace()], _pruned_clauses=_pruned_clauses ) -def constant(prop, dtype, value): +def constant(prop: str, dtype: type, value: str | int | bool) -> PreconditionClause: return PreconditionClause(prop, dtype, PT.CONSTANT, None, {value}) class PreconditionStateIsolationTests(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: previous = logging.root.manager.disable logging.disable(logging.CRITICAL) self.addCleanup(logging.disable, previous) - def assert_phase_condition(self, result): + def assert_phase_condition(self, result: list[Precondition]) -> None: self.assertEqual( result, [Precondition([constant("meta_vars.phase", str, "train")])] ) - def test_prior_independent_solve_does_not_prune_required_clause(self): + def test_prior_independent_solve_does_not_prune_required_clause(self) -> None: self.assertEqual( solve("A"), [Precondition([constant("meta_vars.signal", bool, True)])], ) self.assert_phase_condition(solve("B")) - def test_reversed_independent_order_preserves_later_result(self): + def test_reversed_independent_order_preserves_later_result(self) -> None: self.assert_phase_condition(solve("B")) solve("A") self.assert_phase_condition(solve("B")) - def test_repeated_independent_solve(self): + def test_repeated_independent_solve(self) -> None: for _ in range(3): self.assert_phase_condition(solve("B")) - def test_explicit_empty_set_is_updated_and_reused(self): - state = set() + def test_explicit_empty_set_is_updated_and_reused(self) -> None: + state: set[PreconditionClause] = set() solve("A", _pruned_clauses=state) self.assertIn(constant("meta_vars.phase", str, "train"), state) # Explicit state still means to suppress these clauses in this solve. self.assertEqual(solve("B", _pruned_clauses=state), []) self.assert_phase_condition(solve("B")) - def test_recursive_siblings_share_state_but_new_roots_do_not(self): + def test_recursive_siblings_share_state_but_new_roots_do_not(self) -> None: original = precondition.find_precondition_from_single_group - root_children = [] + root_children: list[list[ObservedChild]] = [] for _ in range(2): - children = [] - - def observe(*args, **kwargs): - children.append( - ( - kwargs["_pruned_clauses"], - kwargs["_skip_pruning"], - kwargs["_current_depth"], - ) + children: list[ObservedChild] = [] + + def observe( + positive_examples: list[Example], + negative_examples: list[Example], + traces: list[Trace], + keys_to_skip: list[str], + *, + _pruned_clauses: set[PreconditionClause], + _skip_pruning: bool, + _current_depth: int, + ) -> list[Precondition]: + children.append((_pruned_clauses, _skip_pruning, _current_depth)) + return original( + positive_examples, + negative_examples, + traces, + keys_to_skip, + _pruned_clauses=_pruned_clauses, + _skip_pruning=_skip_pruning, + _current_depth=_current_depth, ) - return original(*args, **kwargs) positive = [example("train"), example("eval")] negative = [example("test")] @@ -125,8 +144,8 @@ def observe(*args, **kwargs): root_children.append(children) self.assertIsNot(root_children[0][0][0], root_children[1][0][0]) - def test_fresh_and_warm_processes_find_same_clause(self): - outputs = [] + def test_fresh_and_warm_processes_find_same_clause(self) -> None: + outputs: list[str] = [] for order in (("B",), ("A", "B"), ("B", "A", "B")): script = ( "import json\n" @@ -141,11 +160,14 @@ def test_fresh_and_warm_processes_find_same_clause(self): text=True, check=True, ) - outputs.append(json.loads(child.stdout)) + outputs.append(child.stdout.strip()) self.assertEqual(outputs[0], outputs[1]) self.assertEqual(outputs[0], outputs[2]) - self.assertEqual(len(outputs[0]), 1) - self.assertEqual(outputs[0][0]["clauses"][0]["prop_name"], "meta_vars.phase") + expected = [Precondition([constant("meta_vars.phase", str, "train")])] + self.assertEqual( + outputs[0], + json.dumps([item.to_dict() for item in expected], sort_keys=True), + ) if __name__ == "__main__": From c6385665083550ac7f3750c932cfabe54a4c0906 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:23:33 +0800 Subject: [PATCH 22/36] Type deterministic sampling and inference task contracts --- traincheck/infer_engine.py | 27 +++++++++++++------ traincheck/inference_sampling.py | 45 +++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/traincheck/infer_engine.py b/traincheck/infer_engine.py index f5958efb..d98182a5 100644 --- a/traincheck/infer_engine.py +++ b/traincheck/infer_engine.py @@ -19,8 +19,10 @@ find_precondition, relation_pool, ) +from traincheck.invariant.base_cls import GroupedPreconditions from traincheck.parallel import config_snapshot, get_worker_traces, worker_init from traincheck.trace import MDNONEJSONEncoder, select_trace_implementation +from traincheck.trace.trace import Trace from traincheck.utils import register_custom_excepthook register_custom_excepthook() @@ -28,7 +30,14 @@ logger = logging.getLogger(__name__) -def _generate_hypothesis_task(trace_idx, relation, trace): +GenerationTask = tuple[int, type[Relation]] +GenerationResult = tuple[int, type[Relation], list[Hypothesis]] +PreconditionTask = tuple[str, Hypothesis] + + +def _generate_hypothesis_task( + trace_idx: int, relation: type[Relation], trace: Trace +) -> list[Hypothesis]: """Use the same logical task RNG in serial and process-based generation.""" with sampling_task( "generate", trace_idx, relation.__module__, relation.__qualname__ @@ -36,7 +45,7 @@ def _generate_hypothesis_task(trace_idx, relation, trace): return relation.generate_hypothesis(trace) -def _hypothesis_sampling_key(hypothesis): +def _hypothesis_sampling_key(hypothesis: Hypothesis) -> str: """Identify a hypothesis before dispatch, excluding mutable results/counts.""" relation = hypothesis.invariant.relation identity = { @@ -47,13 +56,15 @@ def _hypothesis_sampling_key(hypothesis): } # Normalize JSON object keys and MD_NONE with the invariant-output encoder, # then sort keys for insertion-order independence. - normalized = json.loads(json.dumps(identity, cls=MDNONEJSONEncoder)) + normalized: object = json.loads(json.dumps(identity, cls=MDNONEJSONEncoder)) return json.dumps( normalized, sort_keys=True, separators=(",", ":"), ensure_ascii=False ) -def _find_precondition_task(hypothesis, traces, key): +def _find_precondition_task( + hypothesis: Hypothesis, traces: list[Trace], key: str +) -> GroupedPreconditions | None: with sampling_task("precondition", key): return find_precondition(hypothesis, traces) @@ -64,7 +75,7 @@ def _find_precondition_task(hypothesis, traces, key): # worker via worker_init, not re-pickled per task). -def _genhypo_worker_task(task: tuple): +def _genhypo_worker_task(task: GenerationTask) -> GenerationResult: """Generate hypotheses for one (trace_idx, relation) pair in a worker process. Returns (trace_idx, relation, inferred_hypos); the parent merges them in a @@ -76,7 +87,7 @@ def _genhypo_worker_task(task: tuple): return trace_idx, relation, hypotheses -def _precond_worker_task(task): +def _precond_worker_task(task: PreconditionTask) -> GroupedPreconditions | None: """Run precondition inference for a single hypothesis in a worker process. Returns just the precondition (picklable); the parent applies it to its own @@ -177,7 +188,7 @@ def _merge_hypotheses( def _generate_hypothesis_serial( self, - active_relations: list, + active_relations: list[type[Relation]], hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], ) -> None: @@ -218,7 +229,7 @@ def _generate_hypothesis_serial( def _generate_hypothesis_parallel( self, - active_relations: list, + active_relations: list[type[Relation]], hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], ) -> None: diff --git a/traincheck/inference_sampling.py b/traincheck/inference_sampling.py index a4e26949..1ca94b52 100644 --- a/traincheck/inference_sampling.py +++ b/traincheck/inference_sampling.py @@ -9,7 +9,32 @@ import hashlib import json import random -from contextvars import ContextVar +from collections.abc import Sequence +from contextvars import ContextVar, Token +from types import TracebackType +from typing import Literal, Protocol, TypeAlias, TypeVar, Union + +# The JSON encoder accepts tuples as arrays too; keep that existing contract. +SamplingIdentity: TypeAlias = Union[ + None, + bool, + int, + float, + str, + list["SamplingIdentity"], + tuple["SamplingIdentity", ...], + dict[str, "SamplingIdentity"], +] +_Sample = TypeVar("_Sample") + + +class SamplingRNG(Protocol): + """The shared sampling surface of random.Random and the random module.""" + + def random(self) -> float: ... + + def sample(self, population: Sequence[_Sample], k: int) -> list[_Sample]: ... + _TASK_RNG: ContextVar[random.Random | None] = ContextVar( "inference_task_rng", default=None @@ -26,10 +51,11 @@ class sampling_task: and isolates concurrently executing threads. """ - def __init__(self, *identity): + def __init__(self, *identity: SamplingIdentity) -> None: self.identity = identity + self.token: Token[random.Random | None] - def __enter__(self): + def __enter__(self) -> random.Random: payload = json.dumps( ["traincheck-inference-sampling-v1", 0, *self.identity], sort_keys=True, @@ -41,14 +67,21 @@ def __enter__(self): self.token = _TASK_RNG.set(rng) return rng - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> Literal[False]: # A class context manager also preserves frozen exception objects; the # generator-based contextlib helper may assign exception.__traceback__. _TASK_RNG.reset(self.token) return False -def sampling_rng(): +def sampling_rng() -> SamplingRNG: """Return the task's RNG, or the legacy module for standalone relation calls.""" rng = _TASK_RNG.get() - return random if rng is None else rng + if rng is None: + return random + return rng From ba5268668f46db580f002643262f55495be97308 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:25:00 +0800 Subject: [PATCH 23/36] Define reservoir worker configuration and mutation contracts --- traincheck/invariant/base_cls.py | 6 +++--- traincheck/parallel.py | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/traincheck/invariant/base_cls.py b/traincheck/invariant/base_cls.py index 59ddd2c3..d2f340d2 100644 --- a/traincheck/invariant/base_cls.py +++ b/traincheck/invariant/base_cls.py @@ -1685,10 +1685,10 @@ class ExampleList: uniform joint sample across lists. """ - def __init__(self, group_names: set[str]): + def __init__(self, group_names: set[str]) -> None: self.group_names = group_names self.examples: list[Example] = [] - self.max_size = config.MAX_EXAMPLE_LIST_SIZE + self.max_size: int | None = config.MAX_EXAMPLE_LIST_SIZE self._validate_cap(self.max_size) self.num_seen = 0 self._rng = random.Random(config.EXAMPLE_SAMPLING_SEED) @@ -1719,7 +1719,7 @@ def _validate_state(self) -> None: f"retained {len(self.examples)}, expected {expected} for {self.num_seen} seen" ) - def add_example(self, example: Example): + def add_example(self, example: Example) -> None: if len(self.group_names) == 0: assert len(self.examples) == 0 self.group_names = example.get_group_names() diff --git a/traincheck/parallel.py b/traincheck/parallel.py index b1c35dac..943922ef 100644 --- a/traincheck/parallel.py +++ b/traincheck/parallel.py @@ -13,13 +13,26 @@ access the traces through ``get_worker_traces()``. """ +from typing import TypedDict + import traincheck.config.config as config import traincheck.utils as _tc_utils +from traincheck.trace.trace import Trace + + +class InferenceConfigSnapshot(TypedDict): + """Required runtime settings transferred between our parent and workers.""" + + ENABLE_PRECOND_SAMPLING: bool + PRECOND_SAMPLING_THRESHOLD: int + MAX_EXAMPLE_LIST_SIZE: int | None + EXAMPLE_SAMPLING_SEED: int + -_WORKER_TRACES: list | None = None +_WORKER_TRACES: list[Trace] | None = None -def config_snapshot() -> dict: +def config_snapshot() -> InferenceConfigSnapshot: """Runtime inference config values that must be propagated to workers. These include CLI sampling options and programmatic reservoir settings. @@ -35,7 +48,7 @@ def config_snapshot() -> dict: } -def worker_init(traces: list, snapshot: dict) -> None: +def worker_init(traces: list[Trace], snapshot: InferenceConfigSnapshot) -> None: """Initializer run once per worker process. Stores the traces in a module global (one copy per worker, not per task) and @@ -50,7 +63,7 @@ def worker_init(traces: list, snapshot: dict) -> None: _tc_utils._suppress_inner_progress = True -def get_worker_traces() -> list: +def get_worker_traces() -> list[Trace]: """Return the traces stored by worker_init; only valid inside a worker.""" assert _WORKER_TRACES is not None, "Worker traces were not initialized" return _WORKER_TRACES From aba4ee8e164f4d24eedae97ea641cd9a6f69c343 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:26:51 +0800 Subject: [PATCH 24/36] Type tracing suppression and buffer failure contracts --- tests/test_tracing_contracts.py | 73 +++++++++++++++++++++++++++++++ traincheck/instrumentor/dumper.py | 19 ++++---- traincheck/instrumentor/tracer.py | 34 ++++++++++---- 3 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 tests/test_tracing_contracts.py diff --git a/tests/test_tracing_contracts.py b/tests/test_tracing_contracts.py new file mode 100644 index 00000000..1564692d --- /dev/null +++ b/tests/test_tracing_contracts.py @@ -0,0 +1,73 @@ +"""Failure contracts for recorder state and buffered writes.""" + +import threading +import unittest +from queue import Queue +from unittest.mock import patch + +from traincheck.instrumentor import dumper, tracer + + +class TracingContractTests(unittest.TestCase): + def test_application_scope_restores_suppression_after_exception(self) -> None: + error = ValueError("application failure") + with tracer.suppress_tracing(): + with self.assertRaises(ValueError) as caught: + with tracer.application_tracing(): + self.assertFalse(tracer.tracing_suppressed()) + raise error + self.assertIs(caught.exception, error) + self.assertTrue(tracer.tracing_suppressed()) + self.assertFalse(tracer.tracing_suppressed()) + + def test_new_thread_initializes_independent_state(self) -> None: + observations: list[bool] = [] + + def observe() -> None: + observations.append(tracer.tracing_suppressed()) + with tracer.suppress_tracing(): + observations.append(tracer.tracing_suppressed()) + observations.append(tracer.tracing_suppressed()) + + with tracer.suppress_tracing(): + thread = threading.Thread(target=observe) + thread.start() + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertTrue(tracer.tracing_suppressed()) + self.assertEqual(observations, [False, True, False]) + self.assertFalse(tracer.tracing_suppressed()) + + def test_failed_enqueue_retains_batch_and_propagates(self) -> None: + queue: Queue[str] = Queue() + with patch.object(threading.Thread, "start"): + buffer = dumper.TraceBuffer(lambda: queue, buffer_size=100) + buffer.add_trace({"id": 1}) + error = RuntimeError("queue unavailable") + with patch.object(queue, "put", side_effect=error): + with self.assertRaises(RuntimeError) as caught: + buffer._flush() + self.assertIs(caught.exception, error) + buffer._flush() + self.assertEqual(queue.get_nowait(), '{"id":1}') + self.assertTrue(queue.empty()) + buffer._flush() + self.assertTrue(queue.empty()) + + def test_serialization_failure_preserves_existing_batch(self) -> None: + queue: Queue[str] = Queue() + with patch.object(threading.Thread, "start"): + buffer = dumper.TraceBuffer(lambda: queue, buffer_size=100) + buffer.add_trace({"id": 1}) + error = TypeError("unsupported record") + with patch.object(dumper, "serialize", side_effect=error): + with self.assertRaises(TypeError) as caught: + buffer.add_trace({"id": object()}) + self.assertIs(caught.exception, error) + buffer._flush() + self.assertEqual(queue.get_nowait(), '{"id":1}') + self.assertTrue(queue.empty()) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index 1bf7120f..5a852d33 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -4,7 +4,7 @@ import threading import time from queue import Empty, Queue -from typing import Hashable +from typing import Callable, Hashable import orjson import torch @@ -188,19 +188,22 @@ class TraceBuffer: """ def __init__( - self, queue_getter, buffer_size=BUFFER_SIZE, flush_interval=FLUSH_INTERVAL - ): + self, + queue_getter: Callable[[], Queue[str]], + buffer_size: int = BUFFER_SIZE, + flush_interval: float = FLUSH_INTERVAL, + ) -> None: self.queue = queue_getter() self.buffer_size = buffer_size self.flush_interval = flush_interval - self.buffer = [] + self.buffer: list[str] = [] self._lock = threading.Lock() self.last_flush_time = time.time() self.flush_thread = threading.Thread(target=self._flush_periodically) self.flush_thread.daemon = True self.flush_thread.start() - def add_trace(self, trace): + def add_trace(self, trace: dict[str, object]) -> None: # Snapshot mutable values in the producer, before returning to the caller. serialized = serialize(trace) with self._lock: @@ -211,11 +214,11 @@ def add_trace(self, trace): ): self._flush_locked() - def _flush(self): + def _flush(self) -> None: with self._lock: self._flush_locked() - def _flush_locked(self): + def _flush_locked(self) -> None: if not self.buffer: return # Keep enqueue and clear atomic with respect to the producer and timer. @@ -224,7 +227,7 @@ def _flush_locked(self): self.buffer.clear() self.last_flush_time = time.time() - def _flush_periodically(self): + def _flush_periodically(self) -> None: while not stop_event.is_set(): time.sleep(self.flush_interval) self._flush() diff --git a/traincheck/instrumentor/tracer.py b/traincheck/instrumentor/tracer.py index 33c8f05e..6a6f11c0 100644 --- a/traincheck/instrumentor/tracer.py +++ b/traincheck/instrumentor/tracer.py @@ -10,7 +10,7 @@ import types from contextlib import ContextDecorator from importlib.util import find_spec -from typing import Callable, Optional +from typing import Callable, Literal, Optional import torch @@ -54,7 +54,16 @@ COLLECT_OVERHEAD_METRICS = os.environ.get("COLLECT_OVERHEAD_METRICS", "0") == "1" -THREAD_DATA = threading.local() +class _TracingThreadState(threading.local): + # IDs remain lazy, matching get_process_thread_id's existing behavior. + thread_id: int + process_id: int + + def __init__(self) -> None: + self.suppression_depth: int = 0 + + +THREAD_DATA = _TracingThreadState() class suppress_tracing(ContextDecorator): @@ -65,16 +74,23 @@ class suppress_tracing(ContextDecorator): Never hold this scope over the original application call. """ - def _recreate_cm(self): + depth: int + + def _recreate_cm(self) -> "suppress_tracing": # Decorated conversions may nest or run concurrently on different threads. return type(self)() - def __enter__(self): - self.depth = getattr(THREAD_DATA, "suppression_depth", 0) + def __enter__(self) -> "suppress_tracing": + self.depth = THREAD_DATA.suppression_depth THREAD_DATA.suppression_depth = self.depth + 1 return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> Literal[False]: THREAD_DATA.suppression_depth = self.depth return False @@ -82,14 +98,14 @@ def __exit__(self, exc_type, exc_value, traceback): class application_tracing(suppress_tracing): """Restore application tracing while advancing a user-supplied generator.""" - def __enter__(self): + def __enter__(self) -> "application_tracing": super().__enter__() THREAD_DATA.suppression_depth = 0 return self -def tracing_suppressed(): - return getattr(THREAD_DATA, "suppression_depth", 0) > 0 +def tracing_suppressed() -> bool: + return THREAD_DATA.suppression_depth > 0 class TraceLineType: From 336fc1b4fa3f16c3c5b7b0c9b83ae402c986f4a7 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:27:20 +0800 Subject: [PATCH 25/36] Specify context index and benchmark data contracts --- tools/benchmarks/trace_pandas_indexes.py | 43 ++++++++++++++++-------- traincheck/trace/trace_pandas.py | 29 ++++++++++++---- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/tools/benchmarks/trace_pandas_indexes.py b/tools/benchmarks/trace_pandas_indexes.py index a9ff2fbe..6f29b7de 100644 --- a/tools/benchmarks/trace_pandas_indexes.py +++ b/tools/benchmarks/trace_pandas_indexes.py @@ -14,7 +14,9 @@ import platform import statistics import time +from collections.abc import Callable from pathlib import Path +from typing import TypedDict import numpy as np import pandas as pd @@ -23,19 +25,34 @@ from tests.fixtures import legacy_trace_pandas as legacy from tests.test_call_index import pair from tests.test_context_index import cycle, encode, event +from traincheck.trace.trace_pandas import TracePandas -def compare(frame, operation, repetitions): - seconds = {"legacy": [], "candidate": []} - expected = None +class ComparisonResult(TypedDict): + elapsed_seconds: dict[str, list[float]] + median_seconds: dict[str, float] + speedup: float + exact_results_equal: bool + + +def compare( + frame: pd.DataFrame, + operation: Callable[[type[TracePandas], pd.DataFrame], object], + repetitions: int, +) -> ComparisonResult: + seconds: dict[str, list[float]] = {"legacy": [], "candidate": []} + expected: object = None for repetition in range(repetitions): - variants = [("legacy", legacy), ("candidate", candidate)] + variants = [ + ("legacy", legacy.TracePandas), + ("candidate", candidate.TracePandas), + ] if repetition % 2: variants.reverse() - for name, module in variants: + for name, trace_type in variants: events = frame.copy(deep=True) started = time.perf_counter() - value = operation(module, events) + value = operation(trace_type, events) seconds[name].append(time.perf_counter() - started) if expected is None: expected = value @@ -49,7 +66,7 @@ def compare(frame, operation, repetitions): } -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--mode", choices=["all", "context", "call"], default="all") parser.add_argument("--repetitions", type=int, default=3) @@ -67,7 +84,7 @@ def main(): if args.noise < 0: parser.error("--noise must be nonnegative") logging.disable(logging.WARNING) - results = {} + results: dict[str, object] = {} if args.mode in ("all", "context"): rows = [] for index in range(args.contexts): @@ -79,10 +96,8 @@ def main(): ) frame = pd.DataFrame(rows) - def prepare(module, events): - return encode( - module.TracePandas(events, truncate_incomplete_func_calls=False) - ) + def prepare(trace_type: type[TracePandas], events: pd.DataFrame) -> object: + return encode(trace_type(events, truncate_incomplete_func_calls=False)) results["context"] = { "rows": len(frame), @@ -102,8 +117,8 @@ def prepare(module, events): f"call-{(index * 117) % args.calls:08}" for index in range(args.queries) ] - def lookup(module, events): - trace = module.TracePandas(events, truncate_incomplete_func_calls=False) + def lookup(trace_type: type[TracePandas], events: pd.DataFrame) -> object: + trace = trace_type(events, truncate_incomplete_func_calls=False) return [ ( trace.get_pre_func_call_record(call_id), diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 2ab42924..61e933ee 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -71,7 +71,7 @@ def __init__(self, events, truncate_incomplete_func_calls=True): ] = {} self.all_func_call_ids: dict[str, list[str]] = {} - self.context_manager_states: dict[PTID, list[ContextManagerState]] + self.context_manager_states: dict[PTID, list[ContextManagerState]] | None self.cache_high_level_events_within_id: dict[ str, list[FuncCallEvent | FuncCallExceptionEvent | VarChangeEvent] ] = {} @@ -293,12 +293,14 @@ def _rm_incomplete_trailing_func_calls(self): # test_dump(self.events) - def _index_context_manager_meta_vars(self): + def _index_context_manager_meta_vars( + self, + ) -> dict[PTID, list[ContextManagerState]] | None: """Identify context manager entry and exit events, and add them to the meta_vars.""" # 1. Find all trace records that are related to __enter__ and __exit__ functions if "function" not in self.events.columns: self.context_manager_states = None - return + return None if ( hasattr(self, "context_manager_states") @@ -330,7 +332,7 @@ def _index_context_manager_meta_vars(self): ] if not init_functions: self.context_manager_states = all_context_managers - return + return None init_records = self.events[ self.events["function"].isin(init_functions) @@ -347,7 +349,7 @@ def _index_context_manager_meta_vars(self): ] if not context_manager_init_pre_records: self.context_manager_states = all_context_managers - return + return None context_names = [ name.removesuffix(".__enter__") for name in context_manager_names @@ -368,7 +370,9 @@ def _index_context_manager_meta_vars(self): ), group_columns + ["time"], ] - event_times = {} + event_times: dict[ + tuple[object, object, object, object, object], tuple[pd.Series, bool] + ] = {} for key, records in matching_events.groupby( group_columns, sort=False, dropna=True, observed=True ): @@ -385,7 +389,17 @@ def _index_context_manager_meta_vars(self): ) event_times[key] = (times, ordered) - def first_time_after(function, event_type, process_id, thread_id, obj_id, time): + # pandas object/extension columns may return arbitrary comparable scalar + # types. Keep Any only for this legacy comparison boundary: narrowing to + # float/int (or coercing) changes nullable, UInt64, and object semantics. + def first_time_after( + function: str, + event_type: str, + process_id: object, + thread_id: object, + obj_id: object, + time: Any, + ) -> Any: if not isinstance(time, Real): # Keep pandas validation for nonnumeric/null scalar thresholds, # including malformed nullable timestamps, even with no group. @@ -477,6 +491,7 @@ def first_time_after(function, event_type, process_id, thread_id, obj_id, time): logger.info(f"Found {len(all_context_managers)} context managers.") self.context_manager_states = all_context_managers + return None def is_stage_annotated(self): # ideally we want to have a static manifest for the trace produced by the instrumentor according to the args From c125d060e5f6461b89ffc8cb6542015ed9ff9f7c Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:29:34 +0800 Subject: [PATCH 26/36] Describe schemaless identity values and benchmark results honestly --- tools/benchmarks/identity_cache.py | 78 +++++++++++++++++++++++------- traincheck/trace/trace.py | 9 ++-- traincheck/trace/trace_pandas.py | 9 ++-- 3 files changed, 70 insertions(+), 26 deletions(-) diff --git a/tools/benchmarks/identity_cache.py b/tools/benchmarks/identity_cache.py index 3a82e1d2..e42582e9 100644 --- a/tools/benchmarks/identity_cache.py +++ b/tools/benchmarks/identity_cache.py @@ -32,6 +32,7 @@ import sys import time from pathlib import Path +from typing import Literal, TypedDict import numpy as np import pandas as pd @@ -39,22 +40,46 @@ import traincheck.trace.trace_pandas as trace_module from traincheck.trace.trace_pandas import TracePandas +TimingField = Literal["construction_seconds", "queries_seconds", "total_seconds"] +TIMING_FIELDS: tuple[TimingField, ...] = ( + "construction_seconds", + "queries_seconds", + "total_seconds", +) + + +class TrialResult(TypedDict): + repetition: int + position: int + variant: str + construction_seconds: float + queries_seconds: float + total_seconds: float + result_sha256: str + none_results: int + empty_dict_results: int + + +class Measurement(TrialResult, total=False): + cached_identity_counts: dict[str, int] + cache_tuple_containers_shallow_bytes: int + class LegacyIdentityTrace(TracePandas): """Original lookup expressions; all other behavior uses the same checkout.""" - def get_process_ids(self): + def get_process_ids(self) -> list[object]: if "process_id" not in self.events.columns: return [] return self.events["process_id"].dropna().unique().tolist() - def get_thread_ids(self): + def get_thread_ids(self) -> list[object]: if "thread_id" not in self.events.columns: return [] return self.events["thread_id"].dropna().unique().tolist() -def synthetic_frame(rows): +def synthetic_frame(rows: int) -> pd.DataFrame: sequence = np.arange(rows, dtype=np.int64) process_ids = ((sequence // 16) % 8 + 100).astype(np.float64) thread_ids = (sequence % 16 + 1000).astype(np.float64) @@ -67,7 +92,7 @@ def synthetic_frame(rows): ) -def fixed_queries(calls): +def fixed_queries(calls: int) -> list[tuple[int, int, int]]: # Six valid, one unknown-process, and one unknown-thread call per cycle. # Process/thread presence remains independent, matching get_meta_vars. return [ @@ -80,20 +105,31 @@ def fixed_queries(calls): ] -def digest(value): +def digest(value: object) -> str: return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() -def trial(cls, data, queries): +def trial( + cls: type[TracePandas], + data: pd.DataFrame, + queries: list[tuple[int, int, int]], + *, + repetition: int, + position: int, + variant: str, +) -> Measurement: gc.collect() begin = time.perf_counter() trace = cls(data, truncate_incomplete_func_calls=False) constructed = time.perf_counter() results = [trace.get_meta_vars(*query) for query in queries] finished = time.perf_counter() - record = { + record: Measurement = { + "repetition": repetition, + "position": position, + "variant": variant, "construction_seconds": constructed - begin, "queries_seconds": finished - constructed, "total_seconds": finished - begin, @@ -112,7 +148,7 @@ def trial(cls, data, queries): return record -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--rows", type=int, default=573494) parser.add_argument("--calls", type=int, default=1000) @@ -129,20 +165,26 @@ def main(): input_hash = hashlib.sha256( pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() ).hexdigest() - expected = [None if i % 8 in (6, 7) else {} for i in range(args.calls)] + expected: list[dict[str, object] | None] = [ + None if i % 8 in (6, 7) else {} for i in range(args.calls) + ] expected_hash = digest(expected) - measurements = [] + measurements: list[Measurement] = [] variants = {"legacy": LegacyIdentityTrace, "candidate": TracePandas} for repetition in range(args.repetitions): order = ["legacy", "candidate"] if repetition % 2: order.reverse() for position, variant in enumerate(order): - record = trial(variants[variant], data, queries) - assert record["result_sha256"] == expected_hash, record - record.update( - repetition=repetition + 1, position=position + 1, variant=variant + record = trial( + variants[variant], + data, + queries, + repetition=repetition + 1, + position=position + 1, + variant=variant, ) + assert record["result_sha256"] == expected_hash, record measurements.append(record) # Catch accidental mutation of the shared input by either implementation. @@ -159,7 +201,7 @@ def main(): for record in measurements if record["variant"] == variant ) - for metric in ("construction_seconds", "queries_seconds", "total_seconds") + for metric in TIMING_FIELDS } for variant in variants } @@ -191,13 +233,13 @@ def main(): ) }, "candidate_source_sha256": hashlib.sha256( - Path(trace_module.__file__).read_bytes() + Path(inspect.getfile(trace_module)).read_bytes() ).hexdigest(), "candidate_commit": subprocess.check_output( [ "git", "-C", - str(Path(trace_module.__file__).resolve().parents[2]), + str(Path(inspect.getfile(trace_module)).resolve().parents[2]), "rev-parse", "HEAD", ], @@ -207,7 +249,7 @@ def main(): [ "git", "-C", - str(Path(trace_module.__file__).resolve().parents[2]), + str(Path(inspect.getfile(trace_module)).resolve().parents[2]), "diff", "--quiet", "HEAD", diff --git a/traincheck/trace/trace.py b/traincheck/trace/trace.py index c04ebeda..d5636d29 100644 --- a/traincheck/trace/trace.py +++ b/traincheck/trace/trace.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Sequence import polars as pl @@ -183,14 +184,14 @@ def get_end_time(self, process_id=None, thread_id=None) -> float: "This function should be implemented in the child class." ) - def get_process_ids(self) -> list[int]: - """Find all process ids from the trace.""" + def get_process_ids(self) -> Sequence[object]: + """Find process identities, preserving the backend scalar types.""" raise NotImplementedError( "This function should be implemented in the child class." ) - def get_thread_ids(self) -> list[int]: - """Find all thread ids from the trace.""" + def get_thread_ids(self) -> Sequence[object]: + """Find thread identities, preserving the backend scalar types.""" raise NotImplementedError( "This function should be implemented in the child class." ) diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index d5e93acc..bedcfd9d 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -115,7 +115,7 @@ def __init__(self, events, truncate_incomplete_func_calls=True): # Keep the frame alive so object identity cannot be reused across frames. self._identity_values_source: pd.DataFrame | None = None self._identity_values_length = -1 - self._identity_values: dict[str, tuple] = {} + self._identity_values: dict[str, tuple[object, ...]] = {} self.column_dtypes_cached = {} self.stage_traces: dict[str, "TracePandas"] = {} @@ -594,12 +594,13 @@ def get_end_time(self, process_id=None, thread_id=None) -> float: return self.events["time"].max() - def _get_identity_values(self, column: str) -> list: + def _get_identity_values(self, column: str) -> list[object]: """Cache stable identity columns; return a fresh list for every caller. As with call-record indexes, frame replacement or length changes rebuild the cache. Same-size in-place identity edits, row reordering, or dtype changes require a new TracePandas. + Identity columns are schemaless: their scalars are not necessarily ints. Keep pandas' dropna/unique conversion to preserve dtype and encounter order instead of replacing it with Python set semantics. """ @@ -616,7 +617,7 @@ def _get_identity_values(self, column: str) -> list: ) return list(self._identity_values[column]) - def get_process_ids(self) -> list[int]: + def get_process_ids(self) -> list[object]: """Find all process ids from the trace.""" if "process_id" not in self.events.columns: logger.warning( @@ -625,7 +626,7 @@ def get_process_ids(self) -> list[int]: return [] return self._get_identity_values("process_id") - def get_thread_ids(self) -> list[int]: + def get_thread_ids(self) -> list[object]: """Find all thread ids from the trace.""" if "thread_id" not in self.events.columns: logger.warning( From fa4e498e5bcfba9844081ca9b077a9b9ce533be2 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:32:42 +0800 Subject: [PATCH 27/36] Validate complete worker snapshots before changing process state --- tests/test_parallel_config.py | 77 +++++++++++++++++++++++++++++++++++ traincheck/parallel.py | 63 +++++++++++++++++++++++----- 2 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 tests/test_parallel_config.py diff --git a/tests/test_parallel_config.py b/tests/test_parallel_config.py new file mode 100644 index 00000000..6e534669 --- /dev/null +++ b/tests/test_parallel_config.py @@ -0,0 +1,77 @@ +"""Worker settings are required and validated before any process state changes.""" + +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +import traincheck.config.config as config +import traincheck.parallel as parallel +import traincheck.utils as utils +from traincheck.trace.trace import Trace + + +class WorkerConfigTests(unittest.TestCase): + def setUp(self) -> None: + stack = ExitStack() + self.addCleanup(stack.close) + stack.enter_context(patch.multiple(config, **parallel.config_snapshot())) + self.original_traces: list[Trace] = [] + stack.enter_context( + patch.object(parallel, "_WORKER_TRACES", self.original_traces) + ) + stack.enter_context(patch.object(utils, "_suppress_inner_progress", False)) + + def assert_rejected_unchanged( + self, snapshot: dict[str, object], error: type[Exception] + ) -> None: + before = parallel.config_snapshot() + with self.assertRaises(error): + parallel.worker_init([], snapshot) + self.assertEqual(parallel.config_snapshot(), before) + self.assertIs(parallel.get_worker_traces(), self.original_traces) + self.assertFalse(utils._suppress_inner_progress) + + def test_all_fields_are_required(self) -> None: + for field in parallel.config_snapshot(): + snapshot: dict[str, object] = dict(parallel.config_snapshot()) + del snapshot[field] + with self.subTest(field=field): + self.assert_rejected_unchanged(snapshot, KeyError) + + def test_unknown_fields_are_rejected(self) -> None: + snapshot: dict[str, object] = dict(parallel.config_snapshot()) + snapshot["UNRECOGNIZED_SETTING"] = 1 + self.assert_rejected_unchanged(snapshot, ValueError) + + def test_malformed_fields_leave_all_state_unchanged(self) -> None: + invalid: dict[str, list[object]] = { + "ENABLE_PRECOND_SAMPLING": [1, None, "true"], + "PRECOND_SAMPLING_THRESHOLD": [True, 1.5, "100"], + "MAX_EXAMPLE_LIST_SIZE": [True, -1, 2.5, "10"], + "EXAMPLE_SAMPLING_SEED": [object(), {}, []], + } + for field, values in invalid.items(): + for value in values: + snapshot: dict[str, object] = dict(parallel.config_snapshot()) + snapshot[field] = value + with self.subTest(field=field, value=value): + self.assert_rejected_unchanged(snapshot, ValueError) + + def test_supported_seeds_and_uncapped_or_zero_capacity(self) -> None: + for seed in (None, 9, -8, 1.5, "seed", b"seed", bytearray(b"seed")): + for capacity in (None, 0, 7): + snapshot: dict[str, object] = { + "ENABLE_PRECOND_SAMPLING": False, + "PRECOND_SAMPLING_THRESHOLD": 0, + "MAX_EXAMPLE_LIST_SIZE": capacity, + "EXAMPLE_SAMPLING_SEED": seed, + } + traces: list[Trace] = [] + parallel.worker_init(traces, snapshot) + self.assertEqual(parallel.config_snapshot(), snapshot) + self.assertIs(parallel.get_worker_traces(), traces) + self.assertTrue(utils._suppress_inner_progress) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/parallel.py b/traincheck/parallel.py index 943922ef..218d40d9 100644 --- a/traincheck/parallel.py +++ b/traincheck/parallel.py @@ -13,12 +13,15 @@ access the traces through ``get_worker_traces()``. """ -from typing import TypedDict +from collections.abc import Mapping +from typing import TypeAlias, TypedDict import traincheck.config.config as config import traincheck.utils as _tc_utils from traincheck.trace.trace import Trace +RandomSeed: TypeAlias = int | float | str | bytes | bytearray | None + class InferenceConfigSnapshot(TypedDict): """Required runtime settings transferred between our parent and workers.""" @@ -26,7 +29,7 @@ class InferenceConfigSnapshot(TypedDict): ENABLE_PRECOND_SAMPLING: bool PRECOND_SAMPLING_THRESHOLD: int MAX_EXAMPLE_LIST_SIZE: int | None - EXAMPLE_SAMPLING_SEED: int + EXAMPLE_SAMPLING_SEED: RandomSeed _WORKER_TRACES: list[Trace] | None = None @@ -48,18 +51,58 @@ def config_snapshot() -> InferenceConfigSnapshot: } -def worker_init(traces: list[Trace], snapshot: InferenceConfigSnapshot) -> None: - """Initializer run once per worker process. +def _validate_snapshot(snapshot: Mapping[str, object]) -> InferenceConfigSnapshot: + """Reject incomplete/corrupt worker settings before changing process state.""" + required = { + "ENABLE_PRECOND_SAMPLING", + "PRECOND_SAMPLING_THRESHOLD", + "MAX_EXAMPLE_LIST_SIZE", + "EXAMPLE_SAMPLING_SEED", + } + unexpected = set(snapshot) - required + if unexpected: + raise ValueError(f"Unexpected inference settings: {sorted(unexpected)}") + # Direct access makes every field required; missing fields retain KeyError. + enabled = snapshot["ENABLE_PRECOND_SAMPLING"] + threshold = snapshot["PRECOND_SAMPLING_THRESHOLD"] + capacity = snapshot["MAX_EXAMPLE_LIST_SIZE"] + seed = snapshot["EXAMPLE_SAMPLING_SEED"] + if type(enabled) is not bool: # noqa: E721 - booleans are not integer settings + raise ValueError("ENABLE_PRECOND_SAMPLING must be a bool") + if type(threshold) is not int: # noqa: E721 - match the integer CLI contract + raise ValueError("PRECOND_SAMPLING_THRESHOLD must be an integer") + if capacity is not None and ( + type(capacity) is not int or capacity < 0 # noqa: E721 - reservoir contract + ): + raise ValueError("MAX_EXAMPLE_LIST_SIZE must be a nonnegative integer or None") + # Preserve the documented random.Random seed types, including None and bytes. + if seed is not None and not isinstance(seed, (int, float, str, bytes, bytearray)): + raise ValueError("EXAMPLE_SAMPLING_SEED must be a supported random.Random seed") + return { + "ENABLE_PRECOND_SAMPLING": enabled, + "PRECOND_SAMPLING_THRESHOLD": threshold, + "MAX_EXAMPLE_LIST_SIZE": capacity, + "EXAMPLE_SAMPLING_SEED": seed, + } + + +def worker_init(traces: list[Trace], snapshot: Mapping[str, object]) -> None: + """Validate required settings, then initialize this worker atomically. - Stores the traces in a module global (one copy per worker, not per task) and - restores runtime inference config values (see config_snapshot). Also suppresses - the per-invariant inner progress bars, otherwise every worker would fight - over the terminal. + A malformed snapshot must not install some settings or replace worker traces. + The parent supplies config_snapshot(); accepting a Mapping also lets this + process boundary validate deserialized or otherwise untrusted payloads. """ + settings = _validate_snapshot(snapshot) global _WORKER_TRACES + config.ENABLE_PRECOND_SAMPLING = settings["ENABLE_PRECOND_SAMPLING"] + config.PRECOND_SAMPLING_THRESHOLD = settings["PRECOND_SAMPLING_THRESHOLD"] + # The legacy config module infers int from defaults; programmatic reservoir + # options also support None and the other stdlib seed types. Fixed attribute + # names avoid pretending arbitrary keys are supported configuration fields. + setattr(config, "MAX_EXAMPLE_LIST_SIZE", settings["MAX_EXAMPLE_LIST_SIZE"]) + setattr(config, "EXAMPLE_SAMPLING_SEED", settings["EXAMPLE_SAMPLING_SEED"]) _WORKER_TRACES = traces - for name, value in snapshot.items(): - setattr(config, name, value) _tc_utils._suppress_inner_progress = True From d4ebb12e3c5fd7531080d5623447f2af0a1ab3e9 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:32:52 +0800 Subject: [PATCH 28/36] Enforce TorchScript compatibility metadata contracts --- tests/test_torchscript_compat.py | 10 +- tests/test_torchscript_contracts.py | 161 ++++++++++++++++++ traincheck/instrumentor/torchscript_compat.py | 80 +++++++-- traincheck/instrumentor/tracer.py | 58 ++++--- 4 files changed, 270 insertions(+), 39 deletions(-) create mode 100644 tests/test_torchscript_contracts.py diff --git a/tests/test_torchscript_compat.py b/tests/test_torchscript_compat.py index 05f2b8bb..8d6c7937 100644 --- a/tests/test_torchscript_compat.py +++ b/tests/test_torchscript_compat.py @@ -47,16 +47,16 @@ def evaluate(function): } -def probe(mode): +def probe(mode: str) -> dict[str, object]: """Run one control in its own interpreter so JIT caches cannot leak.""" from traincheck.instrumentor import tracer torch.set_num_threads(1) torch.set_num_interop_threads(1) - records = [] + records: list[dict[str, object]] = [] registry = torch.jit._builtins original = torch.tanh - result = { + result: dict[str, object] = { "mode": mode, "torch": torch.__version__, "initial_registry_initialized": registry._builtin_table is not None, @@ -87,7 +87,9 @@ def probe(mode): # Disable only registration to reproduce the former wrapper; # the helper controls then apply the production helper directly. with patch.object( - tracer, "preserve_torchscript_builtin", return_value=None + tracer, + "preserve_torchscript_builtin", + return_value={"status": "not-registered", "operator": None}, ): wrapped = tracer.wrapper( original, diff --git a/tests/test_torchscript_contracts.py b/tests/test_torchscript_contracts.py new file mode 100644 index 00000000..fa841ed1 --- /dev/null +++ b/tests/test_torchscript_contracts.py @@ -0,0 +1,161 @@ +"""Explicit unsupported, malformed, and failed TorchScript boundary contracts.""" + +import threading +import unittest +import warnings +from types import SimpleNamespace +from typing import Callable +from unittest.mock import patch + +from traincheck.instrumentor import torchscript_compat, tracer +from traincheck.instrumentor.torchscript_compat import preserve_torchscript_builtin + + +def original(value: int = 1) -> int: + return value + 1 + + +def candidate(value: int = 1) -> int: + return original(value) + + +def framework(find: object, register: object) -> SimpleNamespace: + return SimpleNamespace( + jit=SimpleNamespace( + _builtins=SimpleNamespace(_find_builtin=find, _register_builtin=register) + ) + ) + + +class TorchScriptContractTests(unittest.TestCase): + def test_present_noncallable_callback_is_rejected(self) -> None: + def find(function: Callable[..., object]) -> str: + return "aten::tanh" + + def register(function: Callable[..., object], operator: str) -> None: + self.fail("Invalid capabilities must not register a wrapper") + + malformed_callbacks: tuple[object, ...] = (None, 17, "callable", []) + for malformed in malformed_callbacks: + for name, fake in ( + ("_find_builtin", framework(malformed, register)), + ("_register_builtin", framework(find, malformed)), + ): + with self.subTest(name=name, malformed=malformed): + with self.assertRaisesRegex(TypeError, name): + preserve_torchscript_builtin(original, candidate, fake) + + def test_missing_callback_does_not_hide_an_invalid_present_callback(self) -> None: + fake = SimpleNamespace( + jit=SimpleNamespace(_builtins=SimpleNamespace(_register_builtin=None)) + ) + with self.assertRaisesRegex(TypeError, "_register_builtin"): + preserve_torchscript_builtin(original, candidate, fake) + + def test_malformed_operator_is_rejected_before_registration(self) -> None: + registered: list[str] = [] + + def register(function: Callable[..., object], operator: str) -> None: + registered.append(operator) + + malformed_operators: tuple[object, ...] = ("", 1, False, [], {}) + for malformed in malformed_operators: + + def find(function: Callable[..., object]) -> object: + return malformed + + with self.subTest(operator=malformed): + with self.assertRaisesRegex(TypeError, "nonempty string or None"): + preserve_torchscript_builtin( + original, candidate, framework(find, register) + ) + self.assertEqual(registered, []) + self.assertNotIn(candidate, torchscript_compat._REGISTERED_WRAPPERS) + + def test_find_failure_propagates_unchanged(self) -> None: + error = RuntimeError("registry lookup failed") + + def find(function: Callable[..., object]) -> str | None: + raise error + + def register(function: Callable[..., object], operator: str) -> None: + self.fail("Failed lookup must not register a wrapper") + + with self.assertRaises(RuntimeError) as caught: + preserve_torchscript_builtin(original, candidate, framework(find, register)) + self.assertIs(caught.exception, error) + self.assertNotIn(candidate, torchscript_compat._REGISTERED_WRAPPERS) + + def test_registration_failure_propagates_without_retention(self) -> None: + error = RuntimeError("registry update failed") + + def find(function: Callable[..., object]) -> str: + return "aten::tanh" + + def register(function: Callable[..., object], operator: str) -> None: + raise error + + with self.assertRaises(RuntimeError) as caught: + preserve_torchscript_builtin(original, candidate, framework(find, register)) + self.assertIs(caught.exception, error) + self.assertNotIn(candidate, torchscript_compat._REGISTERED_WRAPPERS) + + def test_wrapper_exposes_unsupported_and_warns_once(self) -> None: + with ( + patch.object(tracer, "torch", SimpleNamespace()), + patch.object(torchscript_compat, "_UNSUPPORTED_WARNING_EMITTED", False), + warnings.catch_warnings(record=True) as captured, + ): + warnings.simplefilter("always") + first = tracer.wrapper( + original, False, False, False, disable_dump=True, handle_proxy=False + ) + second = tracer.wrapper( + original, False, False, False, disable_dump=True, handle_proxy=False + ) + self.assertEqual(first(3), 4) + self.assertEqual(second(8), 9) + self.assertEqual(len(captured), 1) + self.assertIn( + "Eager instrumentation remains available", str(captured[0].message) + ) + for wrapped in (first, second): + # Function metadata is deliberately public to instrumentation users. + self.assertEqual( + getattr(wrapped, "_traincheck_torchscript_builtin"), + {"status": "unsupported-private-api", "operator": None}, + ) + + def test_unsupported_warning_is_once_across_threads(self) -> None: + with ( + patch.object(torchscript_compat, "_UNSUPPORTED_WARNING_EMITTED", False), + warnings.catch_warnings(record=True) as captured, + ): + warnings.simplefilter("always") + workers = [ + threading.Thread(target=torchscript_compat.warn_unsupported_torchscript) + for _ in range(8) + ] + for worker in workers: + worker.start() + for worker in workers: + worker.join(5) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(captured), 1) + + def test_wrapper_propagates_metadata_failure(self) -> None: + def find(function: Callable[..., object]) -> object: + return 3 + + def register(function: Callable[..., object], operator: str) -> None: + self.fail("Malformed operator must not be registered") + + with patch.object(tracer, "torch", framework(find, register)): + with self.assertRaisesRegex(TypeError, "nonempty string or None"): + tracer.wrapper( + original, False, False, False, disable_dump=True, handle_proxy=False + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/instrumentor/torchscript_compat.py b/traincheck/instrumentor/torchscript_compat.py index 03087d84..4f35f8c8 100644 --- a/traincheck/instrumentor/torchscript_compat.py +++ b/traincheck/instrumentor/torchscript_compat.py @@ -1,27 +1,87 @@ """Preserve TorchScript operator identity for instrumented builtin functions.""" +import threading +import warnings +from typing import Callable, Literal, TypedDict + + +class PreservedBuiltin(TypedDict): + status: Literal["preserved"] + operator: str + + +class UnregisteredBuiltin(TypedDict): + status: Literal["not-registered"] + operator: None + + +class UnsupportedBuiltinAPI(TypedDict): + status: Literal["unsupported-private-api"] + operator: None + + +BuiltinPreservation = PreservedBuiltin | UnregisteredBuiltin | UnsupportedBuiltinAPI + # TorchScript indexes by id only. Keep registered wrappers alive, like PyTorch's # own _builtin_ops list, so a later unrelated object cannot reuse a stale ID. -_REGISTERED_WRAPPERS = {} +# These callables are retained/passed through, never invoked by this module. +_REGISTERED_WRAPPERS: dict[Callable[..., object], str] = {} +_MISSING = object() +_UNSUPPORTED_WARNING_LOCK = threading.Lock() +_UNSUPPORTED_WARNING_EMITTED = False + + +def warn_unsupported_torchscript() -> None: + """Report missing private support once per process, including always filters.""" + global _UNSUPPORTED_WARNING_EMITTED + with _UNSUPPORTED_WARNING_LOCK: + if not _UNSUPPORTED_WARNING_EMITTED: + warnings.warn( + "TrainCheck cannot preserve TorchScript builtin identity: this " + "PyTorch version lacks the private builtin-registration API. " + "Eager instrumentation remains available.", + RuntimeWarning, + stacklevel=2, + ) + _UNSUPPORTED_WARNING_EMITTED = True -def preserve_torchscript_builtin(original, wrapped, torch_module): - """Call before publishing wrapped into its module, while original is visible. +def preserve_torchscript_builtin( + original: Callable[..., object], + wrapped: Callable[..., object], + torch_module: object, +) -> BuiltinPreservation: + """Preserve a builtin mapping before publishing wrapped into its module. TorchScript compiles the existing operator directly and does not run Python recorder hooks inside its graph. Eager calls still execute the wrapper. - Private API feature detection leaves unsupported versions unchanged. + Missing private APIs are unsupported; present malformed APIs or metadata + are errors. Exceptions from framework callbacks propagate unchanged. """ - jit = getattr(torch_module, "jit", None) - builtins = getattr(jit, "_builtins", None) - find_builtin = getattr(builtins, "_find_builtin", None) - register_builtin = getattr(builtins, "_register_builtin", None) - if not callable(find_builtin) or not callable(register_builtin): + # Missing capabilities are expected across PyTorch versions. A sentinel + # distinguishes absence from present-but-invalid None callback attributes. + jit: object = getattr(torch_module, "jit", _MISSING) + builtins: object = getattr(jit, "_builtins", _MISSING) + find_builtin: object = getattr(builtins, "_find_builtin", _MISSING) + register_builtin: object = getattr(builtins, "_register_builtin", _MISSING) + for name, callback in ( + ("_find_builtin", find_builtin), + ("_register_builtin", register_builtin), + ): + if callback is not _MISSING and not callable(callback): + raise TypeError(f"torch.jit._builtins.{name} must be callable") + if find_builtin is _MISSING or register_builtin is _MISSING: return {"status": "unsupported-private-api", "operator": None} + # These checks narrow dynamic framework attributes after the missing case. + assert callable(find_builtin) and callable(register_builtin) # _find_builtin also initializes the lazy table before the first replacement. - operator = find_builtin(original) + operator: object = find_builtin(original) if operator is None: return {"status": "not-registered", "operator": None} + if not isinstance(operator, str) or not operator: + raise TypeError( + "TorchScript builtin operator must be a nonempty string or None" + ) register_builtin(wrapped, operator) _REGISTERED_WRAPPERS[wrapped] = operator return {"status": "preserved", "operator": operator} diff --git a/traincheck/instrumentor/tracer.py b/traincheck/instrumentor/tracer.py index d8eaa9bf..64b5d87d 100644 --- a/traincheck/instrumentor/tracer.py +++ b/traincheck/instrumentor/tracer.py @@ -39,7 +39,10 @@ funcs_to_be_replaced, is_funcs_to_be_unproxied, ) -from traincheck.instrumentor.torchscript_compat import preserve_torchscript_builtin +from traincheck.instrumentor.torchscript_compat import ( + preserve_torchscript_builtin, + warn_unsupported_torchscript, +) from traincheck.utils import get_timestamp_ns, get_unique_id, typename _instancemethod_t = type(torch._C._distributed_c10d.ProcessGroup.broadcast) @@ -200,7 +203,7 @@ def function_wrapper( dump_ret: bool, dump_ret_config, trigger_var_dump: bool, - var_dump_config: dict, + var_dump_config: dict[str, object] | None, need_unproxy_args_kwargs: bool, *args, **kwargs, @@ -445,19 +448,19 @@ def core_wrapper_proxy(original_function, *args, **kwargs): def wrapper( - original_function, - is_bound_method, - scan_proxy_in_args, - dump_stack_trace, - disable_dump=False, - dump_args=True, - dump_args_config=None, - dump_ret=True, - dump_ret_config=None, - handle_proxy=True, - trigger_var_dump=False, - var_dump_config=None, -): + original_function: Callable[..., object], + is_bound_method: bool, + scan_proxy_in_args: bool, + dump_stack_trace: bool, + disable_dump: bool = False, + dump_args: bool = True, + dump_args_config: dict[str, object] | None = None, + dump_ret: bool = True, + dump_ret_config: dict[str, object] | None = None, + handle_proxy: bool = True, + trigger_var_dump: bool = False, + var_dump_config: dict[str, object] | None = None, +) -> Callable[..., object]: is_builtin = is_c_level_function(original_function) need_unproxy_args_kwargs = handle_proxy and ( is_builtin or is_funcs_to_be_unproxied(original_function) @@ -468,7 +471,7 @@ def wrapper( METRIC_INSTRUMENTED_FUNC_LIST["dump"].append(original_function_name) @functools.wraps(original_function) - def wrapped(*args, **kwargs): + def wrapped(*args: object, **kwargs: object) -> object: return function_wrapper( original_function, original_function_name, @@ -491,19 +494,24 @@ def wrapped(*args, **kwargs): if need_unproxy_args_kwargs: @functools.wraps(original_function) - def wrapped(*args, **kwargs): + def wrapped(*args: object, **kwargs: object) -> object: return core_wrapper_proxy(original_function, *args, **kwargs) else: @functools.wraps(original_function) - def wrapped(*args, **kwargs): + def wrapped(*args: object, **kwargs: object) -> object: return original_function(*args, **kwargs) - wrapped._traincheck_original_function = original_function - wrapped._traincheck_instrumented = True - wrapped._traincheck_dump_disabled = disable_dump - preserve_torchscript_builtin(original_function, wrapped, torch) + setattr(wrapped, "_traincheck_original_function", original_function) + setattr(wrapped, "_traincheck_instrumented", True) + setattr(wrapped, "_traincheck_dump_disabled", disable_dump) + builtin_preservation = preserve_torchscript_builtin( + original_function, wrapped, torch + ) + setattr(wrapped, "_traincheck_torchscript_builtin", builtin_preservation) + if builtin_preservation["status"] == "unsupported-private-api": + warn_unsupported_torchscript() return wrapped @@ -955,9 +963,9 @@ def get_wrapped_function(self, func_obj: Callable) -> Callable: if func_name not in self.instr_opts.funcs_instr_opts: return wrapper( func_obj, - is_bound_method=None, - scan_proxy_in_args=None, - dump_stack_trace=None, + is_bound_method=False, + scan_proxy_in_args=False, + dump_stack_trace=False, disable_dump=True, handle_proxy=used_proxy, ) From 8dc1fd180b3d634f40ba4532238844428334b003 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:33:38 +0800 Subject: [PATCH 29/36] Type validated numeric time indexes and benchmark measurements --- tools/benchmarks/time_range_index.py | 56 +++++++++++++++++++++------- traincheck/trace/trace_pandas.py | 50 ++++++++++++++----------- 2 files changed, 70 insertions(+), 36 deletions(-) diff --git a/tools/benchmarks/time_range_index.py b/tools/benchmarks/time_range_index.py index 427a61d7..c7ae93a0 100644 --- a/tools/benchmarks/time_range_index.py +++ b/tools/benchmarks/time_range_index.py @@ -12,6 +12,7 @@ import argparse import gc import hashlib +import inspect import json import logging import os @@ -19,6 +20,7 @@ import statistics import time from pathlib import Path +from typing import Literal, TypedDict import numpy as np import pandas as pd @@ -27,17 +29,48 @@ from traincheck.instrumentor.tracer import TraceLineType from traincheck.trace.trace_pandas import TracePandas +TimingField = Literal[ + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", +] +TIMING_FIELDS: tuple[TimingField, ...] = ( + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", +) + + +class TrialResult(TypedDict): + repetition: int + order: int + variant: str + construction_seconds: float + first_query_seconds: float + remaining_queries_seconds: float + total_seconds: float + result_sha256: str + + +class Measurement(TrialResult, total=False): + indexed_rows: int + identity_groups: int + class LegacyRangeTrace(TracePandas): - def _get_func_call_time_positions(self, *args): + def _get_func_call_time_positions( + self, time_range: object, process_id: object, thread_id: object + ) -> None: return None -def digest(value): +def digest(value: object) -> str: return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--rows", type=int, default=200000) parser.add_argument("--queries", type=int, default=1000) @@ -60,7 +93,7 @@ def main(): "function": "fixture.step", } ) - queries = [] + queries: list[tuple[tuple[float, float], int, int]] = [] for i in range(args.queries): index = (i * 97) % (args.rows // 2) queries.append( @@ -71,13 +104,13 @@ def main(): ) ) - def input_hash(): + def input_hash() -> str: return hashlib.sha256( pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() ).hexdigest() before = input_hash() - records = [] + records: list[Measurement] = [] for repetition in range(args.repetitions): variants = [("legacy", LegacyRangeTrace), ("candidate", TracePandas)] if repetition % 2: @@ -106,7 +139,7 @@ def input_hash(): ] for events in results ] - record = { + record: Measurement = { "repetition": repetition + 1, "order": order + 1, "variant": name, @@ -131,12 +164,7 @@ def input_hash(): metric: statistics.median( record[metric] for record in records if record["variant"] == name ) - for metric in [ - "construction_seconds", - "first_query_seconds", - "remaining_queries_seconds", - "total_seconds", - ] + for metric in TIMING_FIELDS } for name in ["legacy", "candidate"] } @@ -160,7 +188,7 @@ def input_hash(): }, }, "source_sha256": hashlib.sha256( - Path(trace_module.__file__).read_bytes() + Path(inspect.getfile(trace_module)).read_bytes() ).hexdigest(), "input_sha256": before, "input_unchanged": True, diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 588fa64b..b8d35805 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -126,7 +126,7 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self._func_time_source: pd.DataFrame | None = None self._func_time_length = -1 self._func_time_positions: ( - dict[tuple[int, int], tuple[list, list[int]]] | None + dict[tuple[int, int], tuple[list[int | float], list[int]]] | None ) = None self.column_dtypes_cached = {} @@ -1269,7 +1269,7 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: return post_record def _get_func_call_time_positions( - self, time_range, process_id, thread_id + self, time_range: object, process_id: object, thread_id: object ) -> list[int] | None: """Return exact positional matches, or None to retain pandas semantics. @@ -1286,10 +1286,8 @@ def _get_func_call_time_positions( if ( type(time_range) is not tuple # noqa: E721 - exclude custom indexing or len(time_range) != 2 - or any( - type(value) is not int # noqa: E721 - exact numeric comparison - for value in (process_id, thread_id) - ) + or type(process_id) is not int # noqa: E721 - exact numeric comparison + or type(thread_id) is not int # noqa: E721 - exact numeric comparison or any(not -(2**63) <= value < 2**63 for value in (process_id, thread_id)) or any( not isinstance(self.events[name].dtype, np.dtype) @@ -1298,21 +1296,25 @@ def _get_func_call_time_positions( ) ): return None + start_time: object = time_range[0] + end_time: object = time_range[1] time_dtype = self.events["time"].dtype if not isinstance(time_dtype, np.dtype): return None if time_dtype == "int64": - if any( - type(bound) is not int # noqa: E721 - exclude numeric subclasses - or not -(2**63) <= bound < 2**63 - for bound in time_range + if ( + type(start_time) is not int # noqa: E721 - exclude numeric subclasses + or type(end_time) is not int # noqa: E721 + or not -(2**63) <= start_time < 2**63 + or not -(2**63) <= end_time < 2**63 ): return None elif time_dtype == "float64": - if any( - type(bound) is not float # noqa: E721 - preserve pandas coercion - or math.isnan(bound) - for bound in time_range + if ( + type(start_time) is not float # noqa: E721 - preserve pandas coercion + or type(end_time) is not float # noqa: E721 + or math.isnan(start_time) + or math.isnan(end_time) ): return None else: @@ -1322,7 +1324,9 @@ def _get_func_call_time_positions( self.events ): event_types = self.events["type"] - positions = None + positions: ( + dict[tuple[int, int], tuple[list[int | float], list[int]]] | None + ) = None # A custom __eq__ or extension dtype must still execute in pandas. if ( isinstance(event_types.dtype, np.dtype) @@ -1335,11 +1339,13 @@ def _get_func_call_time_positions( for value in event_types ) ): - rows: dict[tuple[int, int], list[tuple[Any, int]]] = {} - types = event_types.tolist() - times = self.events["time"].tolist() - processes = self.events["process_id"].tolist() - threads = self.events["thread_id"].tolist() + rows: dict[tuple[int, int], list[tuple[int | float, int]]] = {} + types: list[object] = event_types.tolist() + # The exact NumPy dtypes validated above make Series.tolist() + # return builtin numeric scalars; extension/object data falls back. + times: list[int | float] = self.events["time"].tolist() + processes: list[int] = self.events["process_id"].tolist() + threads: list[int] = self.events["thread_id"].tolist() for position, (kind, time, process, thread) in enumerate( zip(types, times, processes, threads) ): @@ -1369,8 +1375,8 @@ def _get_func_call_time_positions( if indexed is None: return [] times, row_positions = indexed - left = bisect_right(times, time_range[0]) - right = bisect_left(times, time_range[1]) + left = bisect_right(times, start_time) + right = bisect_left(times, end_time) # pandas groupby preserves input row order within each call. Time order # cannot replace that order, even when traces arrive out of order. return sorted(row_positions[left:right]) From 1a05921c3fdc345b334714cb6bff896c44430d45 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:34:55 +0800 Subject: [PATCH 30/36] Document tracing contract hardening validation --- docs/experiments/trace-integrity.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/experiments/trace-integrity.md b/docs/experiments/trace-integrity.md index 68a1dfc6..4cdff2aa 100644 --- a/docs/experiments/trace-integrity.md +++ b/docs/experiments/trace-integrity.md @@ -41,3 +41,28 @@ OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m unittest tests.test_trace_int The suite covers recorder re-entry, JSON emission re-entry, real tensor snapshots, nested application calls, generators, original and frozen exceptions, thread-local suppression, overlapping flushes, and concurrent producer ordering. [Sanitized results](trace-integrity-results.json) retain controlled counts, workload settings, base/source revisions, and evidence-file hashes. Measurements used TrainCheck base `9eb4f320d3db34d372145261704b805b88e39bda` and adapted benchmark source `ae6601978217bc0f06b9a78479cf00f0bc7242c9`; all instrumented conditions shared the same dynamic-type-name compatibility patch. No private paths, raw training traces, or checkpoint data are included. + +## Contract hardening (2026-10-25) + +Runtime checkpoint `aba4ee8e164f4d24eedae97ea641cd9a6f69c343` makes the +thread-local suppression state, context-manager exception/traceback interface, +and buffer queue/record types explicit. Suppression starts at zero independently +in each thread; process/thread IDs retain their existing lazy initialization. +`__exit__` returns `Literal[False]`, preserving application exceptions. Buffer +serialization and queue failures propagate; a failed enqueue retains its batch +for a later successful flush without dropping or duplicating records. + +Validation used Python 3.10.16, PyTorch 2.2.2, and one OpenMP/MKL/OpenBLAS thread: + +```sh +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/instrumentor/tracer.py traincheck/instrumentor/dumper.py tests/test_tracing_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics tests/test_tracing_contracts.py +``` + +All 15 tests passed (11 existing regressions and 4 state/write-failure contract +tests). All repository-pinned hooks passed; the stronger check of the new test +module used pinned mypy 1.9. Legacy serializer and proxy-wrapper annotations are +outside this narrowly scoped change; this does not claim repository-wide strict +typing. No benchmark timings were rerun for this typing/contract checkpoint; +the experiment numbers above remain evidence for their original revisions. From 7bba4038577c243d835f0f90f7ff3f479499901e Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:35:26 +0800 Subject: [PATCH 31/36] Document TorchScript contract boundaries and validation --- .../torchscript-builtin-validation.md | 41 +++++++++++++++++++ traincheck/instrumentor/tracer.py | 2 + 2 files changed, 43 insertions(+) diff --git a/docs/experiments/torchscript-builtin-validation.md b/docs/experiments/torchscript-builtin-validation.md index 71c9f4b3..b87e8442 100644 --- a/docs/experiments/torchscript-builtin-validation.md +++ b/docs/experiments/torchscript-builtin-validation.md @@ -36,3 +36,44 @@ TrainCheck imports had already initialized the registry in both tested environme Scripted graphs call `aten::tanh` directly and produce no inner Python tanh records. Eager tracing remains intact; complete compiled-graph tracing is outside this change. Full instrument-before-import Megatron startup and checkpoint-shape compatibility were not validated by this focused fixture. Registered wrappers remain alive for the process lifetime; repeatedly creating new wrappers retains additional objects. The helper depends on feature-detected private APIs and does not establish compatibility with every future PyTorch version or wrapped operator. + +## Contract hardening (2026-10-25) + +Runtime checkpoint `d4ebb12e3c5fd7531080d5623447f2af0a1ab3e9` replaces the +untyped cache and status dictionaries with a typed retention registry and a +required, discriminated result: `preserved` has a nonempty string operator, +`not-registered` has no mapping, and `unsupported-private-api` reports missing +private support. Present noncallable callbacks and nonstring/empty operator +metadata raise `TypeError` before registration; lookup/registration exceptions +propagate unchanged. Failed registration does not retain the wrapper. + +The wrapper publishes `_traincheck_torchscript_builtin` metadata and warns once +per process when private support is unavailable, including under an `always` +warning filter and concurrent callers. Unsupported support still permits eager +instrumentation. Its factory and nested callables now have explicit annotations; +`var_dump_config=None` is passed unchanged to the existing registry. Disabled +tracing passes explicit false flags for its unused tracing options. + +Validation used Python 3.10.16/PyTorch 2.2.2, one OpenMP/MKL/OpenBLAS thread, +and the repository-pinned hooks (including mypy 1.9): + +```sh +python -m unittest discover -s tests -p 'test_torchscript*.py' +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/instrumentor/tracer.py traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_compat.py tests/test_torchscript_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics --warn-return-any traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +``` + +All 13 TorchScript tests passed, including 8 new negative/capability contracts and +the existing six fresh-process controls for outputs, gradients, compiled graphs, +and eager records. All 15 tracing tests and all pinned hooks passed. The stronger +check above passed without adding `Any`, casts, or ignore directives. Callable +ellipsis parameters describe arbitrary framework functions retained or passed +through without invocation by the compatibility module. + +A separate `--check-untyped-defs` check of the legacy tracer/dumper files found +7 errors at original head `9ac61d1140cf8679c24442a174ec12f4de069980` and +4 after hardening: existing logger-key type inference, two unannotated queue +locals, and owner-class traversal assignment. Legacy serialization/proxy helpers +remain partially annotated; this is not a repository-wide strictness claim. +No timing or benchmark improvement is claimed for this contract-only change. diff --git a/traincheck/instrumentor/tracer.py b/traincheck/instrumentor/tracer.py index 64b5d87d..10bb81ab 100644 --- a/traincheck/instrumentor/tracer.py +++ b/traincheck/instrumentor/tracer.py @@ -503,6 +503,8 @@ def wrapped(*args: object, **kwargs: object) -> object: def wrapped(*args: object, **kwargs: object) -> object: return original_function(*args, **kwargs) + # Function attributes are runtime instrumentation metadata, attached after + # functools.wraps copies the original function's attributes. setattr(wrapped, "_traincheck_original_function", original_function) setattr(wrapped, "_traincheck_instrumented", True) setattr(wrapped, "_traincheck_dump_disabled", disable_dump) From b64023b0651278ab987d3276aaad9141e8bf4894 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:37:29 +0800 Subject: [PATCH 32/36] Type dynamic name contracts and retain failure assertions --- docs/experiments/dynamic-type-names.md | 34 ++++++++++++++ tests/test_dynamic_typename.py | 65 ++++++++++++++++++++------ traincheck/utils.py | 2 +- 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md index 2a7ab3e6..a6df3f75 100644 --- a/docs/experiments/dynamic-type-names.md +++ b/docs/experiments/dynamic-type-names.md @@ -65,3 +65,37 @@ The study later stopped at a shape-only negative-control selector because no shape invariant existed, after collection and healthy checks had succeeded. That harness issue and its preserved evidence are separate from the type-name fix. Evidence hashes for the successful collection are included in the JSON. + +## Contract hardening (2026-10-25) + +`typename` now declares its existing `object`/`bool` inputs and string result. +The dynamic-name guards, reported-class fallback, and final string assertions +are unchanged. All dynamic-name test helpers now have explicit annotations. +Two additional tests preserve the negative contract: an invalid fallback class +name still fails the final assertion, and an unexpected metadata exception +propagates unchanged. This does not change `safe_getattr`'s pre-existing framework +compatibility exception handling. + +The composed branch includes the signed #142 and #143 hardening checkpoints. +Validation uses Python 3.10.16/PyTorch 2.2.2 with one OpenMP/MKL/OpenBLAS thread: + +```sh +python -m unittest discover -s tests -p 'test_dynamic_typename.py' +python -m unittest discover -s tests -p 'test_torchscript*.py' +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/utils.py tests/test_dynamic_typename.py traincheck/instrumentor/tracer.py traincheck/instrumentor/dumper.py traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_compat.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics --warn-return-any traincheck/instrumentor/torchscript_compat.py tests/test_dynamic_typename.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +``` + +All 37 tests passed: 9 dynamic-name, 13 TorchScript (including six fresh-process +controls), and 15 tracing tests. All pinned hooks and the stronger scoped check +passed using mypy 1.9. The existing general utility/serialization modules are +not claimed to be strictly typed in full; see the #143 evidence for the remaining +legacy tracer/dumper diagnostics. No runtime benchmark was rerun for this +annotation/contract update, so earlier performance numbers retain their original +revision and workload scope. + +With `--check-untyped-defs`, `utils.py` reports the same single pre-existing +`threading.__excepthook__` stub diagnostic at both original head +`2333682cb2f3f48d338ff27a6d70682f0f4e4702` and this update; no new diagnostic +was introduced by the `typename` annotation. diff --git a/tests/test_dynamic_typename.py b/tests/test_dynamic_typename.py index d5f96194..1ff4d61a 100644 --- a/tests/test_dynamic_typename.py +++ b/tests/test_dynamic_typename.py @@ -10,14 +10,14 @@ class DynamicMetadata: __module__ = "fixtures.dynamic" - def __getattr__(self, name): + def __getattr__(self, name: str) -> object: if name in {"__qualname__", "__name__"}: return lambda: None raise AttributeError(name) class UnsafeTruthValue: - def __bool__(self): + def __bool__(self) -> bool: raise AssertionError("Metadata truth value must not be evaluated") @@ -28,26 +28,28 @@ class VisibleType: class ReportedClass: __module__ = "fixtures.dynamic" - @property - def __class__(self): - return VisibleType + def __getattribute__(self, name: str) -> object: + if name == "__class__": + return VisibleType + attribute: object = super().__getattribute__(name) + return attribute class DynamicTypenameTests(unittest.TestCase): - def test_dynamic_callable_metadata_uses_class_name(self): + def test_dynamic_callable_metadata_uses_class_name(self) -> None: for runtime in [False, True]: self.assertEqual( typename(DynamicMetadata(), is_runtime=runtime), "fixtures.dynamic.DynamicMetadata", ) - def test_non_string_name_values_fall_back_without_truth_testing(self): + def test_non_string_name_values_fall_back_without_truth_testing(self) -> None: for value in [object(), 17, ["name"], UnsafeTruthValue()]: class Metadata: __module__ = "fixtures.dynamic" - def __getattr__(self, name): + def __getattr__(self, name: str) -> object: if name == "__name__": return value raise AttributeError(name) @@ -58,11 +60,11 @@ def __getattr__(self, name): "fixtures.dynamic.Metadata", ) - def test_valid_string_names_still_take_precedence(self): + def test_valid_string_names_still_take_precedence(self) -> None: class Metadata: __module__ = "fixtures.dynamic" - def __getattr__(self, name): + def __getattr__(self, name: str) -> object: if name == "__qualname__": return "Outer.call" if name == "__name__": @@ -71,22 +73,55 @@ def __getattr__(self, name): self.assertEqual(typename(Metadata()), "fixtures.dynamic.Outer.call") - def test_valid_unqualified_name_is_preserved(self): + def test_valid_unqualified_name_is_preserved(self) -> None: value = DynamicMetadata() - value.__name__ = "exported" + setattr(value, "__name__", "exported") self.assertEqual(typename(value), "fixtures.dynamic.exported") - def test_reported_class_fallback_is_preserved(self): + def test_reported_class_fallback_is_preserved(self) -> None: self.assertEqual(typename(ReportedClass()), "fixtures.dynamic.VisibleType") - def test_standard_functions_descriptors_tensors_and_parameters(self): + def test_standard_functions_descriptors_tensors_and_parameters(self) -> None: self.assertEqual(typename(len), "len") self.assertEqual(typename(str.upper), "str.upper") tensor = torch.zeros(1) self.assertEqual(typename(tensor), tensor.type()) self.assertEqual(typename(torch.nn.Parameter(tensor)), "torch.nn.Parameter") - def test_real_pytorch_dynamic_ops_wrapper(self): + def test_invalid_reported_class_name_retains_final_assertion(self) -> None: + class InvalidName: + def __getattribute__(self, name: str) -> object: + if name == "__module__": + return "fixtures.dynamic" + if name in {"__qualname__", "__name__"}: + return None + if name == "__class__": + return InvalidClassName() + raise AttributeError(name) + + class InvalidClassName: + __name__ = 17 + + for runtime in (False, True): + with self.assertRaisesRegex( + AssertionError, "prefix and last_name should be str" + ): + typename(InvalidName(), is_runtime=runtime) + + def test_unexpected_metadata_failure_propagates(self) -> None: + error = ValueError("metadata unavailable") + + class FailingMetadata: + def __getattr__(self, name: str) -> object: + if name == "__qualname__": + raise error + raise AttributeError(name) + + with self.assertRaises(ValueError) as caught: + typename(FailingMetadata()) + self.assertIs(caught.exception, error) + + def test_real_pytorch_dynamic_ops_wrapper(self) -> None: from torch._inductor.virtualized import ops self.assertEqual(typename(ops), "torch._inductor.virtualized.OpsWrapper") diff --git a/traincheck/utils.py b/traincheck/utils.py index f62971fd..30de83bc 100644 --- a/traincheck/utils.py +++ b/traincheck/utils.py @@ -47,7 +47,7 @@ def typename_compile(o): return "compile_stage" -def typename(o, is_runtime=False): +def typename(o: object, is_runtime: bool = False) -> str: if isinstance(o, torch.nn.Parameter): return "torch.nn.Parameter" if isinstance(o, torch.Tensor): From 409d57ca8ec3dd11396f854112801507a8746e16 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:42:23 +0800 Subject: [PATCH 33/36] docs: record hardened inference replay parity --- ...ference-hardening-validation-20261025.json | 61 +++++++++++++++++++ .../inference-time-range-index.md | 36 +++++++++++ 2 files changed, 97 insertions(+) create mode 100644 docs/measurements/inference-hardening-validation-20261025.json diff --git a/docs/measurements/inference-hardening-validation-20261025.json b/docs/measurements/inference-hardening-validation-20261025.json new file mode 100644 index 00000000..85dc80ba --- /dev/null +++ b/docs/measurements/inference-hardening-validation-20261025.json @@ -0,0 +1,61 @@ +{ + "scope": "Engineering hardening verification; one bounded unprofiled replay, not a new before/after performance experiment", + "inference_source_commit": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "common_correctness_patch_commit": "aec748ea46b35b6b43652985f00820afa813175c", + "patch_scope": "Only the two production pruning-state hunks, preserving deterministic sampling", + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0" + }, + "workload": { + "trace_rows": 573494, + "workers": 1, + "native_threads": 1, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ] + }, + "success": true, + "source_unchanged": true, + "traces_unchanged": true, + "validation_run_wall_seconds": 76.79424141719937, + "parity": { + "source": "ce25c12 + aec748e production state patch", + "comparison": "Complete structural JSON multisets; object-key and file-line order only; duplicate multiplicity and array order retained", + "outputs": { + "invariants.jsonl": { + "before_records": 836, + "after_records": 836, + "equal": true, + "raw_bytes_equal": true, + "candidate_sha256": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0" + }, + "invariants.jsonl.failed": { + "before_records": 104, + "after_records": 104, + "equal": true, + "raw_bytes_equal": true, + "candidate_sha256": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2" + } + }, + "contexts": { + "contexts-0.json": true, + "contexts-1.json": true + } + }, + "whole_production_checks": { + "mypy_1_9": "77 files pass", + "ruff_0_3_3": "pass", + "isort_5_10_1": "pass", + "black_24_3_0": "3 unchanged legacy files fail; original base has6 failures", + "strict_typing_limit": "Scoped to newly introduced modules/tools/tests; legacy modules retain documented pre-existing diagnostics", + "composition": { + "inference": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "tracing": "b64023b0651278ab987d3276aaad9141e8bf4894", + "state_patch": "aec748ea46b35b6b43652985f00820afa813175c" + } + } +} diff --git a/docs/measurements/inference-time-range-index.md b/docs/measurements/inference-time-range-index.md index 5212bdb8..301fa5f7 100644 --- a/docs/measurements/inference-time-range-index.md +++ b/docs/measurements/inference-time-range-index.md @@ -155,3 +155,39 @@ immutable source snapshots, full outputs, and the verified harness/helper hashes retained in the evidence archive. Reproducing this full-trace result requires access to the saved trace and retained harness; the synthetic benchmark and regression tests are self-contained in the checkout. + +## Engineering hardening verification + +The inference stack now types task payloads and RNG scopes, validates required +worker settings before any worker/global mutation, and types each benchmark's +inputs and measurement output. Identity APIs describe their actual schemaless +pandas scalar values instead of promising integers. The time index uses numeric +containers only after the existing conservative dtype checks. Legacy context +comparison retains one documented pandas scalar `Any` boundary; no values are +coerced to satisfy typing and the cache stability contract is unchanged. + +All 97 affected inference tests pass, and the configured Ruff/Mypy/Black/isort +hooks pass for changed files. Pinned Mypy 1.9 also passes the sampling and worker +modules, three benchmark tools, and worker-contract tests with +`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics +--follow-imports=silent --ignore-missing-imports`. New worker-contract tests +produce 18 failures before runtime validation and pass afterward; missing, +unknown, and malformed fields are rejected before state changes. + +A new bounded Linux replay of signed inference head `ce25c12`, with the same +separate #145 production patch applied, completed on the original 573,494-event +trace. **All 836 invariants, 104 failed-hypothesis records, and both context files +are byte-for-byte identical to the corrected reference.** Source and input hashes +remain unchanged. This one verification run took 76.794 seconds; it is not a new +paired performance experiment, and the earlier performance claims above remain +tied to their original measured revisions. + +The separately composed production source (inference, tracing, and pruning-state +fixes) passes whole-source Mypy on 77 files, Ruff, and isort. Whole-source Black +still reports three unchanged legacy files (`config/config.py`, +`instrumentor/proxy_wrapper/hash.py`, and `instrumentor/proxy_wrapper/subclass.py`); +the original base reports six. Scoped strict checks do not certify fully strict +typing of every legacy body. + +[Compact hardening evidence](inference-hardening-validation-20261025.json) +records exact composition, runtime, output hashes, and limits. From f3804c0386454ddd1f97bf17b7a1ab9287cd23d2 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:43:35 +0800 Subject: [PATCH 34/36] Harden fresh-validation typing and report contracts --- .../fresh-validation-contract-hardening.md | 90 +++ tests/test_fresh_validation_contracts.py | 277 +++++++++ tools/fresh_validation/benchmark_workload.py | 196 ++++-- tools/fresh_validation/check_coverage.py | 562 +++++++++++++++--- tools/fresh_validation/compare_study.py | 384 ++++++++---- tools/fresh_validation/contracts.py | 91 +++ tools/fresh_validation/fault_control.py | 173 ++++-- tools/fresh_validation/prepare_source.py | 36 +- tools/fresh_validation/run_phase.py | 37 +- tools/fresh_validation/run_study.py | 19 +- 10 files changed, 1563 insertions(+), 302 deletions(-) create mode 100644 docs/measurements/fresh-validation-contract-hardening.md create mode 100644 tests/test_fresh_validation_contracts.py create mode 100644 tools/fresh_validation/contracts.py diff --git a/docs/measurements/fresh-validation-contract-hardening.md b/docs/measurements/fresh-validation-contract-hardening.md new file mode 100644 index 00000000..2e7b85b0 --- /dev/null +++ b/docs/measurements/fresh-validation-contract-hardening.md @@ -0,0 +1,90 @@ +# Fresh validation: typing and report contracts + +This hardens the reusable harness introduced in PR #148. The comparison base is +`27e67aafc1a526ed8e69e0a8e59ff30b8ff827cb`. Frozen predicates, training workloads, +and the original measured results are unchanged by this pass. + +## Problem and change + +All 36 original helper functions had incomplete type signatures. Worker/CLI +inputs, model factories, phase results, mutation reports, checker observation +state, and study comparisons now have explicit contracts. The eight helper +modules now contain 61 fully annotated function definitions, including boundary +validators. The new contract tests are annotated too. + +JSON enters through `object` and is narrowed at required-field boundaries. +Missing fields, wrong types, out-of-range invariant indexes, incomplete phases, +and loss rows inconsistent with the claimed step count fail explicitly. The +single `cast` documents a checked string-keyed, read-only mapping boundary; +there are no new `Any` annotations or type-checker ignores. + +The observer now requires each family's exact identity field. API containment +requires `parent_func_call_id`; the three single-call consistency/threshold +families require `func_call_id`; distinct-argument decisions require both event +records and their string call IDs. It never substitutes another family's field +or `None` when checker source changes. The proxy-skip flag is also required when +that precondition is requested. + +`ObservationReport` uses optional detail fields while the mutable report is being +constructed. `finalize_report` enforces the conditional schema before returning +or aggregating it: completed checks require their checker payload and boolean +outcome; observed coverage requires counters, groups, decisions, source metadata, +and the family-specific call/pair fields. A captured observer failure remains +primary even if the checker catches it and later returns or raises a different +exception. The latter exception remains available as secondary error evidence. + +The benchmark adapter validates dynamically loaded model classes and initializer +callbacks. Its helper-import bootstrap appends the harness path, preserving the +separately composed TrainCheck source selected by `PYTHONPATH`. + +## Report compatibility + +Coverage output now declares `schema_version: 2`. Existing completed-check fields, +checker payloads, raw identities, and decision records keep their meaning. +`coverage_status` distinguishes `observed`, `disabled`, `unsupported`, and `error`. +`observer_error` is separate from `checker_error`; either causes a nonzero CLI +result rather than a successful validation. + +Aggregate active/dormant/identity-active totals are `null` when any coverage is +unobserved. `coverage_unobserved_invariants` and `observed_*` subset counts explain +that limit. Likewise, unknown checker outcomes produce a `null` violation total, +with `check_outcome_unknown_invariants` and `observed_violated_invariants` retained. +A direct checker run can have a known outcome while its coverage remains unknown. +These are deliberate schema corrections, not new performance measurements. + +## Evidence + +The [before/after fixtures](data/fresh-validation-contracts-before-after.json) run +both original and hardened observers against three controlled checker-source +changes: a renamed identity local, a `None` identity, and a checker that catches +the observer exception. Originally all three were labeled `completed` and emitted +one `func_call_id: null` decision. After hardening, all three report +`observer_error` and emit no fabricated decisions: 3/3 false completions become +0/3. A fourth regression covers a subsequent different checker exception. + +Validation on Python 3.10.16, Torch 2.2.2, and Pandas 2.2.3: + +- All 28 targeted tests pass: the 13 original checker/fault tests and 15 contract + tests, including observer/direct parity, dormant rules, early failure, + required identities, malformed reports, and composed-source import precedence. +- Pinned Mypy 1.9.0 checks all eight helper modules and the new test module with + `--check-untyped-defs --disallow-untyped-defs --disallow-any-generics + --warn-return-any --explicit-package-bases --follow-imports=silent + --ignore-missing-imports`. Its `--python-executable` points at the installed + TrainCheck runtime, so Torch annotations resolve. This is a stronger scoped + check, not a claim of repository-wide `--strict` compliance. +- Configured Ruff 0.3.3, Black 24.3.0, and isort 5.10.1 hooks pass on the changed + paths. The configured mypy hook does not select `tools/`; the explicit command + above covers that gap. `git diff --check` passes. + +Run the focused tests with: + +```sh +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 \ +python -m unittest tests.test_fresh_validation_coverage \ + tests.test_fresh_validation_fault tests.test_fresh_validation_contracts -q +``` + +The source-builder recipe remains pinned to its original, reproducible composition. +Full saved-trace replay and the newly composed PR stack require separate integration +validation. This change makes no new training-overhead or generalization claim. diff --git a/tests/test_fresh_validation_contracts.py b/tests/test_fresh_validation_contracts.py new file mode 100644 index 00000000..1379c61d --- /dev/null +++ b/tests/test_fresh_validation_contracts.py @@ -0,0 +1,277 @@ +"""Negative-path contracts for emitted reports and observed call identities.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import ( + ObserverContractError, + decision_record, + finalize_report, + observe, + summarize, +) +from tools.fresh_validation.compare_study import compare, losses +from tools.fresh_validation.contracts import boolean, integer, json_object, required +from tools.fresh_validation.fault_control import digest, mutate +from traincheck.invariant.base_cls import CheckerResult, Invariant +from traincheck.invariant.consistency_transient_vars import ConsistentOutputRelation +from traincheck.trace.trace import Trace + + +def renamed_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + # This still has the exact output-predicate AST but its local identity was + # renamed. The previous observer silently emitted {func_call_id: None}. + for renamed_call in trace.get_func_call_ids("torch.fake"): + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + return CheckerResult(None, inv, True, True) + + +def malformed_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for real_call in trace.get_func_call_ids("torch.fake"): + func_call_event = trace.query_func_call_event(real_call) + func_call_id = None + returned_tensors = [func_call_event, func_call_id] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + return CheckerResult(None, inv, True, True) + + +def catching_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for renamed_call in trace.get_func_call_ids("torch.fake"): + try: + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + except ValueError: + # The observer must retain failure even when the checker catches it. + continue + return CheckerResult(None, inv, True, True) + + +def catching_then_failing_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for renamed_call in trace.get_func_call_ids("torch.fake"): + try: + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + except ValueError: + pass + raise RuntimeError("checker failed after catching observer error") + + +class UnsupportedOutput(ConsistentOutputRelation): + pass + + +class ValidationContractTests(unittest.TestCase): + def test_required_family_identity_does_not_use_another_family_fallback( + self, + ) -> None: + with self.assertRaisesRegex(ObserverContractError, "func_call_id"): + decision_record( + "ConsistentOutputRelation", {"parent_func_call_id": "parent"} + ) + with self.assertRaisesRegex(ObserverContractError, "parent_func_call_id"): + decision_record("APIContainRelation", {"func_call_id": "child"}) + + def test_pair_requires_both_record_identities(self) -> None: + with self.assertRaisesRegex(ObserverContractError, "func_call_id"): + decision_record( + "DistinctArgumentRelation", + {"event1": {"func_call_id": "a"}, "event2": {}}, + ) + + def test_identity_wrong_types_fail_explicitly(self) -> None: + for identity in (None, False, 42, ["call"]): + with ( + self.subTest(identity=identity), + self.assertRaises(ObserverContractError), + ): + decision_record("ConsistentOutputRelation", {"func_call_id": identity}) + + def test_checker_source_drift_is_observer_error_not_successful_null_identity( + self, + ) -> None: + for checker in ( + renamed_local_checker, + malformed_local_checker, + catching_local_checker, + ): + with ( + self.subTest(checker=checker.__name__), + patch.object(ConsistentOutputRelation, "static_check_all", checker), + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + output_rule(), + ) + self.assertEqual(report["status"], "observer_error") + self.assertEqual(report["coverage_status"], "error") + self.assertIsNone(report["check_passed"]) + self.assertIsNone(report["active"]) + self.assertFalse(report["coverage_observed"]) + self.assertEqual(report["decision_records"], []) + + def test_observer_failure_remains_primary_if_checker_then_raises(self) -> None: + with patch.object( + ConsistentOutputRelation, "static_check_all", catching_then_failing_checker + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + output_rule(), + ) + self.assertEqual(report["status"], "observer_error") + self.assertEqual(report["coverage_status"], "error") + self.assertFalse(report["coverage_observed"]) + self.assertIsNone(report["active"]) + self.assertEqual(report["error"]["type"], "ObserverContractError") + self.assertEqual(report["checker_error_after_observer"]["type"], "RuntimeError") + + def test_disabled_and_unsupported_counts_are_unknown_not_zero(self) -> None: + trace = frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]) + direct = observe(trace, output_rule(), enabled=False) + rule = output_rule() + rule.relation = UnsupportedOutput + unsupported = observe(trace, rule) + for report in (direct, unsupported): + summary = summarize([report]) + self.assertIsNone(summary["active_invariants"]) + self.assertIsNone(summary["dormant_invariants"]) + self.assertEqual(summary["coverage_unobserved_invariants"], 1) + self.assertEqual(summary["observed_active_invariants"], 0) + self.assertEqual(direct["coverage_status"], "disabled") + self.assertEqual(unsupported["coverage_status"], "unsupported") + self.assertIsNone(summarize([unsupported])["violated_invariants"]) + self.assertEqual(summarize([direct])["violated_invariants"], 0) + + def test_completed_report_requires_checker_payload(self) -> None: + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), output_rule() + ) + del report["checker_result"] + with self.assertRaisesRegex(ObserverContractError, "checker_result"): + finalize_report(report) + + def test_observed_report_requires_decision_counters(self) -> None: + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), output_rule() + ) + del report["decisions_completed"] + with self.assertRaisesRegex(ObserverContractError, "decisions_completed"): + finalize_report(report) + + def test_checker_error_does_not_imply_zero_violations(self) -> None: + rule = output_rule() + with patch.object( + rule, "check", side_effect=ValueError("fixture checker error") + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + rule, + enabled=False, + ) + self.assertEqual(report["status"], "checker_error") + self.assertIsNone(summarize([report])["violated_invariants"]) + self.assertEqual(summarize([report])["observed_violated_invariants"], 0) + + def test_cli_bootstrap_preserves_composed_traincheck_source(self) -> None: + adapter = ( + Path(__file__).resolve().parents[1] + / "tools/fresh_validation/benchmark_workload.py" + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + composed = root / "composed" + (composed / "traincheck").mkdir(parents=True) + module = composed / "traincheck/__init__.py" + module.write_text("SOURCE = 'fixture'\n") + code = ( + "import runpy; runpy.run_path(" + + repr(str(adapter)) + + "); import traincheck; print(traincheck.__file__)" + ) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=root, + env={ + **os.environ, + "PYTHONPATH": str(composed), + "PYTHONDONTWRITEBYTECODE": "1", + }, + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(Path(result.stdout.strip()), module) + + def test_json_required_fields_and_wrong_types_are_not_coerced(self) -> None: + with self.assertRaisesRegex(ValueError, "required"): + required({}, "required") + with self.assertRaises(ValueError): + boolean("false", "active") + with self.assertRaises(ValueError): + integer(True, "count") + with self.assertRaises(ValueError): + json_object({"invalid": object()}) + + def test_empty_loss_records_do_not_establish_equality(self) -> None: + for steps in ([], [{"step": 1}]): + with self.subTest(steps=steps), self.assertRaisesRegex(ValueError, "loss"): + losses({"steps": steps}) + + def test_truncated_loss_rows_do_not_match_claimed_completed_steps(self) -> None: + with self.assertRaisesRegex(ValueError, "completed_steps"): + losses({"steps": [{"step": 1, "loss": 1.0}], "completed_steps": 2}) + + def test_incomplete_phase_set_cannot_report_study_completed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaisesRegex(ValueError, "Missing completed phases"): + compare(Path(temporary)) + + def test_fault_index_cannot_silently_select_last_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace, rules, coverage = ( + root / name for name in ("trace.jsonl", "rules.jsonl", "coverage.json") + ) + trace.write_text( + json.dumps({"type": "function_call (post)", "func_call_id": "a"}) + "\n" + ) + rules.write_text(json.dumps(output_rule().to_dict()) + "\n") + coverage.write_text( + json.dumps( + { + "inputs_unchanged": True, + "observer_enabled": True, + "input_sha256": {str(p): digest(p) for p in (trace, rules)}, + "per_invariant": [{"invariant_index": -1}], + } + ) + ) + with self.assertRaisesRegex(ValueError, "out of range"): + mutate([trace], rules, coverage, root / "fault") + self.assertFalse((root / "fault").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/fresh_validation/benchmark_workload.py b/tools/fresh_validation/benchmark_workload.py index 90355957..50379d24 100644 --- a/tools/fresh_validation/benchmark_workload.py +++ b/tools/fresh_validation/benchmark_workload.py @@ -6,8 +6,11 @@ not claims to reproduce the complete benchmark's data or end-to-end training. """ +from __future__ import annotations + import argparse import ast +import copy import gzip import hashlib import json @@ -15,8 +18,66 @@ import random import struct import subprocess +import sys import time +from collections.abc import Callable, Mapping, Sequence from pathlib import Path +from typing import TYPE_CHECKING, Literal, Protocol, TypedDict + +if TYPE_CHECKING: + import torch + import torch.nn as nn + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import JSONObject, json_value # noqa: E402 + + +class TracingConfig(Protocol): + DISABLE_WRAPPER: bool + MODEL_TRACKER_STYLE: str | None + INSTR_DESCRIPTORS: bool + INSTRUMENTATION_POLICY: dict[str, int] + + +def configure_tracing(config: TracingConfig) -> None: + config.DISABLE_WRAPPER = True + config.MODEL_TRACKER_STYLE = None + config.INSTR_DESCRIPTORS = False + config.INSTRUMENTATION_POLICY.update(interval=1, warm_up=0) + + +class WorkloadOptions(argparse.Namespace): + benchmark_root: Path + workload: Literal["mnist", "dcgan", "transformer"] + steps: int + data_steps: int | None + seed: int + output: Path + mode: Literal["baseline", "instrumented"] + device: Literal["cpu", "cuda"] + batch_size: int + cpu_threads: int + data_root: Path | None + save_checkpoint: bool + + +class TensorMetadata(TypedDict): + shape: list[int] + dtype: str + + +class WorkloadMetrics(TypedDict): + success: bool + error: dict[str, str] | None + completed_steps: int + steps: list[dict[str, int | float]] + initial_model_state_sha256: str + final_model_state_sha256: str + state_shapes_unchanged: bool + state_shapes: dict[str, TensorMetadata] + timing: str + SOURCES = { "mnist": "instr-overhead/overhead-e2e/mnist/main.py", @@ -25,15 +86,15 @@ } -def sha256(path): +def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def write_json(path, value): +def write_json(path: Path, value: object) -> None: path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") -def definitions(source, names): +def definitions(source: Path, names: Sequence[str]) -> str: """Extract exact source spans, never execute the benchmark's top-level CLI.""" text = source.read_text() nodes = { @@ -44,12 +105,16 @@ def definitions(source, names): missing = set(names) - nodes.keys() if missing: raise ValueError("Missing benchmark definitions: " + str(sorted(missing))) - return ( - "\n\n".join(ast.get_source_segment(text, nodes[name]) for name in names) + "\n" - ) + segments: list[str] = [] + for name in names: + segment = ast.get_source_segment(text, nodes[name]) + if segment is None: + raise ValueError(f"Benchmark definition {name} has no source span") + segments.append(segment) + return "\n\n".join(segments) + "\n" -def mnist_raw(data_root, count): +def mnist_raw(data_root: Path, count: int) -> tuple[bytes, bytes, JSONObject]: """Read local torchvision-style IDX cache without network or torchvision.""" candidates = [data_root / "MNIST" / "raw", data_root / "raw", data_root] for root in candidates: @@ -76,10 +141,12 @@ def mnist_raw(data_root, count): "--data-root must contain the cached MNIST training IDX images and labels" ) - def prefix(path, length): - opener = gzip.open if path.suffix == ".gz" else open - with opener(path, "rb") as handle: - return handle.read(length) + def prefix(path: Path, length: int) -> bytes: + if path.suffix == ".gz": + with gzip.open(path, "rb") as handle: + return handle.read(length) + with path.open("rb") as raw_handle: + return raw_handle.read(length) images = prefix(image, 16 + count * 28 * 28) targets = prefix(labels, 8 + count) @@ -106,7 +173,7 @@ def prefix(path, length): ) -def text_prefix(path, count): +def text_prefix(path: Path, count: int) -> tuple[list[int], int, JSONObject]: words = [] with path.open() as handle: for line in handle: @@ -116,7 +183,7 @@ def text_prefix(path, count): if len(words) < count: raise ValueError("WikiText source is shorter than the bounded workload") words = words[:count] - vocab = {} + vocab: dict[str, int] = {} ids = [vocab.setdefault(word, len(vocab)) for word in words] return ( ids, @@ -133,9 +200,21 @@ def text_prefix(path, count): ) -def prepare(args, torch): +def module_class(namespace: Mapping[str, object], name: str) -> type[nn.Module]: + import torch.nn as nn + + value = namespace[name] + if not isinstance(value, type) or not issubclass(value, nn.Module): + raise ValueError(f"Benchmark definition {name} must be an nn.Module class") + return value + + +def prepare( + args: WorkloadOptions, +) -> tuple[dict[str, nn.Module], Callable[[int], dict[str, torch.Tensor]], JSONObject]: import math + import torch import torch.nn as nn import torch.nn.functional as F @@ -147,7 +226,7 @@ def prepare(args, torch): }[args.workload] extracted = definitions(source, symbols) (args.output / "benchmark_definitions.py").write_text(extracted) - ns = { + ns: dict[str, object] = { "__name__": "trace_impact_benchmark", "torch": torch, "nn": nn, @@ -159,11 +238,11 @@ def prepare(args, torch): "ndf": 8, } exec(compile(extracted, str(args.output / "benchmark_definitions.py"), "exec"), ns) - manifest = { + manifest: JSONObject = { "workload_kind": "bounded_model_micro_workload", "benchmark_source": str(source), "benchmark_source_sha256": sha256(source), - "extracted_symbols": symbols, + "extracted_symbols": [symbol for symbol in symbols], "extracted_definitions_sha256": hashlib.sha256(extracted.encode()).hexdigest(), "steps": args.steps, "seed": args.seed, @@ -190,17 +269,17 @@ def prepare(args, torch): count = args.steps * batch device = torch.device(args.device) if args.workload == "mnist": - model = ns["Net"]().to(device) + model = module_class(ns, "Net")().to(device) optimizer = torch.optim.Adadelta(model.parameters(), lr=1.0) if args.data_root: - images, labels, provenance = mnist_raw(args.data_root, count) + raw_images, raw_labels, provenance = mnist_raw(args.data_root, count) x = ( - torch.tensor(list(images), dtype=torch.float32).reshape( + torch.tensor(list(raw_images), dtype=torch.float32).reshape( count, 1, 28, 28 ) / 255 ) - y = torch.tensor(list(labels), dtype=torch.int64) + y = torch.tensor(list(raw_labels), dtype=torch.int64) else: x = torch.rand(count, 1, 28, 28) y = torch.randint(0, 10, (count,)) @@ -214,7 +293,7 @@ def prepare(args, torch): y = y.to(device) model.train() - def step(index): + def step(index: int) -> dict[str, torch.Tensor]: optimizer.zero_grad() output = model(x[index * batch : (index + 1) * batch]) loss = F.nll_loss(output, y[index * batch : (index + 1) * batch]) @@ -229,10 +308,19 @@ def step(index): update="Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", ) elif args.workload == "dcgan": - generator = ns["Generator"](1).to(device) - generator.apply(ns["weights_init"]) - discriminator = ns["Discriminator"](1).to(device) - discriminator.apply(ns["weights_init"]) + generator = module_class(ns, "Generator")(1).to(device) + initializer = ns["weights_init"] + if not callable(initializer): + raise ValueError("Benchmark weights_init must be callable") + + def initialize(module: nn.Module) -> None: + result: object = initializer(module) + if result is not None: + raise ValueError("Benchmark weights_init must return None") + + generator.apply(initialize) + discriminator = module_class(ns, "Discriminator")(1).to(device) + discriminator.apply(initialize) optimizer_d = torch.optim.Adam( discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999) ) @@ -247,7 +335,7 @@ def step(index): generator.train() discriminator.train() - def step(index): + def step(index: int) -> dict[str, torch.Tensor]: discriminator.zero_grad() real = images[index * batch : (index + 1) * batch] label = torch.full((batch,), 1, dtype=real.dtype, device=device) @@ -298,10 +386,12 @@ def step(index): .contiguous() .to(device) ) - model = ns["TransformerModel"](vocab, 32, 2, 64, 2, 0.2).to(device) + model = module_class(ns, "TransformerModel")(vocab, 32, 2, 64, 2, 0.2).to( + device + ) model.train() - def step(index): + def step(index: int) -> dict[str, torch.Tensor]: start = index * seq inputs = data[start : start + seq] targets = data[start + 1 : start + seq + 1].reshape(-1) @@ -334,15 +424,24 @@ def step(index): return models, step, manifest -def model_digest(models): +def model_digest( + models: Mapping[str, nn.Module] +) -> tuple[str, dict[str, TensorMetadata]]: """Hash tensor contents, names, dtype and shape; independent of torch.save metadata.""" + import torch + digest = hashlib.sha256() - shapes = {} + shapes: dict[str, TensorMetadata] = {} for model_name, model in sorted(models.items()): for name, value in sorted(model.state_dict().items()): key = model_name + "." + name + if not isinstance(value, torch.Tensor): + raise ValueError(f"Benchmark state {key} is not a tensor") tensor = value.detach().cpu().contiguous() - metadata = {"shape": list(tensor.shape), "dtype": str(tensor.dtype)} + metadata: TensorMetadata = { + "shape": list(tensor.shape), + "dtype": str(tensor.dtype), + } shapes[key] = metadata digest.update( key.encode() @@ -354,7 +453,7 @@ def model_digest(models): return digest.hexdigest(), shapes -def run(args): +def run(args: WorkloadOptions) -> None: os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") os.environ["TRAINCHECK_OUTPUT_DIR"] = str(args.output / "traces") import torch @@ -368,11 +467,11 @@ def run(args): torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True - prepare_args = argparse.Namespace(**vars(args)) + prepare_args = copy.copy(args) prepare_args.steps = args.data_steps or args.steps if prepare_args.steps < args.steps: raise ValueError("data_steps must cover every training step") - models, step, manifest = prepare(prepare_args, torch) + models, step, manifest = prepare(prepare_args) manifest.update( steps=args.steps, prepared_data_steps=prepare_args.steps, @@ -387,7 +486,7 @@ def run(args): p.numel() for model in models.values() for p in model.parameters() ) manifest["initial_model_state_sha256"] = initial_hash - manifest["state_shapes"] = shapes + manifest["state_shapes"] = json_value(shapes) manifest["instrumentation"] = { "target": "torch", "models_to_track": [], @@ -399,20 +498,19 @@ def run(args): } manifest["python_version"] = __import__("sys").version write_json(args.output / "adaptation.json", manifest) - tc_config = None - start_step = None + tc_config: TracingConfig | None = None + start_step: Callable[[], object] | None = None if args.mode == "instrumented": (args.output / "traces").mkdir() import traincheck.instrumentor.tracer as tracer_module - from traincheck.config import config as tc_config - from traincheck.instrumentor.control import start_step + from traincheck.config import config as active_config + from traincheck.instrumentor.control import start_step as active_start_step from traincheck.instrumentor.tracer import Instrumentor + tc_config = active_config + start_step = active_start_step manifest["tracer_module"] = str(Path(tracer_module.__file__).resolve()) - tc_config.DISABLE_WRAPPER = True - tc_config.MODEL_TRACKER_STYLE = None - tc_config.INSTR_DESCRIPTORS = False - tc_config.INSTRUMENTATION_POLICY.update(interval=1, warm_up=0) + configure_tracing(tc_config) if os.environ.get("TC_ORIGIN_AUDIT"): from origin_audit import install @@ -435,9 +533,9 @@ def run(args): torch.cuda.manual_seed_all(args.seed + 1) manifest["training_rng_seed"] = args.seed + 1 write_json(args.output / "adaptation.json", manifest) - records = [] + records: list[dict[str, int | float]] = [] success = False - error = None + error: dict[str, str] | None = None try: for index in range(args.steps): if args.device == "cuda": @@ -470,7 +568,7 @@ def run(args): if tc_config is not None: tc_config.DISABLE_WRAPPER = True final_hash, final_shapes = model_digest(models) - result = { + result: WorkloadMetrics = { "success": success, "error": error, "completed_steps": len(records), @@ -489,7 +587,7 @@ def run(args): ) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--benchmark-root", type=Path, required=True) parser.add_argument("--workload", choices=sorted(SOURCES), required=True) @@ -509,7 +607,7 @@ def main(): "--data-root", type=Path, help="Optional local MNIST IDX cache; never downloads" ) parser.add_argument("--save-checkpoint", action="store_true") - args = parser.parse_args() + args = parser.parse_args(namespace=WorkloadOptions()) if args.steps < 1 or args.batch_size < 1 or args.cpu_threads < 1: parser.error("steps, batch-size, cpu-threads must be positive") if args.data_root and args.workload != "mnist": diff --git a/tools/fresh_validation/check_coverage.py b/tools/fresh_validation/check_coverage.py index 14773529..f0db31de 100644 --- a/tools/fresh_validation/check_coverage.py +++ b/tools/fresh_validation/check_coverage.py @@ -16,21 +16,340 @@ import os import sys from collections import Counter +from collections.abc import Callable, Mapping from pathlib import Path +from types import FrameType +from typing import Literal, TypeAlias, TypedDict from unittest.mock import patch from traincheck.invariant import read_inv_file -from traincheck.invariant.base_cls import GroupedPreconditions +from traincheck.invariant.base_cls import APIParam, GroupedPreconditions, Invariant from traincheck.invariant.DistinctArgumentRelation import ( get_event_data_per_function_per_step, ) -from traincheck.trace.trace_pandas import read_trace_file_Pandas +from traincheck.trace.trace import Trace +from traincheck.trace.trace_pandas import TracePandas, read_trace_file_Pandas from traincheck.trace.types import ( FuncCallExceptionEvent, IncompleteFuncCallEvent, MDNONEJSONEncoder, ) +# Direct CLI execution must resolve this checkout's harness package even when +# PYTHONPATH intentionally points to a separately composed TrainCheck source. +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + json_list, + json_object, + json_value, + mapping, + required, + sequence, + string, +) + + +class ObserverContractError(ValueError): + """Checker internals no longer satisfy the observer's verified contract.""" + + +class ObserverSites(TypedDict): + starts: dict[int, str] + ends: dict[int, int] + visits: set[int] + method_source_sha256: str + + +class CallDecision(TypedDict): + func_call_id: str + + +class PairDecision(TypedDict): + pair_func_call_ids: list[str] + + +Decision: TypeAlias = CallDecision | PairDecision + + +class PendingDecision(TypedDict): + start: int + end: int + record: Decision + + +class GroupCounts(TypedDict): + attempts: int + passes: int + failures: int + errors: int + + +class CandidateCounts(TypedDict): + candidate_calls: int + opportunities: int + opportunity_unit: str + incomplete_or_exception_calls: int | None + + +class OptionalReportFields(TypedDict, total=False): + invariant_index: int + checker_result: JSONObject + error: dict[str, str] + checker_error_after_observer: dict[str, str] + candidate_calls: int + opportunities: int + opportunity_unit: str + incomplete_or_exception_calls: int | None + method_source_sha256: str + decisions_started: int + decisions_completed: int + decisions_not_completed: int + visited_calls: int | None + precondition_group_attempts: int + precondition_group_passes: int + precondition_group_failures: int + precondition_group_errors: int + precondition_groups: dict[str, GroupCounts] + decision_records: list[Decision] + unvisited_calls_after_stop: int | None + unvisited_pairs_after_stop: int + variable_precondition_requested: bool + variable_check_skipped: bool + fully_validated_active: bool + + +class ObservationReport(OptionalReportFields): + # Optional detail fields are present only for the matching status. Core + # outcome/observation fields are always present; unknown is never false. + relation: str + description: str | None + identity_clauses: list[JSONObject] + observer_supported: bool + status: Literal[ + "completed", "checker_error", "observer_error", "unsupported_coverage" + ] + coverage_status: Literal["observed", "disabled", "unsupported", "error"] + coverage_observed: bool + active: bool | None + dormant: bool | None + check_passed: bool | None + checker_triggered: bool | None + + +class RuntimeMetadata(TypedDict): + python: str + executable: str + trace_module: str + packages: dict[str, str] + + +class CoverageOutput(TypedDict): + schema_version: Literal[2] + pid: int + check_relation_first: Literal[False] + observer_enabled: bool + runtime: RuntimeMetadata + harness_sha256: str + raw_process_ids: list[JSONValue] + raw_thread_ids: list[JSONValue] + trace_rows: int + input_sha256: dict[str, str] + inputs_unchanged: bool + summary: dict[str, int | None] + per_invariant: list[ObservationReport] + + +TraceCallback: TypeAlias = Callable[[FrameType, str, object], "TraceCallback | None"] + + +def decision_record(family: str, local_values: Mapping[str, object]) -> Decision: + """Read the exact family-specific identity fields, failing closed on drift.""" + try: + if family == "DistinctArgumentRelation": + ids = [] + for name in ("event1", "event2"): + event = mapping(required(local_values, name), name) + ids.append( + string(required(event, "func_call_id"), f"{name}.func_call_id") + ) + return {"pair_func_call_ids": ids} + name = ( + "parent_func_call_id" if family == "APIContainRelation" else "func_call_id" + ) + if family not in SUPPORTED: + raise ValueError(f"Unsupported observer family: {family}") + return {"func_call_id": string(required(local_values, name), name)} + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + + +def finalize_report(report: ObservationReport) -> ObservationReport: + """Validate the conditional schema before exposing a mutable report. + + OptionalReportFields is an internal construction accumulator, not permission + to omit completed-result or observed-coverage fields in emitted reports. + """ + status = report["status"] + coverage_status = report["coverage_status"] + if status not in { + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + }: + raise ObserverContractError(f"Unknown checker status: {status}") + if coverage_status not in {"observed", "disabled", "unsupported", "error"}: + raise ObserverContractError(f"Unknown coverage status: {coverage_status}") + try: + if status == "completed": + checker = mapping(required(report, "checker_result"), "checker_result") + passed = boolean(report["check_passed"], "check_passed") + triggered = boolean(report["checker_triggered"], "checker_triggered") + if ( + boolean( + required(checker, "check_passed"), "checker_result.check_passed" + ) + != passed + or boolean(required(checker, "triggered"), "checker_result.triggered") + != triggered + ): + raise ValueError("Checker outcome and report outcome disagree") + mapping(required(checker, "invariant"), "checker_result.invariant") + else: + if ( + report["check_passed"] is not None + or report["checker_triggered"] is not None + ): + raise ValueError("Incomplete checking must have unknown outcome") + if status in {"checker_error", "observer_error"}: + error = mapping(required(report, "error"), "error") + string(required(error, "type"), "error.type") + string(required(error, "message"), "error.message") + if coverage_status == "observed": + if report["coverage_observed"] is not True or status not in { + "completed", + "checker_error", + }: + raise ValueError("Observed coverage has incompatible checker status") + boolean(report["active"], "active") + boolean(report["dormant"], "dormant") + for key in ( + "candidate_calls", + "opportunities", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + ): + if integer(required(report, key), key) < 0: + raise ValueError(f"{key} cannot be negative") + for key in ( + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active", + ): + boolean(required(report, key), key) + for key in ("opportunity_unit", "method_source_sha256"): + string(required(report, key), key) + groups = mapping( + required(report, "precondition_groups"), "precondition_groups" + ) + for name, raw_group in groups.items(): + group = mapping(raw_group, name) + for key in ("attempts", "passes", "failures", "errors"): + if integer(required(group, key), f"{name}.{key}") < 0: + raise ValueError(f"{name}.{key} cannot be negative") + if report["relation"] == "DistinctArgumentRelation": + if ( + required(report, "visited_calls") is not None + or required(report, "unvisited_calls_after_stop") is not None + ): + raise ValueError("Pair coverage must not claim call visit counts") + integer( + required(report, "unvisited_pairs_after_stop"), + "unvisited_pairs_after_stop", + ) + else: + integer(required(report, "visited_calls"), "visited_calls") + integer( + required(report, "unvisited_calls_after_stop"), + "unvisited_calls_after_stop", + ) + for raw in sequence( + required(report, "decision_records"), "decision_records" + ): + decision = mapping(raw, "decision") + if report["relation"] == "DistinctArgumentRelation": + pair = sequence( + required(decision, "pair_func_call_ids"), "pair_func_call_ids" + ) + if len(pair) != 2: + raise ValueError("Pair decisions require exactly two call IDs") + for identity in pair: + string(identity, "pair call ID") + else: + string(required(decision, "func_call_id"), "decision.func_call_id") + elif ( + report["coverage_observed"] is not False + or report["active"] is not None + or report["dormant"] is not None + ): + raise ValueError( + "Unobserved coverage must have unknown active/dormant states" + ) + if status == "unsupported_coverage" and coverage_status != "unsupported": + raise ValueError("Unsupported checker coverage must be marked unsupported") + if status == "observer_error" and coverage_status != "error": + raise ValueError("Observer failure must mark coverage as an error") + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + return report + + +def summarize(results: list[ObservationReport]) -> dict[str, int | None]: + counts: Counter[str] = Counter() + unobserved = 0 + unknown_outcomes = 0 + for result in results: + finalize_report(result) + counts[result["status"]] += 1 + observed = result["coverage_status"] == "observed" + if observed: + active, dormant = result["active"], result["dormant"] + assert active is not None and dormant is not None + counts["active_invariants"] += active + counts["dormant_invariants"] += dormant + counts["identity_bearing_active"] += ( + bool(result["identity_clauses"]) and active + ) + else: + unobserved += 1 + unknown_outcomes += result["check_passed"] is None + counts["violated_invariants"] += result["check_passed"] is False + counts["identity_bearing_invariants"] += bool(result["identity_clauses"]) + summary: dict[str, int | None] = dict(counts) + summary["coverage_unobserved_invariants"] = unobserved + summary["check_outcome_unknown_invariants"] = unknown_outcomes + summary["observed_violated_invariants"] = counts["violated_invariants"] + summary["violated_invariants"] = ( + None if unknown_outcomes else counts["violated_invariants"] + ) + # A disabled/unsupported/error observer cannot establish the total active or + # dormant population. Keep the observed subset explicitly separate. + for name in ("active_invariants", "dormant_invariants", "identity_bearing_active"): + summary["observed_" + name] = counts[name] + summary[name] = None if unobserved else counts[name] + return summary + + SUPPORTED = { "APIContainRelation", "ConsistentOutputRelation", @@ -40,11 +359,11 @@ } -def sha256(path): +def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def observer_sites(function, family): +def observer_sites(function: Callable[..., object], family: str) -> ObserverSites: """Locate exact predicate ASTs in the installed method, failing closed on drift.""" lines, first = inspect.getsourcelines(function) # Dedent code by its method indentation. A zero-indented line inside a @@ -54,8 +373,10 @@ def observer_sites(function, family): line[indent:] if line.startswith(" " * indent) else line for line in lines ) tree = ast.parse(source) - starts, ends, visits = {}, {}, set() - predicates = { + starts: dict[int, str] = {} + ends: dict[int, int] = {} + visits: set[int] = set() + predicates: dict[str, Callable[[str], bool]] = { "APIContainRelation": lambda s: "skip_var_unchanged_check" in s and "found_expected_child_event" in s, "ConsistentInputOutputRelation": lambda s: s == "input_value != output_value", @@ -69,6 +390,8 @@ def observer_sites(function, family): if isinstance(node, ast.If) and predicates[family](ast.unparse(node.test)): line = first + node.lineno - 1 starts[line] = ast.unparse(node.test) + if node.test.end_lineno is None: + raise ObserverContractError("Predicate has no ending source line") ends[line] = first + node.test.end_lineno - 1 if isinstance(node, ast.Assign) and any( isinstance(t, ast.Name) and t.id in {"func_call_event", "parent_event_type"} @@ -77,7 +400,9 @@ def observer_sites(function, family): visits.add(first + node.lineno - 1) expected = 2 if family in {"ThresholdRelation", "DistinctArgumentRelation"} else 1 if len(starts) != expected: - raise ValueError(f"Unrecognized checker source for {family}: {starts}") + raise ObserverContractError( + f"Unrecognized checker source for {family}: {starts}" + ) return { "starts": starts, "ends": ends, @@ -86,13 +411,15 @@ def observer_sites(function, family): } -def candidate_counts(trace, inv): +def candidate_counts(trace: TracePandas, inv: Invariant) -> CandidateCounts: family = inv.relation.__name__ api = ( inv.params[1] if family in {"ConsistentInputOutputRelation", "ThresholdRelation"} else inv.params[0] ) + if not isinstance(api, APIParam): + raise ObserverContractError(f"{family} expected an APIParam") calls = trace.get_func_call_ids(api.api_full_name) if family == "DistinctArgumentRelation": _, groups = get_event_data_per_function_per_step(trace, {api.api_full_name}) @@ -104,6 +431,7 @@ def candidate_counts(trace, inv): "candidate_calls": len(calls), "opportunities": pairs, "opportunity_unit": "within_step_argument_pair", + "incomplete_or_exception_calls": None, } if family == "APIContainRelation": eligible = len(calls) @@ -127,10 +455,10 @@ def candidate_counts(trace, inv): } -def id_clauses(value): - found = [] +def id_clauses(value: JSONValue) -> list[JSONObject]: + found: list[JSONObject] = [] if isinstance(value, dict): - if value.get("prop_name") in { + if isinstance(value.get("prop_name"), str) and value.get("prop_name") in { "process_id", "thread_id", "meta_vars.process_id", @@ -145,14 +473,30 @@ def id_clauses(value): return found -def observe(trace, inv, enabled=True): +def observe( + trace: TracePandas, inv: Invariant, enabled: bool = True +) -> ObservationReport: """Run one unchanged inv.check(False), counting only observed completed decisions.""" family = inv.relation.__name__ - report = { + assert inv.precondition is not None, "Invariant precondition must be set" + report: ObservationReport = { "relation": family, "description": inv.text_description, - "identity_clauses": id_clauses(inv.precondition.to_dict()), + "identity_clauses": id_clauses( + json_value( + json.loads( + json.dumps(inv.precondition.to_dict(), cls=MDNONEJSONEncoder) + ) + ) + ), "observer_supported": family in SUPPORTED, + "status": "unsupported_coverage", + "coverage_status": "disabled" if not enabled else "unsupported", + "coverage_observed": False, + "active": None, + "dormant": None, + "check_passed": None, + "checker_triggered": None, } if not enabled: # Direct oracle: no opportunity pre-scan, source parsing, or monkeypatches. @@ -162,28 +506,34 @@ def observe(trace, inv, enabled=True): trace.get_start_time(), trace.get_end_time() ) report.update( - status="completed", - checker_result=result.to_dict(), - check_passed=result.check_passed, - checker_triggered=result.triggered, + { + "status": "completed", + "checker_result": json_object( + json.loads(json.dumps(result.to_dict(), cls=MDNONEJSONEncoder)) + ), + "check_passed": boolean( + result.check_passed, "checker.check_passed" + ), + "checker_triggered": boolean(result.triggered, "checker.triggered"), + } ) except Exception as exc: report.update( - status="checker_error", - error={"type": type(exc).__name__, "message": str(exc)}, - check_passed=None, + { + "status": "checker_error", + "error": {"type": type(exc).__name__, "message": str(exc)}, + "check_passed": None, + } ) - report.update(coverage_observed=False, active=None, dormant=None) - return report + report.update({"coverage_observed": False, "active": None, "dormant": None}) + return finalize_report(report) if family not in SUPPORTED: - report.update( - status="unsupported_coverage", check_passed=None, decisions_completed=None - ) - return report - report.update(candidate_counts(trace, inv)) + return finalize_report(report) + candidates = candidate_counts(trace, inv) + report.update(candidates) sites = observer_sites(inv.relation.static_check_all, family) report["method_source_sha256"] = sites["method_source_sha256"] - counts = Counter( + counts: Counter[str] = Counter( decisions_started=0, decisions_completed=0, visited_calls=0, @@ -192,8 +542,8 @@ def observe(trace, inv, enabled=True): precondition_group_failures=0, precondition_group_errors=0, ) - groups = {} - decision_records = [] + groups: dict[str, GroupCounts] = {} + decision_records: list[Decision] = [] variable_precondition_requested = ( family == "APIContainRelation" and "var_events" in inv.precondition.get_group_names() @@ -202,10 +552,16 @@ def observe(trace, inv, enabled=True): ) variable_check_skipped = False method_code = inv.relation.static_check_all.__code__ - pending = None + pending: PendingDecision | None = None + observer_failure: ObserverContractError | None = None original_verify = GroupedPreconditions.verify - def verify(precondition, records, group_name, trace_for_metavars=None): + def verify( + precondition: GroupedPreconditions, + records: list[object], + group_name: str, + trace_for_metavars: Trace | None = None, + ) -> bool: group = groups.setdefault( group_name, {"attempts": 0, "passes": 0, "failures": 0, "errors": 0} ) @@ -219,12 +575,14 @@ def verify(precondition, records, group_name, trace_for_metavars=None): counts["precondition_group_errors"] += 1 group["errors"] += 1 raise - label = "passes" if result else "failures" + label: Literal["passes", "failures"] = "passes" if result else "failures" group[label] += 1 counts["precondition_group_" + label] += 1 return result - def trace_event(frame, event, arg): + def trace_event_impl( + frame: FrameType, event: str, arg: object + ) -> TraceCallback | None: nonlocal pending, variable_check_skipped if frame.f_code is not method_code: return None @@ -246,22 +604,18 @@ def trace_event(frame, event, arg): if frame.f_lineno in sites["visits"]: counts["visited_calls"] += 1 if frame.f_lineno in sites["starts"] and pending is None: - if variable_precondition_requested and frame.f_locals.get( - "skip_var_unchanged_check" - ): - variable_check_skipped = True + if variable_precondition_requested: + try: + skipped = boolean( + required(frame.f_locals, "skip_var_unchanged_check"), + "skip_var_unchanged_check", + ) + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + if skipped: + variable_check_skipped = True + record = decision_record(family, frame.f_locals) counts["decisions_started"] += 1 - call = frame.f_locals.get( - "func_call_id", frame.f_locals.get("parent_func_call_id") - ) - record = {"func_call_id": call} - if family == "DistinctArgumentRelation": - record = { - "pair_func_call_ids": [ - frame.f_locals[e].get("func_call_id") - for e in ("event1", "event2") - ] - } pending = { "start": frame.f_lineno, "end": sites["ends"][frame.f_lineno], @@ -269,37 +623,68 @@ def trace_event(frame, event, arg): } return trace_event + def trace_event(frame: FrameType, event: str, arg: object) -> TraceCallback | None: + nonlocal observer_failure + try: + return trace_event_impl(frame, event, arg) + except ObserverContractError as exc: + # A checker may catch an exception raised by sys.settrace. Retain it + # independently so source drift cannot become a completed check. + observer_failure = exc + raise + previous_trace = sys.gettrace() if enabled and previous_trace is not None: raise RuntimeError("A pre-existing Python trace observer would be overwritten") try: - with ( - patch.object(GroupedPreconditions, "verify", verify) - if enabled - else patch.object(GroupedPreconditions, "verify", original_verify) - ): - if enabled: - sys.settrace(trace_event) + with patch.object(GroupedPreconditions, "verify", verify): + sys.settrace(trace_event) result = inv.check(trace, check_relation_first=False) + if observer_failure is not None: + raise observer_failure result.calc_and_set_time_precentage( trace.get_start_time(), trace.get_end_time() ) report.update( - status="completed", - checker_result=result.to_dict(), - check_passed=result.check_passed, - checker_triggered=result.triggered, + { + "status": "completed", + "checker_result": json_object( + json.loads(json.dumps(result.to_dict(), cls=MDNONEJSONEncoder)) + ), + "check_passed": boolean( + result.check_passed, "checker.check_passed" + ), + "checker_triggered": boolean(result.triggered, "checker.triggered"), + } ) except Exception as exc: + failure = observer_failure if observer_failure is not None else exc report.update( - status="checker_error", - error={"type": type(exc).__name__, "message": str(exc)}, - check_passed=None, + { + "status": ( + "observer_error" + if isinstance(failure, ObserverContractError) + else "checker_error" + ), + "error": {"type": type(failure).__name__, "message": str(failure)}, + "check_passed": None, + } ) + if observer_failure is not None and exc is not observer_failure: + report["checker_error_after_observer"] = { + "type": type(exc).__name__, + "message": str(exc), + } finally: if enabled: sys.settrace(previous_trace) - report.update(counts) + report["decisions_started"] = counts["decisions_started"] + report["decisions_completed"] = counts["decisions_completed"] + report["visited_calls"] = counts["visited_calls"] + report["precondition_group_attempts"] = counts["precondition_group_attempts"] + report["precondition_group_passes"] = counts["precondition_group_passes"] + report["precondition_group_failures"] = counts["precondition_group_failures"] + report["precondition_group_errors"] = counts["precondition_group_errors"] report["precondition_groups"] = groups report["decision_records"] = decision_records report["decisions_not_completed"] = ( @@ -310,36 +695,47 @@ def trace_event(frame, event, arg): report["visited_calls"] = None report["unvisited_calls_after_stop"] = None report["unvisited_pairs_after_stop"] = ( - max(0, report["opportunities"] - counts["decisions_started"]) + max(0, candidates["opportunities"] - counts["decisions_started"]) if stopped else 0 ) else: report["unvisited_calls_after_stop"] = ( - max(0, report["candidate_calls"] - counts["visited_calls"]) + max(0, candidates["candidate_calls"] - counts["visited_calls"]) if stopped else 0 ) report["active"] = counts["decisions_completed"] > 0 report["dormant"] = report["status"] == "completed" and not report["active"] - report["coverage_observed"] = True + report["coverage_observed"] = report["status"] != "observer_error" + report["coverage_status"] = "observed" if report["coverage_observed"] else "error" + if not report["coverage_observed"]: + report["active"] = None + report["dormant"] = None report["variable_precondition_requested"] = variable_precondition_requested report["variable_check_skipped"] = variable_check_skipped report["fully_validated_active"] = ( - report["active"] + report["active"] is True and not variable_check_skipped and report["status"] == "completed" ) - return report + return finalize_report(report) + +class CoverageOptions(argparse.Namespace): + traces: list[Path] + invariants: Path + output: Path + observer_off: bool -def main(): + +def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--traces", nargs="+", type=Path, required=True) parser.add_argument("--invariants", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--observer-off", action="store_true") - args = parser.parse_args() + args = parser.parse_args(namespace=CoverageOptions()) logging.basicConfig(level=logging.ERROR) import traincheck.utils @@ -347,24 +743,16 @@ def main(): before = {str(p): sha256(p) for p in [*args.traces, args.invariants]} trace = read_trace_file_Pandas(list(map(str, args.traces))) invariants = read_inv_file(str(args.invariants)) - results = [] + results: list[ObservationReport] = [] for index, inv in enumerate(invariants): result = observe(trace, inv, enabled=not args.observer_off) result["invariant_index"] = index results.append(result) - summary = Counter() - for result in results: - summary[result["status"]] += 1 - summary["active_invariants"] += bool(result.get("active")) - summary["dormant_invariants"] += bool(result.get("dormant")) - summary["violated_invariants"] += result.get("check_passed") is False - summary["identity_bearing_invariants"] += bool(result.get("identity_clauses")) - summary["identity_bearing_active"] += bool( - result.get("identity_clauses") - ) and bool(result.get("active")) + summary = summarize(results) import traincheck.trace.trace_pandas as trace_module - output = { + output: CoverageOutput = { + "schema_version": 2, "pid": os.getpid(), "check_relation_first": False, "observer_enabled": not args.observer_off, @@ -377,8 +765,8 @@ def main(): }, }, "harness_sha256": sha256(Path(__file__)), - "raw_process_ids": trace.get_process_ids(), - "raw_thread_ids": trace.get_thread_ids(), + "raw_process_ids": json_list(trace.get_process_ids(), "raw_process_ids"), + "raw_thread_ids": json_list(trace.get_thread_ids(), "raw_thread_ids"), "trace_rows": len(trace.events), "input_sha256": before, "inputs_unchanged": all(sha256(Path(p)) == h for p, h in before.items()), @@ -387,7 +775,7 @@ def main(): } args.output.write_text(json.dumps(output, cls=MDNONEJSONEncoder, indent=2) + "\n") print(json.dumps(summary), flush=True) - return int(bool(summary["checker_error"] or summary["unsupported_coverage"])) + return int(any(result["status"] != "completed" for result in results)) if __name__ == "__main__": diff --git a/tools/fresh_validation/compare_study.py b/tools/fresh_validation/compare_study.py index b79fccba..1822556c 100644 --- a/tools/fresh_validation/compare_study.py +++ b/tools/fresh_validation/compare_study.py @@ -1,35 +1,144 @@ -"""Verify the paired study's actual outputs; do not label dormant rules as passes.""" +"""Verify actual paired-study outputs; missing observation never becomes a pass.""" import argparse import hashlib import json import math +import sys from collections import Counter +from collections.abc import Mapping from pathlib import Path +from typing import TypedDict +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + boolean, + integer, + json_object, + json_value, + mapping, + number, + read_object, + required, + sequence, + string, +) -def canonical(value): + +class TrainingComparison(TypedDict): + loss_trajectory_equal: bool + initial_state_sha256_equal: bool + final_state_sha256_equal: bool + shapes_equal: bool + + +class StudyComparison(TypedDict): + status: str + invariants_sha256: str + training_comparisons: dict[str, TrainingComparison] + benchmark_data_runtime_source_and_harness_matched: bool + observer_matches_direct_checker: bool + per_run_summary: dict[str, JSONObject] + per_relation: dict[str, dict[str, dict[str, int]]] + raw_ids: dict[str, JSONObject] + fault: JSONObject + limitations: list[str] + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def canonical(value: object) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":")) -def compare(root): - def read(name): - return json.loads((root / name).read_text()) +def object_field(data: Mapping[str, object], key: str) -> Mapping[str, object]: + return mapping(required(data, key), key) - phases = { - p.parent.name: json.loads(p.read_text()) - for p in root.glob("*/phase-result.json") - } - assert phases and all( - p["exit_code"] == 0 and p["stop_reason"] is None and p["source_unchanged"] - for p in phases.values() - ), phases - metrics = { - role: read(role + "/metrics.json") - for role in ("reference", "control", "holdout", "baseline-holdout") + +def objects_field(data: Mapping[str, object], key: str) -> list[Mapping[str, object]]: + return [mapping(item, key) for item in sequence(required(data, key), key)] + + +def bool_field(data: Mapping[str, object], key: str) -> bool: + return boolean(required(data, key), key) + + +def int_field(data: Mapping[str, object], key: str) -> int: + return integer(required(data, key), key) + + +def str_field(data: Mapping[str, object], key: str) -> str: + return string(required(data, key), key) + + +def losses(metric: Mapping[str, object]) -> list[dict[str, int | float]]: + rows = [ + { + key: number(value, key) + for key, value in step.items() + if key.startswith("loss") + } + for step in objects_field(metric, "steps") + ] + require( + bool(rows) and all(row for row in rows), + "Every training step must contain a loss", + ) + require( + all(math.isfinite(value) for row in rows for value in row.values()), + "Non-finite loss", + ) + require( + int_field(metric, "completed_steps") == len(rows), + "Loss row count does not match completed_steps", + ) + return rows + + +def compare(root: Path) -> StudyComparison: + def read(name: str) -> JSONObject: + return read_object(root / name) + + phases = {p.parent.name: read_object(p) for p in root.glob("*/phase-result.json")} + expected_phases = { + "reference", + "control", + "holdout", + "baseline-holdout", + "inference", + "check-reference", + "check-control", + "check-holdout", + "check-holdout-direct", + "prepare-fault", + "check-fault", } - adaptations = {role: read(role + "/adaptation.json") for role in metrics} - assert all(m["success"] and m["state_shapes_unchanged"] for m in metrics.values()) + require( + expected_phases <= phases.keys(), + f"Missing completed phases: {sorted(expected_phases - phases.keys())}", + ) + for name, phase in phases.items(): + require( + int_field(phase, "exit_code") == 0 + and required(phase, "stop_reason") is None + and bool_field(phase, "source_unchanged"), + f"Phase {name} did not complete with unchanged source", + ) + roles = ("reference", "control", "holdout", "baseline-holdout") + metrics = {role: read(role + "/metrics.json") for role in roles} + adaptations = {role: read(role + "/adaptation.json") for role in roles} + require( + all( + bool_field(m, "success") and bool_field(m, "state_shapes_unchanged") + for m in metrics.values() + ), + "Training failed or state shapes changed", + ) workload_keys = ( "benchmark_source_sha256", "extracted_definitions_sha256", @@ -45,145 +154,202 @@ def read(name): "torch_version", "python_version", ) - for role in metrics: + for role in roles: for key in workload_keys: - assert adaptations[role][key] == adaptations["reference"][key], (role, key) + require( + required(adaptations[role], key) + == required(adaptations["reference"], key), + f"Mismatched {role} adaptation field: {key}", + ) invocations = [read(name + "/invocation.json") for name in phases] - assert all( - i["source_sha256"] == invocations[0]["source_sha256"] for i in invocations - ) - assert all( - i["harness_sha256"] == invocations[0]["harness_sha256"] for i in invocations + for key in ("source_sha256", "harness_sha256"): + require( + all( + object_field(i, key) == object_field(invocations[0], key) + for i in invocations + ), + f"Mismatched phase {key}", + ) + require( + len({int_field(adaptations[role], "process_id") for role in roles}) == 4, + "Training phases must use distinct processes", ) - - assert len({adaptations[r]["process_id"] for r in metrics}) == 4 - for role in metrics: - assert metrics[role]["completed_steps"] == ( - 10 if role in ("reference", "control") else 20 + for role in roles: + require( + int_field(metrics[role], "completed_steps") + == (10 if role in ("reference", "control") else 20), + f"Incorrect completed step count for {role}", ) if role != "baseline-holdout": - assert adaptations[role]["tracer_module"] == str( - Path(adaptations[role]["source_root"]) - / "traincheck/instrumentor/tracer.py" + require( + str_field(adaptations[role], "tracer_module") + == str( + Path(str_field(adaptations[role], "source_root")) + / "traincheck/instrumentor/tracer.py" + ), + f"Wrong tracer source for {role}", ) - if adaptations[role]["data"]["kind"].startswith("synthetic"): - raise AssertionError("Synthetic data is outside this study protocol") - - def losses(metric): - rows = [ - {k: v for k, v in step.items() if k.startswith("loss")} - for step in metric["steps"] - ] - assert all(math.isfinite(v) for row in rows for v in row.values()) - return rows - - paired = {} + require( + not str_field(object_field(adaptations[role], "data"), "kind").startswith( + "synthetic" + ), + "Synthetic data is outside this study protocol", + ) + + paired: dict[str, TrainingComparison] = {} for name, left, right in ( ("same_seed_fresh_control", "reference", "control"), ("holdout_instrumentation_comparison", "holdout", "baseline-holdout"), ): m1, m2 = metrics[left], metrics[right] - checks = { + checks: TrainingComparison = { "loss_trajectory_equal": losses(m1) == losses(m2), - "initial_state_sha256_equal": m1["initial_model_state_sha256"] - == m2["initial_model_state_sha256"], - "final_state_sha256_equal": m1["final_model_state_sha256"] - == m2["final_model_state_sha256"], - "shapes_equal": m1["state_shapes"] == m2["state_shapes"], + "initial_state_sha256_equal": str_field(m1, "initial_model_state_sha256") + == str_field(m2, "initial_model_state_sha256"), + "final_state_sha256_equal": str_field(m1, "final_model_state_sha256") + == str_field(m2, "final_model_state_sha256"), + "shapes_equal": object_field(m1, "state_shapes") + == object_field(m2, "state_shapes"), } - assert all(checks.values()), (name, checks) + require(all(checks.values()), f"Training comparison failed: {name}: {checks}") paired[name] = checks - coverage = { role: read("check-" + role + "/coverage.json") for role in ("reference", "control", "holdout", "holdout-direct", "fault") } - assert all(c["inputs_unchanged"] for c in coverage.values()) - frozen_hash = hashlib.sha256( - (root / "inference/invariants.jsonl").read_bytes() - ).hexdigest() + require( + all(bool_field(c, "inputs_unchanged") for c in coverage.values()), + "Coverage inputs changed", + ) + frozen_path = root / "inference/invariants.jsonl" + frozen_hash = hashlib.sha256(frozen_path.read_bytes()).hexdigest() for checked in coverage.values(): - assert ( - checked["input_sha256"][str(root / "inference/invariants.jsonl")] - == frozen_hash + require( + str_field(object_field(checked, "input_sha256"), str(frozen_path)) + == frozen_hash, + "Frozen invariants changed", + ) + observed = objects_field(coverage["holdout"], "per_invariant") + direct = objects_field(coverage["holdout-direct"], "per_invariant") + require(len(observed) == len(direct), "Observer/direct result counts differ") + for a, b in zip(observed, direct): + require( + str_field(a, "status") == str_field(b, "status") == "completed", + "Checker did not complete", + ) + require( + canonical(object_field(a, "checker_result")) + == canonical(object_field(b, "checker_result")), + "Observer changed checker result", + ) + require( + required(b, "active") is None + and required(b, "dormant") is None + and not bool_field(b, "coverage_observed"), + "Disabled observation must remain unknown", ) - observed = coverage["holdout"]["per_invariant"] - direct = coverage["holdout-direct"]["per_invariant"] - assert len(observed) == len(direct) - assert all( - canonical(a["checker_result"]) == canonical(b["checker_result"]) - for a, b in zip(observed, direct) - ) - assert all(item["active"] is None and item["dormant"] is None for item in direct) mutation = read("fault/mutation.json") - selected = mutation["invariant_index"] - healthy = observed[selected] - faulty = coverage["fault"]["per_invariant"][selected] - assert healthy["check_passed"] and healthy["active"] - assert faulty["status"] == "completed" and not faulty["check_passed"] - assert faulty["decisions_completed"] > 0 and faulty["precondition_group_passes"] > 0 - assert any( - d.get("func_call_id") == mutation["func_call_id"] - for d in faulty["decision_records"] + selected = int_field(mutation, "invariant_index") + fault_results = objects_field(coverage["fault"], "per_invariant") + require( + 0 <= selected < len(observed) and selected < len(fault_results), + "Fault invariant index is out of range", ) - assert any( - event.get("func_call_id") == mutation["func_call_id"] - for event in faulty["checker_result"]["trace"] + healthy, faulty = observed[selected], fault_results[selected] + require( + bool_field(healthy, "check_passed") and bool_field(healthy, "active"), + "Fault target was not healthy and active", ) - assert ( - mutation["ids_unchanged"] - and mutation["precondition_inputs_unchanged"] - and len(mutation["edits"]) == 1 + require( + str_field(faulty, "status") == "completed" + and not bool_field(faulty, "check_passed"), + "Fault did not cause a completed violation", ) - assert ( - coverage["fault"]["raw_process_ids"] == coverage["holdout"]["raw_process_ids"] + require( + int_field(faulty, "decisions_completed") > 0 + and int_field(faulty, "precondition_group_passes") > 0, + "Fault target did not activate", ) - assert coverage["fault"]["raw_thread_ids"] == coverage["holdout"]["raw_thread_ids"] - families = {} + fault_identity = str_field(mutation, "func_call_id") + # This selected rule is ConsistentOutputRelation; both records require a + # single call identity. A pair record or missing ID is a malformed report. + require( + any( + str_field(d, "func_call_id") == fault_identity + for d in objects_field(faulty, "decision_records") + ), + "Fault decision identity not observed", + ) + require( + any( + str_field(event, "func_call_id") == fault_identity + for event in objects_field(object_field(faulty, "checker_result"), "trace") + ), + "Fault identity absent from checker violation", + ) + edits = objects_field(mutation, "edits") + require( + bool_field(mutation, "ids_unchanged") + and bool_field(mutation, "precondition_inputs_unchanged") + and len(edits) == 1, + "Fault changed more than one permitted field", + ) + for key in ("raw_process_ids", "raw_thread_ids"): + require( + required(coverage["fault"], key) == required(coverage["holdout"], key), + f"Fault altered {key}", + ) + families: dict[str, dict[str, dict[str, int]]] = {} for role in ("reference", "control", "holdout", "fault"): - family_counts = {} - for item in coverage[role]["per_invariant"]: - assert item["status"] == "completed", item - counts = family_counts.setdefault(item["relation"], Counter()) + family_counts: dict[str, Counter[str]] = {} + for item in objects_field(coverage[role], "per_invariant"): + require( + str_field(item, "status") == "completed" + and bool_field(item, "coverage_observed"), + "Incomplete or unobserved coverage", + ) + counts = family_counts.setdefault(str_field(item, "relation"), Counter()) counts.update( invariants=1, - active=int(item["active"]), - dormant=int(item["dormant"]), - violations=int(not item["check_passed"]), - opportunities=item["opportunities"], - precondition_group_passes=item["precondition_group_passes"], - actual_decisions=item["decisions_completed"], - proxy_checks_skipped=int(item["variable_check_skipped"]), - fully_validated_active=int(item["fully_validated_active"]), + active=int(bool_field(item, "active")), + dormant=int(bool_field(item, "dormant")), + violations=int(not bool_field(item, "check_passed")), + opportunities=int_field(item, "opportunities"), + precondition_group_passes=int_field(item, "precondition_group_passes"), + actual_decisions=int_field(item, "decisions_completed"), + proxy_checks_skipped=int(bool_field(item, "variable_check_skipped")), + fully_validated_active=int(bool_field(item, "fully_validated_active")), ) families[role] = {name: dict(counts) for name, counts in family_counts.items()} - output = { + output: StudyComparison = { "status": "study_completed", "invariants_sha256": frozen_hash, "training_comparisons": paired, "benchmark_data_runtime_source_and_harness_matched": True, "observer_matches_direct_checker": True, "per_run_summary": { - r: coverage[r]["summary"] + r: json_object(required(coverage[r], "summary")) for r in ("reference", "control", "holdout", "fault") }, "per_relation": families, "raw_ids": { r: { - "process": coverage[r]["raw_process_ids"], - "thread": coverage[r]["raw_thread_ids"], + "process": json_value(required(coverage[r], "raw_process_ids")), + "thread": json_value(required(coverage[r], "raw_thread_ids")), } for r in ("reference", "control", "holdout") }, "fault": { - "kind": mutation["kind"], + "kind": str_field(mutation, "kind"), "invariant_index": selected, - "healthy_decisions": healthy["decisions_completed"], - "fault_decisions": faulty["decisions_completed"], - "fault_precondition_group_passes": faulty["precondition_group_passes"], + "healthy_decisions": int_field(healthy, "decisions_completed"), + "fault_decisions": int_field(faulty, "decisions_completed"), + "fault_precondition_group_passes": int_field( + faulty, "precondition_group_passes" + ), "violation": True, - "edit": mutation["edits"][0], + "edit": json_object(edits[0]), }, "limitations": [ "One reference seed and one holdout seed per workload; no generalization estimate.", diff --git a/tools/fresh_validation/contracts.py b/tools/fresh_validation/contracts.py new file mode 100644 index 00000000..831ebe17 --- /dev/null +++ b/tools/fresh_validation/contracts.py @@ -0,0 +1,91 @@ +"""Validated JSON boundaries shared by the correctness-study tools. + +Unknown JSON and frame metadata enter as ``object``. These helpers never replace +required fields with defaults, and distinguish booleans from numeric counts. +""" + +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import TypeAlias, cast + +JSONValue: TypeAlias = ( + None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] +) +JSONObject: TypeAlias = dict[str, JSONValue] + + +def mapping(value: object, context: str) -> Mapping[str, object]: + if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): + raise ValueError(f"{context} must be an object with string keys") + # The check establishes the key type; read-only Mapping does not promise + # anything about external values before the individual field validators run. + return cast(Mapping[str, object], value) + + +def sequence(value: object, context: str) -> Sequence[object]: + if not isinstance(value, list): + raise ValueError(f"{context} must be a JSON array") + return value + + +def string(value: object, context: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string") + return value + + +def integer(value: object, context: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{context} must be an integer") + return value + + +def boolean(value: object, context: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{context} must be a boolean") + return value + + +def number(value: object, context: str) -> int | float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{context} must be a number") + return value + + +def required(data: Mapping[str, object], key: str) -> object: + if key not in data: + raise ValueError(f"Missing required field: {key}") + return data[key] + + +def json_value(value: object, context: str = "JSON") -> JSONValue: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, list): + return [json_value(child, context) for child in value] + if isinstance(value, Mapping): + return { + key: json_value(child, f"{context}.{key}") + for key, child in mapping(value, context).items() + } + raise ValueError(f"{context} has a non-JSON value of type {type(value).__name__}") + + +def json_object(value: object, context: str = "JSON") -> JSONObject: + result = json_value(value, context) + if not isinstance(result, dict): + raise ValueError(f"{context} must be an object") + return result + + +def json_list(value: object, context: str) -> list[JSONValue]: + result = json_value(value, context) + if not isinstance(result, list): + raise ValueError(f"{context} must be an array") + return result + + +def read_object(path: Path) -> JSONObject: + value: object = json.loads(path.read_text()) + return json_object(value, str(path)) diff --git a/tools/fresh_validation/fault_control.py b/tools/fresh_validation/fault_control.py index f86dab36..66e64e34 100644 --- a/tools/fresh_validation/fault_control.py +++ b/tools/fresh_validation/fault_control.py @@ -8,21 +8,79 @@ import copy import hashlib import json +import sys from pathlib import Path +from typing import TypedDict +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + json_object, + mapping, + read_object, + required, + sequence, + string, +) -def digest(path): + +class Edit(TypedDict): + file: str + zero_based_line: int + path: list[str | int] + attribute: str + before_value: JSONValue + after_value: JSONValue + raw_process_id: JSONValue + raw_thread_id: JSONValue + + +class Fingerprints(TypedDict): + original_sha256: str + fault_sha256: str + + +class MutationReport(TypedDict): + kind: str + selection: str + attribute: str + invariant_index: int + invariant: JSONObject + func_call_id: str + edits: list[Edit] + traces: dict[str, Fingerprints] + invariants_sha256: str + healthy_coverage_sha256: str + precondition_inputs_unchanged: bool + ids_unchanged: bool + invariants_not_retrained_or_rewritten: bool + + +def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def differences(before, after, path=()): +def differences( + before: object, after: object, path: tuple[str | int, ...] = () +) -> list[tuple[str | int, ...]]: if type(before) is not type(after): return [path] if isinstance(before, dict): - if before.keys() != after.keys(): + before_dict = mapping(before, "before") + after_dict = mapping(after, "after") + if before_dict.keys() != after_dict.keys(): return [path] - return [p for k in before for p in differences(before[k], after[k], (*path, k))] + return [ + p + for k in before_dict + for p in differences(before_dict[k], after_dict[k], (*path, k)) + ] if isinstance(before, list): + assert isinstance(after, list) if len(before) != len(after): return [path] return [ @@ -37,7 +95,7 @@ def differences(before, after, path=()): return [path] -def replacement(attributes, attribute): +def replacement(attributes: JSONValue, attribute: str) -> JSONValue: """Return one valid same-type unequal metadata value, or no eligible edit.""" if not isinstance(attributes, dict): return None @@ -55,35 +113,74 @@ def replacement(attributes, attribute): return None -def mutate(traces, invariants, healthy, output): - rules = [json.loads(line) for line in invariants.read_text().splitlines()] - coverage = json.loads(healthy.read_text()) - assert coverage["inputs_unchanged"] and coverage["observer_enabled"] - assert coverage["input_sha256"][str(invariants)] == digest(invariants) - chosen = None - call_id = None - post_events = {} +def mutate( + traces: list[Path], invariants: Path, healthy: Path, output: Path +) -> MutationReport: + rules = [ + json_object(json.loads(line), "invariant") + for line in invariants.read_text().splitlines() + ] + coverage = read_object(healthy) + if not boolean( + required(coverage, "inputs_unchanged"), "inputs_unchanged" + ) or not boolean(required(coverage, "observer_enabled"), "observer_enabled"): + raise ValueError( + "Fault selection requires unchanged inputs and enabled observation" + ) + input_hashes = mapping(required(coverage, "input_sha256"), "input_sha256") + + def verify_hash(path: Path) -> None: + if string(required(input_hashes, str(path)), str(path)) != digest(path): + raise ValueError(f"Input changed since coverage collection: {path}") + + verify_hash(invariants) + chosen: JSONObject | None = None + call_id: str | None = None + selected_index: int | None = None + post_events: dict[str, list[JSONObject]] = {} for trace in traces: - assert coverage["input_sha256"][str(trace)] == digest(trace) + verify_hash(trace) for line in trace.read_bytes().splitlines(): - event = json.loads(line) - if event.get("type") == "function_call (post)": - post_events.setdefault(event["func_call_id"], []).append(event) + event = json_object(json.loads(line), "trace event") + if string(required(event, "type"), "event.type") == "function_call (post)": + identity = string(required(event, "func_call_id"), "event.func_call_id") + post_events.setdefault(identity, []).append(event) # Prefer shape; dtype is a prospective fallback when no live mutable shape # rule exists. Selection remains post hoc and does not change learned rules. for attribute in ("shape", "dtype"): - for item in coverage["per_invariant"]: - rule = rules[item["invariant_index"]] + for raw_item in sequence(required(coverage, "per_invariant"), "per_invariant"): + item = json_object(raw_item, "per_invariant item") + index = integer(required(item, "invariant_index"), "invariant_index") + if not 0 <= index < len(rules): + raise ValueError(f"Invariant index out of range: {index}") + rule = rules[index] + if ( + string(required(rule, "relation"), "relation") + != "ConsistentOutputRelation" + ): + continue + params = sequence(required(rule, "params"), "params") + if len(params) != 2: + raise ValueError("ConsistentOutputRelation requires two parameters") + output_param = mapping(params[1], "output parameter") if not ( - rule["relation"] == "ConsistentOutputRelation" - and rule["params"][1]["attr_name"] == attribute - and item["active"] - and item["check_passed"] - and item["decisions_completed"] > 0 + string(required(output_param, "attr_name"), "attr_name") == attribute + and boolean(required(item, "active"), "active") + and boolean(required(item, "check_passed"), "check_passed") + and integer( + required(item, "decisions_completed"), "decisions_completed" + ) + > 0 ): continue - for decision in item["decision_records"]: - events = post_events.get(decision["func_call_id"], []) + for raw_decision in sequence( + required(item, "decision_records"), "decision_records" + ): + decision = mapping(raw_decision, "decision") + identity = string( + required(decision, "func_call_id"), "decision.func_call_id" + ) + events = post_events.get(identity, []) if len(events) != 1: continue values = events[0]["return_values"] @@ -95,7 +192,7 @@ def mutate(traces, invariants, healthy, output): for name, attributes in tensor.items() ) if eligible: - chosen, call_id = item, decision["func_call_id"] + chosen, call_id, selected_index = item, identity, index break if chosen is not None: break @@ -103,20 +200,22 @@ def mutate(traces, invariants, healthy, output): break if chosen is None: raise ValueError("No healthy, actually evaluated mutable shape or dtype rule") - rule = rules[chosen["invariant_index"]] - assert call_id is not None + assert call_id is not None and selected_index is not None + rule = rules[selected_index] output.mkdir(parents=True, exist_ok=False) (output / "traces").mkdir() - edits, fingerprints = [], {} + edits: list[Edit] = [] + fingerprints: dict[str, Fingerprints] = {} for trace in traces: - assert coverage["input_sha256"][str(trace)] == digest(trace) + verify_hash(trace) raw_lines = trace.read_bytes().splitlines(keepends=True) transformed = list(raw_lines) for index, raw in enumerate(raw_lines): - event = json.loads(raw) + event = json_object(json.loads(raw), "trace event") if ( - event.get("func_call_id") != call_id - or event.get("type") != "function_call (post)" + string(required(event, "type"), "event.type") != "function_call (post)" + or string(required(event, "func_call_id"), "event.func_call_id") + != call_id ): continue before = copy.deepcopy(event) @@ -126,7 +225,7 @@ def mutate(traces, invariants, healthy, output): if not isinstance(value, dict): continue for typename, attributes in value.items(): - if "Tensor" not in typename: + if "Tensor" not in typename or not isinstance(attributes, dict): continue new_value = replacement(attributes, attribute) if new_value is None: @@ -173,11 +272,11 @@ def mutate(traces, invariants, healthy, output): "fault_sha256": digest(target), } assert len(edits) == 1, edits - manifest = { + manifest: MutationReport = { "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", "attribute": attribute, - "invariant_index": chosen["invariant_index"], + "invariant_index": selected_index, "invariant": rule, "func_call_id": call_id, "edits": edits, diff --git a/tools/fresh_validation/prepare_source.py b/tools/fresh_validation/prepare_source.py index a2d498ba..39eedf6d 100644 --- a/tools/fresh_validation/prepare_source.py +++ b/tools/fresh_validation/prepare_source.py @@ -7,6 +7,18 @@ import subprocess import tarfile from pathlib import Path +from typing import TypedDict + + +class SourceManifest(TypedDict): + base: str + trace_integrity: str + torchscript: str + state_fix_hunks_from: str + changed_python_files: list[str] + dynamic_typename_fix: str + source_sha256: dict[str, str] + BASE = "99d16905296d83495570fdcafa91370b242d2528" INTEGRITY = "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4" @@ -15,12 +27,12 @@ TYPENAME = "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c" -def digest(data): +def digest(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def build(repo, destination): - def git(*args): +def build(repo: Path, destination: Path) -> SourceManifest: + def git(*args: str) -> bytes: return subprocess.check_output(["git", "-C", str(repo), *args]) destination.mkdir(parents=True, exist_ok=False) @@ -29,9 +41,13 @@ def git(*args): ) as tar: for member in tar: if member.isfile(): - path = destination / member.name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(tar.extractfile(member).read()) + member_path = destination / member.name + member_path.parent.mkdir(parents=True, exist_ok=True) + extracted = tar.extractfile(member) + assert ( + extracted is not None + ), "Regular archive member must have contents" + member_path.write_bytes(extracted.read()) old_hashes = { str(p.relative_to(destination)): digest(p.read_bytes()) for p in destination.rglob("*.py") @@ -58,8 +74,8 @@ def git(*args): } for path, commit in replacements.items(): (destination / path).write_bytes(git("show", f"{commit}:{path}")) - path = destination / "traincheck/invariant/precondition.py" - source = path.read_text() + source_path = destination / "traincheck/invariant/precondition.py" + source = source_path.read_text() old = "_pruned_clauses: set[PreconditionClause] = set()," new = "_pruned_clauses: set[PreconditionClause] | None = None," assert source.count(old) == 1 @@ -75,13 +91,13 @@ def git(*args): assert source.count(anchor) == 1 source = source.replace(anchor, insertion + anchor, 1) assert "group_rng = random.Random(42)" in source - path.write_text(source) + source_path.write_text(source) hashes = { str(p.relative_to(destination)): digest(p.read_bytes()) for p in destination.rglob("*.py") } changed = sorted(p for p, h in hashes.items() if h != old_hashes.get(p)) - manifest = { + manifest: SourceManifest = { "base": BASE, "trace_integrity": INTEGRITY, "torchscript": TORCHSCRIPT, diff --git a/tools/fresh_validation/run_phase.py b/tools/fresh_validation/run_phase.py index 3ab1e64f..9e52a598 100644 --- a/tools/fresh_validation/run_phase.py +++ b/tools/fresh_validation/run_phase.py @@ -8,10 +8,33 @@ import subprocess import time from pathlib import Path +from typing import Literal, TypedDict -def group_memory(group): - total, processes = 0, [] +class PhaseOptions(argparse.Namespace): + source: Path + output: Path + timeout: float + memory_gib: float + trace_mib: float + command: list[str] + + +StopReason = Literal["process_group_rss_budget", "timeout", "trace_size_budget"] + + +class PhaseResult(TypedDict): + exit_code: int + stop_reason: StopReason | None + source_unchanged: bool + pid: int + elapsed_seconds_operational_only: float + peak_process_group_rss_bytes_operational_only: int + + +def group_memory(group: int) -> tuple[int, list[int]]: + total = 0 + processes: list[int] = [] for entry in Path("/proc").iterdir(): if not entry.name.isdigit(): continue @@ -31,7 +54,7 @@ def group_memory(group): return total, processes -def run(args): +def run(args: PhaseOptions) -> int: args.output.mkdir(parents=True, exist_ok=False) source_hashes = { str(p.relative_to(args.source)): hashlib.sha256(p.read_bytes()).hexdigest() @@ -51,7 +74,8 @@ def run(args): VECLIB_MAXIMUM_THREADS="1", ) command = args.command[1:] if args.command[0] == "--" else args.command - started, peak, reason = time.monotonic(), 0, None + started, peak = time.monotonic(), 0 + reason: StopReason | None = None with (args.output / "stdout.log").open("w") as log: proc = subprocess.Popen( ["nice", "-n", "10", *command], @@ -107,7 +131,8 @@ def run(args): hashlib.sha256((args.source / p).read_bytes()).hexdigest() == h for p, h in source_hashes.items() ) - result = { + assert proc.returncode is not None, "Supervisor must reap its child" + result: PhaseResult = { "exit_code": proc.returncode, "stop_reason": reason, "source_unchanged": unchanged, @@ -128,7 +153,7 @@ def run(args): parser.add_argument("--memory-gib", type=float, default=8) parser.add_argument("--trace-mib", type=float, default=256) parser.add_argument("command", nargs=argparse.REMAINDER) - options = parser.parse_args() + options = parser.parse_args(namespace=PhaseOptions()) if not options.command: parser.error("A command is required") raise SystemExit(run(options)) diff --git a/tools/fresh_validation/run_study.py b/tools/fresh_validation/run_study.py index e6c1c7d8..0cbd0f48 100644 --- a/tools/fresh_validation/run_study.py +++ b/tools/fresh_validation/run_study.py @@ -9,10 +9,21 @@ import hashlib import json import subprocess +from collections.abc import Sequence from pathlib import Path +from typing import Literal -def main(): +class StudyOptions(argparse.Namespace): + source: Path + python: Path + benchmark_root: Path + data_root: Path | None + output: Path + workload: Literal["mnist", "transformer"] + + +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source", type=Path, required=True) parser.add_argument("--python", type=Path, required=True) @@ -20,7 +31,7 @@ def main(): parser.add_argument("--data-root", type=Path) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--workload", choices=("mnist", "transformer"), required=True) - args = parser.parse_args() + args = parser.parse_args(namespace=StudyOptions()) if args.workload == "mnist" and args.data_root is None: parser.error("MNIST requires an existing real IDX data cache") args.output = args.output.resolve() @@ -47,7 +58,7 @@ def main(): + "\n" ) - def phase(name, command): + def phase(name: str, command: Sequence[str | Path | int]) -> None: subprocess.run( [ str(args.python), @@ -69,7 +80,7 @@ def phase(name, command): ("holdout", 20, 43, "instrumented"), ("baseline-holdout", 20, 43, "baseline"), ): - command = [ + command: list[str | Path | int] = [ harness / "benchmark_workload.py", "--benchmark-root", args.benchmark_root.resolve(), From db80c19f6ce5066096b413eb9ce4db54b9230caf Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:44:36 +0800 Subject: [PATCH 35/36] Record observer contract regression evidence --- ...esh-validation-contracts-before-after.json | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/measurements/data/fresh-validation-contracts-before-after.json diff --git a/docs/measurements/data/fresh-validation-contracts-before-after.json b/docs/measurements/data/fresh-validation-contracts-before-after.json new file mode 100644 index 00000000..f5120925 --- /dev/null +++ b/docs/measurements/data/fresh-validation-contracts-before-after.json @@ -0,0 +1,35 @@ +[ + { + "fixture": "renamed_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + }, + { + "fixture": "malformed_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + }, + { + "fixture": "catching_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + } +] From 91d151edc047964066e72300549e8dfe7d3e91e0 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:56:04 +0800 Subject: [PATCH 36/36] docs: verify hardened observer against updated PR stack --- ...resh-validation-hardening-integration.json | 186 ++++++++++++++++++ .../fresh-validation-contract-hardening.md | 30 +++ 2 files changed, 216 insertions(+) create mode 100644 docs/measurements/data/fresh-validation-hardening-integration.json diff --git a/docs/measurements/data/fresh-validation-hardening-integration.json b/docs/measurements/data/fresh-validation-hardening-integration.json new file mode 100644 index 00000000..9b2147cf --- /dev/null +++ b/docs/measurements/data/fresh-validation-hardening-integration.json @@ -0,0 +1,186 @@ +{ + "scope": "Frozen saved-study observer verification; no new training or performance claim", + "harness_source_commit": "f3804c0386454ddd1f97bf17b7a1ab9287cd23d2", + "source_composition": { + "inference": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "tracing": "b64023b0651278ab987d3276aaad9141e8bf4894", + "state_patch": "aec748ea46b35b6b43652985f00820afa813175c" + }, + "git_history": "No local merge of #144 into #148; updated production source validated in an isolated snapshot", + "schema_version": 2, + "comparison": "Every original per-invariant field, summary field, raw trace PID/TID, input hash, row count and checking mode preserved; additive schema fields allowed. JSON NaN tokens compare as retained NaN, without deleting/coercing fields. Only operational PID/runtime-source and harness-provenance fields are excluded from semantic equality. Runtime source path verified explicitly.", + "tests": { + "composed_production_linux": 140, + "harness_linux": 28, + "stronger_mypy_1_9_files": 9 + }, + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0" + }, + "per_rule_reports": 3072, + "runs": [ + { + "workload": "mnist", + "phase": "check-reference", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-control", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-holdout", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-fault", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 3, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 3, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-reference", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-control", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 2, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-holdout", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 2, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-fault", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 4, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 4, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + } + ] +} diff --git a/docs/measurements/fresh-validation-contract-hardening.md b/docs/measurements/fresh-validation-contract-hardening.md index 2e7b85b0..850bba25 100644 --- a/docs/measurements/fresh-validation-contract-hardening.md +++ b/docs/measurements/fresh-validation-contract-hardening.md @@ -88,3 +88,33 @@ python -m unittest tests.test_fresh_validation_coverage \ The source-builder recipe remains pinned to its original, reproducible composition. Full saved-trace replay and the newly composed PR stack require separate integration validation. This change makes no new training-overhead or generalization claim. + +## Updated production-stack integration + +An isolated snapshot combines signed inference `ce25c12`, tracing `b64023b`, +and only the #145 production state-isolation patch from `aec748e`, plus the exact +signed harness source `f3804c0`. It does not change this PR's Git history or its +reproducible original source-builder recipe. The combined production passes 140 +Linux regressions; all 28 harness regressions also pass against that source. +Stronger Mypy 1.9 checks pass all nine harness/test files against the updated +production interfaces with actual Torch type resolution. + +All eight frozen saved-study checker runs (reference, same-seed control, +holdout, and controlled fault for each workload) complete successfully with +unchanged sources/inputs and the expected production module path. Every original +semantic field in **3,072 per-rule reports** matches, including decisions, +coverage, raw trace IDs, checker payloads, and summary counts. Additive schema-2 +fields are allowed; meaningful fields and array order are never dropped. Retained +JSON NaN tokens compare as NaN rather than using Python's non-reflexive float +equality. Operational process IDs and changed harness/source provenance are +separate from semantic equality. + +| Workload | Active rules in each run | Reference violations | Control | Holdout | Fault | +|---|---:|---:|---:|---:|---:| +| MNIST | 204 | 0 | 0 | 0 | 3 | +| Transformer | 564 | 0 | 2 | 2 | 4 | + +The Transformer's two documented healthy false alarms remain visible. These are +saved-input checker replays, not new model training or a generalization claim. +[Compact integration evidence](data/fresh-validation-hardening-integration.json) +records exact source composition, runtime, counts, and comparison rules.