From 24e7cb0f7a573ff6225a388eff189d9e4af5e8b5 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:34:03 +0800 Subject: [PATCH 1/5] Index function-call time windows without changing pandas grouping --- tests/test_time_range_index.py | 325 +++++++++++++++++++++++++++++++ traincheck/trace/trace_pandas.py | 158 +++++++++++++-- 2 files changed, 466 insertions(+), 17 deletions(-) create mode 100644 tests/test_time_range_index.py diff --git a/tests/test_time_range_index.py b/tests/test_time_range_index.py new file mode 100644 index 00000000..7ea110e9 --- /dev/null +++ b/tests/test_time_range_index.py @@ -0,0 +1,325 @@ +"""Differential tests against the unchanged pandas filtering path.""" + +import copy +import logging +import math +import pickle +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from traincheck.instrumentor.tracer import TraceLineType as T +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyRangeTrace(TracePandas): + def _get_func_call_time_positions(self, *args): + return None + + +def frame(times=(1, 2, 3, 4), ids=("z", "z", "a", "a")): + n = len(times) + return pd.DataFrame( + { + "time": times, + "type": [T.FUNC_CALL_PRE, T.FUNC_CALL_POST] * (n // 2), + "process_id": pd.Series([1] * n, dtype="int64"), + "thread_id": pd.Series([2] * n, dtype="int64"), + "func_call_id": ids, + "function": ["fixture.step"] * n, + "args": [{} for _ in range(n)], + "kwargs": [{} for _ in range(n)], + "exception": ["fixture failure"] * n, + } + ) + + +def make(data, cls=TracePandas): + return cls(data.copy(deep=True), truncate_incomplete_func_calls=False) + + +def value(v): + if isinstance(v, dict): + return {k: value(x) for k, x in v.items()} + if isinstance(v, list): + return [value(x) for x in v] + if isinstance(v, float) and math.isnan(v): + return (type(v).__name__, "nan") + return (type(v).__name__, v) + + +def outcome(trace, bounds, process=1, thread=2): + try: + events = trace.query_func_call_events_within_time(bounds, process, thread) + return ("result", [(type(e).__name__, value(e.__dict__)) for e in events]) + except Exception as error: + return ("exception", type(error).__name__, str(error)) + + +class IntLike(int): + pass + + +class EqualType: + def __init__(self, text): + self.text = text + + def __eq__(self, other): + return self.text == other + + def __hash__(self): + return hash(self.text) + + +class TimeRangeIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def compare(self, data, queries): + old, new = make(data, LegacyRangeTrace), make(data) + for query in queries: + with self.subTest(query=query, dtypes=str(data.dtypes)): + self.assertEqual(outcome(old, *query), outcome(new, *query)) + return old, new + + def test_strict_bounds_ties_reverse_and_empty(self): + data = frame((1, 2, 2, 3)) + _, new = self.compare( + data, + [ + ((a, b),) + for a, b in [(0, 4), (1, 3), (1, 2), (2, 3), (2, 2), (4, 0), (9, 10)] + ], + ) + self.assertIsNotNone(new._func_time_positions) + self.assertEqual( + [ + e.pre_record["func_call_id"] + for e in new.query_func_call_events_within_time((0, 4), 1, 2) + ], + ["a", "z"], + ) + + def test_empty_native_frame_and_extreme_identity_keys(self): + _, empty = self.compare(frame().iloc[:0], [((0, 5),)]) + self.assertEqual(empty._get_func_call_time_positions((0, 5), 1, 2), []) + for pid, tid in [(-(2**63), 2**63 - 1), (2**63 - 1, -(2**63))]: + data = frame() + data["process_id"] = pd.Series([pid] * 4, dtype="int64") + data["thread_id"] = pd.Series([tid] * 4, dtype="int64") + _, trace = self.compare(data, [((0, 5), pid, tid)]) + self.assertEqual( + trace._get_func_call_time_positions((0, 5), pid, tid), [0, 1, 2, 3] + ) + + def test_float_nan_inf_signed_zero(self): + data = frame( + (-math.inf, -0.0, 0.0, math.inf, math.nan, math.nan), + ("z", "z", "a", "a", "missing", "missing"), + ) + self.compare( + data, + [ + ((a, b),) + for a, b in [ + (-math.inf, math.inf), + (-1.0, 1.0), + (-0.0, 0.0), + (math.inf, -math.inf), + (math.nan, 9.0), + ] + ], + ) + + def test_integer_extremes_and_precision_fallback(self): + for times in [ + (2**53, 2**53 + 1, 2**53 + 2, 2**53 + 3), + (-(2**63), -(2**63) + 1, 2**63 - 2, 2**63 - 1), + ]: + data = frame(times) + old, new = self.compare( + data, + [ + ((times[0], times[-1]),), + ((float(times[0]), float(times[-1])),), + ((-(2**63) - 1, 2**63),), + ], + ) + self.assertIsNone( + new._get_func_call_time_positions( + (float(times[0]), float(times[-1])), 1, 2 + ) + ) + data = frame(tuple(float(2**53 + i) for i in range(4))) + self.compare( + data, [((2**53 + 1, 2**53 + 3),), ((float(2**53), float(2**53 + 4)),)] + ) + + def test_numeric_query_types_fallback(self): + data = frame() + for process, thread in [ + (True, 2), + (1.0, 2.0), + (np.int64(1), np.int64(2)), + (IntLike(1), 2), + (2**64, 2), + (None, 2), + ]: + _, new = self.compare(data, [((0, 5), process, thread)]) + self.assertIsNone( + new._get_func_call_time_positions((0, 5), process, thread) + ) + for bounds in [ + (np.int64(0), np.int64(5)), + (IntLike(0), 5), + [0, 5], + (0,), + None, + ("a", "b"), + ]: + self.compare(data, [(bounds,)]) + + def test_other_processes_threads_and_missing_identity(self): + data = pd.concat([frame(), frame()], ignore_index=True) + data.loc[4:, "process_id"] = 3 + data.loc[4:, "thread_id"] = 4 + self.compare( + data, [((0, 5), pid, tid) for pid, tid in [(1, 2), (3, 4), (1, 4), (99, 2)]] + ) + + def test_unsorted_input_duplicate_labels_preserves_row_order(self): + data = frame((3, 4, 1, 2)) + data.index = [4, 4, 1, 1] + self.compare(data, [((0, 5),), ((0, 3),)]) + backwards = data.iloc[[1, 0, 2, 3]].copy() + self.assertEqual( + self.compare(backwards, [((0, 5),)])[1].query_func_call_events_within_time( + (0, 0), 1, 2 + ), + [], + ) + self.assertEqual(outcome(make(backwards), (0, 5))[1], "AssertionError") + + def test_duplicate_incomplete_and_exception_records(self): + for data in [ + frame().iloc[:3], + pd.concat([frame(), frame().iloc[:1]], ignore_index=True), + ]: + self.compare(data, [((0, 5),), ((0, 3),)]) + data = frame() + data.loc[1, "type"] = T.FUNC_CALL_POST_EXCEPTION + self.compare(data, [((0, 5),), ((0, 3),)]) + + def test_null_mixed_and_categorical_call_ids(self): + for ids in [ + pd.Series(["z", "z", None, None]), + pd.Series([1, 1, "a", "a"], dtype=object), + pd.Categorical(["z", "z", "a", "a"], categories=["z", "a", "unused"]), + pd.Series(["z", "z", "a", "a"], dtype="string"), + ]: + data = frame() + data["func_call_id"] = ids + self.compare(data, [((0, 5),), ((9, 10),)]) + + def test_time_and_identity_dtype_fallback(self): + for column in ["time", "process_id", "thread_id"]: + for dtype in ["object", "Int64", "Float64", "category", "uint64", "int32"]: + data = frame() + data[column] = data[column].astype(dtype) + _, new = self.compare(data, [((0, 5),)]) + self.assertIsNone(new._get_func_call_time_positions((0, 5), 1, 2)) + data = frame() + data[column] = data[column].astype(float) + data.loc[0, column] = np.nan + self.compare(data, [((0, 5),), ((0.0, 5.0),)]) + + def test_type_nulls_extensions_and_custom_comparison(self): + for dtype in ["string", "category"]: + data = frame() + data["type"] = data["type"].astype(dtype) + _, new = self.compare(data, [((0, 5),)]) + self.assertIsNone(new._get_func_call_time_positions((0, 5), 1, 2)) + for kind in [None, np.nan, pd.NA, EqualType(T.FUNC_CALL_PRE)]: + data = frame() + data.at[0, "type"] = kind + self.compare(data, [((0, 5),), ((9, 10),)]) + + def test_non_function_events_are_excluded(self): + data = pd.concat([frame(), frame()], ignore_index=True) + data.loc[4:, "type"] = "unrelated" + self.compare(data, [((0, 5),)]) + + def test_missing_and_duplicate_columns_keep_exceptions(self): + for col in ["type", "process_id", "thread_id", "func_call_id"]: + self.compare(frame().drop(columns=col), [((0, 5),)]) + data = pd.concat([frame(), frame()[["type"]]], axis=1) + self.compare(data, [((0, 5),)]) + + def test_cache_rebuild_on_replacement_and_length_change(self): + old, new = self.compare(frame(), [((0, 5),)]) + for trace in [old, new]: + trace.events = frame((10, 11, 12, 13)) + self.assertEqual(outcome(old, (9, 14)), outcome(new, (9, 14))) + for trace in [old, new]: + trace.events.drop(index=[2, 3], inplace=True) + self.assertEqual(outcome(old, (9, 14)), outcome(new, (9, 14))) + self.assertEqual(new._func_time_length, 2) + + def test_fresh_events_and_record_dicts(self): + trace = make(frame()) + first = trace.query_func_call_events_within_time((0, 5), 1, 2) + second = trace.query_func_call_events_within_time((0, 5), 1, 2) + self.assertIsNot(first[0], second[0]) + self.assertIsNot(first[0].pre_record, second[0].pre_record) + first[0].pre_record["function"] = "mutated" + self.assertEqual(second[0].pre_record["function"], "fixture.step") + self.assertEqual( + trace.query_func_call_events_within_time((0, 5), 1, 2)[0].pre_record[ + "function" + ], + "fixture.step", + ) + + def test_repeated_query_avoids_type_isin_rescan(self): + trace = make(frame()) + with patch.object( + pd.Series, "isin", side_effect=AssertionError("unexpected full scan") + ): + for _ in range(10): + self.assertEqual( + len(trace.query_func_call_events_within_time((0, 5), 1, 2)), 2 + ) + + def test_random_windows_and_permuted_rows(self): + rng = np.random.default_rng(91) + data = pd.concat( + [ + frame( + (i * 4 + 1, i * 4 + 2, i * 4 + 3, i * 4 + 4), + (f"{i}z", f"{i}z", f"{i}a", f"{i}a"), + ) + for i in range(20) + ], + ignore_index=True, + ) + for _ in range(10): + permuted = data.iloc[rng.permutation(len(data))].copy() + self.compare( + permuted, + [((int(a), int(b)),) for a, b in rng.integers(-2, 83, size=(20, 2))], + ) + + def test_pickled_index_retains_frame_and_results(self): + trace = make(frame()) + expected = outcome(trace, (0, 5)) + restored = pickle.loads(pickle.dumps(trace)) + self.assertIs(restored._func_time_source, restored.events) + self.assertEqual(outcome(restored, (0, 5)), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index e7e20cfe..49c65828 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -1,8 +1,11 @@ import logging +import math import re +from bisect import bisect_left, bisect_right from numbers import Real from typing import Any +import numpy as np import pandas as pd from traincheck.config import config @@ -117,6 +120,15 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self._identity_values_length = -1 self._identity_values: dict[str, tuple] = {} + # As with the other caches, time/type/PID/TID values, dtypes, and row + # order must remain stable. Same-size in-place edits require a new trace. + # Frame replacement or length changes invalidate this positional index. + self._func_time_source: pd.DataFrame | None = None + self._func_time_length = -1 + self._func_time_positions: ( + dict[tuple[int, int], tuple[list, list[int]]] | None + ) = None + self.column_dtypes_cached = {} self.stage_traces: dict[str, "TracePandas"] = {} @@ -1240,6 +1252,113 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: return post_record + def _get_func_call_time_positions( + self, time_range, process_id, thread_id + ) -> list[int] | None: + """Return exact positional matches, or None to retain pandas semantics. + + The fast path deliberately excludes extension/object numeric columns and + mixed scalar comparisons: pandas may coerce those differently from Python + ordering (notably integer/float comparisons above 2**53). + """ + columns = ("type", "time", "process_id", "thread_id") + if any( + name not in self.events or not isinstance(self.events[name], pd.Series) + for name in columns + ): + return None + if ( + type(time_range) is not tuple # noqa: E721 - exclude custom indexing + or len(time_range) != 2 + or any( + type(value) is not int # noqa: E721 - exact numeric comparison + for value in (process_id, thread_id) + ) + or any(not -(2**63) <= value < 2**63 for value in (process_id, thread_id)) + or any( + not isinstance(self.events[name].dtype, np.dtype) + or self.events[name].dtype != "int64" + for name in columns[2:] + ) + ): + return None + time_dtype = self.events["time"].dtype + if not isinstance(time_dtype, np.dtype): + return None + if time_dtype == "int64": + if any( + type(bound) is not int # noqa: E721 - exclude numeric subclasses + or not -(2**63) <= bound < 2**63 + for bound in time_range + ): + return None + elif time_dtype == "float64": + if any( + type(bound) is not float # noqa: E721 - preserve pandas coercion + or math.isnan(bound) + for bound in time_range + ): + return None + else: + return None + + if self._func_time_source is not self.events or self._func_time_length != len( + self.events + ): + event_types = self.events["type"] + positions = None + # A custom __eq__ or extension dtype must still execute in pandas. + if ( + isinstance(event_types.dtype, np.dtype) + and pd.api.types.is_object_dtype(event_types.dtype) + and all( + type(value) is str # noqa: E721 - exclude custom equality + or value is None + or value is pd.NA + or (type(value) is float and math.isnan(value)) # noqa: E721 + for value in event_types + ) + ): + rows: dict[tuple[int, int], list[tuple[Any, int]]] = {} + types = event_types.tolist() + times = self.events["time"].tolist() + processes = self.events["process_id"].tolist() + threads = self.events["thread_id"].tolist() + for position, (kind, time, process, thread) in enumerate( + zip(types, times, processes, threads) + ): + if type(kind) is not str or kind not in ( # noqa: E721 + TraceLineType.FUNC_CALL_PRE, + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ): + continue + if time_dtype == "float64" and math.isnan(time): + continue + rows.setdefault((process, thread), []).append((time, position)) + positions = {} + for identity, records in rows.items(): + records.sort() + positions[identity] = ( + [time for time, _ in records], + [position for _, position in records], + ) + self._func_time_positions = positions + self._func_time_source = self.events + self._func_time_length = len(self.events) + + if self._func_time_positions is None: + return None + indexed = self._func_time_positions.get((process_id, thread_id)) + if indexed is None: + return [] + times, row_positions = indexed + left = bisect_right(times, time_range[0]) + right = bisect_left(times, time_range[1]) + # pandas groupby preserves input row order within each call. Time order + # cannot replace that order, even when traces arrive out of order. + return sorted(row_positions[left:right]) + def query_func_call_events_within_time( self, time_range: tuple[int | float, int | float], @@ -1250,25 +1369,30 @@ def query_func_call_events_within_time( # TODO: check if this is correct func_call_events: list[FuncCallEvent | FuncCallExceptionEvent] = [] - func_call_records = self.events[ - ( - self.events["type"].isin( - [ - TraceLineType.FUNC_CALL_PRE, - TraceLineType.FUNC_CALL_POST, - TraceLineType.FUNC_CALL_POST_EXCEPTION, - ] + positions = self._get_func_call_time_positions( + time_range, process_id, thread_id + ) + if positions is not None: + func_call_records = self.events.iloc[positions] + else: + func_call_records = self.events[ + ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_PRE, + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) ) - ) - & ( - self.events["time"].between( - time_range[0], time_range[1], inclusive="neither" + & ( + self.events["time"].between( + time_range[0], time_range[1], inclusive="neither" + ) ) - ) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] # group function calls by func_call_id func_call_groups = func_call_records.groupby("func_call_id") From 39152aded8b97ed83988e3292bb89c6e3adcf750 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:36:13 +0800 Subject: [PATCH 2/5] Add alternating time-window query benchmark with build costs --- tools/benchmarks/time_range_index.py | 179 +++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tools/benchmarks/time_range_index.py diff --git a/tools/benchmarks/time_range_index.py b/tools/benchmarks/time_range_index.py new file mode 100644 index 00000000..427a61d7 --- /dev/null +++ b/tools/benchmarks/time_range_index.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Compare exact pandas filtering with the time-range index on synthetic calls. + +Run: python -m tools.benchmarks.time_range_index > results.json +Each trial uses a fresh TracePandas. Construction and the first lazy index build +are included, along with fresh event/record construction for every query. Input +creation, result hashing, and garbage collection are outside the timed region. +Repetitions alternate legacy/candidate order. This isolates window queries; it +is not full inference and does not measure peak process memory. +""" + +import argparse +import gc +import hashlib +import json +import logging +import os +import platform +import statistics +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as trace_module +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyRangeTrace(TracePandas): + def _get_func_call_time_positions(self, *args): + return None + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=200000) + parser.add_argument("--queries", type=int, default=1000) + parser.add_argument("--repetitions", type=int, default=3) + args = parser.parse_args() + if args.rows < 4 or args.rows % 2 or args.queries < 2 or args.repetitions < 1: + parser.error("require even rows >= 4, queries >= 2, repetitions >= 1") + logging.disable(logging.CRITICAL) + row = np.arange(args.rows, dtype=np.int64) + call = row // 2 + data = pd.DataFrame( + { + "time": row.astype(np.float64), + "type": np.where( + row % 2 == 0, TraceLineType.FUNC_CALL_PRE, TraceLineType.FUNC_CALL_POST + ), + "process_id": 100 + call % 4, + "thread_id": 1000 + call % 8, + "func_call_id": [f"call-{i}" for i in call], + "function": "fixture.step", + } + ) + queries = [] + for i in range(args.queries): + index = (i * 97) % (args.rows // 2) + queries.append( + ( + (float(index * 2) - 0.5, float(index * 2) + 1.5), + 100 + index % 4, + 1000 + index % 8, + ) + ) + + def input_hash(): + return hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + + before = input_hash() + records = [] + for repetition in range(args.repetitions): + variants = [("legacy", LegacyRangeTrace), ("candidate", TracePandas)] + if repetition % 2: + variants.reverse() + for order, (name, cls) in enumerate(variants): + gc.collect() + start = time.perf_counter() + trace = cls(data, truncate_incomplete_func_calls=False) + constructed = time.perf_counter() + results = [trace.query_func_call_events_within_time(*queries[0])] + first = time.perf_counter() + results.extend( + trace.query_func_call_events_within_time(*query) + for query in queries[1:] + ) + end = time.perf_counter() + assert all(len(events) == 1 for events in results) + summary = [ + [ + { + "class": type(event).__name__, + "pre": event.pre_record, + "post": event.post_record, + } + for event in events + ] + for events in results + ] + record = { + "repetition": repetition + 1, + "order": order + 1, + "variant": name, + "construction_seconds": constructed - start, + "first_query_seconds": first - constructed, + "remaining_queries_seconds": end - first, + "total_seconds": end - start, + "result_sha256": digest(summary), + } + if name == "candidate": + assert trace._func_time_positions is not None + record["indexed_rows"] = sum( + len(positions) + for _, positions in trace._func_time_positions.values() + ) + record["identity_groups"] = len(trace._func_time_positions) + records.append(record) + assert input_hash() == before + assert len({record["result_sha256"] for record in records}) == 1 + medians = { + name: { + metric: statistics.median( + record[metric] for record in records if record["variant"] == name + ) + for metric in [ + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", + ] + } + for name in ["legacy", "candidate"] + } + print( + json.dumps( + { + "scope": "synthetic function-call window queries, including construction and lazy build; not full inference or peak memory", + "settings": vars(args), + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "pandas": pd.__version__, + "numpy": np.__version__, + "native_threads": { + name: os.environ.get(name) + for name in [ + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + ] + }, + }, + "source_sha256": hashlib.sha256( + Path(trace_module.__file__).read_bytes() + ).hexdigest(), + "input_sha256": before, + "input_unchanged": True, + "all_ordered_results_equal": True, + "measurements": records, + "medians": medians, + "median_total_ratio": medians["legacy"]["total_seconds"] + / medians["candidate"]["total_seconds"], + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() From 927707d319e36337979a287e9d184073547cf309 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:01:49 +0800 Subject: [PATCH 3/5] Document audited time-window inference measurements and tradeoffs --- .../inference-time-range-index-20261025.json | 872 ++++++++++++++++++ .../inference-time-range-index.md | 157 ++++ 2 files changed, 1029 insertions(+) create mode 100644 docs/measurements/inference-time-range-index-20261025.json create mode 100644 docs/measurements/inference-time-range-index.md diff --git a/docs/measurements/inference-time-range-index-20261025.json b/docs/measurements/inference-time-range-index-20261025.json new file mode 100644 index 00000000..2ae789d6 --- /dev/null +++ b/docs/measurements/inference-time-range-index-20261025.json @@ -0,0 +1,872 @@ +{ + "scope": "Three alternating sequential unprofiled CPU saved-trace inference pairs; both variants include the common pruning-state correction.", + "date_utc": "2026-10-25", + "started_utc": "2026-10-25T10:44:10.532938+00:00", + "ended_utc": "2026-10-25T10:53:12.729963+00:00", + "source": { + "base_branch_commit": "99d16905296d83495570fdcafa91370b242d2528", + "candidate_production_commit": "24e7cb0f7a573ff6225a388eff189d9e4af5e8b5", + "candidate_benchmark_commit": "39152aded8b97ed83988e3292bb89c6e3adcf750", + "common_state_fix_commit": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "only_production_difference": [ + "traincheck/trace/trace_pandas.py" + ], + "production_sha256": { + "baseline": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "c78bda77cfe83be10b6b385b55563fa6ca72ecc116f8bcc1557544d2e5d2df98", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/tracer.py": "5476b155823b4287f9cb01e498f70b99921063a052512a9f0d5d5b76974850f4", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "c36764ec7b9d7224c82071abf2f83d5ee9952e6868774dff62d851da9d0270e1" + }, + "candidate": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "c78bda77cfe83be10b6b385b55563fa6ca72ecc116f8bcc1557544d2e5d2df98", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/tracer.py": "5476b155823b4287f9cb01e498f70b99921063a052512a9f0d5d5b76974850f4", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "c36764ec7b9d7224c82071abf2f83d5ee9952e6868774dff62d851da9d0270e1" + } + }, + "all_source_files_unchanged_after_all_runs": true + }, + "runtime": { + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "packages": { + "numpy": "2.1.0", + "pandas": "2.3.3", + "torch": "2.16.0.dev20261012+cpu" + }, + "torch_git_revision": "8f9045d72f71e7aa9e16b6d7a055ec5bc1f720b5", + "original_training_runtime_difference": "Original training used Python 3.12.3 and CUDA 13.0; inference uses Python 3.11.4 and CPU torch from the same source revision." + }, + "workload": { + "saved_model_parameters": 6163712, + "training_steps": 10, + "sequence_length": 256, + "batch_size": 1, + "seed": 42, + "trace_events": 573494, + "raw_bytes": 286335289, + "traces": { + "trace_API_1088885_23438542501568.log": { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + "trace_API_1088885_23456244348224.log": { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + }, + "all_traces_unchanged_after_all_runs": true, + "enabled_relation_count": 6, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "eligible_api_count": 41, + "api_coverage_limit": "Original name set was not recorded; matching count alone does not establish exact original-environment API coverage." + }, + "settings": { + "backend": "pandas", + "workers": 1, + "native_threads": 1, + "CUDA_VISIBLE_DEVICES": "", + "PYTHONHASHSEED": "0", + "timeout_seconds": 1800, + "sampled_child_session_pss_budget_gib": 12, + "memory_sample_interval_seconds": 1, + "cprofile": false, + "periodic_traceback_dumps": false, + "fatal_faulthandler": true, + "fresh_process_and_trace_per_run": true, + "lazy_index_build_included": true, + "cpu_affinity": null, + "exclusive_host": false, + "run_order": [ + [ + "baseline", + "candidate" + ], + [ + "candidate", + "baseline" + ], + [ + "baseline", + "candidate" + ] + ] + }, + "runs": [ + { + "pair": 1, + "order": 1, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2327513088, + "rss_bytes": 2335973376, + "uss_bytes": 2324221952 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 75.47, + "precondition_and_output_sort_seconds": 12.565 + }, + "markers": { + "collection_start": "2026-10-25T06:45:38.025000", + "generation_start": "2026-10-25T06:44:22.555000", + "inference_end": "2026-10-25T06:45:50.590000", + "precondition_start": "2026-10-25T06:45:38.025000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 88.07923460006714, + "load_seconds": 10.64183259010315 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 102.65519127622247 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 1.29, + 1.24, + 1.16 + ], + "host_cpu_percent_sample": 4.6, + "available_memory_bytes": 52341108736 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 1.93, + 1.5, + 1.27 + ], + "host_cpu_percent_sample": 3.5, + "available_memory_bytes": 52397568000 + } + } + }, + { + "pair": 1, + "order": 2, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328130560, + "rss_bytes": 2336595968, + "uss_bytes": 2324832256 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 49.225, + "precondition_and_output_sort_seconds": 12.592 + }, + "markers": { + "collection_start": "2026-10-25T06:46:55.427000", + "generation_start": "2026-10-25T06:46:06.202000", + "inference_end": "2026-10-25T06:47:08.019000", + "precondition_start": "2026-10-25T06:46:55.427000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.86267161369324, + "load_seconds": 10.646003484725952 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.7478895187378 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 1.93, + 1.5, + 1.27 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52395216896 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.14, + 1.64, + 1.34 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52372901888 + } + } + }, + { + "pair": 2, + "order": 1, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328266752, + "rss_bytes": 2336739328, + "uss_bytes": 2324983808 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 49.337, + "precondition_and_output_sort_seconds": 12.469 + }, + "markers": { + "collection_start": "2026-10-25T06:48:13.227000", + "generation_start": "2026-10-25T06:47:23.890000", + "inference_end": "2026-10-25T06:48:25.697000", + "precondition_start": "2026-10-25T06:48:13.228000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.853256940841675, + "load_seconds": 10.614852905273438 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.78805723041296 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.14, + 1.64, + 1.34 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52375351296 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.0, + 1.73, + 1.4 + ], + "host_cpu_percent_sample": 3.7, + "available_memory_bytes": 52377493504 + } + } + }, + { + "pair": 2, + "order": 2, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286166016, + "rss_bytes": 2294603776, + "uss_bytes": 2282872832 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 73.854, + "precondition_and_output_sort_seconds": 12.484 + }, + "markers": { + "collection_start": "2026-10-25T06:49:55.626000", + "generation_start": "2026-10-25T06:48:41.772000", + "inference_end": "2026-10-25T06:50:08.110000", + "precondition_start": "2026-10-25T06:49:55.626000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 86.3845362663269, + "load_seconds": 10.706615447998047 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 101.79567445069551 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.0, + 1.73, + 1.4 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52377260032 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.19, + 1.88, + 1.49 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52357947392 + } + } + }, + { + "pair": 3, + "order": 1, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286102528, + "rss_bytes": 2294554624, + "uss_bytes": 2282823680 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 73.811, + "precondition_and_output_sort_seconds": 12.432 + }, + "markers": { + "collection_start": "2026-10-25T06:51:38.197000", + "generation_start": "2026-10-25T06:50:24.386000", + "inference_end": "2026-10-25T06:51:50.629000", + "precondition_start": "2026-10-25T06:51:38.197000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 86.28747224807739, + "load_seconds": 10.534409046173096 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 101.61026244610548 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.19, + 1.88, + 1.49 + ], + "host_cpu_percent_sample": 3.4, + "available_memory_bytes": 52357136384 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.25, + 1.98, + 1.57 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52370046976 + } + } + }, + { + "pair": 3, + "order": 2, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328313856, + "rss_bytes": 2336751616, + "uss_bytes": 2325057536 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 49.004, + "precondition_and_output_sort_seconds": 12.445 + }, + "markers": { + "collection_start": "2026-10-25T06:52:56.197000", + "generation_start": "2026-10-25T06:52:07.193000", + "inference_end": "2026-10-25T06:53:08.643000", + "precondition_start": "2026-10-25T06:52:56.198000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.49606490135193, + "load_seconds": 10.764497995376587 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.6580608561635 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.25, + 1.98, + 1.57 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52370046976 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.26, + 2.03, + 1.62 + ], + "host_cpu_percent_sample": 3.5, + "available_memory_bytes": 52363665408 + } + } + } + ], + "medians": { + "baseline": { + "wall_seconds": 101.79567445069551, + "wall_min_seconds": 101.61026244610548, + "wall_max_seconds": 102.65519127622247, + "peak_pss_bytes": 2286166016, + "load_seconds": 10.64183259010315, + "inference_seconds": 86.3845362663269, + "generation_and_pruning_seconds": 73.854, + "precondition_and_output_sort_seconds": 12.484 + }, + "candidate": { + "wall_seconds": 76.7478895187378, + "wall_min_seconds": 76.6580608561635, + "wall_max_seconds": 76.78805723041296, + "peak_pss_bytes": 2328266752, + "load_seconds": 10.646003484725952, + "inference_seconds": 61.853256940841675, + "generation_and_pruning_seconds": 49.225, + "precondition_and_output_sort_seconds": 12.469 + } + }, + "ratio_of_median_wall_times": 1.3263644784113884, + "median_wall_reduction_percent": 24.605942312499295, + "median_peak_pss_increase_bytes": 42100736, + "equivalence": { + "all_outputs_equal_corrected_reference": true, + "comparison": "Complete structural JSON multisets; duplicate multiplicity and array order preserved, object key and record line order ignored.", + "outputs": { + "invariants": { + "records": 836, + "distinct_records": 836, + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb" + }, + "failed_hypotheses": { + "records": 104, + "distinct_records": 104, + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798" + } + }, + "all_six_output_files_also_byte_identical": true, + "raw_output_sha256": { + "invariants.jsonl": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0", + "invariants.jsonl.failed": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2" + }, + "all_full_and_stage_context_files_equal_reference": true, + "context_sha256": { + "contexts-0.json": "bea2a909d42bc621be2fea66ad7de4e7e871b7f40903f546934e3d7e4d9c1207", + "contexts-1.json": "bea2a909d42bc621be2fea66ad7de4e7e871b7f40903f546934e3d7e4d9c1207" + } + }, + "fast_path_probe": { + "rows": 573494, + "dtypes": { + "type": "object", + "time": "int64", + "process_id": "int64", + "thread_id": "int64", + "func_call_id": "object" + }, + "sampled_queries": 24, + "all_sampled_queries_hit": true, + "indexed_rows": 573494, + "identity_groups": 2, + "limit": "Sampled real-call queries in a separate probe, not a measured hit count for every inference query." + }, + "validation": { + "local_combined_tests": 61, + "matching_runtime_combined_tests": 67, + "matching_runtime_tests_note": "Includes six common pruning-state tests. First packaging attempt lacked tests.fixtures; completed test-support-only rerun passed without production changes.", + "matching_runtime_test_log_sha256": "84cfeba80e7c2032b2e7a28baebc4c7bd2fc2e778f27bf6eb16362b4f857b5d3", + "independent_differential_audit": { + "parent_commit": "99d16905296d83495570fdcafa91370b242d2528", + "parent_source_sha256": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "candidate_source_sha256": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "pandas": "2.3.3", + "numpy": "2.1.0", + "cases": 144, + "comparisons": 6480, + "differences": [] + }, + "independent_evidence_audit": { + "passed": true, + "archive_files_verified": 295, + "all_outputs_and_contexts_equal_corrected_reference": true, + "only_signed_production_delta_confirmed": true, + "raw_trace_hash_verification_scope": "Before/after hashes computed remotely and reviewed; raw input is not in evidence archive and was not rehashed locally." + } + }, + "synthetic_benchmark": { + "scope": "synthetic function-call window queries, including construction and lazy build; not full inference or peak memory", + "settings": { + "rows": 200000, + "queries": 1000, + "repetitions": 3 + }, + "runtime": { + "python": "3.11.4", + "platform": "Linux-6.1.0-1027-oem-x86_64-with-glibc2.35", + "pandas": "2.3.3", + "numpy": "2.1.0", + "native_threads": { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1" + } + }, + "source_sha256": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "input_sha256": "4d2a33e4e4c692eaa5339751b3db19fbb8a2335cb1c9ebd26ba073fbbf407c1f", + "input_unchanged": true, + "all_ordered_results_equal": true, + "measurements": [ + { + "repetition": 1, + "order": 1, + "variant": "legacy", + "construction_seconds": 0.04303494095802307, + "first_query_seconds": 0.006073214113712311, + "remaining_queries_seconds": 2.613212414085865, + "total_seconds": 2.6623205691576004, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 1, + "order": 2, + "variant": "candidate", + "construction_seconds": 0.0409860722720623, + "first_query_seconds": 0.07652264088392258, + "remaining_queries_seconds": 0.21955957636237144, + "total_seconds": 0.3370682895183563, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + }, + { + "repetition": 2, + "order": 1, + "variant": "candidate", + "construction_seconds": 0.044426146894693375, + "first_query_seconds": 0.07581401616334915, + "remaining_queries_seconds": 0.21997691318392754, + "total_seconds": 0.34021707624197006, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + }, + { + "repetition": 2, + "order": 2, + "variant": "legacy", + "construction_seconds": 0.04476165026426315, + "first_query_seconds": 0.003517802804708481, + "remaining_queries_seconds": 2.71007227152586, + "total_seconds": 2.7583517245948315, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 3, + "order": 1, + "variant": "legacy", + "construction_seconds": 0.04182035103440285, + "first_query_seconds": 0.0035678334534168243, + "remaining_queries_seconds": 2.596189245581627, + "total_seconds": 2.6415774300694466, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 3, + "order": 2, + "variant": "candidate", + "construction_seconds": 0.04158874601125717, + "first_query_seconds": 0.0771775059401989, + "remaining_queries_seconds": 0.22500132396817207, + "total_seconds": 0.34376757591962814, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + } + ], + "medians": { + "legacy": { + "construction_seconds": 0.04303494095802307, + "first_query_seconds": 0.0035678334534168243, + "remaining_queries_seconds": 2.613212414085865, + "total_seconds": 2.6623205691576004 + }, + "candidate": { + "construction_seconds": 0.04158874601125717, + "first_query_seconds": 0.07652264088392258, + "remaining_queries_seconds": 0.21997691318392754, + "total_seconds": 0.34021707624197006 + } + }, + "median_total_ratio": 7.825358440456698 + }, + "provenance": { + "evidence_archive_name": "timing-evidence.tar.gz", + "evidence_archive_bytes": 1705821, + "evidence_archive_sha256": "7f45cee3ca5bf8f4b8b1fa2f29ee54234fafd1fe81b57fd0badb7cb38fa81c36", + "evidence_archive_access": "Retained study artifact; raw trace/output data are not published in this repository.", + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "helper_sha256": "1fc8409ec68d23da26223424a41dcaeff252ede84723b4256fd7dc5797f8e994", + "controller_sha256": "416db6554b02299ccc44ef73ed4eba7735bfe925a0e68a708701ae0f6baf1271", + "paired_runs_json_sha256": "5304104c25281c369b00ff217fb2989a0be712dc9f4cfe08ac4d54c2b3af9b27", + "snapshot_manifest_sha256": "2f85ca6e20061565b6b6cf02ad2760f3c8abb70978ce76eb5d1ed69cd032d750", + "final_integrity_validation_sha256": "0bf446f2fe7387e4fdcc9e30f0c4e9012162251b340158e844d8e0712f1a2913", + "independent_audit_json_sha256": "6ca49fd92ede07e153c81c373d3242fa49c379204e3dc2e8fc3d1497057cfdd8", + "fast_path_probe_json_sha256": "dbc0be5aacdab32979a46e21a11c55fa6f4bcdf49a25c4d369113925a22006c7", + "process_check_utc": "2026-10-25T10:54:52.654222+00:00", + "remaining_study_processes": [] + }, + "limits": [ + "Three pairs on one saved trace and six enabled relations, with unrelated background load and no exclusive host or CPU affinity.", + "Measures offline inference, not training overhead or distributed scalability; output equality is not independent invariant ground truth.", + "Wall time includes child startup/imports/output writes and monitor polling, excludes integrity hashing; load includes harness context serialization.", + "Reported inference ends before invariant JSON writes; marker durations are subphases, not additive extra time.", + "One-second sampled concurrent child-session PSS excludes the monitor and depends on shared pages; the 12-GiB budget is a monitor threshold, not an OS hard limit.", + "The per-frame index stores O(N) values/positions and adds first-query build cost; same-size in-place edits to cached fields or row order require a new TracePandas.", + "This incremental speedup must not be multiplied by the previous single-pair identity-cache measurement as an independently measured combined speedup." + ] +} diff --git a/docs/measurements/inference-time-range-index.md b/docs/measurements/inference-time-range-index.md new file mode 100644 index 00000000..5212bdb8 --- /dev/null +++ b/docs/measurements/inference-time-range-index.md @@ -0,0 +1,157 @@ +# Index function-call time windows + +Repeated calls to `query_func_call_events_within_time` previously filtered every +trace row by type, time, process, and thread before grouping the selected calls. +The new lazy index uses binary search within each process/thread pair, restores +the selected rows to their original order, and retains the existing pandas +`groupby`, record construction, and assertions. A conservative fast path handles +ordinary int64 identities with int64 or float64 time; all unsupported dtypes, +query scalar types, and comparison cases use the original pandas mask. + +## Complete saved-trace inference + +Three alternating sequential pairs on labmach reduced median total wall time from +**101.796 s to 76.748 s (1.326x, 24.61% less time)**. Both variants include the +identity cache and the independent pruning-state correction from PR #145. The +only production difference between the measured snapshots is the time-window +index in `trace_pandas.py`. + +| Pair | Execution order | Baseline wall | Candidate wall | Baseline peak PSS | Candidate peak PSS | +|---|---|---:|---:|---:|---:| +| 1 | Baseline, candidate | 102.655 s | 76.748 s | 2.167665 GiB | 2.168241 GiB | +| 2 | Candidate, baseline | 101.796 s | 76.788 s | 2.129158 GiB | 2.168367 GiB | +| 3 | Baseline, candidate | 101.610 s | 76.658 s | 2.129099 GiB | 2.168411 GiB | + +All six runs completed and produced the same **836 invariant records and 104 +saved failed-hypothesis records** as the corrected reference. Complete structural +JSON multisets match, preserving duplicate multiplicity and array order. The raw +output files also match byte for byte across the six runs. Full/stage context +files match the corrected reference byte for byte, and all source/input hashes +remained unchanged. Independent review verified all 295 archived files, source +snapshots, complete outputs, contexts, and calculations. Trace integrity checks +ran before and after inference on the remote host; the raw 286-MB input is not +in the downloaded archive and was not independently rehashed locally. This is +an incremental measurement over the corrected cached +baseline; it should not be multiplied by the earlier single-pair identity-cache +speedup as an independently measured combined result. + +| Median measurement | Baseline | Time-window index | +|---|---:|---:| +| Total child wall time | 101.796 s | 76.748 s | +| Trace loading/construction | 10.642 s | 10.646 s | +| Reported inference | 86.385 s | 61.853 s | +| Generation and pruning (part of inference) | 73.854 s | 49.225 s | +| Final preconditions and sort (part of inference) | 12.484 s | 12.469 s | +| Peak sampled child-session PSS | 2.129 GiB | 2.168 GiB | + +The median sampled memory peak increased by **40.15 MiB**. The first baseline +peak was higher than the other two and close to its paired candidate peak; +per-run values above retain that variation. Each trial uses a fresh process and +trace, including the first lazy index construction. The index stores +O(N) timestamp/position values and requires temporary build storage. It is a +runtime/memory tradeoff, and its first-query cost matters for short-lived traces. + +The input is both complete trace files from the saved 6,163,712-parameter, +10-step reference run: 573,494 events and 286,335,289 raw bytes, sequence length +256, batch size 1, seed 42. No training was rerun. A separate probe confirmed that +all 24 sampled real-call queries hit the new fast path, with int64 time/PID/TID +columns and built-in integer bounds/identities. This is sampled eligibility +validation, not a hit count for every query in the timed inference runs. + +Both variants use pandas, one worker, native thread settings of one, no GPU, +`PYTHONHASHSEED=0`, and six enabled relations; `FunctionLeadRelation` and +`FunctionCoverRelation` remain disabled. Every run has a 1,800-second cutoff and +12-GiB sampled child-session PSS budget. The runtime is Python 3.11.4, +Torch 2.16.0.dev20261012+cpu, pandas 2.3.3, and NumPy 2.1.0. Torch's source revision +matches the original CUDA wheel, but training used Python 3.12.3 and CUDA 13.0. +The eligible API count is 41; the original name set was not retained, so matching +the count does not establish exact original-environment API coverage. + +The host was shared, with no CPU affinity or exclusive reservation. Other owned +experiment jobs were paused during the six timed runs; unrelated background +work was left untouched. Before-run one-minute load averages ranged from 1.29 +to 2.25, and sampled whole-host CPU utilization from 3.3% to 4.6%. These snapshots +do not bound interference during each run. The result covers one saved workload +and relation set, not training overhead, distributed scaling, or invariant +ground-truth correctness. + +Total wall time includes child startup, imports, output writes, and monitor +polling (about one-second resolution), but excludes integrity hashing. The load +timer includes harness context serialization. The inference timer ends before +invariant JSON writes. Its two phases in the table are log-marker intervals, +not additional time. Memory is sampled once per second for the concurrent child +session, excludes the monitor, and depends on shared pages. The memory budget is +a monitor threshold, not an OS hard limit. cProfile and periodic traceback dumps +were disabled; fatal-error traceback handling remained enabled. + +[Compact measurements and provenance](inference-time-range-index-20261025.json) +retain every run's timing, memory peak, background-load snapshots, source/input +hashes, output digests, and archive identity. Raw traces and output records are +retained as study artifacts rather than published in this repository. + +## Correctness and cache contract + +The 18 new differential tests compare the indexed path with the unchanged pandas +fallback. Together with context/call/identity tests, 61 tests pass locally. The +matching Linux runtime passes those tests plus six common pruning-state tests +(67 total). An independent oracle extracted from the exact parent method checks +144 dtype/offset cases and 6,480 queries with no differences in each of two +pandas/NumPy environments: 2.2.3/1.26.4 and 2.3.3/2.1.0. + +The tests cover strict boundaries, tied/reversed bounds, signed integer extremes, +mixed integer/float precision above 2**53, NaNs/infinities/signed zero, malformed +bounds and scalar subclasses, null/extension/categorical dtypes, empty frames, +unsorted rows and duplicate labels, categorical/null call IDs, incomplete or +exception records, fresh returned objects, and cache invalidation. Empty +selections still run the original grouping logic, including its assertions. + +Time, event type, process/thread identity values and dtypes, and row order must +remain stable within a cached frame. Replacement or length changes invalidate +the cache; same-size in-place changes require a new `TracePandas`. The cache holds +a source-frame reference rather than copying the DataFrame, but can retain an old +frame until invalidation. Existing pandas group ordering, category/dropna +behavior, within-group row order, assertion messages, and fresh record/event +objects remain unchanged. + +## Reproduction + +The portable synthetic benchmark needs no saved trace: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m tools.benchmarks.time_range_index --rows 200000 --queries 1000 --repetitions 3 + +python -m unittest tests.test_context_index tests.test_call_index \ + tests.test_identity_cache tests.test_time_range_index +``` + +On the matching Linux runtime, its median construction-plus-query time fell from +**2.662321 s to 0.340217 s (7.825x)**. Ordered outputs match in all six trials, and +the input remains unchanged. Candidate first-query median was **0.076523 s** +versus **0.003568 s** for legacy filtering, exposing the lazy build cost. Input +generation, garbage collection, and result hashing are excluded; fresh record +construction is included. This synthetic result is separate from full inference. + +For the saved-trace measurement, reconstruct the baseline from commit +`99d16905296d83495570fdcafa91370b242d2528` plus only the pruning-state correction +hunks from `1b5aedffbeefd4cdd9d30036dbb0a833521526cf`. The candidate adds the production +change at `24e7cb0f7a573ff6225a388eff189d9e4af5e8b5`; the benchmark was checkpointed +at `39152aded8b97ed83988e3292bb89c6e3adcf750`. Match the recorded per-file hashes. +With the retained `inference_replay_fatal_only.py` and `inference_scaling.py` +harness files, invoke each fresh variant as follows, using the two trace files +identified by digest in the evidence JSON: + +```sh +"$INFERENCE_PYTHON" "$STUDY/inference_replay_fatal_only.py" \ + --mode infer --source "$SOURCE_SNAPSHOT" --traces "$TRACE_A" "$TRACE_B" \ + --output "$NEW_OUTPUT_DIRECTORY" --timeout 1800 --memory-limit-gib 12 +``` + +Run baseline/candidate, candidate/baseline, baseline/candidate sequentially, with +no `--profile` flag. The harness fixes workers/native threads/hash seed and CPU +mode, verifies source/input hashes, and saves phase/memory/context evidence. The +controller compares complete outputs. `run_pairs.py`, exact invocations, both +immutable source snapshots, full outputs, and the verified harness/helper hashes are +retained in the evidence archive. Reproducing this full-trace result requires +access to the saved trace and retained harness; the synthetic benchmark and +regression tests are self-contained in the checkout. From 8dc1fd180b3d634f40ba4532238844428334b003 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:33:38 +0800 Subject: [PATCH 4/5] Type validated numeric time indexes and benchmark measurements --- tools/benchmarks/time_range_index.py | 56 +++++++++++++++++++++------- traincheck/trace/trace_pandas.py | 50 ++++++++++++++----------- 2 files changed, 70 insertions(+), 36 deletions(-) diff --git a/tools/benchmarks/time_range_index.py b/tools/benchmarks/time_range_index.py index 427a61d7..c7ae93a0 100644 --- a/tools/benchmarks/time_range_index.py +++ b/tools/benchmarks/time_range_index.py @@ -12,6 +12,7 @@ import argparse import gc import hashlib +import inspect import json import logging import os @@ -19,6 +20,7 @@ import statistics import time from pathlib import Path +from typing import Literal, TypedDict import numpy as np import pandas as pd @@ -27,17 +29,48 @@ from traincheck.instrumentor.tracer import TraceLineType from traincheck.trace.trace_pandas import TracePandas +TimingField = Literal[ + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", +] +TIMING_FIELDS: tuple[TimingField, ...] = ( + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", +) + + +class TrialResult(TypedDict): + repetition: int + order: int + variant: str + construction_seconds: float + first_query_seconds: float + remaining_queries_seconds: float + total_seconds: float + result_sha256: str + + +class Measurement(TrialResult, total=False): + indexed_rows: int + identity_groups: int + class LegacyRangeTrace(TracePandas): - def _get_func_call_time_positions(self, *args): + def _get_func_call_time_positions( + self, time_range: object, process_id: object, thread_id: object + ) -> None: return None -def digest(value): +def digest(value: object) -> str: return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--rows", type=int, default=200000) parser.add_argument("--queries", type=int, default=1000) @@ -60,7 +93,7 @@ def main(): "function": "fixture.step", } ) - queries = [] + queries: list[tuple[tuple[float, float], int, int]] = [] for i in range(args.queries): index = (i * 97) % (args.rows // 2) queries.append( @@ -71,13 +104,13 @@ def main(): ) ) - def input_hash(): + def input_hash() -> str: return hashlib.sha256( pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() ).hexdigest() before = input_hash() - records = [] + records: list[Measurement] = [] for repetition in range(args.repetitions): variants = [("legacy", LegacyRangeTrace), ("candidate", TracePandas)] if repetition % 2: @@ -106,7 +139,7 @@ def input_hash(): ] for events in results ] - record = { + record: Measurement = { "repetition": repetition + 1, "order": order + 1, "variant": name, @@ -131,12 +164,7 @@ def input_hash(): metric: statistics.median( record[metric] for record in records if record["variant"] == name ) - for metric in [ - "construction_seconds", - "first_query_seconds", - "remaining_queries_seconds", - "total_seconds", - ] + for metric in TIMING_FIELDS } for name in ["legacy", "candidate"] } @@ -160,7 +188,7 @@ def input_hash(): }, }, "source_sha256": hashlib.sha256( - Path(trace_module.__file__).read_bytes() + Path(inspect.getfile(trace_module)).read_bytes() ).hexdigest(), "input_sha256": before, "input_unchanged": True, diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 588fa64b..b8d35805 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -126,7 +126,7 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self._func_time_source: pd.DataFrame | None = None self._func_time_length = -1 self._func_time_positions: ( - dict[tuple[int, int], tuple[list, list[int]]] | None + dict[tuple[int, int], tuple[list[int | float], list[int]]] | None ) = None self.column_dtypes_cached = {} @@ -1269,7 +1269,7 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: return post_record def _get_func_call_time_positions( - self, time_range, process_id, thread_id + self, time_range: object, process_id: object, thread_id: object ) -> list[int] | None: """Return exact positional matches, or None to retain pandas semantics. @@ -1286,10 +1286,8 @@ def _get_func_call_time_positions( if ( type(time_range) is not tuple # noqa: E721 - exclude custom indexing or len(time_range) != 2 - or any( - type(value) is not int # noqa: E721 - exact numeric comparison - for value in (process_id, thread_id) - ) + or type(process_id) is not int # noqa: E721 - exact numeric comparison + or type(thread_id) is not int # noqa: E721 - exact numeric comparison or any(not -(2**63) <= value < 2**63 for value in (process_id, thread_id)) or any( not isinstance(self.events[name].dtype, np.dtype) @@ -1298,21 +1296,25 @@ def _get_func_call_time_positions( ) ): return None + start_time: object = time_range[0] + end_time: object = time_range[1] time_dtype = self.events["time"].dtype if not isinstance(time_dtype, np.dtype): return None if time_dtype == "int64": - if any( - type(bound) is not int # noqa: E721 - exclude numeric subclasses - or not -(2**63) <= bound < 2**63 - for bound in time_range + if ( + type(start_time) is not int # noqa: E721 - exclude numeric subclasses + or type(end_time) is not int # noqa: E721 + or not -(2**63) <= start_time < 2**63 + or not -(2**63) <= end_time < 2**63 ): return None elif time_dtype == "float64": - if any( - type(bound) is not float # noqa: E721 - preserve pandas coercion - or math.isnan(bound) - for bound in time_range + if ( + type(start_time) is not float # noqa: E721 - preserve pandas coercion + or type(end_time) is not float # noqa: E721 + or math.isnan(start_time) + or math.isnan(end_time) ): return None else: @@ -1322,7 +1324,9 @@ def _get_func_call_time_positions( self.events ): event_types = self.events["type"] - positions = None + positions: ( + dict[tuple[int, int], tuple[list[int | float], list[int]]] | None + ) = None # A custom __eq__ or extension dtype must still execute in pandas. if ( isinstance(event_types.dtype, np.dtype) @@ -1335,11 +1339,13 @@ def _get_func_call_time_positions( for value in event_types ) ): - rows: dict[tuple[int, int], list[tuple[Any, int]]] = {} - types = event_types.tolist() - times = self.events["time"].tolist() - processes = self.events["process_id"].tolist() - threads = self.events["thread_id"].tolist() + rows: dict[tuple[int, int], list[tuple[int | float, int]]] = {} + types: list[object] = event_types.tolist() + # The exact NumPy dtypes validated above make Series.tolist() + # return builtin numeric scalars; extension/object data falls back. + times: list[int | float] = self.events["time"].tolist() + processes: list[int] = self.events["process_id"].tolist() + threads: list[int] = self.events["thread_id"].tolist() for position, (kind, time, process, thread) in enumerate( zip(types, times, processes, threads) ): @@ -1369,8 +1375,8 @@ def _get_func_call_time_positions( if indexed is None: return [] times, row_positions = indexed - left = bisect_right(times, time_range[0]) - right = bisect_left(times, time_range[1]) + left = bisect_right(times, start_time) + right = bisect_left(times, end_time) # pandas groupby preserves input row order within each call. Time order # cannot replace that order, even when traces arrive out of order. return sorted(row_positions[left:right]) From 409d57ca8ec3dd11396f854112801507a8746e16 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:42:23 +0800 Subject: [PATCH 5/5] docs: record hardened inference replay parity --- ...ference-hardening-validation-20261025.json | 61 +++++++++++++++++++ .../inference-time-range-index.md | 36 +++++++++++ 2 files changed, 97 insertions(+) create mode 100644 docs/measurements/inference-hardening-validation-20261025.json diff --git a/docs/measurements/inference-hardening-validation-20261025.json b/docs/measurements/inference-hardening-validation-20261025.json new file mode 100644 index 00000000..85dc80ba --- /dev/null +++ b/docs/measurements/inference-hardening-validation-20261025.json @@ -0,0 +1,61 @@ +{ + "scope": "Engineering hardening verification; one bounded unprofiled replay, not a new before/after performance experiment", + "inference_source_commit": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "common_correctness_patch_commit": "aec748ea46b35b6b43652985f00820afa813175c", + "patch_scope": "Only the two production pruning-state hunks, preserving deterministic sampling", + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0" + }, + "workload": { + "trace_rows": 573494, + "workers": 1, + "native_threads": 1, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ] + }, + "success": true, + "source_unchanged": true, + "traces_unchanged": true, + "validation_run_wall_seconds": 76.79424141719937, + "parity": { + "source": "ce25c12 + aec748e production state patch", + "comparison": "Complete structural JSON multisets; object-key and file-line order only; duplicate multiplicity and array order retained", + "outputs": { + "invariants.jsonl": { + "before_records": 836, + "after_records": 836, + "equal": true, + "raw_bytes_equal": true, + "candidate_sha256": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0" + }, + "invariants.jsonl.failed": { + "before_records": 104, + "after_records": 104, + "equal": true, + "raw_bytes_equal": true, + "candidate_sha256": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2" + } + }, + "contexts": { + "contexts-0.json": true, + "contexts-1.json": true + } + }, + "whole_production_checks": { + "mypy_1_9": "77 files pass", + "ruff_0_3_3": "pass", + "isort_5_10_1": "pass", + "black_24_3_0": "3 unchanged legacy files fail; original base has6 failures", + "strict_typing_limit": "Scoped to newly introduced modules/tools/tests; legacy modules retain documented pre-existing diagnostics", + "composition": { + "inference": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "tracing": "b64023b0651278ab987d3276aaad9141e8bf4894", + "state_patch": "aec748ea46b35b6b43652985f00820afa813175c" + } + } +} diff --git a/docs/measurements/inference-time-range-index.md b/docs/measurements/inference-time-range-index.md index 5212bdb8..301fa5f7 100644 --- a/docs/measurements/inference-time-range-index.md +++ b/docs/measurements/inference-time-range-index.md @@ -155,3 +155,39 @@ immutable source snapshots, full outputs, and the verified harness/helper hashes retained in the evidence archive. Reproducing this full-trace result requires access to the saved trace and retained harness; the synthetic benchmark and regression tests are self-contained in the checkout. + +## Engineering hardening verification + +The inference stack now types task payloads and RNG scopes, validates required +worker settings before any worker/global mutation, and types each benchmark's +inputs and measurement output. Identity APIs describe their actual schemaless +pandas scalar values instead of promising integers. The time index uses numeric +containers only after the existing conservative dtype checks. Legacy context +comparison retains one documented pandas scalar `Any` boundary; no values are +coerced to satisfy typing and the cache stability contract is unchanged. + +All 97 affected inference tests pass, and the configured Ruff/Mypy/Black/isort +hooks pass for changed files. Pinned Mypy 1.9 also passes the sampling and worker +modules, three benchmark tools, and worker-contract tests with +`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics +--follow-imports=silent --ignore-missing-imports`. New worker-contract tests +produce 18 failures before runtime validation and pass afterward; missing, +unknown, and malformed fields are rejected before state changes. + +A new bounded Linux replay of signed inference head `ce25c12`, with the same +separate #145 production patch applied, completed on the original 573,494-event +trace. **All 836 invariants, 104 failed-hypothesis records, and both context files +are byte-for-byte identical to the corrected reference.** Source and input hashes +remain unchanged. This one verification run took 76.794 seconds; it is not a new +paired performance experiment, and the earlier performance claims above remain +tied to their original measured revisions. + +The separately composed production source (inference, tracing, and pruning-state +fixes) passes whole-source Mypy on 77 files, Ruff, and isort. Whole-source Black +still reports three unchanged legacy files (`config/config.py`, +`instrumentor/proxy_wrapper/hash.py`, and `instrumentor/proxy_wrapper/subclass.py`); +the original base reports six. Scoped strict checks do not certify fully strict +typing of every legacy body. + +[Compact hardening evidence](inference-hardening-validation-20261025.json) +records exact composition, runtime, output hashes, and limits.