From f9b221f1240fcab33370eb85a4e5770b06ab1838 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Wed, 28 Oct 2026 01:46:12 +0800 Subject: [PATCH 01/13] Reduce repeated instrumenter metadata discovery and scalar JSON probes --- tests/test_immutable_metadata_suffix.py | 158 ++++++++++++++++++++ tests/test_serializable_scalars.py | 157 +++++++++++++++++++ traincheck/instrumentor/dumper.py | 23 ++- traincheck/instrumentor/metadata_methods.py | 65 +++++++- 4 files changed, 400 insertions(+), 3 deletions(-) create mode 100644 tests/test_immutable_metadata_suffix.py create mode 100644 tests/test_serializable_scalars.py diff --git a/tests/test_immutable_metadata_suffix.py b/tests/test_immutable_metadata_suffix.py new file mode 100644 index 00000000..aacf1a5f --- /dev/null +++ b/tests/test_immutable_metadata_suffix.py @@ -0,0 +1,158 @@ +"""Immutable namespace caching must preserve live application class lookup.""" + +import types +import unittest +from collections.abc import Mapping +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper, metadata_methods + + +class ImmutableMetadataSuffixTests(unittest.TestCase): + def test_metaclass_cannot_spoof_immutable_type_flags(self) -> None: + visits: list[str] = [] + + class SpoofedFlags(type): + @property + def __flags__(cls) -> int: + visits.append("flags") + return 1 << 8 + + class Mutable(metaclass=SpoofedFlags): + marker = 1 + + mro, namespaces = metadata_methods._class_layout(Mutable) + mutable, suffix = metadata_methods._immutable_suffix(mro, namespaces) + self.assertEqual(visits, []) + self.assertIs(mutable[0], namespaces[0]) + self.assertNotIn("marker", suffix) + Mutable.marker = 2 + self.assertEqual(mutable[0]["marker"], 2) + + def test_mutable_middle_and_head_remain_live(self) -> None: + class Middle(list[object]): + middle_marker = 1 + + class Head(Middle): + head_marker = 2 + + mro, namespaces = metadata_methods._class_layout(Head) + mutable, suffix = metadata_methods._immutable_suffix(mro, namespaces) + self.assertIs(mutable[0], namespaces[0]) + self.assertIs(mutable[1], namespaces[1]) + self.assertNotIn("middle_marker", suffix) + self.assertNotIn("head_marker", suffix) + Middle.middle_marker = 3 + self.assertEqual(mutable[1]["middle_marker"], 3) + setattr(Head, "inserted", 4) + self.assertEqual(mutable[0]["inserted"], 4) + delattr(Head, "inserted") + self.assertNotIn("inserted", mutable[0]) + + def assert_uncached_binding( + self, + mutable: tuple[Mapping[object, object], ...], + suffix: Mapping[str, object], + ) -> None: + self.assertIs(mutable, metadata_methods._TENSOR_NAMESPACES) + self.assertEqual(suffix, {}) + tensor = torch.ones(1) + with patch.object(metadata_methods, "_TENSOR_MUTABLE", mutable): + with patch.object(metadata_methods, "_TENSOR_SUFFIX", suffix): + binding = metadata_methods.tensor_method_binding(tensor, "add") + assert binding is not None + self.assertIs(binding.__self__, tensor) + + def test_other_runtime_uses_original_live_namespaces(self) -> None: + with patch.object( + metadata_methods.platform, "python_implementation", return_value="PyPy" + ): + mutable, suffix = metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, metadata_methods._TENSOR_NAMESPACES + ) + self.assert_uncached_binding(mutable, suffix) + + def test_absent_native_flags_uses_original_live_namespaces(self) -> None: + namespace: dict[str, object] = dict(vars(type)) + del namespace["__flags__"] + with patch.object(metadata_methods, "_TYPE_NAMESPACE", namespace): + mutable, suffix = metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, metadata_methods._TENSOR_NAMESPACES + ) + self.assert_uncached_binding(mutable, suffix) + + def test_malformed_present_flags_is_not_absent(self) -> None: + visits: list[object] = [] + + def application_flags(value: object) -> int: + visits.append(value) + return 1 << 8 + + for malformed in (None, object(), property(application_flags)): + with self.subTest(metadata_type=type(malformed)): + namespace: dict[str, object] = dict(vars(type)) + namespace["__flags__"] = malformed + with patch.object(metadata_methods, "_TYPE_NAMESPACE", namespace): + with self.assertRaises(AssertionError): + metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, + metadata_methods._TENSOR_NAMESPACES, + ) + self.assertEqual(visits, []) + + def test_replacing_and_restoring_inherited_method_is_observed(self) -> None: + tensor = torch.ones(1) + original = metadata_methods.tensor_method_binding(tensor, "add") + assert original is not None + self.assertIs(original.__self__, tensor) + + def replacement(value: torch.Tensor) -> int: + return 37 + + with patch.object(torch.Tensor, "add", property(replacement)): + self.assertIsNone(metadata_methods.tensor_method_binding(tensor, "add")) + self.assertEqual( + dumper.convert_var_to_dict(tensor, False, {"add": None}), {"add": 37} + ) + restored = metadata_methods.tensor_method_binding(tensor, "add") + assert restored is not None + self.assertIs(restored.__self__, tensor) + self.assertIs(type(restored), type(original)) + + def test_new_head_function_binding_and_deletion_are_observed(self) -> None: + tensor = torch.ones(1) + + def inserted(value: torch.Tensor) -> int: + return 41 + + name = "traincheck_native_suffix_test_method" + self.assertIsNone(metadata_methods.tensor_method_binding(tensor, name)) + with patch.object(torch.Tensor, name, inserted, create=True): + binding = metadata_methods.tensor_method_binding(tensor, name) + assert isinstance(binding, types.MethodType) + self.assertIs(binding.__func__, inserted) + self.assertIs(binding.__self__, tensor) + self.assertIsNone(metadata_methods.tensor_method_binding(tensor, name)) + + def test_earlier_property_can_shadow_cached_base_method(self) -> None: + tensor = torch.ones(1) + + def replacement(value: torch.Tensor) -> int: + return 43 + + def prepare(value: torch.Tensor) -> int: + setattr(torch.Tensor, "add", property(replacement)) + return 1 + + with patch.object(torch.Tensor, "add", torch.Tensor.add): + with patch.object(torch.Tensor, "prepare", property(prepare), create=True): + result = dumper.convert_var_to_dict( + tensor, False, {"prepare": None, "add": None} + ) + self.assertEqual(result, {"prepare": 1, "add": 43}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_serializable_scalars.py b/tests/test_serializable_scalars.py new file mode 100644 index 00000000..2757ebbf --- /dev/null +++ b/tests/test_serializable_scalars.py @@ -0,0 +1,157 @@ +"""Compatibility boundaries for skipping redundant exact-scalar JSON probes.""" + +import json +import math +import sys +import unittest +from collections.abc import Iterator +from unittest.mock import patch + +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from traincheck.config import config +from traincheck.instrumentor import dumper + + +class WatchedList(list[int]): + def __init__(self) -> None: + super().__init__([1, 2]) + self.iterations = 0 + + def __iter__(self) -> Iterator[int]: + self.iterations += 1 + return super().__iter__() + + +class MetadataDictionary(dict[str, int]): + @property + def status(self) -> str: + return "kept" + + +class EqualityTrap(type): + def __eq__(self, other: object) -> bool: + raise AssertionError("Scalar type guards must not invoke class equality") + + def __hash__(self) -> int: + raise AssertionError("Scalar type guards must not hash user classes") + + +class CustomInteger(int, metaclass=EqualityTrap): + pass + + +class SerializableScalarTests(unittest.TestCase): + def setUp(self) -> None: + compile_guard = patch.object(config, "USE_TORCH_COMPILE", False) + compile_guard.start() + self.addCleanup(compile_guard.stop) + + def test_exact_scalar_values_keep_runtime_labels_and_original_objects(self) -> None: + values: tuple[tuple[str, object], ...] = ( + ("NoneType", None), + ("bool", True), + ("bool", False), + ("int", -(2**63)), + ("int", 2**63 - 1), + ("float", 1.25), + ("str", "雪\ud800"), + ) + for name, value in values: + with self.subTest(name=name, value=value): + result = dumper.var_to_serializable(value) + self.assertEqual(list(result), [name]) + self.assertIs(result[name], value) + + def test_nonfinite_and_negative_zero_are_not_replaced_or_coerced(self) -> None: + for value in (float("nan"), float("inf"), float("-inf"), -0.0): + result = dumper.var_to_serializable(value) + self.assertIs(result["float"], value) + self.assertEqual( + dumper.serialize(result), dumper.serialize({"float": value}) + ) + result_zero = dumper.var_to_serializable(-0.0)["float"] + assert isinstance(result_zero, float) + self.assertEqual(math.copysign(1, result_zero), -1) + + def test_large_integers_outside_fast_path_still_keep_value(self) -> None: + for value in (-(2**63) - 1, 2**63, 2**128): + self.assertIs(dumper.var_to_serializable(value)["int"], value) + + def test_huge_integer_preserves_configured_json_value_error(self) -> None: + # Python exposes this optional security setting on current 3.10+ patch + # releases. Older interpreters without it cannot exercise this contract. + get_limit: object = getattr(sys, "get_int_max_str_digits", None) + set_limit: object = getattr(sys, "set_int_max_str_digits", None) + if not callable(get_limit) or not callable(set_limit): + self.skipTest("Interpreter has no configurable integer digit limit") + previous: object = get_limit() + assert isinstance(previous, int) + try: + set_limit(640) + value = 10**1000 + with self.assertRaises(ValueError) as expected: + json.dumps(value) + with self.assertRaises(ValueError) as actual: + dumper.var_to_serializable(value) + self.assertEqual(str(actual.exception), str(expected.exception)) + finally: + set_limit(previous) + + def test_custom_scalar_class_equality_and_hash_are_not_invoked(self) -> None: + value = CustomInteger(7) + result = dumper.var_to_serializable(value) + self.assertEqual(len(result), 1) + self.assertIs(next(iter(result.values())), value) + self.assertTrue(next(iter(result)).endswith("CustomInteger")) + + def test_list_subclass_keeps_json_iterator_side_effect(self) -> None: + value = WatchedList() + result = dumper.var_to_serializable(value) + self.assertEqual(value.iterations, 1) + self.assertIs(next(iter(result.values())), value) + + def test_dictionary_subclass_keeps_selected_metadata_conversion(self) -> None: + value = MetadataDictionary(hidden=3) + result = dumper.var_to_serializable(value, {"status": None}) + self.assertEqual(next(iter(result.values())), {"status": "kept"}) + + def test_mixed_containers_keep_identity_and_nested_contents(self) -> None: + value: list[object] = [1, "雪", None, {"flag": True}, (2.0, False)] + result = dumper.var_to_serializable(value) + self.assertIs(result["list"], value) + self.assertEqual(json.dumps(result), json.dumps({"list": value})) + + def test_cyclic_container_still_raises_at_conversion(self) -> None: + value: list[object] = [] + value.append(value) + with self.assertRaisesRegex(ValueError, "Circular reference"): + dumper.var_to_serializable(value) + + def test_existing_compile_gate_runs_before_scalar_shortcut(self) -> None: + # The existing compile-mode heuristic treats objects without a device + # as fake. This optimization must preserve that established gate. + with patch.object(config, "USE_TORCH_COMPILE", True): + self.assertEqual( + dumper.var_to_serializable(3), + {"builtins.int": {"fake": True, "shape": None, "dtype": None}}, + ) + + def test_actual_fake_tensor_keeps_dedicated_metadata_path(self) -> None: + with FakeTensorMode() as mode: + value = mode.from_tensor(torch.empty(2, 3)) + with patch.object(config, "USE_TORCH_COMPILE", True): + result = dumper.var_to_serializable(value) + self.assertEqual( + next(iter(result.values())), + {"fake": True, "shape": (2, 3), "dtype": "torch.float32"}, + ) + + def test_primitive_keeps_ignoring_uninterpreted_dump_configuration(self) -> None: + value = 42 + self.assertIs(dumper.var_to_serializable(value, object())["int"], value) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index fcb9416b..974be95a 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -549,12 +549,16 @@ def obj_to_serializable(obj, dump_config=None) -> dict[str, object]: return {str(type(obj)): None} -def var_to_serializable(obj, dump_config=None) -> dict[str, object]: +def var_to_serializable(obj: object, dump_config: object = None) -> dict[str, object]: """Convert any object to a serializable dictionary. Note that this function is largely a wrapper of convert_var_to_dict to add some heuristics about how to dump a few types of objects. and it does not dump the `data` attribute of a tensor. If you want to dump the `data` attribute of a tensor, use `convert_var_to_dict` and set `include_tensor_data=True`. + + Primitive/JSON-compatible values ignore dump_config. Other values delegate + its interpretation to obj_to_serializable unchanged; this boundary does not + claim that arbitrary caller-supplied configuration has been validated. """ if is_fake_tensor(obj): @@ -563,6 +567,23 @@ def var_to_serializable(obj, dump_config=None) -> dict[str, object]: if issubclass(type(obj), dict) and type(obj) != dict: # noqa E721 return obj_to_serializable(obj, dump_config=dump_config) + obj_type = type(obj) + if ( + obj_type is type(None) + or obj_type is bool + or obj_type is float + or obj_type is str + ): + # These exact built-ins always pass the standard JSON compatibility + # probe. Keep subclasses and custom objects on the original path. + return {typename(obj, is_runtime=True): obj} + if obj_type is int: + assert isinstance(obj, int) + # Larger integers can fail under Python's configurable digit limit. + # Preserve that synchronous failure by retaining their original probe. + if -(2**63) <= obj < 2**63: + return {typename(obj, is_runtime=True): obj} + try: json.dumps( obj diff --git a/traincheck/instrumentor/metadata_methods.py b/traincheck/instrumentor/metadata_methods.py index f8e22249..d71e1e8a 100644 --- a/traincheck/instrumentor/metadata_methods.py +++ b/traincheck/instrumentor/metadata_methods.py @@ -1,5 +1,6 @@ """Bind verified native tensor methods without per-attribute warning contexts.""" +import platform import types from collections.abc import Mapping from typing import TypeAlias @@ -54,6 +55,58 @@ def _class_layout( _PARAMETER_MRO, _PARAMETER_NAMESPACES = _class_layout(_PARAMETER_TYPE) +def _immutable_suffix( + mro: tuple[object, ...], namespaces: tuple[Mapping[object, object], ...] +) -> tuple[tuple[Mapping[object, object], ...], Mapping[str, object]]: + """Keep mutable namespaces live and cache only a proven immutable tail. + + CPython's Py_TPFLAGS_IMMUTABLETYPE (1 << 8) disallows attribute changes + through the Python type API. Read that flag through type's native descriptor, + bypassing metaclass hooks. Other runtimes or an absent capability keep every + namespace live. Present malformed native metadata remains an internal error. + + The cache owns references already held by immutable type namespaces, and its + mapping cannot be modified. An unchanged MRO is required before callers use + it; no mutable class lookup decision is cached. + """ + assert len(mro) == len(namespaces) + empty: Mapping[str, object] = types.MappingProxyType({}) + if platform.python_implementation() != "CPython": + return namespaces, empty + flags_descriptor = _TYPE_NAMESPACE.get("__flags__", _ABSENT) + if flags_descriptor is _ABSENT: + return namespaces, empty + assert isinstance( + flags_descriptor, (types.GetSetDescriptorType, types.MemberDescriptorType) + ) + offset = len(mro) + for index in range(len(mro) - 1, -1, -1): + owner = mro[index] + assert isinstance(owner, type) + flags: object = flags_descriptor.__get__(owner) + flags_type = type(flags) + assert flags_type is int + assert isinstance(flags, int) + if not flags & (1 << 8): + break + offset = index + suffix: dict[str, object] = {} + for namespace in namespaces[offset:]: + for name, descriptor in namespace.items(): + name_type = type(name) + assert name_type is str + assert isinstance(name, str) + if name not in suffix: + suffix[name] = descriptor + return namespaces[:offset], types.MappingProxyType(suffix) + + +_TENSOR_MUTABLE, _TENSOR_SUFFIX = _immutable_suffix(_TENSOR_MRO, _TENSOR_NAMESPACES) +_PARAMETER_MUTABLE, _PARAMETER_SUFFIX = _immutable_suffix( + _PARAMETER_MRO, _PARAMETER_NAMESPACES +) + + def _native_tensor_dictionary() -> types.GetSetDescriptorType | None: # Absence/replacement before import disables this optional fast path. A # genuine native descriptor remains valid for its owner after a user replaces @@ -90,9 +143,11 @@ def tensor_method_binding(receiver: object, name: str) -> BoundTensorMethod | No return None receiver_type = type(receiver) if receiver_type is _TENSOR_TYPE: - mro, namespaces = _TENSOR_MRO, _TENSOR_NAMESPACES + mro, namespaces = _TENSOR_MRO, _TENSOR_MUTABLE + suffix = _TENSOR_SUFFIX elif receiver_type is _PARAMETER_TYPE: - mro, namespaces = _PARAMETER_MRO, _PARAMETER_NAMESPACES + mro, namespaces = _PARAMETER_MRO, _PARAMETER_MUTABLE + suffix = _PARAMETER_SUFFIX else: return None if _TENSOR_DICTIONARY is None or _MRO_DESCRIPTOR.__get__(receiver_type) is not mro: @@ -107,6 +162,12 @@ def tensor_method_binding(receiver: object, name: str) -> BoundTensorMethod | No getattribute = namespace.get("__getattribute__", _ABSENT) if descriptor is _ABSENT: descriptor = namespace.get(name, _ABSENT) + if "__getattr__" in suffix: + return None + if getattribute is _ABSENT: + getattribute = suffix.get("__getattribute__", _ABSENT) + if descriptor is _ABSENT: + descriptor = suffix.get(name, _ABSENT) if getattribute is not _OBJECT_GETATTRIBUTE: return None From 161313a93512b07410fa2b1072613333a5a2e492 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Wed, 28 Oct 2026 01:58:58 +0800 Subject: [PATCH 02/13] Cache only immutable lookup facts without retaining descriptors --- tests/test_immutable_metadata_suffix.py | 69 +++++++++++++++++++-- traincheck/instrumentor/metadata_methods.py | 58 +++++++++++------ 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/tests/test_immutable_metadata_suffix.py b/tests/test_immutable_metadata_suffix.py index aacf1a5f..be825ea0 100644 --- a/tests/test_immutable_metadata_suffix.py +++ b/tests/test_immutable_metadata_suffix.py @@ -1,5 +1,8 @@ """Immutable namespace caching must preserve live application class lookup.""" +from __future__ import annotations + +import sys import types import unittest from collections.abc import Mapping @@ -11,6 +14,60 @@ class ImmutableMetadataSuffixTests(unittest.TestCase): + def test_custom_namespace_key_keeps_hook_lookups_on_original_path(self) -> None: + visits: list[object] = [] + + class CustomName(str): + def __hash__(self) -> int: + return str.__hash__(self) + + def __eq__(self, other: object) -> bool: + visits.append(other) + raise AssertionError("Cache construction must not compare custom keys") + + state: dict[object, object] = {CustomName("__getattr__"): False} + namespace: Mapping[object, object] = types.MappingProxyType(state) + namespaces = (namespace,) + mutable, suffix = metadata_methods._immutable_suffix((object,), namespaces) + self.assertIs(mutable, namespaces) + self.assertEqual(suffix.namespaces, ()) + self.assertEqual(visits, []) + + def test_cache_does_not_change_observable_metadata_reference_counts(self) -> None: + tensor = torch.ones(1) + + def descriptor_references(value: torch.Tensor) -> int: + return sys.getrefcount(torch.Tensor.add) + + def name_references(value: torch.Tensor) -> int: + return sys.getrefcount(torch.Tensor.add.__name__) + + selected: dict[str, object] = { + "metadata_native_refcount": None, + "metadata_native_name_refcount": None, + } + with patch.object( + torch.Tensor, + "metadata_native_refcount", + property(descriptor_references), + create=True, + ): + with patch.object( + torch.Tensor, + "metadata_native_name_refcount", + property(name_references), + create=True, + ): + before = dumper.convert_var_to_dict(tensor, False, selected) + additional_cache = metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, + metadata_methods._TENSOR_NAMESPACES, + ) + after = dumper.convert_var_to_dict(tensor, False, selected) + self.assertEqual(after, before) + # Keep the additional cache alive through both observations. + self.assertIsNotNone(additional_cache) + def test_metaclass_cannot_spoof_immutable_type_flags(self) -> None: visits: list[str] = [] @@ -27,7 +84,7 @@ class Mutable(metaclass=SpoofedFlags): mutable, suffix = metadata_methods._immutable_suffix(mro, namespaces) self.assertEqual(visits, []) self.assertIs(mutable[0], namespaces[0]) - self.assertNotIn("marker", suffix) + self.assertTrue(all(view is not namespaces[0] for view in suffix.namespaces)) Mutable.marker = 2 self.assertEqual(mutable[0]["marker"], 2) @@ -42,8 +99,8 @@ class Head(Middle): mutable, suffix = metadata_methods._immutable_suffix(mro, namespaces) self.assertIs(mutable[0], namespaces[0]) self.assertIs(mutable[1], namespaces[1]) - self.assertNotIn("middle_marker", suffix) - self.assertNotIn("head_marker", suffix) + self.assertTrue(all(view is not namespaces[0] for view in suffix.namespaces)) + self.assertTrue(all(view is not namespaces[1] for view in suffix.namespaces)) Middle.middle_marker = 3 self.assertEqual(mutable[1]["middle_marker"], 3) setattr(Head, "inserted", 4) @@ -54,10 +111,12 @@ class Head(Middle): def assert_uncached_binding( self, mutable: tuple[Mapping[object, object], ...], - suffix: Mapping[str, object], + suffix: metadata_methods._ImmutableSuffix, ) -> None: self.assertIs(mutable, metadata_methods._TENSOR_NAMESPACES) - self.assertEqual(suffix, {}) + self.assertEqual(suffix.namespaces, ()) + self.assertFalse(suffix.has_getattr) + self.assertFalse(suffix.object_getattribute) tensor = torch.ones(1) with patch.object(metadata_methods, "_TENSOR_MUTABLE", mutable): with patch.object(metadata_methods, "_TENSOR_SUFFIX", suffix): diff --git a/traincheck/instrumentor/metadata_methods.py b/traincheck/instrumentor/metadata_methods.py index d71e1e8a..6ae71f54 100644 --- a/traincheck/instrumentor/metadata_methods.py +++ b/traincheck/instrumentor/metadata_methods.py @@ -3,7 +3,7 @@ import platform import types from collections.abc import Mapping -from typing import TypeAlias +from typing import NamedTuple, TypeAlias import torch @@ -55,22 +55,30 @@ def _class_layout( _PARAMETER_MRO, _PARAMETER_NAMESPACES = _class_layout(_PARAMETER_TYPE) +class _ImmutableSuffix(NamedTuple): + namespaces: tuple[Mapping[object, object], ...] + has_getattr: bool + object_getattribute: bool + + def _immutable_suffix( mro: tuple[object, ...], namespaces: tuple[Mapping[object, object], ...] -) -> tuple[tuple[Mapping[object, object], ...], Mapping[str, object]]: - """Keep mutable namespaces live and cache only a proven immutable tail. +) -> tuple[tuple[Mapping[object, object], ...], _ImmutableSuffix]: + """Keep namespaces live and cache only immutable lookup-hook facts. CPython's Py_TPFLAGS_IMMUTABLETYPE (1 << 8) disallows attribute changes through the Python type API. Read that flag through type's native descriptor, bypassing metaclass hooks. Other runtimes or an absent capability keep every namespace live. Present malformed native metadata remains an internal error. - The cache owns references already held by immutable type namespaces, and its - mapping cannot be modified. An unchanged MRO is required before callers use - it; no mutable class lookup decision is cached. + Retain only existing instrumenter-owned namespace views and booleans. Even + immutable descriptors and their names must not acquire extra cache references: + application properties can observe their reference counts. Each descriptor + is still read from its original namespace. An unchanged MRO is required before + callers use the facts; no mutable class lookup decision is cached. """ assert len(mro) == len(namespaces) - empty: Mapping[str, object] = types.MappingProxyType({}) + empty = _ImmutableSuffix((), False, False) if platform.python_implementation() != "CPython": return namespaces, empty flags_descriptor = _TYPE_NAMESPACE.get("__flags__", _ABSENT) @@ -90,15 +98,24 @@ def _immutable_suffix( if not flags & (1 << 8): break offset = index - suffix: dict[str, object] = {} - for namespace in namespaces[offset:]: - for name, descriptor in namespace.items(): + immutable = namespaces[offset:] + # A custom dictionary key can execute equality during hook lookup. Such a + # namespace keeps the original live path rather than moving those calls. + for namespace in immutable: + for name in namespace: name_type = type(name) - assert name_type is str - assert isinstance(name, str) - if name not in suffix: - suffix[name] = descriptor - return namespaces[:offset], types.MappingProxyType(suffix) + if name_type is not str: + return namespaces, empty + getattribute: object = _ABSENT + has_getattr = False + for namespace in immutable: + if "__getattr__" in namespace: + has_getattr = True + if getattribute is _ABSENT: + getattribute = namespace.get("__getattribute__", _ABSENT) + return namespaces[:offset], _ImmutableSuffix( + immutable, has_getattr, getattribute is _OBJECT_GETATTRIBUTE + ) _TENSOR_MUTABLE, _TENSOR_SUFFIX = _immutable_suffix(_TENSOR_MRO, _TENSOR_NAMESPACES) @@ -162,12 +179,15 @@ def tensor_method_binding(receiver: object, name: str) -> BoundTensorMethod | No getattribute = namespace.get("__getattribute__", _ABSENT) if descriptor is _ABSENT: descriptor = namespace.get(name, _ABSENT) - if "__getattr__" in suffix: + if suffix.has_getattr: return None - if getattribute is _ABSENT: - getattribute = suffix.get("__getattribute__", _ABSENT) + if getattribute is _ABSENT and suffix.object_getattribute: + getattribute = _OBJECT_GETATTRIBUTE if descriptor is _ABSENT: - descriptor = suffix.get(name, _ABSENT) + for namespace in suffix.namespaces: + descriptor = namespace.get(name, _ABSENT) + if descriptor is not _ABSENT: + break if getattribute is not _OBJECT_GETATTRIBUTE: return None From aebb88b69559c5facf0c5f353705be3e65aeb743 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Wed, 28 Oct 2026 02:09:52 +0800 Subject: [PATCH 03/13] Record corrected instrumenter microbenchmarks and CPU training validation --- .../instrumenter-hotpaths-20261028.md | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 docs/measurements/instrumenter-hotpaths-20261028.md diff --git a/docs/measurements/instrumenter-hotpaths-20261028.md b/docs/measurements/instrumenter-hotpaths-20261028.md new file mode 100644 index 00000000..f69612a6 --- /dev/null +++ b/docs/measurements/instrumenter-hotpaths-20261028.md @@ -0,0 +1,251 @@ +# Instrumenter hot-path optimization, 2026-10-28 + +This study compares signed implementation +`161313a93512b07410fa2b1072613333a5a2e492` with +`bccc56fab83d525198286f1855b9dd7f6866135f` (PR #162). PR #163 depends on +#162. The work targets metadata collection; it changes neither the trace format +nor the point at which mutable values are serialized into an owned snapshot. + +## Implementation and contracts + +1. Native Tensor/Parameter metadata lookup caches two lookup-hook facts for a + contiguous immutable suffix of the class MRO. CPython's native `type.__flags__` + descriptor supplies + `Py_TPFLAGS_IMMUTABLETYPE`; class names or package names never establish + immutability. All descriptors, mutable class namespaces and receiver + dictionaries remain live, + and every binding still checks the current MRO, lookup hooks and descriptor + owner. Real method bindings and the existing classifier preserve observable + temporary lifetimes. Only booleans and existing instrumenter-owned namespace + views are retained; no descriptor or attribute-name references are added. + Other runtimes or an absent capability retain live + lookup; malformed present metadata is an internal error. +2. Exact None, bool, float, str and signed-64-bit integers avoid a throwaway + `json.dumps` compatibility probe. The existing compile/FakeTensor gate still + runs first. Subclasses, containers and other objects retain their original + path. Larger integers keep the probe, including Python's configurable + integer-to-string limit and its synchronous errors. Returned objects and + runtime type labels are unchanged. + +No metadata values or mutable class decisions are cached. No tensor fields, +identity fields, warnings from application getters, trace records or checks are +dropped. Final `orjson`/stdlib JSON fallback, argument observation order, +serialization timing, producer buffering and writer behavior are unchanged. + +## Ordinary labmach measurements + +| Timed body | Baseline median | Candidate median | Less time | +|---|---:|---:|---:| +| 800 add/mm calls | 1.516760 s | 1.479685 s | 2.44% | +| MNIST, 10 training steps | 0.667927 s | 0.646459 s | 3.21% | +| Transformer, 10 training steps | 1.771281 s | 1.725027 s | 2.61% | + +Training uses the existing benchmark adapters, batch size 2, Transformer +sequence length 16, and ten matched data/training steps. These are complete +short training loops, not full dataset epochs. Traced training bodies include +forward, loss, backward and optimizer work; setup, evidence capture, final +queue wait and shutdown remain outside the body. MNIST trial ranges overlap; +every Transformer candidate trial is faster than every baseline in this set. +All before/after numbers in this table use the corrected revision. + +MNIST has 1,199,882 parameters and uses the first 20 cached training examples, +no shuffling, NLL loss and Adadelta. The small Transformer has **28,267 +parameters**, two layers, width 32, two attention heads and feed-forward width +64. Its WikiText-2 prefix has 322 tokens and a 171-token vocabulary. The prefix +SHA256 is `c2122261adb584adc0e9cd4672758896de1260e7dae0ed732e8f07394bce9712`; +the underlying text SHA256 is +`9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f`. +This differs from the earlier three-step study's smaller vocabulary/model; +the fresh baseline and candidate both use the same ten-step data and model. +The training harness SHA256 is +`8a8b351a8baf0a98215461855cc72bfded7dbc94a667f27af789170a2deedda6`, +and the frozen adapter SHA256 is +`5b6852940601d663c9aedad3e096c869fea1631d2038d21956e883ae6e4c7a31`. + +The corrected candidate reduces the median timed operation loop from +**1.516760 to 1.479685 seconds, 2.44% less time (1.025x)**. There are four fresh +processes per revision in A1/B1/B2/A2/A3/B3/B4/A4 order. Baseline samples are +1.521204, 1.518426, 1.515093 and 1.513667 seconds; candidate samples are +1.544820, 1.481346, 1.475127 and 1.478023 seconds. One candidate run is slower +than baseline, so the result must not be described as an improvement in every +trial. Percentages divide saved time by baseline median time. + +Each trial makes 800 calls: 200 repetitions of `torch.add` and `torch.mm` with +exact Tensor/Parameter arguments, using 16x16 float32 inputs. All 800 outputs +remain retained during the loop in every variant. Full API argument and return +metadata produce 1,600 records per trial. Model tracking is disabled. Timing +includes the identical output-list retention and excludes setup, state evidence, +final queue wait and process shutdown. + +Every traced run matches the fresh plain control's complete recorded input, +output and Python/NumPy/Torch RNG evidence. Four complete ordered raw-record +comparisons retain every field and differ only in original process-specific +identities and timestamps; raw byte equality is not claimed. Source inventories +are verified before and after the study. Recorded final queue-wait time is not +total I/O cost: writing also overlaps the operation loop and evidence capture. + +The unchanged lab runtime is Python 3.11.4, Torch 2.16.0.dev20261012+cpu, +NumPy 2.1.0 and orjson 3.13.0. Runs use one pinned performance core (CPU8) and +one native/Torch thread. There is no GPU, selective-instrumentation policy, +online checker or distributed execution in this comparison. +The single plain control establishes state/output parity, not a stable +uninstrumented timing baseline; no tracing-overhead multiplier is inferred. + +Across all 24 instrumented trials, complete recorded state/loss/RNG comparisons +against the three fresh plain controls pass. Twelve ordered raw comparisons +retain all IDs/timestamps and find no other field, shape, type, array or exception +differences. Per trial, MNIST emits 1,584 records/1,066,058 bytes; Transformer +4,446 records/3,370,348 bytes; add/mm 1,600 records/1,863,600 bytes. Each has zero +VAR records because model tracking is disabled in the timed series. + +| Workload | Observed peak RSS median, baseline to candidate | Final queue-wait median, baseline to candidate | +|---|---:|---:| +| Add/mm (process group) | 508.160 to 507.867 MiB | 0.023853 to 0.327049 ms | +| MNIST (training process) | 542.705 to 548.924 MiB | 0.208640 to 0.193022 ms | +| Transformer (training process) | 518.225 to 518.201 MiB | 0.283403 to 0.248324 ms | + +These are operational observations, not isolated memory costs attributable to +the optimization. In particular MNIST's observed RSS is higher; no process-memory +saving is claimed. All min/median/max values and 24 individual samples are in +`measurements-summary.json`. Queue wait occurs after state evidence capture, +when background writes have already progressed; it is not the total I/O cost. + +## Separate model-tracker correctness smoke + +Six fresh three-step runs cover a plain control and baseline/candidate combined +API + subclass model tracking for each model. All four combined cases match +their plain control's complete recorded initial/final model, optimizer, gradient, +stored-value, loss and RNG evidence. Parameter identity and optimizer references +are checked within each run. Complete ordered API/VAR comparisons retain all +fields and find only original ID/timestamp differences. + +Each MNIST combined run emits 520 API and 64 VAR records. Each Transformer run +emits 1,340 API and 64 VAR records. The three-step Transformer smoke uses 21,052 +parameters and its shorter data prefix, distinct from the 28,267-parameter +ten-step timed workload. No performance result is inferred from these smoke runs. + +Observer coverage is explicitly limited: MNIST observes all eight parameters +before/after each step, with eight updates at steps 2 and 3. Transformer observes +only `model.input_emb.weight` before/after each step; all 29 parameter names +appear in update records at steps 2 and 3. Coverage matches the baseline but is +not complete Transformer pre/post coverage. No invariant checker ran in this +study, and trace integrity is not a claim of invariant activation or success. + +## Component measurements + +These local CPU measurements are separate from training. Percentages divide +saved elapsed time by the baseline elapsed time in the same row. + +| Component | Baseline | Candidate | Less time | +|---|---:|---:|---:| +| 120,000 exact scalar conversions, first series | 166.130 ms | 107.213 ms | 35.46% | +| Same scalar workload, second series | 167.685 ms | 105.716 ms | 36.96% | + +Scalar measurements use Python 3.10.16/Torch 2.2.2 and six fixed values repeated +20,000 times, four runs per +variant in A1/B1/B2/A2/A3/B3/B4/A4 order. The baseline function comes from the +frozen baseline source; the candidate is the actual scalar implementation in +`f9b221f`. Its entire dumper module is unchanged in corrected `161313a`, SHA256 +`60e626d77dcbcbdc98317dd061a0bc09cde638b3d0bf70a96a3da510f46fc7de`. +These component measurements do not validate the superseded descriptor cache. +The two repetitions are labeled by the accompanying replay trace files, but the +scalar workload is the same synthetic workload in both; they are not MNIST or +Transformer training timings. No profiling hooks run inside these measurements. + +An earlier descriptor-map cache passed the ordinary workload comparisons but +failed a new regression: a custom getter observed additional descriptor/name +references through `sys.getrefcount`. That implementation and its performance +results remain historical evidence. The corrected cache records only lookup-hook +facts and existing namespace views. The regression fails against the earlier +implementation and passes after correction. + +The corrected cache's fixed containers occupy **344 bytes per process** in the +local runtime, measured with `sys.getsizeof`. This excludes the NamedTuple +class/code objects and allocator overhead; it is not peak RSS. The helper's +SHA256 is `374383debe3dcdb76d13e6b901c25c9441021a4c1cf5df8f1d9bb5f8f520b3d9`. + +Two actual-source local native-metadata batches each have nine alternating-order +repeats. Full Tensor conversion is 893.6 to 887.9 us (0.64% less) in the first +batch and 869.6 to 836.1 us (3.85% less) in the second. The first batch has clear +scheduling outliers; one bounded repeat was performed and both complete batches +are retained. This does not support one stable general converter percentage. +The ordinary pinned lab comparisons provide the primary pipeline measurements. + +## Binary capture and other rejected experiments + +Replay uses one saved 520-record MNIST trace, reconstructing positional integer +keys while retaining every decoded field. The trace SHA256 is +`5f941cfb23dee11054af901056fbbac489f440ba32ce005f859c8476323dac59`. +These are exclusively owned decoded fixtures, not arbitrary live objects. +Forty passes process 20,800 records; each comparison uses four runs per variant. + +| Replay experiment | Current serializer | Experiment | Result | +|---|---:|---:|---| +| Pickle capture alone, without validation | 58.912 ms | 31.892 ms | 45.86% less producer work in this optimistic fixture | +| Python validation followed by pickle | 58.773 ms | 124.984 ms | 112.65% more producer work | +| Pickle capture, decode, then existing JSON serialization | 58.773 ms | 143.648 ms | 144.41% more total sequential work | + +Pickle payloads occupy 286,757 bytes versus 350,553 UTF-8 JSON bytes, excluding +framing. Exact final JSON strings match after round-trip. This does **not** prove +a correct live-object snapshot: custom pickle hooks can run application code, +and Python validation followed by pickling leaves a concurrent mutation gap. +IPC, scheduling, copies, backpressure, worker memory and drain costs are excluded. +Parallel execution might hide some worker work, but was not measured here. +The binary approach remains exploratory, with no production change or training +speedup claim. + +The same reconstructed MNIST corpus sends 219/520 records (42.12%) through +stdlib fallback because positional argument keys are integers. A callback-free +prefix guard before bypassing the failed `orjson` attempt was **128.56% slower** +than the existing serializer. Simply enabling non-string keys changes codec +semantics, including nonfinite values, so it was not adopted. UTF-8 byte batching +showed only 0.09% difference in that replay and was not adopted. Direct native +slot dispatch and cached mappingproxy getter methods also failed to provide a +meaningful gain. Preserved prototypes and raw results distinguish these rejected +options from the final implementation. + +## Validation + +- Python 3.10: 70 focused metadata/scalar/trace-integrity regressions pass. +- Python 3.12: 43 metadata tests and 12 scalar tests pass. +- Full Python 3.10 suite: 357 pass, with two failures reproduced in an isolated + unchanged baseline archive: `test_traincheck_internal_attr_hidden` and + `test_inject_eval_loop`. This is not a claim that the full suite is green. +- Configured changed-file pre-commit hooks pass. New tests also pass explicit + Ruff/Black/isort checks. Strict scoped mypy checks the metadata helper and both + new test modules; ordinary repository mypy checks the changed dumper and its + callers. Existing untyped metadata delegates remain outside strict coverage. +- Independent cross-review found no actionable defect. Signed-commit verification + succeeded with the configured signing key. +- All 329 timed-evidence files and 89 combined-evidence files passed checksum + verification after download. All study processes have finished. + +## Evidence and reproduction + +Durable raw evidence lives under +`outputs/instrumenter-optimization-20261028/` in the canonical TrainCheck +checkout, outside Git. Relevant files include `CHECKPOINT.md`, +`native-lookup-lifetime-repair/REPORT.md`, its two actual-source timing datasets, +`native-lookup-lifetime-repair/final-cache-size.json`, `mnist-final-component-probe.json`, +`transformer-final-component-probe.json`, `binary-capture-probe-checked.json`, +and the test logs. Scripts, full timing repetitions, source hashes and limitations +are retained beside those results. Exploratory older results are preserved and +are not substituted for the final-candidate numbers above. + +The isolated labmach harness is under `labmach/`; its README documents the frozen +source inventories, existing runtime/data, exact invocations, resource bounds +and separate `--stage micro` / `--stage training` gates. The candidate study is +`/home/yuxuan/experiments/instrumenter-optimization-20261028/161313a93512`. +Its preparation archive SHA256 is +`e6d8929a4bff0e1d5101f38cbb2c7f4ba8583241f0f8c31b4f1a6fefc40f93d3`. +Local final study artifacts are under `labmach/studies/161313a93512/`. + +| Artifact | SHA256 | +|---|---| +| `timed-evidence.tar.gz` (23,645,408 bytes) | `1fe4188c382816424e21496327a018a0610caaa8eeaa93d91ecd9f59eaf5471b` | +| `combined-evidence.tar.gz` (3,850,102 bytes) | `e356b9393168c00758f20c7382c57fe877d0798d7851d7e56dd0018f6b396747` | +| `verified-analysis.json` | `1320d25c89f540c8a041d756b928ab3f4d649f4b5789b76f5bb02f31b26f62c2` | +| `measurements-summary.json` | `48302b7201c607ffbbc0b8c56b12fab550816b85ce4ebe7e4c3106bafbd2e867` | +| `verified-combined-analysis.json` | `4a0614b078383279aa286eaf0efa988edff18027167cbecae223f36a123f4dba` | + +No H100 experiment is part of this development pass. From f83785ded34f2822adbceb8e8577a76b723b2ec8 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 15:36:50 +0800 Subject: [PATCH 04/13] Add an agent workflow that reports invariant outcomes and coverage explicitly --- skills/traincheck/SKILL.md | 104 ++++++ skills/traincheck/scripts/summarize_check.py | 318 +++++++++++++++++++ tests/test_traincheck_skill_report.py | 263 +++++++++++++++ 3 files changed, 685 insertions(+) create mode 100644 skills/traincheck/SKILL.md create mode 100644 skills/traincheck/scripts/summarize_check.py create mode 100644 tests/test_traincheck_skill_report.py diff --git a/skills/traincheck/SKILL.md b/skills/traincheck/SKILL.md new file mode 100644 index 00000000..493e900a --- /dev/null +++ b/skills/traincheck/SKILL.md @@ -0,0 +1,104 @@ +--- +name: traincheck +description: Diagnose PyTorch training behavior with TrainCheck reference traces, inferred invariants, and offline checks. Use when an agent needs evidence about a suspected training bug or the behavioral effect of an allowed training change. +--- + +Use TrainCheck to obtain evidence for the user's training task. A violated +invariant is a difference from its reference, not automatically a bug or a reason +to undo an intentional algorithm change. The task's evaluator remains the measure +of success. + +## Establish the experiment + +- Use the supplied TrainCheck source and Python environment. Keep the source on + `PYTHONPATH`; the coverage observer below is a repository tool, not a wheel entry + point. Do not install this skill globally during a treatment/control experiment. +- Use a legitimate public baseline, an authorized known-good run, or supplied + reference artifacts. Never obtain a reference from hidden tests or a withheld + solution. Record the reference's code, data, configuration, and framework + versions. The Llama experiment's invariant file is not a universal rule set. +- Start with a short run that reaches the operation of interest. Use matching + runtime/data/configuration for reference and candidate. Check a fresh healthy + run before treating an alarm as a regression; object IDs, thread IDs, and CUDA + stream handles can make learned rules process-specific. +- Keep benchmark code changes within the task's allowed edit scope. If training + code cannot be changed, run instrumentation in an isolated diagnostic copy and + submit only permitted artifacts. Instrumented training time is not an ordinary + benchmark score. + +## Collect, infer, and check + +Example commands, using absolute paths and fresh output directories: + +```sh +export PYTHONPATH="$TRAINCHECK_SOURCE" +"$PYTHON" -m traincheck.collect_trace --pyscript "$REFERENCE_SCRIPT" \ + --output-dir "$RUN/reference" --sampling-interval 1 --warm-up-steps 0 +"$PYTHON" -m traincheck.infer_engine -f "$RUN/reference" \ + -o "$RUN/invariants.json" --num-workers 1 \ + --disable-relation FunctionLeadRelation FunctionCoverRelation +"$PYTHON" -m traincheck.collect_trace --pyscript "$TARGET_SCRIPT" \ + --invariants "$RUN/invariants.json" --output-dir "$RUN/target" \ + --sampling-interval 1 --warm-up-steps 0 +"$PYTHON" -m traincheck.checker -f "$RUN/target" \ + -i "$RUN/invariants.json" --output-dir "$RUN/check" --num-workers 1 +``` + +This example collects API evidence. For a task that needs parameter/gradient +updates, add `--models-to-track MODEL_VARIABLE --model-tracker-style subclass` to +both collection commands, then verify the generated policy and actual variable +records. A requested tracker alone does not prove those checks ran. Distributed +FSDP/DTensor model tracking and concurrent checking have separate compatibility +limits; use the offline path for the initial pilot. Do not switch to a different +tracker to make a failed comparison pass. + +The two ordering relations are disabled in this bounded starting recipe to match +the validated offline workflow; record any chosen relation filter. Add a needed +family deliberately rather than silently changing rules after seeing a result. +For launchers or script arguments, inspect `collect_trace --help` and the supplied +task adapter. Do not guess a different training entry point. + +Measure activation with the existing observer, using the complete trace file +list from that run, including variable traces when present: + +```sh +"$PYTHON" "$TRAINCHECK_SOURCE/tools/fresh_validation/check_coverage.py" \ + --traces "$RUN/target"/trace_*.log --invariants "$RUN/invariants.json" \ + --output "$RUN/coverage.json" +"$PYTHON" "$TRAINCHECK_SKILL/scripts/summarize_check.py" \ + --coverage "$RUN/coverage.json" \ + --collection-policy "$RUN/target/instr_opts.json" \ + --output "$RUN/agent_report.json" +``` + +Inspect actual output filenames before invoking the observer; do not pass an +unexpanded glob or omit a `proxy_log.json` required by the chosen tracker. The +summary helper optionally accepts `--violations-summary` pointing to the direct +checker's `violations_summary.json`. That is a count cross-check, not a claim of +complete result equivalence. Complete evidence stays in the hashed source files. + +## Interpret and act + +- Exit code 0 from the checker means it ran, not that training is correct. Read + `agent_report.json`: observed violations, active passes, dormant rules, unknown + outcomes, skipped variable checks, and observer/checker failures are distinct. + A report with no completed relation decisions provides no checking evidence. +- For an alarm, inspect its `per_invariant` entry in the original coverage file: + the complete `checker_result`, invariant, trace records, and decision identities + identify what differed. The compact summary lists at most ten problem entries; + it preserves the source file and tells you how many remain. +- Keep literal-ID alarms visible and explain their portability limitation. Do + not delete rules, rewrite IDs, or replace missing values to manufacture parity. + Check whether the same alarm appears on a fresh healthy control. +- If the task intentionally changes an optimizer, dtype, model shape, or other + learned behavior, re-evaluate whether the reference is applicable. Record that + mismatch explicitly. Do not infer new rules from an unverified candidate and + use those rules to certify the same candidate. +- Make a hypothesis from the evidence, apply a task-permitted fix, rerun the + relevant check, and separately run the task's real evaluator. Report diagnosis, + coverage, result, and total cost without equating fewer alarms with a better + model. If no relevant rule activates, say so and use another diagnostic. + +Keep raw traces, checkpoints, and `env_dump.txt` local to the experiment. The +collector records environment variables; share the selected diagnostic evidence, +not the entire output directory by default. diff --git a/skills/traincheck/scripts/summarize_check.py b/skills/traincheck/scripts/summarize_check.py new file mode 100644 index 00000000..32d2aa66 --- /dev/null +++ b/skills/traincheck/scripts/summarize_check.py @@ -0,0 +1,318 @@ +"""Summarize existing TrainCheck coverage without rerunning or changing its rules. + +Use the TrainCheck checkout on PYTHONPATH. This reader validates the fields used +for its conclusions; complete rule/trace evidence stays in the hashed input file. +It never treats a checker process's exit status as an invariant outcome. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +from tools.fresh_validation.contracts import ( + JSONObject, + boolean, + integer, + json_object, + mapping, + required, + sequence, + string, +) + + +def nullable_bool(value: object, label: str) -> bool | None: + return None if value is None else boolean(value, label) + + +def count(value: object, label: str) -> int: + result = integer(value, label) + if result < 0: + raise ValueError(f"{label} cannot be negative") + return result + + +@dataclass(frozen=True) +class RuleView: + index: int + relation: str + description: str | None + status: str + coverage_status: str + passed: bool | None + active: bool | None + dormant: bool | None + skipped: bool | None + decisions: int | None + identity_bearing: bool + + +def read_rule(value: object, position: int) -> RuleView: + rule = mapping(value, f"per_invariant[{position}]") + index = count(required(rule, "invariant_index"), "invariant_index") + if index != position: + raise ValueError("Invariant indices must retain their complete source order") + status = string(required(rule, "status"), "status") + coverage = string(required(rule, "coverage_status"), "coverage_status") + if status not in { + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + }: + raise ValueError(f"Unknown checker status: {status}") + if coverage not in {"observed", "disabled", "unsupported", "error"}: + raise ValueError(f"Unknown coverage status: {coverage}") + passed = nullable_bool(required(rule, "check_passed"), "check_passed") + triggered = nullable_bool(required(rule, "checker_triggered"), "checker_triggered") + active = nullable_bool(required(rule, "active"), "active") + dormant = nullable_bool(required(rule, "dormant"), "dormant") + observed = boolean(required(rule, "coverage_observed"), "coverage_observed") + if status == "completed": + checker = mapping(required(rule, "checker_result"), "checker_result") + mapping(required(checker, "invariant"), "checker_result.invariant") + if passed is None or triggered is None: + raise ValueError("Completed checking requires a known outcome") + if ( + boolean(required(checker, "check_passed"), "checker_result.check_passed") + != passed + or boolean(required(checker, "triggered"), "checker_result.triggered") + != triggered + ): + raise ValueError("Checker result and coverage outcome disagree") + elif passed is not None or triggered is not None: + raise ValueError("Incomplete checking must retain unknown outcomes") + if status in {"checker_error", "observer_error"}: + error = mapping(required(rule, "error"), "error") + string(required(error, "type"), "error.type") + string(required(error, "message"), "error.message") + decisions = None + skipped = None + if coverage == "observed": + if not observed or status not in {"completed", "checker_error"}: + raise ValueError("Observed coverage has an incompatible checker status") + decisions = count(required(rule, "decisions_completed"), "decisions_completed") + if active is not (decisions > 0) or dormant is not ( + status == "completed" and not active + ): + raise ValueError("Activation disagrees with completed relation decisions") + skipped = boolean( + required(rule, "variable_check_skipped"), "variable_check_skipped" + ) + fully_validated = boolean( + required(rule, "fully_validated_active"), "fully_validated_active" + ) + if fully_validated != (active and not skipped and status == "completed"): + raise ValueError("Fully validated status disagrees with activity or skips") + elif observed or active is not None or dormant is not None: + raise ValueError("Unobserved activation must remain unknown") + if status == "unsupported_coverage" and coverage != "unsupported": + raise ValueError("Unsupported coverage must be explicitly marked") + if status == "observer_error" and coverage != "error": + raise ValueError("Observer errors must have error coverage") + description = required(rule, "description") + if description is not None: + description = string(description, "description") + identity = sequence(required(rule, "identity_clauses"), "identity_clauses") + for clause in identity: + mapping(clause, "identity clause") + return RuleView( + index, + string(required(rule, "relation"), "relation"), + description, + status, + coverage, + passed, + active, + dormant, + skipped, + decisions, + bool(identity), + ) + + +def summarize(coverage_value: object) -> JSONObject: + coverage = mapping(coverage_value, "coverage") + if integer(required(coverage, "schema_version"), "schema_version") != 3: + raise ValueError("Expected coverage schema_version 3") + if not boolean(required(coverage, "inputs_unchanged"), "inputs_unchanged"): + raise ValueError("Coverage inputs changed during checking") + rows = count(required(coverage, "trace_rows"), "trace_rows") + hashes = mapping(required(coverage, "input_sha256"), "input_sha256") + if not hashes: + raise ValueError("Coverage requires input hashes") + for path, digest in hashes.items(): + text = string(digest, f"input_sha256[{path}]") + if len(text) != 64 or any(c not in "0123456789abcdef" for c in text): + raise ValueError("Malformed input SHA256") + rules = [ + read_rule(raw, i) + for i, raw in enumerate( + sequence(required(coverage, "per_invariant"), "per_invariant") + ) + ] + counts: Counter[str] = Counter() + problems: list[JSONObject] = [] + for rule in rules: + counts[rule.status] += 1 + counts["observed_active"] += rule.active is True + counts["observed_dormant"] += rule.dormant is True + counts["observed_violations"] += rule.passed is False + counts["observed_active_passes"] += rule.active is True and rule.passed is True + counts["unknown_outcomes"] += rule.passed is None + counts["unobserved_coverage"] += rule.coverage_status != "observed" + counts["skipped_variable_checks"] += rule.skipped is True + counts["completed_decisions"] += 0 if rule.decisions is None else rule.decisions + if rule.passed is not True or rule.active is not True or rule.skipped: + problems.append( + json_object( + { + "invariant_index": rule.index, + "relation": rule.relation, + "description": rule.description, + "status": rule.status, + "coverage_status": rule.coverage_status, + "check_passed": rule.passed, + "active": rule.active, + "dormant": rule.dormant, + "variable_check_skipped": rule.skipped, + "identity_bearing": rule.identity_bearing, + "evidence_pointer": f"per_invariant[{rule.index}]", + } + ) + ) + source_summary = mapping(required(coverage, "summary"), "summary") + for source_key, derived_key in ( + ("observed_active_invariants", "observed_active"), + ("observed_dormant_invariants", "observed_dormant"), + ("observed_violated_invariants", "observed_violations"), + ("check_outcome_unknown_invariants", "unknown_outcomes"), + ("coverage_unobserved_invariants", "unobserved_coverage"), + ): + if ( + count(required(source_summary, source_key), source_key) + != counts[derived_key] + ): + raise ValueError( + f"Coverage summary disagrees with its rule records: {source_key}" + ) + incomplete = any( + counts[key] + for key in ( + "checker_error", + "observer_error", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "observed_dormant", + ) + ) + if counts["observed_violations"]: + outcome = "violations" + elif not rules or rows == 0 or counts["completed_decisions"] == 0: + outcome = "no_evidence" + elif incomplete: + outcome = "incomplete" + else: + outcome = "active_checks_passed" + return json_object( + { + "schema_version": 1, + "outcome": outcome, + "coverage_incomplete": incomplete, + "trace_rows": rows, + "invariants": len(rules), + "counts": { + key: counts[key] + for key in ( + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + "observed_active", + "observed_active_passes", + "observed_dormant", + "observed_violations", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "completed_decisions", + ) + }, + "problem_rules": problems[:10], + "additional_problem_rules": max(0, len(problems) - 10), + "input_sha256": hashes, + "interpretation": "These are observed rule outcomes, not task success or proof that training is correct.", + } + ) + + +def read_artifact(path: Path) -> tuple[JSONObject, JSONObject]: + contents = path.read_bytes() + value: object = json.loads(contents) + return json_object(value, str(path)), { + "path": str(path.resolve()), + "sha256": hashlib.sha256(contents).hexdigest(), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--coverage", type=Path, required=True) + parser.add_argument("--collection-policy", type=Path) + parser.add_argument("--violations-summary", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(namespace=Options()) + coverage, source = read_artifact(args.coverage) + report = summarize(coverage) + report["coverage_source"] = source + report["collection_mode"] = "unknown: no collection policy supplied" + if args.collection_policy is not None: + policy, policy_source = read_artifact(args.collection_policy) + raw_style = required(policy, "model_tracker_style") + style = None if raw_style is None else string(raw_style, "model_tracker_style") + if style is not None and style not in {"proxy", "subclass", "sampler"}: + raise ValueError("Unsupported model_tracker_style in collection policy") + report["collection_mode"] = ( + "api-only" if style is None else "model-tracker-requested" + ) + report["configured_model_tracker"] = style + report["collection_policy_source"] = policy_source + if args.violations_summary is not None: + direct, direct_source = read_artifact(args.violations_summary) + total = count( + required(direct, "distinct_invariants_violated"), + "distinct_invariants_violated", + ) + if total != len(sequence(required(direct, "violations"), "violations")): + raise ValueError( + "Direct checker summary has an inconsistent violation count" + ) + report_counts = mapping(report["counts"], "counts") + if total != report_counts["observed_violations"]: + raise ValueError("Direct and observed checker violation counts disagree") + report["direct_summary_source"] = direct_source + report["direct_count_check"] = ( + "matched; complete result equivalence not established" + ) + # Never overwrite an earlier report or either input artifact during a retry. + with args.output.open("x") as stream: + stream.write(json.dumps(report, indent=2) + "\n") + print(json.dumps({"outcome": report["outcome"], "counts": report["counts"]})) + return 0 + + +class Options(argparse.Namespace): + coverage: Path + collection_policy: Path | None + violations_summary: Path | None + output: Path + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_traincheck_skill_report.py b/tests/test_traincheck_skill_report.py new file mode 100644 index 00000000..c2adfe5c --- /dev/null +++ b/tests/test_traincheck_skill_report.py @@ -0,0 +1,263 @@ +"""Agent summaries distinguish real checking from absent or malformed evidence.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from skills.traincheck.scripts.summarize_check import main, read_artifact, summarize +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import ObservationReport, observe +from tools.fresh_validation.check_coverage import summarize as summarize_observations +from tools.fresh_validation.contracts import ( + JSONObject, + JSONValue, + json_object, + mapping, + sequence, +) + + +def observation(*, valid: bool = True, enabled: bool = True) -> ObservationReport: + dtype = "torch.float32" if valid else "torch.float64" + return observe( + frame([{"torch.FloatTensor": {"shape": [2], "dtype": dtype}}]), + output_rule(), + enabled=enabled, + ) + + +def coverage(*reports: ObservationReport, rows: int = 2) -> JSONObject: + copied = [copy.deepcopy(report) for report in reports] + for index, record in enumerate(copied): + record["invariant_index"] = index + return json_object( + { + "schema_version": 3, + "inputs_unchanged": True, + "trace_rows": rows, + "input_sha256": {"/source/trace.log": "a" * 64}, + "summary": summarize_observations(copied), + "per_invariant": copied, + } + ) + + +class SkillReportTests(unittest.TestCase): + def test_real_active_checks_and_real_violations_remain_distinct(self) -> None: + passed = summarize(coverage(observation())) + failed = summarize(coverage(observation(valid=False))) + self.assertEqual(passed["outcome"], "active_checks_passed") + self.assertEqual(failed["outcome"], "violations") + counts = mapping(failed["counts"], "counts") + self.assertEqual(counts["observed_violations"], 1) + self.assertEqual(counts["completed_decisions"], 1) + + def test_dormant_disabled_and_empty_are_not_active_success(self) -> None: + dormant = observe( + frame([{"torch.FloatTensor": {"shape": [2]}}]), output_rule(False) + ) + self.assertEqual(summarize(coverage(dormant))["outcome"], "no_evidence") + mixed = summarize(coverage(observation(), dormant)) + self.assertEqual(mixed["outcome"], "incomplete") + disabled = summarize(coverage(observation(enabled=False))) + self.assertEqual(disabled["outcome"], "no_evidence") + self.assertTrue(disabled["coverage_incomplete"]) + self.assertEqual(summarize(coverage())["outcome"], "no_evidence") + self.assertEqual( + summarize(coverage(observation(), rows=0))["outcome"], "no_evidence" + ) + + def test_error_unknown_outcome_and_skipped_variable_checks_stay_visible( + self, + ) -> None: + error = observation() + error.update( + { + "status": "checker_error", + "check_passed": None, + "checker_triggered": None, + "error": {"type": "ValueError", "message": "malformed tensor"}, + "fully_validated_active": False, + } + ) + report = summarize(coverage(error)) + self.assertEqual(report["outcome"], "incomplete") + self.assertEqual(mapping(report["counts"], "counts")["unknown_outcomes"], 1) + skipped = observation() + skipped["variable_check_skipped"] = True + skipped["fully_validated_active"] = False + self.assertEqual(summarize(coverage(skipped))["outcome"], "incomplete") + + def test_unsupported_coverage_is_not_zero_violations_or_success(self) -> None: + unsupported = observation(enabled=False) + unsupported.update( + { + "status": "unsupported_coverage", + "coverage_status": "unsupported", + "check_passed": None, + "checker_triggered": None, + } + ) + report = summarize(coverage(unsupported)) + self.assertEqual(report["outcome"], "no_evidence") + counts = mapping(report["counts"], "counts") + self.assertEqual(counts["unsupported_coverage"], 1) + self.assertEqual(counts["unknown_outcomes"], 1) + self.assertEqual(counts["unobserved_coverage"], 1) + + def test_required_fields_types_and_contradictory_evidence_are_rejected( + self, + ) -> None: + good = coverage(observation()) + cases: list[JSONObject] = [] + for field in ("per_invariant", "trace_rows", "input_sha256", "summary"): + missing = copy.deepcopy(good) + del missing[field] + cases.append(missing) + top_level_changes: list[tuple[str, JSONValue]] = [ + ("trace_rows", True), + ("trace_rows", -1), + ("schema_version", 2), + ("inputs_unchanged", False), + ("input_sha256", {}), + ] + for field, value in top_level_changes: + bad = copy.deepcopy(good) + bad[field] = value + cases.append(bad) + for bad in cases: + with self.subTest(bad=bad), self.assertRaises(ValueError): + summarize(bad) + rule_changes: list[tuple[str, JSONValue]] = [ + ("status", "success"), + ("active", False), + ("decisions_completed", True), + ("check_passed", False), + ("invariant_index", 3), + ] + for field, value in rule_changes: + rule = json_object(observation()) + rule["invariant_index"] = 0 + rule[field] = value + bad = copy.deepcopy(good) + bad["per_invariant"] = [rule] + with self.subTest(field=field), self.assertRaises(ValueError): + summarize(bad) + + def test_identity_alarm_and_complete_source_are_preserved(self) -> None: + rule = observation(valid=False) + rule["identity_clauses"] = [{"object_id": "literal-007", "thread_id": 3.5}] + source = coverage(rule) + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "coverage.json" + encoded = json.dumps(source).encode() + path.write_bytes(encoded) + loaded, provenance = read_artifact(path) + result = summarize(loaded) + self.assertEqual(loaded, source) + self.assertEqual(path.read_bytes(), encoded) + self.assertEqual(provenance["sha256"], hashlib.sha256(encoded).hexdigest()) + self.assertEqual(result["outcome"], "violations") + self.assertIn('"identity_bearing": true', json.dumps(result)) + + def test_compact_display_does_not_hide_total_problems_or_change_source( + self, + ) -> None: + source = coverage(*([observation(valid=False)] * 12)) + before = copy.deepcopy(source) + result = summarize(source) + self.assertEqual(len(sequence(result["problem_rules"], "problem_rules")), 10) + self.assertEqual(result["additional_problem_rules"], 2) + self.assertEqual(mapping(result["counts"], "counts")["observed_violations"], 12) + self.assertEqual(source, before) + + def test_aggregate_claim_cannot_override_individual_checker_evidence(self) -> None: + source = coverage(observation(valid=False)) + aggregate = json_object(source["summary"]) + aggregate["observed_violated_invariants"] = 0 + source["summary"] = aggregate + with self.assertRaisesRegex(ValueError, "summary disagrees"): + summarize(source) + + def test_cli_records_actual_policy_count_check_and_refuses_overwrite(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, policy, direct, output = ( + root / name + for name in ( + "coverage.json", + "policy.json", + "direct.json", + "agent.json", + ) + ) + source.write_text(json.dumps(coverage(observation(valid=False)))) + policy.write_text(json.dumps({"model_tracker_style": None})) + direct.write_text( + json.dumps({"distinct_invariants_violated": 1, "violations": [{}]}) + ) + command = [ + "summarize_check.py", + "--coverage", + str(source), + "--collection-policy", + str(policy), + "--violations-summary", + str(direct), + "--output", + str(output), + ] + with patch("sys.argv", command): + self.assertEqual(main(), 0) + result, _ = read_artifact(output) + self.assertEqual(result["outcome"], "violations") + self.assertEqual(result["collection_mode"], "api-only") + with patch("sys.argv", command), self.assertRaises(FileExistsError): + main() + + def test_direct_mismatch_and_malformed_policy_fail_before_output(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, policy, direct, output = ( + root / name + for name in ( + "coverage.json", + "policy.json", + "direct.json", + "agent.json", + ) + ) + source.write_text(json.dumps(coverage(observation()))) + policy.write_text(json.dumps({"model_tracker_style": []})) + direct.write_text( + json.dumps({"distinct_invariants_violated": 1, "violations": [{}]}) + ) + for flag, path in ( + ("--collection-policy", policy), + ("--violations-summary", direct), + ): + with patch( + "sys.argv", + [ + "report", + "--coverage", + str(source), + flag, + str(path), + "--output", + str(output), + ], + ): + with self.assertRaises(ValueError): + main() + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() From 0a0280c7ea13ca136fedacefdca25992a564d67f Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 15:59:24 +0800 Subject: [PATCH 05/13] Freeze bounded RSI agent trial protocol and preserve trainer behavior during tracing --- docs/measurements/rsi-pilot-20261030.md | 130 +++++++ tests/test_rsi_pilot_client.py | 309 ++++++++++++++++ tests/test_rsi_trace_adaptive.py | 191 ++++++++++ tools/rsi_pilot/run_candidate.py | 470 ++++++++++++++++++++++++ 4 files changed, 1100 insertions(+) create mode 100644 docs/measurements/rsi-pilot-20261030.md create mode 100644 tests/test_rsi_pilot_client.py create mode 100644 tests/test_rsi_trace_adaptive.py create mode 100644 tools/rsi_pilot/run_candidate.py diff --git a/docs/measurements/rsi-pilot-20261030.md b/docs/measurements/rsi-pilot-20261030.md new file mode 100644 index 00000000..e4e59725 --- /dev/null +++ b/docs/measurements/rsi-pilot-20261030.md @@ -0,0 +1,130 @@ +# TrainCheck agent pilot: adaptive latent reasoning + +Status: protocol frozen before solver trials; results pending. + +The decision is whether TrainCheck gives research agents useful diagnostic +evidence that justifies a larger evaluation. This is a reduced-compute development +pilot, not an official RSI benchmark submission or leaderboard score. + +## Task and scope + +The public Scale AI RSI repository is pinned to +`aa82cfc29e541e68b5fb79d789b3f09227851739`, task +`tasks/adaptive-latent-reasoning`. Its public local trainer already supports CPU +execution. The unchanged 51,335-parameter looped Transformer learns prefix +addition/subtraction modulo seven with one through five operations. The original +model, eight training loops, data generator and tensor-program interpreter are +retained. Agents edit a loss function and optionally optimizer equations through +that interpreter. They cannot change the model, data, runner, training profile or +evaluator. This pilot fixes inference depths to `[1, 2, 4, 4]` to isolate training +changes; it does not evaluate the task's full inference-policy search space. + +The official task uses an H100, batch 4,096, up to 50,000 updates and a signed +trusted training service. This pilot uses labmach CPU, batch 64 and 10,000 +updates, the public content-hash partition and unsigned local checkpoints. +Learning-rate warmup is 100 updates and cosine decay ends at 10,000. The initial +learning rate, backbone, optimizer defaults and baseline depth weights stay as +in the public task. Data pool construction is disabled in favor of the upstream +on-demand sampler. Runtime differs from the official lockfile: the existing +Python 3.11.4 / Torch 2.16 development CPU environment is used with an isolated +safetensors 0.8.0 overlay. Exact profiles and hashes are retained with artifacts. + +## Calibration, before solver access + +The built-in objective at 200, 1,000 and 3,000 updates stayed near chance on the +public development panel. A 10,000-update run using the exact public +`baseline_research.py` through the actual tensor-program interpreter completed +in 132.716 seconds of training (134.21 seconds process wall time). Development +accuracy at depths 1/2/4/8 was 22.220/24.173/23.548/23.235 percent. Gains were +concentrated in one- and two-operation expressions; longer expressions remained +near chance. This is one deterministic training run, not a repeated timing +estimate. The one-operation development stratum has only 17 distinct examples; +macro averages weight it equally with the four larger strata. + +## Comparison protocol + +- Three matched pairs of fresh agents, trial order C1/T1, T2/C2, C3/T3. Each + receives the same public source interface, baseline program, baseline + development feedback, frozen training profile and optimization objective. +- All solver agents start without the parent conversation. They use the same + inherited model/effort configuration. No solution, repository task metadata + containing solution explanations, private verifier, other trial or final-test + result is provided. The operator inspected task metadata during selection; + that context is deliberately excluded from solver prompts. +- Each agent may submit at most two candidate programs for 10,000-update runs. + Failed requests consume this allowance. Treatment agents additionally receive + the TrainCheck skill and may request one short diagnostic rerun for a candidate. + This additional computation is measured and included in cost reporting. + Both arms may select the supplied baseline instead of a candidate. +- The intended reasoning window is 15 minutes per solver, excluding time queued + for shared compute. The client enforces candidate and diagnostic counts; + agent reasoning/token limits are not mechanically enforced by this native + subagent interface. Actual elapsed time and any overrun will be reported. +- Candidate training is serialized on the same four idle performance cores + (8, 10, 12, 14). Original upstream code uses four Torch threads. A shared lock + prevents candidate/diagnostic overlap. Process wall time includes queue and + transfer costs; training-body time is reported separately. +- Final selection is written before held-out scoring. The operator then uses + the original model, public generator's test partition, seed 20261030 and up to + 512 distinct examples per operation stratum. No candidate is tuned on these + results. Final scoring uses upstream accuracy/FLOP accounting and reward + formula for the fixed policy; it is still a pilot metric, not an official + score or trusted-service receipt. + +## TrainCheck treatment and interpretation + +Reference inference uses the unmodified public baseline program and an explicit +three-update probe. Target probes use the same runtime, model and batch size. +The probe's `total_steps` is three, so step-dependent candidate programs may +behave differently from their full 10,000-update runs. This limitation is exposed +to solvers. Tracing is API-only; no claim of variable-tracker or complete gradient +invariant coverage is made. The interpreter rejects Tensor subclasses, so the +model tracker is not silently enabled. + +The adapter checks unmodified trainer source and traces only the training body. +Full reference tracing is installed before the research interpreter captures +Torch callable references. Plain/reference controls must match model, gradient, +optimizer, buffer, loss, Torch RNG and local data-generator state. Unseeded +global Python/NumPy RNGs must remain unchanged within each run. Invariants are +frozen before candidate checks, and no rule is discarded to reduce alarms. +FunctionLead/FunctionCover relations are excluded as in the validated bounded +offline workflow; remaining families are retained. + +An invariant violation means a difference from reference behavior. Loss and +optimizer changes are expressly allowed, so many differences may be legitimate. +The treatment receives the fresh healthy-run report and full evidence paths to +help distinguish such differences. Checker exit code zero does not mean no +violations; errors, dormant rules, skipped checks and missing evidence stay +explicit. Both ordinary training and diagnostic failures are retained. + +## Decision criteria + +Record task success, selected-program identity, development and held-out reward, +all attempted candidates, measured training/diagnostic/setup cost, actual tool +use and whether any diagnostic changed an agent decision. A useful mechanistic +signal is an agent using relevant evidence to correct an unintended training +behavior, followed by independent validation. Fewer alarms alone do not count +as improvement. A higher score in one of three pairs does not establish a +general effect or statistical significance. + +If diagnostics are unused, irrelevant, dominated by healthy/reference-mismatch +alarms or add cost without improving decisions, the recommendation is to pause +broader performance work and target the demonstrated integration/coverage gap. +If the treatment shows a reproducible useful mechanism, select additional tasks +and increase repeats before making claims about model quality or efficiency. + +## Artifacts and engineering checkpoint + +Local large artifacts: `outputs/rsi-pilot-20261030/` in the experiment owner's +canonical TrainCheck checkout. Remote: labmach +`/home/yuxuan/experiments/traincheck-rsi-pilot-20261030/`. Only allowlisted public +task files are copied into solver workspaces; raw traces/checkpoints stay out of +Git. The native solver is instruction-isolated, not an OS-enforced hostile-agent +sandbox; this limits claims about benchmark integrity against deliberate abuse. + +The pilot branch integrates the reviewed #163 changes that had not reached main. +Composed regression checks passed 134 tests on labmach. The exportable skill and +report helper passed ten tests, explicit strict typing and configured checks. +A separate four-step linear-model workflow inferred 91 rules; its fresh healthy +control produced 87 active passes and four containment alarms, with matching +recorded losses. That smoke is separate from this RSI task. diff --git a/tests/test_rsi_pilot_client.py b/tests/test_rsi_pilot_client.py new file mode 100644 index 00000000..6fd5385b --- /dev/null +++ b/tests/test_rsi_pilot_client.py @@ -0,0 +1,309 @@ +"""Boundary checks for the pilot client; training is tested on the real runtime.""" + +import hashlib +import json +import shlex +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.fresh_validation.contracts import JSONObject, JSONValue, read_object +from tools.rsi_pilot.run_candidate import ( + configure, + execute, + main, + program_text, + remote_command, + run, + train_arguments, + validate_diagnostics, + validate_results, +) + + +def trial_fixture(root: Path, arm: str = "control") -> tuple[Path, Path]: + trial, workspace = root / "trial", root / "solver" + trial.mkdir() + workspace.mkdir() + program = workspace / "research.py" + program.write_text( + "def loss(logits, tokens, labels, step, total_steps):\n return logits.sum()\n" + ) + profile = root / "profile.toml" + profile.write_text("[training]\nsteps = 5\n# PILOT_RESEARCH_SOURCE\n") + manifest = { + "workspace": str(workspace), + "arm": arm, + "profile": str(profile), + "profile_sha256": hashlib.sha256(profile.read_bytes()).hexdigest(), + "max_candidates": 3, + "max_diagnostics": 1 if arm == "traincheck" else 0, + "expected_steps": 5, + "host": "unused-host", + "remote_trial": "/experiment/trial", + "remote_workspace": "/experiment/workspace", + "remote_overlay": "/experiment/overlay", + "remote_python": "/experiment/python", + "affinity": "8,10,12,14", + "diagnostic_driver": "/experiment/diagnose", + "remote_compute_lock": "/experiment/compute.lock", + } + (trial / "trial.json").write_text(json.dumps(manifest)) + return trial, program + + +def training_result() -> JSONObject: + return { + "completed_steps": 5, + "training_seconds": 0.25, + "device_backend": "cpu", + "final": [{"loops": 1, "macro_accuracy_percent": 20.0}], + } + + +class PilotClientTests(unittest.TestCase): + def test_program_is_data_not_shell_code(self) -> None: + source = 'def loss(logits, tokens, labels, step, total_steps):\n return "`touch sentinel` $HOME"\n' + config = configure("[training]\n# PILOT_RESEARCH_SOURCE\n", source) + self.assertIn('research_source = "def loss', config) + self.assertIn("\\n", config) + self.assertNotIn("\n return", config) + arguments = ["python", "path with spaces", "$(touch sentinel)", "x'y"] + command = remote_command({"host": "labmach"}, arguments) + self.assertEqual(shlex.split(command[-1]), arguments) + + def test_profile_marker_is_required_and_unique(self) -> None: + for profile in ("", "# PILOT_RESEARCH_SOURCE\n# PILOT_RESEARCH_SOURCE"): + with self.assertRaises(ValueError): + configure(profile, "pass") + + def test_unicode_source_is_not_encoded_as_invalid_toml_surrogates(self) -> None: + source = 'def loss(logits, tokens, labels, step, total_steps):\n "😀"\n return logits.sum()\n' + encoded = configure("# PILOT_RESEARCH_SOURCE", source) + self.assertNotIn("\\ud83d", encoded) + self.assertEqual(json.loads(encoded.removeprefix("research_source = ")), source) + + def test_only_contained_small_utf8_programs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + workspace = root / "solver" + workspace.mkdir() + program = workspace / "research.py" + program.write_text("pass\n") + self.assertEqual(program_text(program, workspace), "pass\n") + outside = root / "outside.py" + outside.write_text("pass\n") + with self.assertRaises(ValueError): + program_text(outside, workspace) + program.write_bytes(b"x" * 32769) + with self.assertRaises(ValueError): + program_text(program, workspace) + program.write_bytes(b"\xff") + with self.assertRaises(UnicodeDecodeError): + program_text(program, workspace) + + def test_failed_uploads_are_charged_and_have_nonempty_error_receipts(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch( + "tools.rsi_pilot.run_candidate.upload", + side_effect=subprocess.CalledProcessError(255, ["scp"]), + ), + ): + for index in range(1, 4): + with self.assertRaises(subprocess.CalledProcessError): + run(trial, program, False) + receipt = read_object( + trial / "runs" / f"candidate-{index:02d}" / "run.json" + ) + self.assertEqual(receipt["status"], "client_error") + self.assertIn( + '"phase": "upload_config"', json.dumps(receipt["errors"]) + ) + self.assertIn( + "non-zero exit status 255", json.dumps(receipt["errors"]) + ) + with self.assertRaisesRegex(ValueError, "Candidate budget exhausted"): + run(trial, program, False) + self.assertEqual( + read_object(trial / "usage.json"), + { + "candidates_reserved": 3, + "diagnostics_reserved": 0, + }, + ) + + def test_one_diagnostic_attempt_is_reserved_even_when_transfer_fails(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary), "traincheck") + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch( + "tools.rsi_pilot.run_candidate.upload", + side_effect=OSError("connection lost"), + ), + ): + with self.assertRaises(OSError): + run(trial, program, True) + with self.assertRaisesRegex(ValueError, "Diagnostic budget exhausted"): + run(trial, program, True) + self.assertEqual( + read_object(trial / "usage.json"), + { + "candidates_reserved": 1, + "diagnostics_reserved": 1, + }, + ) + + def test_control_cannot_request_diagnostics(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with self.assertRaisesRegex(ValueError, "not enabled"): + run(trial, program, True) + self.assertFalse((trial / "usage.json").exists()) + + def test_remote_training_failure_remains_failure_without_results(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch("tools.rsi_pilot.run_candidate.execute", return_value=124), + patch("tools.rsi_pilot.run_candidate.download") as downloader, + ): + result = run(trial, program, False) + downloader.assert_not_called() + self.assertEqual(result["status"], "training_failed") + self.assertNotIn("development_depths", result) + self.assertIn("Training exited 124", json.dumps(result["errors"])) + + def test_failed_result_download_has_receipt_and_does_not_claim_completion( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch("tools.rsi_pilot.run_candidate.execute", return_value=0), + patch( + "tools.rsi_pilot.run_candidate.download", + side_effect=OSError("download lost"), + ), + ): + with self.assertRaises(OSError): + run(trial, program, False) + receipt = read_object(trial / "runs/candidate-01/run.json") + self.assertEqual(receipt["status"], "client_error") + self.assertEqual(receipt["training_exit_code"], 0) + self.assertNotIn("development_depths", receipt) + self.assertIn("download_results", json.dumps(receipt["errors"])) + + def test_invalid_result_metadata_is_not_accepted_as_success(self) -> None: + manifest: JSONObject = {"expected_steps": 5} + valid = training_result() + self.assertEqual(validate_results(valid, manifest)[0], 5) + changes: list[tuple[str, JSONValue]] = [ + ("completed_steps", True), + ("completed_steps", 4), + ("training_seconds", float("nan")), + ("training_seconds", -1), + ("device_backend", "cuda"), + ("unrelated_metric", float("nan")), + ("final", []), + ("final", [{"loops": 1, "macro_accuracy_percent": float("inf")}]), + ] + for key, value in changes: + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + validate_results({**valid, key: value}, manifest) + del valid["completed_steps"] + with self.assertRaises(ValueError): + validate_results(valid, manifest) + + def test_malformed_diagnostic_cannot_be_successfully_downloaded_evidence( + self, + ) -> None: + with self.assertRaises(ValueError): + validate_diagnostics({}) + with self.assertRaises(ValueError): + validate_diagnostics({"schema_version": 1, "outcome": "success"}) + + def test_cli_reports_nonzero_for_a_recorded_training_failure(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + with ( + patch( + "sys.argv", + [ + "run_candidate", + "--trial", + str(root), + "--program", + str(root / "program.py"), + ], + ), + patch( + "tools.rsi_pilot.run_candidate.run", + return_value={ + "status": "training_failed", + "errors": [{"message": "Training exited 1"}], + }, + ), + ): + self.assertEqual(main(), 1) + + def test_malformed_usage_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + (trial / "usage.json").write_text( + '{"candidates_reserved": false, "diagnostics_reserved": 0}' + ) + with self.assertRaises(ValueError): + run(trial, program, False) + self.assertFalse((trial / "runs/candidate-01").exists()) + + def test_cpu_thread_settings_match_calibration(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, _ = trial_fixture(Path(temporary)) + command = train_arguments(read_object(trial / "trial.json"), "/run") + for setting in ( + "OMP_NUM_THREADS=4", + "MKL_NUM_THREADS=4", + "OPENBLAS_NUM_THREADS=1", + ): + self.assertIn(setting, command) + self.assertEqual(command[-3:], ["--device", "cpu", "--require-complete"]) + self.assertEqual( + command[:4], ["flock", "-w", "360", "/experiment/compute.lock"] + ) + self.assertIn("300s", command) + + def test_timeout_stops_owned_descendants(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + sentinel = root / "descendant-ran" + child = ( + "import pathlib,time; time.sleep(2); pathlib.Path(" + + repr(str(sentinel)) + + ").write_text('leaked')" + ) + parent = ( + "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c'," + + repr(child) + + "]); time.sleep(30)" + ) + code = execute([sys.executable, "-c", parent], root / "process.log", 1) + self.assertEqual(code, 124) + time.sleep(1.5) + self.assertFalse(sentinel.exists()) + self.assertIn("CLIENT_TIMEOUT", (root / "process.log").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rsi_trace_adaptive.py b/tests/test_rsi_trace_adaptive.py new file mode 100644 index 00000000..6d7e56d8 --- /dev/null +++ b/tests/test_rsi_trace_adaptive.py @@ -0,0 +1,191 @@ +"""Behavioral contracts for the narrow, unchanged-source RSI trace window.""" + +from __future__ import annotations + +import hashlib +import sys +import tempfile +import unittest +from pathlib import Path +from types import FrameType, FunctionType + +import torch + +from tools.rsi_pilot.trace_adaptive import ( + StepWindow, + locate_window, + run, + state_value, + tensor_evidence, +) + +SOURCE = """def helper(): + return 3 + +def main(optimizer, fail=False): + completed = 0 + for step in range(1, 3): + helper() + optimizer.zero_grad() + loss = torch.tensor(float(step)) + helper() + if fail: + raise ValueError('original failure') + completed = step + helper() + return completed +""" + + +class FakeOptimizer: + def __init__(self, flags: list[bool]) -> None: + self.flags = flags + self.observed: list[bool] = [] + + def zero_grad(self) -> None: + self.observed.append(self.flags[0]) + + +class TestWindow(unittest.TestCase): + def test_restricts_window_preserves_return_and_captures_outside(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "train.py" + source.write_text(SOURCE) + lines = locate_window( + source, hashlib.sha256(source.read_bytes()).hexdigest() + ) + namespace: dict[str, object] = {"torch": torch} + exec(compile(SOURCE, str(source), "exec"), namespace) + entry = namespace["main"] + assert isinstance(entry, FunctionType) + flags = [False] + snapshots: list[int] = [] + + def enable() -> None: + flags[0] = True + + def disable() -> None: + flags[0] = False + + def snapshot(frame: FrameType) -> dict[str, object]: + self.assertFalse(flags[0]) + step: object = frame.f_locals["step"] + assert isinstance(step, int) + snapshots.append(step) + return {"step": step} + + window = StepWindow(entry.__code__, lines, enable, disable, snapshot) + optimizer = FakeOptimizer(flags) + prior = sys.gettrace() + try: + sys.settrace(window) + result: object = entry(optimizer) + finally: + sys.settrace(prior) + window.close() + self.assertEqual(result, 2) + self.assertEqual(optimizer.observed, [True, True]) + self.assertEqual(snapshots, [1, 1, 2, 2]) + self.assertEqual([row["loss"] for row in window.records], [1.0, 2.0]) + self.assertFalse(window.failed) + self.assertFalse(flags[0]) + self.assertIsNone(window(sys._getframe(), "line", None)) + + def test_exception_propagates_and_disarms(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "train.py" + source.write_text(SOURCE) + lines = locate_window( + source, hashlib.sha256(source.read_bytes()).hexdigest() + ) + namespace: dict[str, object] = {"torch": torch} + exec(compile(SOURCE, str(source), "exec"), namespace) + entry = namespace["main"] + assert isinstance(entry, FunctionType) + flags = [False] + + def enable() -> None: + flags[0] = True + + def disable() -> None: + flags[0] = False + + def snapshot(frame: FrameType) -> dict[str, object]: + return {} + + window = StepWindow(entry.__code__, lines, enable, disable, snapshot) + prior = sys.gettrace() + try: + sys.settrace(window) + with self.assertRaisesRegex(ValueError, "original failure"): + entry(FakeOptimizer(flags), True) + finally: + sys.settrace(prior) + window.close() + self.assertTrue(window.failed) + self.assertFalse(flags[0]) + self.assertEqual(window.records, []) + + def test_source_drift_and_duplicate_anchors_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "train.py" + source.write_text(SOURCE) + with self.assertRaisesRegex(ValueError, "pinned source hash"): + locate_window(source, "0" * 64) + duplicate = SOURCE.replace( + "optimizer.zero_grad()", + "optimizer.zero_grad()\n optimizer.zero_grad()", + ) + source.write_text(duplicate) + with self.assertRaisesRegex(ValueError, "unique"): + locate_window(source, hashlib.sha256(source.read_bytes()).hexdigest()) + + def test_foreign_hook_is_not_replaced(self) -> None: + def prior(frame: FrameType, event: str, arg: object) -> None: + return None + + old = sys.gettrace() + try: + sys.settrace(prior) + with self.assertRaisesRegex(RuntimeError, "existing Python trace hook"): + run(Path("missing"), Path("unused"), "plain", None, []) + self.assertIs(sys.gettrace(), prior) + finally: + sys.settrace(old) + + def test_adapter_owned_arguments_and_policy_contract(self) -> None: + with self.assertRaisesRegex(ValueError, "exactly for selective"): + run(Path("missing"), Path("unused"), "selective", None, []) + with self.assertRaisesRegex(ValueError, "owns upstream"): + run(Path("missing"), Path("unused"), "plain", None, ["--device=cuda"]) + + def test_snapshot_copies_values_and_preserves_container_kinds(self) -> None: + value = torch.tensor([1.0, 2.0]) + before = tensor_evidence(value) + copied = state_value({0: (value,), "group": [True, None]}) + value.add_(1) + self.assertNotEqual(before, tensor_evidence(value)) + self.assertEqual( + copied, + { + "kind": "dict", + "items": [ + [ + 0, + { + "kind": "tuple", + "items": [{"kind": "tensor", "value": before}], + }, + ], + ["group", {"kind": "list", "items": [True, None]}], + ], + }, + ) + with self.assertRaisesRegex(TypeError, "state keys"): + state_value({(1, 2): 3}) + with self.assertRaisesRegex(TypeError, "Unsupported"): + state_value(object()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/rsi_pilot/run_candidate.py b/tools/rsi_pilot/run_candidate.py new file mode 100644 index 00000000..f67b3772 --- /dev/null +++ b/tools/rsi_pilot/run_candidate.py @@ -0,0 +1,470 @@ +"""Bounded client for the CPU adaptive-reasoning development pilot. + +The operator owns the trial manifest, trainer, reference invariants and evaluator. +Solvers supply only a research program interpreted by the upstream TensorProgram. +This client is an experiment harness, not a security boundary against its owner. +""" + +import argparse +import fcntl +import hashlib +import json +import math +import os +import shlex +import signal +import subprocess +import sys +import time +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + mapping, + number, + read_object, + required, + sequence, + string, +) + + +def checked_text(record: JSONObject, key: str) -> str: + value = string(required(record, key), key) + if not value: + raise ValueError(f"{key} must be nonempty") + return value + + +def program_text(path: Path, workspace: Path) -> str: + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(workspace.resolve(strict=True)): + raise ValueError("Research program must be inside this solver's workspace") + raw = resolved.read_bytes() + if not 0 < len(raw) <= 32768: + raise ValueError("Research program must contain 1..32768 UTF-8 bytes") + return raw.decode("utf-8") + + +def configure(template: str, source: str) -> str: + """Insert source as a TOML basic string; never interpolate it into shell code.""" + marker = "# PILOT_RESEARCH_SOURCE" + if template.count(marker) != 1: + raise ValueError("Frozen profile must contain exactly one source marker") + # TOML forbids JSON's surrogate-pair escapes for non-BMP Unicode characters. + return template.replace( + marker, "research_source = " + json.dumps(source, ensure_ascii=False) + ) + + +def remote_command(manifest: JSONObject, arguments: list[str]) -> list[str]: + """All remote arguments are separately shell-quoted for OpenSSH's command.""" + return [ + "ssh", + "-o", + "BatchMode=yes", + checked_text(manifest, "host"), + shlex.join(arguments), + ] + + +def stop_process_group(process: subprocess.Popen[bytes]) -> None: + """Stop only the session created for this command, including descendants.""" + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass # The owned process group has already exited. + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + # A leader may exit on TERM while its descendants ignore it. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + +def execute(command: list[str], log: Path, timeout: int) -> int: + with log.open("wb") as handle: + with subprocess.Popen( + command, stdout=handle, stderr=subprocess.STDOUT, start_new_session=True + ) as process: + try: + return process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # GNU timeout also bounds the remote process, independently of + # local SSH cleanup if the connection disappears. + stop_process_group(process) + handle.write(b"\nCLIENT_TIMEOUT\n") + return 124 + except BaseException: + stop_process_group(process) + raise + + +def checked_execute(command: list[str], log: Path, timeout: int = 60) -> None: + code = execute(command, log, timeout) + if code: + raise subprocess.CalledProcessError(code, command) + + +def upload(manifest: JSONObject, source: Path, target: str, log: Path) -> None: + checked_execute( + [ + "scp", + "-q", + "-o", + "BatchMode=yes", + str(source), + checked_text(manifest, "host") + ":" + shlex.quote(target), + ], + log, + ) + + +def download(manifest: JSONObject, source: str, target: Path, log: Path) -> None: + checked_execute( + [ + "scp", + "-q", + "-o", + "BatchMode=yes", + checked_text(manifest, "host") + ":" + shlex.quote(source), + str(target), + ], + log, + ) + + +def train_arguments(manifest: JSONObject, remote: str) -> list[str]: + workspace = checked_text(manifest, "remote_workspace") + # Upstream recipe_hashes resolves its workspace from module __file__, not cwd. + return [ + "flock", + "-w", + "360", + checked_text(manifest, "remote_compute_lock"), + "timeout", + "--signal=TERM", + "--kill-after=10s", + "300s", + "taskset", + "-c", + checked_text(manifest, "affinity"), + "env", + "PYTHONDONTWRITEBYTECODE=1", + "PYTHONHASHSEED=0", + "CUDA_VISIBLE_DEVICES=", + "OMP_NUM_THREADS=4", + "MKL_NUM_THREADS=4", + "OPENBLAS_NUM_THREADS=1", + "PYTHONPATH=" + checked_text(manifest, "remote_overlay") + ":" + workspace, + checked_text(manifest, "remote_python"), + "-m", + "latent_reasoning.train", + "--config", + remote + "/config.toml", + "--output", + remote + "/training", + "--device", + "cpu", + "--require-complete", + ] + + +def write_record(path: Path, record: JSONObject) -> None: + """Replace one receipt atomically; callers hold this trial's exclusive lock.""" + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(record, indent=2, allow_nan=False) + "\n") + temporary.replace(path) + + +def reserve(trial: Path, manifest: JSONObject, diagnose: bool) -> tuple[int, int]: + """Reserve attempts before remote work; failed attempts remain charged. + + Missing max_diagnostics explicitly disables diagnostics. A missing usage + ledger is a new/legacy trial: existing complete run receipts reconstruct its + reservations. Malformed/missing receipts fail closed instead of resetting it. + """ + limit = integer(required(manifest, "max_candidates"), "max_candidates") + if not 1 <= limit <= 3: + raise ValueError("max_candidates must be between 1 and 3") + diagnostic_limit = ( + integer(manifest["max_diagnostics"], "max_diagnostics") + if "max_diagnostics" in manifest + else 0 + ) + if not 0 <= diagnostic_limit <= 1: + raise ValueError("max_diagnostics must be 0 or 1") + ledger_path = trial / "usage.json" + if ledger_path.exists(): + ledger = read_object(ledger_path) + candidates = integer( + required(ledger, "candidates_reserved"), "candidates_reserved" + ) + diagnostics = integer( + required(ledger, "diagnostics_reserved"), "diagnostics_reserved" + ) + else: + existing = sorted((trial / "runs").glob("candidate-*")) + candidates = len(existing) + diagnostics = sum( + boolean( + required(read_object(path / "run.json"), "diagnostics_requested"), + "diagnostics_requested", + ) + for path in existing + ) + if candidates < 0 or diagnostics < 0 or diagnostics > candidates: + raise ValueError("Invalid reservation ledger") + if candidates >= limit: + raise ValueError(f"Candidate budget exhausted ({limit})") + if diagnose and diagnostics >= diagnostic_limit: + raise ValueError(f"Diagnostic budget exhausted ({diagnostic_limit})") + candidates += 1 + diagnostics += diagnose + write_record( + ledger_path, + { + "candidates_reserved": candidates, + "diagnostics_reserved": diagnostics, + }, + ) + return candidates, diagnostics + + +def validate_results( + results: JSONObject, manifest: JSONObject +) -> tuple[int, int | float, list[JSONValue]]: + # The upstream result writer disallows nonfinite values everywhere. Validate + # before assigning payloads to the receipt, so cleanup can always serialize it. + json.dumps(results, allow_nan=False) + steps = integer(required(results, "completed_steps"), "completed_steps") + if steps < 1: + raise ValueError("Training must report a positive completed step count") + if "expected_steps" in manifest: + expected = integer(manifest["expected_steps"], "expected_steps") + if expected < 1 or steps != expected: + raise ValueError( + "Completed steps do not match the frozen trial requirement" + ) + elapsed = number(required(results, "training_seconds"), "training_seconds") + if not math.isfinite(elapsed) or elapsed < 0: + raise ValueError("Training duration must be finite and nonnegative") + if checked_text(results, "device_backend") != "cpu": + raise ValueError("This pilot requires CPU results") + final = sequence(required(results, "final"), "final") + if not final: + raise ValueError("Training must return nonempty development evaluation results") + for entry in final: + row = mapping(entry, "development result") + depth = integer(required(row, "loops"), "loops") + accuracy = number( + required(row, "macro_accuracy_percent"), "macro_accuracy_percent" + ) + if depth < 1 or not math.isfinite(accuracy) or not 0 <= accuracy <= 100: + raise ValueError("Malformed development depth or accuracy") + # read_object has already validated recursive JSON values; preserve them. + stored_final = results["final"] + assert isinstance(stored_final, list) + return steps, elapsed, stored_final + + +def validate_diagnostics(report: JSONObject) -> None: + json.dumps(report, allow_nan=False) + if integer(required(report, "schema_version"), "schema_version") != 1: + raise ValueError("Expected agent report schema_version 1") + if checked_text(report, "outcome") not in { + "violations", + "incomplete", + "no_evidence", + "active_checks_passed", + }: + raise ValueError("Unknown TrainCheck report outcome") + boolean(required(report, "coverage_incomplete"), "coverage_incomplete") + counts = mapping(required(report, "counts"), "counts") + for name in ( + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + "observed_active", + "observed_active_passes", + "observed_dormant", + "observed_violations", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "completed_decisions", + ): + if integer(required(counts, name), name) < 0: + raise ValueError(f"Negative diagnostic count: {name}") + provenance = mapping(required(report, "coverage_source"), "coverage_source") + string(required(provenance, "path"), "coverage_source.path") + string(required(provenance, "sha256"), "coverage_source.sha256") + + +def run(trial: Path, source_path: Path, diagnose: bool) -> JSONObject: + manifest_path = trial / "trial.json" + manifest = read_object(manifest_path) + workspace = Path(checked_text(manifest, "workspace")) + source = program_text(source_path, workspace) + arm = checked_text(manifest, "arm") + if arm not in ("control", "traincheck"): + raise ValueError("Unknown trial arm") + if diagnose and arm != "traincheck": + raise ValueError("TrainCheck diagnostics are not enabled for this arm") + template_path = Path(checked_text(manifest, "profile")) + template_bytes = template_path.read_bytes() + if hashlib.sha256(template_bytes).hexdigest() != checked_text( + manifest, "profile_sha256" + ): + raise ValueError("Frozen training profile changed") + runs = trial / "runs" + runs.mkdir(exist_ok=True) + with (trial / "run.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + attempt, diagnostic_attempts = reserve(trial, manifest, diagnose) + candidate = f"candidate-{attempt:02d}" + output = runs / candidate + output.mkdir() + remote = checked_text(manifest, "remote_trial") + "/" + candidate + errors: list[JSONValue] = [] + record: JSONObject = { + "candidate": candidate, + "arm": arm, + "diagnostics_requested": diagnose, + "program_sha256": hashlib.sha256(source.encode()).hexdigest(), + "profile_sha256": hashlib.sha256(template_bytes).hexdigest(), + "trial_manifest_sha256": hashlib.sha256( + manifest_path.read_bytes() + ).hexdigest(), + "remote_directory": remote, + "status": "started", + "candidates_reserved": attempt, + "diagnostics_reserved": diagnostic_attempts, + "errors": errors, + } + write_record(output / "run.json", record) + started = time.monotonic() + phase = "prepare_local" + try: + (output / "program.py").write_text(source) + (output / "config.toml").write_text( + configure(template_bytes.decode(), source) + ) + phase = "prepare_remote" + checked_execute( + remote_command(manifest, ["mkdir", "-p", remote]), output / "setup.log" + ) + phase = "upload_config" + upload( + manifest, + output / "config.toml", + remote + "/config.toml", + output / "upload.log", + ) + phase = "training" + code = execute( + remote_command(manifest, train_arguments(manifest, remote)), + output / "training.log", + 700, + ) + record["training_exit_code"] = code + record["status"] = "training_complete" if code == 0 else "training_failed" + if code: + errors.append( + { + "phase": phase, + "type": "ProcessExit", + "message": f"Training exited {code}; see training.log", + } + ) + if code == 0: + phase = "download_results" + download( + manifest, + remote + "/training/results.json", + output / "results.json", + output / "download-results.log", + ) + phase = "validate_results" + results = read_object(output / "results.json") + steps, elapsed, final = validate_results(results, manifest) + record["completed_steps"] = steps + record["training_seconds"] = elapsed + record["development_depths"] = final + if diagnose: + # This operator-supplied driver performs a short independently + # rerun trace and offline check. It does not alter the checkpoint. + phase = "diagnostics" + # The frozen driver owns the same compute lock and its 240s + # timeout. Do not nest that lock or charge queue wait to tracing. + diagnostic = [checked_text(manifest, "diagnostic_driver"), remote] + code = execute( + remote_command(manifest, diagnostic), + output / "diagnostics.log", + 700, + ) + record["diagnostic_exit_code"] = code + if code == 0: + phase = "download_diagnostics" + download( + manifest, + remote + "/diagnostics/agent_report.json", + output / "agent_report.json", + output / "download-diagnostics.log", + ) + phase = "read_diagnostics" + report = read_object(output / "agent_report.json") + validate_diagnostics(report) + record["traincheck_report"] = report + record["diagnostic_status"] = "complete" + else: + record["diagnostic_status"] = "execution_failed" + errors.append( + { + "phase": phase, + "type": "ProcessExit", + "message": f"Diagnostics exited {code}; see diagnostics.log", + } + ) + return record + except BaseException as exc: + record["status"] = "client_error" + errors.append( + { + "phase": phase, + "type": type(exc).__name__, + "message": str(exc) or repr(exc), + } + ) + raise + finally: + record["operational_seconds_including_transfer_and_diagnostics"] = ( + time.monotonic() - started + ) + write_record(output / "run.json", record) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trial", type=Path, required=True) + parser.add_argument("--program", type=Path, required=True) + parser.add_argument("--diagnose", action="store_true") + args = parser.parse_args() + result = run(args.trial.resolve(strict=True), args.program, args.diagnose) + print(json.dumps(result, indent=2, allow_nan=False)) + return int(bool(sequence(required(result, "errors"), "errors"))) + + +if __name__ == "__main__": + raise SystemExit(main()) From 6075e4886bd6847ada4a2fc4fbdb933c2daa0bce Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 15:59:52 +0800 Subject: [PATCH 06/13] Include the tested RSI tracing adapter excluded by the trace artifact glob --- tools/rsi_pilot/trace_adaptive.py | 454 ++++++++++++++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 tools/rsi_pilot/trace_adaptive.py diff --git a/tools/rsi_pilot/trace_adaptive.py b/tools/rsi_pilot/trace_adaptive.py new file mode 100644 index 00000000..b84fe770 --- /dev/null +++ b/tools/rsi_pilot/trace_adaptive.py @@ -0,0 +1,454 @@ +"""Trace only training updates in the pinned RSI adaptive-reasoning workload. + +Run in a fresh interpreter. The upstream trainer is imported unchanged; a line +hook gates API tracing from optimizer.zero_grad through the optimizer update. +Initialization, data sampling, evaluation, and evidence capture stay untraced. +This CPU development adapter does not produce official benchmark receipts. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import importlib +import json +import os +import pickle +import random +import sys +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import CodeType, FrameType, FunctionType +from typing import Literal, TypedDict + +import numpy as np +import torch + +Mode = Literal["plain", "reference", "selective"] +TRAIN_SHA256 = "809b62dfe68b4e9feb8bc38f24c85aad93df1f58ba0e9967f42e6560d8a6c08f" + + +@dataclass(frozen=True) +class WindowLines: + main: int + start: int + stop: int + + +def locate_window(path: Path, expected_sha256: str) -> WindowLines: + """Reject source drift and ambiguous/moved anchors instead of guessing.""" + source = path.read_bytes() + if hashlib.sha256(source).hexdigest() != expected_sha256: + raise ValueError("Upstream train.py differs from the pinned source hash") + tree = ast.parse(source, filename=str(path)) + mains = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "main" + ] + if len(mains) != 1: + raise ValueError("Expected exactly one upstream main function") + main = mains[0] + starts = [ + node + for node in ast.walk(main) + if isinstance(node, ast.Expr) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Attribute) + and isinstance(node.value.func.value, ast.Name) + and node.value.func.value.id == "optimizer" + and node.value.func.attr == "zero_grad" + ] + stops = [ + node + for node in ast.walk(main) + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "completed" + and isinstance(node.value, ast.Name) + and node.value.id == "step" + ] + if len(starts) != 1 or len(stops) != 1: + raise ValueError("Expected unique zero_grad and completed = step anchors") + start, stop = starts[0], stops[0] + loops = [ + node + for node in ast.walk(main) + if isinstance(node, ast.For) + and start in tuple(ast.walk(node)) + and stop in tuple(ast.walk(node)) + ] + if len(loops) != 1 or start.lineno >= stop.lineno: + raise ValueError("Training anchors must belong to one ordered for loop") + return WindowLines(main.lineno, start.lineno, stop.lineno) + + +def tensor_evidence(value: torch.Tensor) -> dict[str, object]: + if value.device.type != "cpu" or value.layout != torch.strided: + raise ValueError("This pilot snapshots only strided CPU tensors") + raw = value.detach().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + return { + "dtype": str(value.dtype), + "shape": list(value.shape), + "requires_grad": value.requires_grad, + "sha256": hashlib.sha256(raw).hexdigest(), + } + + +def state_value(value: object) -> object: + """Snapshot optimizer containers without retaining live mutable references.""" + if isinstance(value, torch.Tensor): + return {"kind": "tensor", "value": tensor_evidence(value)} + if value is None or type(value) in (bool, int, float, str): + return value + if isinstance(value, (list, tuple)): + return { + "kind": "tuple" if isinstance(value, tuple) else "list", + "items": [state_value(item) for item in value], + } + if isinstance(value, dict): + items: list[list[object]] = [] + for key, item in value.items(): + if type(key) not in (int, str): + raise TypeError("Optimizer state keys must be integers or strings") + items.append([key, state_value(item)]) + return {"kind": "dict", "items": items} + raise TypeError(f"Unsupported optimizer state value: {type(value).__name__}") + + +def state_snapshot(frame: FrameType) -> dict[str, object]: + model: object = frame.f_locals["model"] + optimizer: object = frame.f_locals["optimizer"] + rng: object = frame.f_locals["rng"] + if not isinstance(model, torch.nn.Module): + raise TypeError("Upstream model must be a torch.nn.Module") + if not isinstance(optimizer, torch.optim.Optimizer): + raise TypeError("Upstream optimizer must be a torch.optim.Optimizer") + if not isinstance(rng, random.Random): + raise TypeError("Upstream rng must be random.Random") + parameter_names = {id(value): name for name, value in model.named_parameters()} + optimizer_parameters: list[list[str]] = [] + for group in optimizer.param_groups: + names: list[str] = [] + for parameter in group["params"]: + if id(parameter) not in parameter_names: + raise ValueError("Optimizer references a tensor outside the model") + names.append(parameter_names[id(parameter)]) + optimizer_parameters.append(names) + result: dict[str, object] = { + "parameters": { + name: tensor_evidence(value) for name, value in model.named_parameters() + }, + "gradients": { + name: None if value.grad is None else tensor_evidence(value.grad) + for name, value in model.named_parameters() + }, + "buffers": { + name: tensor_evidence(value) for name, value in model.named_buffers() + }, + "optimizer": state_value(optimizer.state_dict()), + "optimizer_parameter_names": optimizer_parameters, + "custom_optimizer_states": state_value(frame.f_locals["optimizer_states"]), + "torch_rng": tensor_evidence(torch.get_rng_state()), + # These are trusted local RNG snapshots, serialized only for a byte digest. + "python_rng": hashlib.sha256(pickle.dumps(random.getstate())).hexdigest(), + "numpy_rng": hashlib.sha256(pickle.dumps(np.random.get_state())).hexdigest(), + "sampler_rng": hashlib.sha256(pickle.dumps(rng.getstate())).hexdigest(), + } + if "sampler" in frame.f_locals: + sampler: object = frame.f_locals["sampler"] + if not isinstance(sampler, torch.Generator): + raise TypeError("Upstream sampler must be a torch.Generator") + result["pool_rng"] = tensor_evidence(sampler.get_state()) + return result + + +class StepRecord(TypedDict): + step: int + seconds: float + before: dict[str, object] + after: dict[str, object] + loss: float + + +class StepWindow: + """A trace callback which returns itself only for the original main frame.""" + + def __init__( + self, + code: CodeType, + lines: WindowLines, + enable: Callable[[], None], + disable: Callable[[], None], + snapshot: Callable[[FrameType], dict[str, object]] = state_snapshot, + ) -> None: + self.code = code + self.lines = lines + self.enable = enable + self.disable = disable + self.snapshot = snapshot + self.active = False + self.records: list[StepRecord] = [] + self.before: dict[str, object] | None = None + self.started: float | None = None + self.step: int | None = None + self.failed = False + + def __call__(self, frame: FrameType, event: str, arg: object) -> StepWindow | None: + if frame.f_code is not self.code: + return None + if event in ("exception", "return") and self.active: + self.disable() + self.active = False + self.failed = True + if event != "line": + return self + if frame.f_lineno == self.lines.start: + if self.active or self.failed: + raise RuntimeError("Training window was reentered or previously failed") + step: object = frame.f_locals["step"] + if not isinstance(step, int) or isinstance(step, bool) or step < 1: + raise TypeError("Upstream step must be a positive integer") + if step != len(self.records) + 1: + raise ValueError("Training steps must be consecutive from one") + self.step = step + self.before = self.snapshot(frame) + self.started = time.perf_counter() + self.active = True + self.enable() + elif frame.f_lineno == self.lines.stop: + if not self.active: + raise RuntimeError("Training end reached without an active window") + self.disable() + assert self.started is not None + elapsed = time.perf_counter() - self.started + self.active = False + assert self.before is not None and self.step is not None + after = self.snapshot(frame) + loss: object = frame.f_locals["loss"] + if not isinstance(loss, torch.Tensor) or loss.ndim != 0: + raise TypeError("Upstream loss must be a scalar tensor") + self.records.append( + { + "step": self.step, + "seconds": elapsed, + "before": self.before, + "after": after, + "loss": float(loss.detach()), + } + ) + self.before = None + return self + + def close(self) -> None: + self.disable() + if self.active: + self.failed = True + self.active = False + + +@dataclass(frozen=True) +class Instrumentation: + enable: Callable[[], None] + disable: Callable[[], None] + drain: Callable[[], None] + instrumented_apis: int + + +def noop() -> None: + pass + + +def install_instrumentation( + mode: Mode, trace_dir: Path, invariants: Path | None +) -> Instrumentation: + if mode == "plain": + return Instrumentation(noop, noop, noop, 0) + from traincheck.config import config + from traincheck.config.config import InstrOpt + from traincheck.instrumentor.caches import META_VARS + from traincheck.instrumentor.control import start_step + from traincheck.instrumentor.proxy_wrapper.proxy_config import tensor_dump_format + + config.DISABLE_WRAPPER = True + config.INSTRUMENTATION_POLICY = {"interval": 1, "warm_up": 0} + config.INSTR_OPTS = None + config.MODEL_TRACKER_STYLE = None + config.INSTR_OPTS_FILE = str(trace_dir / "instr_opts.json") + if META_VARS["step"] != 0: + raise RuntimeError("Trace adapter requires a fresh interpreter") + for key in ("dump_tensor_hash", "dump_tensor_stats", "dump_tensor_full"): + tensor_dump_format[key] = False + if mode == "selective": + from traincheck.collect_trace import ( + get_model_tracker_instr_opts, + get_per_func_instr_opts, + ) + from traincheck.invariant.base_cls import read_inv_file + + assert invariants is not None + rules = read_inv_file(str(invariants)) + if not rules: + raise ValueError("Selective mode requires nonempty invariants") + if get_model_tracker_instr_opts(rules, "subclass") is not None: + raise ValueError("API-only pilot cannot satisfy variable-tracker rules") + options = InstrOpt(get_per_func_instr_opts(rules), None, True) + Path(config.INSTR_OPTS_FILE).write_text(options.to_json() + "\n") + loaded: object = config.load_instr_opts() + if not isinstance(loaded, InstrOpt) or loaded.model_tracker_style is not None: + raise TypeError("Saved API-only instrumentation policy failed validation") + from traincheck.instrumentor.tracer import Instrumentor + + count = Instrumentor( + torch, + scan_proxy_in_args=False, + use_full_instr=False, + API_dump_stack_trace=False, + sampling_interval=1, + warm_up_steps=0, + ).instrument() + + def enable() -> None: + start_step() + + def disable() -> None: + config.DISABLE_WRAPPER = True + + def drain() -> None: + from traincheck.instrumentor import dumper + + for name in ("api_trace_buffer", "var_trace_buffer"): + # Absence means no event was emitted for that stream. + buffer: object = getattr(dumper.THREAD_DATA, name, None) + if buffer is not None: + if not isinstance(buffer, dumper.TraceBuffer): + raise TypeError(f"Unexpected trace buffer: {name}") + buffer._flush() + buffer.queue.join() + + return Instrumentation(enable, disable, drain, count) + + +def run( + workspace: Path, + output: Path, + mode: Mode, + invariants: Path | None, + training_args: Sequence[str], +) -> None: + if mode not in ("plain", "reference", "selective"): + raise ValueError("Unknown tracing mode") + if (mode == "selective") != (invariants is not None): + raise ValueError("--invariants is required exactly for selective mode") + if sys.gettrace() is not None: + raise RuntimeError("Cannot replace an existing Python trace hook") + if any(arg.split("=", 1)[0] in ("--output", "--device") for arg in training_args): + raise ValueError("Adapter owns upstream --output and --device") + workspace, output = workspace.resolve(), output.resolve() + source = workspace / "latent_reasoning" / "train.py" + lines = locate_window(source, TRAIN_SHA256) + output.mkdir(parents=True, exist_ok=False) + trace_dir = output / "traces" + trace_dir.mkdir() + os.environ["TRAINCHECK_OUTPUT_DIR"] = str(trace_dir) + # research.py captures Torch callables in its interpreter registry at import. + # Install wrappers first so those calls participate in reference/selective traces. + instrumentation = install_instrumentation(mode, trace_dir, invariants) + sys.path.insert(0, str(workspace)) + module = importlib.import_module("latent_reasoning.train") + entry: object = getattr(module, "main") + if not isinstance(entry, FunctionType): + raise TypeError("Upstream main must be a Python function") + if ( + Path(entry.__code__.co_filename).resolve() != source + or entry.__code__.co_firstlineno != lines.main + ): + raise ValueError("Imported trainer does not match the verified source") + window = StepWindow( + entry.__code__, lines, instrumentation.enable, instrumentation.disable + ) + manifest: dict[str, object] = { + "schema_version": 1, + "mode": mode, + "source_sha256": TRAIN_SHA256, + "adapter_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "invariants_sha256": ( + None + if invariants is None + else hashlib.sha256(invariants.read_bytes()).hexdigest() + ), + "policy_sha256": ( + hashlib.sha256((trace_dir / "instr_opts.json").read_bytes()).hexdigest() + if mode == "selective" + else None + ), + "window_start_line": lines.start, + "window_stop_line": lines.stop, + "model_tracker": None, + "tensor_value_capture": False, + "instrumented_apis": instrumentation.instrumented_apis, + "training_args": list(training_args), + "scope": "zero_grad through update; CPU; unsigned reduced-compute pilot", + "status": "running", + } + old_argv, old_cwd = sys.argv, Path.cwd() + sys.argv = [ + str(source), + "--output", + str(output / "training"), + "--device", + "cpu", + *training_args, + ] + try: + os.chdir(workspace) + sys.settrace(window) + entry() + if window.active or window.failed or not window.records: + raise RuntimeError("Training did not complete a valid traced update") + manifest["status"] = "complete" + except BaseException as error: + manifest["status"] = "failed" + manifest["error_type"] = type(error).__name__ + raise + finally: + sys.settrace(None) + window.close() + sys.argv = old_argv + os.chdir(old_cwd) + instrumentation.drain() + manifest["steps"] = window.records + (output / "trace_manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n" + ) + + +def main(argv: Sequence[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--mode", choices=("plain", "reference", "selective"), required=True + ) + parser.add_argument("--invariants", type=Path) + parser.add_argument("training_args", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + mode: Mode + if args.mode == "plain": + mode = "plain" + elif args.mode == "reference": + mode = "reference" + else: + mode = "selective" + forwarded: list[str] = args.training_args + if forwarded[:1] == ["--"]: + forwarded = forwarded[1:] + run(args.workspace, args.output, mode, args.invariants, forwarded) + + +if __name__ == "__main__": + main() From ff088d28082c05a80b0ef176fd7f0f4367e5a369 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 16:11:42 +0800 Subject: [PATCH 07/13] Validate diagnostic evidence and freeze the independent RSI scorer --- skills/traincheck/SKILL.md | 4 + skills/traincheck/scripts/summarize_check.py | 4 +- tests/test_rsi_diagnose_adaptive.py | 110 +++++++++++ tests/test_rsi_pilot_client.py | 38 ++++ tests/test_traincheck_skill_report.py | 44 ++++- tools/rsi_pilot/diagnose_adaptive.py | 195 +++++++++++++++++++ tools/rsi_pilot/run_candidate.py | 47 +++++ tools/rsi_pilot/score_adaptive.py | 147 ++++++++++++++ 8 files changed, 586 insertions(+), 3 deletions(-) create mode 100644 tests/test_rsi_diagnose_adaptive.py create mode 100644 tools/rsi_pilot/diagnose_adaptive.py create mode 100644 tools/rsi_pilot/score_adaptive.py diff --git a/skills/traincheck/SKILL.md b/skills/traincheck/SKILL.md index 493e900a..40f1d25c 100644 --- a/skills/traincheck/SKILL.md +++ b/skills/traincheck/SKILL.md @@ -90,6 +90,10 @@ complete result equivalence. Complete evidence stays in the hashed source files. - Keep literal-ID alarms visible and explain their portability limitation. Do not delete rules, rewrite IDs, or replace missing values to manufacture parity. Check whether the same alarm appears on a fresh healthy control. + `has_identity_precondition` describes only the observer's identity precondition + clauses. A false value does not establish portability: literal IDs can also + occur in `checker_result.invariant.params`, including stored argument fields + such as `_TRAINCHECK_positional_ID`. Inspect those arguments for the alarm. - If the task intentionally changes an optimizer, dtype, model shape, or other learned behavior, re-evaluate whether the reference is applicable. Record that mismatch explicitly. Do not infer new rules from an unverified candidate and diff --git a/skills/traincheck/scripts/summarize_check.py b/skills/traincheck/scripts/summarize_check.py index 32d2aa66..ae6d4860 100644 --- a/skills/traincheck/scripts/summarize_check.py +++ b/skills/traincheck/scripts/summarize_check.py @@ -49,7 +49,7 @@ class RuleView: dormant: bool | None skipped: bool | None decisions: int | None - identity_bearing: bool + has_identity_precondition: bool def read_rule(value: object, position: int) -> RuleView: @@ -181,7 +181,7 @@ def summarize(coverage_value: object) -> JSONObject: "active": rule.active, "dormant": rule.dormant, "variable_check_skipped": rule.skipped, - "identity_bearing": rule.identity_bearing, + "has_identity_precondition": rule.has_identity_precondition, "evidence_pointer": f"per_invariant[{rule.index}]", } ) diff --git a/tests/test_rsi_diagnose_adaptive.py b/tests/test_rsi_diagnose_adaptive.py new file mode 100644 index 00000000..e1a64617 --- /dev/null +++ b/tests/test_rsi_diagnose_adaptive.py @@ -0,0 +1,110 @@ +"""Fail-closed contracts for candidate diagnostic phase results.""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.rsi_pilot.diagnose_adaptive import Driver, main + + +class TestDiagnosticPhases(unittest.TestCase): + def phase_result(self, output: Path, **changes: object) -> None: + result: dict[str, object] = { + "exit_code": 0, + "stop_reason": None, + "source_unchanged": True, + } + result.update(changes) + (output / "phase-result.json").write_text(json.dumps(result)) + + def test_checker_errors_can_be_reported_but_collection_errors_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + driver = Driver(output, Path("/python")) + self.phase_result(output, exit_code=1) + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 1) + ): + driver.phase( + output, ["checker.py"], timeout=90, allow_checker_failure=True + ) + with self.assertRaisesRegex(RuntimeError, "Diagnostic phase failed"): + driver.phase(output, ["collector.py"], timeout=90) + + def test_stopped_or_changed_source_never_becomes_checker_result(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + driver = Driver(output, Path("/python")) + cases: tuple[dict[str, object], ...] = ( + {"stop_reason": "timeout"}, + {"source_unchanged": False}, + {"exit_code": -15}, + ) + for changes in cases: + self.phase_result(output, **changes) + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 0) + ): + with self.assertRaisesRegex( + RuntimeError, "Diagnostic phase failed" + ): + driver.phase( + output, + ["checker.py"], + timeout=90, + allow_checker_failure=True, + ) + + def test_missing_required_result_field_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + (output / "phase-result.json").write_text('{"exit_code":0}') + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 0) + ): + with self.assertRaises((ValueError, KeyError)): + Driver(output, Path("/python")).phase( + output, ["check.py"], timeout=90 + ) + + def test_candidate_config_is_required(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + with self.assertRaisesRegex(FileNotFoundError, "configuration missing"): + Driver(output, Path("/python")).diagnose(output) + self.assertFalse((output / "diagnostics").exists()) + + def test_cli_preserves_virtualenv_python_symlink(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + base_python = root / "base-python" + base_python.touch() + venv_python = root / "venv-python" + venv_python.symlink_to(base_python) + argv = [ + "diagnose_adaptive.py", + "--study", + str(root), + "--python", + str(venv_python), + str(root / "candidate"), + ] + with ( + patch("sys.argv", argv), + patch( + "tools.rsi_pilot.diagnose_adaptive.Driver", autospec=True + ) as driver, + patch("builtins.print"), + ): + driver.return_value.diagnose.return_value = root / "report.json" + main() + driver.assert_called_once_with(root.resolve(), venv_python, "8,10,12,14") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rsi_pilot_client.py b/tests/test_rsi_pilot_client.py index 6fd5385b..13f0cee8 100644 --- a/tests/test_rsi_pilot_client.py +++ b/tests/test_rsi_pilot_client.py @@ -20,6 +20,7 @@ remote_command, run, train_arguments, + validate_diagnostic_artifacts, validate_diagnostics, validate_results, ) @@ -268,6 +269,43 @@ def test_malformed_usage_fails_closed(self) -> None: run(trial, program, False) self.assertFalse((trial / "runs/candidate-01").exists()) + def test_diagnostic_drilldown_requires_matching_bytes_and_candidate(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + coverage_path = output / "coverage.json" + coverage_path.write_text('{"schema_version": 3, "per_invariant": []}\n') + config = output / "config.toml" + config.write_text("[training]\nsteps = 5\n") + diagnostic: JSONObject = { + "schema_version": 1, + "status": "complete", + "candidate_config_sha256": hashlib.sha256( + config.read_bytes() + ).hexdigest(), + } + (output / "diagnostic_manifest.json").write_text(json.dumps(diagnostic)) + source_path = "/remote/candidate/diagnostics/coverage.json" + report: JSONObject = { + "coverage_source": { + "path": source_path, + "sha256": hashlib.sha256(coverage_path.read_bytes()).hexdigest(), + } + } + evidence = validate_diagnostic_artifacts(report, output) + self.assertIn(str(coverage_path), json.dumps(evidence)) + self.assertIn(source_path, json.dumps(report)) + config.write_text("[training]\nsteps = 6\n") + with self.assertRaisesRegex(ValueError, "different candidate"): + validate_diagnostic_artifacts(report, output) + coverage_path.write_text("{}\n") + with self.assertRaisesRegex(ValueError, "does not match"): + validate_diagnostic_artifacts(report, output) + + def test_missing_diagnostic_drilldown_is_an_error(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(FileNotFoundError): + validate_diagnostic_artifacts({}, Path(temporary)) + def test_cpu_thread_settings_match_calibration(self) -> None: with tempfile.TemporaryDirectory() as temporary: trial, _ = trial_fixture(Path(temporary)) diff --git a/tests/test_traincheck_skill_report.py b/tests/test_traincheck_skill_report.py index c2adfe5c..0e00a9aa 100644 --- a/tests/test_traincheck_skill_report.py +++ b/tests/test_traincheck_skill_report.py @@ -164,7 +164,49 @@ def test_identity_alarm_and_complete_source_are_preserved(self) -> None: self.assertEqual(path.read_bytes(), encoded) self.assertEqual(provenance["sha256"], hashlib.sha256(encoded).hexdigest()) self.assertEqual(result["outcome"], "violations") - self.assertIn('"identity_bearing": true', json.dumps(result)) + self.assertIn('"has_identity_precondition": true', json.dumps(result)) + + def test_precondition_flag_does_not_claim_absence_of_relation_argument_ids( + self, + ) -> None: + for field, value in ( + ("_TRAINCHECK_positional_ID", "literal-007"), + ("sequence_length", "16"), + ): + rule = observation(valid=False) + rule["identity_clauses"] = [] + checker = rule["checker_result"] + invariant = json_object(checker["invariant"]) + invariant["params"] = [ + { + "param_type": "APIParam", + "api_full_name": "torch.fake", + "arguments": {"args": {"self": {"model": {field: value}}}}, + } + ] + checker["invariant"] = invariant + source = coverage(rule) + before = copy.deepcopy(source) + report = summarize(source) + problem = mapping( + sequence(report["problem_rules"], "problem_rules")[0], "problem" + ) + with self.subTest(field=field): + self.assertIs(problem["has_identity_precondition"], False) + self.assertNotIn("identity_bearing", problem) + self.assertEqual(problem["evidence_pointer"], "per_invariant[0]") + self.assertEqual(source, before) + self.assertEqual(report["outcome"], "violations") + + def test_missing_identity_preconditions_are_unknown_not_false(self) -> None: + source = coverage(observation(valid=False)) + rule = json_object(sequence(source["per_invariant"], "per_invariant")[0]) + del rule["identity_clauses"] + source["per_invariant"] = [rule] + with self.assertRaisesRegex( + ValueError, "Missing required field: identity_clauses" + ): + summarize(source) def test_compact_display_does_not_hide_total_problems_or_change_source( self, diff --git a/tools/rsi_pilot/diagnose_adaptive.py b/tools/rsi_pilot/diagnose_adaptive.py new file mode 100644 index 00000000..7825c394 --- /dev/null +++ b/tools/rsi_pilot/diagnose_adaptive.py @@ -0,0 +1,195 @@ +"""Operator driver for a bounded three-update TrainCheck diagnostic probe. + +The launcher must hold the experiment's CPU lock and apply an overall timeout. +Each expensive phase also uses the existing process-group resource supervisor. +No invariant is inferred, modified, or removed during candidate diagnostics. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import time +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +from tools.fresh_validation.contracts import boolean, integer, read_object, required + + +@dataclass(frozen=True) +class Driver: + study: Path + python: Path + cpus: str = "8,10,12,14" + + @property + def source(self) -> Path: + return self.study / "source" + + def phase( + self, + output: Path, + command: Sequence[str], + *, + timeout: int, + allow_checker_failure: bool = False, + ) -> None: + command_line = [ + "taskset", + "-c", + self.cpus, + str(self.python), + str(self.source / "tools/fresh_validation/run_phase.py"), + "--source", + str(self.source), + "--output", + str(output), + "--timeout", + str(timeout), + "--memory-gib", + "4", + "--trace-mib", + "256", + "--", + "env", + f"PYTHONPATH={self.source}:{self.study / 'overlay'}", + str(self.python), + *command, + ] + process = subprocess.run(command_line, check=False) + result = read_object(output / "phase-result.json") + child_exit = integer(required(result, "exit_code"), "exit_code") + unchanged = boolean(required(result, "source_unchanged"), "source_unchanged") + stopped = required(result, "stop_reason") + # The coverage observer returns 1 for a checker/observer error. Its + # explicit error records are still summarized; limits and missing output + # never become an empty or passing report. + allowed = (0, 1) if allow_checker_failure else (0,) + if ( + child_exit not in allowed + or not unchanged + or stopped is not None + or process.returncode not in allowed + ): + raise RuntimeError( + f"Diagnostic phase failed: {output.name}; inspect {output}" + ) + + def diagnose(self, candidate: Path) -> Path: + candidate = candidate.resolve() + config = candidate / "config.toml" + if not config.is_file(): + raise FileNotFoundError(f"Candidate configuration missing: {config}") + invariants = self.study / "reference/invariants.json" + digest = hashlib.sha256(invariants.read_bytes()).hexdigest() + output = candidate / "diagnostics" + output.mkdir(exist_ok=False) + started = time.monotonic() + trace_output = output / "collection/result" + self.phase( + output / "collection", + [ + str(self.study / "bin/trace_adaptive.py"), + "--workspace", + str(self.study / "adaptive-workspace"), + "--output", + str(trace_output), + "--mode", + "selective", + "--invariants", + str(invariants), + "--", + "--config", + str(config), + "--steps", + "3", + "--pool-size", + "0", + "--require-complete", + ], + timeout=180, + ) + traces = sorted((trace_output / "traces").glob("trace_API*.log")) + if not traces or not all(path.stat().st_size > 0 for path in traces): + raise RuntimeError( + "Successful diagnostic collection must produce API records" + ) + coverage = output / "coverage.json" + self.phase( + output / "checking", + [ + str(self.source / "tools/fresh_validation/check_coverage.py"), + "--traces", + *map(str, traces), + "--invariants", + str(invariants), + "--output", + str(coverage), + ], + timeout=90, + allow_checker_failure=True, + ) + report = output / "agent_report.json" + self.phase( + output / "summary", + [ + str(self.study / "bin/summarize_check.py"), + "--coverage", + str(coverage), + "--collection-policy", + str(trace_output / "traces/instr_opts.json"), + "--output", + str(report), + ], + timeout=20, + ) + if hashlib.sha256(invariants.read_bytes()).hexdigest() != digest: + raise RuntimeError("Frozen reference invariants changed during diagnosis") + manifest = { + "schema_version": 1, + "status": "complete", + "invariants_sha256": digest, + "candidate_config_sha256": hashlib.sha256(config.read_bytes()).hexdigest(), + "elapsed_seconds_including_collection_checking_and_summary": time.monotonic() + - started, + "scope": "Three fresh CPU updates, API-only selective collection and offline checking", + "limitations": [ + "Probe overrides steps to three; research loss receives total_steps=3, not the full-training horizon.", + "A clean probe does not establish full-run health or improved model quality.", + "Changed training methods can legitimately violate baseline invariants.", + "No model-variable tracker or online checker is enabled.", + ], + } + (output / "diagnostic_manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n" + ) + return report + + +class Options(argparse.Namespace): + study: Path + python: Path + candidate: Path + cpus: str + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--study", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--cpus", default="8,10,12,14") + parser.add_argument("candidate", type=Path) + args = parser.parse_args(namespace=Options()) + # Resolving a venv's Python symlink switches to the base interpreter and + # loses its installed dependencies; preserve the executable's venv path. + report = Driver(args.study.resolve(), args.python.absolute(), args.cpus).diagnose( + args.candidate + ) + print(json.dumps({"report": str(report)}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/rsi_pilot/run_candidate.py b/tools/rsi_pilot/run_candidate.py index f67b3772..a334a317 100644 --- a/tools/rsi_pilot/run_candidate.py +++ b/tools/rsi_pilot/run_candidate.py @@ -312,6 +312,41 @@ def validate_diagnostics(report: JSONObject) -> None: string(required(provenance, "sha256"), "coverage_source.sha256") +def validate_diagnostic_artifacts(report: JSONObject, output: Path) -> JSONObject: + """Link the compact report to locally readable, unchanged complete evidence.""" + coverage_path = output / "coverage.json" + manifest_path = output / "diagnostic_manifest.json" + coverage_bytes = coverage_path.read_bytes() + coverage_digest = hashlib.sha256(coverage_bytes).hexdigest() + provenance = mapping(required(report, "coverage_source"), "coverage_source") + if coverage_digest != string( + required(provenance, "sha256"), "coverage_source.sha256" + ): + raise ValueError("Downloaded coverage does not match the agent report hash") + # The original report/source paths stay unchanged; local locations are separate. + read_object(coverage_path) + diagnostic = read_object(manifest_path) + if ( + integer(required(diagnostic, "schema_version"), "diagnostic schema_version") + != 1 + ): + raise ValueError("Expected diagnostic manifest schema_version 1") + if checked_text(diagnostic, "status") != "complete": + raise ValueError("Diagnostic manifest does not report completion") + config_digest = hashlib.sha256((output / "config.toml").read_bytes()).hexdigest() + if checked_text(diagnostic, "candidate_config_sha256") != config_digest: + raise ValueError( + "Diagnostic manifest belongs to a different candidate configuration" + ) + return { + "coverage": {"path": str(coverage_path.resolve()), "sha256": coverage_digest}, + "diagnostic_manifest": { + "path": str(manifest_path.resolve()), + "sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), + }, + } + + def run(trial: Path, source_path: Path, diagnose: bool) -> JSONObject: manifest_path = trial / "trial.json" manifest = read_object(manifest_path) @@ -426,6 +461,18 @@ def run(trial: Path, source_path: Path, diagnose: bool) -> JSONObject: phase = "read_diagnostics" report = read_object(output / "agent_report.json") validate_diagnostics(report) + for name in ("coverage.json", "diagnostic_manifest.json"): + phase = "download_" + name + download( + manifest, + remote + "/diagnostics/" + name, + output / name, + output / ("download-" + name + ".log"), + ) + phase = "validate_diagnostic_artifacts" + record["traincheck_evidence"] = validate_diagnostic_artifacts( + report, output + ) record["traincheck_report"] = report record["diagnostic_status"] = "complete" else: diff --git a/tools/rsi_pilot/score_adaptive.py b/tools/rsi_pilot/score_adaptive.py new file mode 100644 index 00000000..e9b5631d --- /dev/null +++ b/tools/rsi_pilot/score_adaptive.py @@ -0,0 +1,147 @@ +"""Operator-only scoring for the reduced-compute adaptive-reasoning pilot. + +Uses the public upstream generator, model and FLOP accounting. It does not +produce an official RSI result: CPU training has no trusted-service receipt and +uses the public partition instead of the official service's private partition. +""" + +import argparse +import hashlib +import json +import math +import sys +from pathlib import Path +from typing import Literal + +import torch +from safetensors.torch import load_file + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + integer, + json_list, + json_object, + mapping, + number, + read_object, + required, + sequence, +) + + +def score( + run: Path, + output: Path, + seed: int, + per_bin: int, + split: Literal["validation", "test"] = "test", +) -> JSONObject: + # These public upstream modules are supplied by the pinned workspace on + # PYTHONPATH. Their accepted output is validated before calculating scores. + from latent_reasoning.data import evaluation_set + from latent_reasoning.evaluate import evaluate + from latent_reasoning.model import LoopedTransformer, ModelConfig + from latent_reasoning.reward import aggregate_reward + + config = read_object(run / "config.json") + model_config = mapping(required(config, "model"), "model") + expected = { + "modulus": 7, + "width": 64, + "heads": 4, + "layers": 1, + "max_loops": 8, + "max_length": 32, + } + if set(model_config) != set(expected) or any( + integer(required(model_config, key), key) != value + for key, value in expected.items() + ): + raise ValueError("Checkpoint must use the unchanged upstream backbone") + torch.set_num_threads(4) + model = LoopedTransformer(ModelConfig()) + model.load_state_dict(load_file(run / "model.safetensors"), strict=True) + model.to(torch.device("cpu")) + # Test panels are generated only after solver finalization. Validation mode + # lets the operator check scoring parity before freezing the experiment. + examples = evaluation_set(split, [1, 2, 3, 4, 5], per_bin, 7, ("+", "-"), seed) + rows_value: object = evaluate(model, examples, [1, 2, 4], torch.device("cpu")) + rows = [ + json_object(row, "depth result") + for row in sequence(rows_value, "depth results") + ] + if [integer(required(row, "loops"), "loops") for row in rows] != [1, 2, 4]: + raise ValueError("Expected exactly the frozen fixed-depth policy evaluations") + points = [rows[0], rows[1], rows[2], rows[2]] + # This JSON round-trip validates that the retained raw evidence is serializable + # with no NaN/Infinity before using the upstream's numeric range validation. + json.dumps(rows, allow_nan=False) + reward_value: object = aggregate_reward( + [ + { + "accuracy_percent": number( + required(row, "macro_accuracy_percent"), "accuracy" + ), + "mean_dense_matmul_flops": number( + required(row, "macro_mean_dense_matmul_flops"), "work" + ), + } + for row in points + ] + ) + reward = number(reward_value, "reward") + if not math.isfinite(reward): + raise ValueError("Reward must be finite") + panel = [ + { + "tokens": list(example.tokens), + "label": example.label, + "operations": example.operations, + } + for example in examples + ] + panel_bytes = json.dumps(panel, sort_keys=True, separators=(",", ":")).encode() + report: JSONObject = { + "kind": "reduced_compute_pilot_not_official_RSI_score", + "split": split, + "seed": seed, + "requested_examples_per_bin": per_bin, + "actual_examples": len(examples), + "fixed_policy_depths": [1, 2, 4, 4], + "reward": reward, + "depths": json_list(rows, "depths"), + "panel_sha256": hashlib.sha256(panel_bytes).hexdigest(), + "checkpoint_sha256": hashlib.sha256( + (run / "model.safetensors").read_bytes() + ).hexdigest(), + "config_sha256": hashlib.sha256((run / "config.json").read_bytes()).hexdigest(), + } + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + raise ValueError("Scoring output already exists") + output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--seed", required=True, type=int) + parser.add_argument("--per-bin", default=512, type=int) + parser.add_argument("--split", choices=("validation", "test"), default="test") + args = parser.parse_args() + if args.per_bin < 1: + parser.error("per-bin must be positive") + print( + json.dumps( + score(args.run, args.output, args.seed, args.per_bin, args.split), indent=2 + ) + ) + + +if __name__ == "__main__": + main() From 1ab5bf7677491d66bf6c8403b6a2d0f33c0ab5ae Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 16:29:15 +0800 Subject: [PATCH 08/13] Restore free upstream program validation before bounded RSI trials --- tests/test_rsi_pilot_client.py | 140 +++++++++++++++++++++ tools/rsi_pilot/run_candidate.py | 203 ++++++++++++++++++++++++++++++- 2 files changed, 341 insertions(+), 2 deletions(-) diff --git a/tests/test_rsi_pilot_client.py b/tests/test_rsi_pilot_client.py index 13f0cee8..87913856 100644 --- a/tests/test_rsi_pilot_client.py +++ b/tests/test_rsi_pilot_client.py @@ -13,6 +13,7 @@ from tools.fresh_validation.contracts import JSONObject, JSONValue, read_object from tools.rsi_pilot.run_candidate import ( + check_program, configure, execute, main, @@ -22,6 +23,7 @@ train_arguments, validate_diagnostic_artifacts, validate_diagnostics, + validate_preflight, validate_results, ) @@ -67,6 +69,144 @@ def training_result() -> JSONObject: class PilotClientTests(unittest.TestCase): + def test_preflight_valid_and_invalid_programs_do_not_charge_budgets(self) -> None: + cases: list[tuple[int, JSONObject, str]] = [ + (0, {"schema_version": 1, "valid": True, "error": None}, "program_valid"), + ( + 2, + { + "schema_version": 1, + "valid": False, + "error": { + "type": "ValueError", + "message": "unsupported research constant", + }, + }, + "program_invalid", + ), + ] + for arm in ("control", "traincheck"): + for code, payload, status in cases: + with ( + self.subTest(arm=arm, status=status), + tempfile.TemporaryDirectory() as temporary, + ): + trial, program = trial_fixture(Path(temporary), arm) + + def provide_result( + manifest: JSONObject, source: str, target: Path, log: Path + ) -> None: + target.write_text(json.dumps(payload)) + + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch( + "tools.rsi_pilot.run_candidate.download", + side_effect=provide_result, + ), + patch( + "tools.rsi_pilot.run_candidate.execute", return_value=code + ) as executor, + ): + result = check_program(trial, program) + self.assertFalse((trial / "usage.json").exists()) + ledger = ( + b'{"candidates_reserved": 3, "diagnostics_reserved": 1}\n' + ) + (trial / "usage.json").write_bytes(ledger) + repeated = check_program(trial, program) + self.assertEqual((trial / "usage.json").read_bytes(), ledger) + self.assertFalse((trial / "runs").exists()) + self.assertEqual(result["status"], status) + self.assertEqual(repeated["preflight"], "preflight-02") + saved = read_object( + trial / "preflights/preflight-01/preflight.json" + ) + self.assertEqual(saved, result) + self.assertEqual(saved["result"], payload) + self.assertNotIn("latent_reasoning.train", str(executor.call_args)) + + def test_preflight_backend_failure_is_not_program_rejection(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch("tools.rsi_pilot.run_candidate.execute", return_value=1), + patch("tools.rsi_pilot.run_candidate.download") as downloader, + ): + result = check_program(trial, program) + downloader.assert_not_called() + self.assertEqual(result["status"], "execution_failed") + self.assertNotIn("valid", result) + self.assertFalse((trial / "usage.json").exists()) + + def test_preflight_transfer_failure_is_saved_without_budget_charge(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch( + "tools.rsi_pilot.run_candidate.upload", + side_effect=OSError("connection lost"), + ), + ): + with self.assertRaises(OSError): + check_program(trial, program) + receipt = read_object(trial / "preflights/preflight-01/preflight.json") + self.assertEqual(receipt["status"], "client_error") + self.assertIn("connection lost", json.dumps(receipt["errors"])) + self.assertFalse((trial / "usage.json").exists()) + + def test_preflight_rejects_malformed_or_contradictory_receipts(self) -> None: + cases: list[tuple[JSONObject, int]] = [ + ({}, 0), + ({"schema_version": 1, "valid": 1, "error": None}, 0), + ({"schema_version": 1, "valid": True}, 0), + ({"schema_version": 1, "valid": True, "error": None}, 2), + ({"schema_version": 1, "valid": False, "error": None}, 2), + ( + { + "schema_version": 1, + "valid": False, + "error": {"type": "ImportError", "message": "torch missing"}, + }, + 2, + ), + ] + for payload, code in cases: + with self.subTest(payload=payload), self.assertRaises(ValueError): + validate_preflight(payload, code) + + def test_check_program_cli_is_exclusive_and_preserves_rejection_status( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + arguments = [ + "run_candidate", + "--trial", + str(root), + "--program", + str(root / "program.py"), + "--check-program", + ] + with ( + patch("sys.argv", arguments), + patch( + "tools.rsi_pilot.run_candidate.check_program", + return_value={"status": "program_invalid", "errors": []}, + ), + patch("tools.rsi_pilot.run_candidate.run") as trainer, + ): + self.assertEqual(main(), 2) + trainer.assert_not_called() + with patch("sys.argv", [*arguments, "--diagnose"]): + with self.assertRaises(SystemExit) as failure: + main() + self.assertEqual(failure.exception.code, 2) + def test_program_is_data_not_shell_code(self) -> None: source = 'def loss(logits, tokens, labels, step, total_steps):\n return "`touch sentinel` $HOME"\n' config = configure("[training]\n# PILOT_RESEARCH_SOURCE\n", source) diff --git a/tools/rsi_pilot/run_candidate.py b/tools/rsi_pilot/run_candidate.py index a334a317..f203b5c3 100644 --- a/tools/rsi_pilot/run_candidate.py +++ b/tools/rsi_pilot/run_candidate.py @@ -34,6 +34,34 @@ string, ) +# Only this fixed operator-owned program executes. The research source is loaded +# as TOML data and handed to the unchanged upstream AST interpreter. +PREFLIGHT_PROGRAM = """import json +import sys +import tomllib +from pathlib import Path +from latent_reasoning.research import TensorProgram + +with Path(sys.argv[1]).open("rb") as handle: + config = tomllib.load(handle) +training = config["training"] +if not isinstance(training, dict): + raise TypeError("training must be a table") +source = training["research_source"] +if not isinstance(source, str): + raise TypeError("research_source must be text") +result: dict[str, object] +try: + TensorProgram(source) +except ValueError as exc: + result = {"schema_version": 1, "valid": False, + "error": {"type": type(exc).__name__, "message": str(exc)}} +else: + result = {"schema_version": 1, "valid": True, "error": None} +Path(sys.argv[2]).write_text(json.dumps(result, allow_nan=False) + "\\n") +raise SystemExit(0 if result["valid"] else 2) +""" + def checked_text(record: JSONObject, key: str) -> str: value = string(required(record, key), key) @@ -180,6 +208,168 @@ def train_arguments(manifest: JSONObject, remote: str) -> list[str]: ] +def preflight_arguments(manifest: JSONObject, remote: str) -> list[str]: + """Run syntax validation under the same runtime, without invoking training.""" + return [ + "flock", + "-w", + "360", + checked_text(manifest, "remote_compute_lock"), + "timeout", + "--signal=TERM", + "--kill-after=10s", + "30s", + "taskset", + "-c", + checked_text(manifest, "affinity"), + "env", + "PYTHONDONTWRITEBYTECODE=1", + "PYTHONHASHSEED=0", + "CUDA_VISIBLE_DEVICES=", + "OMP_NUM_THREADS=4", + "MKL_NUM_THREADS=4", + "OPENBLAS_NUM_THREADS=1", + "PYTHONPATH=" + + checked_text(manifest, "remote_overlay") + + ":" + + checked_text(manifest, "remote_workspace"), + checked_text(manifest, "remote_python"), + "-c", + PREFLIGHT_PROGRAM, + remote + "/config.toml", + remote + "/result.json", + ] + + +def validate_preflight(result: JSONObject, code: int) -> bool: + """Reject malformed or contradictory backend receipts, including exit status.""" + if integer(required(result, "schema_version"), "schema_version") != 1: + raise ValueError("Expected preflight schema_version 1") + valid = boolean(required(result, "valid"), "valid") + error = required(result, "error") + if valid: + if code != 0 or error is not None: + raise ValueError("Valid preflight must have exit code 0 and no error") + else: + detail = mapping(error, "error") + if code != 2 or string(required(detail, "type"), "error.type") != "ValueError": + raise ValueError("Invalid program must have exit code 2 and ValueError") + if not string(required(detail, "message"), "error.message"): + raise ValueError("Program rejection must include an error message") + return valid + + +def check_program(trial: Path, source_path: Path) -> JSONObject: + """Provide the upstream free syntax check without reserving any train budget. + + Preflights use the same exclusive trial lock as training. Their immutable + receipts live separately; neither successful nor failed syntax checks create + or modify usage.json, candidate directories, or diagnostic reservations. + """ + manifest_path = trial / "trial.json" + manifest = read_object(manifest_path) + source = program_text(source_path, Path(checked_text(manifest, "workspace"))) + arm = checked_text(manifest, "arm") + if arm not in ("control", "traincheck"): + raise ValueError("Unknown trial arm") + template_bytes = Path(checked_text(manifest, "profile")).read_bytes() + profile_digest = hashlib.sha256(template_bytes).hexdigest() + if profile_digest != checked_text(manifest, "profile_sha256"): + raise ValueError("Frozen training profile changed") + with (trial / "run.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + preflights = trial / "preflights" + preflights.mkdir(exist_ok=True) + attempt = 1 + while (preflights / f"preflight-{attempt:02d}").exists(): + attempt += 1 + name = f"preflight-{attempt:02d}" + output = preflights / name + output.mkdir() + remote = checked_text(manifest, "remote_trial") + "/preflights/" + name + errors: list[JSONValue] = [] + record: JSONObject = { + "schema_version": 1, + "preflight": name, + "arm": arm, + "status": "started", + "program_sha256": hashlib.sha256(source.encode()).hexdigest(), + "profile_sha256": profile_digest, + "trial_manifest_sha256": hashlib.sha256( + manifest_path.read_bytes() + ).hexdigest(), + "remote_directory": remote, + "errors": errors, + } + write_record(output / "preflight.json", record) + started = time.monotonic() + phase = "prepare_local" + try: + (output / "program.py").write_text(source) + (output / "config.toml").write_text( + configure(template_bytes.decode(), source) + ) + phase = "prepare_remote" + checked_execute( + remote_command(manifest, ["mkdir", "-p", remote]), output / "setup.log" + ) + phase = "upload_config" + upload( + manifest, + output / "config.toml", + remote + "/config.toml", + output / "upload.log", + ) + phase = "check_program" + code = execute( + remote_command(manifest, preflight_arguments(manifest, remote)), + output / "check.log", + 430, + ) + record["check_exit_code"] = code + if code not in (0, 2): + record["status"] = "execution_failed" + errors.append( + { + "phase": phase, + "type": "ProcessExit", + "message": f"Syntax check exited {code}; see check.log", + } + ) + else: + phase = "download_result" + download( + manifest, + remote + "/result.json", + output / "result.json", + output / "download.log", + ) + phase = "validate_result" + result = read_object(output / "result.json") + valid = validate_preflight(result, code) + record["valid"] = valid + record["result"] = result + record["status"] = "program_valid" if valid else "program_invalid" + if not valid: + errors.append({"phase": phase, "error": result["error"]}) + return record + except BaseException as exc: + record["status"] = "client_error" + errors.append( + { + "phase": phase, + "type": type(exc).__name__, + "message": str(exc) or repr(exc), + } + ) + raise + finally: + record["operational_seconds_including_transfer"] = ( + time.monotonic() - started + ) + write_record(output / "preflight.json", record) + + def write_record(path: Path, record: JSONObject) -> None: """Replace one receipt atomically; callers hold this trial's exclusive lock.""" temporary = path.with_name(path.name + ".tmp") @@ -506,10 +696,19 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--trial", type=Path, required=True) parser.add_argument("--program", type=Path, required=True) - parser.add_argument("--diagnose", action="store_true") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--diagnose", action="store_true") + mode.add_argument("--check-program", action="store_true") args = parser.parse_args() - result = run(args.trial.resolve(strict=True), args.program, args.diagnose) + trial = args.trial.resolve(strict=True) + result = ( + check_program(trial, args.program) + if args.check_program + else run(trial, args.program, args.diagnose) + ) print(json.dumps(result, indent=2, allow_nan=False)) + if args.check_program and result["status"] == "program_invalid": + return 2 return int(bool(sequence(required(result, "errors"), "errors"))) From 759ebe81085c81807e73c5c33b92ebadf6720e68 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 16:34:25 +0800 Subject: [PATCH 09/13] Freeze paired pilot selections and audit held-out evidence and costs --- docs/measurements/rsi-pilot-20261030.md | 37 +- tests/test_rsi_pilot_analysis.py | 453 +++++++++++++++ tools/rsi_pilot/analyze_trials.py | 726 ++++++++++++++++++++++++ 3 files changed, 1211 insertions(+), 5 deletions(-) create mode 100644 tests/test_rsi_pilot_analysis.py create mode 100644 tools/rsi_pilot/analyze_trials.py diff --git a/docs/measurements/rsi-pilot-20261030.md b/docs/measurements/rsi-pilot-20261030.md index e4e59725..94bd7924 100644 --- a/docs/measurements/rsi-pilot-20261030.md +++ b/docs/measurements/rsi-pilot-20261030.md @@ -1,6 +1,7 @@ # TrainCheck agent pilot: adaptive latent reasoning -Status: protocol frozen before solver trials; results pending. +Status: revised comparison protocol recorded before held-out scoring; results +pending. The decision is whether TrainCheck gives research agents useful diagnostic evidence that justifies a larger evaluation. This is a reduced-compute development @@ -43,7 +44,7 @@ macro averages weight it equally with the four larger strata. ## Comparison protocol -- Three matched pairs of fresh agents, trial order C1/T1, T2/C2, C3/T3. Each +- Two matched pairs of fresh agents, trial order B1C/B1T, B2T/B2C. Each receives the same public source interface, baseline program, baseline development feedback, frozen training profile and optimization objective. - All solver agents start without the parent conversation. They use the same @@ -52,7 +53,11 @@ macro averages weight it equally with the four larger strata. result is provided. The operator inspected task metadata during selection; that context is deliberately excluded from solver prompts. - Each agent may submit at most two candidate programs for 10,000-update runs. - Failed requests consume this allowance. Treatment agents additionally receive + Failed requests consume this allowance. Both arms have free program validation + through the upstream tensor-program interpreter, matching the task's documented + `--check-program` interface. Validation does not train or consume candidate + allowance; its receipts, failures and duration are reported separately. + Treatment agents additionally receive the TrainCheck skill and may request one short diagnostic rerun for a candidate. This additional computation is measured and included in cost reporting. Both arms may select the supplied baseline instead of a candidate. @@ -69,7 +74,29 @@ macro averages weight it equally with the four larger strata. 512 distinct examples per operation stratum. No candidate is tuned on these results. Final scoring uses upstream accuracy/FLOP accounting and reward formula for the fixed policy; it is still a pilot metric, not an official - score or trusted-service receipt. + score or trusted-service receipt. Each unique checkpoint/configuration pair + is scored once. Identical selections reuse that immutable report, recorded + in `scores/reuse.json`; these reused durations are not independent timing + measurements. + +### Setup revision before held-out evaluation + +The initial C1/T1/T2 agents encountered source-parser rejections, including +docstrings exceeding the interpreter's string limit. The pilot client had omitted +the free program validator documented in the public task. All three selected +the baseline. These runs demonstrate a harness integration gap and are retained +as setup evidence, with their failed requests and costs; they are excluded from +the paired outcome comparison. The initially planned C2/C3/T3 agents were never +launched. + +The revised comparison restores that validator and supplies the public +interpreter source equally to all four fresh agents. The model, data, training +budget, development panel, held-out panel and treatment remain unchanged. +`study_protocol.json` records the reason, trial order and two pairs before any +held-out examples or scores are generated. The original six-trial +`heldout-evaluation-plan.json` is preserved as provenance; the revision changes +its completion gate to all four B submissions. This setup revision does not +establish an effect of TrainCheck on research performance. ## TrainCheck treatment and interpretation @@ -104,7 +131,7 @@ all attempted candidates, measured training/diagnostic/setup cost, actual tool use and whether any diagnostic changed an agent decision. A useful mechanistic signal is an agent using relevant evidence to correct an unintended training behavior, followed by independent validation. Fewer alarms alone do not count -as improvement. A higher score in one of three pairs does not establish a +as improvement. A higher score in one of two pairs does not establish a general effect or statistical significance. If diagnostics are unused, irrelevant, dominated by healthy/reference-mismatch diff --git a/tests/test_rsi_pilot_analysis.py b/tests/test_rsi_pilot_analysis.py new file mode 100644 index 00000000..3100c877 --- /dev/null +++ b/tests/test_rsi_pilot_analysis.py @@ -0,0 +1,453 @@ +"""Evidence failures must prevent an apparently successful pilot comparison.""" + +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from tools.fresh_validation.contracts import JSONObject, JSONValue, mapping, read_object +from tools.rsi_pilot.analyze_trials import ( + analyze, + begin_scoring, + collect, + digest, + freeze, + metric, +) +from tools.rsi_pilot.run_candidate import configure + +ORDER = ("B1C", "B1T", "B2T", "B2C") + + +def write(path: Path, value: JSONObject) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, allow_nan=False) + "\n") + + +def depths(accuracy: float, include_eight: bool = True) -> list[JSONValue]: + return [ + { + "loops": depth, + "macro_accuracy_percent": accuracy, + "macro_mean_dense_matmul_flops": work, + } + for depth, work in ((1, 805760), (2, 1610624), (4, 3220352), (8, 6439808)) + if depth != 8 or include_eight + ] + + +class TestPilotAnalysis(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory[str]() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.profile = self.root / "frozen-profile.toml" + self.profile.write_text("[training]\nsteps = 10000\n# PILOT_RESEARCH_SOURCE\n") + self.baseline = "def loss():\n return 1\n" + self.baseline_config: JSONObject = { + "model": {"width": 64}, + "training": {"steps": 10000, "research_source": self.baseline}, + } + write(self.root / "selected/baseline/config.json", self.baseline_config) + (self.root / "selected/baseline/model.safetensors").write_bytes( + b"baseline model" + ) + write( + self.root / "study_protocol.json", + { + "revised_at": "2026-10-30T00:00:00+00:00", + "revision_reason": "Restore omitted free validation before the scored pilot.", + "trial_order": list(ORDER), + "pairs": [ + {"control": "B1C", "traincheck": "B1T"}, + {"control": "B2C", "traincheck": "B2T"}, + ], + "excluded_setup_trials": ["C1"], + }, + ) + write( + self.root / "heldout-evaluation-plan.json", + {"seed": 20261030, "requested_examples_per_bin": 512}, + ) + for trial_id in (*ORDER, "C1"): + self.make_trial(trial_id) + source = ( + Path(__file__).resolve().parents[1] / "tools/rsi_pilot/run_candidate.py" + ) + write( + self.root / "harness-freeze-B.json", + {"files": {"tools/rsi_pilot/run_candidate.py": digest(source)}}, + ) + inventory: JSONObject = { + trial_id: { + "baseline_research.py": digest( + self.root / "trials" / trial_id / "workspace/baseline_research.py" + ) + } + for trial_id in ORDER + } + write( + self.root / "solver-packet-manifest-B.json", {"packet_inventory": inventory} + ) + + def make_trial(self, trial_id: str) -> None: + trial = self.root / "trials" / trial_id + workspace = trial / "workspace" + workspace.mkdir(parents=True) + arm = "traincheck" if trial_id.endswith("T") else "control" + write( + trial / "trial.json", + { + "workspace": str(workspace), + "arm": arm, + "max_candidates": 2, + "max_diagnostics": int(arm == "traincheck"), + "expected_steps": 10000, + "profile": str(self.profile), + "profile_sha256": digest(self.profile), + }, + ) + (workspace / "baseline_research.py").write_text(self.baseline) + (workspace / "assignment.txt").write_text( + "Improve the objective within this trial's budget." + ) + (workspace / "summary.md").write_text( + "Selected the baseline after inspecting development results." + ) + write(workspace / "baseline_development.json", {"final": depths(20.0)}) + write(workspace / "submission.json", {"candidate": "baseline"}) + write(self.root / "selected" / trial_id / "config.json", self.baseline_config) + (self.root / "selected" / trial_id / "model.safetensors").write_bytes( + b"baseline model" + ) + + def add_run( + self, + trial_id: str, + *, + successful: bool, + diagnose: bool = False, + crlf: bool = False, + ) -> Path: + trial = self.root / "trials" / trial_id + manifest = read_object(trial / "trial.json") + run = trial / "runs/candidate-01" + run.mkdir(parents=True) + program = "def loss():\n return 2\n" + if crlf: + program = program.replace("\n", "\r\n") + (run / "program.py").write_bytes(program.encode()) + (run / "config.toml").write_text(configure(self.profile.read_text(), program)) + receipt: JSONObject = { + "candidate": "candidate-01", + "arm": manifest["arm"], + "diagnostics_requested": diagnose, + "program_sha256": digest(run / "program.py"), + "profile_sha256": digest(self.profile), + "trial_manifest_sha256": digest(trial / "trial.json"), + "candidates_reserved": 1, + "diagnostics_reserved": int(diagnose), + "status": "training_complete" if successful else "training_failed", + "training_exit_code": 0 if successful else 1, + "errors": ( + [] + if successful + else [ + { + "phase": "training", + "type": "ProcessExit", + "message": "Training failed", + } + ] + ), + "operational_seconds_including_transfer_and_diagnostics": 8.0, + } + if successful: + rows = depths(40.0) + receipt.update( + { + "completed_steps": 10000, + "training_seconds": 5.0, + "development_depths": rows, + } + ) + write( + run / "results.json", + { + "completed_steps": 10000, + "training_seconds": 5.0, + "device_backend": "cpu", + "final": rows, + }, + ) + write(trial / "workspace/submission.json", {"candidate": "candidate-01"}) + write( + self.root / "selected" / trial_id / "config.json", + { + "model": {"width": 64}, + "training": {"steps": 10000, "research_source": program}, + }, + ) + (self.root / "selected" / trial_id / "model.safetensors").write_bytes( + b"candidate model" + ) + if diagnose: + receipt.update( + { + "diagnostic_status": "execution_failed", + "diagnostic_exit_code": 1, + "errors": [ + { + "phase": "diagnostics", + "type": "ProcessExit", + "message": "Diagnostic failed", + } + ], + } + ) + write(run / "run.json", receipt) + write( + trial / "usage.json", + {"candidates_reserved": 1, "diagnostics_reserved": int(diagnose)}, + ) + return run + + def scores(self) -> None: + freeze(self.root) + begin_scoring(self.root) + for trial_id in ("baseline", *ORDER): + raw = depths(30.0, include_eight=False) + values = metric(raw, [1, 2, 4]) + selected = self.root / "selected" / trial_id + write( + self.root / "scores" / (trial_id + ".json"), + { + "kind": "reduced_compute_pilot_not_official_RSI_score", + "split": "test", + "fixed_policy_depths": [1, 2, 4, 4], + "seed": 20261030, + "requested_examples_per_bin": 512, + "actual_examples": 2048, + "checkpoint_sha256": digest(selected / "model.safetensors"), + "config_sha256": digest(selected / "config.json"), + "panel_sha256": hashlib.sha256(b"shared heldout panel").hexdigest(), + "depths": raw, + "reward": values["reward"], + }, + ) + + def test_missing_trial_or_submission_fails(self) -> None: + path = self.root / "trials/B2C/workspace/submission.json" + path.unlink() + with self.assertRaises(FileNotFoundError): + freeze(self.root) + + def test_untrained_selected_candidate_fails(self) -> None: + write( + self.root / "trials/B1C/workspace/submission.json", + {"candidate": "candidate-02"}, + ) + with self.assertRaisesRegex(ValueError, "no validated completed training"): + freeze(self.root) + + def test_unknown_failed_training_time_is_separate(self) -> None: + self.add_run("C1", successful=False) + self.scores() + result = analyze(self.root) + setup = result["excluded_setup_trials"] + self.assertIsInstance(setup, list) + assert isinstance(setup, list) + first = mapping(setup[0], "setup") + self.assertEqual(first["training_seconds_known"], 0.0) + self.assertEqual(first["training_duration_unknown_attempts"], ["candidate-01"]) + self.assertEqual( + first["operational_seconds_including_queue_transfer_diagnostics"], 8.0 + ) + + def test_diagnostic_failure_does_not_discard_trained_selection(self) -> None: + self.add_run("B1T", successful=True, diagnose=True, crlf=True) + self.scores() + reports = analyze(self.root)["trials"] + assert isinstance(reports, list) + treatment = mapping(reports[1], "treatment") + self.assertEqual(treatment["selection"], "candidate-01") + self.assertEqual(treatment["diagnostic_execution_failures"], 1) + self.assertEqual(treatment["diagnostic_skip_counts_unknown"], 1) + self.assertEqual(treatment["training_seconds_known"], 5.0) + + def test_bad_ledger_is_not_repaired(self) -> None: + self.add_run("B1C", successful=False) + write( + self.root / "trials/B1C/usage.json", + {"candidates_reserved": True, "diagnostics_reserved": 0}, + ) + with self.assertRaisesRegex(ValueError, "must be an integer"): + freeze(self.root) + + def test_changed_program_or_manifest_hash_fails(self) -> None: + run = self.add_run("B1C", successful=True) + (run / "program.py").write_text("changed source") + with self.assertRaisesRegex(ValueError, "program_sha256 mismatch"): + freeze(self.root) + + def test_unfinished_receipt_fails(self) -> None: + run = self.add_run("B1C", successful=False) + receipt = read_object(run / "run.json") + receipt["status"] = "started" + write(run / "run.json", receipt) + with self.assertRaisesRegex(ValueError, "has not finished"): + freeze(self.root) + + def test_missing_operational_time_is_not_zero(self) -> None: + run = self.add_run("B1C", successful=False) + receipt = read_object(run / "run.json") + del receipt["operational_seconds_including_transfer_and_diagnostics"] + write(run / "run.json", receipt) + with self.assertRaisesRegex(ValueError, "Missing required field"): + freeze(self.root) + + def test_selected_checkpoint_mismatch_fails(self) -> None: + self.scores() + score_path = self.root / "scores/B1C.json" + score = read_object(score_path) + score["checkpoint_sha256"] = "0" * 64 + write(score_path, score) + with self.assertRaisesRegex(ValueError, "checkpoint_sha256 mismatch"): + analyze(self.root) + + def test_false_or_fabricated_reward_fails(self) -> None: + self.scores() + path = self.root / "scores/B1T.json" + score = read_object(path) + for reward in (False, 100.0): + score["reward"] = reward + write(path, score) + with self.assertRaises(ValueError): + analyze(self.root) + + def test_altered_frozen_assignment_fails(self) -> None: + self.scores() + (self.root / "trials/B1T/workspace/assignment.txt").write_text( + "Changed after selection" + ) + with self.assertRaisesRegex(ValueError, "frozen file hash mismatch"): + analyze(self.root) + + def test_scoring_before_finalization_is_rejected(self) -> None: + self.scores() + path = self.root / "scores/provenance.json" + provenance = read_object(path) + provenance["scoring_started_at"] = "2026-01-01T00:00:00+00:00" + write(path, provenance) + with self.assertRaisesRegex(ValueError, "must follow finalization"): + analyze(self.root) + + def test_duplicate_pair_or_setup_overlap_is_rejected(self) -> None: + path = self.root / "study_protocol.json" + record = read_object(path) + record["excluded_setup_trials"] = ["B1C"] + write(path, record) + with self.assertRaisesRegex(ValueError, "overlapping trial IDs"): + collect(self.root) + + def test_scores_cannot_exist_when_freezing(self) -> None: + write(self.root / "scores/B1C.json", {}) + with self.assertRaisesRegex(ValueError, "already exist"): + freeze(self.root) + + def test_preflight_cost_is_separate_and_does_not_create_training_budget( + self, + ) -> None: + trial = self.root / "trials/B1C" + path = trial / "preflights/preflight-01" + path.mkdir(parents=True) + (path / "program.py").write_text(self.baseline) + (path / "config.toml").write_text( + configure(self.profile.read_text(), self.baseline) + ) + backend: JSONObject = { + "schema_version": 1, + "valid": False, + "error": {"type": "ValueError", "message": "Unsupported syntax"}, + } + write(path / "result.json", backend) + write( + path / "preflight.json", + { + "schema_version": 1, + "preflight": "preflight-01", + "arm": "control", + "profile_sha256": digest(self.profile), + "trial_manifest_sha256": digest(trial / "trial.json"), + "program_sha256": digest(path / "program.py"), + "status": "program_invalid", + "errors": [{"phase": "validate_result", "error": backend["error"]}], + "check_exit_code": 2, + "valid": False, + "result": backend, + "operational_seconds_including_transfer": 1.5, + }, + ) + self.scores() + reports = analyze(self.root)["trials"] + assert isinstance(reports, list) + report = mapping(reports[0], "trial") + self.assertEqual(report["attempts"], []) + self.assertEqual(report["operational_seconds_including_preflights"], 1.5) + self.assertFalse((trial / "usage.json").exists()) + (path / "program.py").write_text("changed after finalization") + with self.assertRaisesRegex(ValueError, "program_sha256 mismatch"): + analyze(self.root) + + def test_incomplete_preflight_is_not_a_success(self) -> None: + path = self.root / "trials/B1C/preflights/preflight-01" + write(path / "preflight.json", {"status": "started"}) + with self.assertRaises(ValueError): + freeze(self.root) + + def test_baseline_selection_must_match_reference_score(self) -> None: + self.scores() + path = self.root / "scores/B1C.json" + score = read_object(path) + score["depths"] = depths(80.0, include_eight=False) + score["reward"] = metric(score["depths"], [1, 2, 4])["reward"] + write(path, score) + with self.assertRaisesRegex(ValueError, "baseline selection score mismatch"): + analyze(self.root) + + def test_score_panel_size_must_match(self) -> None: + self.scores() + path = self.root / "scores/B1C.json" + score = read_object(path) + score["actual_examples"] = 100 + write(path, score) + with self.assertRaisesRegex(ValueError, "test panel size mismatch"): + analyze(self.root) + + def test_harness_inventory_mismatch_fails(self) -> None: + write( + self.root / "harness-freeze-B.json", + {"files": {"tools/rsi_pilot/run_candidate.py": "0" * 64}}, + ) + with self.assertRaisesRegex(ValueError, "inventory hash mismatch"): + freeze(self.root) + + def test_complete_training_without_results_is_not_success(self) -> None: + run = self.add_run("B1C", successful=True) + (run / "results.json").unlink() + with self.assertRaisesRegex(ValueError, "no validated results"): + freeze(self.root) + + def test_missing_requested_diagnostic_is_not_empty_success(self) -> None: + run = self.add_run("B1T", successful=True, diagnose=True) + receipt = read_object(run / "run.json") + del receipt["diagnostic_status"] + del receipt["diagnostic_exit_code"] + write(run / "run.json", receipt) + with self.assertRaisesRegex(ValueError, "missing its requested diagnostic"): + freeze(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/rsi_pilot/analyze_trials.py b/tools/rsi_pilot/analyze_trials.py new file mode 100644 index 00000000..686a25c1 --- /dev/null +++ b/tools/rsi_pilot/analyze_trials.py @@ -0,0 +1,726 @@ +"""Freeze completed RSI pilot selections, then audit matched held-out results. + +Timestamps come from explicit operator actions, never file mtimes. Hashes bind +the saved evidence; they do not establish a security boundary against its owner. +The operator scores each unique checkpoint/configuration once and reuses that +immutable report for identical selections. Exact baseline-score equality checks +that reuse contract; it is not a claim of identical repeated execution timings. +""" + +import argparse +import hashlib +import json +import math +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.fresh_validation.contracts import JSONObject # noqa: E402 +from tools.fresh_validation.contracts import ( + JSONValue, + boolean, + integer, + json_object, + mapping, + number, + read_object, + required, + sequence, + string, +) +from tools.rsi_pilot.run_candidate import configure # noqa: E402 +from tools.rsi_pilot.run_candidate import ( + validate_diagnostic_artifacts, + validate_diagnostics, + validate_preflight, + validate_results, +) + +POLICY: list[JSONValue] = [1, 2, 4, 4] + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def text(path: Path) -> str: + return path.read_bytes().decode("utf-8") + + +def finite(value: object, label: str) -> float: + result = float(number(value, label)) + if not math.isfinite(result) or result < 0: + raise ValueError(f"{label} must be finite and nonnegative") + return result + + +def require_equal(actual: object, expected: object, label: str) -> None: + if type(actual) is not type(expected) or actual != expected: + raise ValueError(f"{label} mismatch") + if isinstance(actual, dict) and isinstance(expected, dict): + left_mapping, right_mapping = mapping(actual, label), mapping(expected, label) + for key in left_mapping: + require_equal(left_mapping[key], right_mapping[key], label) + if isinstance(actual, list) and isinstance(expected, list): + for left, right in zip(sequence(actual, label), sequence(expected, label)): + require_equal(left, right, label) + + +def protocol(root: Path) -> tuple[JSONObject, list[str], dict[str, str], list[str]]: + record = read_object(root / "study_protocol.json") + timestamp(required(record, "revised_at")) + if not string(required(record, "revision_reason"), "revision_reason"): + raise ValueError("Protocol revision must explain the excluded setup trials") + order = [ + string(value, "trial ID") + for value in sequence(required(record, "trial_order"), "trial_order") + ] + excluded = [ + string(value, "setup trial ID") + for value in sequence( + required(record, "excluded_setup_trials"), "excluded_setup_trials" + ) + ] + if any(re.fullmatch(r"[A-Za-z0-9_-]+", name) is None for name in order + excluded): + raise ValueError("Invalid trial directory ID") + if len(set(order + excluded)) != len(order + excluded): + raise ValueError("Duplicate or overlapping trial IDs") + arms: dict[str, str] = {} + pairs = sequence(required(record, "pairs"), "pairs") + if len(pairs) != 2: + raise ValueError("This pilot requires exactly two matched pairs") + for item in pairs: + pair = mapping(item, "pair") + for arm in ("control", "traincheck"): + name = string(required(pair, arm), arm) + if name in arms: + raise ValueError("Trial repeated across pairs") + arms[name] = arm + require_equal(set(order), set(arms), "paired trial set") + return record, order, arms, excluded + + +def metric(value: object, expected_depths: list[int]) -> JSONObject: + rows = [mapping(row, "depth") for row in sequence(value, "depths")] + require_equal( + [integer(required(r, "loops"), "loops") for r in rows], + expected_depths, + "depth order", + ) + accuracies = [ + finite(required(r, "macro_accuracy_percent"), "accuracy") for r in rows + ] + work = [finite(required(r, "macro_mean_dense_matmul_flops"), "FLOPs") for r in rows] + if any(a > 100 for a in accuracies) or any(w <= 0 for w in work): + raise ValueError("Invalid accuracy or work") + selected = [0, 1, 2, 2] + ratios = [ + work[i] / ceiling + for i, ceiling in zip(selected, [805760, 1610624, 3220352, 6439808]) + ] + if any(ratio > 1 for ratio in ratios): + raise ValueError("Fixed policy exceeds its work ceiling") + accuracy = sum(accuracies[i] for i in selected) / 4 + return { + "fixed_policy_depths": list(POLICY), + "accuracy_percent": accuracy, + # Preserve the upstream aggregate_reward evaluation order as well as + # its mathematical formula; exact receipt validation must not round. + "reward": sum( + 0.25 * (0.8 * accuracies[i] + 0.2 * (100 * (1 - ratio))) + for i, ratio in zip(selected, ratios) + ), + } + + +def preflight_summary( + trial: Path, manifest: JSONObject, files: set[Path] +) -> JSONObject: + """Free program checks have their own receipts and costs, never train budget.""" + attempts: list[JSONValue] = [] + seconds = 0.0 + paths = sorted((trial / "preflights").glob("preflight-*")) + if [path.name for path in paths] != [ + f"preflight-{i:02d}" for i in range(1, len(paths) + 1) + ]: + raise ValueError("Malformed preflight attempt sequence") + profile = Path(string(required(manifest, "profile"), "profile")) + for path in paths: + receipt = read_object(path / "preflight.json") + status = string(required(receipt, "status"), "preflight status") + if status not in ( + "program_valid", + "program_invalid", + "execution_failed", + "client_error", + ): + raise ValueError("Preflight has not finished") + for key, expected in ( + ("schema_version", 1), + ("preflight", path.name), + ("arm", manifest["arm"]), + ("profile_sha256", digest(profile)), + ("trial_manifest_sha256", digest(trial / "trial.json")), + ("program_sha256", digest(path / "program.py")), + ): + require_equal(required(receipt, key), expected, key) + require_equal( + text(path / "config.toml"), + configure(text(profile), text(path / "program.py")), + "preflight config", + ) + errors = sequence(required(receipt, "errors"), "preflight errors") + if (status == "program_valid") != (len(errors) == 0): + raise ValueError("Preflight status/error mismatch") + for error in errors: + detail = mapping(error, "preflight error") + if not string(required(detail, "phase"), "preflight error phase"): + raise ValueError("Empty preflight error phase") + # Program rejections nest the original interpreter error; process + # and client failures store the exception directly in the receipt. + cause = ( + mapping(detail["error"], "interpreter error") + if "error" in detail + else detail + ) + for key in ("type", "message"): + if not string(required(cause, key), key): + raise ValueError("Empty preflight error evidence") + if status in ("program_valid", "program_invalid"): + result = read_object(path / "result.json") + valid = validate_preflight( + result, integer(required(receipt, "check_exit_code"), "check_exit_code") + ) + require_equal(required(receipt, "valid"), valid, "preflight valid") + require_equal( + required(receipt, "result"), result, "preflight backend result" + ) + require_equal(valid, status == "program_valid", "preflight status") + elif status == "execution_failed" and integer( + required(receipt, "check_exit_code"), "check_exit_code" + ) in (0, 2): + raise ValueError( + "Preflight execution failure has a normal interpreter exit" + ) + duration = finite( + required(receipt, "operational_seconds_including_transfer"), + "preflight duration", + ) + seconds += duration + files.update(item for item in path.iterdir() if item.is_file()) + attempts.append( + { + "preflight": path.name, + "status": status, + "operational_seconds": duration, + "errors": receipt["errors"], + } + ) + return { + "attempts": attempts, + "count": len(paths), + "operational_seconds_including_queue_transfer": seconds, + } + + +def trial_summary( + root: Path, trial_id: str, files: set[Path], expected_arm: str | None +) -> JSONObject: + trial = root / "trials" / trial_id + manifest_path = trial / "trial.json" + manifest = read_object(manifest_path) + workspace = trial / "workspace" + require_equal( + Path(string(required(manifest, "workspace"), "workspace")).resolve(), + workspace.resolve(), + "workspace", + ) + arm = string(required(manifest, "arm"), "arm") + if arm not in ("control", "traincheck"): + raise ValueError("Unknown arm") + if expected_arm is not None: + require_equal(arm, expected_arm, "arm") + for key, expected_limit in ( + ("max_candidates", 2), + ("max_diagnostics", int(arm == "traincheck")), + ("expected_steps", 10000), + ): + require_equal(integer(required(manifest, key), key), expected_limit, key) + profile = root / "frozen-profile.toml" + require_equal( + Path(string(required(manifest, "profile"), "profile")).resolve(), + profile.resolve(), + "profile path", + ) + require_equal(required(manifest, "profile_sha256"), digest(profile), "profile hash") + files.update( + { + manifest_path, + profile, + workspace / "assignment.txt", + workspace / "baseline_research.py", + workspace / "baseline_development.json", + workspace / "summary.md", + workspace / "submission.json", + } + ) + preflights = preflight_summary(trial, manifest, files) + paths = sorted((trial / "runs").glob("candidate-*")) + if len(paths) > 2 or [p.name for p in paths] != [ + f"candidate-{i:02d}" for i in range(1, len(paths) + 1) + ]: + raise ValueError("Malformed or excessive candidate attempts") + attempts: list[JSONValue] = [] + valid: dict[str, JSONObject] = {} + training_known = operational = 0.0 + unknown_training: list[JSONValue] = [] + diagnostic_count = diagnostic_complete = diagnostic_failed = diagnostic_skips = ( + error_count + ) = 0 + for index, path in enumerate(paths, 1): + receipt_path = path / "run.json" + receipt = read_object(receipt_path) + requested = boolean( + required(receipt, "diagnostics_requested"), "diagnostics_requested" + ) + diagnostic_count += requested + for key, expected_text in ( + ("candidate", path.name), + ("arm", arm), + ("profile_sha256", digest(profile)), + ("trial_manifest_sha256", digest(manifest_path)), + ("program_sha256", digest(path / "program.py")), + ): + require_equal(required(receipt, key), expected_text, key) + for key, expected_count in ( + ("candidates_reserved", index), + ("diagnostics_reserved", diagnostic_count), + ): + require_equal(integer(required(receipt, key), key), expected_count, key) + require_equal( + text(path / "config.toml"), + configure(text(profile), text(path / "program.py")), + "candidate config", + ) + status = string(required(receipt, "status"), "status") + if status not in ("training_complete", "training_failed", "client_error"): + raise ValueError("Candidate has not finished") + errors = sequence(required(receipt, "errors"), "errors") + error_count += len(errors) + for error in errors: + for key in ("phase", "type", "message"): + if not string(required(mapping(error, "error"), key), key): + raise ValueError("Empty error evidence") + if status != "training_complete" and not errors: + raise ValueError("Failed candidate has no error evidence") + duration = finite( + required(receipt, "operational_seconds_including_transfer_and_diagnostics"), + "operational duration", + ) + operational += duration + files.update({receipt_path, path / "program.py", path / "config.toml"}) + files.update(path.glob("*.log")) + elapsed: float | None = None + if "training_exit_code" in receipt: + code = integer(receipt["training_exit_code"], "training_exit_code") + if status == "training_failed" and code == 0: + raise ValueError("Training failure has a successful process exit") + if code == 0 and (path / "results.json").exists(): + results = read_object(path / "results.json") + _, seconds, _ = validate_results(results, manifest) + for key, expected_result in ( + ("training_seconds", seconds), + ("completed_steps", 10000), + ("development_depths", results["final"]), + ): + require_equal(required(receipt, key), expected_result, key) + elapsed = finite(seconds, "training duration") + valid[path.name] = metric(results["final"], [1, 2, 4, 8]) + files.add(path / "results.json") + elif status != "client_error": + raise ValueError("Finished training receipt has no process exit") + if status == "training_complete" and elapsed is None: + raise ValueError("Complete training has no validated results") + if elapsed is None: + unknown_training.append(path.name) + else: + training_known += elapsed + diagnostic: JSONObject = {"requested": requested, "status": "not_requested"} + if requested: + diagnostic["status"] = "unknown_after_client_error" + if "diagnostic_status" in receipt: + diagnostic_status = string( + receipt["diagnostic_status"], "diagnostic_status" + ) + if diagnostic_status not in ("complete", "execution_failed"): + raise ValueError("Unknown diagnostic status") + diagnostic["status"] = diagnostic_status + exit_code = integer( + required(receipt, "diagnostic_exit_code"), "diagnostic_exit_code" + ) + if (diagnostic_status == "complete") != (exit_code == 0): + raise ValueError("Diagnostic exit/status mismatch") + if diagnostic_status == "complete": + diagnostic_report = read_object(path / "agent_report.json") + validate_diagnostics(diagnostic_report) + validate_diagnostic_artifacts(diagnostic_report, path) + require_equal( + required(receipt, "traincheck_report"), + diagnostic_report, + "diagnostic report", + ) + diagnostic["report"] = diagnostic_report + diagnostic_complete += 1 + diagnostic_skips += integer( + required( + mapping(diagnostic_report["counts"], "counts"), + "skipped_variable_checks", + ), + "skipped_variable_checks", + ) + files.update( + { + path / name + for name in ( + "agent_report.json", + "coverage.json", + "diagnostic_manifest.json", + ) + } + ) + else: + diagnostic_failed += 1 + elif status != "client_error": + raise ValueError( + "Finished candidate is missing its requested diagnostic" + ) + elif any( + key in receipt + for key in ( + "traincheck_report", + "diagnostic_status", + "diagnostic_exit_code", + ) + ): + raise ValueError("Unrequested diagnostic evidence") + attempts.append( + { + "candidate": path.name, + "status": status, + "training_seconds": elapsed, + "operational_seconds": duration, + "diagnostic": diagnostic, + "errors": receipt["errors"], + } + ) + if diagnostic_count > int(arm == "traincheck"): + raise ValueError("Diagnostic budget exceeded") + ledger_path = trial / "usage.json" + if paths or ledger_path.exists(): + ledger = read_object(ledger_path) + require_equal( + integer(required(ledger, "candidates_reserved"), "candidates_reserved"), + len(paths), + "ledger candidates", + ) + require_equal( + integer(required(ledger, "diagnostics_reserved"), "diagnostics_reserved"), + diagnostic_count, + "ledger diagnostics", + ) + files.add(ledger_path) + report: JSONObject = { + "trial": trial_id, + "arm": arm, + "attempts": attempts, + "preflights": preflights, + "training_seconds_known": training_known, + "training_duration_unknown_attempts": unknown_training, + "operational_seconds_including_queue_transfer_diagnostics": operational, + "operational_seconds_including_preflights": operational + + finite( + preflights["operational_seconds_including_queue_transfer"], + "preflight duration", + ), + "diagnostics_requested": diagnostic_count, + "diagnostics_completed": diagnostic_complete, + "diagnostic_execution_failures": diagnostic_failed, + "diagnostic_status_unknown": diagnostic_count + - diagnostic_complete + - diagnostic_failed, + "diagnostic_skips_known": diagnostic_skips, + "diagnostic_skip_counts_unknown": diagnostic_count - diagnostic_complete, + "recorded_errors": error_count, + } + if expected_arm is None: + report["role"] = "excluded_setup_only" + return report + submission_path = workspace / "submission.json" + selection = string(required(read_object(submission_path), "candidate"), "candidate") + if selection not in ("baseline", "candidate-01", "candidate-02"): + raise ValueError("Unknown selected candidate") + files.update({submission_path, workspace / "summary.md"}) + baseline = read_object(workspace / "baseline_development.json") + development = metric(required(baseline, "final"), [1, 2, 4, 8]) + source = workspace / "baseline_research.py" + if selection != "baseline": + if selection not in valid: + raise ValueError("Selected candidate has no validated completed training") + development, source = ( + valid[selection], + trial / "runs" / selection / "program.py", + ) + selected = root / "selected" / trial_id + selected_config = read_object(selected / "config.json") + baseline_config = read_object(root / "selected" / "baseline" / "config.json") + training = dict(mapping(required(selected_config, "training"), "training")) + require_equal( + required(training, "research_source"), text(source), "selected program" + ) + training["research_source"] = required( + mapping(required(baseline_config, "training"), "baseline training"), + "research_source", + ) + normalized = dict(selected_config) + normalized["training"] = json_object(training, "selected training") + require_equal(normalized, baseline_config, "frozen training configuration") + if selection == "baseline": + for name in ("config.json", "model.safetensors"): + require_equal( + digest(selected / name), + digest(root / "selected" / "baseline" / name), + "baseline artifact", + ) + files.update({selected / "config.json", selected / "model.safetensors"}) + summary = (workspace / "summary.md").read_text() + report.update( + { + "selection": selection, + "development": development, + "agent_summary_excerpt": summary[:2000], + "agent_summary_truncated": len(summary) > 2000, + } + ) + return report + + +def collect(root: Path) -> tuple[list[JSONValue], list[JSONValue], set[Path]]: + _, order, arms, excluded = protocol(root) + files = { + root / "selected" / "baseline" / name + for name in ("config.json", "model.safetensors") + } + files.add(root / "heldout-evaluation-plan.json") + files.add(root / "study_protocol.json") + harness_path = root / "harness-freeze-B.json" + packets_path = root / "solver-packet-manifest-B.json" + files.update({harness_path, packets_path}) + verify_inventory( + Path(__file__).resolve().parents[2], + required(read_object(harness_path), "files"), + files, + ) + inventory = mapping( + required(read_object(packets_path), "packet_inventory"), "packet inventory" + ) + require_equal(set(inventory), set(order), "packet trial set") + for trial in order: + verify_inventory(root / "trials" / trial / "workspace", inventory[trial], files) + reports: list[JSONValue] = [ + trial_summary(root, trial, files, arms[trial]) for trial in order + ] + setup: list[JSONValue] = [ + trial_summary(root, trial, files, None) for trial in excluded + ] + files.update(Path(__file__).resolve().parent.glob("*.py")) + return reports, setup, files + + +def verify_inventory(base: Path, value: object, files: set[Path]) -> None: + inventory = mapping(value, "hash inventory") + if not inventory: + raise ValueError("Empty evidence inventory") + for name, saved_hash in inventory.items(): + path = (base / name).resolve(strict=True) + if not path.is_relative_to(base.resolve()): + raise ValueError("Inventory path outside its root") + require_equal(digest(path), string(saved_hash, "saved hash"), "inventory hash") + files.add(path) + + +def timestamp(value: object) -> datetime: + parsed = datetime.fromisoformat(string(value, "timestamp").replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.utcoffset() != timezone.utc.utcoffset(parsed): + raise ValueError("Timestamp must explicitly identify UTC") + return parsed + + +def save_new(path: Path, value: JSONObject) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as handle: + handle.write(json.dumps(value, indent=2, allow_nan=False) + "\n") + + +def freeze(root: Path) -> None: + _, _, files = collect(root) + if (root / "scores").exists() and any((root / "scores").iterdir()): + raise ValueError("Scores already exist before finalization") + now = datetime.now(timezone.utc) + study, _, _, _ = protocol(root) + if timestamp(study["revised_at"]) >= now: + raise ValueError("Protocol revision must precede finalization") + save_new( + root / "finalization.json", + { + "finalized_at": now.isoformat(), + "files": {str(path.resolve()): digest(path) for path in sorted(files)}, + }, + ) + + +def verify_freeze(root: Path) -> JSONObject: + frozen = read_object(root / "finalization.json") + _, _, files = collect(root) + expected = mapping(required(frozen, "files"), "frozen files") + require_equal( + set(expected), {str(path.resolve()) for path in files}, "frozen file set" + ) + for path, saved_hash in expected.items(): + require_equal( + digest(Path(path)), string(saved_hash, "hash"), "frozen file hash" + ) + timestamp(required(frozen, "finalized_at")) + return frozen + + +def begin_scoring(root: Path) -> None: + verify_freeze(root) + if (root / "scores").exists() and any((root / "scores").iterdir()): + raise ValueError("Scoring has already started") + save_new( + root / "scores" / "provenance.json", + { + "scoring_started_at": datetime.now(timezone.utc).isoformat(), + "finalization_sha256": digest(root / "finalization.json"), + }, + ) + + +def analyze(root: Path) -> JSONObject: + frozen = verify_freeze(root) + provenance = read_object(root / "scores" / "provenance.json") + require_equal( + required(provenance, "finalization_sha256"), + digest(root / "finalization.json"), + "finalization provenance", + ) + if timestamp(required(provenance, "scoring_started_at")) <= timestamp( + required(frozen, "finalized_at") + ): + raise ValueError("Held-out scoring must follow finalization") + reports, setup, _ = collect(root) + study, order, _, _ = protocol(root) + scores: dict[str, JSONObject] = {} + panel: str | None = None + panel_size: int | None = None + plan = read_object(root / "heldout-evaluation-plan.json") + for trial in ("baseline", *order): + score = read_object(root / "scores" / (trial + ".json")) + for key, expected in ( + ("kind", "reduced_compute_pilot_not_official_RSI_score"), + ("split", "test"), + ("fixed_policy_depths", POLICY), + ("seed", required(plan, "seed")), + ( + "requested_examples_per_bin", + required(plan, "requested_examples_per_bin"), + ), + ): + require_equal(required(score, key), expected, key) + for key, name in ( + ("checkpoint_sha256", "model.safetensors"), + ("config_sha256", "config.json"), + ): + require_equal( + required(score, key), digest(root / "selected" / trial / name), key + ) + current_panel = string(required(score, "panel_sha256"), "panel hash") + if re.fullmatch(r"[0-9a-f]{64}", current_panel) is None: + raise ValueError("Malformed test panel hash") + current_size = integer(required(score, "actual_examples"), "actual_examples") + if current_size < 1: + raise ValueError("Test panel must contain observed examples") + if panel is not None: + require_equal(current_panel, panel, "test panel") + require_equal(current_size, panel_size, "test panel size") + panel = current_panel + panel_size = current_size + values = metric(required(score, "depths"), [1, 2, 4]) + require_equal( + finite(required(score, "reward"), "reward"), + values["reward"], + "reward calculation", + ) + values["raw_depths"] = score["depths"] + scores[trial] = values + for entry in reports: + trial_report = mapping(entry, "trial report") + if trial_report["selection"] == "baseline": + require_equal( + scores[string(trial_report["trial"], "trial")], + scores["baseline"], + "baseline selection score", + ) + pairs: list[JSONValue] = [] + for index, pair_entry in enumerate(sequence(study["pairs"], "pairs"), 1): + pair = mapping(pair_entry, "pair") + treatment = string(required(pair, "traincheck"), "traincheck") + control = string(required(pair, "control"), "control") + pairs.append( + { + "pair": index, + "control": control, + "traincheck": treatment, + "treatment_minus_control_reward": finite( + scores[treatment]["reward"], "reward" + ) + - finite(scores[control]["reward"], "reward"), + } + ) + return { + "kind": "two_pair_development_pilot_no_statistical_claim", + "trials": reports, + "excluded_setup_trials": setup, + "exclusion_reason": study["revision_reason"], + "heldout": json_object(scores, "heldout scores"), + "pairs": pairs, + "test_panel_sha256": panel, + "finalization_sha256": digest(root / "finalization.json"), + "limitations": [ + "Training time for failed or unobserved training attempts remains unknown, not zero.", + "Operational durations include queue wait and transfer; they are not agent wall time or token usage.", + "Agent summary excerpts are verbatim self-reports, not verified causal explanations.", + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("freeze", "begin-scoring", "analyze")) + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + root = args.root.resolve(strict=True) + if args.mode == "freeze": + freeze(root) + elif args.mode == "begin-scoring": + begin_scoring(root) + else: + print(json.dumps(analyze(root), indent=2, allow_nan=False)) + + +if __name__ == "__main__": + main() From 400c387e5d54a0b47d90368e67f20861b1c905fa Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 16:41:00 +0800 Subject: [PATCH 10/13] Bind selected checkpoints and diagnostic invariants to their frozen origins --- tests/test_rsi_pilot_analysis.py | 132 ++++++++++++++++++++++++++++++ tools/rsi_pilot/analyze_trials.py | 61 ++++++++++++++ 2 files changed, 193 insertions(+) diff --git a/tests/test_rsi_pilot_analysis.py b/tests/test_rsi_pilot_analysis.py index 3100c877..53b305c8 100644 --- a/tests/test_rsi_pilot_analysis.py +++ b/tests/test_rsi_pilot_analysis.py @@ -90,6 +90,35 @@ def setUp(self) -> None: write( self.root / "solver-packet-manifest-B.json", {"packet_inventory": inventory} ) + write( + self.root / "solver-packet-manifest.json", + {"baseline_remote_training": "/remote/baseline"}, + ) + self.save_origins() + + def save_origins(self) -> None: + artifacts: JSONObject = {} + for trial_id in ("baseline", *ORDER): + remote = "/remote/baseline" + if trial_id != "baseline": + submission = read_object( + self.root / "trials" / trial_id / "workspace/submission.json" + ) + candidate = submission["candidate"] + assert isinstance(candidate, str) + if candidate != "baseline": + remote = f"/remote/trials/{trial_id}/{candidate}/training" + artifacts[trial_id] = { + "remote_directory": remote, + "files": { + name: digest(self.root / "selected" / trial_id / name) + for name in ("config.json", "model.safetensors") + }, + } + write( + self.root / "selected/origins.json", + {"schema_version": 1, "artifacts": artifacts}, + ) def make_trial(self, trial_id: str) -> None: trial = self.root / "trials" / trial_id @@ -106,6 +135,7 @@ def make_trial(self, trial_id: str) -> None: "expected_steps": 10000, "profile": str(self.profile), "profile_sha256": digest(self.profile), + "remote_trial": f"/remote/trials/{trial_id}", }, ) (workspace / "baseline_research.py").write_text(self.baseline) @@ -211,6 +241,7 @@ def add_run( trial / "usage.json", {"candidates_reserved": 1, "diagnostics_reserved": int(diagnose)}, ) + self.save_origins() return run def scores(self) -> None: @@ -316,6 +347,107 @@ def test_selected_checkpoint_mismatch_fails(self) -> None: with self.assertRaisesRegex(ValueError, "checkpoint_sha256 mismatch"): analyze(self.root) + def test_wrong_downloaded_checkpoint_fails_before_finalization(self) -> None: + self.add_run("B1C", successful=True) + (self.root / "selected/B1C/model.safetensors").write_bytes(b"wrong model") + with self.assertRaisesRegex(ValueError, "remote artifact hash mismatch"): + freeze(self.root) + + def test_origin_directory_and_required_fields_are_checked(self) -> None: + self.add_run("B1C", successful=True) + path = self.root / "selected/origins.json" + original = path.read_bytes() + for change in ("wrong_directory", "missing_hash", "missing_trial"): + with self.subTest(change=change): + path.write_bytes(original) + origins = read_object(path) + artifacts = origins["artifacts"] + assert isinstance(artifacts, dict) + candidate = artifacts["B1C"] + assert isinstance(candidate, dict) + if change == "wrong_directory": + candidate["remote_directory"] = ( + "/remote/trials/B1C/candidate-02/training" + ) + elif change == "missing_hash": + hashes = candidate["files"] + assert isinstance(hashes, dict) + del hashes["config.json"] + else: + del artifacts["B1T"] + write(path, origins) + with self.assertRaises(ValueError): + freeze(self.root) + + def test_origin_evidence_is_frozen(self) -> None: + self.scores() + path = self.root / "selected/origins.json" + origins = read_object(path) + origins["host"] = "changed host after finalization" + write(path, origins) + with self.assertRaisesRegex(ValueError, "frozen file hash mismatch"): + analyze(self.root) + + def test_completed_diagnostic_requires_frozen_reference(self) -> None: + run = self.add_run("B1T", successful=True, diagnose=True) + invariant_path = self.root / "trials/B1T/workspace/invariants.json" + invariant_path.write_text("[]\n") + write(run / "coverage.json", {"schema_version": 3, "per_invariant": []}) + diagnostic: JSONObject = { + "schema_version": 1, + "status": "complete", + "candidate_config_sha256": digest(run / "config.toml"), + "invariants_sha256": digest(invariant_path), + } + write(run / "diagnostic_manifest.json", diagnostic) + report: JSONObject = { + "schema_version": 1, + "outcome": "no_evidence", + "coverage_incomplete": True, + "coverage_source": { + "path": "/remote/coverage.json", + "sha256": digest(run / "coverage.json"), + }, + "counts": { + key: 0 + for key in ( + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + "observed_active", + "observed_active_passes", + "observed_dormant", + "observed_violations", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "completed_decisions", + ) + }, + } + write(run / "agent_report.json", report) + receipt = read_object(run / "run.json") + receipt.update( + diagnostic_status="complete", + diagnostic_exit_code=0, + errors=[], + traincheck_report=report, + ) + write(run / "run.json", receipt) + _, _, frozen_files = collect(self.root) + self.assertIn(invariant_path, frozen_files) + diagnostic["invariants_sha256"] = "0" * 64 + write(run / "diagnostic_manifest.json", diagnostic) + with self.assertRaisesRegex( + ValueError, "diagnostic reference invariants mismatch" + ): + collect(self.root) + del diagnostic["invariants_sha256"] + write(run / "diagnostic_manifest.json", diagnostic) + with self.assertRaisesRegex(ValueError, "Missing required field"): + collect(self.root) + def test_false_or_fabricated_reward_fails(self) -> None: self.scores() path = self.root / "scores/B1T.json" diff --git a/tools/rsi_pilot/analyze_trials.py b/tools/rsi_pilot/analyze_trials.py index 686a25c1..b8e9a982 100644 --- a/tools/rsi_pilot/analyze_trials.py +++ b/tools/rsi_pilot/analyze_trials.py @@ -368,6 +368,16 @@ def trial_summary( diagnostic_report = read_object(path / "agent_report.json") validate_diagnostics(diagnostic_report) validate_diagnostic_artifacts(diagnostic_report, path) + invariant_path = workspace / "invariants.json" + require_equal( + required( + read_object(path / "diagnostic_manifest.json"), + "invariants_sha256", + ), + digest(invariant_path), + "diagnostic reference invariants", + ) + files.add(invariant_path) require_equal( required(receipt, "traincheck_report"), diagnostic_report, @@ -508,6 +518,56 @@ def trial_summary( return report +def verify_selected_origins(root: Path, order: list[str], files: set[Path]) -> None: + """Bind downloaded selections to hashes read at their original remote paths. + + The operator records these hashes directly on the remote host before copying + the artifacts. This checks accidental path/content mismatch, not adversarial + tampering by the experiment owner. + """ + origins_path = root / "selected/origins.json" + origins = read_object(origins_path) + require_equal(required(origins, "schema_version"), 1, "origin schema version") + artifacts = mapping(required(origins, "artifacts"), "origin artifacts") + require_equal(set(artifacts), {"baseline", *order}, "origin artifact set") + packet_path = root / "solver-packet-manifest.json" + baseline_remote = string( + required(read_object(packet_path), "baseline_remote_training"), + "baseline remote directory", + ) + if not baseline_remote.startswith("/"): + raise ValueError("Baseline remote directory must be absolute") + for trial_id in ("baseline", *order): + expected_remote = baseline_remote + if trial_id != "baseline": + trial = root / "trials" / trial_id + selection = string( + required(read_object(trial / "workspace/submission.json"), "candidate"), + "selected candidate", + ) + if selection != "baseline": + remote_trial = string( + required(read_object(trial / "trial.json"), "remote_trial"), + "remote trial directory", + ) + expected_remote = remote_trial + "/" + selection + "/training" + origin = mapping(artifacts[trial_id], "artifact origin") + require_equal( + required(origin, "remote_directory"), expected_remote, "origin directory" + ) + hashes = mapping(required(origin, "files"), "remote artifact hashes") + require_equal( + set(hashes), {"config.json", "model.safetensors"}, "origin file set" + ) + for name, remote_hash in hashes.items(): + require_equal( + string(remote_hash, "remote artifact hash"), + digest(root / "selected" / trial_id / name), + "remote artifact hash", + ) + files.update({origins_path, packet_path}) + + def collect(root: Path) -> tuple[list[JSONValue], list[JSONValue], set[Path]]: _, order, arms, excluded = protocol(root) files = { @@ -536,6 +596,7 @@ def collect(root: Path) -> tuple[list[JSONValue], list[JSONValue], set[Path]]: setup: list[JSONValue] = [ trial_summary(root, trial, files, None) for trial in excluded ] + verify_selected_origins(root, order, files) files.update(Path(__file__).resolve().parent.glob("*.py")) return reports, setup, files From ca4bf61aad0b6c6e763ae448949e047bd15588b1 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 16:48:54 +0800 Subject: [PATCH 11/13] Clean up owned phase processes when outer supervision sends a signal --- tests/test_fresh_validation_supervisor.py | 194 ++++++++++++++++++++++ tools/fresh_validation/run_phase.py | 158 ++++++++++++------ 2 files changed, 301 insertions(+), 51 deletions(-) create mode 100644 tests/test_fresh_validation_supervisor.py diff --git a/tests/test_fresh_validation_supervisor.py b/tests/test_fresh_validation_supervisor.py new file mode 100644 index 00000000..2cf01d04 --- /dev/null +++ b/tests/test_fresh_validation_supervisor.py @@ -0,0 +1,194 @@ +"""Linux process-tree regressions for the bounded correctness supervisor. + +Set TRAINCHECK_TEST_SUPERVISOR to test an archived CLI against the same external +signal cases. The tests always clean up their own recorded process groups. +""" + +import contextlib +import io +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.fresh_validation import run_phase +from tools.fresh_validation.contracts import boolean, integer, read_object, required + +DEFAULT_SUPERVISOR = Path(run_phase.__file__).resolve() +SUPERVISOR = ( + Path(os.environ["TRAINCHECK_TEST_SUPERVISOR"]) + if "TRAINCHECK_TEST_SUPERVISOR" in os.environ + else DEFAULT_SUPERVISOR +) + +GRANDCHILD = """import os, signal, sys, time +from pathlib import Path +signal.signal(signal.SIGTERM, signal.SIG_IGN) +Path(sys.argv[1]).write_text(str(os.getpid())) +time.sleep(60) +""" +LEADER = """import subprocess, sys, time +from pathlib import Path +child = subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]]) +while not Path(sys.argv[2]).exists(): + time.sleep(0.01) +Path(sys.argv[3]).write_text('ready') +time.sleep(60) +""" + + +def live(pid: int) -> bool: + """An orphan zombie is no longer running; only its parent can reap it.""" + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except FileNotFoundError: + return False + return fields[0] != "Z" + + +def wait_ready(path: Path, timeout: float = 5) -> None: + deadline = time.monotonic() + timeout + while not path.exists(): + if time.monotonic() >= deadline: + raise TimeoutError(f"Child did not become ready: {path}") + time.sleep(0.01) + + +@unittest.skipUnless(sys.platform == "linux", "Supervisor inspects Linux /proc") +class TestPhaseSupervisor(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory[str]() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / "source" + self.source.mkdir() + (self.source / "unchanged.py").write_text("VALUE = 1\n") + + def options(self, command: list[str]) -> run_phase.PhaseOptions: + return run_phase.PhaseOptions( + source=self.source, + output=self.root / "phase", + timeout=20.0, + memory_gib=1.0, + trace_mib=1.0, + command=command, + ) + + def assert_stopped(self, pid: int) -> None: + deadline = time.monotonic() + 3 + while live(pid) and time.monotonic() < deadline: + time.sleep(0.01) + self.assertFalse(live(pid), f"Owned child {pid} survived supervisor cleanup") + + def external_signal(self, signum: signal.Signals, expected: str) -> None: + output = self.root / "phase" + ready, child_pid_file = self.root / "ready", self.root / "grandchild.pid" + command = [ + sys.executable, + str(SUPERVISOR), + "--source", + str(self.source), + "--output", + str(output), + "--timeout", + "30", + "--memory-gib", + "1", + "--trace-mib", + "1", + "--", + sys.executable, + "-c", + LEADER, + GRANDCHILD, + str(child_pid_file), + str(ready), + ] + with (self.root / "supervisor.log").open("wb") as log: + process = subprocess.Popen( + command, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + child_group: int | None = None + try: + wait_ready(ready) + invocation = read_object(output / "invocation.json") + child_group = integer(required(invocation, "pid"), "child PID") + descendant = int(child_pid_file.read_text()) + os.kill(process.pid, signum) + code = process.wait(timeout=8) + self.assert_stopped(descendant) + self.assert_stopped(child_group) + self.assertEqual(code, 1) + result = read_object(output / "phase-result.json") + self.assertEqual(required(result, "stop_reason"), expected) + self.assertTrue( + boolean(required(result, "source_unchanged"), "source unchanged") + ) + self.assertNotEqual( + integer(required(result, "exit_code"), "child exit"), 0 + ) + finally: + # This also cleans the leaked child in the archived buggy CLI. + if child_group is not None: + with contextlib.suppress(ProcessLookupError): + os.killpg(child_group, signal.SIGKILL) + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=3) + + def test_sigterm_cleans_descendant_after_leader_exits(self) -> None: + self.external_signal(signal.SIGTERM, "signal_sigterm") + + def test_sigint_cleans_descendant_after_leader_exits(self) -> None: + self.external_signal(signal.SIGINT, "signal_sigint") + + def test_success_preserves_result_and_restores_handlers(self) -> None: + previous = (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)) + with contextlib.redirect_stdout(io.StringIO()): + result = run_phase.run(self.options([sys.executable, "-c", "pass"])) + self.assertEqual(result, 0) + self.assertEqual( + (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)), + previous, + ) + receipt = read_object(self.root / "phase/phase-result.json") + self.assertIsNone(required(receipt, "stop_reason")) + self.assertEqual(integer(required(receipt, "exit_code"), "child exit"), 0) + self.assertTrue( + boolean(required(receipt, "source_unchanged"), "source unchanged") + ) + + def test_monitor_exception_kills_term_resistant_child_and_restores_handlers( + self, + ) -> None: + ready = self.root / "child.pid" + previous = (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)) + + def failed_monitor(group: int) -> tuple[int, list[int]]: + wait_ready(ready) + raise ValueError("monitor failed") + + try: + with patch.object(run_phase, "group_memory", side_effect=failed_monitor): + with self.assertRaisesRegex(ValueError, "monitor failed"): + run_phase.run( + self.options([sys.executable, "-c", GRANDCHILD, str(ready)]) + ) + self.assert_stopped(int(ready.read_text())) + self.assertEqual( + (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)), + previous, + ) + finally: + if ready.exists(): + with contextlib.suppress(ProcessLookupError): + os.killpg(int(ready.read_text()), signal.SIGKILL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/fresh_validation/run_phase.py b/tools/fresh_validation/run_phase.py index 9e52a598..5dc83028 100644 --- a/tools/fresh_validation/run_phase.py +++ b/tools/fresh_validation/run_phase.py @@ -8,6 +8,7 @@ import subprocess import time from pathlib import Path +from types import FrameType from typing import Literal, TypedDict @@ -20,7 +21,13 @@ class PhaseOptions(argparse.Namespace): command: list[str] -StopReason = Literal["process_group_rss_budget", "timeout", "trace_size_budget"] +StopReason = Literal[ + "process_group_rss_budget", + "timeout", + "trace_size_budget", + "signal_sigterm", + "signal_sigint", +] class PhaseResult(TypedDict): @@ -54,7 +61,34 @@ def group_memory(group: int) -> tuple[int, list[int]]: return total, processes +def terminate_group(process: subprocess.Popen[bytes]) -> None: + """Stop the owned session, including descendants whose leader already exited. + + A terminating leader does not prove its process group is empty. Always send + KILL to any remaining members after a bounded TERM grace period, then reap + the direct child. A child that cannot be reaped is an explicit failure. + """ + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass # The owned group has already exited. + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + + def run(args: PhaseOptions) -> int: + """Supervise from the main thread, temporarily handling SIGTERM and SIGINT. + + Cleanup covers handled signals, normal completion and Python exceptions. + Uncatchable termination such as SIGKILL cannot run user-space cleanup. + """ args.output.mkdir(parents=True, exist_ok=False) source_hashes = { str(p.relative_to(args.source)): hashlib.sha256(p.read_bytes()).hexdigest() @@ -76,57 +110,79 @@ def run(args: PhaseOptions) -> int: command = args.command[1:] if args.command[0] == "--" else args.command started, peak = time.monotonic(), 0 reason: StopReason | None = None - with (args.output / "stdout.log").open("w") as log: - proc = subprocess.Popen( - ["nice", "-n", "10", *command], - cwd=args.output, - env=env, - stdout=log, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - invocation = { - "command": command, - "pid": proc.pid, - "native_threads": 1, - "timeout_seconds": args.timeout, - "memory_limit_gib": args.memory_gib, - "trace_limit_mib": args.trace_mib, - "source_sha256": source_hashes, - "harness_sha256": { - p.name: hashlib.sha256(p.read_bytes()).hexdigest() - for p in Path(__file__).resolve().parent.glob("*.py") - }, - } - (args.output / "invocation.json").write_text( - json.dumps(invocation, indent=2) + "\n" - ) + signal_reason: StopReason | None = None + + def request_stop(signum: int, frame: FrameType | None) -> None: + nonlocal signal_reason + assert signum in (signal.SIGTERM, signal.SIGINT) + if signal_reason is None: + signal_reason = ( + "signal_sigterm" if signum == signal.SIGTERM else "signal_sigint" + ) + + # The workload has a separate session. An outer timeout's signal to this + # supervisor must therefore trigger explicit child-group cleanup, not the + # default immediate supervisor exit. Signals only request a stop so repeated + # signals cannot interrupt TERM/KILL/reap midway through cleanup. + prior_term = signal.signal(signal.SIGTERM, request_stop) + try: + prior_int = signal.signal(signal.SIGINT, request_stop) try: - while proc.poll() is None: - memory, _ = group_memory(proc.pid) - peak = max(peak, memory) - if memory > args.memory_gib * 1024**3: - reason = "process_group_rss_budget" - if time.monotonic() - started > args.timeout: - reason = "timeout" - if ( - sum(p.stat().st_size for p in args.output.rglob("trace_API*.log")) - > args.trace_mib * 1024**2 - ): - reason = "trace_size_budget" - if reason: - os.killpg(proc.pid, signal.SIGTERM) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait() - break - time.sleep(0.25) - except BaseException: - os.killpg(proc.pid, signal.SIGTERM) - proc.wait(timeout=10) - raise + with (args.output / "stdout.log").open("w") as log: + proc = subprocess.Popen( + ["nice", "-n", "10", *command], + cwd=args.output, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + invocation = { + "command": command, + "pid": proc.pid, + "native_threads": 1, + "timeout_seconds": args.timeout, + "memory_limit_gib": args.memory_gib, + "trace_limit_mib": args.trace_mib, + "source_sha256": source_hashes, + "harness_sha256": { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in Path(__file__).resolve().parent.glob("*.py") + }, + } + (args.output / "invocation.json").write_text( + json.dumps(invocation, indent=2) + "\n" + ) + while proc.poll() is None: + memory, _ = group_memory(proc.pid) + peak = max(peak, memory) + if memory > args.memory_gib * 1024**3: + reason = "process_group_rss_budget" + if time.monotonic() - started > args.timeout: + reason = "timeout" + if ( + sum( + p.stat().st_size + for p in args.output.rglob("trace_API*.log") + ) + > args.trace_mib * 1024**2 + ): + reason = "trace_size_budget" + if signal_reason is not None: + reason = signal_reason + if reason is not None: + break + time.sleep(0.25) + finally: + terminate_group(proc) + # A signal may arrive while the leader exits or during cleanup. + if signal_reason is not None: + reason = signal_reason + finally: + signal.signal(signal.SIGINT, prior_int) + finally: + signal.signal(signal.SIGTERM, prior_term) unchanged = all( hashlib.sha256((args.source / p).read_bytes()).hexdigest() == h for p, h in source_hashes.items() From cd36805c2521a36766efd6e072cee8a36879ba61 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 17:08:54 +0800 Subject: [PATCH 12/13] Use the upstream finite-support census for held-out pilot evaluation --- docs/measurements/rsi-pilot-20261030.md | 18 ++++++++++++++++++ tools/rsi_pilot/score_adaptive.py | 23 +++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/measurements/rsi-pilot-20261030.md b/docs/measurements/rsi-pilot-20261030.md index 94bd7924..319e5715 100644 --- a/docs/measurements/rsi-pilot-20261030.md +++ b/docs/measurements/rsi-pilot-20261030.md @@ -98,6 +98,24 @@ held-out examples or scores are generated. The original six-trial its completion gate to all four B submissions. This setup revision does not establish an effect of TrainCheck on research performance. +### Evaluation repair after selection + +After all four selections and 305 evidence files were frozen, the first held-out +attempt failed during panel construction, before any model evaluation. The +original public `evaluation_set` requires 512 distinct two-operation test +expressions, but this public partition contains only 399. Partial examples had +been generated; no accuracy or reward had been computed. + +Test scoring now uses the public task's existing `calibrated_panel`: a census of +the one- and two-operation strata (14 and 399 examples), plus 512 unique examples +in each of the three larger strata, totaling 1,949. The seed remains 20261030, +but the new builder changes the sample sequence; it is not the original panel. +Validation scoring retains the original builder. Models, selections, training, +fixed inference policy and scoring formula are unchanged. The original failed +attempt and freeze are retained, and the evaluation amendment verifies every +original frozen hash except the explicitly changed scorer and harness inventory +before recording a new freeze. No model outcome informed this repair. + ## TrainCheck treatment and interpretation Reference inference uses the unmodified public baseline program and an explicit diff --git a/tools/rsi_pilot/score_adaptive.py b/tools/rsi_pilot/score_adaptive.py index e9b5631d..fe23e7dd 100644 --- a/tools/rsi_pilot/score_adaptive.py +++ b/tools/rsi_pilot/score_adaptive.py @@ -41,7 +41,11 @@ def score( ) -> JSONObject: # These public upstream modules are supplied by the pinned workspace on # PYTHONPATH. Their accepted output is validated before calculating scores. - from latent_reasoning.data import evaluation_set + from latent_reasoning.data import ( + calibrated_panel, + evaluation_set, + runtime_partition_key, + ) from latent_reasoning.evaluate import evaluate from latent_reasoning.model import LoopedTransformer, ModelConfig from latent_reasoning.reward import aggregate_reward @@ -67,7 +71,17 @@ def score( model.to(torch.device("cpu")) # Test panels are generated only after solver finalization. Validation mode # lets the operator check scoring parity before freezing the experiment. - examples = evaluation_set(split, [1, 2, 3, 4, 5], per_bin, 7, ("+", "-"), seed) + if runtime_partition_key() is not None: + raise ValueError("This pilot requires the public content-hash partition") + # The two-operation test partition has fewer than 512 distinct expressions. + # The public task's calibrated panel censuses both finite small strata and + # samples the requested count in each larger stratum without duplicates. + # Keep validation generation unchanged for native-trainer parity checks. + examples = ( + calibrated_panel(split, per_bin, seed) + if split == "test" + else evaluation_set(split, [1, 2, 3, 4, 5], per_bin, 7, ("+", "-"), seed) + ) rows_value: object = evaluate(model, examples, [1, 2, 4], torch.device("cpu")) rows = [ json_object(row, "depth result") @@ -109,6 +123,11 @@ def score( "split": split, "seed": seed, "requested_examples_per_bin": per_bin, + "panel_generator": ( + "upstream_calibrated_panel" + if split == "test" + else "upstream_evaluation_set" + ), "actual_examples": len(examples), "fixed_policy_depths": [1, 2, 4, 4], "reward": reward, From 754277ae87e2189cebf16b405940bef1fee5904c Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Fri, 30 Oct 2026 17:17:59 +0800 Subject: [PATCH 13/13] Report held-out RSI pilot outcomes and diagnostic decision usefulness --- docs/measurements/rsi-pilot-20261030.md | 148 +++++++++++++++++++++++- 1 file changed, 144 insertions(+), 4 deletions(-) diff --git a/docs/measurements/rsi-pilot-20261030.md b/docs/measurements/rsi-pilot-20261030.md index 319e5715..6c062225 100644 --- a/docs/measurements/rsi-pilot-20261030.md +++ b/docs/measurements/rsi-pilot-20261030.md @@ -1,12 +1,111 @@ # TrainCheck agent pilot: adaptive latent reasoning -Status: revised comparison protocol recorded before held-out scoring; results -pending. +Status: completed; four independent agents, eight completed training requests, +two completed TrainCheck probes, and five independently scored checkpoints. The decision is whether TrainCheck gives research agents useful diagnostic evidence that justifies a larger evaluation. This is a reduced-compute development pilot, not an official RSI benchmark submission or leaderboard score. +## Result and investment decision + +The workflow works, but this pilot does not demonstrate that TrainCheck improves +agent decisions. Treatment wins one pair and loses one; mean held-out accuracy +differs by only 0.652 percentage points. Both treatment agents explicitly report +that diagnostic evidence did not change their candidate design or selection. +Two pairs on one task cannot establish benefit, harm, or statistical equivalence. + +| Pair | Without TrainCheck | With TrainCheck | Treatment minus control | +| --- | ---: | ---: | ---: | +| 1, held-out accuracy | 56.769% | 51.411% | -5.358 percentage points | +| 2, held-out accuracy | 42.259% | 48.921% | +6.662 percentage points | +| Mean of two agents per arm | 49.514% | 50.166% | +0.652 percentage points | + +The unchanged baseline scores 21.198% on this panel. Both arms found useful +subtree-supervision objectives; those gains over baseline are not evidence of a +TrainCheck effect. All results use the predeclared mean of macro accuracies at +depths `[1, 2, 4, 4]`. The five operation strata each receive equal weight despite +their unequal census/sample sizes. + +| Trial | Selected request | Development accuracy | Held-out accuracy | Held-out reward | +| --- | --- | ---: | ---: | ---: | +| B1C | candidate-02 | 55.857% | 56.769% | 47.914772 | +| B1T | candidate-02 | 50.288% | 51.411% | 43.628470 | +| B2C | candidate-01 | 43.316% | 42.259% | 36.306728 | +| B2T | candidate-02 | 51.560% | 48.921% | 41.636263 | + +Differences depend on expression difficulty. The following percentages first +average the four fixed depth points, then average the two selected agents in an +arm. These are descriptive strata, not additional selection criteria. + +| Operations | Test examples | Baseline | Control mean | TrainCheck mean | +| --- | ---: | ---: | ---: | ---: | +| 1 | 14 | 28.57% | 83.93% | 96.43% | +| 2 | 399 | 32.21% | 76.63% | 80.67% | +| 3 | 512 | 14.65% | 46.39% | 41.24% | +| 4 | 512 | 15.38% | 23.32% | 17.80% | +| 5 | 512 | 15.19% | 17.31% | 14.70% | + +Recommendation: pause broad RSI scaling and further performance work undertaken +solely to enable it. The next useful validation is one training-bug task with a +known fault and demonstrated relevant invariant coverage. This objective-design +task supplies no observed mechanism by which diagnostics improved a decision. + +## What the diagnostics contributed + +Both treatment agents requested a probe on candidate-01 and ultimately selected +candidate-02 using ordinary development feedback. The selected candidates were +not probed; no claim of their full-run diagnostic certification is made. + +- B1T: 357 active rules, 355 passes, and the same two known healthy alarms as the + reference. No dormant rules, checker/observer errors, or unknown outcomes. +- B2T: 357 active rules, 354 passes, the two known alarms, and one additional + `DistinctArgumentRelation` alarm for `Sequential.forward`. No dormant rules, + checker/observer errors, or unknown outcomes. +- The two known alarms arise from synthetic parameter-identity metadata missing + in selective collection; their learned relation arguments also contain literal + process identities. The child APIs actually execute. Rules were retained. +- The extra alarm establishes equality of recorded argument metadata, including + a repeated `_TRAINCHECK_data_ID`. Tensor values/hashes were not collected. + It occurs before the candidate loss executes. First-update model, optimizer, + Torch RNG and sampler RNG snapshots match both healthy/full references. + Temporary `.data` wrapper-ID reuse is plausible but unconfirmed; this is + neither a confirmed candidate defect nor a confirmed false positive. + +Active-rule counts do not mean every opportunity was examined: the extra rule +stops after its first failing comparison (94 of 360 comparisons completed). +API-only collection provides no model-variable coverage, even when the report +lists zero skipped variable checks. The three-update probe also supplies +`total_steps=3`, so candidate schedules differ from their full-run horizons. + +## Measured cost + +| Trial | Ordinary training, two requests | Client operations including preflight, queue, transfer and diagnostics | Approximate recorded agent wall span | +| --- | ---: | ---: | ---: | +| B1C | 336.48 s | 549.41 s | 12.49 min | +| B1T | 312.93 s | 667.72 s | 18.01 min | +| B2C | 316.90 s | 523.13 s | 12.72 min | +| B2T | 321.48 s | 621.93 s | 14.71 min | + +Eight ordinary training runs consumed 1,287.78 seconds of training-body time. +The two probes added 8.955 and 8.935 seconds of collection/checking/summary time: +17.890 seconds, or 2.82% relative to the 634.40 seconds in four ordinary treatment +training runs. This measures optional short-probe cost, not full-training +instrumentation slowdown. CPU computation uses four fixed cores. + +Client totals include shared-queue delays and SSH transfer, so differences are +not causal tool-latency measurements. Agent spans come from operator-recorded +launch/completion timestamps and include queue/approval waits; they are +approximate. The 15-minute reasoning allowance was advisory, and reasoning-only +time/overruns and token usage are unknown. These totals exclude calibration, +engineering, tests, and other setup work. + +All eight revised training requests completed. Three free preflight connection +attempts failed in the local sandbox, then succeeded with authorized access; +they consumed no training allowance. Excluded initial setup trials retain six +training requests: two completed (273.164 seconds known training) and four failed +with training durations unknown. Their total client time was 520.370 seconds. + ## Task and scope The public Scale AI RSI repository is pinned to @@ -168,8 +267,49 @@ Git. The native solver is instruction-isolated, not an OS-enforced hostile-agent sandbox; this limits claims about benchmark integrity against deliberate abuse. The pilot branch integrates the reviewed #163 changes that had not reached main. -Composed regression checks passed 134 tests on labmach. The exportable skill and -report helper passed ten tests, explicit strict typing and configured checks. +Composed regression checks passed 134 tests on labmach. Final focused validation +passed 70 tests locally and four Linux process tests on labmach. Explicit strict +mypy passed all 13 helper/test modules; configured hooks and diff checks passed. +The finite-panel repair also passed a real upstream regression: the old builder +fails, the census builder returns 1,949 distinct, correctly labelled test +expressions, and repeating the seed reproduces the exact panel. The updated +scorer passed strict typing/checks and all five real held-out evaluations. A separate four-step linear-model workflow inferred 91 rules; its fresh healthy control produced 87 active passes and four containment alarms, with matching recorded losses. That smoke is separate from this RSI task. + +Review also reproduced an outer-signal cleanup defect in the existing phase +supervisor. Both baseline TERM/INT regressions left resistant descendants alive; +the fix passes four Linux tests, terminates the owned group, reaps its direct +child and restores handlers. It does not make SIGKILL catchable. This fix was +not deployed into the frozen pilot runtime; both actual probes finished normally +and the post-diagnostic process audit found no active diagnostic phase commands. + +Experimental TrainCheck core revision is `be4ddc3332753c4bf7ecb3cae422db54b5594749`; +client revision is `1ab5bf7677491d66bf6c8403b6a2d0f33c0ab5ae`; the scored-panel +repair is in `cd36805`. Exact per-file hashes bind the remaining adapters and +analyzer. Upstream/runtime details are above. The final panel SHA256 is +`d0893310003a1e0f3eff2a1c904d4f54d2c983b1f1000892cbb36340225574f1`. +All five checkpoint/configuration pairs were distinct and scored once. + +Durable evidence includes `analysis.json`, `numbers.json`, `selected/origins.json`, +all raw trial receipts and agent summaries, `scores/`, both finalization manifests, +`evaluation-amendment.json`, and `reference/rule274-b2t-audit.{json,md}`. The complete +remote trial archive has SHA256 +`8364dc54d9d75b0b3252279b82e40ef52fe25212fee592a9796444136f0d0c6a`. +The amendment preserves 303 original frozen files byte-for-byte; only scorer and +harness-inventory hashes change before successful scoring. + +Reproduce a candidate with `tools/rsi_pilot/run_candidate.py --trial TRIAL +--program PROGRAM`, using `--check-program` first and `--diagnose` only within +the treatment allowance. New repetitions require new trial directories/budgets; +do not reset this study's ledgers. After collecting final selections, the +operator runs `analyze_trials.py freeze`, `begin-scoring`, the original public +scorer with `--split test --seed 20261030 --per-bin 512`, then `analyze`. +Full concrete commands remain in the artifact receipts and operator scripts. + +Use the repository Python 3.10 environment for the exact reward audit, matching +the native Python 3.11 summation behavior. A Python 3.12 audit rejected one reward +by one ULP because that version changed floating-point `sum`; rerunning the +unchanged auditor under Python 3.10 passed. No stored score or tolerance was +changed. This is an explicit analysis-runtime limitation.