diff --git a/docs/measurements/instrumenter-hotpaths-20261028.md b/docs/measurements/instrumenter-hotpaths-20261028.md new file mode 100644 index 00000000..f69612a6 --- /dev/null +++ b/docs/measurements/instrumenter-hotpaths-20261028.md @@ -0,0 +1,251 @@ +# Instrumenter hot-path optimization, 2026-10-28 + +This study compares signed implementation +`161313a93512b07410fa2b1072613333a5a2e492` with +`bccc56fab83d525198286f1855b9dd7f6866135f` (PR #162). PR #163 depends on +#162. The work targets metadata collection; it changes neither the trace format +nor the point at which mutable values are serialized into an owned snapshot. + +## Implementation and contracts + +1. Native Tensor/Parameter metadata lookup caches two lookup-hook facts for a + contiguous immutable suffix of the class MRO. CPython's native `type.__flags__` + descriptor supplies + `Py_TPFLAGS_IMMUTABLETYPE`; class names or package names never establish + immutability. All descriptors, mutable class namespaces and receiver + dictionaries remain live, + and every binding still checks the current MRO, lookup hooks and descriptor + owner. Real method bindings and the existing classifier preserve observable + temporary lifetimes. Only booleans and existing instrumenter-owned namespace + views are retained; no descriptor or attribute-name references are added. + Other runtimes or an absent capability retain live + lookup; malformed present metadata is an internal error. +2. Exact None, bool, float, str and signed-64-bit integers avoid a throwaway + `json.dumps` compatibility probe. The existing compile/FakeTensor gate still + runs first. Subclasses, containers and other objects retain their original + path. Larger integers keep the probe, including Python's configurable + integer-to-string limit and its synchronous errors. Returned objects and + runtime type labels are unchanged. + +No metadata values or mutable class decisions are cached. No tensor fields, +identity fields, warnings from application getters, trace records or checks are +dropped. Final `orjson`/stdlib JSON fallback, argument observation order, +serialization timing, producer buffering and writer behavior are unchanged. + +## Ordinary labmach measurements + +| Timed body | Baseline median | Candidate median | Less time | +|---|---:|---:|---:| +| 800 add/mm calls | 1.516760 s | 1.479685 s | 2.44% | +| MNIST, 10 training steps | 0.667927 s | 0.646459 s | 3.21% | +| Transformer, 10 training steps | 1.771281 s | 1.725027 s | 2.61% | + +Training uses the existing benchmark adapters, batch size 2, Transformer +sequence length 16, and ten matched data/training steps. These are complete +short training loops, not full dataset epochs. Traced training bodies include +forward, loss, backward and optimizer work; setup, evidence capture, final +queue wait and shutdown remain outside the body. MNIST trial ranges overlap; +every Transformer candidate trial is faster than every baseline in this set. +All before/after numbers in this table use the corrected revision. + +MNIST has 1,199,882 parameters and uses the first 20 cached training examples, +no shuffling, NLL loss and Adadelta. The small Transformer has **28,267 +parameters**, two layers, width 32, two attention heads and feed-forward width +64. Its WikiText-2 prefix has 322 tokens and a 171-token vocabulary. The prefix +SHA256 is `c2122261adb584adc0e9cd4672758896de1260e7dae0ed732e8f07394bce9712`; +the underlying text SHA256 is +`9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f`. +This differs from the earlier three-step study's smaller vocabulary/model; +the fresh baseline and candidate both use the same ten-step data and model. +The training harness SHA256 is +`8a8b351a8baf0a98215461855cc72bfded7dbc94a667f27af789170a2deedda6`, +and the frozen adapter SHA256 is +`5b6852940601d663c9aedad3e096c869fea1631d2038d21956e883ae6e4c7a31`. + +The corrected candidate reduces the median timed operation loop from +**1.516760 to 1.479685 seconds, 2.44% less time (1.025x)**. There are four fresh +processes per revision in A1/B1/B2/A2/A3/B3/B4/A4 order. Baseline samples are +1.521204, 1.518426, 1.515093 and 1.513667 seconds; candidate samples are +1.544820, 1.481346, 1.475127 and 1.478023 seconds. One candidate run is slower +than baseline, so the result must not be described as an improvement in every +trial. Percentages divide saved time by baseline median time. + +Each trial makes 800 calls: 200 repetitions of `torch.add` and `torch.mm` with +exact Tensor/Parameter arguments, using 16x16 float32 inputs. All 800 outputs +remain retained during the loop in every variant. Full API argument and return +metadata produce 1,600 records per trial. Model tracking is disabled. Timing +includes the identical output-list retention and excludes setup, state evidence, +final queue wait and process shutdown. + +Every traced run matches the fresh plain control's complete recorded input, +output and Python/NumPy/Torch RNG evidence. Four complete ordered raw-record +comparisons retain every field and differ only in original process-specific +identities and timestamps; raw byte equality is not claimed. Source inventories +are verified before and after the study. Recorded final queue-wait time is not +total I/O cost: writing also overlaps the operation loop and evidence capture. + +The unchanged lab runtime is Python 3.11.4, Torch 2.16.0.dev20261012+cpu, +NumPy 2.1.0 and orjson 3.13.0. Runs use one pinned performance core (CPU8) and +one native/Torch thread. There is no GPU, selective-instrumentation policy, +online checker or distributed execution in this comparison. +The single plain control establishes state/output parity, not a stable +uninstrumented timing baseline; no tracing-overhead multiplier is inferred. + +Across all 24 instrumented trials, complete recorded state/loss/RNG comparisons +against the three fresh plain controls pass. Twelve ordered raw comparisons +retain all IDs/timestamps and find no other field, shape, type, array or exception +differences. Per trial, MNIST emits 1,584 records/1,066,058 bytes; Transformer +4,446 records/3,370,348 bytes; add/mm 1,600 records/1,863,600 bytes. Each has zero +VAR records because model tracking is disabled in the timed series. + +| Workload | Observed peak RSS median, baseline to candidate | Final queue-wait median, baseline to candidate | +|---|---:|---:| +| Add/mm (process group) | 508.160 to 507.867 MiB | 0.023853 to 0.327049 ms | +| MNIST (training process) | 542.705 to 548.924 MiB | 0.208640 to 0.193022 ms | +| Transformer (training process) | 518.225 to 518.201 MiB | 0.283403 to 0.248324 ms | + +These are operational observations, not isolated memory costs attributable to +the optimization. In particular MNIST's observed RSS is higher; no process-memory +saving is claimed. All min/median/max values and 24 individual samples are in +`measurements-summary.json`. Queue wait occurs after state evidence capture, +when background writes have already progressed; it is not the total I/O cost. + +## Separate model-tracker correctness smoke + +Six fresh three-step runs cover a plain control and baseline/candidate combined +API + subclass model tracking for each model. All four combined cases match +their plain control's complete recorded initial/final model, optimizer, gradient, +stored-value, loss and RNG evidence. Parameter identity and optimizer references +are checked within each run. Complete ordered API/VAR comparisons retain all +fields and find only original ID/timestamp differences. + +Each MNIST combined run emits 520 API and 64 VAR records. Each Transformer run +emits 1,340 API and 64 VAR records. The three-step Transformer smoke uses 21,052 +parameters and its shorter data prefix, distinct from the 28,267-parameter +ten-step timed workload. No performance result is inferred from these smoke runs. + +Observer coverage is explicitly limited: MNIST observes all eight parameters +before/after each step, with eight updates at steps 2 and 3. Transformer observes +only `model.input_emb.weight` before/after each step; all 29 parameter names +appear in update records at steps 2 and 3. Coverage matches the baseline but is +not complete Transformer pre/post coverage. No invariant checker ran in this +study, and trace integrity is not a claim of invariant activation or success. + +## Component measurements + +These local CPU measurements are separate from training. Percentages divide +saved elapsed time by the baseline elapsed time in the same row. + +| Component | Baseline | Candidate | Less time | +|---|---:|---:|---:| +| 120,000 exact scalar conversions, first series | 166.130 ms | 107.213 ms | 35.46% | +| Same scalar workload, second series | 167.685 ms | 105.716 ms | 36.96% | + +Scalar measurements use Python 3.10.16/Torch 2.2.2 and six fixed values repeated +20,000 times, four runs per +variant in A1/B1/B2/A2/A3/B3/B4/A4 order. The baseline function comes from the +frozen baseline source; the candidate is the actual scalar implementation in +`f9b221f`. Its entire dumper module is unchanged in corrected `161313a`, SHA256 +`60e626d77dcbcbdc98317dd061a0bc09cde638b3d0bf70a96a3da510f46fc7de`. +These component measurements do not validate the superseded descriptor cache. +The two repetitions are labeled by the accompanying replay trace files, but the +scalar workload is the same synthetic workload in both; they are not MNIST or +Transformer training timings. No profiling hooks run inside these measurements. + +An earlier descriptor-map cache passed the ordinary workload comparisons but +failed a new regression: a custom getter observed additional descriptor/name +references through `sys.getrefcount`. That implementation and its performance +results remain historical evidence. The corrected cache records only lookup-hook +facts and existing namespace views. The regression fails against the earlier +implementation and passes after correction. + +The corrected cache's fixed containers occupy **344 bytes per process** in the +local runtime, measured with `sys.getsizeof`. This excludes the NamedTuple +class/code objects and allocator overhead; it is not peak RSS. The helper's +SHA256 is `374383debe3dcdb76d13e6b901c25c9441021a4c1cf5df8f1d9bb5f8f520b3d9`. + +Two actual-source local native-metadata batches each have nine alternating-order +repeats. Full Tensor conversion is 893.6 to 887.9 us (0.64% less) in the first +batch and 869.6 to 836.1 us (3.85% less) in the second. The first batch has clear +scheduling outliers; one bounded repeat was performed and both complete batches +are retained. This does not support one stable general converter percentage. +The ordinary pinned lab comparisons provide the primary pipeline measurements. + +## Binary capture and other rejected experiments + +Replay uses one saved 520-record MNIST trace, reconstructing positional integer +keys while retaining every decoded field. The trace SHA256 is +`5f941cfb23dee11054af901056fbbac489f440ba32ce005f859c8476323dac59`. +These are exclusively owned decoded fixtures, not arbitrary live objects. +Forty passes process 20,800 records; each comparison uses four runs per variant. + +| Replay experiment | Current serializer | Experiment | Result | +|---|---:|---:|---| +| Pickle capture alone, without validation | 58.912 ms | 31.892 ms | 45.86% less producer work in this optimistic fixture | +| Python validation followed by pickle | 58.773 ms | 124.984 ms | 112.65% more producer work | +| Pickle capture, decode, then existing JSON serialization | 58.773 ms | 143.648 ms | 144.41% more total sequential work | + +Pickle payloads occupy 286,757 bytes versus 350,553 UTF-8 JSON bytes, excluding +framing. Exact final JSON strings match after round-trip. This does **not** prove +a correct live-object snapshot: custom pickle hooks can run application code, +and Python validation followed by pickling leaves a concurrent mutation gap. +IPC, scheduling, copies, backpressure, worker memory and drain costs are excluded. +Parallel execution might hide some worker work, but was not measured here. +The binary approach remains exploratory, with no production change or training +speedup claim. + +The same reconstructed MNIST corpus sends 219/520 records (42.12%) through +stdlib fallback because positional argument keys are integers. A callback-free +prefix guard before bypassing the failed `orjson` attempt was **128.56% slower** +than the existing serializer. Simply enabling non-string keys changes codec +semantics, including nonfinite values, so it was not adopted. UTF-8 byte batching +showed only 0.09% difference in that replay and was not adopted. Direct native +slot dispatch and cached mappingproxy getter methods also failed to provide a +meaningful gain. Preserved prototypes and raw results distinguish these rejected +options from the final implementation. + +## Validation + +- Python 3.10: 70 focused metadata/scalar/trace-integrity regressions pass. +- Python 3.12: 43 metadata tests and 12 scalar tests pass. +- Full Python 3.10 suite: 357 pass, with two failures reproduced in an isolated + unchanged baseline archive: `test_traincheck_internal_attr_hidden` and + `test_inject_eval_loop`. This is not a claim that the full suite is green. +- Configured changed-file pre-commit hooks pass. New tests also pass explicit + Ruff/Black/isort checks. Strict scoped mypy checks the metadata helper and both + new test modules; ordinary repository mypy checks the changed dumper and its + callers. Existing untyped metadata delegates remain outside strict coverage. +- Independent cross-review found no actionable defect. Signed-commit verification + succeeded with the configured signing key. +- All 329 timed-evidence files and 89 combined-evidence files passed checksum + verification after download. All study processes have finished. + +## Evidence and reproduction + +Durable raw evidence lives under +`outputs/instrumenter-optimization-20261028/` in the canonical TrainCheck +checkout, outside Git. Relevant files include `CHECKPOINT.md`, +`native-lookup-lifetime-repair/REPORT.md`, its two actual-source timing datasets, +`native-lookup-lifetime-repair/final-cache-size.json`, `mnist-final-component-probe.json`, +`transformer-final-component-probe.json`, `binary-capture-probe-checked.json`, +and the test logs. Scripts, full timing repetitions, source hashes and limitations +are retained beside those results. Exploratory older results are preserved and +are not substituted for the final-candidate numbers above. + +The isolated labmach harness is under `labmach/`; its README documents the frozen +source inventories, existing runtime/data, exact invocations, resource bounds +and separate `--stage micro` / `--stage training` gates. The candidate study is +`/home/yuxuan/experiments/instrumenter-optimization-20261028/161313a93512`. +Its preparation archive SHA256 is +`e6d8929a4bff0e1d5101f38cbb2c7f4ba8583241f0f8c31b4f1a6fefc40f93d3`. +Local final study artifacts are under `labmach/studies/161313a93512/`. + +| Artifact | SHA256 | +|---|---| +| `timed-evidence.tar.gz` (23,645,408 bytes) | `1fe4188c382816424e21496327a018a0610caaa8eeaa93d91ecd9f59eaf5471b` | +| `combined-evidence.tar.gz` (3,850,102 bytes) | `e356b9393168c00758f20c7382c57fe877d0798d7851d7e56dd0018f6b396747` | +| `verified-analysis.json` | `1320d25c89f540c8a041d756b928ab3f4d649f4b5789b76f5bb02f31b26f62c2` | +| `measurements-summary.json` | `48302b7201c607ffbbc0b8c56b12fab550816b85ce4ebe7e4c3106bafbd2e867` | +| `verified-combined-analysis.json` | `4a0614b078383279aa286eaf0efa988edff18027167cbecae223f36a123f4dba` | + +No H100 experiment is part of this development pass. diff --git a/docs/measurements/rsi-pilot-20261030.md b/docs/measurements/rsi-pilot-20261030.md new file mode 100644 index 00000000..6c062225 --- /dev/null +++ b/docs/measurements/rsi-pilot-20261030.md @@ -0,0 +1,315 @@ +# TrainCheck agent pilot: adaptive latent reasoning + +Status: completed; four independent agents, eight completed training requests, +two completed TrainCheck probes, and five independently scored checkpoints. + +The decision is whether TrainCheck gives research agents useful diagnostic +evidence that justifies a larger evaluation. This is a reduced-compute development +pilot, not an official RSI benchmark submission or leaderboard score. + +## Result and investment decision + +The workflow works, but this pilot does not demonstrate that TrainCheck improves +agent decisions. Treatment wins one pair and loses one; mean held-out accuracy +differs by only 0.652 percentage points. Both treatment agents explicitly report +that diagnostic evidence did not change their candidate design or selection. +Two pairs on one task cannot establish benefit, harm, or statistical equivalence. + +| Pair | Without TrainCheck | With TrainCheck | Treatment minus control | +| --- | ---: | ---: | ---: | +| 1, held-out accuracy | 56.769% | 51.411% | -5.358 percentage points | +| 2, held-out accuracy | 42.259% | 48.921% | +6.662 percentage points | +| Mean of two agents per arm | 49.514% | 50.166% | +0.652 percentage points | + +The unchanged baseline scores 21.198% on this panel. Both arms found useful +subtree-supervision objectives; those gains over baseline are not evidence of a +TrainCheck effect. All results use the predeclared mean of macro accuracies at +depths `[1, 2, 4, 4]`. The five operation strata each receive equal weight despite +their unequal census/sample sizes. + +| Trial | Selected request | Development accuracy | Held-out accuracy | Held-out reward | +| --- | --- | ---: | ---: | ---: | +| B1C | candidate-02 | 55.857% | 56.769% | 47.914772 | +| B1T | candidate-02 | 50.288% | 51.411% | 43.628470 | +| B2C | candidate-01 | 43.316% | 42.259% | 36.306728 | +| B2T | candidate-02 | 51.560% | 48.921% | 41.636263 | + +Differences depend on expression difficulty. The following percentages first +average the four fixed depth points, then average the two selected agents in an +arm. These are descriptive strata, not additional selection criteria. + +| Operations | Test examples | Baseline | Control mean | TrainCheck mean | +| --- | ---: | ---: | ---: | ---: | +| 1 | 14 | 28.57% | 83.93% | 96.43% | +| 2 | 399 | 32.21% | 76.63% | 80.67% | +| 3 | 512 | 14.65% | 46.39% | 41.24% | +| 4 | 512 | 15.38% | 23.32% | 17.80% | +| 5 | 512 | 15.19% | 17.31% | 14.70% | + +Recommendation: pause broad RSI scaling and further performance work undertaken +solely to enable it. The next useful validation is one training-bug task with a +known fault and demonstrated relevant invariant coverage. This objective-design +task supplies no observed mechanism by which diagnostics improved a decision. + +## What the diagnostics contributed + +Both treatment agents requested a probe on candidate-01 and ultimately selected +candidate-02 using ordinary development feedback. The selected candidates were +not probed; no claim of their full-run diagnostic certification is made. + +- B1T: 357 active rules, 355 passes, and the same two known healthy alarms as the + reference. No dormant rules, checker/observer errors, or unknown outcomes. +- B2T: 357 active rules, 354 passes, the two known alarms, and one additional + `DistinctArgumentRelation` alarm for `Sequential.forward`. No dormant rules, + checker/observer errors, or unknown outcomes. +- The two known alarms arise from synthetic parameter-identity metadata missing + in selective collection; their learned relation arguments also contain literal + process identities. The child APIs actually execute. Rules were retained. +- The extra alarm establishes equality of recorded argument metadata, including + a repeated `_TRAINCHECK_data_ID`. Tensor values/hashes were not collected. + It occurs before the candidate loss executes. First-update model, optimizer, + Torch RNG and sampler RNG snapshots match both healthy/full references. + Temporary `.data` wrapper-ID reuse is plausible but unconfirmed; this is + neither a confirmed candidate defect nor a confirmed false positive. + +Active-rule counts do not mean every opportunity was examined: the extra rule +stops after its first failing comparison (94 of 360 comparisons completed). +API-only collection provides no model-variable coverage, even when the report +lists zero skipped variable checks. The three-update probe also supplies +`total_steps=3`, so candidate schedules differ from their full-run horizons. + +## Measured cost + +| Trial | Ordinary training, two requests | Client operations including preflight, queue, transfer and diagnostics | Approximate recorded agent wall span | +| --- | ---: | ---: | ---: | +| B1C | 336.48 s | 549.41 s | 12.49 min | +| B1T | 312.93 s | 667.72 s | 18.01 min | +| B2C | 316.90 s | 523.13 s | 12.72 min | +| B2T | 321.48 s | 621.93 s | 14.71 min | + +Eight ordinary training runs consumed 1,287.78 seconds of training-body time. +The two probes added 8.955 and 8.935 seconds of collection/checking/summary time: +17.890 seconds, or 2.82% relative to the 634.40 seconds in four ordinary treatment +training runs. This measures optional short-probe cost, not full-training +instrumentation slowdown. CPU computation uses four fixed cores. + +Client totals include shared-queue delays and SSH transfer, so differences are +not causal tool-latency measurements. Agent spans come from operator-recorded +launch/completion timestamps and include queue/approval waits; they are +approximate. The 15-minute reasoning allowance was advisory, and reasoning-only +time/overruns and token usage are unknown. These totals exclude calibration, +engineering, tests, and other setup work. + +All eight revised training requests completed. Three free preflight connection +attempts failed in the local sandbox, then succeeded with authorized access; +they consumed no training allowance. Excluded initial setup trials retain six +training requests: two completed (273.164 seconds known training) and four failed +with training durations unknown. Their total client time was 520.370 seconds. + +## Task and scope + +The public Scale AI RSI repository is pinned to +`aa82cfc29e541e68b5fb79d789b3f09227851739`, task +`tasks/adaptive-latent-reasoning`. Its public local trainer already supports CPU +execution. The unchanged 51,335-parameter looped Transformer learns prefix +addition/subtraction modulo seven with one through five operations. The original +model, eight training loops, data generator and tensor-program interpreter are +retained. Agents edit a loss function and optionally optimizer equations through +that interpreter. They cannot change the model, data, runner, training profile or +evaluator. This pilot fixes inference depths to `[1, 2, 4, 4]` to isolate training +changes; it does not evaluate the task's full inference-policy search space. + +The official task uses an H100, batch 4,096, up to 50,000 updates and a signed +trusted training service. This pilot uses labmach CPU, batch 64 and 10,000 +updates, the public content-hash partition and unsigned local checkpoints. +Learning-rate warmup is 100 updates and cosine decay ends at 10,000. The initial +learning rate, backbone, optimizer defaults and baseline depth weights stay as +in the public task. Data pool construction is disabled in favor of the upstream +on-demand sampler. Runtime differs from the official lockfile: the existing +Python 3.11.4 / Torch 2.16 development CPU environment is used with an isolated +safetensors 0.8.0 overlay. Exact profiles and hashes are retained with artifacts. + +## Calibration, before solver access + +The built-in objective at 200, 1,000 and 3,000 updates stayed near chance on the +public development panel. A 10,000-update run using the exact public +`baseline_research.py` through the actual tensor-program interpreter completed +in 132.716 seconds of training (134.21 seconds process wall time). Development +accuracy at depths 1/2/4/8 was 22.220/24.173/23.548/23.235 percent. Gains were +concentrated in one- and two-operation expressions; longer expressions remained +near chance. This is one deterministic training run, not a repeated timing +estimate. The one-operation development stratum has only 17 distinct examples; +macro averages weight it equally with the four larger strata. + +## Comparison protocol + +- Two matched pairs of fresh agents, trial order B1C/B1T, B2T/B2C. Each + receives the same public source interface, baseline program, baseline + development feedback, frozen training profile and optimization objective. +- All solver agents start without the parent conversation. They use the same + inherited model/effort configuration. No solution, repository task metadata + containing solution explanations, private verifier, other trial or final-test + result is provided. The operator inspected task metadata during selection; + that context is deliberately excluded from solver prompts. +- Each agent may submit at most two candidate programs for 10,000-update runs. + Failed requests consume this allowance. Both arms have free program validation + through the upstream tensor-program interpreter, matching the task's documented + `--check-program` interface. Validation does not train or consume candidate + allowance; its receipts, failures and duration are reported separately. + Treatment agents additionally receive + the TrainCheck skill and may request one short diagnostic rerun for a candidate. + This additional computation is measured and included in cost reporting. + Both arms may select the supplied baseline instead of a candidate. +- The intended reasoning window is 15 minutes per solver, excluding time queued + for shared compute. The client enforces candidate and diagnostic counts; + agent reasoning/token limits are not mechanically enforced by this native + subagent interface. Actual elapsed time and any overrun will be reported. +- Candidate training is serialized on the same four idle performance cores + (8, 10, 12, 14). Original upstream code uses four Torch threads. A shared lock + prevents candidate/diagnostic overlap. Process wall time includes queue and + transfer costs; training-body time is reported separately. +- Final selection is written before held-out scoring. The operator then uses + the original model, public generator's test partition, seed 20261030 and up to + 512 distinct examples per operation stratum. No candidate is tuned on these + results. Final scoring uses upstream accuracy/FLOP accounting and reward + formula for the fixed policy; it is still a pilot metric, not an official + score or trusted-service receipt. Each unique checkpoint/configuration pair + is scored once. Identical selections reuse that immutable report, recorded + in `scores/reuse.json`; these reused durations are not independent timing + measurements. + +### Setup revision before held-out evaluation + +The initial C1/T1/T2 agents encountered source-parser rejections, including +docstrings exceeding the interpreter's string limit. The pilot client had omitted +the free program validator documented in the public task. All three selected +the baseline. These runs demonstrate a harness integration gap and are retained +as setup evidence, with their failed requests and costs; they are excluded from +the paired outcome comparison. The initially planned C2/C3/T3 agents were never +launched. + +The revised comparison restores that validator and supplies the public +interpreter source equally to all four fresh agents. The model, data, training +budget, development panel, held-out panel and treatment remain unchanged. +`study_protocol.json` records the reason, trial order and two pairs before any +held-out examples or scores are generated. The original six-trial +`heldout-evaluation-plan.json` is preserved as provenance; the revision changes +its completion gate to all four B submissions. This setup revision does not +establish an effect of TrainCheck on research performance. + +### Evaluation repair after selection + +After all four selections and 305 evidence files were frozen, the first held-out +attempt failed during panel construction, before any model evaluation. The +original public `evaluation_set` requires 512 distinct two-operation test +expressions, but this public partition contains only 399. Partial examples had +been generated; no accuracy or reward had been computed. + +Test scoring now uses the public task's existing `calibrated_panel`: a census of +the one- and two-operation strata (14 and 399 examples), plus 512 unique examples +in each of the three larger strata, totaling 1,949. The seed remains 20261030, +but the new builder changes the sample sequence; it is not the original panel. +Validation scoring retains the original builder. Models, selections, training, +fixed inference policy and scoring formula are unchanged. The original failed +attempt and freeze are retained, and the evaluation amendment verifies every +original frozen hash except the explicitly changed scorer and harness inventory +before recording a new freeze. No model outcome informed this repair. + +## TrainCheck treatment and interpretation + +Reference inference uses the unmodified public baseline program and an explicit +three-update probe. Target probes use the same runtime, model and batch size. +The probe's `total_steps` is three, so step-dependent candidate programs may +behave differently from their full 10,000-update runs. This limitation is exposed +to solvers. Tracing is API-only; no claim of variable-tracker or complete gradient +invariant coverage is made. The interpreter rejects Tensor subclasses, so the +model tracker is not silently enabled. + +The adapter checks unmodified trainer source and traces only the training body. +Full reference tracing is installed before the research interpreter captures +Torch callable references. Plain/reference controls must match model, gradient, +optimizer, buffer, loss, Torch RNG and local data-generator state. Unseeded +global Python/NumPy RNGs must remain unchanged within each run. Invariants are +frozen before candidate checks, and no rule is discarded to reduce alarms. +FunctionLead/FunctionCover relations are excluded as in the validated bounded +offline workflow; remaining families are retained. + +An invariant violation means a difference from reference behavior. Loss and +optimizer changes are expressly allowed, so many differences may be legitimate. +The treatment receives the fresh healthy-run report and full evidence paths to +help distinguish such differences. Checker exit code zero does not mean no +violations; errors, dormant rules, skipped checks and missing evidence stay +explicit. Both ordinary training and diagnostic failures are retained. + +## Decision criteria + +Record task success, selected-program identity, development and held-out reward, +all attempted candidates, measured training/diagnostic/setup cost, actual tool +use and whether any diagnostic changed an agent decision. A useful mechanistic +signal is an agent using relevant evidence to correct an unintended training +behavior, followed by independent validation. Fewer alarms alone do not count +as improvement. A higher score in one of two pairs does not establish a +general effect or statistical significance. + +If diagnostics are unused, irrelevant, dominated by healthy/reference-mismatch +alarms or add cost without improving decisions, the recommendation is to pause +broader performance work and target the demonstrated integration/coverage gap. +If the treatment shows a reproducible useful mechanism, select additional tasks +and increase repeats before making claims about model quality or efficiency. + +## Artifacts and engineering checkpoint + +Local large artifacts: `outputs/rsi-pilot-20261030/` in the experiment owner's +canonical TrainCheck checkout. Remote: labmach +`/home/yuxuan/experiments/traincheck-rsi-pilot-20261030/`. Only allowlisted public +task files are copied into solver workspaces; raw traces/checkpoints stay out of +Git. The native solver is instruction-isolated, not an OS-enforced hostile-agent +sandbox; this limits claims about benchmark integrity against deliberate abuse. + +The pilot branch integrates the reviewed #163 changes that had not reached main. +Composed regression checks passed 134 tests on labmach. Final focused validation +passed 70 tests locally and four Linux process tests on labmach. Explicit strict +mypy passed all 13 helper/test modules; configured hooks and diff checks passed. +The finite-panel repair also passed a real upstream regression: the old builder +fails, the census builder returns 1,949 distinct, correctly labelled test +expressions, and repeating the seed reproduces the exact panel. The updated +scorer passed strict typing/checks and all five real held-out evaluations. +A separate four-step linear-model workflow inferred 91 rules; its fresh healthy +control produced 87 active passes and four containment alarms, with matching +recorded losses. That smoke is separate from this RSI task. + +Review also reproduced an outer-signal cleanup defect in the existing phase +supervisor. Both baseline TERM/INT regressions left resistant descendants alive; +the fix passes four Linux tests, terminates the owned group, reaps its direct +child and restores handlers. It does not make SIGKILL catchable. This fix was +not deployed into the frozen pilot runtime; both actual probes finished normally +and the post-diagnostic process audit found no active diagnostic phase commands. + +Experimental TrainCheck core revision is `be4ddc3332753c4bf7ecb3cae422db54b5594749`; +client revision is `1ab5bf7677491d66bf6c8403b6a2d0f33c0ab5ae`; the scored-panel +repair is in `cd36805`. Exact per-file hashes bind the remaining adapters and +analyzer. Upstream/runtime details are above. The final panel SHA256 is +`d0893310003a1e0f3eff2a1c904d4f54d2c983b1f1000892cbb36340225574f1`. +All five checkpoint/configuration pairs were distinct and scored once. + +Durable evidence includes `analysis.json`, `numbers.json`, `selected/origins.json`, +all raw trial receipts and agent summaries, `scores/`, both finalization manifests, +`evaluation-amendment.json`, and `reference/rule274-b2t-audit.{json,md}`. The complete +remote trial archive has SHA256 +`8364dc54d9d75b0b3252279b82e40ef52fe25212fee592a9796444136f0d0c6a`. +The amendment preserves 303 original frozen files byte-for-byte; only scorer and +harness-inventory hashes change before successful scoring. + +Reproduce a candidate with `tools/rsi_pilot/run_candidate.py --trial TRIAL +--program PROGRAM`, using `--check-program` first and `--diagnose` only within +the treatment allowance. New repetitions require new trial directories/budgets; +do not reset this study's ledgers. After collecting final selections, the +operator runs `analyze_trials.py freeze`, `begin-scoring`, the original public +scorer with `--split test --seed 20261030 --per-bin 512`, then `analyze`. +Full concrete commands remain in the artifact receipts and operator scripts. + +Use the repository Python 3.10 environment for the exact reward audit, matching +the native Python 3.11 summation behavior. A Python 3.12 audit rejected one reward +by one ULP because that version changed floating-point `sum`; rerunning the +unchanged auditor under Python 3.10 passed. No stored score or tolerance was +changed. This is an explicit analysis-runtime limitation. diff --git a/skills/traincheck/SKILL.md b/skills/traincheck/SKILL.md new file mode 100644 index 00000000..40f1d25c --- /dev/null +++ b/skills/traincheck/SKILL.md @@ -0,0 +1,108 @@ +--- +name: traincheck +description: Diagnose PyTorch training behavior with TrainCheck reference traces, inferred invariants, and offline checks. Use when an agent needs evidence about a suspected training bug or the behavioral effect of an allowed training change. +--- + +Use TrainCheck to obtain evidence for the user's training task. A violated +invariant is a difference from its reference, not automatically a bug or a reason +to undo an intentional algorithm change. The task's evaluator remains the measure +of success. + +## Establish the experiment + +- Use the supplied TrainCheck source and Python environment. Keep the source on + `PYTHONPATH`; the coverage observer below is a repository tool, not a wheel entry + point. Do not install this skill globally during a treatment/control experiment. +- Use a legitimate public baseline, an authorized known-good run, or supplied + reference artifacts. Never obtain a reference from hidden tests or a withheld + solution. Record the reference's code, data, configuration, and framework + versions. The Llama experiment's invariant file is not a universal rule set. +- Start with a short run that reaches the operation of interest. Use matching + runtime/data/configuration for reference and candidate. Check a fresh healthy + run before treating an alarm as a regression; object IDs, thread IDs, and CUDA + stream handles can make learned rules process-specific. +- Keep benchmark code changes within the task's allowed edit scope. If training + code cannot be changed, run instrumentation in an isolated diagnostic copy and + submit only permitted artifacts. Instrumented training time is not an ordinary + benchmark score. + +## Collect, infer, and check + +Example commands, using absolute paths and fresh output directories: + +```sh +export PYTHONPATH="$TRAINCHECK_SOURCE" +"$PYTHON" -m traincheck.collect_trace --pyscript "$REFERENCE_SCRIPT" \ + --output-dir "$RUN/reference" --sampling-interval 1 --warm-up-steps 0 +"$PYTHON" -m traincheck.infer_engine -f "$RUN/reference" \ + -o "$RUN/invariants.json" --num-workers 1 \ + --disable-relation FunctionLeadRelation FunctionCoverRelation +"$PYTHON" -m traincheck.collect_trace --pyscript "$TARGET_SCRIPT" \ + --invariants "$RUN/invariants.json" --output-dir "$RUN/target" \ + --sampling-interval 1 --warm-up-steps 0 +"$PYTHON" -m traincheck.checker -f "$RUN/target" \ + -i "$RUN/invariants.json" --output-dir "$RUN/check" --num-workers 1 +``` + +This example collects API evidence. For a task that needs parameter/gradient +updates, add `--models-to-track MODEL_VARIABLE --model-tracker-style subclass` to +both collection commands, then verify the generated policy and actual variable +records. A requested tracker alone does not prove those checks ran. Distributed +FSDP/DTensor model tracking and concurrent checking have separate compatibility +limits; use the offline path for the initial pilot. Do not switch to a different +tracker to make a failed comparison pass. + +The two ordering relations are disabled in this bounded starting recipe to match +the validated offline workflow; record any chosen relation filter. Add a needed +family deliberately rather than silently changing rules after seeing a result. +For launchers or script arguments, inspect `collect_trace --help` and the supplied +task adapter. Do not guess a different training entry point. + +Measure activation with the existing observer, using the complete trace file +list from that run, including variable traces when present: + +```sh +"$PYTHON" "$TRAINCHECK_SOURCE/tools/fresh_validation/check_coverage.py" \ + --traces "$RUN/target"/trace_*.log --invariants "$RUN/invariants.json" \ + --output "$RUN/coverage.json" +"$PYTHON" "$TRAINCHECK_SKILL/scripts/summarize_check.py" \ + --coverage "$RUN/coverage.json" \ + --collection-policy "$RUN/target/instr_opts.json" \ + --output "$RUN/agent_report.json" +``` + +Inspect actual output filenames before invoking the observer; do not pass an +unexpanded glob or omit a `proxy_log.json` required by the chosen tracker. The +summary helper optionally accepts `--violations-summary` pointing to the direct +checker's `violations_summary.json`. That is a count cross-check, not a claim of +complete result equivalence. Complete evidence stays in the hashed source files. + +## Interpret and act + +- Exit code 0 from the checker means it ran, not that training is correct. Read + `agent_report.json`: observed violations, active passes, dormant rules, unknown + outcomes, skipped variable checks, and observer/checker failures are distinct. + A report with no completed relation decisions provides no checking evidence. +- For an alarm, inspect its `per_invariant` entry in the original coverage file: + the complete `checker_result`, invariant, trace records, and decision identities + identify what differed. The compact summary lists at most ten problem entries; + it preserves the source file and tells you how many remain. +- Keep literal-ID alarms visible and explain their portability limitation. Do + not delete rules, rewrite IDs, or replace missing values to manufacture parity. + Check whether the same alarm appears on a fresh healthy control. + `has_identity_precondition` describes only the observer's identity precondition + clauses. A false value does not establish portability: literal IDs can also + occur in `checker_result.invariant.params`, including stored argument fields + such as `_TRAINCHECK_positional_ID`. Inspect those arguments for the alarm. +- If the task intentionally changes an optimizer, dtype, model shape, or other + learned behavior, re-evaluate whether the reference is applicable. Record that + mismatch explicitly. Do not infer new rules from an unverified candidate and + use those rules to certify the same candidate. +- Make a hypothesis from the evidence, apply a task-permitted fix, rerun the + relevant check, and separately run the task's real evaluator. Report diagnosis, + coverage, result, and total cost without equating fewer alarms with a better + model. If no relevant rule activates, say so and use another diagnostic. + +Keep raw traces, checkpoints, and `env_dump.txt` local to the experiment. The +collector records environment variables; share the selected diagnostic evidence, +not the entire output directory by default. diff --git a/skills/traincheck/scripts/summarize_check.py b/skills/traincheck/scripts/summarize_check.py new file mode 100644 index 00000000..ae6d4860 --- /dev/null +++ b/skills/traincheck/scripts/summarize_check.py @@ -0,0 +1,318 @@ +"""Summarize existing TrainCheck coverage without rerunning or changing its rules. + +Use the TrainCheck checkout on PYTHONPATH. This reader validates the fields used +for its conclusions; complete rule/trace evidence stays in the hashed input file. +It never treats a checker process's exit status as an invariant outcome. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +from tools.fresh_validation.contracts import ( + JSONObject, + boolean, + integer, + json_object, + mapping, + required, + sequence, + string, +) + + +def nullable_bool(value: object, label: str) -> bool | None: + return None if value is None else boolean(value, label) + + +def count(value: object, label: str) -> int: + result = integer(value, label) + if result < 0: + raise ValueError(f"{label} cannot be negative") + return result + + +@dataclass(frozen=True) +class RuleView: + index: int + relation: str + description: str | None + status: str + coverage_status: str + passed: bool | None + active: bool | None + dormant: bool | None + skipped: bool | None + decisions: int | None + has_identity_precondition: bool + + +def read_rule(value: object, position: int) -> RuleView: + rule = mapping(value, f"per_invariant[{position}]") + index = count(required(rule, "invariant_index"), "invariant_index") + if index != position: + raise ValueError("Invariant indices must retain their complete source order") + status = string(required(rule, "status"), "status") + coverage = string(required(rule, "coverage_status"), "coverage_status") + if status not in { + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + }: + raise ValueError(f"Unknown checker status: {status}") + if coverage not in {"observed", "disabled", "unsupported", "error"}: + raise ValueError(f"Unknown coverage status: {coverage}") + passed = nullable_bool(required(rule, "check_passed"), "check_passed") + triggered = nullable_bool(required(rule, "checker_triggered"), "checker_triggered") + active = nullable_bool(required(rule, "active"), "active") + dormant = nullable_bool(required(rule, "dormant"), "dormant") + observed = boolean(required(rule, "coverage_observed"), "coverage_observed") + if status == "completed": + checker = mapping(required(rule, "checker_result"), "checker_result") + mapping(required(checker, "invariant"), "checker_result.invariant") + if passed is None or triggered is None: + raise ValueError("Completed checking requires a known outcome") + if ( + boolean(required(checker, "check_passed"), "checker_result.check_passed") + != passed + or boolean(required(checker, "triggered"), "checker_result.triggered") + != triggered + ): + raise ValueError("Checker result and coverage outcome disagree") + elif passed is not None or triggered is not None: + raise ValueError("Incomplete checking must retain unknown outcomes") + if status in {"checker_error", "observer_error"}: + error = mapping(required(rule, "error"), "error") + string(required(error, "type"), "error.type") + string(required(error, "message"), "error.message") + decisions = None + skipped = None + if coverage == "observed": + if not observed or status not in {"completed", "checker_error"}: + raise ValueError("Observed coverage has an incompatible checker status") + decisions = count(required(rule, "decisions_completed"), "decisions_completed") + if active is not (decisions > 0) or dormant is not ( + status == "completed" and not active + ): + raise ValueError("Activation disagrees with completed relation decisions") + skipped = boolean( + required(rule, "variable_check_skipped"), "variable_check_skipped" + ) + fully_validated = boolean( + required(rule, "fully_validated_active"), "fully_validated_active" + ) + if fully_validated != (active and not skipped and status == "completed"): + raise ValueError("Fully validated status disagrees with activity or skips") + elif observed or active is not None or dormant is not None: + raise ValueError("Unobserved activation must remain unknown") + if status == "unsupported_coverage" and coverage != "unsupported": + raise ValueError("Unsupported coverage must be explicitly marked") + if status == "observer_error" and coverage != "error": + raise ValueError("Observer errors must have error coverage") + description = required(rule, "description") + if description is not None: + description = string(description, "description") + identity = sequence(required(rule, "identity_clauses"), "identity_clauses") + for clause in identity: + mapping(clause, "identity clause") + return RuleView( + index, + string(required(rule, "relation"), "relation"), + description, + status, + coverage, + passed, + active, + dormant, + skipped, + decisions, + bool(identity), + ) + + +def summarize(coverage_value: object) -> JSONObject: + coverage = mapping(coverage_value, "coverage") + if integer(required(coverage, "schema_version"), "schema_version") != 3: + raise ValueError("Expected coverage schema_version 3") + if not boolean(required(coverage, "inputs_unchanged"), "inputs_unchanged"): + raise ValueError("Coverage inputs changed during checking") + rows = count(required(coverage, "trace_rows"), "trace_rows") + hashes = mapping(required(coverage, "input_sha256"), "input_sha256") + if not hashes: + raise ValueError("Coverage requires input hashes") + for path, digest in hashes.items(): + text = string(digest, f"input_sha256[{path}]") + if len(text) != 64 or any(c not in "0123456789abcdef" for c in text): + raise ValueError("Malformed input SHA256") + rules = [ + read_rule(raw, i) + for i, raw in enumerate( + sequence(required(coverage, "per_invariant"), "per_invariant") + ) + ] + counts: Counter[str] = Counter() + problems: list[JSONObject] = [] + for rule in rules: + counts[rule.status] += 1 + counts["observed_active"] += rule.active is True + counts["observed_dormant"] += rule.dormant is True + counts["observed_violations"] += rule.passed is False + counts["observed_active_passes"] += rule.active is True and rule.passed is True + counts["unknown_outcomes"] += rule.passed is None + counts["unobserved_coverage"] += rule.coverage_status != "observed" + counts["skipped_variable_checks"] += rule.skipped is True + counts["completed_decisions"] += 0 if rule.decisions is None else rule.decisions + if rule.passed is not True or rule.active is not True or rule.skipped: + problems.append( + json_object( + { + "invariant_index": rule.index, + "relation": rule.relation, + "description": rule.description, + "status": rule.status, + "coverage_status": rule.coverage_status, + "check_passed": rule.passed, + "active": rule.active, + "dormant": rule.dormant, + "variable_check_skipped": rule.skipped, + "has_identity_precondition": rule.has_identity_precondition, + "evidence_pointer": f"per_invariant[{rule.index}]", + } + ) + ) + source_summary = mapping(required(coverage, "summary"), "summary") + for source_key, derived_key in ( + ("observed_active_invariants", "observed_active"), + ("observed_dormant_invariants", "observed_dormant"), + ("observed_violated_invariants", "observed_violations"), + ("check_outcome_unknown_invariants", "unknown_outcomes"), + ("coverage_unobserved_invariants", "unobserved_coverage"), + ): + if ( + count(required(source_summary, source_key), source_key) + != counts[derived_key] + ): + raise ValueError( + f"Coverage summary disagrees with its rule records: {source_key}" + ) + incomplete = any( + counts[key] + for key in ( + "checker_error", + "observer_error", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "observed_dormant", + ) + ) + if counts["observed_violations"]: + outcome = "violations" + elif not rules or rows == 0 or counts["completed_decisions"] == 0: + outcome = "no_evidence" + elif incomplete: + outcome = "incomplete" + else: + outcome = "active_checks_passed" + return json_object( + { + "schema_version": 1, + "outcome": outcome, + "coverage_incomplete": incomplete, + "trace_rows": rows, + "invariants": len(rules), + "counts": { + key: counts[key] + for key in ( + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + "observed_active", + "observed_active_passes", + "observed_dormant", + "observed_violations", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "completed_decisions", + ) + }, + "problem_rules": problems[:10], + "additional_problem_rules": max(0, len(problems) - 10), + "input_sha256": hashes, + "interpretation": "These are observed rule outcomes, not task success or proof that training is correct.", + } + ) + + +def read_artifact(path: Path) -> tuple[JSONObject, JSONObject]: + contents = path.read_bytes() + value: object = json.loads(contents) + return json_object(value, str(path)), { + "path": str(path.resolve()), + "sha256": hashlib.sha256(contents).hexdigest(), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--coverage", type=Path, required=True) + parser.add_argument("--collection-policy", type=Path) + parser.add_argument("--violations-summary", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(namespace=Options()) + coverage, source = read_artifact(args.coverage) + report = summarize(coverage) + report["coverage_source"] = source + report["collection_mode"] = "unknown: no collection policy supplied" + if args.collection_policy is not None: + policy, policy_source = read_artifact(args.collection_policy) + raw_style = required(policy, "model_tracker_style") + style = None if raw_style is None else string(raw_style, "model_tracker_style") + if style is not None and style not in {"proxy", "subclass", "sampler"}: + raise ValueError("Unsupported model_tracker_style in collection policy") + report["collection_mode"] = ( + "api-only" if style is None else "model-tracker-requested" + ) + report["configured_model_tracker"] = style + report["collection_policy_source"] = policy_source + if args.violations_summary is not None: + direct, direct_source = read_artifact(args.violations_summary) + total = count( + required(direct, "distinct_invariants_violated"), + "distinct_invariants_violated", + ) + if total != len(sequence(required(direct, "violations"), "violations")): + raise ValueError( + "Direct checker summary has an inconsistent violation count" + ) + report_counts = mapping(report["counts"], "counts") + if total != report_counts["observed_violations"]: + raise ValueError("Direct and observed checker violation counts disagree") + report["direct_summary_source"] = direct_source + report["direct_count_check"] = ( + "matched; complete result equivalence not established" + ) + # Never overwrite an earlier report or either input artifact during a retry. + with args.output.open("x") as stream: + stream.write(json.dumps(report, indent=2) + "\n") + print(json.dumps({"outcome": report["outcome"], "counts": report["counts"]})) + return 0 + + +class Options(argparse.Namespace): + coverage: Path + collection_policy: Path | None + violations_summary: Path | None + output: Path + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_fresh_validation_supervisor.py b/tests/test_fresh_validation_supervisor.py new file mode 100644 index 00000000..2cf01d04 --- /dev/null +++ b/tests/test_fresh_validation_supervisor.py @@ -0,0 +1,194 @@ +"""Linux process-tree regressions for the bounded correctness supervisor. + +Set TRAINCHECK_TEST_SUPERVISOR to test an archived CLI against the same external +signal cases. The tests always clean up their own recorded process groups. +""" + +import contextlib +import io +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.fresh_validation import run_phase +from tools.fresh_validation.contracts import boolean, integer, read_object, required + +DEFAULT_SUPERVISOR = Path(run_phase.__file__).resolve() +SUPERVISOR = ( + Path(os.environ["TRAINCHECK_TEST_SUPERVISOR"]) + if "TRAINCHECK_TEST_SUPERVISOR" in os.environ + else DEFAULT_SUPERVISOR +) + +GRANDCHILD = """import os, signal, sys, time +from pathlib import Path +signal.signal(signal.SIGTERM, signal.SIG_IGN) +Path(sys.argv[1]).write_text(str(os.getpid())) +time.sleep(60) +""" +LEADER = """import subprocess, sys, time +from pathlib import Path +child = subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]]) +while not Path(sys.argv[2]).exists(): + time.sleep(0.01) +Path(sys.argv[3]).write_text('ready') +time.sleep(60) +""" + + +def live(pid: int) -> bool: + """An orphan zombie is no longer running; only its parent can reap it.""" + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except FileNotFoundError: + return False + return fields[0] != "Z" + + +def wait_ready(path: Path, timeout: float = 5) -> None: + deadline = time.monotonic() + timeout + while not path.exists(): + if time.monotonic() >= deadline: + raise TimeoutError(f"Child did not become ready: {path}") + time.sleep(0.01) + + +@unittest.skipUnless(sys.platform == "linux", "Supervisor inspects Linux /proc") +class TestPhaseSupervisor(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory[str]() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / "source" + self.source.mkdir() + (self.source / "unchanged.py").write_text("VALUE = 1\n") + + def options(self, command: list[str]) -> run_phase.PhaseOptions: + return run_phase.PhaseOptions( + source=self.source, + output=self.root / "phase", + timeout=20.0, + memory_gib=1.0, + trace_mib=1.0, + command=command, + ) + + def assert_stopped(self, pid: int) -> None: + deadline = time.monotonic() + 3 + while live(pid) and time.monotonic() < deadline: + time.sleep(0.01) + self.assertFalse(live(pid), f"Owned child {pid} survived supervisor cleanup") + + def external_signal(self, signum: signal.Signals, expected: str) -> None: + output = self.root / "phase" + ready, child_pid_file = self.root / "ready", self.root / "grandchild.pid" + command = [ + sys.executable, + str(SUPERVISOR), + "--source", + str(self.source), + "--output", + str(output), + "--timeout", + "30", + "--memory-gib", + "1", + "--trace-mib", + "1", + "--", + sys.executable, + "-c", + LEADER, + GRANDCHILD, + str(child_pid_file), + str(ready), + ] + with (self.root / "supervisor.log").open("wb") as log: + process = subprocess.Popen( + command, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + child_group: int | None = None + try: + wait_ready(ready) + invocation = read_object(output / "invocation.json") + child_group = integer(required(invocation, "pid"), "child PID") + descendant = int(child_pid_file.read_text()) + os.kill(process.pid, signum) + code = process.wait(timeout=8) + self.assert_stopped(descendant) + self.assert_stopped(child_group) + self.assertEqual(code, 1) + result = read_object(output / "phase-result.json") + self.assertEqual(required(result, "stop_reason"), expected) + self.assertTrue( + boolean(required(result, "source_unchanged"), "source unchanged") + ) + self.assertNotEqual( + integer(required(result, "exit_code"), "child exit"), 0 + ) + finally: + # This also cleans the leaked child in the archived buggy CLI. + if child_group is not None: + with contextlib.suppress(ProcessLookupError): + os.killpg(child_group, signal.SIGKILL) + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=3) + + def test_sigterm_cleans_descendant_after_leader_exits(self) -> None: + self.external_signal(signal.SIGTERM, "signal_sigterm") + + def test_sigint_cleans_descendant_after_leader_exits(self) -> None: + self.external_signal(signal.SIGINT, "signal_sigint") + + def test_success_preserves_result_and_restores_handlers(self) -> None: + previous = (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)) + with contextlib.redirect_stdout(io.StringIO()): + result = run_phase.run(self.options([sys.executable, "-c", "pass"])) + self.assertEqual(result, 0) + self.assertEqual( + (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)), + previous, + ) + receipt = read_object(self.root / "phase/phase-result.json") + self.assertIsNone(required(receipt, "stop_reason")) + self.assertEqual(integer(required(receipt, "exit_code"), "child exit"), 0) + self.assertTrue( + boolean(required(receipt, "source_unchanged"), "source unchanged") + ) + + def test_monitor_exception_kills_term_resistant_child_and_restores_handlers( + self, + ) -> None: + ready = self.root / "child.pid" + previous = (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)) + + def failed_monitor(group: int) -> tuple[int, list[int]]: + wait_ready(ready) + raise ValueError("monitor failed") + + try: + with patch.object(run_phase, "group_memory", side_effect=failed_monitor): + with self.assertRaisesRegex(ValueError, "monitor failed"): + run_phase.run( + self.options([sys.executable, "-c", GRANDCHILD, str(ready)]) + ) + self.assert_stopped(int(ready.read_text())) + self.assertEqual( + (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)), + previous, + ) + finally: + if ready.exists(): + with contextlib.suppress(ProcessLookupError): + os.killpg(int(ready.read_text()), signal.SIGKILL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_immutable_metadata_suffix.py b/tests/test_immutable_metadata_suffix.py new file mode 100644 index 00000000..be825ea0 --- /dev/null +++ b/tests/test_immutable_metadata_suffix.py @@ -0,0 +1,217 @@ +"""Immutable namespace caching must preserve live application class lookup.""" + +from __future__ import annotations + +import sys +import types +import unittest +from collections.abc import Mapping +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper, metadata_methods + + +class ImmutableMetadataSuffixTests(unittest.TestCase): + def test_custom_namespace_key_keeps_hook_lookups_on_original_path(self) -> None: + visits: list[object] = [] + + class CustomName(str): + def __hash__(self) -> int: + return str.__hash__(self) + + def __eq__(self, other: object) -> bool: + visits.append(other) + raise AssertionError("Cache construction must not compare custom keys") + + state: dict[object, object] = {CustomName("__getattr__"): False} + namespace: Mapping[object, object] = types.MappingProxyType(state) + namespaces = (namespace,) + mutable, suffix = metadata_methods._immutable_suffix((object,), namespaces) + self.assertIs(mutable, namespaces) + self.assertEqual(suffix.namespaces, ()) + self.assertEqual(visits, []) + + def test_cache_does_not_change_observable_metadata_reference_counts(self) -> None: + tensor = torch.ones(1) + + def descriptor_references(value: torch.Tensor) -> int: + return sys.getrefcount(torch.Tensor.add) + + def name_references(value: torch.Tensor) -> int: + return sys.getrefcount(torch.Tensor.add.__name__) + + selected: dict[str, object] = { + "metadata_native_refcount": None, + "metadata_native_name_refcount": None, + } + with patch.object( + torch.Tensor, + "metadata_native_refcount", + property(descriptor_references), + create=True, + ): + with patch.object( + torch.Tensor, + "metadata_native_name_refcount", + property(name_references), + create=True, + ): + before = dumper.convert_var_to_dict(tensor, False, selected) + additional_cache = metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, + metadata_methods._TENSOR_NAMESPACES, + ) + after = dumper.convert_var_to_dict(tensor, False, selected) + self.assertEqual(after, before) + # Keep the additional cache alive through both observations. + self.assertIsNotNone(additional_cache) + + def test_metaclass_cannot_spoof_immutable_type_flags(self) -> None: + visits: list[str] = [] + + class SpoofedFlags(type): + @property + def __flags__(cls) -> int: + visits.append("flags") + return 1 << 8 + + class Mutable(metaclass=SpoofedFlags): + marker = 1 + + mro, namespaces = metadata_methods._class_layout(Mutable) + mutable, suffix = metadata_methods._immutable_suffix(mro, namespaces) + self.assertEqual(visits, []) + self.assertIs(mutable[0], namespaces[0]) + self.assertTrue(all(view is not namespaces[0] for view in suffix.namespaces)) + Mutable.marker = 2 + self.assertEqual(mutable[0]["marker"], 2) + + def test_mutable_middle_and_head_remain_live(self) -> None: + class Middle(list[object]): + middle_marker = 1 + + class Head(Middle): + head_marker = 2 + + mro, namespaces = metadata_methods._class_layout(Head) + mutable, suffix = metadata_methods._immutable_suffix(mro, namespaces) + self.assertIs(mutable[0], namespaces[0]) + self.assertIs(mutable[1], namespaces[1]) + self.assertTrue(all(view is not namespaces[0] for view in suffix.namespaces)) + self.assertTrue(all(view is not namespaces[1] for view in suffix.namespaces)) + Middle.middle_marker = 3 + self.assertEqual(mutable[1]["middle_marker"], 3) + setattr(Head, "inserted", 4) + self.assertEqual(mutable[0]["inserted"], 4) + delattr(Head, "inserted") + self.assertNotIn("inserted", mutable[0]) + + def assert_uncached_binding( + self, + mutable: tuple[Mapping[object, object], ...], + suffix: metadata_methods._ImmutableSuffix, + ) -> None: + self.assertIs(mutable, metadata_methods._TENSOR_NAMESPACES) + self.assertEqual(suffix.namespaces, ()) + self.assertFalse(suffix.has_getattr) + self.assertFalse(suffix.object_getattribute) + tensor = torch.ones(1) + with patch.object(metadata_methods, "_TENSOR_MUTABLE", mutable): + with patch.object(metadata_methods, "_TENSOR_SUFFIX", suffix): + binding = metadata_methods.tensor_method_binding(tensor, "add") + assert binding is not None + self.assertIs(binding.__self__, tensor) + + def test_other_runtime_uses_original_live_namespaces(self) -> None: + with patch.object( + metadata_methods.platform, "python_implementation", return_value="PyPy" + ): + mutable, suffix = metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, metadata_methods._TENSOR_NAMESPACES + ) + self.assert_uncached_binding(mutable, suffix) + + def test_absent_native_flags_uses_original_live_namespaces(self) -> None: + namespace: dict[str, object] = dict(vars(type)) + del namespace["__flags__"] + with patch.object(metadata_methods, "_TYPE_NAMESPACE", namespace): + mutable, suffix = metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, metadata_methods._TENSOR_NAMESPACES + ) + self.assert_uncached_binding(mutable, suffix) + + def test_malformed_present_flags_is_not_absent(self) -> None: + visits: list[object] = [] + + def application_flags(value: object) -> int: + visits.append(value) + return 1 << 8 + + for malformed in (None, object(), property(application_flags)): + with self.subTest(metadata_type=type(malformed)): + namespace: dict[str, object] = dict(vars(type)) + namespace["__flags__"] = malformed + with patch.object(metadata_methods, "_TYPE_NAMESPACE", namespace): + with self.assertRaises(AssertionError): + metadata_methods._immutable_suffix( + metadata_methods._TENSOR_MRO, + metadata_methods._TENSOR_NAMESPACES, + ) + self.assertEqual(visits, []) + + def test_replacing_and_restoring_inherited_method_is_observed(self) -> None: + tensor = torch.ones(1) + original = metadata_methods.tensor_method_binding(tensor, "add") + assert original is not None + self.assertIs(original.__self__, tensor) + + def replacement(value: torch.Tensor) -> int: + return 37 + + with patch.object(torch.Tensor, "add", property(replacement)): + self.assertIsNone(metadata_methods.tensor_method_binding(tensor, "add")) + self.assertEqual( + dumper.convert_var_to_dict(tensor, False, {"add": None}), {"add": 37} + ) + restored = metadata_methods.tensor_method_binding(tensor, "add") + assert restored is not None + self.assertIs(restored.__self__, tensor) + self.assertIs(type(restored), type(original)) + + def test_new_head_function_binding_and_deletion_are_observed(self) -> None: + tensor = torch.ones(1) + + def inserted(value: torch.Tensor) -> int: + return 41 + + name = "traincheck_native_suffix_test_method" + self.assertIsNone(metadata_methods.tensor_method_binding(tensor, name)) + with patch.object(torch.Tensor, name, inserted, create=True): + binding = metadata_methods.tensor_method_binding(tensor, name) + assert isinstance(binding, types.MethodType) + self.assertIs(binding.__func__, inserted) + self.assertIs(binding.__self__, tensor) + self.assertIsNone(metadata_methods.tensor_method_binding(tensor, name)) + + def test_earlier_property_can_shadow_cached_base_method(self) -> None: + tensor = torch.ones(1) + + def replacement(value: torch.Tensor) -> int: + return 43 + + def prepare(value: torch.Tensor) -> int: + setattr(torch.Tensor, "add", property(replacement)) + return 1 + + with patch.object(torch.Tensor, "add", torch.Tensor.add): + with patch.object(torch.Tensor, "prepare", property(prepare), create=True): + result = dumper.convert_var_to_dict( + tensor, False, {"prepare": None, "add": None} + ) + self.assertEqual(result, {"prepare": 1, "add": 43}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rsi_diagnose_adaptive.py b/tests/test_rsi_diagnose_adaptive.py new file mode 100644 index 00000000..e1a64617 --- /dev/null +++ b/tests/test_rsi_diagnose_adaptive.py @@ -0,0 +1,110 @@ +"""Fail-closed contracts for candidate diagnostic phase results.""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.rsi_pilot.diagnose_adaptive import Driver, main + + +class TestDiagnosticPhases(unittest.TestCase): + def phase_result(self, output: Path, **changes: object) -> None: + result: dict[str, object] = { + "exit_code": 0, + "stop_reason": None, + "source_unchanged": True, + } + result.update(changes) + (output / "phase-result.json").write_text(json.dumps(result)) + + def test_checker_errors_can_be_reported_but_collection_errors_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + driver = Driver(output, Path("/python")) + self.phase_result(output, exit_code=1) + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 1) + ): + driver.phase( + output, ["checker.py"], timeout=90, allow_checker_failure=True + ) + with self.assertRaisesRegex(RuntimeError, "Diagnostic phase failed"): + driver.phase(output, ["collector.py"], timeout=90) + + def test_stopped_or_changed_source_never_becomes_checker_result(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + driver = Driver(output, Path("/python")) + cases: tuple[dict[str, object], ...] = ( + {"stop_reason": "timeout"}, + {"source_unchanged": False}, + {"exit_code": -15}, + ) + for changes in cases: + self.phase_result(output, **changes) + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 0) + ): + with self.assertRaisesRegex( + RuntimeError, "Diagnostic phase failed" + ): + driver.phase( + output, + ["checker.py"], + timeout=90, + allow_checker_failure=True, + ) + + def test_missing_required_result_field_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + (output / "phase-result.json").write_text('{"exit_code":0}') + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 0) + ): + with self.assertRaises((ValueError, KeyError)): + Driver(output, Path("/python")).phase( + output, ["check.py"], timeout=90 + ) + + def test_candidate_config_is_required(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + with self.assertRaisesRegex(FileNotFoundError, "configuration missing"): + Driver(output, Path("/python")).diagnose(output) + self.assertFalse((output / "diagnostics").exists()) + + def test_cli_preserves_virtualenv_python_symlink(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + base_python = root / "base-python" + base_python.touch() + venv_python = root / "venv-python" + venv_python.symlink_to(base_python) + argv = [ + "diagnose_adaptive.py", + "--study", + str(root), + "--python", + str(venv_python), + str(root / "candidate"), + ] + with ( + patch("sys.argv", argv), + patch( + "tools.rsi_pilot.diagnose_adaptive.Driver", autospec=True + ) as driver, + patch("builtins.print"), + ): + driver.return_value.diagnose.return_value = root / "report.json" + main() + driver.assert_called_once_with(root.resolve(), venv_python, "8,10,12,14") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rsi_pilot_analysis.py b/tests/test_rsi_pilot_analysis.py new file mode 100644 index 00000000..53b305c8 --- /dev/null +++ b/tests/test_rsi_pilot_analysis.py @@ -0,0 +1,585 @@ +"""Evidence failures must prevent an apparently successful pilot comparison.""" + +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from tools.fresh_validation.contracts import JSONObject, JSONValue, mapping, read_object +from tools.rsi_pilot.analyze_trials import ( + analyze, + begin_scoring, + collect, + digest, + freeze, + metric, +) +from tools.rsi_pilot.run_candidate import configure + +ORDER = ("B1C", "B1T", "B2T", "B2C") + + +def write(path: Path, value: JSONObject) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, allow_nan=False) + "\n") + + +def depths(accuracy: float, include_eight: bool = True) -> list[JSONValue]: + return [ + { + "loops": depth, + "macro_accuracy_percent": accuracy, + "macro_mean_dense_matmul_flops": work, + } + for depth, work in ((1, 805760), (2, 1610624), (4, 3220352), (8, 6439808)) + if depth != 8 or include_eight + ] + + +class TestPilotAnalysis(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory[str]() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.profile = self.root / "frozen-profile.toml" + self.profile.write_text("[training]\nsteps = 10000\n# PILOT_RESEARCH_SOURCE\n") + self.baseline = "def loss():\n return 1\n" + self.baseline_config: JSONObject = { + "model": {"width": 64}, + "training": {"steps": 10000, "research_source": self.baseline}, + } + write(self.root / "selected/baseline/config.json", self.baseline_config) + (self.root / "selected/baseline/model.safetensors").write_bytes( + b"baseline model" + ) + write( + self.root / "study_protocol.json", + { + "revised_at": "2026-10-30T00:00:00+00:00", + "revision_reason": "Restore omitted free validation before the scored pilot.", + "trial_order": list(ORDER), + "pairs": [ + {"control": "B1C", "traincheck": "B1T"}, + {"control": "B2C", "traincheck": "B2T"}, + ], + "excluded_setup_trials": ["C1"], + }, + ) + write( + self.root / "heldout-evaluation-plan.json", + {"seed": 20261030, "requested_examples_per_bin": 512}, + ) + for trial_id in (*ORDER, "C1"): + self.make_trial(trial_id) + source = ( + Path(__file__).resolve().parents[1] / "tools/rsi_pilot/run_candidate.py" + ) + write( + self.root / "harness-freeze-B.json", + {"files": {"tools/rsi_pilot/run_candidate.py": digest(source)}}, + ) + inventory: JSONObject = { + trial_id: { + "baseline_research.py": digest( + self.root / "trials" / trial_id / "workspace/baseline_research.py" + ) + } + for trial_id in ORDER + } + write( + self.root / "solver-packet-manifest-B.json", {"packet_inventory": inventory} + ) + write( + self.root / "solver-packet-manifest.json", + {"baseline_remote_training": "/remote/baseline"}, + ) + self.save_origins() + + def save_origins(self) -> None: + artifacts: JSONObject = {} + for trial_id in ("baseline", *ORDER): + remote = "/remote/baseline" + if trial_id != "baseline": + submission = read_object( + self.root / "trials" / trial_id / "workspace/submission.json" + ) + candidate = submission["candidate"] + assert isinstance(candidate, str) + if candidate != "baseline": + remote = f"/remote/trials/{trial_id}/{candidate}/training" + artifacts[trial_id] = { + "remote_directory": remote, + "files": { + name: digest(self.root / "selected" / trial_id / name) + for name in ("config.json", "model.safetensors") + }, + } + write( + self.root / "selected/origins.json", + {"schema_version": 1, "artifacts": artifacts}, + ) + + def make_trial(self, trial_id: str) -> None: + trial = self.root / "trials" / trial_id + workspace = trial / "workspace" + workspace.mkdir(parents=True) + arm = "traincheck" if trial_id.endswith("T") else "control" + write( + trial / "trial.json", + { + "workspace": str(workspace), + "arm": arm, + "max_candidates": 2, + "max_diagnostics": int(arm == "traincheck"), + "expected_steps": 10000, + "profile": str(self.profile), + "profile_sha256": digest(self.profile), + "remote_trial": f"/remote/trials/{trial_id}", + }, + ) + (workspace / "baseline_research.py").write_text(self.baseline) + (workspace / "assignment.txt").write_text( + "Improve the objective within this trial's budget." + ) + (workspace / "summary.md").write_text( + "Selected the baseline after inspecting development results." + ) + write(workspace / "baseline_development.json", {"final": depths(20.0)}) + write(workspace / "submission.json", {"candidate": "baseline"}) + write(self.root / "selected" / trial_id / "config.json", self.baseline_config) + (self.root / "selected" / trial_id / "model.safetensors").write_bytes( + b"baseline model" + ) + + def add_run( + self, + trial_id: str, + *, + successful: bool, + diagnose: bool = False, + crlf: bool = False, + ) -> Path: + trial = self.root / "trials" / trial_id + manifest = read_object(trial / "trial.json") + run = trial / "runs/candidate-01" + run.mkdir(parents=True) + program = "def loss():\n return 2\n" + if crlf: + program = program.replace("\n", "\r\n") + (run / "program.py").write_bytes(program.encode()) + (run / "config.toml").write_text(configure(self.profile.read_text(), program)) + receipt: JSONObject = { + "candidate": "candidate-01", + "arm": manifest["arm"], + "diagnostics_requested": diagnose, + "program_sha256": digest(run / "program.py"), + "profile_sha256": digest(self.profile), + "trial_manifest_sha256": digest(trial / "trial.json"), + "candidates_reserved": 1, + "diagnostics_reserved": int(diagnose), + "status": "training_complete" if successful else "training_failed", + "training_exit_code": 0 if successful else 1, + "errors": ( + [] + if successful + else [ + { + "phase": "training", + "type": "ProcessExit", + "message": "Training failed", + } + ] + ), + "operational_seconds_including_transfer_and_diagnostics": 8.0, + } + if successful: + rows = depths(40.0) + receipt.update( + { + "completed_steps": 10000, + "training_seconds": 5.0, + "development_depths": rows, + } + ) + write( + run / "results.json", + { + "completed_steps": 10000, + "training_seconds": 5.0, + "device_backend": "cpu", + "final": rows, + }, + ) + write(trial / "workspace/submission.json", {"candidate": "candidate-01"}) + write( + self.root / "selected" / trial_id / "config.json", + { + "model": {"width": 64}, + "training": {"steps": 10000, "research_source": program}, + }, + ) + (self.root / "selected" / trial_id / "model.safetensors").write_bytes( + b"candidate model" + ) + if diagnose: + receipt.update( + { + "diagnostic_status": "execution_failed", + "diagnostic_exit_code": 1, + "errors": [ + { + "phase": "diagnostics", + "type": "ProcessExit", + "message": "Diagnostic failed", + } + ], + } + ) + write(run / "run.json", receipt) + write( + trial / "usage.json", + {"candidates_reserved": 1, "diagnostics_reserved": int(diagnose)}, + ) + self.save_origins() + return run + + def scores(self) -> None: + freeze(self.root) + begin_scoring(self.root) + for trial_id in ("baseline", *ORDER): + raw = depths(30.0, include_eight=False) + values = metric(raw, [1, 2, 4]) + selected = self.root / "selected" / trial_id + write( + self.root / "scores" / (trial_id + ".json"), + { + "kind": "reduced_compute_pilot_not_official_RSI_score", + "split": "test", + "fixed_policy_depths": [1, 2, 4, 4], + "seed": 20261030, + "requested_examples_per_bin": 512, + "actual_examples": 2048, + "checkpoint_sha256": digest(selected / "model.safetensors"), + "config_sha256": digest(selected / "config.json"), + "panel_sha256": hashlib.sha256(b"shared heldout panel").hexdigest(), + "depths": raw, + "reward": values["reward"], + }, + ) + + def test_missing_trial_or_submission_fails(self) -> None: + path = self.root / "trials/B2C/workspace/submission.json" + path.unlink() + with self.assertRaises(FileNotFoundError): + freeze(self.root) + + def test_untrained_selected_candidate_fails(self) -> None: + write( + self.root / "trials/B1C/workspace/submission.json", + {"candidate": "candidate-02"}, + ) + with self.assertRaisesRegex(ValueError, "no validated completed training"): + freeze(self.root) + + def test_unknown_failed_training_time_is_separate(self) -> None: + self.add_run("C1", successful=False) + self.scores() + result = analyze(self.root) + setup = result["excluded_setup_trials"] + self.assertIsInstance(setup, list) + assert isinstance(setup, list) + first = mapping(setup[0], "setup") + self.assertEqual(first["training_seconds_known"], 0.0) + self.assertEqual(first["training_duration_unknown_attempts"], ["candidate-01"]) + self.assertEqual( + first["operational_seconds_including_queue_transfer_diagnostics"], 8.0 + ) + + def test_diagnostic_failure_does_not_discard_trained_selection(self) -> None: + self.add_run("B1T", successful=True, diagnose=True, crlf=True) + self.scores() + reports = analyze(self.root)["trials"] + assert isinstance(reports, list) + treatment = mapping(reports[1], "treatment") + self.assertEqual(treatment["selection"], "candidate-01") + self.assertEqual(treatment["diagnostic_execution_failures"], 1) + self.assertEqual(treatment["diagnostic_skip_counts_unknown"], 1) + self.assertEqual(treatment["training_seconds_known"], 5.0) + + def test_bad_ledger_is_not_repaired(self) -> None: + self.add_run("B1C", successful=False) + write( + self.root / "trials/B1C/usage.json", + {"candidates_reserved": True, "diagnostics_reserved": 0}, + ) + with self.assertRaisesRegex(ValueError, "must be an integer"): + freeze(self.root) + + def test_changed_program_or_manifest_hash_fails(self) -> None: + run = self.add_run("B1C", successful=True) + (run / "program.py").write_text("changed source") + with self.assertRaisesRegex(ValueError, "program_sha256 mismatch"): + freeze(self.root) + + def test_unfinished_receipt_fails(self) -> None: + run = self.add_run("B1C", successful=False) + receipt = read_object(run / "run.json") + receipt["status"] = "started" + write(run / "run.json", receipt) + with self.assertRaisesRegex(ValueError, "has not finished"): + freeze(self.root) + + def test_missing_operational_time_is_not_zero(self) -> None: + run = self.add_run("B1C", successful=False) + receipt = read_object(run / "run.json") + del receipt["operational_seconds_including_transfer_and_diagnostics"] + write(run / "run.json", receipt) + with self.assertRaisesRegex(ValueError, "Missing required field"): + freeze(self.root) + + def test_selected_checkpoint_mismatch_fails(self) -> None: + self.scores() + score_path = self.root / "scores/B1C.json" + score = read_object(score_path) + score["checkpoint_sha256"] = "0" * 64 + write(score_path, score) + with self.assertRaisesRegex(ValueError, "checkpoint_sha256 mismatch"): + analyze(self.root) + + def test_wrong_downloaded_checkpoint_fails_before_finalization(self) -> None: + self.add_run("B1C", successful=True) + (self.root / "selected/B1C/model.safetensors").write_bytes(b"wrong model") + with self.assertRaisesRegex(ValueError, "remote artifact hash mismatch"): + freeze(self.root) + + def test_origin_directory_and_required_fields_are_checked(self) -> None: + self.add_run("B1C", successful=True) + path = self.root / "selected/origins.json" + original = path.read_bytes() + for change in ("wrong_directory", "missing_hash", "missing_trial"): + with self.subTest(change=change): + path.write_bytes(original) + origins = read_object(path) + artifacts = origins["artifacts"] + assert isinstance(artifacts, dict) + candidate = artifacts["B1C"] + assert isinstance(candidate, dict) + if change == "wrong_directory": + candidate["remote_directory"] = ( + "/remote/trials/B1C/candidate-02/training" + ) + elif change == "missing_hash": + hashes = candidate["files"] + assert isinstance(hashes, dict) + del hashes["config.json"] + else: + del artifacts["B1T"] + write(path, origins) + with self.assertRaises(ValueError): + freeze(self.root) + + def test_origin_evidence_is_frozen(self) -> None: + self.scores() + path = self.root / "selected/origins.json" + origins = read_object(path) + origins["host"] = "changed host after finalization" + write(path, origins) + with self.assertRaisesRegex(ValueError, "frozen file hash mismatch"): + analyze(self.root) + + def test_completed_diagnostic_requires_frozen_reference(self) -> None: + run = self.add_run("B1T", successful=True, diagnose=True) + invariant_path = self.root / "trials/B1T/workspace/invariants.json" + invariant_path.write_text("[]\n") + write(run / "coverage.json", {"schema_version": 3, "per_invariant": []}) + diagnostic: JSONObject = { + "schema_version": 1, + "status": "complete", + "candidate_config_sha256": digest(run / "config.toml"), + "invariants_sha256": digest(invariant_path), + } + write(run / "diagnostic_manifest.json", diagnostic) + report: JSONObject = { + "schema_version": 1, + "outcome": "no_evidence", + "coverage_incomplete": True, + "coverage_source": { + "path": "/remote/coverage.json", + "sha256": digest(run / "coverage.json"), + }, + "counts": { + key: 0 + for key in ( + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + "observed_active", + "observed_active_passes", + "observed_dormant", + "observed_violations", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "completed_decisions", + ) + }, + } + write(run / "agent_report.json", report) + receipt = read_object(run / "run.json") + receipt.update( + diagnostic_status="complete", + diagnostic_exit_code=0, + errors=[], + traincheck_report=report, + ) + write(run / "run.json", receipt) + _, _, frozen_files = collect(self.root) + self.assertIn(invariant_path, frozen_files) + diagnostic["invariants_sha256"] = "0" * 64 + write(run / "diagnostic_manifest.json", diagnostic) + with self.assertRaisesRegex( + ValueError, "diagnostic reference invariants mismatch" + ): + collect(self.root) + del diagnostic["invariants_sha256"] + write(run / "diagnostic_manifest.json", diagnostic) + with self.assertRaisesRegex(ValueError, "Missing required field"): + collect(self.root) + + def test_false_or_fabricated_reward_fails(self) -> None: + self.scores() + path = self.root / "scores/B1T.json" + score = read_object(path) + for reward in (False, 100.0): + score["reward"] = reward + write(path, score) + with self.assertRaises(ValueError): + analyze(self.root) + + def test_altered_frozen_assignment_fails(self) -> None: + self.scores() + (self.root / "trials/B1T/workspace/assignment.txt").write_text( + "Changed after selection" + ) + with self.assertRaisesRegex(ValueError, "frozen file hash mismatch"): + analyze(self.root) + + def test_scoring_before_finalization_is_rejected(self) -> None: + self.scores() + path = self.root / "scores/provenance.json" + provenance = read_object(path) + provenance["scoring_started_at"] = "2026-01-01T00:00:00+00:00" + write(path, provenance) + with self.assertRaisesRegex(ValueError, "must follow finalization"): + analyze(self.root) + + def test_duplicate_pair_or_setup_overlap_is_rejected(self) -> None: + path = self.root / "study_protocol.json" + record = read_object(path) + record["excluded_setup_trials"] = ["B1C"] + write(path, record) + with self.assertRaisesRegex(ValueError, "overlapping trial IDs"): + collect(self.root) + + def test_scores_cannot_exist_when_freezing(self) -> None: + write(self.root / "scores/B1C.json", {}) + with self.assertRaisesRegex(ValueError, "already exist"): + freeze(self.root) + + def test_preflight_cost_is_separate_and_does_not_create_training_budget( + self, + ) -> None: + trial = self.root / "trials/B1C" + path = trial / "preflights/preflight-01" + path.mkdir(parents=True) + (path / "program.py").write_text(self.baseline) + (path / "config.toml").write_text( + configure(self.profile.read_text(), self.baseline) + ) + backend: JSONObject = { + "schema_version": 1, + "valid": False, + "error": {"type": "ValueError", "message": "Unsupported syntax"}, + } + write(path / "result.json", backend) + write( + path / "preflight.json", + { + "schema_version": 1, + "preflight": "preflight-01", + "arm": "control", + "profile_sha256": digest(self.profile), + "trial_manifest_sha256": digest(trial / "trial.json"), + "program_sha256": digest(path / "program.py"), + "status": "program_invalid", + "errors": [{"phase": "validate_result", "error": backend["error"]}], + "check_exit_code": 2, + "valid": False, + "result": backend, + "operational_seconds_including_transfer": 1.5, + }, + ) + self.scores() + reports = analyze(self.root)["trials"] + assert isinstance(reports, list) + report = mapping(reports[0], "trial") + self.assertEqual(report["attempts"], []) + self.assertEqual(report["operational_seconds_including_preflights"], 1.5) + self.assertFalse((trial / "usage.json").exists()) + (path / "program.py").write_text("changed after finalization") + with self.assertRaisesRegex(ValueError, "program_sha256 mismatch"): + analyze(self.root) + + def test_incomplete_preflight_is_not_a_success(self) -> None: + path = self.root / "trials/B1C/preflights/preflight-01" + write(path / "preflight.json", {"status": "started"}) + with self.assertRaises(ValueError): + freeze(self.root) + + def test_baseline_selection_must_match_reference_score(self) -> None: + self.scores() + path = self.root / "scores/B1C.json" + score = read_object(path) + score["depths"] = depths(80.0, include_eight=False) + score["reward"] = metric(score["depths"], [1, 2, 4])["reward"] + write(path, score) + with self.assertRaisesRegex(ValueError, "baseline selection score mismatch"): + analyze(self.root) + + def test_score_panel_size_must_match(self) -> None: + self.scores() + path = self.root / "scores/B1C.json" + score = read_object(path) + score["actual_examples"] = 100 + write(path, score) + with self.assertRaisesRegex(ValueError, "test panel size mismatch"): + analyze(self.root) + + def test_harness_inventory_mismatch_fails(self) -> None: + write( + self.root / "harness-freeze-B.json", + {"files": {"tools/rsi_pilot/run_candidate.py": "0" * 64}}, + ) + with self.assertRaisesRegex(ValueError, "inventory hash mismatch"): + freeze(self.root) + + def test_complete_training_without_results_is_not_success(self) -> None: + run = self.add_run("B1C", successful=True) + (run / "results.json").unlink() + with self.assertRaisesRegex(ValueError, "no validated results"): + freeze(self.root) + + def test_missing_requested_diagnostic_is_not_empty_success(self) -> None: + run = self.add_run("B1T", successful=True, diagnose=True) + receipt = read_object(run / "run.json") + del receipt["diagnostic_status"] + del receipt["diagnostic_exit_code"] + write(run / "run.json", receipt) + with self.assertRaisesRegex(ValueError, "missing its requested diagnostic"): + freeze(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rsi_pilot_client.py b/tests/test_rsi_pilot_client.py new file mode 100644 index 00000000..87913856 --- /dev/null +++ b/tests/test_rsi_pilot_client.py @@ -0,0 +1,487 @@ +"""Boundary checks for the pilot client; training is tested on the real runtime.""" + +import hashlib +import json +import shlex +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.fresh_validation.contracts import JSONObject, JSONValue, read_object +from tools.rsi_pilot.run_candidate import ( + check_program, + configure, + execute, + main, + program_text, + remote_command, + run, + train_arguments, + validate_diagnostic_artifacts, + validate_diagnostics, + validate_preflight, + validate_results, +) + + +def trial_fixture(root: Path, arm: str = "control") -> tuple[Path, Path]: + trial, workspace = root / "trial", root / "solver" + trial.mkdir() + workspace.mkdir() + program = workspace / "research.py" + program.write_text( + "def loss(logits, tokens, labels, step, total_steps):\n return logits.sum()\n" + ) + profile = root / "profile.toml" + profile.write_text("[training]\nsteps = 5\n# PILOT_RESEARCH_SOURCE\n") + manifest = { + "workspace": str(workspace), + "arm": arm, + "profile": str(profile), + "profile_sha256": hashlib.sha256(profile.read_bytes()).hexdigest(), + "max_candidates": 3, + "max_diagnostics": 1 if arm == "traincheck" else 0, + "expected_steps": 5, + "host": "unused-host", + "remote_trial": "/experiment/trial", + "remote_workspace": "/experiment/workspace", + "remote_overlay": "/experiment/overlay", + "remote_python": "/experiment/python", + "affinity": "8,10,12,14", + "diagnostic_driver": "/experiment/diagnose", + "remote_compute_lock": "/experiment/compute.lock", + } + (trial / "trial.json").write_text(json.dumps(manifest)) + return trial, program + + +def training_result() -> JSONObject: + return { + "completed_steps": 5, + "training_seconds": 0.25, + "device_backend": "cpu", + "final": [{"loops": 1, "macro_accuracy_percent": 20.0}], + } + + +class PilotClientTests(unittest.TestCase): + def test_preflight_valid_and_invalid_programs_do_not_charge_budgets(self) -> None: + cases: list[tuple[int, JSONObject, str]] = [ + (0, {"schema_version": 1, "valid": True, "error": None}, "program_valid"), + ( + 2, + { + "schema_version": 1, + "valid": False, + "error": { + "type": "ValueError", + "message": "unsupported research constant", + }, + }, + "program_invalid", + ), + ] + for arm in ("control", "traincheck"): + for code, payload, status in cases: + with ( + self.subTest(arm=arm, status=status), + tempfile.TemporaryDirectory() as temporary, + ): + trial, program = trial_fixture(Path(temporary), arm) + + def provide_result( + manifest: JSONObject, source: str, target: Path, log: Path + ) -> None: + target.write_text(json.dumps(payload)) + + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch( + "tools.rsi_pilot.run_candidate.download", + side_effect=provide_result, + ), + patch( + "tools.rsi_pilot.run_candidate.execute", return_value=code + ) as executor, + ): + result = check_program(trial, program) + self.assertFalse((trial / "usage.json").exists()) + ledger = ( + b'{"candidates_reserved": 3, "diagnostics_reserved": 1}\n' + ) + (trial / "usage.json").write_bytes(ledger) + repeated = check_program(trial, program) + self.assertEqual((trial / "usage.json").read_bytes(), ledger) + self.assertFalse((trial / "runs").exists()) + self.assertEqual(result["status"], status) + self.assertEqual(repeated["preflight"], "preflight-02") + saved = read_object( + trial / "preflights/preflight-01/preflight.json" + ) + self.assertEqual(saved, result) + self.assertEqual(saved["result"], payload) + self.assertNotIn("latent_reasoning.train", str(executor.call_args)) + + def test_preflight_backend_failure_is_not_program_rejection(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch("tools.rsi_pilot.run_candidate.execute", return_value=1), + patch("tools.rsi_pilot.run_candidate.download") as downloader, + ): + result = check_program(trial, program) + downloader.assert_not_called() + self.assertEqual(result["status"], "execution_failed") + self.assertNotIn("valid", result) + self.assertFalse((trial / "usage.json").exists()) + + def test_preflight_transfer_failure_is_saved_without_budget_charge(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch( + "tools.rsi_pilot.run_candidate.upload", + side_effect=OSError("connection lost"), + ), + ): + with self.assertRaises(OSError): + check_program(trial, program) + receipt = read_object(trial / "preflights/preflight-01/preflight.json") + self.assertEqual(receipt["status"], "client_error") + self.assertIn("connection lost", json.dumps(receipt["errors"])) + self.assertFalse((trial / "usage.json").exists()) + + def test_preflight_rejects_malformed_or_contradictory_receipts(self) -> None: + cases: list[tuple[JSONObject, int]] = [ + ({}, 0), + ({"schema_version": 1, "valid": 1, "error": None}, 0), + ({"schema_version": 1, "valid": True}, 0), + ({"schema_version": 1, "valid": True, "error": None}, 2), + ({"schema_version": 1, "valid": False, "error": None}, 2), + ( + { + "schema_version": 1, + "valid": False, + "error": {"type": "ImportError", "message": "torch missing"}, + }, + 2, + ), + ] + for payload, code in cases: + with self.subTest(payload=payload), self.assertRaises(ValueError): + validate_preflight(payload, code) + + def test_check_program_cli_is_exclusive_and_preserves_rejection_status( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + arguments = [ + "run_candidate", + "--trial", + str(root), + "--program", + str(root / "program.py"), + "--check-program", + ] + with ( + patch("sys.argv", arguments), + patch( + "tools.rsi_pilot.run_candidate.check_program", + return_value={"status": "program_invalid", "errors": []}, + ), + patch("tools.rsi_pilot.run_candidate.run") as trainer, + ): + self.assertEqual(main(), 2) + trainer.assert_not_called() + with patch("sys.argv", [*arguments, "--diagnose"]): + with self.assertRaises(SystemExit) as failure: + main() + self.assertEqual(failure.exception.code, 2) + + def test_program_is_data_not_shell_code(self) -> None: + source = 'def loss(logits, tokens, labels, step, total_steps):\n return "`touch sentinel` $HOME"\n' + config = configure("[training]\n# PILOT_RESEARCH_SOURCE\n", source) + self.assertIn('research_source = "def loss', config) + self.assertIn("\\n", config) + self.assertNotIn("\n return", config) + arguments = ["python", "path with spaces", "$(touch sentinel)", "x'y"] + command = remote_command({"host": "labmach"}, arguments) + self.assertEqual(shlex.split(command[-1]), arguments) + + def test_profile_marker_is_required_and_unique(self) -> None: + for profile in ("", "# PILOT_RESEARCH_SOURCE\n# PILOT_RESEARCH_SOURCE"): + with self.assertRaises(ValueError): + configure(profile, "pass") + + def test_unicode_source_is_not_encoded_as_invalid_toml_surrogates(self) -> None: + source = 'def loss(logits, tokens, labels, step, total_steps):\n "😀"\n return logits.sum()\n' + encoded = configure("# PILOT_RESEARCH_SOURCE", source) + self.assertNotIn("\\ud83d", encoded) + self.assertEqual(json.loads(encoded.removeprefix("research_source = ")), source) + + def test_only_contained_small_utf8_programs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + workspace = root / "solver" + workspace.mkdir() + program = workspace / "research.py" + program.write_text("pass\n") + self.assertEqual(program_text(program, workspace), "pass\n") + outside = root / "outside.py" + outside.write_text("pass\n") + with self.assertRaises(ValueError): + program_text(outside, workspace) + program.write_bytes(b"x" * 32769) + with self.assertRaises(ValueError): + program_text(program, workspace) + program.write_bytes(b"\xff") + with self.assertRaises(UnicodeDecodeError): + program_text(program, workspace) + + def test_failed_uploads_are_charged_and_have_nonempty_error_receipts(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch( + "tools.rsi_pilot.run_candidate.upload", + side_effect=subprocess.CalledProcessError(255, ["scp"]), + ), + ): + for index in range(1, 4): + with self.assertRaises(subprocess.CalledProcessError): + run(trial, program, False) + receipt = read_object( + trial / "runs" / f"candidate-{index:02d}" / "run.json" + ) + self.assertEqual(receipt["status"], "client_error") + self.assertIn( + '"phase": "upload_config"', json.dumps(receipt["errors"]) + ) + self.assertIn( + "non-zero exit status 255", json.dumps(receipt["errors"]) + ) + with self.assertRaisesRegex(ValueError, "Candidate budget exhausted"): + run(trial, program, False) + self.assertEqual( + read_object(trial / "usage.json"), + { + "candidates_reserved": 3, + "diagnostics_reserved": 0, + }, + ) + + def test_one_diagnostic_attempt_is_reserved_even_when_transfer_fails(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary), "traincheck") + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch( + "tools.rsi_pilot.run_candidate.upload", + side_effect=OSError("connection lost"), + ), + ): + with self.assertRaises(OSError): + run(trial, program, True) + with self.assertRaisesRegex(ValueError, "Diagnostic budget exhausted"): + run(trial, program, True) + self.assertEqual( + read_object(trial / "usage.json"), + { + "candidates_reserved": 1, + "diagnostics_reserved": 1, + }, + ) + + def test_control_cannot_request_diagnostics(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with self.assertRaisesRegex(ValueError, "not enabled"): + run(trial, program, True) + self.assertFalse((trial / "usage.json").exists()) + + def test_remote_training_failure_remains_failure_without_results(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch("tools.rsi_pilot.run_candidate.execute", return_value=124), + patch("tools.rsi_pilot.run_candidate.download") as downloader, + ): + result = run(trial, program, False) + downloader.assert_not_called() + self.assertEqual(result["status"], "training_failed") + self.assertNotIn("development_depths", result) + self.assertIn("Training exited 124", json.dumps(result["errors"])) + + def test_failed_result_download_has_receipt_and_does_not_claim_completion( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + with ( + patch("tools.rsi_pilot.run_candidate.checked_execute"), + patch("tools.rsi_pilot.run_candidate.upload"), + patch("tools.rsi_pilot.run_candidate.execute", return_value=0), + patch( + "tools.rsi_pilot.run_candidate.download", + side_effect=OSError("download lost"), + ), + ): + with self.assertRaises(OSError): + run(trial, program, False) + receipt = read_object(trial / "runs/candidate-01/run.json") + self.assertEqual(receipt["status"], "client_error") + self.assertEqual(receipt["training_exit_code"], 0) + self.assertNotIn("development_depths", receipt) + self.assertIn("download_results", json.dumps(receipt["errors"])) + + def test_invalid_result_metadata_is_not_accepted_as_success(self) -> None: + manifest: JSONObject = {"expected_steps": 5} + valid = training_result() + self.assertEqual(validate_results(valid, manifest)[0], 5) + changes: list[tuple[str, JSONValue]] = [ + ("completed_steps", True), + ("completed_steps", 4), + ("training_seconds", float("nan")), + ("training_seconds", -1), + ("device_backend", "cuda"), + ("unrelated_metric", float("nan")), + ("final", []), + ("final", [{"loops": 1, "macro_accuracy_percent": float("inf")}]), + ] + for key, value in changes: + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + validate_results({**valid, key: value}, manifest) + del valid["completed_steps"] + with self.assertRaises(ValueError): + validate_results(valid, manifest) + + def test_malformed_diagnostic_cannot_be_successfully_downloaded_evidence( + self, + ) -> None: + with self.assertRaises(ValueError): + validate_diagnostics({}) + with self.assertRaises(ValueError): + validate_diagnostics({"schema_version": 1, "outcome": "success"}) + + def test_cli_reports_nonzero_for_a_recorded_training_failure(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + with ( + patch( + "sys.argv", + [ + "run_candidate", + "--trial", + str(root), + "--program", + str(root / "program.py"), + ], + ), + patch( + "tools.rsi_pilot.run_candidate.run", + return_value={ + "status": "training_failed", + "errors": [{"message": "Training exited 1"}], + }, + ), + ): + self.assertEqual(main(), 1) + + def test_malformed_usage_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, program = trial_fixture(Path(temporary)) + (trial / "usage.json").write_text( + '{"candidates_reserved": false, "diagnostics_reserved": 0}' + ) + with self.assertRaises(ValueError): + run(trial, program, False) + self.assertFalse((trial / "runs/candidate-01").exists()) + + def test_diagnostic_drilldown_requires_matching_bytes_and_candidate(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + coverage_path = output / "coverage.json" + coverage_path.write_text('{"schema_version": 3, "per_invariant": []}\n') + config = output / "config.toml" + config.write_text("[training]\nsteps = 5\n") + diagnostic: JSONObject = { + "schema_version": 1, + "status": "complete", + "candidate_config_sha256": hashlib.sha256( + config.read_bytes() + ).hexdigest(), + } + (output / "diagnostic_manifest.json").write_text(json.dumps(diagnostic)) + source_path = "/remote/candidate/diagnostics/coverage.json" + report: JSONObject = { + "coverage_source": { + "path": source_path, + "sha256": hashlib.sha256(coverage_path.read_bytes()).hexdigest(), + } + } + evidence = validate_diagnostic_artifacts(report, output) + self.assertIn(str(coverage_path), json.dumps(evidence)) + self.assertIn(source_path, json.dumps(report)) + config.write_text("[training]\nsteps = 6\n") + with self.assertRaisesRegex(ValueError, "different candidate"): + validate_diagnostic_artifacts(report, output) + coverage_path.write_text("{}\n") + with self.assertRaisesRegex(ValueError, "does not match"): + validate_diagnostic_artifacts(report, output) + + def test_missing_diagnostic_drilldown_is_an_error(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(FileNotFoundError): + validate_diagnostic_artifacts({}, Path(temporary)) + + def test_cpu_thread_settings_match_calibration(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + trial, _ = trial_fixture(Path(temporary)) + command = train_arguments(read_object(trial / "trial.json"), "/run") + for setting in ( + "OMP_NUM_THREADS=4", + "MKL_NUM_THREADS=4", + "OPENBLAS_NUM_THREADS=1", + ): + self.assertIn(setting, command) + self.assertEqual(command[-3:], ["--device", "cpu", "--require-complete"]) + self.assertEqual( + command[:4], ["flock", "-w", "360", "/experiment/compute.lock"] + ) + self.assertIn("300s", command) + + def test_timeout_stops_owned_descendants(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + sentinel = root / "descendant-ran" + child = ( + "import pathlib,time; time.sleep(2); pathlib.Path(" + + repr(str(sentinel)) + + ").write_text('leaked')" + ) + parent = ( + "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c'," + + repr(child) + + "]); time.sleep(30)" + ) + code = execute([sys.executable, "-c", parent], root / "process.log", 1) + self.assertEqual(code, 124) + time.sleep(1.5) + self.assertFalse(sentinel.exists()) + self.assertIn("CLIENT_TIMEOUT", (root / "process.log").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rsi_trace_adaptive.py b/tests/test_rsi_trace_adaptive.py new file mode 100644 index 00000000..6d7e56d8 --- /dev/null +++ b/tests/test_rsi_trace_adaptive.py @@ -0,0 +1,191 @@ +"""Behavioral contracts for the narrow, unchanged-source RSI trace window.""" + +from __future__ import annotations + +import hashlib +import sys +import tempfile +import unittest +from pathlib import Path +from types import FrameType, FunctionType + +import torch + +from tools.rsi_pilot.trace_adaptive import ( + StepWindow, + locate_window, + run, + state_value, + tensor_evidence, +) + +SOURCE = """def helper(): + return 3 + +def main(optimizer, fail=False): + completed = 0 + for step in range(1, 3): + helper() + optimizer.zero_grad() + loss = torch.tensor(float(step)) + helper() + if fail: + raise ValueError('original failure') + completed = step + helper() + return completed +""" + + +class FakeOptimizer: + def __init__(self, flags: list[bool]) -> None: + self.flags = flags + self.observed: list[bool] = [] + + def zero_grad(self) -> None: + self.observed.append(self.flags[0]) + + +class TestWindow(unittest.TestCase): + def test_restricts_window_preserves_return_and_captures_outside(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "train.py" + source.write_text(SOURCE) + lines = locate_window( + source, hashlib.sha256(source.read_bytes()).hexdigest() + ) + namespace: dict[str, object] = {"torch": torch} + exec(compile(SOURCE, str(source), "exec"), namespace) + entry = namespace["main"] + assert isinstance(entry, FunctionType) + flags = [False] + snapshots: list[int] = [] + + def enable() -> None: + flags[0] = True + + def disable() -> None: + flags[0] = False + + def snapshot(frame: FrameType) -> dict[str, object]: + self.assertFalse(flags[0]) + step: object = frame.f_locals["step"] + assert isinstance(step, int) + snapshots.append(step) + return {"step": step} + + window = StepWindow(entry.__code__, lines, enable, disable, snapshot) + optimizer = FakeOptimizer(flags) + prior = sys.gettrace() + try: + sys.settrace(window) + result: object = entry(optimizer) + finally: + sys.settrace(prior) + window.close() + self.assertEqual(result, 2) + self.assertEqual(optimizer.observed, [True, True]) + self.assertEqual(snapshots, [1, 1, 2, 2]) + self.assertEqual([row["loss"] for row in window.records], [1.0, 2.0]) + self.assertFalse(window.failed) + self.assertFalse(flags[0]) + self.assertIsNone(window(sys._getframe(), "line", None)) + + def test_exception_propagates_and_disarms(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "train.py" + source.write_text(SOURCE) + lines = locate_window( + source, hashlib.sha256(source.read_bytes()).hexdigest() + ) + namespace: dict[str, object] = {"torch": torch} + exec(compile(SOURCE, str(source), "exec"), namespace) + entry = namespace["main"] + assert isinstance(entry, FunctionType) + flags = [False] + + def enable() -> None: + flags[0] = True + + def disable() -> None: + flags[0] = False + + def snapshot(frame: FrameType) -> dict[str, object]: + return {} + + window = StepWindow(entry.__code__, lines, enable, disable, snapshot) + prior = sys.gettrace() + try: + sys.settrace(window) + with self.assertRaisesRegex(ValueError, "original failure"): + entry(FakeOptimizer(flags), True) + finally: + sys.settrace(prior) + window.close() + self.assertTrue(window.failed) + self.assertFalse(flags[0]) + self.assertEqual(window.records, []) + + def test_source_drift_and_duplicate_anchors_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "train.py" + source.write_text(SOURCE) + with self.assertRaisesRegex(ValueError, "pinned source hash"): + locate_window(source, "0" * 64) + duplicate = SOURCE.replace( + "optimizer.zero_grad()", + "optimizer.zero_grad()\n optimizer.zero_grad()", + ) + source.write_text(duplicate) + with self.assertRaisesRegex(ValueError, "unique"): + locate_window(source, hashlib.sha256(source.read_bytes()).hexdigest()) + + def test_foreign_hook_is_not_replaced(self) -> None: + def prior(frame: FrameType, event: str, arg: object) -> None: + return None + + old = sys.gettrace() + try: + sys.settrace(prior) + with self.assertRaisesRegex(RuntimeError, "existing Python trace hook"): + run(Path("missing"), Path("unused"), "plain", None, []) + self.assertIs(sys.gettrace(), prior) + finally: + sys.settrace(old) + + def test_adapter_owned_arguments_and_policy_contract(self) -> None: + with self.assertRaisesRegex(ValueError, "exactly for selective"): + run(Path("missing"), Path("unused"), "selective", None, []) + with self.assertRaisesRegex(ValueError, "owns upstream"): + run(Path("missing"), Path("unused"), "plain", None, ["--device=cuda"]) + + def test_snapshot_copies_values_and_preserves_container_kinds(self) -> None: + value = torch.tensor([1.0, 2.0]) + before = tensor_evidence(value) + copied = state_value({0: (value,), "group": [True, None]}) + value.add_(1) + self.assertNotEqual(before, tensor_evidence(value)) + self.assertEqual( + copied, + { + "kind": "dict", + "items": [ + [ + 0, + { + "kind": "tuple", + "items": [{"kind": "tensor", "value": before}], + }, + ], + ["group", {"kind": "list", "items": [True, None]}], + ], + }, + ) + with self.assertRaisesRegex(TypeError, "state keys"): + state_value({(1, 2): 3}) + with self.assertRaisesRegex(TypeError, "Unsupported"): + state_value(object()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_serializable_scalars.py b/tests/test_serializable_scalars.py new file mode 100644 index 00000000..2757ebbf --- /dev/null +++ b/tests/test_serializable_scalars.py @@ -0,0 +1,157 @@ +"""Compatibility boundaries for skipping redundant exact-scalar JSON probes.""" + +import json +import math +import sys +import unittest +from collections.abc import Iterator +from unittest.mock import patch + +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from traincheck.config import config +from traincheck.instrumentor import dumper + + +class WatchedList(list[int]): + def __init__(self) -> None: + super().__init__([1, 2]) + self.iterations = 0 + + def __iter__(self) -> Iterator[int]: + self.iterations += 1 + return super().__iter__() + + +class MetadataDictionary(dict[str, int]): + @property + def status(self) -> str: + return "kept" + + +class EqualityTrap(type): + def __eq__(self, other: object) -> bool: + raise AssertionError("Scalar type guards must not invoke class equality") + + def __hash__(self) -> int: + raise AssertionError("Scalar type guards must not hash user classes") + + +class CustomInteger(int, metaclass=EqualityTrap): + pass + + +class SerializableScalarTests(unittest.TestCase): + def setUp(self) -> None: + compile_guard = patch.object(config, "USE_TORCH_COMPILE", False) + compile_guard.start() + self.addCleanup(compile_guard.stop) + + def test_exact_scalar_values_keep_runtime_labels_and_original_objects(self) -> None: + values: tuple[tuple[str, object], ...] = ( + ("NoneType", None), + ("bool", True), + ("bool", False), + ("int", -(2**63)), + ("int", 2**63 - 1), + ("float", 1.25), + ("str", "雪\ud800"), + ) + for name, value in values: + with self.subTest(name=name, value=value): + result = dumper.var_to_serializable(value) + self.assertEqual(list(result), [name]) + self.assertIs(result[name], value) + + def test_nonfinite_and_negative_zero_are_not_replaced_or_coerced(self) -> None: + for value in (float("nan"), float("inf"), float("-inf"), -0.0): + result = dumper.var_to_serializable(value) + self.assertIs(result["float"], value) + self.assertEqual( + dumper.serialize(result), dumper.serialize({"float": value}) + ) + result_zero = dumper.var_to_serializable(-0.0)["float"] + assert isinstance(result_zero, float) + self.assertEqual(math.copysign(1, result_zero), -1) + + def test_large_integers_outside_fast_path_still_keep_value(self) -> None: + for value in (-(2**63) - 1, 2**63, 2**128): + self.assertIs(dumper.var_to_serializable(value)["int"], value) + + def test_huge_integer_preserves_configured_json_value_error(self) -> None: + # Python exposes this optional security setting on current 3.10+ patch + # releases. Older interpreters without it cannot exercise this contract. + get_limit: object = getattr(sys, "get_int_max_str_digits", None) + set_limit: object = getattr(sys, "set_int_max_str_digits", None) + if not callable(get_limit) or not callable(set_limit): + self.skipTest("Interpreter has no configurable integer digit limit") + previous: object = get_limit() + assert isinstance(previous, int) + try: + set_limit(640) + value = 10**1000 + with self.assertRaises(ValueError) as expected: + json.dumps(value) + with self.assertRaises(ValueError) as actual: + dumper.var_to_serializable(value) + self.assertEqual(str(actual.exception), str(expected.exception)) + finally: + set_limit(previous) + + def test_custom_scalar_class_equality_and_hash_are_not_invoked(self) -> None: + value = CustomInteger(7) + result = dumper.var_to_serializable(value) + self.assertEqual(len(result), 1) + self.assertIs(next(iter(result.values())), value) + self.assertTrue(next(iter(result)).endswith("CustomInteger")) + + def test_list_subclass_keeps_json_iterator_side_effect(self) -> None: + value = WatchedList() + result = dumper.var_to_serializable(value) + self.assertEqual(value.iterations, 1) + self.assertIs(next(iter(result.values())), value) + + def test_dictionary_subclass_keeps_selected_metadata_conversion(self) -> None: + value = MetadataDictionary(hidden=3) + result = dumper.var_to_serializable(value, {"status": None}) + self.assertEqual(next(iter(result.values())), {"status": "kept"}) + + def test_mixed_containers_keep_identity_and_nested_contents(self) -> None: + value: list[object] = [1, "雪", None, {"flag": True}, (2.0, False)] + result = dumper.var_to_serializable(value) + self.assertIs(result["list"], value) + self.assertEqual(json.dumps(result), json.dumps({"list": value})) + + def test_cyclic_container_still_raises_at_conversion(self) -> None: + value: list[object] = [] + value.append(value) + with self.assertRaisesRegex(ValueError, "Circular reference"): + dumper.var_to_serializable(value) + + def test_existing_compile_gate_runs_before_scalar_shortcut(self) -> None: + # The existing compile-mode heuristic treats objects without a device + # as fake. This optimization must preserve that established gate. + with patch.object(config, "USE_TORCH_COMPILE", True): + self.assertEqual( + dumper.var_to_serializable(3), + {"builtins.int": {"fake": True, "shape": None, "dtype": None}}, + ) + + def test_actual_fake_tensor_keeps_dedicated_metadata_path(self) -> None: + with FakeTensorMode() as mode: + value = mode.from_tensor(torch.empty(2, 3)) + with patch.object(config, "USE_TORCH_COMPILE", True): + result = dumper.var_to_serializable(value) + self.assertEqual( + next(iter(result.values())), + {"fake": True, "shape": (2, 3), "dtype": "torch.float32"}, + ) + + def test_primitive_keeps_ignoring_uninterpreted_dump_configuration(self) -> None: + value = 42 + self.assertIs(dumper.var_to_serializable(value, object())["int"], value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_traincheck_skill_report.py b/tests/test_traincheck_skill_report.py new file mode 100644 index 00000000..0e00a9aa --- /dev/null +++ b/tests/test_traincheck_skill_report.py @@ -0,0 +1,305 @@ +"""Agent summaries distinguish real checking from absent or malformed evidence.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from skills.traincheck.scripts.summarize_check import main, read_artifact, summarize +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import ObservationReport, observe +from tools.fresh_validation.check_coverage import summarize as summarize_observations +from tools.fresh_validation.contracts import ( + JSONObject, + JSONValue, + json_object, + mapping, + sequence, +) + + +def observation(*, valid: bool = True, enabled: bool = True) -> ObservationReport: + dtype = "torch.float32" if valid else "torch.float64" + return observe( + frame([{"torch.FloatTensor": {"shape": [2], "dtype": dtype}}]), + output_rule(), + enabled=enabled, + ) + + +def coverage(*reports: ObservationReport, rows: int = 2) -> JSONObject: + copied = [copy.deepcopy(report) for report in reports] + for index, record in enumerate(copied): + record["invariant_index"] = index + return json_object( + { + "schema_version": 3, + "inputs_unchanged": True, + "trace_rows": rows, + "input_sha256": {"/source/trace.log": "a" * 64}, + "summary": summarize_observations(copied), + "per_invariant": copied, + } + ) + + +class SkillReportTests(unittest.TestCase): + def test_real_active_checks_and_real_violations_remain_distinct(self) -> None: + passed = summarize(coverage(observation())) + failed = summarize(coverage(observation(valid=False))) + self.assertEqual(passed["outcome"], "active_checks_passed") + self.assertEqual(failed["outcome"], "violations") + counts = mapping(failed["counts"], "counts") + self.assertEqual(counts["observed_violations"], 1) + self.assertEqual(counts["completed_decisions"], 1) + + def test_dormant_disabled_and_empty_are_not_active_success(self) -> None: + dormant = observe( + frame([{"torch.FloatTensor": {"shape": [2]}}]), output_rule(False) + ) + self.assertEqual(summarize(coverage(dormant))["outcome"], "no_evidence") + mixed = summarize(coverage(observation(), dormant)) + self.assertEqual(mixed["outcome"], "incomplete") + disabled = summarize(coverage(observation(enabled=False))) + self.assertEqual(disabled["outcome"], "no_evidence") + self.assertTrue(disabled["coverage_incomplete"]) + self.assertEqual(summarize(coverage())["outcome"], "no_evidence") + self.assertEqual( + summarize(coverage(observation(), rows=0))["outcome"], "no_evidence" + ) + + def test_error_unknown_outcome_and_skipped_variable_checks_stay_visible( + self, + ) -> None: + error = observation() + error.update( + { + "status": "checker_error", + "check_passed": None, + "checker_triggered": None, + "error": {"type": "ValueError", "message": "malformed tensor"}, + "fully_validated_active": False, + } + ) + report = summarize(coverage(error)) + self.assertEqual(report["outcome"], "incomplete") + self.assertEqual(mapping(report["counts"], "counts")["unknown_outcomes"], 1) + skipped = observation() + skipped["variable_check_skipped"] = True + skipped["fully_validated_active"] = False + self.assertEqual(summarize(coverage(skipped))["outcome"], "incomplete") + + def test_unsupported_coverage_is_not_zero_violations_or_success(self) -> None: + unsupported = observation(enabled=False) + unsupported.update( + { + "status": "unsupported_coverage", + "coverage_status": "unsupported", + "check_passed": None, + "checker_triggered": None, + } + ) + report = summarize(coverage(unsupported)) + self.assertEqual(report["outcome"], "no_evidence") + counts = mapping(report["counts"], "counts") + self.assertEqual(counts["unsupported_coverage"], 1) + self.assertEqual(counts["unknown_outcomes"], 1) + self.assertEqual(counts["unobserved_coverage"], 1) + + def test_required_fields_types_and_contradictory_evidence_are_rejected( + self, + ) -> None: + good = coverage(observation()) + cases: list[JSONObject] = [] + for field in ("per_invariant", "trace_rows", "input_sha256", "summary"): + missing = copy.deepcopy(good) + del missing[field] + cases.append(missing) + top_level_changes: list[tuple[str, JSONValue]] = [ + ("trace_rows", True), + ("trace_rows", -1), + ("schema_version", 2), + ("inputs_unchanged", False), + ("input_sha256", {}), + ] + for field, value in top_level_changes: + bad = copy.deepcopy(good) + bad[field] = value + cases.append(bad) + for bad in cases: + with self.subTest(bad=bad), self.assertRaises(ValueError): + summarize(bad) + rule_changes: list[tuple[str, JSONValue]] = [ + ("status", "success"), + ("active", False), + ("decisions_completed", True), + ("check_passed", False), + ("invariant_index", 3), + ] + for field, value in rule_changes: + rule = json_object(observation()) + rule["invariant_index"] = 0 + rule[field] = value + bad = copy.deepcopy(good) + bad["per_invariant"] = [rule] + with self.subTest(field=field), self.assertRaises(ValueError): + summarize(bad) + + def test_identity_alarm_and_complete_source_are_preserved(self) -> None: + rule = observation(valid=False) + rule["identity_clauses"] = [{"object_id": "literal-007", "thread_id": 3.5}] + source = coverage(rule) + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "coverage.json" + encoded = json.dumps(source).encode() + path.write_bytes(encoded) + loaded, provenance = read_artifact(path) + result = summarize(loaded) + self.assertEqual(loaded, source) + self.assertEqual(path.read_bytes(), encoded) + self.assertEqual(provenance["sha256"], hashlib.sha256(encoded).hexdigest()) + self.assertEqual(result["outcome"], "violations") + self.assertIn('"has_identity_precondition": true', json.dumps(result)) + + def test_precondition_flag_does_not_claim_absence_of_relation_argument_ids( + self, + ) -> None: + for field, value in ( + ("_TRAINCHECK_positional_ID", "literal-007"), + ("sequence_length", "16"), + ): + rule = observation(valid=False) + rule["identity_clauses"] = [] + checker = rule["checker_result"] + invariant = json_object(checker["invariant"]) + invariant["params"] = [ + { + "param_type": "APIParam", + "api_full_name": "torch.fake", + "arguments": {"args": {"self": {"model": {field: value}}}}, + } + ] + checker["invariant"] = invariant + source = coverage(rule) + before = copy.deepcopy(source) + report = summarize(source) + problem = mapping( + sequence(report["problem_rules"], "problem_rules")[0], "problem" + ) + with self.subTest(field=field): + self.assertIs(problem["has_identity_precondition"], False) + self.assertNotIn("identity_bearing", problem) + self.assertEqual(problem["evidence_pointer"], "per_invariant[0]") + self.assertEqual(source, before) + self.assertEqual(report["outcome"], "violations") + + def test_missing_identity_preconditions_are_unknown_not_false(self) -> None: + source = coverage(observation(valid=False)) + rule = json_object(sequence(source["per_invariant"], "per_invariant")[0]) + del rule["identity_clauses"] + source["per_invariant"] = [rule] + with self.assertRaisesRegex( + ValueError, "Missing required field: identity_clauses" + ): + summarize(source) + + def test_compact_display_does_not_hide_total_problems_or_change_source( + self, + ) -> None: + source = coverage(*([observation(valid=False)] * 12)) + before = copy.deepcopy(source) + result = summarize(source) + self.assertEqual(len(sequence(result["problem_rules"], "problem_rules")), 10) + self.assertEqual(result["additional_problem_rules"], 2) + self.assertEqual(mapping(result["counts"], "counts")["observed_violations"], 12) + self.assertEqual(source, before) + + def test_aggregate_claim_cannot_override_individual_checker_evidence(self) -> None: + source = coverage(observation(valid=False)) + aggregate = json_object(source["summary"]) + aggregate["observed_violated_invariants"] = 0 + source["summary"] = aggregate + with self.assertRaisesRegex(ValueError, "summary disagrees"): + summarize(source) + + def test_cli_records_actual_policy_count_check_and_refuses_overwrite(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, policy, direct, output = ( + root / name + for name in ( + "coverage.json", + "policy.json", + "direct.json", + "agent.json", + ) + ) + source.write_text(json.dumps(coverage(observation(valid=False)))) + policy.write_text(json.dumps({"model_tracker_style": None})) + direct.write_text( + json.dumps({"distinct_invariants_violated": 1, "violations": [{}]}) + ) + command = [ + "summarize_check.py", + "--coverage", + str(source), + "--collection-policy", + str(policy), + "--violations-summary", + str(direct), + "--output", + str(output), + ] + with patch("sys.argv", command): + self.assertEqual(main(), 0) + result, _ = read_artifact(output) + self.assertEqual(result["outcome"], "violations") + self.assertEqual(result["collection_mode"], "api-only") + with patch("sys.argv", command), self.assertRaises(FileExistsError): + main() + + def test_direct_mismatch_and_malformed_policy_fail_before_output(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, policy, direct, output = ( + root / name + for name in ( + "coverage.json", + "policy.json", + "direct.json", + "agent.json", + ) + ) + source.write_text(json.dumps(coverage(observation()))) + policy.write_text(json.dumps({"model_tracker_style": []})) + direct.write_text( + json.dumps({"distinct_invariants_violated": 1, "violations": [{}]}) + ) + for flag, path in ( + ("--collection-policy", policy), + ("--violations-summary", direct), + ): + with patch( + "sys.argv", + [ + "report", + "--coverage", + str(source), + flag, + str(path), + "--output", + str(output), + ], + ): + with self.assertRaises(ValueError): + main() + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/fresh_validation/run_phase.py b/tools/fresh_validation/run_phase.py index 9e52a598..5dc83028 100644 --- a/tools/fresh_validation/run_phase.py +++ b/tools/fresh_validation/run_phase.py @@ -8,6 +8,7 @@ import subprocess import time from pathlib import Path +from types import FrameType from typing import Literal, TypedDict @@ -20,7 +21,13 @@ class PhaseOptions(argparse.Namespace): command: list[str] -StopReason = Literal["process_group_rss_budget", "timeout", "trace_size_budget"] +StopReason = Literal[ + "process_group_rss_budget", + "timeout", + "trace_size_budget", + "signal_sigterm", + "signal_sigint", +] class PhaseResult(TypedDict): @@ -54,7 +61,34 @@ def group_memory(group: int) -> tuple[int, list[int]]: return total, processes +def terminate_group(process: subprocess.Popen[bytes]) -> None: + """Stop the owned session, including descendants whose leader already exited. + + A terminating leader does not prove its process group is empty. Always send + KILL to any remaining members after a bounded TERM grace period, then reap + the direct child. A child that cannot be reaped is an explicit failure. + """ + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass # The owned group has already exited. + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + + def run(args: PhaseOptions) -> int: + """Supervise from the main thread, temporarily handling SIGTERM and SIGINT. + + Cleanup covers handled signals, normal completion and Python exceptions. + Uncatchable termination such as SIGKILL cannot run user-space cleanup. + """ args.output.mkdir(parents=True, exist_ok=False) source_hashes = { str(p.relative_to(args.source)): hashlib.sha256(p.read_bytes()).hexdigest() @@ -76,57 +110,79 @@ def run(args: PhaseOptions) -> int: command = args.command[1:] if args.command[0] == "--" else args.command started, peak = time.monotonic(), 0 reason: StopReason | None = None - with (args.output / "stdout.log").open("w") as log: - proc = subprocess.Popen( - ["nice", "-n", "10", *command], - cwd=args.output, - env=env, - stdout=log, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - invocation = { - "command": command, - "pid": proc.pid, - "native_threads": 1, - "timeout_seconds": args.timeout, - "memory_limit_gib": args.memory_gib, - "trace_limit_mib": args.trace_mib, - "source_sha256": source_hashes, - "harness_sha256": { - p.name: hashlib.sha256(p.read_bytes()).hexdigest() - for p in Path(__file__).resolve().parent.glob("*.py") - }, - } - (args.output / "invocation.json").write_text( - json.dumps(invocation, indent=2) + "\n" - ) + signal_reason: StopReason | None = None + + def request_stop(signum: int, frame: FrameType | None) -> None: + nonlocal signal_reason + assert signum in (signal.SIGTERM, signal.SIGINT) + if signal_reason is None: + signal_reason = ( + "signal_sigterm" if signum == signal.SIGTERM else "signal_sigint" + ) + + # The workload has a separate session. An outer timeout's signal to this + # supervisor must therefore trigger explicit child-group cleanup, not the + # default immediate supervisor exit. Signals only request a stop so repeated + # signals cannot interrupt TERM/KILL/reap midway through cleanup. + prior_term = signal.signal(signal.SIGTERM, request_stop) + try: + prior_int = signal.signal(signal.SIGINT, request_stop) try: - while proc.poll() is None: - memory, _ = group_memory(proc.pid) - peak = max(peak, memory) - if memory > args.memory_gib * 1024**3: - reason = "process_group_rss_budget" - if time.monotonic() - started > args.timeout: - reason = "timeout" - if ( - sum(p.stat().st_size for p in args.output.rglob("trace_API*.log")) - > args.trace_mib * 1024**2 - ): - reason = "trace_size_budget" - if reason: - os.killpg(proc.pid, signal.SIGTERM) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait() - break - time.sleep(0.25) - except BaseException: - os.killpg(proc.pid, signal.SIGTERM) - proc.wait(timeout=10) - raise + with (args.output / "stdout.log").open("w") as log: + proc = subprocess.Popen( + ["nice", "-n", "10", *command], + cwd=args.output, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + invocation = { + "command": command, + "pid": proc.pid, + "native_threads": 1, + "timeout_seconds": args.timeout, + "memory_limit_gib": args.memory_gib, + "trace_limit_mib": args.trace_mib, + "source_sha256": source_hashes, + "harness_sha256": { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in Path(__file__).resolve().parent.glob("*.py") + }, + } + (args.output / "invocation.json").write_text( + json.dumps(invocation, indent=2) + "\n" + ) + while proc.poll() is None: + memory, _ = group_memory(proc.pid) + peak = max(peak, memory) + if memory > args.memory_gib * 1024**3: + reason = "process_group_rss_budget" + if time.monotonic() - started > args.timeout: + reason = "timeout" + if ( + sum( + p.stat().st_size + for p in args.output.rglob("trace_API*.log") + ) + > args.trace_mib * 1024**2 + ): + reason = "trace_size_budget" + if signal_reason is not None: + reason = signal_reason + if reason is not None: + break + time.sleep(0.25) + finally: + terminate_group(proc) + # A signal may arrive while the leader exits or during cleanup. + if signal_reason is not None: + reason = signal_reason + finally: + signal.signal(signal.SIGINT, prior_int) + finally: + signal.signal(signal.SIGTERM, prior_term) unchanged = all( hashlib.sha256((args.source / p).read_bytes()).hexdigest() == h for p, h in source_hashes.items() diff --git a/tools/rsi_pilot/analyze_trials.py b/tools/rsi_pilot/analyze_trials.py new file mode 100644 index 00000000..b8e9a982 --- /dev/null +++ b/tools/rsi_pilot/analyze_trials.py @@ -0,0 +1,787 @@ +"""Freeze completed RSI pilot selections, then audit matched held-out results. + +Timestamps come from explicit operator actions, never file mtimes. Hashes bind +the saved evidence; they do not establish a security boundary against its owner. +The operator scores each unique checkpoint/configuration once and reuses that +immutable report for identical selections. Exact baseline-score equality checks +that reuse contract; it is not a claim of identical repeated execution timings. +""" + +import argparse +import hashlib +import json +import math +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.fresh_validation.contracts import JSONObject # noqa: E402 +from tools.fresh_validation.contracts import ( + JSONValue, + boolean, + integer, + json_object, + mapping, + number, + read_object, + required, + sequence, + string, +) +from tools.rsi_pilot.run_candidate import configure # noqa: E402 +from tools.rsi_pilot.run_candidate import ( + validate_diagnostic_artifacts, + validate_diagnostics, + validate_preflight, + validate_results, +) + +POLICY: list[JSONValue] = [1, 2, 4, 4] + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def text(path: Path) -> str: + return path.read_bytes().decode("utf-8") + + +def finite(value: object, label: str) -> float: + result = float(number(value, label)) + if not math.isfinite(result) or result < 0: + raise ValueError(f"{label} must be finite and nonnegative") + return result + + +def require_equal(actual: object, expected: object, label: str) -> None: + if type(actual) is not type(expected) or actual != expected: + raise ValueError(f"{label} mismatch") + if isinstance(actual, dict) and isinstance(expected, dict): + left_mapping, right_mapping = mapping(actual, label), mapping(expected, label) + for key in left_mapping: + require_equal(left_mapping[key], right_mapping[key], label) + if isinstance(actual, list) and isinstance(expected, list): + for left, right in zip(sequence(actual, label), sequence(expected, label)): + require_equal(left, right, label) + + +def protocol(root: Path) -> tuple[JSONObject, list[str], dict[str, str], list[str]]: + record = read_object(root / "study_protocol.json") + timestamp(required(record, "revised_at")) + if not string(required(record, "revision_reason"), "revision_reason"): + raise ValueError("Protocol revision must explain the excluded setup trials") + order = [ + string(value, "trial ID") + for value in sequence(required(record, "trial_order"), "trial_order") + ] + excluded = [ + string(value, "setup trial ID") + for value in sequence( + required(record, "excluded_setup_trials"), "excluded_setup_trials" + ) + ] + if any(re.fullmatch(r"[A-Za-z0-9_-]+", name) is None for name in order + excluded): + raise ValueError("Invalid trial directory ID") + if len(set(order + excluded)) != len(order + excluded): + raise ValueError("Duplicate or overlapping trial IDs") + arms: dict[str, str] = {} + pairs = sequence(required(record, "pairs"), "pairs") + if len(pairs) != 2: + raise ValueError("This pilot requires exactly two matched pairs") + for item in pairs: + pair = mapping(item, "pair") + for arm in ("control", "traincheck"): + name = string(required(pair, arm), arm) + if name in arms: + raise ValueError("Trial repeated across pairs") + arms[name] = arm + require_equal(set(order), set(arms), "paired trial set") + return record, order, arms, excluded + + +def metric(value: object, expected_depths: list[int]) -> JSONObject: + rows = [mapping(row, "depth") for row in sequence(value, "depths")] + require_equal( + [integer(required(r, "loops"), "loops") for r in rows], + expected_depths, + "depth order", + ) + accuracies = [ + finite(required(r, "macro_accuracy_percent"), "accuracy") for r in rows + ] + work = [finite(required(r, "macro_mean_dense_matmul_flops"), "FLOPs") for r in rows] + if any(a > 100 for a in accuracies) or any(w <= 0 for w in work): + raise ValueError("Invalid accuracy or work") + selected = [0, 1, 2, 2] + ratios = [ + work[i] / ceiling + for i, ceiling in zip(selected, [805760, 1610624, 3220352, 6439808]) + ] + if any(ratio > 1 for ratio in ratios): + raise ValueError("Fixed policy exceeds its work ceiling") + accuracy = sum(accuracies[i] for i in selected) / 4 + return { + "fixed_policy_depths": list(POLICY), + "accuracy_percent": accuracy, + # Preserve the upstream aggregate_reward evaluation order as well as + # its mathematical formula; exact receipt validation must not round. + "reward": sum( + 0.25 * (0.8 * accuracies[i] + 0.2 * (100 * (1 - ratio))) + for i, ratio in zip(selected, ratios) + ), + } + + +def preflight_summary( + trial: Path, manifest: JSONObject, files: set[Path] +) -> JSONObject: + """Free program checks have their own receipts and costs, never train budget.""" + attempts: list[JSONValue] = [] + seconds = 0.0 + paths = sorted((trial / "preflights").glob("preflight-*")) + if [path.name for path in paths] != [ + f"preflight-{i:02d}" for i in range(1, len(paths) + 1) + ]: + raise ValueError("Malformed preflight attempt sequence") + profile = Path(string(required(manifest, "profile"), "profile")) + for path in paths: + receipt = read_object(path / "preflight.json") + status = string(required(receipt, "status"), "preflight status") + if status not in ( + "program_valid", + "program_invalid", + "execution_failed", + "client_error", + ): + raise ValueError("Preflight has not finished") + for key, expected in ( + ("schema_version", 1), + ("preflight", path.name), + ("arm", manifest["arm"]), + ("profile_sha256", digest(profile)), + ("trial_manifest_sha256", digest(trial / "trial.json")), + ("program_sha256", digest(path / "program.py")), + ): + require_equal(required(receipt, key), expected, key) + require_equal( + text(path / "config.toml"), + configure(text(profile), text(path / "program.py")), + "preflight config", + ) + errors = sequence(required(receipt, "errors"), "preflight errors") + if (status == "program_valid") != (len(errors) == 0): + raise ValueError("Preflight status/error mismatch") + for error in errors: + detail = mapping(error, "preflight error") + if not string(required(detail, "phase"), "preflight error phase"): + raise ValueError("Empty preflight error phase") + # Program rejections nest the original interpreter error; process + # and client failures store the exception directly in the receipt. + cause = ( + mapping(detail["error"], "interpreter error") + if "error" in detail + else detail + ) + for key in ("type", "message"): + if not string(required(cause, key), key): + raise ValueError("Empty preflight error evidence") + if status in ("program_valid", "program_invalid"): + result = read_object(path / "result.json") + valid = validate_preflight( + result, integer(required(receipt, "check_exit_code"), "check_exit_code") + ) + require_equal(required(receipt, "valid"), valid, "preflight valid") + require_equal( + required(receipt, "result"), result, "preflight backend result" + ) + require_equal(valid, status == "program_valid", "preflight status") + elif status == "execution_failed" and integer( + required(receipt, "check_exit_code"), "check_exit_code" + ) in (0, 2): + raise ValueError( + "Preflight execution failure has a normal interpreter exit" + ) + duration = finite( + required(receipt, "operational_seconds_including_transfer"), + "preflight duration", + ) + seconds += duration + files.update(item for item in path.iterdir() if item.is_file()) + attempts.append( + { + "preflight": path.name, + "status": status, + "operational_seconds": duration, + "errors": receipt["errors"], + } + ) + return { + "attempts": attempts, + "count": len(paths), + "operational_seconds_including_queue_transfer": seconds, + } + + +def trial_summary( + root: Path, trial_id: str, files: set[Path], expected_arm: str | None +) -> JSONObject: + trial = root / "trials" / trial_id + manifest_path = trial / "trial.json" + manifest = read_object(manifest_path) + workspace = trial / "workspace" + require_equal( + Path(string(required(manifest, "workspace"), "workspace")).resolve(), + workspace.resolve(), + "workspace", + ) + arm = string(required(manifest, "arm"), "arm") + if arm not in ("control", "traincheck"): + raise ValueError("Unknown arm") + if expected_arm is not None: + require_equal(arm, expected_arm, "arm") + for key, expected_limit in ( + ("max_candidates", 2), + ("max_diagnostics", int(arm == "traincheck")), + ("expected_steps", 10000), + ): + require_equal(integer(required(manifest, key), key), expected_limit, key) + profile = root / "frozen-profile.toml" + require_equal( + Path(string(required(manifest, "profile"), "profile")).resolve(), + profile.resolve(), + "profile path", + ) + require_equal(required(manifest, "profile_sha256"), digest(profile), "profile hash") + files.update( + { + manifest_path, + profile, + workspace / "assignment.txt", + workspace / "baseline_research.py", + workspace / "baseline_development.json", + workspace / "summary.md", + workspace / "submission.json", + } + ) + preflights = preflight_summary(trial, manifest, files) + paths = sorted((trial / "runs").glob("candidate-*")) + if len(paths) > 2 or [p.name for p in paths] != [ + f"candidate-{i:02d}" for i in range(1, len(paths) + 1) + ]: + raise ValueError("Malformed or excessive candidate attempts") + attempts: list[JSONValue] = [] + valid: dict[str, JSONObject] = {} + training_known = operational = 0.0 + unknown_training: list[JSONValue] = [] + diagnostic_count = diagnostic_complete = diagnostic_failed = diagnostic_skips = ( + error_count + ) = 0 + for index, path in enumerate(paths, 1): + receipt_path = path / "run.json" + receipt = read_object(receipt_path) + requested = boolean( + required(receipt, "diagnostics_requested"), "diagnostics_requested" + ) + diagnostic_count += requested + for key, expected_text in ( + ("candidate", path.name), + ("arm", arm), + ("profile_sha256", digest(profile)), + ("trial_manifest_sha256", digest(manifest_path)), + ("program_sha256", digest(path / "program.py")), + ): + require_equal(required(receipt, key), expected_text, key) + for key, expected_count in ( + ("candidates_reserved", index), + ("diagnostics_reserved", diagnostic_count), + ): + require_equal(integer(required(receipt, key), key), expected_count, key) + require_equal( + text(path / "config.toml"), + configure(text(profile), text(path / "program.py")), + "candidate config", + ) + status = string(required(receipt, "status"), "status") + if status not in ("training_complete", "training_failed", "client_error"): + raise ValueError("Candidate has not finished") + errors = sequence(required(receipt, "errors"), "errors") + error_count += len(errors) + for error in errors: + for key in ("phase", "type", "message"): + if not string(required(mapping(error, "error"), key), key): + raise ValueError("Empty error evidence") + if status != "training_complete" and not errors: + raise ValueError("Failed candidate has no error evidence") + duration = finite( + required(receipt, "operational_seconds_including_transfer_and_diagnostics"), + "operational duration", + ) + operational += duration + files.update({receipt_path, path / "program.py", path / "config.toml"}) + files.update(path.glob("*.log")) + elapsed: float | None = None + if "training_exit_code" in receipt: + code = integer(receipt["training_exit_code"], "training_exit_code") + if status == "training_failed" and code == 0: + raise ValueError("Training failure has a successful process exit") + if code == 0 and (path / "results.json").exists(): + results = read_object(path / "results.json") + _, seconds, _ = validate_results(results, manifest) + for key, expected_result in ( + ("training_seconds", seconds), + ("completed_steps", 10000), + ("development_depths", results["final"]), + ): + require_equal(required(receipt, key), expected_result, key) + elapsed = finite(seconds, "training duration") + valid[path.name] = metric(results["final"], [1, 2, 4, 8]) + files.add(path / "results.json") + elif status != "client_error": + raise ValueError("Finished training receipt has no process exit") + if status == "training_complete" and elapsed is None: + raise ValueError("Complete training has no validated results") + if elapsed is None: + unknown_training.append(path.name) + else: + training_known += elapsed + diagnostic: JSONObject = {"requested": requested, "status": "not_requested"} + if requested: + diagnostic["status"] = "unknown_after_client_error" + if "diagnostic_status" in receipt: + diagnostic_status = string( + receipt["diagnostic_status"], "diagnostic_status" + ) + if diagnostic_status not in ("complete", "execution_failed"): + raise ValueError("Unknown diagnostic status") + diagnostic["status"] = diagnostic_status + exit_code = integer( + required(receipt, "diagnostic_exit_code"), "diagnostic_exit_code" + ) + if (diagnostic_status == "complete") != (exit_code == 0): + raise ValueError("Diagnostic exit/status mismatch") + if diagnostic_status == "complete": + diagnostic_report = read_object(path / "agent_report.json") + validate_diagnostics(diagnostic_report) + validate_diagnostic_artifacts(diagnostic_report, path) + invariant_path = workspace / "invariants.json" + require_equal( + required( + read_object(path / "diagnostic_manifest.json"), + "invariants_sha256", + ), + digest(invariant_path), + "diagnostic reference invariants", + ) + files.add(invariant_path) + require_equal( + required(receipt, "traincheck_report"), + diagnostic_report, + "diagnostic report", + ) + diagnostic["report"] = diagnostic_report + diagnostic_complete += 1 + diagnostic_skips += integer( + required( + mapping(diagnostic_report["counts"], "counts"), + "skipped_variable_checks", + ), + "skipped_variable_checks", + ) + files.update( + { + path / name + for name in ( + "agent_report.json", + "coverage.json", + "diagnostic_manifest.json", + ) + } + ) + else: + diagnostic_failed += 1 + elif status != "client_error": + raise ValueError( + "Finished candidate is missing its requested diagnostic" + ) + elif any( + key in receipt + for key in ( + "traincheck_report", + "diagnostic_status", + "diagnostic_exit_code", + ) + ): + raise ValueError("Unrequested diagnostic evidence") + attempts.append( + { + "candidate": path.name, + "status": status, + "training_seconds": elapsed, + "operational_seconds": duration, + "diagnostic": diagnostic, + "errors": receipt["errors"], + } + ) + if diagnostic_count > int(arm == "traincheck"): + raise ValueError("Diagnostic budget exceeded") + ledger_path = trial / "usage.json" + if paths or ledger_path.exists(): + ledger = read_object(ledger_path) + require_equal( + integer(required(ledger, "candidates_reserved"), "candidates_reserved"), + len(paths), + "ledger candidates", + ) + require_equal( + integer(required(ledger, "diagnostics_reserved"), "diagnostics_reserved"), + diagnostic_count, + "ledger diagnostics", + ) + files.add(ledger_path) + report: JSONObject = { + "trial": trial_id, + "arm": arm, + "attempts": attempts, + "preflights": preflights, + "training_seconds_known": training_known, + "training_duration_unknown_attempts": unknown_training, + "operational_seconds_including_queue_transfer_diagnostics": operational, + "operational_seconds_including_preflights": operational + + finite( + preflights["operational_seconds_including_queue_transfer"], + "preflight duration", + ), + "diagnostics_requested": diagnostic_count, + "diagnostics_completed": diagnostic_complete, + "diagnostic_execution_failures": diagnostic_failed, + "diagnostic_status_unknown": diagnostic_count + - diagnostic_complete + - diagnostic_failed, + "diagnostic_skips_known": diagnostic_skips, + "diagnostic_skip_counts_unknown": diagnostic_count - diagnostic_complete, + "recorded_errors": error_count, + } + if expected_arm is None: + report["role"] = "excluded_setup_only" + return report + submission_path = workspace / "submission.json" + selection = string(required(read_object(submission_path), "candidate"), "candidate") + if selection not in ("baseline", "candidate-01", "candidate-02"): + raise ValueError("Unknown selected candidate") + files.update({submission_path, workspace / "summary.md"}) + baseline = read_object(workspace / "baseline_development.json") + development = metric(required(baseline, "final"), [1, 2, 4, 8]) + source = workspace / "baseline_research.py" + if selection != "baseline": + if selection not in valid: + raise ValueError("Selected candidate has no validated completed training") + development, source = ( + valid[selection], + trial / "runs" / selection / "program.py", + ) + selected = root / "selected" / trial_id + selected_config = read_object(selected / "config.json") + baseline_config = read_object(root / "selected" / "baseline" / "config.json") + training = dict(mapping(required(selected_config, "training"), "training")) + require_equal( + required(training, "research_source"), text(source), "selected program" + ) + training["research_source"] = required( + mapping(required(baseline_config, "training"), "baseline training"), + "research_source", + ) + normalized = dict(selected_config) + normalized["training"] = json_object(training, "selected training") + require_equal(normalized, baseline_config, "frozen training configuration") + if selection == "baseline": + for name in ("config.json", "model.safetensors"): + require_equal( + digest(selected / name), + digest(root / "selected" / "baseline" / name), + "baseline artifact", + ) + files.update({selected / "config.json", selected / "model.safetensors"}) + summary = (workspace / "summary.md").read_text() + report.update( + { + "selection": selection, + "development": development, + "agent_summary_excerpt": summary[:2000], + "agent_summary_truncated": len(summary) > 2000, + } + ) + return report + + +def verify_selected_origins(root: Path, order: list[str], files: set[Path]) -> None: + """Bind downloaded selections to hashes read at their original remote paths. + + The operator records these hashes directly on the remote host before copying + the artifacts. This checks accidental path/content mismatch, not adversarial + tampering by the experiment owner. + """ + origins_path = root / "selected/origins.json" + origins = read_object(origins_path) + require_equal(required(origins, "schema_version"), 1, "origin schema version") + artifacts = mapping(required(origins, "artifacts"), "origin artifacts") + require_equal(set(artifacts), {"baseline", *order}, "origin artifact set") + packet_path = root / "solver-packet-manifest.json" + baseline_remote = string( + required(read_object(packet_path), "baseline_remote_training"), + "baseline remote directory", + ) + if not baseline_remote.startswith("/"): + raise ValueError("Baseline remote directory must be absolute") + for trial_id in ("baseline", *order): + expected_remote = baseline_remote + if trial_id != "baseline": + trial = root / "trials" / trial_id + selection = string( + required(read_object(trial / "workspace/submission.json"), "candidate"), + "selected candidate", + ) + if selection != "baseline": + remote_trial = string( + required(read_object(trial / "trial.json"), "remote_trial"), + "remote trial directory", + ) + expected_remote = remote_trial + "/" + selection + "/training" + origin = mapping(artifacts[trial_id], "artifact origin") + require_equal( + required(origin, "remote_directory"), expected_remote, "origin directory" + ) + hashes = mapping(required(origin, "files"), "remote artifact hashes") + require_equal( + set(hashes), {"config.json", "model.safetensors"}, "origin file set" + ) + for name, remote_hash in hashes.items(): + require_equal( + string(remote_hash, "remote artifact hash"), + digest(root / "selected" / trial_id / name), + "remote artifact hash", + ) + files.update({origins_path, packet_path}) + + +def collect(root: Path) -> tuple[list[JSONValue], list[JSONValue], set[Path]]: + _, order, arms, excluded = protocol(root) + files = { + root / "selected" / "baseline" / name + for name in ("config.json", "model.safetensors") + } + files.add(root / "heldout-evaluation-plan.json") + files.add(root / "study_protocol.json") + harness_path = root / "harness-freeze-B.json" + packets_path = root / "solver-packet-manifest-B.json" + files.update({harness_path, packets_path}) + verify_inventory( + Path(__file__).resolve().parents[2], + required(read_object(harness_path), "files"), + files, + ) + inventory = mapping( + required(read_object(packets_path), "packet_inventory"), "packet inventory" + ) + require_equal(set(inventory), set(order), "packet trial set") + for trial in order: + verify_inventory(root / "trials" / trial / "workspace", inventory[trial], files) + reports: list[JSONValue] = [ + trial_summary(root, trial, files, arms[trial]) for trial in order + ] + setup: list[JSONValue] = [ + trial_summary(root, trial, files, None) for trial in excluded + ] + verify_selected_origins(root, order, files) + files.update(Path(__file__).resolve().parent.glob("*.py")) + return reports, setup, files + + +def verify_inventory(base: Path, value: object, files: set[Path]) -> None: + inventory = mapping(value, "hash inventory") + if not inventory: + raise ValueError("Empty evidence inventory") + for name, saved_hash in inventory.items(): + path = (base / name).resolve(strict=True) + if not path.is_relative_to(base.resolve()): + raise ValueError("Inventory path outside its root") + require_equal(digest(path), string(saved_hash, "saved hash"), "inventory hash") + files.add(path) + + +def timestamp(value: object) -> datetime: + parsed = datetime.fromisoformat(string(value, "timestamp").replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.utcoffset() != timezone.utc.utcoffset(parsed): + raise ValueError("Timestamp must explicitly identify UTC") + return parsed + + +def save_new(path: Path, value: JSONObject) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as handle: + handle.write(json.dumps(value, indent=2, allow_nan=False) + "\n") + + +def freeze(root: Path) -> None: + _, _, files = collect(root) + if (root / "scores").exists() and any((root / "scores").iterdir()): + raise ValueError("Scores already exist before finalization") + now = datetime.now(timezone.utc) + study, _, _, _ = protocol(root) + if timestamp(study["revised_at"]) >= now: + raise ValueError("Protocol revision must precede finalization") + save_new( + root / "finalization.json", + { + "finalized_at": now.isoformat(), + "files": {str(path.resolve()): digest(path) for path in sorted(files)}, + }, + ) + + +def verify_freeze(root: Path) -> JSONObject: + frozen = read_object(root / "finalization.json") + _, _, files = collect(root) + expected = mapping(required(frozen, "files"), "frozen files") + require_equal( + set(expected), {str(path.resolve()) for path in files}, "frozen file set" + ) + for path, saved_hash in expected.items(): + require_equal( + digest(Path(path)), string(saved_hash, "hash"), "frozen file hash" + ) + timestamp(required(frozen, "finalized_at")) + return frozen + + +def begin_scoring(root: Path) -> None: + verify_freeze(root) + if (root / "scores").exists() and any((root / "scores").iterdir()): + raise ValueError("Scoring has already started") + save_new( + root / "scores" / "provenance.json", + { + "scoring_started_at": datetime.now(timezone.utc).isoformat(), + "finalization_sha256": digest(root / "finalization.json"), + }, + ) + + +def analyze(root: Path) -> JSONObject: + frozen = verify_freeze(root) + provenance = read_object(root / "scores" / "provenance.json") + require_equal( + required(provenance, "finalization_sha256"), + digest(root / "finalization.json"), + "finalization provenance", + ) + if timestamp(required(provenance, "scoring_started_at")) <= timestamp( + required(frozen, "finalized_at") + ): + raise ValueError("Held-out scoring must follow finalization") + reports, setup, _ = collect(root) + study, order, _, _ = protocol(root) + scores: dict[str, JSONObject] = {} + panel: str | None = None + panel_size: int | None = None + plan = read_object(root / "heldout-evaluation-plan.json") + for trial in ("baseline", *order): + score = read_object(root / "scores" / (trial + ".json")) + for key, expected in ( + ("kind", "reduced_compute_pilot_not_official_RSI_score"), + ("split", "test"), + ("fixed_policy_depths", POLICY), + ("seed", required(plan, "seed")), + ( + "requested_examples_per_bin", + required(plan, "requested_examples_per_bin"), + ), + ): + require_equal(required(score, key), expected, key) + for key, name in ( + ("checkpoint_sha256", "model.safetensors"), + ("config_sha256", "config.json"), + ): + require_equal( + required(score, key), digest(root / "selected" / trial / name), key + ) + current_panel = string(required(score, "panel_sha256"), "panel hash") + if re.fullmatch(r"[0-9a-f]{64}", current_panel) is None: + raise ValueError("Malformed test panel hash") + current_size = integer(required(score, "actual_examples"), "actual_examples") + if current_size < 1: + raise ValueError("Test panel must contain observed examples") + if panel is not None: + require_equal(current_panel, panel, "test panel") + require_equal(current_size, panel_size, "test panel size") + panel = current_panel + panel_size = current_size + values = metric(required(score, "depths"), [1, 2, 4]) + require_equal( + finite(required(score, "reward"), "reward"), + values["reward"], + "reward calculation", + ) + values["raw_depths"] = score["depths"] + scores[trial] = values + for entry in reports: + trial_report = mapping(entry, "trial report") + if trial_report["selection"] == "baseline": + require_equal( + scores[string(trial_report["trial"], "trial")], + scores["baseline"], + "baseline selection score", + ) + pairs: list[JSONValue] = [] + for index, pair_entry in enumerate(sequence(study["pairs"], "pairs"), 1): + pair = mapping(pair_entry, "pair") + treatment = string(required(pair, "traincheck"), "traincheck") + control = string(required(pair, "control"), "control") + pairs.append( + { + "pair": index, + "control": control, + "traincheck": treatment, + "treatment_minus_control_reward": finite( + scores[treatment]["reward"], "reward" + ) + - finite(scores[control]["reward"], "reward"), + } + ) + return { + "kind": "two_pair_development_pilot_no_statistical_claim", + "trials": reports, + "excluded_setup_trials": setup, + "exclusion_reason": study["revision_reason"], + "heldout": json_object(scores, "heldout scores"), + "pairs": pairs, + "test_panel_sha256": panel, + "finalization_sha256": digest(root / "finalization.json"), + "limitations": [ + "Training time for failed or unobserved training attempts remains unknown, not zero.", + "Operational durations include queue wait and transfer; they are not agent wall time or token usage.", + "Agent summary excerpts are verbatim self-reports, not verified causal explanations.", + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("freeze", "begin-scoring", "analyze")) + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + root = args.root.resolve(strict=True) + if args.mode == "freeze": + freeze(root) + elif args.mode == "begin-scoring": + begin_scoring(root) + else: + print(json.dumps(analyze(root), indent=2, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/tools/rsi_pilot/diagnose_adaptive.py b/tools/rsi_pilot/diagnose_adaptive.py new file mode 100644 index 00000000..7825c394 --- /dev/null +++ b/tools/rsi_pilot/diagnose_adaptive.py @@ -0,0 +1,195 @@ +"""Operator driver for a bounded three-update TrainCheck diagnostic probe. + +The launcher must hold the experiment's CPU lock and apply an overall timeout. +Each expensive phase also uses the existing process-group resource supervisor. +No invariant is inferred, modified, or removed during candidate diagnostics. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import time +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +from tools.fresh_validation.contracts import boolean, integer, read_object, required + + +@dataclass(frozen=True) +class Driver: + study: Path + python: Path + cpus: str = "8,10,12,14" + + @property + def source(self) -> Path: + return self.study / "source" + + def phase( + self, + output: Path, + command: Sequence[str], + *, + timeout: int, + allow_checker_failure: bool = False, + ) -> None: + command_line = [ + "taskset", + "-c", + self.cpus, + str(self.python), + str(self.source / "tools/fresh_validation/run_phase.py"), + "--source", + str(self.source), + "--output", + str(output), + "--timeout", + str(timeout), + "--memory-gib", + "4", + "--trace-mib", + "256", + "--", + "env", + f"PYTHONPATH={self.source}:{self.study / 'overlay'}", + str(self.python), + *command, + ] + process = subprocess.run(command_line, check=False) + result = read_object(output / "phase-result.json") + child_exit = integer(required(result, "exit_code"), "exit_code") + unchanged = boolean(required(result, "source_unchanged"), "source_unchanged") + stopped = required(result, "stop_reason") + # The coverage observer returns 1 for a checker/observer error. Its + # explicit error records are still summarized; limits and missing output + # never become an empty or passing report. + allowed = (0, 1) if allow_checker_failure else (0,) + if ( + child_exit not in allowed + or not unchanged + or stopped is not None + or process.returncode not in allowed + ): + raise RuntimeError( + f"Diagnostic phase failed: {output.name}; inspect {output}" + ) + + def diagnose(self, candidate: Path) -> Path: + candidate = candidate.resolve() + config = candidate / "config.toml" + if not config.is_file(): + raise FileNotFoundError(f"Candidate configuration missing: {config}") + invariants = self.study / "reference/invariants.json" + digest = hashlib.sha256(invariants.read_bytes()).hexdigest() + output = candidate / "diagnostics" + output.mkdir(exist_ok=False) + started = time.monotonic() + trace_output = output / "collection/result" + self.phase( + output / "collection", + [ + str(self.study / "bin/trace_adaptive.py"), + "--workspace", + str(self.study / "adaptive-workspace"), + "--output", + str(trace_output), + "--mode", + "selective", + "--invariants", + str(invariants), + "--", + "--config", + str(config), + "--steps", + "3", + "--pool-size", + "0", + "--require-complete", + ], + timeout=180, + ) + traces = sorted((trace_output / "traces").glob("trace_API*.log")) + if not traces or not all(path.stat().st_size > 0 for path in traces): + raise RuntimeError( + "Successful diagnostic collection must produce API records" + ) + coverage = output / "coverage.json" + self.phase( + output / "checking", + [ + str(self.source / "tools/fresh_validation/check_coverage.py"), + "--traces", + *map(str, traces), + "--invariants", + str(invariants), + "--output", + str(coverage), + ], + timeout=90, + allow_checker_failure=True, + ) + report = output / "agent_report.json" + self.phase( + output / "summary", + [ + str(self.study / "bin/summarize_check.py"), + "--coverage", + str(coverage), + "--collection-policy", + str(trace_output / "traces/instr_opts.json"), + "--output", + str(report), + ], + timeout=20, + ) + if hashlib.sha256(invariants.read_bytes()).hexdigest() != digest: + raise RuntimeError("Frozen reference invariants changed during diagnosis") + manifest = { + "schema_version": 1, + "status": "complete", + "invariants_sha256": digest, + "candidate_config_sha256": hashlib.sha256(config.read_bytes()).hexdigest(), + "elapsed_seconds_including_collection_checking_and_summary": time.monotonic() + - started, + "scope": "Three fresh CPU updates, API-only selective collection and offline checking", + "limitations": [ + "Probe overrides steps to three; research loss receives total_steps=3, not the full-training horizon.", + "A clean probe does not establish full-run health or improved model quality.", + "Changed training methods can legitimately violate baseline invariants.", + "No model-variable tracker or online checker is enabled.", + ], + } + (output / "diagnostic_manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n" + ) + return report + + +class Options(argparse.Namespace): + study: Path + python: Path + candidate: Path + cpus: str + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--study", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--cpus", default="8,10,12,14") + parser.add_argument("candidate", type=Path) + args = parser.parse_args(namespace=Options()) + # Resolving a venv's Python symlink switches to the base interpreter and + # loses its installed dependencies; preserve the executable's venv path. + report = Driver(args.study.resolve(), args.python.absolute(), args.cpus).diagnose( + args.candidate + ) + print(json.dumps({"report": str(report)}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/rsi_pilot/run_candidate.py b/tools/rsi_pilot/run_candidate.py new file mode 100644 index 00000000..f203b5c3 --- /dev/null +++ b/tools/rsi_pilot/run_candidate.py @@ -0,0 +1,716 @@ +"""Bounded client for the CPU adaptive-reasoning development pilot. + +The operator owns the trial manifest, trainer, reference invariants and evaluator. +Solvers supply only a research program interpreted by the upstream TensorProgram. +This client is an experiment harness, not a security boundary against its owner. +""" + +import argparse +import fcntl +import hashlib +import json +import math +import os +import shlex +import signal +import subprocess +import sys +import time +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + mapping, + number, + read_object, + required, + sequence, + string, +) + +# Only this fixed operator-owned program executes. The research source is loaded +# as TOML data and handed to the unchanged upstream AST interpreter. +PREFLIGHT_PROGRAM = """import json +import sys +import tomllib +from pathlib import Path +from latent_reasoning.research import TensorProgram + +with Path(sys.argv[1]).open("rb") as handle: + config = tomllib.load(handle) +training = config["training"] +if not isinstance(training, dict): + raise TypeError("training must be a table") +source = training["research_source"] +if not isinstance(source, str): + raise TypeError("research_source must be text") +result: dict[str, object] +try: + TensorProgram(source) +except ValueError as exc: + result = {"schema_version": 1, "valid": False, + "error": {"type": type(exc).__name__, "message": str(exc)}} +else: + result = {"schema_version": 1, "valid": True, "error": None} +Path(sys.argv[2]).write_text(json.dumps(result, allow_nan=False) + "\\n") +raise SystemExit(0 if result["valid"] else 2) +""" + + +def checked_text(record: JSONObject, key: str) -> str: + value = string(required(record, key), key) + if not value: + raise ValueError(f"{key} must be nonempty") + return value + + +def program_text(path: Path, workspace: Path) -> str: + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(workspace.resolve(strict=True)): + raise ValueError("Research program must be inside this solver's workspace") + raw = resolved.read_bytes() + if not 0 < len(raw) <= 32768: + raise ValueError("Research program must contain 1..32768 UTF-8 bytes") + return raw.decode("utf-8") + + +def configure(template: str, source: str) -> str: + """Insert source as a TOML basic string; never interpolate it into shell code.""" + marker = "# PILOT_RESEARCH_SOURCE" + if template.count(marker) != 1: + raise ValueError("Frozen profile must contain exactly one source marker") + # TOML forbids JSON's surrogate-pair escapes for non-BMP Unicode characters. + return template.replace( + marker, "research_source = " + json.dumps(source, ensure_ascii=False) + ) + + +def remote_command(manifest: JSONObject, arguments: list[str]) -> list[str]: + """All remote arguments are separately shell-quoted for OpenSSH's command.""" + return [ + "ssh", + "-o", + "BatchMode=yes", + checked_text(manifest, "host"), + shlex.join(arguments), + ] + + +def stop_process_group(process: subprocess.Popen[bytes]) -> None: + """Stop only the session created for this command, including descendants.""" + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass # The owned process group has already exited. + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + # A leader may exit on TERM while its descendants ignore it. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + +def execute(command: list[str], log: Path, timeout: int) -> int: + with log.open("wb") as handle: + with subprocess.Popen( + command, stdout=handle, stderr=subprocess.STDOUT, start_new_session=True + ) as process: + try: + return process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # GNU timeout also bounds the remote process, independently of + # local SSH cleanup if the connection disappears. + stop_process_group(process) + handle.write(b"\nCLIENT_TIMEOUT\n") + return 124 + except BaseException: + stop_process_group(process) + raise + + +def checked_execute(command: list[str], log: Path, timeout: int = 60) -> None: + code = execute(command, log, timeout) + if code: + raise subprocess.CalledProcessError(code, command) + + +def upload(manifest: JSONObject, source: Path, target: str, log: Path) -> None: + checked_execute( + [ + "scp", + "-q", + "-o", + "BatchMode=yes", + str(source), + checked_text(manifest, "host") + ":" + shlex.quote(target), + ], + log, + ) + + +def download(manifest: JSONObject, source: str, target: Path, log: Path) -> None: + checked_execute( + [ + "scp", + "-q", + "-o", + "BatchMode=yes", + checked_text(manifest, "host") + ":" + shlex.quote(source), + str(target), + ], + log, + ) + + +def train_arguments(manifest: JSONObject, remote: str) -> list[str]: + workspace = checked_text(manifest, "remote_workspace") + # Upstream recipe_hashes resolves its workspace from module __file__, not cwd. + return [ + "flock", + "-w", + "360", + checked_text(manifest, "remote_compute_lock"), + "timeout", + "--signal=TERM", + "--kill-after=10s", + "300s", + "taskset", + "-c", + checked_text(manifest, "affinity"), + "env", + "PYTHONDONTWRITEBYTECODE=1", + "PYTHONHASHSEED=0", + "CUDA_VISIBLE_DEVICES=", + "OMP_NUM_THREADS=4", + "MKL_NUM_THREADS=4", + "OPENBLAS_NUM_THREADS=1", + "PYTHONPATH=" + checked_text(manifest, "remote_overlay") + ":" + workspace, + checked_text(manifest, "remote_python"), + "-m", + "latent_reasoning.train", + "--config", + remote + "/config.toml", + "--output", + remote + "/training", + "--device", + "cpu", + "--require-complete", + ] + + +def preflight_arguments(manifest: JSONObject, remote: str) -> list[str]: + """Run syntax validation under the same runtime, without invoking training.""" + return [ + "flock", + "-w", + "360", + checked_text(manifest, "remote_compute_lock"), + "timeout", + "--signal=TERM", + "--kill-after=10s", + "30s", + "taskset", + "-c", + checked_text(manifest, "affinity"), + "env", + "PYTHONDONTWRITEBYTECODE=1", + "PYTHONHASHSEED=0", + "CUDA_VISIBLE_DEVICES=", + "OMP_NUM_THREADS=4", + "MKL_NUM_THREADS=4", + "OPENBLAS_NUM_THREADS=1", + "PYTHONPATH=" + + checked_text(manifest, "remote_overlay") + + ":" + + checked_text(manifest, "remote_workspace"), + checked_text(manifest, "remote_python"), + "-c", + PREFLIGHT_PROGRAM, + remote + "/config.toml", + remote + "/result.json", + ] + + +def validate_preflight(result: JSONObject, code: int) -> bool: + """Reject malformed or contradictory backend receipts, including exit status.""" + if integer(required(result, "schema_version"), "schema_version") != 1: + raise ValueError("Expected preflight schema_version 1") + valid = boolean(required(result, "valid"), "valid") + error = required(result, "error") + if valid: + if code != 0 or error is not None: + raise ValueError("Valid preflight must have exit code 0 and no error") + else: + detail = mapping(error, "error") + if code != 2 or string(required(detail, "type"), "error.type") != "ValueError": + raise ValueError("Invalid program must have exit code 2 and ValueError") + if not string(required(detail, "message"), "error.message"): + raise ValueError("Program rejection must include an error message") + return valid + + +def check_program(trial: Path, source_path: Path) -> JSONObject: + """Provide the upstream free syntax check without reserving any train budget. + + Preflights use the same exclusive trial lock as training. Their immutable + receipts live separately; neither successful nor failed syntax checks create + or modify usage.json, candidate directories, or diagnostic reservations. + """ + manifest_path = trial / "trial.json" + manifest = read_object(manifest_path) + source = program_text(source_path, Path(checked_text(manifest, "workspace"))) + arm = checked_text(manifest, "arm") + if arm not in ("control", "traincheck"): + raise ValueError("Unknown trial arm") + template_bytes = Path(checked_text(manifest, "profile")).read_bytes() + profile_digest = hashlib.sha256(template_bytes).hexdigest() + if profile_digest != checked_text(manifest, "profile_sha256"): + raise ValueError("Frozen training profile changed") + with (trial / "run.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + preflights = trial / "preflights" + preflights.mkdir(exist_ok=True) + attempt = 1 + while (preflights / f"preflight-{attempt:02d}").exists(): + attempt += 1 + name = f"preflight-{attempt:02d}" + output = preflights / name + output.mkdir() + remote = checked_text(manifest, "remote_trial") + "/preflights/" + name + errors: list[JSONValue] = [] + record: JSONObject = { + "schema_version": 1, + "preflight": name, + "arm": arm, + "status": "started", + "program_sha256": hashlib.sha256(source.encode()).hexdigest(), + "profile_sha256": profile_digest, + "trial_manifest_sha256": hashlib.sha256( + manifest_path.read_bytes() + ).hexdigest(), + "remote_directory": remote, + "errors": errors, + } + write_record(output / "preflight.json", record) + started = time.monotonic() + phase = "prepare_local" + try: + (output / "program.py").write_text(source) + (output / "config.toml").write_text( + configure(template_bytes.decode(), source) + ) + phase = "prepare_remote" + checked_execute( + remote_command(manifest, ["mkdir", "-p", remote]), output / "setup.log" + ) + phase = "upload_config" + upload( + manifest, + output / "config.toml", + remote + "/config.toml", + output / "upload.log", + ) + phase = "check_program" + code = execute( + remote_command(manifest, preflight_arguments(manifest, remote)), + output / "check.log", + 430, + ) + record["check_exit_code"] = code + if code not in (0, 2): + record["status"] = "execution_failed" + errors.append( + { + "phase": phase, + "type": "ProcessExit", + "message": f"Syntax check exited {code}; see check.log", + } + ) + else: + phase = "download_result" + download( + manifest, + remote + "/result.json", + output / "result.json", + output / "download.log", + ) + phase = "validate_result" + result = read_object(output / "result.json") + valid = validate_preflight(result, code) + record["valid"] = valid + record["result"] = result + record["status"] = "program_valid" if valid else "program_invalid" + if not valid: + errors.append({"phase": phase, "error": result["error"]}) + return record + except BaseException as exc: + record["status"] = "client_error" + errors.append( + { + "phase": phase, + "type": type(exc).__name__, + "message": str(exc) or repr(exc), + } + ) + raise + finally: + record["operational_seconds_including_transfer"] = ( + time.monotonic() - started + ) + write_record(output / "preflight.json", record) + + +def write_record(path: Path, record: JSONObject) -> None: + """Replace one receipt atomically; callers hold this trial's exclusive lock.""" + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(record, indent=2, allow_nan=False) + "\n") + temporary.replace(path) + + +def reserve(trial: Path, manifest: JSONObject, diagnose: bool) -> tuple[int, int]: + """Reserve attempts before remote work; failed attempts remain charged. + + Missing max_diagnostics explicitly disables diagnostics. A missing usage + ledger is a new/legacy trial: existing complete run receipts reconstruct its + reservations. Malformed/missing receipts fail closed instead of resetting it. + """ + limit = integer(required(manifest, "max_candidates"), "max_candidates") + if not 1 <= limit <= 3: + raise ValueError("max_candidates must be between 1 and 3") + diagnostic_limit = ( + integer(manifest["max_diagnostics"], "max_diagnostics") + if "max_diagnostics" in manifest + else 0 + ) + if not 0 <= diagnostic_limit <= 1: + raise ValueError("max_diagnostics must be 0 or 1") + ledger_path = trial / "usage.json" + if ledger_path.exists(): + ledger = read_object(ledger_path) + candidates = integer( + required(ledger, "candidates_reserved"), "candidates_reserved" + ) + diagnostics = integer( + required(ledger, "diagnostics_reserved"), "diagnostics_reserved" + ) + else: + existing = sorted((trial / "runs").glob("candidate-*")) + candidates = len(existing) + diagnostics = sum( + boolean( + required(read_object(path / "run.json"), "diagnostics_requested"), + "diagnostics_requested", + ) + for path in existing + ) + if candidates < 0 or diagnostics < 0 or diagnostics > candidates: + raise ValueError("Invalid reservation ledger") + if candidates >= limit: + raise ValueError(f"Candidate budget exhausted ({limit})") + if diagnose and diagnostics >= diagnostic_limit: + raise ValueError(f"Diagnostic budget exhausted ({diagnostic_limit})") + candidates += 1 + diagnostics += diagnose + write_record( + ledger_path, + { + "candidates_reserved": candidates, + "diagnostics_reserved": diagnostics, + }, + ) + return candidates, diagnostics + + +def validate_results( + results: JSONObject, manifest: JSONObject +) -> tuple[int, int | float, list[JSONValue]]: + # The upstream result writer disallows nonfinite values everywhere. Validate + # before assigning payloads to the receipt, so cleanup can always serialize it. + json.dumps(results, allow_nan=False) + steps = integer(required(results, "completed_steps"), "completed_steps") + if steps < 1: + raise ValueError("Training must report a positive completed step count") + if "expected_steps" in manifest: + expected = integer(manifest["expected_steps"], "expected_steps") + if expected < 1 or steps != expected: + raise ValueError( + "Completed steps do not match the frozen trial requirement" + ) + elapsed = number(required(results, "training_seconds"), "training_seconds") + if not math.isfinite(elapsed) or elapsed < 0: + raise ValueError("Training duration must be finite and nonnegative") + if checked_text(results, "device_backend") != "cpu": + raise ValueError("This pilot requires CPU results") + final = sequence(required(results, "final"), "final") + if not final: + raise ValueError("Training must return nonempty development evaluation results") + for entry in final: + row = mapping(entry, "development result") + depth = integer(required(row, "loops"), "loops") + accuracy = number( + required(row, "macro_accuracy_percent"), "macro_accuracy_percent" + ) + if depth < 1 or not math.isfinite(accuracy) or not 0 <= accuracy <= 100: + raise ValueError("Malformed development depth or accuracy") + # read_object has already validated recursive JSON values; preserve them. + stored_final = results["final"] + assert isinstance(stored_final, list) + return steps, elapsed, stored_final + + +def validate_diagnostics(report: JSONObject) -> None: + json.dumps(report, allow_nan=False) + if integer(required(report, "schema_version"), "schema_version") != 1: + raise ValueError("Expected agent report schema_version 1") + if checked_text(report, "outcome") not in { + "violations", + "incomplete", + "no_evidence", + "active_checks_passed", + }: + raise ValueError("Unknown TrainCheck report outcome") + boolean(required(report, "coverage_incomplete"), "coverage_incomplete") + counts = mapping(required(report, "counts"), "counts") + for name in ( + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + "observed_active", + "observed_active_passes", + "observed_dormant", + "observed_violations", + "unknown_outcomes", + "unobserved_coverage", + "skipped_variable_checks", + "completed_decisions", + ): + if integer(required(counts, name), name) < 0: + raise ValueError(f"Negative diagnostic count: {name}") + provenance = mapping(required(report, "coverage_source"), "coverage_source") + string(required(provenance, "path"), "coverage_source.path") + string(required(provenance, "sha256"), "coverage_source.sha256") + + +def validate_diagnostic_artifacts(report: JSONObject, output: Path) -> JSONObject: + """Link the compact report to locally readable, unchanged complete evidence.""" + coverage_path = output / "coverage.json" + manifest_path = output / "diagnostic_manifest.json" + coverage_bytes = coverage_path.read_bytes() + coverage_digest = hashlib.sha256(coverage_bytes).hexdigest() + provenance = mapping(required(report, "coverage_source"), "coverage_source") + if coverage_digest != string( + required(provenance, "sha256"), "coverage_source.sha256" + ): + raise ValueError("Downloaded coverage does not match the agent report hash") + # The original report/source paths stay unchanged; local locations are separate. + read_object(coverage_path) + diagnostic = read_object(manifest_path) + if ( + integer(required(diagnostic, "schema_version"), "diagnostic schema_version") + != 1 + ): + raise ValueError("Expected diagnostic manifest schema_version 1") + if checked_text(diagnostic, "status") != "complete": + raise ValueError("Diagnostic manifest does not report completion") + config_digest = hashlib.sha256((output / "config.toml").read_bytes()).hexdigest() + if checked_text(diagnostic, "candidate_config_sha256") != config_digest: + raise ValueError( + "Diagnostic manifest belongs to a different candidate configuration" + ) + return { + "coverage": {"path": str(coverage_path.resolve()), "sha256": coverage_digest}, + "diagnostic_manifest": { + "path": str(manifest_path.resolve()), + "sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), + }, + } + + +def run(trial: Path, source_path: Path, diagnose: bool) -> JSONObject: + manifest_path = trial / "trial.json" + manifest = read_object(manifest_path) + workspace = Path(checked_text(manifest, "workspace")) + source = program_text(source_path, workspace) + arm = checked_text(manifest, "arm") + if arm not in ("control", "traincheck"): + raise ValueError("Unknown trial arm") + if diagnose and arm != "traincheck": + raise ValueError("TrainCheck diagnostics are not enabled for this arm") + template_path = Path(checked_text(manifest, "profile")) + template_bytes = template_path.read_bytes() + if hashlib.sha256(template_bytes).hexdigest() != checked_text( + manifest, "profile_sha256" + ): + raise ValueError("Frozen training profile changed") + runs = trial / "runs" + runs.mkdir(exist_ok=True) + with (trial / "run.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + attempt, diagnostic_attempts = reserve(trial, manifest, diagnose) + candidate = f"candidate-{attempt:02d}" + output = runs / candidate + output.mkdir() + remote = checked_text(manifest, "remote_trial") + "/" + candidate + errors: list[JSONValue] = [] + record: JSONObject = { + "candidate": candidate, + "arm": arm, + "diagnostics_requested": diagnose, + "program_sha256": hashlib.sha256(source.encode()).hexdigest(), + "profile_sha256": hashlib.sha256(template_bytes).hexdigest(), + "trial_manifest_sha256": hashlib.sha256( + manifest_path.read_bytes() + ).hexdigest(), + "remote_directory": remote, + "status": "started", + "candidates_reserved": attempt, + "diagnostics_reserved": diagnostic_attempts, + "errors": errors, + } + write_record(output / "run.json", record) + started = time.monotonic() + phase = "prepare_local" + try: + (output / "program.py").write_text(source) + (output / "config.toml").write_text( + configure(template_bytes.decode(), source) + ) + phase = "prepare_remote" + checked_execute( + remote_command(manifest, ["mkdir", "-p", remote]), output / "setup.log" + ) + phase = "upload_config" + upload( + manifest, + output / "config.toml", + remote + "/config.toml", + output / "upload.log", + ) + phase = "training" + code = execute( + remote_command(manifest, train_arguments(manifest, remote)), + output / "training.log", + 700, + ) + record["training_exit_code"] = code + record["status"] = "training_complete" if code == 0 else "training_failed" + if code: + errors.append( + { + "phase": phase, + "type": "ProcessExit", + "message": f"Training exited {code}; see training.log", + } + ) + if code == 0: + phase = "download_results" + download( + manifest, + remote + "/training/results.json", + output / "results.json", + output / "download-results.log", + ) + phase = "validate_results" + results = read_object(output / "results.json") + steps, elapsed, final = validate_results(results, manifest) + record["completed_steps"] = steps + record["training_seconds"] = elapsed + record["development_depths"] = final + if diagnose: + # This operator-supplied driver performs a short independently + # rerun trace and offline check. It does not alter the checkpoint. + phase = "diagnostics" + # The frozen driver owns the same compute lock and its 240s + # timeout. Do not nest that lock or charge queue wait to tracing. + diagnostic = [checked_text(manifest, "diagnostic_driver"), remote] + code = execute( + remote_command(manifest, diagnostic), + output / "diagnostics.log", + 700, + ) + record["diagnostic_exit_code"] = code + if code == 0: + phase = "download_diagnostics" + download( + manifest, + remote + "/diagnostics/agent_report.json", + output / "agent_report.json", + output / "download-diagnostics.log", + ) + phase = "read_diagnostics" + report = read_object(output / "agent_report.json") + validate_diagnostics(report) + for name in ("coverage.json", "diagnostic_manifest.json"): + phase = "download_" + name + download( + manifest, + remote + "/diagnostics/" + name, + output / name, + output / ("download-" + name + ".log"), + ) + phase = "validate_diagnostic_artifacts" + record["traincheck_evidence"] = validate_diagnostic_artifacts( + report, output + ) + record["traincheck_report"] = report + record["diagnostic_status"] = "complete" + else: + record["diagnostic_status"] = "execution_failed" + errors.append( + { + "phase": phase, + "type": "ProcessExit", + "message": f"Diagnostics exited {code}; see diagnostics.log", + } + ) + return record + except BaseException as exc: + record["status"] = "client_error" + errors.append( + { + "phase": phase, + "type": type(exc).__name__, + "message": str(exc) or repr(exc), + } + ) + raise + finally: + record["operational_seconds_including_transfer_and_diagnostics"] = ( + time.monotonic() - started + ) + write_record(output / "run.json", record) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trial", type=Path, required=True) + parser.add_argument("--program", type=Path, required=True) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--diagnose", action="store_true") + mode.add_argument("--check-program", action="store_true") + args = parser.parse_args() + trial = args.trial.resolve(strict=True) + result = ( + check_program(trial, args.program) + if args.check_program + else run(trial, args.program, args.diagnose) + ) + print(json.dumps(result, indent=2, allow_nan=False)) + if args.check_program and result["status"] == "program_invalid": + return 2 + return int(bool(sequence(required(result, "errors"), "errors"))) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rsi_pilot/score_adaptive.py b/tools/rsi_pilot/score_adaptive.py new file mode 100644 index 00000000..fe23e7dd --- /dev/null +++ b/tools/rsi_pilot/score_adaptive.py @@ -0,0 +1,166 @@ +"""Operator-only scoring for the reduced-compute adaptive-reasoning pilot. + +Uses the public upstream generator, model and FLOP accounting. It does not +produce an official RSI result: CPU training has no trusted-service receipt and +uses the public partition instead of the official service's private partition. +""" + +import argparse +import hashlib +import json +import math +import sys +from pathlib import Path +from typing import Literal + +import torch +from safetensors.torch import load_file + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + integer, + json_list, + json_object, + mapping, + number, + read_object, + required, + sequence, +) + + +def score( + run: Path, + output: Path, + seed: int, + per_bin: int, + split: Literal["validation", "test"] = "test", +) -> JSONObject: + # These public upstream modules are supplied by the pinned workspace on + # PYTHONPATH. Their accepted output is validated before calculating scores. + from latent_reasoning.data import ( + calibrated_panel, + evaluation_set, + runtime_partition_key, + ) + from latent_reasoning.evaluate import evaluate + from latent_reasoning.model import LoopedTransformer, ModelConfig + from latent_reasoning.reward import aggregate_reward + + config = read_object(run / "config.json") + model_config = mapping(required(config, "model"), "model") + expected = { + "modulus": 7, + "width": 64, + "heads": 4, + "layers": 1, + "max_loops": 8, + "max_length": 32, + } + if set(model_config) != set(expected) or any( + integer(required(model_config, key), key) != value + for key, value in expected.items() + ): + raise ValueError("Checkpoint must use the unchanged upstream backbone") + torch.set_num_threads(4) + model = LoopedTransformer(ModelConfig()) + model.load_state_dict(load_file(run / "model.safetensors"), strict=True) + model.to(torch.device("cpu")) + # Test panels are generated only after solver finalization. Validation mode + # lets the operator check scoring parity before freezing the experiment. + if runtime_partition_key() is not None: + raise ValueError("This pilot requires the public content-hash partition") + # The two-operation test partition has fewer than 512 distinct expressions. + # The public task's calibrated panel censuses both finite small strata and + # samples the requested count in each larger stratum without duplicates. + # Keep validation generation unchanged for native-trainer parity checks. + examples = ( + calibrated_panel(split, per_bin, seed) + if split == "test" + else evaluation_set(split, [1, 2, 3, 4, 5], per_bin, 7, ("+", "-"), seed) + ) + rows_value: object = evaluate(model, examples, [1, 2, 4], torch.device("cpu")) + rows = [ + json_object(row, "depth result") + for row in sequence(rows_value, "depth results") + ] + if [integer(required(row, "loops"), "loops") for row in rows] != [1, 2, 4]: + raise ValueError("Expected exactly the frozen fixed-depth policy evaluations") + points = [rows[0], rows[1], rows[2], rows[2]] + # This JSON round-trip validates that the retained raw evidence is serializable + # with no NaN/Infinity before using the upstream's numeric range validation. + json.dumps(rows, allow_nan=False) + reward_value: object = aggregate_reward( + [ + { + "accuracy_percent": number( + required(row, "macro_accuracy_percent"), "accuracy" + ), + "mean_dense_matmul_flops": number( + required(row, "macro_mean_dense_matmul_flops"), "work" + ), + } + for row in points + ] + ) + reward = number(reward_value, "reward") + if not math.isfinite(reward): + raise ValueError("Reward must be finite") + panel = [ + { + "tokens": list(example.tokens), + "label": example.label, + "operations": example.operations, + } + for example in examples + ] + panel_bytes = json.dumps(panel, sort_keys=True, separators=(",", ":")).encode() + report: JSONObject = { + "kind": "reduced_compute_pilot_not_official_RSI_score", + "split": split, + "seed": seed, + "requested_examples_per_bin": per_bin, + "panel_generator": ( + "upstream_calibrated_panel" + if split == "test" + else "upstream_evaluation_set" + ), + "actual_examples": len(examples), + "fixed_policy_depths": [1, 2, 4, 4], + "reward": reward, + "depths": json_list(rows, "depths"), + "panel_sha256": hashlib.sha256(panel_bytes).hexdigest(), + "checkpoint_sha256": hashlib.sha256( + (run / "model.safetensors").read_bytes() + ).hexdigest(), + "config_sha256": hashlib.sha256((run / "config.json").read_bytes()).hexdigest(), + } + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + raise ValueError("Scoring output already exists") + output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--seed", required=True, type=int) + parser.add_argument("--per-bin", default=512, type=int) + parser.add_argument("--split", choices=("validation", "test"), default="test") + args = parser.parse_args() + if args.per_bin < 1: + parser.error("per-bin must be positive") + print( + json.dumps( + score(args.run, args.output, args.seed, args.per_bin, args.split), indent=2 + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tools/rsi_pilot/trace_adaptive.py b/tools/rsi_pilot/trace_adaptive.py new file mode 100644 index 00000000..b84fe770 --- /dev/null +++ b/tools/rsi_pilot/trace_adaptive.py @@ -0,0 +1,454 @@ +"""Trace only training updates in the pinned RSI adaptive-reasoning workload. + +Run in a fresh interpreter. The upstream trainer is imported unchanged; a line +hook gates API tracing from optimizer.zero_grad through the optimizer update. +Initialization, data sampling, evaluation, and evidence capture stay untraced. +This CPU development adapter does not produce official benchmark receipts. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import importlib +import json +import os +import pickle +import random +import sys +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import CodeType, FrameType, FunctionType +from typing import Literal, TypedDict + +import numpy as np +import torch + +Mode = Literal["plain", "reference", "selective"] +TRAIN_SHA256 = "809b62dfe68b4e9feb8bc38f24c85aad93df1f58ba0e9967f42e6560d8a6c08f" + + +@dataclass(frozen=True) +class WindowLines: + main: int + start: int + stop: int + + +def locate_window(path: Path, expected_sha256: str) -> WindowLines: + """Reject source drift and ambiguous/moved anchors instead of guessing.""" + source = path.read_bytes() + if hashlib.sha256(source).hexdigest() != expected_sha256: + raise ValueError("Upstream train.py differs from the pinned source hash") + tree = ast.parse(source, filename=str(path)) + mains = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "main" + ] + if len(mains) != 1: + raise ValueError("Expected exactly one upstream main function") + main = mains[0] + starts = [ + node + for node in ast.walk(main) + if isinstance(node, ast.Expr) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Attribute) + and isinstance(node.value.func.value, ast.Name) + and node.value.func.value.id == "optimizer" + and node.value.func.attr == "zero_grad" + ] + stops = [ + node + for node in ast.walk(main) + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "completed" + and isinstance(node.value, ast.Name) + and node.value.id == "step" + ] + if len(starts) != 1 or len(stops) != 1: + raise ValueError("Expected unique zero_grad and completed = step anchors") + start, stop = starts[0], stops[0] + loops = [ + node + for node in ast.walk(main) + if isinstance(node, ast.For) + and start in tuple(ast.walk(node)) + and stop in tuple(ast.walk(node)) + ] + if len(loops) != 1 or start.lineno >= stop.lineno: + raise ValueError("Training anchors must belong to one ordered for loop") + return WindowLines(main.lineno, start.lineno, stop.lineno) + + +def tensor_evidence(value: torch.Tensor) -> dict[str, object]: + if value.device.type != "cpu" or value.layout != torch.strided: + raise ValueError("This pilot snapshots only strided CPU tensors") + raw = value.detach().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + return { + "dtype": str(value.dtype), + "shape": list(value.shape), + "requires_grad": value.requires_grad, + "sha256": hashlib.sha256(raw).hexdigest(), + } + + +def state_value(value: object) -> object: + """Snapshot optimizer containers without retaining live mutable references.""" + if isinstance(value, torch.Tensor): + return {"kind": "tensor", "value": tensor_evidence(value)} + if value is None or type(value) in (bool, int, float, str): + return value + if isinstance(value, (list, tuple)): + return { + "kind": "tuple" if isinstance(value, tuple) else "list", + "items": [state_value(item) for item in value], + } + if isinstance(value, dict): + items: list[list[object]] = [] + for key, item in value.items(): + if type(key) not in (int, str): + raise TypeError("Optimizer state keys must be integers or strings") + items.append([key, state_value(item)]) + return {"kind": "dict", "items": items} + raise TypeError(f"Unsupported optimizer state value: {type(value).__name__}") + + +def state_snapshot(frame: FrameType) -> dict[str, object]: + model: object = frame.f_locals["model"] + optimizer: object = frame.f_locals["optimizer"] + rng: object = frame.f_locals["rng"] + if not isinstance(model, torch.nn.Module): + raise TypeError("Upstream model must be a torch.nn.Module") + if not isinstance(optimizer, torch.optim.Optimizer): + raise TypeError("Upstream optimizer must be a torch.optim.Optimizer") + if not isinstance(rng, random.Random): + raise TypeError("Upstream rng must be random.Random") + parameter_names = {id(value): name for name, value in model.named_parameters()} + optimizer_parameters: list[list[str]] = [] + for group in optimizer.param_groups: + names: list[str] = [] + for parameter in group["params"]: + if id(parameter) not in parameter_names: + raise ValueError("Optimizer references a tensor outside the model") + names.append(parameter_names[id(parameter)]) + optimizer_parameters.append(names) + result: dict[str, object] = { + "parameters": { + name: tensor_evidence(value) for name, value in model.named_parameters() + }, + "gradients": { + name: None if value.grad is None else tensor_evidence(value.grad) + for name, value in model.named_parameters() + }, + "buffers": { + name: tensor_evidence(value) for name, value in model.named_buffers() + }, + "optimizer": state_value(optimizer.state_dict()), + "optimizer_parameter_names": optimizer_parameters, + "custom_optimizer_states": state_value(frame.f_locals["optimizer_states"]), + "torch_rng": tensor_evidence(torch.get_rng_state()), + # These are trusted local RNG snapshots, serialized only for a byte digest. + "python_rng": hashlib.sha256(pickle.dumps(random.getstate())).hexdigest(), + "numpy_rng": hashlib.sha256(pickle.dumps(np.random.get_state())).hexdigest(), + "sampler_rng": hashlib.sha256(pickle.dumps(rng.getstate())).hexdigest(), + } + if "sampler" in frame.f_locals: + sampler: object = frame.f_locals["sampler"] + if not isinstance(sampler, torch.Generator): + raise TypeError("Upstream sampler must be a torch.Generator") + result["pool_rng"] = tensor_evidence(sampler.get_state()) + return result + + +class StepRecord(TypedDict): + step: int + seconds: float + before: dict[str, object] + after: dict[str, object] + loss: float + + +class StepWindow: + """A trace callback which returns itself only for the original main frame.""" + + def __init__( + self, + code: CodeType, + lines: WindowLines, + enable: Callable[[], None], + disable: Callable[[], None], + snapshot: Callable[[FrameType], dict[str, object]] = state_snapshot, + ) -> None: + self.code = code + self.lines = lines + self.enable = enable + self.disable = disable + self.snapshot = snapshot + self.active = False + self.records: list[StepRecord] = [] + self.before: dict[str, object] | None = None + self.started: float | None = None + self.step: int | None = None + self.failed = False + + def __call__(self, frame: FrameType, event: str, arg: object) -> StepWindow | None: + if frame.f_code is not self.code: + return None + if event in ("exception", "return") and self.active: + self.disable() + self.active = False + self.failed = True + if event != "line": + return self + if frame.f_lineno == self.lines.start: + if self.active or self.failed: + raise RuntimeError("Training window was reentered or previously failed") + step: object = frame.f_locals["step"] + if not isinstance(step, int) or isinstance(step, bool) or step < 1: + raise TypeError("Upstream step must be a positive integer") + if step != len(self.records) + 1: + raise ValueError("Training steps must be consecutive from one") + self.step = step + self.before = self.snapshot(frame) + self.started = time.perf_counter() + self.active = True + self.enable() + elif frame.f_lineno == self.lines.stop: + if not self.active: + raise RuntimeError("Training end reached without an active window") + self.disable() + assert self.started is not None + elapsed = time.perf_counter() - self.started + self.active = False + assert self.before is not None and self.step is not None + after = self.snapshot(frame) + loss: object = frame.f_locals["loss"] + if not isinstance(loss, torch.Tensor) or loss.ndim != 0: + raise TypeError("Upstream loss must be a scalar tensor") + self.records.append( + { + "step": self.step, + "seconds": elapsed, + "before": self.before, + "after": after, + "loss": float(loss.detach()), + } + ) + self.before = None + return self + + def close(self) -> None: + self.disable() + if self.active: + self.failed = True + self.active = False + + +@dataclass(frozen=True) +class Instrumentation: + enable: Callable[[], None] + disable: Callable[[], None] + drain: Callable[[], None] + instrumented_apis: int + + +def noop() -> None: + pass + + +def install_instrumentation( + mode: Mode, trace_dir: Path, invariants: Path | None +) -> Instrumentation: + if mode == "plain": + return Instrumentation(noop, noop, noop, 0) + from traincheck.config import config + from traincheck.config.config import InstrOpt + from traincheck.instrumentor.caches import META_VARS + from traincheck.instrumentor.control import start_step + from traincheck.instrumentor.proxy_wrapper.proxy_config import tensor_dump_format + + config.DISABLE_WRAPPER = True + config.INSTRUMENTATION_POLICY = {"interval": 1, "warm_up": 0} + config.INSTR_OPTS = None + config.MODEL_TRACKER_STYLE = None + config.INSTR_OPTS_FILE = str(trace_dir / "instr_opts.json") + if META_VARS["step"] != 0: + raise RuntimeError("Trace adapter requires a fresh interpreter") + for key in ("dump_tensor_hash", "dump_tensor_stats", "dump_tensor_full"): + tensor_dump_format[key] = False + if mode == "selective": + from traincheck.collect_trace import ( + get_model_tracker_instr_opts, + get_per_func_instr_opts, + ) + from traincheck.invariant.base_cls import read_inv_file + + assert invariants is not None + rules = read_inv_file(str(invariants)) + if not rules: + raise ValueError("Selective mode requires nonempty invariants") + if get_model_tracker_instr_opts(rules, "subclass") is not None: + raise ValueError("API-only pilot cannot satisfy variable-tracker rules") + options = InstrOpt(get_per_func_instr_opts(rules), None, True) + Path(config.INSTR_OPTS_FILE).write_text(options.to_json() + "\n") + loaded: object = config.load_instr_opts() + if not isinstance(loaded, InstrOpt) or loaded.model_tracker_style is not None: + raise TypeError("Saved API-only instrumentation policy failed validation") + from traincheck.instrumentor.tracer import Instrumentor + + count = Instrumentor( + torch, + scan_proxy_in_args=False, + use_full_instr=False, + API_dump_stack_trace=False, + sampling_interval=1, + warm_up_steps=0, + ).instrument() + + def enable() -> None: + start_step() + + def disable() -> None: + config.DISABLE_WRAPPER = True + + def drain() -> None: + from traincheck.instrumentor import dumper + + for name in ("api_trace_buffer", "var_trace_buffer"): + # Absence means no event was emitted for that stream. + buffer: object = getattr(dumper.THREAD_DATA, name, None) + if buffer is not None: + if not isinstance(buffer, dumper.TraceBuffer): + raise TypeError(f"Unexpected trace buffer: {name}") + buffer._flush() + buffer.queue.join() + + return Instrumentation(enable, disable, drain, count) + + +def run( + workspace: Path, + output: Path, + mode: Mode, + invariants: Path | None, + training_args: Sequence[str], +) -> None: + if mode not in ("plain", "reference", "selective"): + raise ValueError("Unknown tracing mode") + if (mode == "selective") != (invariants is not None): + raise ValueError("--invariants is required exactly for selective mode") + if sys.gettrace() is not None: + raise RuntimeError("Cannot replace an existing Python trace hook") + if any(arg.split("=", 1)[0] in ("--output", "--device") for arg in training_args): + raise ValueError("Adapter owns upstream --output and --device") + workspace, output = workspace.resolve(), output.resolve() + source = workspace / "latent_reasoning" / "train.py" + lines = locate_window(source, TRAIN_SHA256) + output.mkdir(parents=True, exist_ok=False) + trace_dir = output / "traces" + trace_dir.mkdir() + os.environ["TRAINCHECK_OUTPUT_DIR"] = str(trace_dir) + # research.py captures Torch callables in its interpreter registry at import. + # Install wrappers first so those calls participate in reference/selective traces. + instrumentation = install_instrumentation(mode, trace_dir, invariants) + sys.path.insert(0, str(workspace)) + module = importlib.import_module("latent_reasoning.train") + entry: object = getattr(module, "main") + if not isinstance(entry, FunctionType): + raise TypeError("Upstream main must be a Python function") + if ( + Path(entry.__code__.co_filename).resolve() != source + or entry.__code__.co_firstlineno != lines.main + ): + raise ValueError("Imported trainer does not match the verified source") + window = StepWindow( + entry.__code__, lines, instrumentation.enable, instrumentation.disable + ) + manifest: dict[str, object] = { + "schema_version": 1, + "mode": mode, + "source_sha256": TRAIN_SHA256, + "adapter_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "invariants_sha256": ( + None + if invariants is None + else hashlib.sha256(invariants.read_bytes()).hexdigest() + ), + "policy_sha256": ( + hashlib.sha256((trace_dir / "instr_opts.json").read_bytes()).hexdigest() + if mode == "selective" + else None + ), + "window_start_line": lines.start, + "window_stop_line": lines.stop, + "model_tracker": None, + "tensor_value_capture": False, + "instrumented_apis": instrumentation.instrumented_apis, + "training_args": list(training_args), + "scope": "zero_grad through update; CPU; unsigned reduced-compute pilot", + "status": "running", + } + old_argv, old_cwd = sys.argv, Path.cwd() + sys.argv = [ + str(source), + "--output", + str(output / "training"), + "--device", + "cpu", + *training_args, + ] + try: + os.chdir(workspace) + sys.settrace(window) + entry() + if window.active or window.failed or not window.records: + raise RuntimeError("Training did not complete a valid traced update") + manifest["status"] = "complete" + except BaseException as error: + manifest["status"] = "failed" + manifest["error_type"] = type(error).__name__ + raise + finally: + sys.settrace(None) + window.close() + sys.argv = old_argv + os.chdir(old_cwd) + instrumentation.drain() + manifest["steps"] = window.records + (output / "trace_manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n" + ) + + +def main(argv: Sequence[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--mode", choices=("plain", "reference", "selective"), required=True + ) + parser.add_argument("--invariants", type=Path) + parser.add_argument("training_args", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + mode: Mode + if args.mode == "plain": + mode = "plain" + elif args.mode == "reference": + mode = "reference" + else: + mode = "selective" + forwarded: list[str] = args.training_args + if forwarded[:1] == ["--"]: + forwarded = forwarded[1:] + run(args.workspace, args.output, mode, args.invariants, forwarded) + + +if __name__ == "__main__": + main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index ff701c47..0707c171 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -552,12 +552,16 @@ def obj_to_serializable(obj, dump_config=None) -> dict[str, object]: return {str(type(obj)): None} -def var_to_serializable(obj, dump_config=None) -> dict[str, object]: +def var_to_serializable(obj: object, dump_config: object = None) -> dict[str, object]: """Convert any object to a serializable dictionary. Note that this function is largely a wrapper of convert_var_to_dict to add some heuristics about how to dump a few types of objects. and it does not dump the `data` attribute of a tensor. If you want to dump the `data` attribute of a tensor, use `convert_var_to_dict` and set `include_tensor_data=True`. + + Primitive/JSON-compatible values ignore dump_config. Other values delegate + its interpretation to obj_to_serializable unchanged; this boundary does not + claim that arbitrary caller-supplied configuration has been validated. """ if is_fake_tensor(obj): @@ -566,6 +570,23 @@ def var_to_serializable(obj, dump_config=None) -> dict[str, object]: if issubclass(type(obj), dict) and type(obj) != dict: # noqa E721 return obj_to_serializable(obj, dump_config=dump_config) + obj_type = type(obj) + if ( + obj_type is type(None) + or obj_type is bool + or obj_type is float + or obj_type is str + ): + # These exact built-ins always pass the standard JSON compatibility + # probe. Keep subclasses and custom objects on the original path. + return {typename(obj, is_runtime=True): obj} + if obj_type is int: + assert isinstance(obj, int) + # Larger integers can fail under Python's configurable digit limit. + # Preserve that synchronous failure by retaining their original probe. + if -(2**63) <= obj < 2**63: + return {typename(obj, is_runtime=True): obj} + try: json.dumps( obj diff --git a/traincheck/instrumentor/metadata_methods.py b/traincheck/instrumentor/metadata_methods.py index f8e22249..6ae71f54 100644 --- a/traincheck/instrumentor/metadata_methods.py +++ b/traincheck/instrumentor/metadata_methods.py @@ -1,8 +1,9 @@ """Bind verified native tensor methods without per-attribute warning contexts.""" +import platform import types from collections.abc import Mapping -from typing import TypeAlias +from typing import NamedTuple, TypeAlias import torch @@ -54,6 +55,75 @@ def _class_layout( _PARAMETER_MRO, _PARAMETER_NAMESPACES = _class_layout(_PARAMETER_TYPE) +class _ImmutableSuffix(NamedTuple): + namespaces: tuple[Mapping[object, object], ...] + has_getattr: bool + object_getattribute: bool + + +def _immutable_suffix( + mro: tuple[object, ...], namespaces: tuple[Mapping[object, object], ...] +) -> tuple[tuple[Mapping[object, object], ...], _ImmutableSuffix]: + """Keep namespaces live and cache only immutable lookup-hook facts. + + CPython's Py_TPFLAGS_IMMUTABLETYPE (1 << 8) disallows attribute changes + through the Python type API. Read that flag through type's native descriptor, + bypassing metaclass hooks. Other runtimes or an absent capability keep every + namespace live. Present malformed native metadata remains an internal error. + + Retain only existing instrumenter-owned namespace views and booleans. Even + immutable descriptors and their names must not acquire extra cache references: + application properties can observe their reference counts. Each descriptor + is still read from its original namespace. An unchanged MRO is required before + callers use the facts; no mutable class lookup decision is cached. + """ + assert len(mro) == len(namespaces) + empty = _ImmutableSuffix((), False, False) + if platform.python_implementation() != "CPython": + return namespaces, empty + flags_descriptor = _TYPE_NAMESPACE.get("__flags__", _ABSENT) + if flags_descriptor is _ABSENT: + return namespaces, empty + assert isinstance( + flags_descriptor, (types.GetSetDescriptorType, types.MemberDescriptorType) + ) + offset = len(mro) + for index in range(len(mro) - 1, -1, -1): + owner = mro[index] + assert isinstance(owner, type) + flags: object = flags_descriptor.__get__(owner) + flags_type = type(flags) + assert flags_type is int + assert isinstance(flags, int) + if not flags & (1 << 8): + break + offset = index + immutable = namespaces[offset:] + # A custom dictionary key can execute equality during hook lookup. Such a + # namespace keeps the original live path rather than moving those calls. + for namespace in immutable: + for name in namespace: + name_type = type(name) + if name_type is not str: + return namespaces, empty + getattribute: object = _ABSENT + has_getattr = False + for namespace in immutable: + if "__getattr__" in namespace: + has_getattr = True + if getattribute is _ABSENT: + getattribute = namespace.get("__getattribute__", _ABSENT) + return namespaces[:offset], _ImmutableSuffix( + immutable, has_getattr, getattribute is _OBJECT_GETATTRIBUTE + ) + + +_TENSOR_MUTABLE, _TENSOR_SUFFIX = _immutable_suffix(_TENSOR_MRO, _TENSOR_NAMESPACES) +_PARAMETER_MUTABLE, _PARAMETER_SUFFIX = _immutable_suffix( + _PARAMETER_MRO, _PARAMETER_NAMESPACES +) + + def _native_tensor_dictionary() -> types.GetSetDescriptorType | None: # Absence/replacement before import disables this optional fast path. A # genuine native descriptor remains valid for its owner after a user replaces @@ -90,9 +160,11 @@ def tensor_method_binding(receiver: object, name: str) -> BoundTensorMethod | No return None receiver_type = type(receiver) if receiver_type is _TENSOR_TYPE: - mro, namespaces = _TENSOR_MRO, _TENSOR_NAMESPACES + mro, namespaces = _TENSOR_MRO, _TENSOR_MUTABLE + suffix = _TENSOR_SUFFIX elif receiver_type is _PARAMETER_TYPE: - mro, namespaces = _PARAMETER_MRO, _PARAMETER_NAMESPACES + mro, namespaces = _PARAMETER_MRO, _PARAMETER_MUTABLE + suffix = _PARAMETER_SUFFIX else: return None if _TENSOR_DICTIONARY is None or _MRO_DESCRIPTOR.__get__(receiver_type) is not mro: @@ -107,6 +179,15 @@ def tensor_method_binding(receiver: object, name: str) -> BoundTensorMethod | No getattribute = namespace.get("__getattribute__", _ABSENT) if descriptor is _ABSENT: descriptor = namespace.get(name, _ABSENT) + if suffix.has_getattr: + return None + if getattribute is _ABSENT and suffix.object_getattribute: + getattribute = _OBJECT_GETATTRIBUTE + if descriptor is _ABSENT: + for namespace in suffix.namespaces: + descriptor = namespace.get(name, _ABSENT) + if descriptor is not _ABSENT: + break if getattribute is not _OBJECT_GETATTRIBUTE: return None