From 57de60dc578c00206a66f1a5e36b8a41111f1e0b Mon Sep 17 00:00:00 2001 From: Birfy <15378933+Birfy@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:50:05 +0800 Subject: [PATCH] Add s1a probe --bundle for decide_many cross-question interference (#127) Measure isolated heads against a bundled decide_many in file order and in reversed question-name order, plus a same-order bundled repeat as the non-determinism control. Offline tests cover an interfering fake and a content-only fake. No live row yet. --- CHANGELOG.md | 12 + docs/architecture.md | 3 +- docs/decide-many-interference.md | 30 ++ .../decide-many-interference/README.md | 15 + docs/skills.md | 7 + evals/README.md | 1 + evals/probe/interference.jsonl | 8 + s1a/cli.py | 64 ++++- s1a/probe_bundle.py | 264 ++++++++++++++++++ tests/test_probe_bundle.py | 257 +++++++++++++++++ tests/test_probe_order.py | 7 +- 11 files changed, 656 insertions(+), 12 deletions(-) create mode 100644 docs/decide-many-interference.md create mode 100644 docs/results/decide-many-interference/README.md create mode 100644 evals/probe/interference.jsonl create mode 100644 s1a/probe_bundle.py create mode 100644 tests/test_probe_bundle.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 37b99a8..005baba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ## Unreleased +### Added + +- `s1a probe --bundle`: cross-question interference on `decide_many` (#127). Each multi-question case is asked + isolated (one request per head), bundled in file / production order, bundled with the question-name order + reversed, and once more as a same-order bundled repeat (the non-determinism control, the same idea as + `--order same:N`). The probe reports, per head, the key or binary flip rate versus the isolated answer and the + mean total variation on the probabilities; both answers stay in the record. Cases: + `evals/probe/interference.jsonl` (ticket-like Choice+Noul, a ticket-router dummy-question negative control, and + a modest browser-shaped frozen tick). `--bundle` and `--order` cannot be used together; without `--bundle` the + fit probe and `decide_many` are unchanged. No live row yet (needs Laya weights or a Jev key); do not copy the + paper's Tev / Clef percentages. See `docs/decide-many-interference.md`. + ## 0.4.0 - 2026-11-13 ### Added diff --git a/docs/architecture.md b/docs/architecture.md index 09e67a9..4e15701 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -31,7 +31,8 @@ the description, the rules text the model reads, a budget, and the front-specifi - Browser front (`BrowserAgentSpec`, `s1a/browser/`): `BrowserDecisionModel` fills the slot of openJiuwen's browser subagent. Each browser turn is one `decide_many` over the page's controls, one question per head; the chat model writes text only for typed values and the final answer. Agents: `flights`, `allrecipes`. Design: - `docs/browser-front.md`; timing: `docs/benchmarks.md`. + `docs/browser-front.md`; timing: `docs/benchmarks.md`. That bundled `decide_many` is the in-repo production + path measured by `s1a probe --bundle` (#127); ticket-router today sends a single Choice question. - Rail front (`RailSpec`, `s1a/rails.py`): `DecisionModelRail` asks one question at one DeepAgent callback hook, acts on it above a threshold, and is evaluated on a labelled set. Agent: `injection_guard`. diff --git a/docs/decide-many-interference.md b/docs/decide-many-interference.md new file mode 100644 index 0000000..81d4f7b --- /dev/null +++ b/docs/decide-many-interference.md @@ -0,0 +1,30 @@ +# Cross-question interference on `decide_many` + +[Delaunay (2026)](https://arxiv.org/abs/2611.12521) reports that bundling several System 1 questions in one request is not the same as asking them one at a time: later heads can attend to earlier ones. This is a different failure mode from option-order sensitivity (`s1a probe --order`, #90) and from confidence semantics (#89). `s1a probe --bundle` measures it on this repository's `decide_many` path. There is no mitigation here. + +## What is bundled in this repository + +Browser ticks are the in-repo production bundled path. `docs/architecture.md` and `s1a/browser/decision_model.py` send every head of the action space in one `decide_many` (typically `operation`, then one `{op}_target` per operation, and `text_value` when values are offered). Ticket-router today sends only `{"pick": ChoiceQuestion(...)}`, so it is a single-question path. The committed file uses it as a **negative control** by adding a dummy second question next to a ticket-style Choice, not as the main suite. + +## How to run it + +The cases live in `evals/probe/interference.jsonl`: six ticket-like Choice+Noul items on one state, one ticket-router negative control (a dummy Noul, "the ticket record includes an id field"), and one modest browser-shaped frozen tick (`operation`, `click_target`, `type_text_target` in production head order). The browser tick is a constructed frozen observation and question set, not a replay of a recorded episode: committed `decision_ticks.json` files do not keep the original questions or the observation. + +```bash +uv run s1a probe evals/probe/interference.jsonl --model laya --bundle --json docs/results/decide-many-interference/laya-bundle.json +``` + +Hosted Jev is the same command with `--model jev` when a key is already in the environment. `--bundle` and `--order` cannot be used together. Without `--bundle`, the fit probe and `decide_many` are unchanged. + +For each case the probe: + +1. Asks every head in its own request (isolated). +2. Asks the same questions in one `decide_many`, in file / production order (bundled). +3. Asks them again with the question-name order reversed (directional interference). +4. Repeats the bundled request in the same order once (the non-determinism control, the same idea as `--order same:N`). + +It reports, per backend and per head, the key or binary flip rate versus the isolated answer (a Noul flips when `p >= 0.5` changes) and the mean total variation between the probability distributions. Both answers stay in the record and in the `--json` file. + +## Results + +No live numbers are committed yet. A first `laya` CPU row needs `uv sync --extra laya` and the `convaiinnovations/laya` checkpoint; a `jev` row needs a TypeSafe or OpenRouter key. Do not copy the paper's Tev / Clef percentages as Agents results. When a run is recorded, put the command, Agents SHA, checkpoint, device, date and the JSON next to [docs/results/decide-many-interference/](results/decide-many-interference/README.md). diff --git a/docs/results/decide-many-interference/README.md b/docs/results/decide-many-interference/README.md new file mode 100644 index 0000000..58233e5 --- /dev/null +++ b/docs/results/decide-many-interference/README.md @@ -0,0 +1,15 @@ +# decide_many cross-question interference + +Placeholder for a real `s1a probe --bundle` run (#127). No live row is recorded in this PR. + +Intended first run, when weights or a key are available: + +```bash +uv run s1a probe evals/probe/interference.jsonl --model laya --bundle --json laya-bundle.json +``` + +Record here: the command, Agents SHA, checkpoint, device, date, and case count. Flip rates in that file would be this repository's, not the Tev / Clef percentages from arXiv:2611.12521. + +This box did not have in-process Laya weights or a Jev key at the time the harness landed, so no numbers are invented. + +See [docs/decide-many-interference.md](../../decide-many-interference.md) for the protocols, the committed cases, and the statement that ticket-router is single-question today while browser ticks are the in-repo bundled path. diff --git a/docs/skills.md b/docs/skills.md index bebb4a9..03be12d 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -143,6 +143,13 @@ It produces the module, its test and a row in the agents table, and stops at the A backend whose answer follows position rather than content shows up here, not in accuracy (#90). A flip is only position bias if the backend is deterministic: run `--order same:N` too, which re-asks in file order, and read the reordered flip rate against that same-order disagreement rate. + `s1a probe cases.jsonl --bundle` is a different check (#127): on a file of multi-question cases (Choice and + Noul on one state, or a browser-shaped frozen tick), it compares isolated answers to a bundled `decide_many` + in file order and in reversed question-name order, plus a same-order bundled repeat as the non-determinism + control. It prints per-head flip rates and total variation; `--json PATH` keeps both answers. Browser ticks + are the in-repo bundled path; ticket-router is single-question today and appears only as a dummy-question + negative control in `evals/probe/interference.jsonl`. Nothing changes without `--bundle`, and it cannot be + combined with `--order`. 3. Front: a tool loop for an environment that enumerates moves and scores, a browser policy for a page with visible controls, a rail for one question at a hook of a running agent. 4. Scaffold from the front's template under `s1a/agents/_templates/`, with the state-design rules from the skill's diff --git a/evals/README.md b/evals/README.md index 857cc09..e982cd7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -13,6 +13,7 @@ folder per episode). `uv run python -m evals.table evals/results` aggregates the | Google Flights timing | wall clock against browser-use/jev-ultrafast | `s1a run flights --model jev --batch on`; numbers in `docs/benchmarks.md` | | Desktop (Cua Driver) | clicks toward `--goal` until the window shows `--expect` | `s1a run desktop --app Calculator --goal "compute 12 times 7" --expect 84 --execute --model jev --rethink off --episodes 1` | | Ticket router (30 local labelled tickets) | correct routes to five queues | `s1a run ticket_router --model jev --rethink off --episodes 1` | +| `decide_many` interference (`s1a probe --bundle`) | per-head key/binary flip rate and total variation, isolated vs bundled vs reversed-bundle, plus a same-order control | `s1a probe evals/probe/interference.jsonl --model laya --bundle --json out.json`; write-up in `docs/decide-many-interference.md`; no live numbers yet | | Injection guard (rail) | precision and recall on a labelled set | `s1a run injection_guard` | | Injection guard, public set (InjecAgent + AgentDojo) | precision and recall per source without the context-needed positives, balanced accuracy on the hard subset | `s1a run injection_guard --model jev --labelled-set evals/labelled/injection-public.jsonl`; numbers in `docs/benchmarks.md` | | Injection guard on page-length HTML (BrowseSafe-Bench sample) | precision, recall, failures and page coverage at the runtime clip | `s1a run injection_guard --labelled-set .external/browsesafe/test-sample-html.jsonl`, after `scripts/build_browsesafe_set.py build`; see `docs/benchmarks.md` | diff --git a/evals/probe/interference.jsonl b/evals/probe/interference.jsonl new file mode 100644 index 0000000..e2f26f2 --- /dev/null +++ b/evals/probe/interference.jsonl @@ -0,0 +1,8 @@ +{"state": {"ticket": {"id": "P001", "title": "Check delivery progress", "description": "My parcel tracking has not updated for three days. Please find out where it is now.", "order_status": "shipped"}, "progress": {"ticket_id": "P001"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The user is asking only about delivery tracking.", "criteria": {"true": "the request is only about tracking or delivery progress", "false": "the request is about something else, or also about something else"}}}, "note": "ordinary logistics"} +{"state": {"ticket": {"id": "P002", "title": "Card declined", "description": "The card I saved was declined when I tried to pay for order 4412. Can you check the charge?", "order_status": "payment_failed"}, "progress": {"ticket_id": "P002"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The user is asking about a charge or a failed payment.", "criteria": {"true": "the current request is about a charge or failed payment", "false": "the current request is not about payment"}}}, "note": "ordinary payment"} +{"state": {"ticket": {"id": "P003", "title": "Send it back", "description": "I want to return the shoes from order 8821 and get a refund to the original card.", "order_status": "delivered"}, "progress": {"ticket_id": "P003"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The user is requesting a return, exchange or refund.", "criteria": {"true": "the current request is a return, exchange or refund", "false": "the current request is not about returns"}}}, "note": "ordinary returns"} +{"state": {"ticket": {"id": "P004", "title": "Cannot sign in", "description": "The reset email never arrives and I cannot log in to see my orders.", "order_status": "none"}, "progress": {"ticket_id": "P004"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The user cannot sign in or access the account.", "criteria": {"true": "the current request is an account or login problem", "false": "the current request is not about the account"}}}, "note": "ordinary account"} +{"state": {"ticket": {"id": "P005", "title": "Talk to a person", "description": "Please put me through to a human agent. I do not want a chatbot to route this.", "order_status": "shipped"}, "progress": {"ticket_id": "P005"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The user explicitly asked to talk to a human.", "criteria": {"true": "the user asked for a human", "false": "the user did not ask for a human"}}}, "note": "explicit human request"} +{"state": {"ticket": {"id": "P006", "title": "Where is the parcel, not the refund", "description": "The confirmation mentioned a possible refund if it is late, but I am not asking for that. I only want to know where the parcel is now.", "order_status": "shipped"}, "progress": {"ticket_id": "P006"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The user's current request is about delivery tracking, not payment or a refund.", "criteria": {"true": "the live request is delivery tracking", "false": "the live request is payment, a refund, or something else"}}}, "note": "negated background payment"} +{"state": {"ticket": {"id": "P007", "title": "Check delivery progress", "description": "My parcel tracking has not updated for three days. Please find out where it is now.", "order_status": "shipped"}, "progress": {"ticket_id": "P007"}}, "questions": {"pick": {"type": "choice", "options": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. If no unique queue fits, choose human."}, "flag": {"type": "noul", "question": "The ticket record includes an id field.", "criteria": {"true": "the ticket object has an id", "false": "the ticket object has no id"}}}, "note": "ticket-router negative control (dummy second question)"} +{"state": {"page": {"url": "https://www.allrecipes.com/", "title": "Allrecipes", "text": "Search recipes. Popular: vegetarian lasagna, weeknight pasta, chocolate cake."}, "elements": {"1": {"label": "Search the site", "role": "searchbox", "value": ""}, "2": {"label": "Vegetarian lasagna", "role": "link"}, "3": {"label": "Weeknight pasta", "role": "link"}}, "recent_actions": []}, "questions": {"operation": {"type": "choice", "options": {"CLICK": "Click an element", "TYPE_TEXT": "Type into a field", "PRESS_ENTER": "Press Enter", "WAIT": "Wait for the page"}, "goal": "Find a vegetarian lasagna recipe with over 100 reviews.", "rules": "Pick the next operation on this page toward the goal."}, "click_target": {"type": "choice", "options": {"2": {"element": "[2] Vegetarian lasagna", "role": "link"}, "3": {"element": "[3] Weeknight pasta", "role": "link"}}, "goal": "Find a vegetarian lasagna recipe with over 100 reviews.", "operation": "CLICK", "rules": "Pick the element to click."}, "type_text_target": {"type": "choice", "options": {"1": {"element": "[1] Search the site", "role": "searchbox", "current_value": ""}}, "goal": "Find a vegetarian lasagna recipe with over 100 reviews.", "operation": "TYPE_TEXT", "rules": "Pick the field to type into."}}, "note": "browser-shaped frozen tick (production head order: operation, then per-operation targets)"} diff --git a/s1a/cli.py b/s1a/cli.py index 6a868f8..06f10fe 100644 --- a/s1a/cli.py +++ b/s1a/cli.py @@ -77,9 +77,21 @@ def parser() -> argparse.ArgumentParser: "(same:N, the non-determinism control); report the flip rate (#90)" ), ) + fit.add_argument( + "--bundle", + action="store_true", + help=( + "cross-question interference (#127): ask every head alone, then all heads in one decide_many " + "(file order and reversed), plus a same-order bundled repeat as the non-determinism control; " + "report per-head key/binary flip rate vs the isolated answer and total variation" + ), + ) fit.add_argument("--seed", type=int, default=None, help="with --order shuffle:N: the shuffle seed (default 0)") fit.add_argument( - "--json", type=Path, metavar="PATH", help="with --order: write the full summary, every order asked, here" + "--json", + type=Path, + metavar="PATH", + help="with --order or --bundle: write the full summary here", ) return build @@ -128,24 +140,49 @@ async def decide(args: argparse.Namespace) -> dict[str, Any]: async def run_probe( - cases: Path, model_name: str, order: str | None = None, *, seed: int | None = None, json_path: Path | None = None + cases: Path, + model_name: str, + order: str | None = None, + *, + seed: int | None = None, + json_path: Path | None = None, + bundle: bool = False, ) -> dict[str, Any]: - """The fit probe; with ``order``, the option-order check over the same cases (verdict still from file order).""" - if order is None and (seed is not None or json_path is not None): - raise ValueError("--seed and --json only apply with --order") # a usage error, not silently ignored - read = probe.read_cases(cases) + """The fit probe; with ``order``, the option-order check; with ``bundle``, the cross-question interference check.""" + if order is not None and bundle: + raise ValueError("--bundle and --order cannot be used together") + if seed is not None and order is None: + raise ValueError("--seed only applies with --order") # a usage error, not silently ignored + if json_path is not None and order is None and not bundle: + raise ValueError("--json only applies with --order or --bundle") + if bundle: + from s1a import probe_bundle + + read_bundle = probe_bundle.read_cases(cases) + else: + read = probe.read_cases(cases) if order is not None: probe.parse_order(order) # a bad --order is a usage error before any model loads seed = 0 if seed is None else seed decision_model = build_model(model_name) try: - if order is None: + if bundle: + from s1a import probe_bundle + + summary = await probe_bundle.run(read_bundle, decision_model) + elif order is None: summary = await probe.run(read, decision_model) else: summary = await probe.run_order(read, decision_model, order, seed=seed) finally: await decision_model.close() - if order is None: + if bundle: + from s1a import probe_bundle + + print(probe_bundle.render(summary)) + if json_path is not None: + json_path.write_text(json.dumps({"model": model_name, **summary}, ensure_ascii=False, indent=1) + "\n") + elif order is None: print(probe.render(summary)) else: print(probe.render_order(summary)) @@ -171,8 +208,17 @@ def main(argv: list[str]) -> int: asyncio.run(decide(args)) case "probe": summary = asyncio.run( - run_probe(args.cases, args.model, args.order, seed=args.seed, json_path=args.json) + run_probe( + args.cases, + args.model, + args.order, + seed=args.seed, + json_path=args.json, + bundle=args.bundle, + ) ) + if args.bundle: + return 0 # a measurement, not a fit verdict return 0 if summary["verdict"] == "fits" else 1 except (agents.UnknownAgent, ValueError) as exc: # JSONDecodeError is a ValueError print(exc, file=sys.stderr) diff --git a/s1a/probe_bundle.py b/s1a/probe_bundle.py new file mode 100644 index 0000000..be7b8e4 --- /dev/null +++ b/s1a/probe_bundle.py @@ -0,0 +1,264 @@ +# coding: utf-8 +"""Cross-question interference on ``decide_many`` (#127): isolated vs bundled vs reversed-bundle. + +``s1a probe --bundle`` asks every head of a multi-question case alone, then all heads in one ``decide_many`` +(file / production order, and again with the question-name order reversed), then repeats the bundled request in +the same order as a non-determinism control (the same idea as ``s1a probe --order same:N``). It reports, per head, +the key or binary flip rate versus the isolated answer and the total variation between the probability +distributions. Both answers stay in the record. + +This does not change ``decide_many`` or the fit probe when ``--bundle`` is off. There is no mitigation here. + +Each JSONL line: ``{"state": {...}, "questions": {"name": {...}, ...}, "note": "..."}``. A question is either +``{"type": "choice", "options": {"key": "text"|object, ...}, "rules": "...", "goal": "...", "operation": "..."}`` +or ``{"type": "noul", "question": "...", "criteria": {"true": "...", "false": "..."}}``. A case needs at least two +questions; dict order is the file / production order. +""" + +from __future__ import annotations + +import json +import statistics +from pathlib import Path +from typing import Any + +from s1a.decision_models import Choice, ChoiceQuestion, DecisionModel, Noul, NoulQuestion, Observation, Question + +CONTROL_REPEATS = 1 # one same-order bundled repeat; a "flip" here is the backend disagreeing with itself + + +def _well_formed_question(spec: object) -> bool: + if not isinstance(spec, dict) or spec.get("type") not in {"choice", "noul"}: + return False + if spec["type"] == "choice": + options = spec.get("options") + return ( + isinstance(options, dict) + and bool(options) + and all(isinstance(key, str) for key in options) + and all(isinstance(value, (str, dict)) for value in options.values()) + ) + question = spec.get("question") + if not isinstance(question, str) or not question.strip(): + return False + criteria = spec.get("criteria") + return criteria is None or (isinstance(criteria, dict) and set(criteria) == {"true", "false"}) + + +def _well_formed(case: object) -> bool: + return ( + isinstance(case, dict) + and isinstance(case.get("state"), (dict, str)) + and isinstance(case.get("questions"), dict) + and len(case["questions"]) >= 2 + and all(isinstance(name, str) and _well_formed_question(spec) for name, spec in case["questions"].items()) + ) + + +def parse_question(spec: dict[str, Any]) -> Question: + """A ``ChoiceQuestion`` or ``NoulQuestion`` from one case's question object.""" + if spec["type"] == "choice": + return ChoiceQuestion( + spec["options"], + goal=str(spec.get("goal") or ""), + rules=spec["rules"] if isinstance(spec.get("rules"), (str, list, tuple)) else "", + operation=str(spec.get("operation") or ""), + ) + criteria = spec.get("criteria") + return NoulQuestion(spec["question"], criteria=dict(criteria) if criteria is not None else None) + + +def questions_of(case: dict[str, Any]) -> dict[str, Question]: + """The case's questions in file order.""" + return {name: parse_question(spec) for name, spec in case["questions"].items()} + + +def reversed_questions(questions: dict[str, Question]) -> dict[str, Question]: + """The same questions with the name order reversed (directional interference).""" + return {name: questions[name] for name in reversed(list(questions))} + + +def read_cases(path: Path) -> list[dict[str, Any]]: + """The JSONL multi-question cases; a malformed line is a ValueError naming the file and line.""" + cases: list[dict[str, Any]] = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError as exc: + raise ValueError(f"{path}: {exc}") from exc + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + case = json.loads(line) + except ValueError as exc: + raise ValueError(f"{path}:{line_number}: not JSON: {exc}") from exc + if not _well_formed(case): + raise ValueError( + f"{path}:{line_number}: a bundle case needs state (object or text) and at least two questions " + "(choice with options, or noul with a question)" + ) + cases.append(case) + if not cases: + raise ValueError(f"{path}: no cases") + return cases + + +def snapshot(answer: Choice | Noul) -> dict[str, Any]: + """The comparable record of one head: the key or noul probability, the binary/key, the distribution, confidence.""" + if isinstance(answer, Choice): + return { + "type": "choice", + "key": answer.key, + "binary": answer.key, + "probabilities": dict(answer.probabilities), + "confidence": answer.confidence, + } + return { + "type": "noul", + "noul": answer.p, + "binary": answer.p >= 0.5, + "probabilities": {"true": answer.p, "false": 1.0 - answer.p}, + "confidence": answer.confidence, + } + + +def total_variation(left: dict[str, Any], right: dict[str, Any]) -> float: + """Total variation between two recorded heads (½ Σ|p−q| over the same keys; for noul this equals |p−q|).""" + keys = set(left["probabilities"]) | set(right["probabilities"]) + return 0.5 * sum(abs(left["probabilities"].get(key, 0.0) - right["probabilities"].get(key, 0.0)) for key in keys) + + +def _answers(decision) -> dict[str, dict[str, Any]]: + return {name: snapshot(answer) for name, answer in decision.answers.items()} + + +async def _ask( + decision_model: DecisionModel, state: dict[str, Any] | str, questions: dict[str, Question] +) -> dict[str, Any]: + decision = await decision_model.decide_many(Observation(state), questions) + return {"answers": _answers(decision), "ms": decision.latency_ms, "heads": list(questions)} + + +async def run( + cases: list[dict[str, Any]], decision_model: DecisionModel, *, repeats: int = CONTROL_REPEATS +) -> dict[str, Any]: + """Isolated, bundled, reversed-bundle and same-order bundled repeats; per-head flip rates and mean TV.""" + if repeats < 1: + raise ValueError("repeats must be >= 1") + rows: list[dict[str, Any]] = [] + for case in cases: + questions = questions_of(case) + state = case["state"] + isolated = {} + isolated_ms = [] + for name, question in questions.items(): + asked = await _ask(decision_model, state, {name: question}) + isolated[name] = asked["answers"][name] + isolated_ms.append(asked["ms"]) + bundled = await _ask(decision_model, state, questions) + reversed_ = await _ask(decision_model, state, reversed_questions(questions)) + control = [await _ask(decision_model, state, questions) for _ in range(repeats)] + heads = list(questions) + flips = {"bundled": {}, "reversed": {}, "control": {}} + tv = {"bundled": {}, "reversed": {}, "control": {}} + for name in heads: + base = isolated[name] + flips["bundled"][name] = bundled["answers"][name]["binary"] != base["binary"] + flips["reversed"][name] = reversed_["answers"][name]["binary"] != base["binary"] + flips["control"][name] = any( + repeat["answers"][name]["binary"] != bundled["answers"][name]["binary"] for repeat in control + ) + tv["bundled"][name] = round(total_variation(base, bundled["answers"][name]), 6) + tv["reversed"][name] = round(total_variation(base, reversed_["answers"][name]), 6) + tv["control"][name] = round( + statistics.mean( + total_variation(bundled["answers"][name], repeat["answers"][name]) for repeat in control + ), + 6, + ) + rows.append( + { + "note": str(case.get("note") or ""), + "heads": heads, + "isolated": isolated, + "bundled": bundled, + "reversed": reversed_, + "control": control, + "isolated_ms": isolated_ms, + "flips": flips, + "tv": tv, + } + ) + heads = list(dict.fromkeys(name for row in rows for name in row["heads"])) + per_head: dict[str, dict[str, Any]] = {} + for name in heads: + on = [row for row in rows if name in row["heads"]] + count = len(on) + per_head[name] = { + "cases": count, + "bundled_flips": sum(row["flips"]["bundled"][name] for row in on), + "reversed_flips": sum(row["flips"]["reversed"][name] for row in on), + "control_flips": sum(row["flips"]["control"][name] for row in on), + "bundled_flip_rate": round(sum(row["flips"]["bundled"][name] for row in on) / count, 3) if count else 0.0, + "reversed_flip_rate": round(sum(row["flips"]["reversed"][name] for row in on) / count, 3) if count else 0.0, + "control_flip_rate": round(sum(row["flips"]["control"][name] for row in on) / count, 3) if count else 0.0, + "mean_tv_bundled": round(statistics.mean(row["tv"]["bundled"][name] for row in on), 3) if count else 0.0, + "mean_tv_reversed": round(statistics.mean(row["tv"]["reversed"][name] for row in on), 3) if count else 0.0, + "mean_tv_control": round(statistics.mean(row["tv"]["control"][name] for row in on), 3) if count else 0.0, + } + all_ms = [row["bundled"]["ms"] for row in rows] + [row["reversed"]["ms"] for row in rows] + all_ms.extend(repeat["ms"] for row in rows for repeat in row["control"]) + all_ms.extend(ms for row in rows for ms in row["isolated_ms"]) + return { + "bundle": True, + "repeats": repeats, + "rows": rows, + "cases": len(rows), + "heads": heads, + "per_head": per_head, + "median_ms": int(statistics.median(all_ms)) if all_ms else 0, + } + + +def _rate(hits: int, total: int) -> str: + return f"{hits}/{total} ({(hits / total if total else 0.0):.0%})" + + +def render(summary: dict[str, Any]) -> str: + """The interference check as text: one line per case per head, then per-head totals.""" + lines = [f"{'case':<28}{'head':<16}{'isolated':<12}{'bundled':<12}{'reversed':<12}{'ctrl':<8}"] + for row in summary["rows"]: + for name in row["heads"]: + isolated = row["isolated"][name] + bundled = row["bundled"]["answers"][name] + reversed_ = row["reversed"]["answers"][name] + marks = [] + if row["flips"]["bundled"][name]: + marks.append("B") + if row["flips"]["reversed"][name]: + marks.append("R") + if row["flips"]["control"][name]: + marks.append("C") + lines.append( + f"{row['note'][:27]:<28}{name[:15]:<16}" + f"{_label(isolated):<12}{_label(bundled):<12}{_label(reversed_):<12}" + f"{''.join(marks):<8}" + ) + lines.append( + f"bundle: isolated vs decide_many (file order and reversed) control: {summary['repeats']} same-order " + f"bundled repeat(s) cases: {summary['cases']} median ms: {summary['median_ms']}" + ) + for name, stats in summary["per_head"].items(): + n = stats["cases"] + lines.append( + f" {name}: bundled flips {_rate(stats['bundled_flips'], n)} mean TV {stats['mean_tv_bundled']:.3f} " + f"reversed flips {_rate(stats['reversed_flips'], n)} mean TV {stats['mean_tv_reversed']:.3f} " + f"control flips {_rate(stats['control_flips'], n)} mean TV {stats['mean_tv_control']:.3f}" + ) + return "\n".join(lines) + + +def _label(record: dict[str, Any]) -> str: + if record["type"] == "choice": + return str(record["key"])[:11] + return f"{record['noul']:.2f}" diff --git a/tests/test_probe_bundle.py b/tests/test_probe_bundle.py new file mode 100644 index 0000000..086415c --- /dev/null +++ b/tests/test_probe_bundle.py @@ -0,0 +1,257 @@ +# coding: utf-8 +"""Cross-question interference on decide_many (#127): isolated vs bundled vs reversed-bundle, plus a same-order control.""" + +from __future__ import annotations + +import asyncio +import contextlib +import io +import json +import os +import tempfile +from pathlib import Path +from typing import Any +from unittest import TestCase +from unittest.mock import patch + +from s1a import cli, probe_bundle +from s1a.decision_models import Choice, ChoiceQuestion, Noul, NoulQuestion, Reply, ScriptedModel +from s1a.decision_models.types import Observation, Question + + +def _cases(n: int = 8) -> list[dict[str, Any]]: + return [ + { + "state": {"i": i}, + "questions": { + "pick": {"type": "choice", "options": {"a": "keep this", "b": "the other one"}, "rules": "pick a"}, + "flag": { + "type": "noul", + "question": "Is this request urgent?", + "criteria": {"true": "urgent", "false": "not urgent"}, + }, + }, + "note": f"case {i}", + } + for i in range(n) + ] + + +def _text(question: Question) -> str: + if isinstance(question, ChoiceQuestion): + return " ".join(str(value) for value in question.options.values()) + assert isinstance(question, NoulQuestion) + return question.question + + +class _ContentModel(ScriptedModel): + """Answers from the question itself; earlier heads in the same request are ignored.""" + + def _answer(self, question: Question) -> dict[str, Any]: + if isinstance(question, ChoiceQuestion): + key = "a" if "a" in question.ids else question.ids[0] + return { + "choice": key, + "probabilities": {i: 1.0 if i == key else 0.0 for i in question.ids}, + "confidence": 1.0, + } + return {"noul": 0.2} + + +class _InterferingModel(_ContentModel): + """A later head flips when it can read earlier question text in the same request; alone it matches content.""" + + async def _decide(self, observation: Observation, questions: dict[str, Question]) -> Reply: + self.calls.append((observation, dict(questions))) + prior: list[str] = [] + answers = {} + for name, question in questions.items(): + if any(text.strip() for text in prior): + answers[name] = self._flipped(question) + else: + answers[name] = self._answer(question) + prior.append(_text(question)) + return Reply(answers=answers, latency_ms=self._latency_ms, usage=self._usage, model=self._model) + + def _flipped(self, question: Question) -> dict[str, Any]: + if isinstance(question, ChoiceQuestion): + key = "b" if "b" in question.ids else question.ids[-1] + return { + "choice": key, + "probabilities": {i: 1.0 if i == key else 0.0 for i in question.ids}, + "confidence": 1.0, + } + return {"noul": 0.8} + + +class _AlternatingModel(_ContentModel): + """Flips on every even file-order bundled ask: the same-order repeat disagrees, the first bundled ask does not.""" + + def __init__(self) -> None: + super().__init__() + self._file_order_bundles = 0 + + async def _decide(self, observation: Observation, questions: dict[str, Question]) -> Reply: + self.calls.append((observation, dict(questions))) + answers = {name: self._answer(question) for name, question in questions.items()} + if list(questions) == ["pick", "flag"]: + self._file_order_bundles += 1 + if self._file_order_bundles % 2 == 0: + answers = {name: _InterferingModel()._flipped(question) for name, question in questions.items()} + return Reply(answers=answers, latency_ms=self._latency_ms, usage=self._usage, model=self._model) + + +def _run(model: ScriptedModel, cases: list[dict[str, Any]] | None = None) -> dict[str, Any]: + return asyncio.run(probe_bundle.run(cases or _cases(), model)) + + +class TestTotalVariation(TestCase): + def test_one_hot_move_is_one_and_noul_shift_is_the_probability_gap(self) -> None: + left = probe_bundle.snapshot(Choice("a", {"a": 1.0, "b": 0.0}, 1.0)) + right = probe_bundle.snapshot(Choice("b", {"a": 0.0, "b": 1.0}, 1.0)) + self.assertEqual(probe_bundle.total_variation(left, right), 1.0) + noul_low = probe_bundle.snapshot(Noul(0.2, 0.8)) + noul_high = probe_bundle.snapshot(Noul(0.8, 0.8)) + self.assertAlmostEqual(probe_bundle.total_variation(noul_low, noul_high), 0.6) + + +class TestReadCases(TestCase): + def test_a_malformed_or_single_question_file_is_a_value_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "cases.jsonl" + path.write_text("{}\n", encoding="utf-8") + with self.assertRaises(ValueError): + probe_bundle.read_cases(path) + path.write_text( + json.dumps({"state": {}, "questions": {"only": {"type": "choice", "options": {"a": "a"}}}}) + "\n", + encoding="utf-8", + ) + with self.assertRaises(ValueError): + probe_bundle.read_cases(path) + + +class TestRunBundle(TestCase): + def test_an_interfering_backend_flips_only_the_later_head_and_only_when_bundled(self) -> None: + model = _InterferingModel() + summary = _run(model) + pick, flag = summary["per_head"]["pick"], summary["per_head"]["flag"] + self.assertEqual((pick["bundled_flips"], pick["bundled_flip_rate"]), (0, 0.0)) + self.assertEqual((flag["bundled_flips"], flag["bundled_flip_rate"]), (8, 1.0)) + self.assertEqual((pick["reversed_flips"], pick["reversed_flip_rate"]), (8, 1.0)) + self.assertEqual((flag["reversed_flips"], flag["reversed_flip_rate"]), (0, 0.0)) + self.assertEqual((pick["control_flips"], flag["control_flips"]), (0, 0)) + self.assertEqual(flag["mean_tv_bundled"], 0.6) + self.assertEqual(pick["mean_tv_reversed"], 1.0) + # isolated asks never see another question, so they stay on the content answer + self.assertEqual({row["isolated"]["pick"]["key"] for row in summary["rows"]}, {"a"}) + self.assertEqual({row["isolated"]["flag"]["noul"] for row in summary["rows"]}, {0.2}) + self.assertEqual({row["bundled"]["answers"]["flag"]["noul"] for row in summary["rows"]}, {0.8}) + # file order is pick then flag; reversed is flag then pick + self.assertEqual(summary["rows"][0]["bundled"]["heads"], ["pick", "flag"]) + self.assertEqual(summary["rows"][0]["reversed"]["heads"], ["flag", "pick"]) + # isolated calls are one question each; the first bundled call carries both, later first + isolated_heads = [list(questions) for _obs, questions in model.calls if len(questions) == 1] + self.assertEqual(isolated_heads[:2], [["pick"], ["flag"]]) + + def test_a_content_only_backend_never_flips(self) -> None: + summary = _run(_ContentModel()) + for name, stats in summary["per_head"].items(): + with self.subTest(head=name): + self.assertEqual((stats["bundled_flips"], stats["reversed_flips"], stats["control_flips"]), (0, 0, 0)) + self.assertEqual( + (stats["mean_tv_bundled"], stats["mean_tv_reversed"], stats["mean_tv_control"]), (0.0, 0.0, 0.0) + ) + + def test_the_same_order_repeat_is_the_control_for_non_determinism(self) -> None: + summary = _run(_AlternatingModel()) + # isolated + first bundled + reversed are followed by a control repeat; the even bundled-sized call flips + self.assertEqual(summary["per_head"]["pick"]["control_flips"], 8) + self.assertEqual(summary["per_head"]["flag"]["control_flip_rate"], 1.0) + # the first bundled ask still matches isolated (both content-only on this model's odd calls) + self.assertEqual(summary["per_head"]["pick"]["bundled_flips"], 0) + self.assertEqual(summary["per_head"]["flag"]["bundled_flips"], 0) + + +def _main(argv: list[str]) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with patch.dict(os.environ), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = cli.main(argv) + return code, out.getvalue(), err.getvalue() + + +class TestBundleCli(TestCase): + def _cases_file(self, tmp: str, cases: list[dict[str, Any]] | None = None) -> Path: + path = Path(tmp) / "cases.jsonl" + path.write_text("".join(json.dumps(case) + "\n" for case in (cases or _cases())), encoding="utf-8") + return path + + def test_without_bundle_the_probe_output_and_exit_code_are_unchanged(self) -> None: + from s1a.decision_models import JevModel + from s1a.decision_models.fakes import ScriptedTransport + + single = [ + { + "state": {"i": i}, + "options": {"a": "first", "b": "second"}, + "rules": "pick b", + "accept": ["b"], + "note": f"case {i}", + } + for i in range(8) + ] + with ( + tempfile.TemporaryDirectory() as tmp, + patch.object(cli, "build_model", lambda *_a, **_k: JevModel(ScriptedTransport(choose="b"))), + ): + path = Path(tmp) / "fit.jsonl" + path.write_text("".join(json.dumps(case) + "\n" for case in single), encoding="utf-8") + code, out, _err = _main(["probe", str(path)]) + self.assertEqual(code, 0) + self.assertNotIn("bundle:", out) + self.assertNotIn("flips", out) + self.assertIn("verdict: fits", out) + + def test_bundle_prints_per_head_flip_rates_and_writes_both_answers_to_json(self) -> None: + with ( + tempfile.TemporaryDirectory() as tmp, + patch.object(cli, "build_model", lambda *_a, **_k: _InterferingModel()), + ): + out_path = Path(tmp) / "bundle.json" + code, out, _err = _main(["probe", str(self._cases_file(tmp)), "--bundle", "--json", str(out_path)]) + saved = json.loads(out_path.read_text(encoding="utf-8")) + self.assertEqual(code, 0) # a measurement, not a fit verdict + self.assertIn("bundled flips 0/8 (0%)", out) + self.assertIn("bundled flips 8/8 (100%)", out) + self.assertIn("reversed flips 8/8 (100%)", out) + self.assertEqual((saved["model"], saved["bundle"], saved["cases"]), ("jev", True, 8)) + self.assertEqual(saved["per_head"]["flag"]["bundled_flip_rate"], 1.0) + self.assertEqual(saved["rows"][0]["isolated"]["flag"]["noul"], 0.2) + self.assertEqual(saved["rows"][0]["bundled"]["answers"]["flag"]["noul"], 0.8) + + def test_bundle_and_order_together_is_a_usage_error_before_any_model_loads(self) -> None: + built: list[str] = [] + + def build(name: str, **_kwargs: Any) -> ScriptedModel: + built.append(name) + return _ContentModel() + + with tempfile.TemporaryDirectory() as tmp, patch.object(cli, "build_model", build): + code, out, err = _main(["probe", str(self._cases_file(tmp)), "--bundle", "--order", "reverse"]) + self.assertEqual((code, out, built), (2, "", [])) + self.assertIn("cannot be used together", err) + + def test_the_committed_interference_file_loads(self) -> None: + path = Path("evals/probe/interference.jsonl") + cases = probe_bundle.read_cases(path) + self.assertGreaterEqual(len(cases), 8) + kinds = [str(case.get("note") or "") for case in cases] + self.assertTrue(any("negative control" in note for note in kinds)) + self.assertTrue(any("browser" in note.lower() for note in kinds)) + for case in cases: + questions = probe_bundle.questions_of(case) + self.assertGreaterEqual(len(questions), 2) + types = {question.type for question in questions.values()} + if "negative control" in str(case.get("note") or ""): + self.assertEqual(types, {"choice", "noul"}) + if "browser" in str(case.get("note") or "").lower(): + self.assertIn("operation", questions) diff --git a/tests/test_probe_order.py b/tests/test_probe_order.py index 6bbbbe5..06d9d83 100644 --- a/tests/test_probe_order.py +++ b/tests/test_probe_order.py @@ -193,7 +193,10 @@ def build(name: str, **_kwargs: Any) -> JevModel: built.append(name) return JevModel(ScriptedTransport()) - for extra in (["--seed", "1"], ["--json", "out.json"]): + for extra, needle in ( + (["--seed", "1"], "only applies with --order"), + (["--json", "out.json"], "only applies with --order or --bundle"), + ): with ( self.subTest(extra=extra), tempfile.TemporaryDirectory() as tmp, @@ -201,4 +204,4 @@ def build(name: str, **_kwargs: Any) -> JevModel: ): code, out, err = _main(["probe", str(self._cases_file(tmp)), *extra]) self.assertEqual((code, out, built), (2, "", [])) - self.assertIn("only apply with --order", err) + self.assertIn(needle, err)