diff --git a/agentdescent/cli.py b/agentdescent/cli.py index 5d0cafba..4fd560c7 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -2768,6 +2768,11 @@ def cmd_evolve(a: argparse.Namespace) -> int: "test_tasks": getattr(result, "test_tasks", 0), "select_tasks": getattr(result, "select_tasks", 0), "outcomes": result.outcomes(), "error": result.error, "dir": rd.path} + # Which limit fired, in the machine output too: `stop_reason` alone cannot + # tell the operator ceiling from the caller's own cap (#1190). Present only + # when set, so an uncapped payload keeps yesterday's bytes. + if st.stop_detail: + payload["stop_detail"] = st.stop_detail from .diagnose import SEED_SOLVED_HINT, seed_already_solved seed_solved = seed_already_solved(init, result.final_reward, result.stop_reason) if seed_solved: @@ -3117,6 +3122,10 @@ def show_payload(run_id: str, *, store: Optional[str] = None, diff: bool = True, result = rd.result() payload: Dict[str, Any] = {"run_id": run_id, "state": st.state, "stop_reason": st.stop_reason, "error": st.error} + # Same rule as `evolve --json` (#1190): name the cap only when one tripped, + # so a run that never hit a limit reports the shape it always had. + if st.stop_detail: + payload["stop_detail"] = st.stop_detail if result is None: # A failed, cancelled or still-running run has no artifact and no diff, # but what it did is on disk: the rounds that finished, the reflector diff --git a/changelog.d/1190-json-stop-detail.md b/changelog.d/1190-json-stop-detail.md new file mode 100644 index 00000000..838b6854 --- /dev/null +++ b/changelog.d/1190-json-stop-detail.md @@ -0,0 +1,5 @@ +Fixed +`--json evolve` and `--json show` now include `stop_detail` when a run-level +cap stopped the run, so the machine output names the limit that fired instead +of only `"stop_reason": "budget"`. The key is omitted when no cap tripped, so +an uncapped payload keeps its previous shape (#1190). diff --git a/tests/test_cli_json_stop_detail.py b/tests/test_cli_json_stop_detail.py new file mode 100644 index 00000000..744f6479 --- /dev/null +++ b/tests/test_cli_json_stop_detail.py @@ -0,0 +1,92 @@ +"""`--json evolve` and `--json show` say which cap stopped the run (#1190). + +``status.json`` and the human status line carry ``stop_detail`` (#1166), but the +machine output of the two commands that end or report a run dropped it: a +caller reading ``"stop_reason": "budget"`` could not tell the operator ceiling +from its own ``--max-calls``. The key is additive: a run with no cap set must +produce exactly yesterday's payload, so the absent row below is the guard +against a blanket ``None``. +""" + +from __future__ import annotations + +import io +import json +from contextlib import redirect_stdout + +from agentdescent import cli +from agentdescent.evolvespec import EvolveSpec + +#: One entry per model call the spec's agent actually makes. +CALLS = [] + + +def counting_agent(*, usage=None, tag="agent"): + """A spec-nameable model factory that records each call.""" + + def complete(prompt): + CALLS.append(tag) + return "ok" + + return complete + + +def _spec(): + model = {"ref": "tests.test_cli_json_stop_detail:counting_agent", "tag": "agent"} + return EvolveSpec( + kind="text", target="You are a helpful assistant.", + data={"inline": [{"prompt": f"q{i}", "gold": f"a{i}"} for i in range(8)]}, + agent=model, reflect=model, allow=["tests."], + evolve={"rounds": 4, "n_workers": 4, "seed": 0, "patience": 10}) + + +def _cli(*args): + """Run the CLI in-process; returns (exit code, stdout).""" + buf = io.StringIO() + with redirect_stdout(buf): + code = cli.main(list(args)) + return code, buf.getvalue() + + +def _evolve_json(tmp_path, *, max_calls=None): + CALLS.clear() + path = tmp_path / "spec.json" + path.write_text(json.dumps(_spec().to_dict())) + args = ["--store", str(tmp_path / "runs"), "--json", "evolve", str(path)] + if max_calls is not None: + args += ["--max-calls", str(max_calls)] + code, out = _cli(*args) + assert code == 0, out + return json.loads(out) + + +def _show_json(tmp_path, run_id): + code, out = _cli("--store", str(tmp_path / "runs"), "--json", "show", run_id) + assert code == 0, out + return json.loads(out) + + +def test_evolve_json_omits_stop_detail_without_a_cap(tmp_path): + payload = _evolve_json(tmp_path) + assert payload["stop_reason"] != "budget", payload + assert "stop_detail" not in payload, payload + + +def test_evolve_json_names_the_limit_that_fired(tmp_path): + payload = _evolve_json(tmp_path, max_calls=1) + assert payload["stop_reason"] == "budget", payload + assert "max_calls" in payload["stop_detail"], payload + + +def test_show_json_omits_stop_detail_without_a_cap(tmp_path): + payload = _evolve_json(tmp_path) + shown = _show_json(tmp_path, payload["run_id"]) + assert shown["stop_reason"] != "budget", shown + assert "stop_detail" not in shown, shown + + +def test_show_json_names_the_limit_that_fired(tmp_path): + payload = _evolve_json(tmp_path, max_calls=1) + shown = _show_json(tmp_path, payload["run_id"]) + assert shown["stop_reason"] == "budget", shown + assert "max_calls" in shown["stop_detail"], shown