diff --git a/agentdescent/mcp.py b/agentdescent/mcp.py index 5b0ddbd9..c5d879de 100644 --- a/agentdescent/mcp.py +++ b/agentdescent/mcp.py @@ -42,7 +42,7 @@ require_resume_not_solved, resolve_run_id, show_payload, status_payload) from .evolvespec import (EvolveSpec, SpecError, budget_round_overshoot_warning, - compose, estimate) + build_call_limit, compose, estimate) __all__ = ["ADVANCED_ENV", "DEFAULT_HTTP_HOST", "DEFAULT_HTTP_PORT", "PATH_A_TOOLS", "RESCAN_ALLOW_ENV", "STREAMABLE_HTTP_REQUIRES", "TRANSPORTS", @@ -163,13 +163,18 @@ def advanced_from_env(environ: Optional[Dict[str, str]] = None) -> bool: "one minimal call per model-backed agent (the same preflight as " "`plan --probe`); a refusal returns `probe`, and prints the same refusal " "contract as the CLI (`ok: false`, `ready: false`) with `next` naming the " - "fix, so do not call start then. Show the user the " + "fix, so do not call start then. Pass max_calls (and/or max_usd with " + "usd_per_call) to cap the run at the model call; the reply's `call_limit` is " + "what start would enforce. Show the user the " "spec and the estimate and get a yes before calling start."), "start": ( "Start an evolution run in the background and return its run_id at once. The run " "is a detached process; poll `status` about once per round (not more often) and " "read `show` when it is done. Never call start without having shown the user the " - "plan. A run costs real agent calls: rounds x n_workers x tasks. When the spec has " + "plan. A run costs real agent calls: rounds x n_workers x tasks. Pass max_calls " + "(and/or max_usd with usd_per_call) to stop it at that many model calls; a bad " + "value refuses before any run exists, and an operator ceiling lowers the number " + "rather than raising it. When the spec has " "an `audit` block, `status` reports `audit_store` once records exist -- pass that " "path to audit_status to find out how far the run's scorer is from the truth."), "status": ( @@ -426,6 +431,40 @@ def _with_probe(payload: Dict[str, Any], spec: EvolveSpec) -> Dict[str, Any]: return payload +def _call_limit_payload(max_calls: Optional[int], max_usd: Optional[float], + usd_per_call: Optional[float]) -> Optional[Dict[str, Any]]: + """The run-level cap an MCP ``plan`` / ``start`` will enforce, or ``None`` (#1184). + + Validated by the same :func:`~agentdescent.evolvespec.build_call_limit` the + CLI and the run store use, so a bad number -- and a ``max_usd`` with no + ``usd_per_call`` -- refuses with the CLI's own sentence instead of a second + wording for this surface. The operator's ceiling is applied with the same + ``min`` as ``runstore.launch`` / ``execute``: a caller may lower it, never + raise it, and a caller that names no cap still gets it. + + ``source`` and ``max_calls_requested`` appear only when the ceiling is what + binds, the same distinction ``AGENTDESCENT_MAX_CALLS_REQUESTED`` carries + into the child: without them a host cannot tell "the ceiling lowered your 9 + to 2" from "you asked for 2". + """ + limit = build_call_limit(max_calls=max_calls, max_usd=max_usd, + usd_per_call=usd_per_call) + ceiling = runstore.operator_max_calls() + if limit is None and ceiling is None: + return None + enforced = None if limit is None else limit.max_calls + payload: Dict[str, Any] = {"max_calls": enforced, + "max_usd": None if limit is None else limit.max_usd} + if ceiling is not None and (enforced is None or ceiling < enforced): + # A run that names no cap still gets the ceiling -- and then the reply + # has to say so, or `start` would look unbounded while it is capped. + payload["max_calls"] = ceiling + payload["source"] = runstore.OPERATOR_MAX_CALLS_ENV + if max_calls is not None: + payload["max_calls_requested"] = int(max_calls) + return payload + + class Tools: """The tool bodies, SDK-free. One instance per server, bound to a run store.""" @@ -668,15 +707,23 @@ def algorithm_spec(self, name: str, path: str, cases: Optional[str] = None, def plan(self, spec: Dict[str, Any], usd_per_call: Optional[float] = None, budget_usd: Optional[float] = None, - probe: bool = False) -> Dict[str, Any]: + probe: bool = False, max_calls: Optional[int] = None, + max_usd: Optional[float] = None) -> Dict[str, Any]: try: # Relative paths mean "where this server was launched"; the detached # run has a different cwd, so they are resolved here, once. es = EvolveSpec.from_dict(spec).absolutise() payload = plan_payload(es, usd_per_call=usd_per_call, budget_usd=budget_usd) + limit = _call_limit_payload(max_calls, max_usd, usd_per_call) except SpecError as e: return {"ok": False, "error": str(e)} + except runstore.RunStoreError as e: + return {"ok": False, "error": str(e)} + if limit is not None: + # What `start` would enforce, so the host can show the number the + # user is agreeing to -- and the clamped one when a ceiling binds. + payload["call_limit"] = limit if not probe or not payload.get("ok", True): return payload return _with_probe(payload, es) @@ -712,13 +759,18 @@ def diagnose(self, run_id: str) -> Dict[str, Any]: # -- side effects ----------------------------------------------------------- def start(self, spec: Dict[str, Any], budget_usd: Optional[float] = None, - usd_per_call: Optional[float] = None) -> Dict[str, Any]: + usd_per_call: Optional[float] = None, + max_calls: Optional[int] = None, + max_usd: Optional[float] = None) -> Dict[str, Any]: try: es = EvolveSpec.from_dict(spec).absolutise() comp = compose(es) # fail here, with the field, not in the child # Before create: same rule as CLI -- budget without price must not # return ok/started and then fail only in log.txt (#342). runstore.require_budget_pair(budget_usd, usd_per_call) + # A bad cap is a refusal too, not a run whose log.txt explains it + # later; same sentence the CLI prints (#1184). + limit = _call_limit_payload(max_calls, max_usd, usd_per_call) except SpecError as e: return {"ok": False, "error": str(e)} except runstore.RunStoreError as e: @@ -754,7 +806,11 @@ def start(self, spec: Dict[str, Any], budget_usd: Optional[float] = None, # The run is a detached process, so a `host_model` reference in the # spec resolves there, not here -- it needs the bridge's address in # its environment or it has no session to borrow. + # The caller's own numbers go in, not `limit`'s clamped ones: + # `launch` applies the same min and is what records what was asked + # for across the detach boundary (#1184). st = runstore.launch(rd, budget_usd=budget_usd, usd_per_call=usd_per_call, + max_calls=max_calls, max_usd=max_usd, env=self.host_model_env()) except runstore.RunStoreError as e: return {"ok": False, "error": str(e), "run_id": rd.run_id, "dir": rd.path} @@ -767,7 +823,9 @@ def start(self, spec: Dict[str, Any], budget_usd: Optional[float] = None, "host_model_route": ("sampling" if self.bridge else (self.host_cli or None)), **({} if (self.bridge or self.host_cli) - else {"host_model_unavailable": self.bridge_error})} + else {"host_model_unavailable": self.bridge_error}), + # Enforced value, and the ceiling that bound it when one did. + **({"call_limit": limit} if limit is not None else {})} if budget_warn: out["warnings"] = [budget_warn] return out @@ -1392,8 +1450,9 @@ def algorithm_spec(name: str, path: str, cases: Optional[str] = None, @maybe_tool("plan") def plan(spec: Dict[str, Any], usd_per_call: Optional[float] = None, budget_usd: Optional[float] = None, - probe: bool = False) -> Dict[str, Any]: - return t.plan(spec, usd_per_call, budget_usd, probe) + probe: bool = False, max_calls: Optional[int] = None, + max_usd: Optional[float] = None) -> Dict[str, Any]: + return t.plan(spec, usd_per_call, budget_usd, probe, max_calls, max_usd) # `start` and `resume` are the two tools that launch a process, so they are # the two that take a Context. The annotation is set after the definition @@ -1402,7 +1461,9 @@ def plan(spec: Dict[str, Any], usd_per_call: Optional[float] = None, context_cls = _context_class() async def start(spec: Dict[str, Any], ctx, budget_usd: Optional[float] = None, - usd_per_call: Optional[float] = None) -> Dict[str, Any]: + usd_per_call: Optional[float] = None, + max_calls: Optional[int] = None, + max_usd: Optional[float] = None) -> Dict[str, Any]: import asyncio import anyio.to_thread @@ -1410,7 +1471,7 @@ async def start(spec: Dict[str, Any], ctx, budget_usd: Optional[float] = None, attach_bridge(ctx, asyncio.get_running_loop()) # The body forks a process, so it stays off the loop the bridge needs. return await anyio.to_thread.run_sync( - lambda: t.start(spec, budget_usd, usd_per_call)) + lambda: t.start(spec, budget_usd, usd_per_call, max_calls, max_usd)) start.__annotations__["ctx"] = context_cls maybe_tool("start")(start) diff --git a/changelog.d/1184-mcp-call-limit.md b/changelog.d/1184-mcp-call-limit.md new file mode 100644 index 00000000..941ed5a4 --- /dev/null +++ b/changelog.d/1184-mcp-call-limit.md @@ -0,0 +1,8 @@ +Added +The MCP `plan` and `start` tools take `max_calls` and `max_usd`, so a host can +bound a run's model calls per call instead of only through the process-wide +`AGENTDESCENT_MAX_CALLS`. Both validate through the same rule the CLI uses (a +`max_usd` without a price refuses with the CLI's own sentence) and report +`call_limit` with what was asked for and what will be enforced; the operator's +ceiling lowers a caller's number and never raises it, and `start` launches the +detached run under the clamped cap (#1184). diff --git a/docs/plugin-design.md b/docs/plugin-design.md index d55449c7..20eb1e56 100644 --- a/docs/plugin-design.md +++ b/docs/plugin-design.md @@ -238,8 +238,8 @@ descriptions are written for the *calling model*, since that is who reads them. | Tool | Returns | Notes | |---|---|---| | `doctor()` | available agents, providers, missing keys | call first | -| `plan(spec)` | validated spec + cost estimate | **does not run**; `validate()` resolves refs and builds `Policies`, so composition errors surface here | -| `start(spec, budget_usd?)` | `run_id` | detaches; returns in < 1 s | +| `plan(spec, usd_per_call?, budget_usd?, probe?, max_calls?, max_usd?)` | validated spec + cost estimate, and `call_limit` when a cap was asked for | **does not run**; `validate()` resolves refs and builds `Policies`, so composition errors surface here | +| `start(spec, budget_usd?, usd_per_call?, max_calls?, max_usd?)` | `run_id` | detaches; returns in < 1 s; `max_calls` / `max_usd` cap the run's model calls (#1184) | | `status(run_id?)` | `status.json` (+ last 3 rounds) | cheap; safe to poll | | `show(run_id, diff=true)` | evolved tree, `write_to(dry_run=True)` plan, unified diff, `outcomes()` | what the user reads before deciding | | `apply(run_id, to?)` | files written, backup path | **destructive**; host must confirm with the user | @@ -601,7 +601,10 @@ Two guards keep this from folding in on itself: child starts with the clamped `--max-calls` and learns what the caller asked for through an internal label-only variable, so `status` says which knob bound it from either path. So "a worker asked for an unbounded run" is answered by - the environment, not by review. + the environment, not by review. The MCP surface takes the same numbers — + `plan` / `start` accept `max_calls` / `max_usd` and report `call_limit` with + what was asked for and what will be enforced, so a host sees a request of 9 + come back as 2 with `source: AGENTDESCENT_MAX_CALLS` (#1184). * **The controller never hot-swaps itself.** A self-evolved plugin reaches the user's real installation only through `apply`, which is a human decision on a diff, after an L1 oracle gate, with a backup. The loop can improve its own diff --git a/tests/test_mcp_call_limit.py b/tests/test_mcp_call_limit.py new file mode 100644 index 00000000..33716a51 --- /dev/null +++ b/tests/test_mcp_call_limit.py @@ -0,0 +1,154 @@ +"""An MCP host can cap a run: `plan` / `start` take `max_calls` / `max_usd` (#1184). + +The CLI and the run store already carry the run-level cap (#1166) and the +operator ceiling (#1082). This is the third surface: a host that calls `start` +from a tool can now say "at most N model calls for this run", hears the CLI's +own refusal when it asks for a dollar cap with no price, and sees the clamped +number (requested vs enforced) when ``AGENTDESCENT_MAX_CALLS`` binds. +""" + +from __future__ import annotations + +import subprocess +import time + +import pytest + +from agentdescent import runstore +from agentdescent.cli import NESTED_ENV +from agentdescent.evolvespec import SpecError, build_call_limit +from agentdescent.mcp import Tools + +from tests.test_evolvespec import _dir_spec # noqa: F401 + + +@pytest.fixture +def store(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home")) + monkeypatch.setenv("AGENTDESCENT_SURFACE_WRITE_DIRS", str(tmp_path)) + return str(tmp_path / "runs") + + +def _wait(rd, timeout=120): + deadline = time.time() + timeout + while time.time() < deadline: + st = rd.status() + if st.state in ("done", "failed", "cancelled"): + return st + time.sleep(0.2) + raise AssertionError(rd.status()) + + +def test_plan_reports_the_cap_it_will_enforce(store, tmp_path): + t = Tools(store) + payload = t.plan(_dir_spec(str(tmp_path)).to_dict(), max_calls=3) + assert payload["ok"] is True, payload + assert payload["call_limit"] == {"max_calls": 3, "max_usd": None} + + +def test_plan_without_a_cap_is_unchanged(store, tmp_path): + t = Tools(store) + payload = t.plan(_dir_spec(str(tmp_path)).to_dict()) + assert payload["ok"] is True, payload + assert "call_limit" not in payload + + +def test_start_caps_the_run_and_status_shows_it(store, tmp_path): + t = Tools(store) + started = t.start(_dir_spec(str(tmp_path)).to_dict(), max_calls=1) + assert started["ok"] is True, started + assert started["call_limit"] == {"max_calls": 1, "max_usd": None} + st = _wait(runstore.get(started["run_id"], store=store)) + assert st.state == "done", st + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail + assert st.calls == 1, st + + +def test_start_without_a_cap_is_unchanged(store, tmp_path): + t = Tools(store) + started = t.start(_dir_spec(str(tmp_path)).to_dict()) + assert started["ok"] is True, started + assert "call_limit" not in started + + +def test_max_usd_alone_is_refused_with_the_cli_sentence(store, tmp_path): + t = Tools(store) + spec = _dir_spec(str(tmp_path)).to_dict() + plan = t.plan(spec, max_usd=1.0) + start = t.start(spec, max_usd=1.0) + with pytest.raises(SpecError) as raised: + build_call_limit(max_usd=1.0) + for payload in (plan, start): + assert payload["ok"] is False, payload + # The CLI's wording, not a second sentence for this surface. + assert payload["error"] == str(raised.value), payload["error"] + assert t.status()["runs"] == [], "a refused cap must not leave a run behind" + + +def test_a_caller_cap_above_the_operator_ceiling_is_clamped_and_labelled( + store, tmp_path, monkeypatch): + monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2") + t = Tools(store) + spec = _dir_spec(str(tmp_path)).to_dict() + assert t.plan(spec, max_calls=9)["call_limit"] == { + "max_calls": 2, "max_usd": None, + "source": runstore.OPERATOR_MAX_CALLS_ENV, "max_calls_requested": 9} + started = t.start(spec, max_calls=9) + assert started["call_limit"] == { + "max_calls": 2, "max_usd": None, + "source": runstore.OPERATOR_MAX_CALLS_ENV, "max_calls_requested": 9}, started + st = _wait(runstore.get(started["run_id"], store=store)) + assert st.calls == 2, f"the ceiling let {st.calls} calls through" + assert st.stop_reason == "budget", st + assert runstore.OPERATOR_MAX_CALLS_ENV in (st.stop_detail or ""), st.stop_detail + + +def test_a_cap_equal_to_the_ceiling_is_not_reported_as_bound( + store, tmp_path, monkeypatch): + """Same rule as `_clamped_max_calls`: requested == ceiling did not bind. + + The payload and `stop_detail` are derived in two places (`mcp.py` here, + `runstore._clamped_max_calls` for the run), so a host that was told "the + ceiling lowered you" about a caller who asked for exactly the ceiling would + be reading a lie the run does not share. + """ + monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2") + payload = Tools(store).plan(_dir_spec(str(tmp_path)).to_dict(), max_calls=2) + assert payload["call_limit"] == {"max_calls": 2, "max_usd": None}, payload + + +def test_a_run_that_names_no_cap_still_gets_the_ceiling( + store, tmp_path, monkeypatch): + monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2") + payload = Tools(store).plan( + _dir_spec(str(tmp_path)).to_dict(), usd_per_call=None) + assert payload["call_limit"] == { + "max_calls": 2, "max_usd": None, + "source": runstore.OPERATOR_MAX_CALLS_ENV}, payload + + +def test_start_hands_the_clamped_cap_to_the_detached_child( + store, tmp_path, monkeypatch): + monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2") + seen = {} + + def fake_run(argv, **kwargs): + seen["argv"] = argv + return subprocess.CompletedProcess(argv, 0, stdout="4242", stderr="") + + monkeypatch.setattr(runstore.subprocess, "run", fake_run) + started = Tools(store).start(_dir_spec(str(tmp_path)).to_dict(), max_calls=9) + assert started["ok"] is True, started + argv = seen["argv"] + assert argv[argv.index("--max-calls") + 1] == "2", argv + + +def test_nested_start_is_unchanged(store, tmp_path, monkeypatch): + monkeypatch.setenv(NESTED_ENV, "1") + t = Tools(store) + payload = t.start(_dir_spec(str(tmp_path)).to_dict(), max_calls=1) + assert payload == { + "ok": True, "run_id": "nested-stub", "state": "refused", "nested": True, + "reason": "nested-run", "note": payload["note"]}, payload + assert t.status()["runs"] == []