Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 71 additions & 10 deletions agentdescent/mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
require_resume_not_solved, resolve_run_id, show_payload,
status_payload)
from .evolvespec import (EvolveSpec, SpecError, budget_round_overshoot_warning,
compose, estimate)
build_call_limit, compose, estimate)

__all__ = ["ADVANCED_ENV", "DEFAULT_HTTP_HOST", "DEFAULT_HTTP_PORT", "PATH_A_TOOLS",
"RESCAN_ALLOW_ENV", "STREAMABLE_HTTP_REQUIRES", "TRANSPORTS",
Expand Down Expand Up @@ -163,13 +163,18 @@ def advanced_from_env(environ: Optional[Dict[str, str]] = None) -> bool:
"one minimal call per model-backed agent (the same preflight as "
"`plan --probe`); a refusal returns `probe`, and prints the same refusal "
"contract as the CLI (`ok: false`, `ready: false`) with `next` naming the "
"fix, so do not call start then. Show the user the "
"fix, so do not call start then. Pass max_calls (and/or max_usd with "
"usd_per_call) to cap the run at the model call; the reply's `call_limit` is "
"what start would enforce. Show the user the "
"spec and the estimate and get a yes before calling start."),
"start": (
"Start an evolution run in the background and return its run_id at once. The run "
"is a detached process; poll `status` about once per round (not more often) and "
"read `show` when it is done. Never call start without having shown the user the "
"plan. A run costs real agent calls: rounds x n_workers x tasks. When the spec has "
"plan. A run costs real agent calls: rounds x n_workers x tasks. Pass max_calls "
"(and/or max_usd with usd_per_call) to stop it at that many model calls; a bad "
"value refuses before any run exists, and an operator ceiling lowers the number "
"rather than raising it. When the spec has "
"an `audit` block, `status` reports `audit_store` once records exist -- pass that "
"path to audit_status to find out how far the run's scorer is from the truth."),
"status": (
Expand Down Expand Up @@ -426,6 +431,40 @@ def _with_probe(payload: Dict[str, Any], spec: EvolveSpec) -> Dict[str, Any]:
return payload


def _call_limit_payload(max_calls: Optional[int], max_usd: Optional[float],
usd_per_call: Optional[float]) -> Optional[Dict[str, Any]]:
"""The run-level cap an MCP ``plan`` / ``start`` will enforce, or ``None`` (#1184).

Validated by the same :func:`~agentdescent.evolvespec.build_call_limit` the
CLI and the run store use, so a bad number -- and a ``max_usd`` with no
``usd_per_call`` -- refuses with the CLI's own sentence instead of a second
wording for this surface. The operator's ceiling is applied with the same
``min`` as ``runstore.launch`` / ``execute``: a caller may lower it, never
raise it, and a caller that names no cap still gets it.

``source`` and ``max_calls_requested`` appear only when the ceiling is what
binds, the same distinction ``AGENTDESCENT_MAX_CALLS_REQUESTED`` carries
into the child: without them a host cannot tell "the ceiling lowered your 9
to 2" from "you asked for 2".
"""
limit = build_call_limit(max_calls=max_calls, max_usd=max_usd,
usd_per_call=usd_per_call)
ceiling = runstore.operator_max_calls()
if limit is None and ceiling is None:
return None
enforced = None if limit is None else limit.max_calls
payload: Dict[str, Any] = {"max_calls": enforced,
"max_usd": None if limit is None else limit.max_usd}
if ceiling is not None and (enforced is None or ceiling < enforced):
# A run that names no cap still gets the ceiling -- and then the reply
# has to say so, or `start` would look unbounded while it is capped.
payload["max_calls"] = ceiling
payload["source"] = runstore.OPERATOR_MAX_CALLS_ENV
if max_calls is not None:
payload["max_calls_requested"] = int(max_calls)
return payload


class Tools:
"""The tool bodies, SDK-free. One instance per server, bound to a run store."""

Expand Down Expand Up @@ -668,15 +707,23 @@ def algorithm_spec(self, name: str, path: str, cases: Optional[str] = None,

def plan(self, spec: Dict[str, Any], usd_per_call: Optional[float] = None,
budget_usd: Optional[float] = None,
probe: bool = False) -> Dict[str, Any]:
probe: bool = False, max_calls: Optional[int] = None,
max_usd: Optional[float] = None) -> Dict[str, Any]:
try:
# Relative paths mean "where this server was launched"; the detached
# run has a different cwd, so they are resolved here, once.
es = EvolveSpec.from_dict(spec).absolutise()
payload = plan_payload(es, usd_per_call=usd_per_call,
budget_usd=budget_usd)
limit = _call_limit_payload(max_calls, max_usd, usd_per_call)
except SpecError as e:
return {"ok": False, "error": str(e)}
except runstore.RunStoreError as e:
return {"ok": False, "error": str(e)}
if limit is not None:
# What `start` would enforce, so the host can show the number the
# user is agreeing to -- and the clamped one when a ceiling binds.
payload["call_limit"] = limit
if not probe or not payload.get("ok", True):
return payload
return _with_probe(payload, es)
Expand Down Expand Up @@ -712,13 +759,18 @@ def diagnose(self, run_id: str) -> Dict[str, Any]:
# -- side effects -----------------------------------------------------------

def start(self, spec: Dict[str, Any], budget_usd: Optional[float] = None,
usd_per_call: Optional[float] = None) -> Dict[str, Any]:
usd_per_call: Optional[float] = None,
max_calls: Optional[int] = None,
max_usd: Optional[float] = None) -> Dict[str, Any]:
try:
es = EvolveSpec.from_dict(spec).absolutise()
comp = compose(es) # fail here, with the field, not in the child
# Before create: same rule as CLI -- budget without price must not
# return ok/started and then fail only in log.txt (#342).
runstore.require_budget_pair(budget_usd, usd_per_call)
# A bad cap is a refusal too, not a run whose log.txt explains it
# later; same sentence the CLI prints (#1184).
limit = _call_limit_payload(max_calls, max_usd, usd_per_call)
except SpecError as e:
return {"ok": False, "error": str(e)}
except runstore.RunStoreError as e:
Expand Down Expand Up @@ -754,7 +806,11 @@ def start(self, spec: Dict[str, Any], budget_usd: Optional[float] = None,
# The run is a detached process, so a `host_model` reference in the
# spec resolves there, not here -- it needs the bridge's address in
# its environment or it has no session to borrow.
# The caller's own numbers go in, not `limit`'s clamped ones:
# `launch` applies the same min and is what records what was asked
# for across the detach boundary (#1184).
st = runstore.launch(rd, budget_usd=budget_usd, usd_per_call=usd_per_call,
max_calls=max_calls, max_usd=max_usd,
env=self.host_model_env())
except runstore.RunStoreError as e:
return {"ok": False, "error": str(e), "run_id": rd.run_id, "dir": rd.path}
Expand All @@ -767,7 +823,9 @@ def start(self, spec: Dict[str, Any], budget_usd: Optional[float] = None,
"host_model_route": ("sampling" if self.bridge
else (self.host_cli or None)),
**({} if (self.bridge or self.host_cli)
else {"host_model_unavailable": self.bridge_error})}
else {"host_model_unavailable": self.bridge_error}),
# Enforced value, and the ceiling that bound it when one did.
**({"call_limit": limit} if limit is not None else {})}
if budget_warn:
out["warnings"] = [budget_warn]
return out
Expand Down Expand Up @@ -1392,8 +1450,9 @@ def algorithm_spec(name: str, path: str, cases: Optional[str] = None,
@maybe_tool("plan")
def plan(spec: Dict[str, Any], usd_per_call: Optional[float] = None,
budget_usd: Optional[float] = None,
probe: bool = False) -> Dict[str, Any]:
return t.plan(spec, usd_per_call, budget_usd, probe)
probe: bool = False, max_calls: Optional[int] = None,
max_usd: Optional[float] = None) -> Dict[str, Any]:
return t.plan(spec, usd_per_call, budget_usd, probe, max_calls, max_usd)

# `start` and `resume` are the two tools that launch a process, so they are
# the two that take a Context. The annotation is set after the definition
Expand All @@ -1402,15 +1461,17 @@ def plan(spec: Dict[str, Any], usd_per_call: Optional[float] = None,
context_cls = _context_class()

async def start(spec: Dict[str, Any], ctx, budget_usd: Optional[float] = None,
usd_per_call: Optional[float] = None) -> Dict[str, Any]:
usd_per_call: Optional[float] = None,
max_calls: Optional[int] = None,
max_usd: Optional[float] = None) -> Dict[str, Any]:
import asyncio

import anyio.to_thread

attach_bridge(ctx, asyncio.get_running_loop())
# The body forks a process, so it stays off the loop the bridge needs.
return await anyio.to_thread.run_sync(
lambda: t.start(spec, budget_usd, usd_per_call))
lambda: t.start(spec, budget_usd, usd_per_call, max_calls, max_usd))

start.__annotations__["ctx"] = context_cls
maybe_tool("start")(start)
Expand Down
8 changes: 8 additions & 0 deletions changelog.d/1184-mcp-call-limit.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Added
The MCP `plan` and `start` tools take `max_calls` and `max_usd`, so a host can
bound a run's model calls per call instead of only through the process-wide
`AGENTDESCENT_MAX_CALLS`. Both validate through the same rule the CLI uses (a
`max_usd` without a price refuses with the CLI's own sentence) and report
`call_limit` with what was asked for and what will be enforced; the operator's
ceiling lowers a caller's number and never raises it, and `start` launches the
detached run under the clamped cap (#1184).
9 changes: 6 additions & 3 deletions docs/plugin-design.md
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,8 @@ descriptions are written for the *calling model*, since that is who reads them.
| Tool | Returns | Notes |
|---|---|---|
| `doctor()` | available agents, providers, missing keys | call first |
| `plan(spec)` | validated spec + cost estimate | **does not run**; `validate()` resolves refs and builds `Policies`, so composition errors surface here |
| `start(spec, budget_usd?)` | `run_id` | detaches; returns in < 1 s |
| `plan(spec, usd_per_call?, budget_usd?, probe?, max_calls?, max_usd?)` | validated spec + cost estimate, and `call_limit` when a cap was asked for | **does not run**; `validate()` resolves refs and builds `Policies`, so composition errors surface here |
| `start(spec, budget_usd?, usd_per_call?, max_calls?, max_usd?)` | `run_id` | detaches; returns in < 1 s; `max_calls` / `max_usd` cap the run's model calls (#1184) |
| `status(run_id?)` | `status.json` (+ last 3 rounds) | cheap; safe to poll |
| `show(run_id, diff=true)` | evolved tree, `write_to(dry_run=True)` plan, unified diff, `outcomes()` | what the user reads before deciding |
| `apply(run_id, to?)` | files written, backup path | **destructive**; host must confirm with the user |
Expand Down Expand Up @@ -601,7 +601,10 @@ Two guards keep this from folding in on itself:
child starts with the clamped `--max-calls` and learns what the caller asked
for through an internal label-only variable, so `status` says which knob bound
it from either path. So "a worker asked for an unbounded run" is answered by
the environment, not by review.
the environment, not by review. The MCP surface takes the same numbers —
`plan` / `start` accept `max_calls` / `max_usd` and report `call_limit` with
what was asked for and what will be enforced, so a host sees a request of 9
come back as 2 with `source: AGENTDESCENT_MAX_CALLS` (#1184).
* **The controller never hot-swaps itself.** A self-evolved plugin reaches the
user's real installation only through `apply`, which is a human decision on
a diff, after an L1 oracle gate, with a backup. The loop can improve its own
Expand Down
154 changes: 154 additions & 0 deletions tests/test_mcp_call_limit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
"""An MCP host can cap a run: `plan` / `start` take `max_calls` / `max_usd` (#1184).

The CLI and the run store already carry the run-level cap (#1166) and the
operator ceiling (#1082). This is the third surface: a host that calls `start`
from a tool can now say "at most N model calls for this run", hears the CLI's
own refusal when it asks for a dollar cap with no price, and sees the clamped
number (requested vs enforced) when ``AGENTDESCENT_MAX_CALLS`` binds.
"""

from __future__ import annotations

import subprocess
import time

import pytest

from agentdescent import runstore
from agentdescent.cli import NESTED_ENV
from agentdescent.evolvespec import SpecError, build_call_limit
from agentdescent.mcp import Tools

from tests.test_evolvespec import _dir_spec # noqa: F401


@pytest.fixture
def store(tmp_path, monkeypatch):
monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home"))
monkeypatch.setenv("AGENTDESCENT_SURFACE_WRITE_DIRS", str(tmp_path))
return str(tmp_path / "runs")


def _wait(rd, timeout=120):
deadline = time.time() + timeout
while time.time() < deadline:
st = rd.status()
if st.state in ("done", "failed", "cancelled"):
return st
time.sleep(0.2)
raise AssertionError(rd.status())


def test_plan_reports_the_cap_it_will_enforce(store, tmp_path):
t = Tools(store)
payload = t.plan(_dir_spec(str(tmp_path)).to_dict(), max_calls=3)
assert payload["ok"] is True, payload
assert payload["call_limit"] == {"max_calls": 3, "max_usd": None}


def test_plan_without_a_cap_is_unchanged(store, tmp_path):
t = Tools(store)
payload = t.plan(_dir_spec(str(tmp_path)).to_dict())
assert payload["ok"] is True, payload
assert "call_limit" not in payload


def test_start_caps_the_run_and_status_shows_it(store, tmp_path):
t = Tools(store)
started = t.start(_dir_spec(str(tmp_path)).to_dict(), max_calls=1)
assert started["ok"] is True, started
assert started["call_limit"] == {"max_calls": 1, "max_usd": None}
st = _wait(runstore.get(started["run_id"], store=store))
assert st.state == "done", st
assert st.stop_reason == "budget", st
assert "max_calls" in (st.stop_detail or ""), st.stop_detail
assert st.calls == 1, st


def test_start_without_a_cap_is_unchanged(store, tmp_path):
t = Tools(store)
started = t.start(_dir_spec(str(tmp_path)).to_dict())
assert started["ok"] is True, started
assert "call_limit" not in started


def test_max_usd_alone_is_refused_with_the_cli_sentence(store, tmp_path):
t = Tools(store)
spec = _dir_spec(str(tmp_path)).to_dict()
plan = t.plan(spec, max_usd=1.0)
start = t.start(spec, max_usd=1.0)
with pytest.raises(SpecError) as raised:
build_call_limit(max_usd=1.0)
for payload in (plan, start):
assert payload["ok"] is False, payload
# The CLI's wording, not a second sentence for this surface.
assert payload["error"] == str(raised.value), payload["error"]
assert t.status()["runs"] == [], "a refused cap must not leave a run behind"


def test_a_caller_cap_above_the_operator_ceiling_is_clamped_and_labelled(
store, tmp_path, monkeypatch):
monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2")
t = Tools(store)
spec = _dir_spec(str(tmp_path)).to_dict()
assert t.plan(spec, max_calls=9)["call_limit"] == {
"max_calls": 2, "max_usd": None,
"source": runstore.OPERATOR_MAX_CALLS_ENV, "max_calls_requested": 9}
started = t.start(spec, max_calls=9)
assert started["call_limit"] == {
"max_calls": 2, "max_usd": None,
"source": runstore.OPERATOR_MAX_CALLS_ENV, "max_calls_requested": 9}, started
st = _wait(runstore.get(started["run_id"], store=store))
assert st.calls == 2, f"the ceiling let {st.calls} calls through"
assert st.stop_reason == "budget", st
assert runstore.OPERATOR_MAX_CALLS_ENV in (st.stop_detail or ""), st.stop_detail


def test_a_cap_equal_to_the_ceiling_is_not_reported_as_bound(
store, tmp_path, monkeypatch):
"""Same rule as `_clamped_max_calls`: requested == ceiling did not bind.

The payload and `stop_detail` are derived in two places (`mcp.py` here,
`runstore._clamped_max_calls` for the run), so a host that was told "the
ceiling lowered you" about a caller who asked for exactly the ceiling would
be reading a lie the run does not share.
"""
monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2")
payload = Tools(store).plan(_dir_spec(str(tmp_path)).to_dict(), max_calls=2)
assert payload["call_limit"] == {"max_calls": 2, "max_usd": None}, payload


def test_a_run_that_names_no_cap_still_gets_the_ceiling(
store, tmp_path, monkeypatch):
monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2")
payload = Tools(store).plan(
_dir_spec(str(tmp_path)).to_dict(), usd_per_call=None)
assert payload["call_limit"] == {
"max_calls": 2, "max_usd": None,
"source": runstore.OPERATOR_MAX_CALLS_ENV}, payload


def test_start_hands_the_clamped_cap_to_the_detached_child(
store, tmp_path, monkeypatch):
monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2")
seen = {}

def fake_run(argv, **kwargs):
seen["argv"] = argv
return subprocess.CompletedProcess(argv, 0, stdout="4242", stderr="")

monkeypatch.setattr(runstore.subprocess, "run", fake_run)
started = Tools(store).start(_dir_spec(str(tmp_path)).to_dict(), max_calls=9)
assert started["ok"] is True, started
argv = seen["argv"]
assert argv[argv.index("--max-calls") + 1] == "2", argv


def test_nested_start_is_unchanged(store, tmp_path, monkeypatch):
monkeypatch.setenv(NESTED_ENV, "1")
t = Tools(store)
payload = t.start(_dir_spec(str(tmp_path)).to_dict(), max_calls=1)
assert payload == {
"ok": True, "run_id": "nested-stub", "state": "refused", "nested": True,
"reason": "nested-run", "note": payload["note"]}, payload
assert t.status()["runs"] == []
Loading