Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions agentdescent/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@
from . import runstore
from .evolution import (SELECT_SET_NOTE, _check_apply_paths, _file_tree_backup_path,
reward_set_line)
from .evolvespec import (KINDS, EvolveSpec, SpecError, budget_round_overshoot_warning,
from .evolvespec import (CLI_CAP_HINT, KINDS, EvolveSpec, SpecError,
budget_round_overshoot_warning,
build_call_limit, compose, estimate, fold_kind, load_spec,
resolve_kind, _POLICY_SLOTS)
from .probe import first_failed_next
Expand Down Expand Up @@ -2726,11 +2727,12 @@ def cmd_evolve(a: argparse.Namespace) -> int:
# whose status later reads as "process ended without reporting" (#342).
runstore.require_budget_pair(a.budget, a.usd_per_call)
# The run-level cap (#1166): refused here, before a run directory exists,
# with the field and the value named -- a bad flag is not a live run.
# with the field and the value named -- a bad flag is not a live run. The
# CLI passes its own hint: this caller can type the flag (#1196).
try:
build_call_limit(max_calls=getattr(a, "max_calls", None),
max_usd=getattr(a, "max_usd", None),
usd_per_call=a.usd_per_call)
usd_per_call=a.usd_per_call, usd_hint=CLI_CAP_HINT)
except SpecError as e:
return _spec_error(a, e)
# Same gate as plan / MCP start: missing CLI binary must not report started (#346).
Expand Down
23 changes: 19 additions & 4 deletions agentdescent/evolvespec.py
Original file line number Diff line number Diff line change
Expand Up @@ -1110,12 +1110,27 @@ def _cap_value(field: str, value: Any, *, integer: bool) -> float:
return int(value) if integer else float(value)


#: What an unpriced dollar cap tells the caller to add on the command line. A
#: caller that is not the CLI passes its own ``usd_hint`` (#1196).
CLI_CAP_HINT = "pass --usd-per-call on the command line"

#: The default hint names no surface: ``runstore.launch`` / ``execute`` reach
#: this through Python, where a flag or a tool argument would be unactionable.
NEUTRAL_CAP_HINT = "set usd_per_call beside it"


def build_call_limit(max_calls: Any = None, max_usd: Any = None,
usd_per_call: Any = None) -> Optional["CallLimit"]:
usd_per_call: Any = None, *,
usd_hint: str = NEUTRAL_CAP_HINT) -> Optional["CallLimit"]:
"""A validated run-level call cap, or ``None`` when neither limit is set.

Shared by ``compose`` (spec knobs) and the run store / CLI (run options), so
the refusal reads the same wherever the number was typed (#1166).
Shared by ``compose`` (spec knobs) and the run store / CLI (run options): the
reason is the same wherever the number was typed, while the actionable hint
belongs to the surface -- the CLI flag, the tool's argument, or the field
alone for a Python caller (#1166, #1196).

``usd_hint`` carries that surface; its default names the field without
assuming one, and a caller that has a flag or an argument passes its own.
"""
from .budget import CallLimit

Expand All @@ -1131,7 +1146,7 @@ def build_call_limit(max_calls: Any = None, max_usd: Any = None,
if limit.max_usd is not None and not limit.usd_per_call:
raise SpecError(
"max_usd needs usd_per_call: a dollar cap cannot be charged without "
"a per-call price (pass --usd-per-call on the command line)")
f"a per-call price ({usd_hint})")
return limit


Expand Down
14 changes: 8 additions & 6 deletions agentdescent/mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,19 +436,21 @@ def _call_limit_payload(max_calls: Optional[int], max_usd: Optional[float],
"""The run-level cap an MCP ``plan`` / ``start`` will enforce, or ``None`` (#1184).

Validated by the same :func:`~agentdescent.evolvespec.build_call_limit` the
CLI and the run store use, so a bad number -- and a ``max_usd`` with no
``usd_per_call`` -- refuses with the CLI's own sentence instead of a second
wording for this surface. The operator's ceiling is applied with the same
``min`` as ``runstore.launch`` / ``execute``: a caller may lower it, never
raise it, and a caller that names no cap still gets it.
CLI and the run store use: the reason is the same, while the actionable hint
belongs to the surface. This one names ``usd_per_call``, the argument the
host passes in the same tool call -- it has no command line to type
``--usd-per-call`` on (#1196). The operator's ceiling is applied with the
same ``min`` as ``runstore.launch`` / ``execute``: a caller may lower it,
never raise it, and a caller that names no cap still gets it.

``source`` and ``max_calls_requested`` appear only when the ceiling is what
binds, the same distinction ``AGENTDESCENT_MAX_CALLS_REQUESTED`` carries
into the child: without them a host cannot tell "the ceiling lowered your 9
to 2" from "you asked for 2".
"""
limit = build_call_limit(max_calls=max_calls, max_usd=max_usd,
usd_per_call=usd_per_call)
usd_per_call=usd_per_call,
usd_hint="usd_per_call in the same tool call")
ceiling = runstore.operator_max_calls()
if limit is None and ceiling is None:
return None
Expand Down
6 changes: 6 additions & 0 deletions changelog.d/1196-unpriced-cap-hint.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Fixed
The MCP `plan` / `start` tools no longer answer an unpriced `max_usd` with the
CLI's `--usd-per-call` sentence: they now name `usd_per_call`, the argument the
host passes in the same tool call. A Python `runstore.launch` / `execute` caller
is told to set `usd_per_call` beside it, and the command line keeps its own
sentence unchanged (#1196).
86 changes: 83 additions & 3 deletions tests/test_max_usd_refusal_hint.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
"""An unpriced ``max_usd`` names the field to add, not a flag that cannot help (#1183).
"""An unpriced ``max_usd`` names the field the caller can set (#1183, #1196).

``compose`` refuses a spec whose ``evolve`` block caps dollars without a
per-call price before ``cmd_evolve`` ever reads ``--usd-per-call``, so telling
the author to pass that flag sent them in a circle: the same sentence came back
at the same exit code. The price such a run can use lives in the spec.

The same rule then applies one surface at a time: a person typing the command
line can add ``--usd-per-call``, an MCP host can pass ``usd_per_call`` in the
same tool call, and a Python ``runstore.launch`` caller can do neither -- so the
builder's default names the field and no surface at all (#1196).
"""

from __future__ import annotations
Expand All @@ -12,8 +17,16 @@

import pytest

from agentdescent import cli
from agentdescent.evolvespec import EvolveSpec, SpecError, compose
from agentdescent import cli, runstore
from agentdescent.evolvespec import (EvolveSpec, SpecError, build_call_limit,
compose)
from agentdescent.mcp import Tools

from tests.test_evolvespec import _dir_spec # noqa: F401 (the MCP row's spec)

CLI_SENTENCE = (
"max_usd needs usd_per_call: a dollar cap cannot be charged without "
"a per-call price (pass --usd-per-call on the command line)")


def _spec(**evolve):
Expand Down Expand Up @@ -48,3 +61,70 @@ def test_a_priced_spec_still_composes(tmp_path):
assert priced.call_limit.max_usd == 1.0, priced.call_limit
assert priced.call_limit.usd_per_call == 0.02, priced.call_limit
assert compose(_spec(max_calls=3)).call_limit.max_calls == 3


def test_the_command_line_keeps_its_flag_sentence(tmp_path, capsys):
"""A person can type the flag, so the CLI still names it, byte for byte.

Asserted through the real command rather than against the helper's default,
so a neutral default leaking into the CLI turns this red (#1196).
"""
path = tmp_path / "spec.json"
path.write_text(json.dumps(_spec().to_dict()), encoding="utf-8")
code = cli.main(["--store", str(tmp_path / "runs"), "--json",
"evolve", str(path), "--max-usd", "1"])
captured = capsys.readouterr().out
refusal = json.loads(captured[captured.index("{"):])
assert code == 2, code
assert refusal["error"] == CLI_SENTENCE, refusal["error"]


def test_a_tool_refusal_names_the_argument_it_can_pass(tmp_path, monkeypatch):
"""A host has no command line: the hint names ``usd_per_call`` (#1196).

The sentence is asserted whole, not by "contains": the CLI row pins the
shared base text, so an exact suffix is what makes a later reword on the
tool path alone turn red instead of quietly passing.
"""
monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home"))
monkeypatch.setenv("AGENTDESCENT_SURFACE_WRITE_DIRS", str(tmp_path))
tool_sentence = (
"max_usd needs usd_per_call: a dollar cap cannot be charged without "
"a per-call price (usd_per_call in the same tool call)")
t = Tools(str(tmp_path / "runs"))
spec = _dir_spec(str(tmp_path)).to_dict()
plan = t.plan(spec, max_usd=1.0)
start = t.start(spec, max_usd=1.0)
for payload in (plan, start):
assert payload["ok"] is False, payload
assert payload["error"] == tool_sentence, payload["error"]
assert set(plan) == {"ok", "error"}, plan
assert t.status()["runs"] == [], "a refused cap must not leave a run behind"


def test_a_python_caller_is_told_to_set_the_field_not_a_flag(tmp_path):
"""``runstore.launch`` is Python, so its hint names the field and no flag.

No ``RunDir`` is built: the refusal comes before ``launch`` touches the
store, which is exactly what makes "nothing was created" assertable (#1196).
"""
store = tmp_path / "runs"
with pytest.raises(runstore.RunStoreError) as caught:
runstore.launch(store, max_usd=1.0)
message = str(caught.value)
assert "usd_per_call" in message, message
assert "--usd-per-call" not in message, message
assert not store.exists(), "a refused cap created a run store"


def test_the_default_hint_names_no_surface():
"""The default is what a caller we do not know yet inherits: no flag, no API.

Teeth for the row above: if the default goes back to the CLI sentence, or
grows any ``--`` flag, this turns red (#1196).
"""
with pytest.raises(SpecError) as caught:
build_call_limit(max_usd=1.0)
message = str(caught.value)
assert message.endswith("(set usd_per_call beside it)"), message
assert "--" not in message, message
23 changes: 5 additions & 18 deletions tests/test_mcp_call_limit.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

The CLI and the run store already carry the run-level cap (#1166) and the
operator ceiling (#1082). This is the third surface: a host that calls `start`
from a tool can now say "at most N model calls for this run", hears the CLI's
own refusal when it asks for a dollar cap with no price, and sees the clamped
number (requested vs enforced) when ``AGENTDESCENT_MAX_CALLS`` binds.
from a tool can now say "at most N model calls for this run", gets a refusal
naming the argument it can pass when it asks for a dollar cap with no price
(the wording per surface lives in ``tests/test_max_usd_refusal_hint.py``,
#1196), and sees the clamped number (requested vs enforced) when
``AGENTDESCENT_MAX_CALLS`` binds.
"""

from __future__ import annotations
Expand All @@ -16,7 +18,6 @@

from agentdescent import runstore
from agentdescent.cli import NESTED_ENV
from agentdescent.evolvespec import SpecError, build_call_limit
from agentdescent.mcp import Tools

from tests.test_evolvespec import _dir_spec # noqa: F401
Expand Down Expand Up @@ -72,20 +73,6 @@ def test_start_without_a_cap_is_unchanged(store, tmp_path):
assert "call_limit" not in started


def test_max_usd_alone_is_refused_with_the_cli_sentence(store, tmp_path):
t = Tools(store)
spec = _dir_spec(str(tmp_path)).to_dict()
plan = t.plan(spec, max_usd=1.0)
start = t.start(spec, max_usd=1.0)
with pytest.raises(SpecError) as raised:
build_call_limit(max_usd=1.0)
for payload in (plan, start):
assert payload["ok"] is False, payload
# The CLI's wording, not a second sentence for this surface.
assert payload["error"] == str(raised.value), payload["error"]
assert t.status()["runs"] == [], "a refused cap must not leave a run behind"


def test_a_caller_cap_above_the_operator_ceiling_is_clamped_and_labelled(
store, tmp_path, monkeypatch):
monkeypatch.setenv(runstore.OPERATOR_MAX_CALLS_ENV, "2")
Expand Down
Loading