Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions agentdescent/evolvespec.py
Original file line number Diff line number Diff line change
Expand Up @@ -1142,14 +1142,24 @@ def _limit_from_knobs(knobs: Dict[str, Any]) -> Optional["CallLimit"]:
leaves ``max_calls`` in place, so the barrier budget of the same name still
applies -- a per-call cap of N and a barrier cap of N cannot disagree.
"""
limit = build_call_limit(
max_calls=(None if knobs.get("max_calls") is None
else _cap_value("evolve.max_calls", knobs["max_calls"], integer=True)),
max_usd=(None if knobs.get("max_usd") is None
else _cap_value("evolve.max_usd", knobs["max_usd"], integer=False)),
usd_per_call=(None if knobs.get("usd_per_call") is None
else _cap_value("evolve.usd_per_call", knobs["usd_per_call"],
integer=False)))
# Validate all three values first, then refuse the pair: testing the pair on
# the raw knob would answer a non-numeric `max_usd` with "needs usd_per_call"
# and lose the field that is actually wrong (#1183).
max_calls = (None if knobs.get("max_calls") is None
else _cap_value("evolve.max_calls", knobs["max_calls"], integer=True))
max_usd = (None if knobs.get("max_usd") is None
else _cap_value("evolve.max_usd", knobs["max_usd"], integer=False))
usd_per_call = (None if knobs.get("usd_per_call") is None
else _cap_value("evolve.usd_per_call", knobs["usd_per_call"],
integer=False))
# A spec can only be priced by its own field: `compose` runs before the CLI
# reads `--usd-per-call`, so that advice cannot be acted on here (#1183).
if max_usd is not None and not usd_per_call:
raise SpecError(
"evolve.max_usd needs evolve.usd_per_call beside it: a dollar cap "
"cannot be charged without a per-call price")
limit = build_call_limit(max_calls=max_calls, max_usd=max_usd,
usd_per_call=usd_per_call)
knobs.pop("max_usd", None)
knobs.pop("usd_per_call", None)
return limit
Expand Down
5 changes: 5 additions & 0 deletions changelog.d/1183-unpriced-spec-usd.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fixed
A spec whose `evolve` block sets `max_usd` without `usd_per_call` is refused with
`evolve.max_usd needs evolve.usd_per_call beside it` instead of advising a
`--usd-per-call` flag that `compose` refuses before the CLI reads it; the CLI and
run-store wording is unchanged (#1183).
50 changes: 50 additions & 0 deletions tests/test_max_usd_refusal_hint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"""An unpriced ``max_usd`` names the field to add, not a flag that cannot help (#1183).

``compose`` refuses a spec whose ``evolve`` block caps dollars without a
per-call price before ``cmd_evolve`` ever reads ``--usd-per-call``, so telling
the author to pass that flag sent them in a circle: the same sentence came back
at the same exit code. The price such a run can use lives in the spec.
"""

from __future__ import annotations

import json

import pytest

from agentdescent import cli
from agentdescent.evolvespec import EvolveSpec, SpecError, compose


def _spec(**evolve):
return EvolveSpec(
kind="text", target="You are a helpful assistant.",
data={"inline": [{"prompt": "q0", "gold": "a0"}]},
agent={"ref": "echo"}, evolve=evolve)


def test_an_unpriced_spec_dollar_cap_names_the_spec_field(tmp_path, capsys):
"""Refused with ``evolve.usd_per_call`` named, and no run store created."""
with pytest.raises(SpecError) as caught:
compose(_spec(max_usd=1))
message = str(caught.value)
assert "evolve.usd_per_call" in message, message
assert "--usd-per-call" not in message, message

path = tmp_path / "spec.json"
path.write_text(json.dumps(_spec(max_usd=1).to_dict()), encoding="utf-8")
code = cli.main(["--store", str(tmp_path / "runs"), "--json",
"evolve", str(path)])
captured = capsys.readouterr().out
refusal = json.loads(captured[captured.index("{"):])
assert code == 2, code
assert "evolve.usd_per_call" in refusal["error"], refusal
assert not (tmp_path / "runs").exists(), "a refused spec created a run store"


def test_a_priced_spec_still_composes(tmp_path):
"""The pair still works, and an unpriced cap with no dollar limit is fine."""
priced = compose(_spec(max_usd=1, usd_per_call=0.02))
assert priced.call_limit.max_usd == 1.0, priced.call_limit
assert priced.call_limit.usd_per_call == 0.02, priced.call_limit
assert compose(_spec(max_calls=3)).call_limit.max_calls == 3
Loading