diff --git a/agentdescent/agents.py b/agentdescent/agents.py index acfe23b5..a8a8fbd2 100644 --- a/agentdescent/agents.py +++ b/agentdescent/agents.py @@ -28,7 +28,7 @@ import urllib.request from dataclasses import dataclass, field from typing import (Any, Callable, Dict, List, Mapping, Optional, Protocol, - Sequence, runtime_checkable) + Sequence, Tuple, runtime_checkable) from .evolvable import ContractError from ._subprocess import decode_output @@ -76,6 +76,18 @@ class Usage: #: network outage mid-sweep and its 43-minute wall carried ~17 minutes of #: stall that had nothing to do with the architecture under test. failure_seconds: float = 0.0 + #: Failed **attempts** whose provider reported usage -- a 4xx body or a + #: stream that died after the last chunk can carry the tokens already billed. + #: Their tokens are in ``prompt_tokens``/``completion_tokens`` like any other + #: call (money already spent, so the ``max_tokens`` meter must see it); these + #: two only make that part of the bill visible. "Attempts", not calls, is the + #: honest unit: on the direct path each record is a call, but a spec run's + #: sink sees provider attempts and one actor call can be retried, so this may + #: exceed ``calls`` there. A failure that reported nothing adds to neither. + failed_usage_attempts: int = 0 + #: Tokens of the failures counted by ``failed_usage_attempts``; always a subset + #: of ``total_tokens`` (same ``record`` call, never recomputed). + failed_usage_tokens: int = 0 #: In-flight invocations, token -> phase, in :meth:`begin` order. Read #: through :attr:`pending` / :attr:`phase`. _inflight: Dict[int, Optional[str]] = field( @@ -153,10 +165,16 @@ def end(self, token: int) -> None: def record(self, *, prompt_tokens: int = 0, completion_tokens: int = 0, seconds: float = 0.0, failed: bool = False, - finishes: Optional[int] = None) -> None: + finishes: Optional[int] = None, + reported_usage: bool = False) -> None: """Account one finished call. ``finishes`` is the :meth:`begin` token of the invocation this record completes; without it, in-flight state - is left alone (an unpaired provider attempt is not the actor call).""" + is left alone (an unpaired provider attempt is not the actor call). + + ``reported_usage`` says the provider returned usage numbers for this + call, so a failed call's tokens are in the totals *and* in the + failed-with-usage column. Both are incremented here, in one pass, so + the two can never disagree.""" with self._lock: self.calls += 1 self.prompt_tokens += prompt_tokens @@ -165,6 +183,9 @@ def record(self, *, prompt_tokens: int = 0, completion_tokens: int = 0, if failed: self.failures += 1 self.failure_seconds += seconds + if reported_usage: + self.failed_usage_attempts += 1 + self.failed_usage_tokens += prompt_tokens + completion_tokens if finishes is not None: self._inflight.pop(finishes, None) self._notify() @@ -179,6 +200,20 @@ def add_tokens(self, prompt_tokens: int = 0, completion_tokens: int = 0) -> None self.completion_tokens += completion_tokens self._notify() + def note_failed_usage(self, prompt_tokens: int = 0, + completion_tokens: int = 0) -> None: + """Count a usage-bearing failed attempt without touching the totals. + + The sink path has already added the tokens with :meth:`add_tokens`, so + adding them again here would double the bill. Used by + :class:`TokenSink` for a failed provider attempt that reported usage + (#1202); on that path the count is usage-bearing **attempts**, not + actor calls, because one call can be retried several times.""" + with self._lock: + self.failed_usage_attempts += 1 + self.failed_usage_tokens += prompt_tokens + completion_tokens + self._notify() + def token_sink(self) -> "TokenSink": """A ``usage=`` for a model adapter that adds only its **tokens** here. @@ -200,20 +235,33 @@ def summary(self) -> str: f"{self.completion_tokens:,} completion tokens, " f"{self.seconds:.1f}s in the model" + (f", {self.failures} failed ({self.failure_seconds:.1f}s lost)" - if self.failures else "")) + if self.failures else "") + + (f", {self.failed_usage_attempts:,} of them reported " + f"{self.failed_usage_tokens:,} tokens" + if self.failed_usage_attempts else "")) class TokenSink: - """What :meth:`Usage.token_sink` returns: ``record`` keeps only the tokens.""" + """What :meth:`Usage.token_sink` returns: ``record`` keeps only the tokens. + + The engine counts calls and seconds itself on a spec run, so this object + only forwards tokens -- plus, for a failed attempt that reported usage, the + failed-with-usage column (:meth:`Usage.note_failed_usage`). That column is + then a count of usage-bearing provider **attempts**: one actor call can be + retried, so it may exceed ``calls`` -- unlike the direct path, where every + provider record is also a call.""" def __init__(self, target: Usage) -> None: self.target = target def record(self, *, prompt_tokens: int = 0, completion_tokens: int = 0, seconds: float = 0.0, failed: bool = False, - finishes: Optional[int] = None) -> None: + finishes: Optional[int] = None, + reported_usage: bool = False) -> None: if prompt_tokens or completion_tokens: self.target.add_tokens(prompt_tokens, completion_tokens) + if reported_usage and failed: + self.target.note_failed_usage(prompt_tokens, completion_tokens) def metered(completion: Completion, usage: Usage) -> Completion: @@ -765,7 +813,45 @@ def complete(prompt: str) -> str: return with_retries(complete, attempts=retries) if retries > 1 else complete -def _read_sse(response: Any) -> Dict[str, Any]: +def _usage_counts(payload: Any) -> Optional[Tuple[int, int]]: + """``(prompt, completion)`` tokens a provider reported, or ``None``. + + Accepts the two shapes an error body uses -- ``{"usage": {...}}`` and a + top-level mapping -- and both wire formats (``prompt_tokens`` / + ``completion_tokens``, ``input_tokens`` / ``output_tokens``). Only + non-negative ints count: a proxy's string, a boolean or a negative number + must not move a budget. A body that carries only ``total_tokens`` is *not* + split into halves -- that would be a guess -- so it is no evidence. + """ + if not isinstance(payload, dict): + return None + inner = payload.get("usage") + counts = inner if isinstance(inner, dict) else payload + + def _int(*names: str) -> Optional[int]: + for name in names: + value = counts.get(name) + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return None + + prompt = _int("prompt_tokens", "input_tokens") + completion = _int("completion_tokens", "output_tokens") + if prompt is None and completion is None: + return None + return (prompt or 0, completion or 0) + + +def _usage_from_body(body: str) -> Optional[Tuple[int, int]]: + """The usage an error body carries, parsed from the **whole** body.""" + try: + payload = json.loads(body) + except (TypeError, ValueError): + return None + return _usage_counts(payload) + + +def _read_sse(response: Any, seen: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Reassemble an OpenAI-shaped SSE stream into the response it stands for. Returns the same shape the non-streaming path parses -- ``choices[0].message`` @@ -793,6 +879,10 @@ def _read_sse(response: Any) -> Dict[str, Any]: continue if isinstance(chunk.get("usage"), dict): usage_payload = chunk["usage"] + if seen is not None: + # A stream that dies after this chunk still billed these tokens; + # the caller reads `seen` on the way out of its error path. + seen["usage"] = usage_payload for choice in chunk.get("choices") or (): delta = choice.get("delta") or {} piece = delta.get("content") @@ -932,21 +1022,32 @@ def complete(prompt: str) -> str: f"{base}/chat/completions", data=body, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}) t0 = time.time() + seen: Dict[str, Any] = {} try: with urllib.request.urlopen(req, timeout=timeout) as resp: - data = _read_sse(resp) if stream else json.load(resp) + data = _read_sse(resp, seen) if stream else json.load(resp) except urllib.error.HTTPError as e: # The body carries the only useful part -- "rate limit: retry in 12s", # "context length exceeded", "insufficient quota" -- and it lives on # e.read(), so re-raising bare collapses every 4xx to "HTTP Error 429: # Too Many Requests". `_git` and `_CliAgent` both surface the # underlying detail; this was the one provider path that did not. - if usage is not None: - usage.record(seconds=time.time() - t0, failed=True) + # + # Read before `record` (a body reads once) and outside the + # `usage is not None` guard: a caller with no Usage still gets the + # detail in its message. A body that carries usage is money already + # billed, so its counts go into the same `record` call (#1202). try: - detail = e.read().decode("utf-8", "replace").strip()[:400] + body_text = e.read().decode("utf-8", "replace") except Exception: # noqa: BLE001 - the body is best-effort - detail = "" + body_text = "" + detail = body_text.strip()[:400] + if usage is not None: + spent = _usage_from_body(body_text) + usage.record(prompt_tokens=spent[0] if spent else 0, + completion_tokens=spent[1] if spent else 0, + seconds=time.time() - t0, failed=True, + reported_usage=spent is not None) message = (f"{base} returned HTTP {e.code} for model {model!r}" + (f": {detail}" if detail else "")) if e.code in (429, 503): @@ -984,8 +1085,14 @@ def complete(prompt: str) -> str: raise ProviderConfigError(f"{message} -- {hint}") from e raise RuntimeError(message) from e except Exception as e: + # A stream can bill usage in a chunk and then die; `seen` holds the + # last usage the reader got before the transport failed. + spent = _usage_counts(seen.get("usage")) if seen else None if usage is not None: - usage.record(seconds=time.time() - t0, failed=True) + usage.record(prompt_tokens=spent[0] if spent else 0, + completion_tokens=spent[1] if spent else 0, + seconds=time.time() - t0, failed=True, + reported_usage=spent is not None) if _is_timeout(e): # A bare "The read operation timed out" names nothing (#467). raise RuntimeError(_timeout_message( @@ -1076,13 +1183,20 @@ def complete(prompt: str) -> str: data = json.load(resp) except urllib.error.HTTPError as e: # Same reasoning as `openai_compatible`: the body carries the only - # useful part of a 4xx and it lives on `e.read()`. - if usage is not None: - usage.record(seconds=time.time() - t0, failed=True) + # useful part of a 4xx and it lives on `e.read()`. Read it first, + # outside the usage guard, so a `usage=None` caller keeps its detail + # and a usage-bearing body lands in one `record` call (#1202). try: - detail = e.read().decode("utf-8", "replace").strip()[:400] + body_text = e.read().decode("utf-8", "replace") except Exception: # noqa: BLE001 - the body is best-effort - detail = "" + body_text = "" + detail = body_text.strip()[:400] + if usage is not None: + spent = _usage_from_body(body_text) + usage.record(prompt_tokens=spent[0] if spent else 0, + completion_tokens=spent[1] if spent else 0, + seconds=time.time() - t0, failed=True, + reported_usage=spent is not None) message = (f"{base} returned HTTP {e.code} for model {model!r}" + (f": {detail}" if detail else "")) if e.code in (429, 503): diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index 1a4cc459..e1c4a86c 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -1965,7 +1965,12 @@ def save(self, path: str) -> None: "prompt_tokens": self.usage.prompt_tokens, "completion_tokens": self.usage.completion_tokens, "seconds": self.usage.seconds, - "failures": self.usage.failures}, + "failures": self.usage.failures, + # Additive: the part of the bill that came from failed + # calls whose provider reported usage. Not a required + # key for readers; absent in older payloads. + "failed_usage_attempts": self.usage.failed_usage_attempts, + "failed_usage_tokens": self.usage.failed_usage_tokens}, "wallclock": self.wallclock, "rollouts": self.rollouts, "rollout_seconds": self.rollout_seconds, diff --git a/changelog.d/1202-failed-usage-tokens.md b/changelog.d/1202-failed-usage-tokens.md new file mode 100644 index 00000000..eef2221f --- /dev/null +++ b/changelog.d/1202-failed-usage-tokens.md @@ -0,0 +1,7 @@ +Fixed +A failed model call now records the tokens its provider reported -- a 4xx body, +or a stream that died after its last chunk -- so the run's totals and the +`max_tokens` meter no longer under-report what was spent. `Usage` gains +`failed_usage_attempts` and `failed_usage_tokens`, which report that part of +the bill; both are additive fields with defaults, and the accounting basis is +unchanged (#1202). diff --git a/docs/api.md b/docs/api.md index 3213b896..87206ad4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -879,6 +879,8 @@ Usage( seconds: float = 0.0, failures: int = 0, failure_seconds: float = 0.0, + failed_usage_attempts: int = 0, + failed_usage_tokens: int = 0, on_change: Optional[Callable[[], None]] = None, _lock: threading.Lock = ) -> None @@ -890,6 +892,7 @@ Usage( | `begin(phase: Optional[str] = None) -> int` | Mark an actor invocation as started; returns its token. | | `end(token: int) -> None` | Finish the invocation `begin` returned `token` for. | | `estimated_cost(per_1m_prompt: float, per_1m_completion: float) -> float` | Cost at the given per-million-token prices (both provider-specific). | +| `note_failed_usage(prompt_tokens: int = 0, completion_tokens: int = 0) -> None` | Count a usage-bearing failed attempt without touching the totals. | | `record(...)` | Account one finished call. `finishes` is the `begin` token of the invocation this record completes; without it, in-flight state is left alone (an unpaired provider attempt is not the actor call). | | `token_sink() -> 'TokenSink'` | A `usage=` for a model adapter that adds only its **tokens** here. | diff --git a/tests/test_failed_call_usage.py b/tests/test_failed_call_usage.py new file mode 100644 index 00000000..2426d44a --- /dev/null +++ b/tests/test_failed_call_usage.py @@ -0,0 +1,364 @@ +"""Tokens a provider charged on a *failed* call must not vanish (#1202). + +A failed call already records its wall-clock (``Usage.failure_seconds``) but not +the tokens the provider reported -- and a 4xx body or a stream that dies after +the last chunk can carry ``usage``. That is money already spent, so it lands in +the ordinary counters (and therefore in the ``max_tokens`` meter), while the +report keeps it visible as its own column: + +- ``failed_usage_attempts`` / ``failed_usage_tokens`` count only failures that came + with usage numbers; a failure without them records **0** tokens and is only a + failed call. +- The accumulation happens in the same ``record()`` call that adds the tokens, + so the failure column and the totals cannot drift apart. + +Both fields are additive with defaults: an old ``Usage`` payload that lacks them +loads unchanged. +""" + +import io +import json +import urllib.error + +import pytest + +from agentdescent.agents import ( + AgentError, Usage, claude, cli_agent, openai_compatible, +) + + +def _http_error(body, code=429, url="https://api.example/v1/chat/completions"): + return urllib.error.HTTPError(url, code, "err", {}, io.BytesIO(body.encode())) + + +def _openai_ok(monkeypatch, *, prompt=10, completion=20, calls=None): + """urlopen that answers 200 with a well-formed chat completion.""" + def urlopen(req, timeout=None): + if calls is not None: + calls.append(req) + body = {"choices": [{"message": {"content": "ok"}}], + "usage": {"prompt_tokens": prompt, "completion_tokens": completion}} + return io.BytesIO(json.dumps(body).encode()) + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + + +def _openai_error(monkeypatch, body, code=429): + def urlopen(req, timeout=None): + raise _http_error(body, code=code, url=req.full_url) + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + + +# --- the two new fields ---------------------------------------------------- + +def test_a_failed_call_with_reported_usage_counts_in_totals_and_in_its_own_column(): + u = Usage() + u.record(prompt_tokens=40, completion_tokens=60, seconds=1.0, + failed=True, reported_usage=True) + assert (u.prompt_tokens, u.completion_tokens) == (40, 60) + assert u.total_tokens == 100 + assert u.failures == 1 + assert u.failed_usage_attempts == 1 + assert u.failed_usage_tokens == 100 + + +def test_a_failed_call_without_usage_records_zero_tokens(): + u = Usage() + u.record(seconds=2.0, failed=True) + assert (u.prompt_tokens, u.completion_tokens, u.total_tokens) == (0, 0, 0) + assert u.failures == 1 + assert u.failed_usage_attempts == 0 + assert u.failed_usage_tokens == 0 + + +def test_a_successful_call_records_the_same_numbers_as_today(): + u = Usage() + u.record(prompt_tokens=10, completion_tokens=20, seconds=0.5) + assert u.total_tokens == 30 + assert u.failures == 0 and u.failed_usage_attempts == 0 + + +def test_the_failure_column_is_a_subset_of_the_totals_not_a_parallel_tally(): + u = Usage() + u.record(prompt_tokens=40, completion_tokens=60, failed=True, + reported_usage=True) + u.record(seconds=0.1, failed=True) # no usage: failure only + assert u.failures == 2 + assert u.failed_usage_attempts == 1 + assert u.failed_usage_tokens <= u.total_tokens == 100 + # Direct ``Usage`` path only: every provider record is also a call here. + # The sink path counts attempts instead, so this bound does not hold there. + assert u.failed_usage_attempts <= u.calls + + +# --- the two body-reading sites ------------------------------------------- + +def test_an_openai_error_body_with_usage_records_them_and_still_raises(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"error": {"message": "boom"}, + "usage": {"prompt_tokens": 7, "completion_tokens": 3}}) + _openai_error(monkeypatch, body, code=429) + u = Usage() + call = openai_compatible("m", usage=u, retries=0) + with pytest.raises(Exception) as raised: + call("hi") + assert "boom" in str(raised.value) + assert (u.prompt_tokens, u.completion_tokens) == (7, 3) + assert u.failures == 1 and u.failed_usage_attempts == 1 + assert u.failed_usage_tokens == 10 + + +def test_an_error_body_without_usage_is_only_a_failure(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + _openai_error(monkeypatch, json.dumps({"error": {"message": "rate limit"}})) + u = Usage() + call = openai_compatible("m", usage=u, retries=0) + with pytest.raises(Exception): + call("hi") + assert u.total_tokens == 0 + assert u.failures == 1 and u.failed_usage_attempts == 0 + + +def test_the_detail_stops_at_400_bytes_while_usage_parses_from_the_whole_body(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + padding = "x" * 600 + body = json.dumps({"error": {"message": padding}, + "usage": {"prompt_tokens": 5, "completion_tokens": 6}}) + _openai_error(monkeypatch, body) + u = Usage() + call = openai_compatible("m", usage=u, retries=0) + with pytest.raises(Exception) as raised: + call("hi") + message = str(raised.value) + assert padding[:400] not in message.split(": ", 1)[-1][:400] + "zzz" + assert len(message.split(": ", 1)[-1]) <= 400 + assert u.failed_usage_tokens == 11 + + +def test_an_anthropic_error_body_with_usage_records_input_and_output(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "k") + body = json.dumps({"type": "error", + "usage": {"input_tokens": 8, "output_tokens": 2}}) + def urlopen(req, timeout=None): + raise _http_error(body, code=500, url=req.full_url) + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + u = Usage() + from agentdescent.agents import anthropic_compatible + call = anthropic_compatible("m", usage=u, retries=0) + with pytest.raises(Exception): + call("hi") + assert (u.prompt_tokens, u.completion_tokens) == (8, 2) + assert u.failed_usage_attempts == 1 + + +def test_a_caller_without_usage_gets_the_same_error_text(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"error": {"message": "quota"}}) + _openai_error(monkeypatch, body, code=400) + calls, texts = [], [] + for usage in (None, Usage()): + calls.append(openai_compatible("m", usage=usage, retries=0)) + with pytest.raises(Exception) as raised: + calls[-1]("hi") + texts.append(str(raised.value)) + assert texts[0] == texts[1] + assert "quota" in texts[0] + + +def test_a_stream_that_dies_after_usage_still_counts_the_tokens(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + def urlopen(req, timeout=None): + def frames(): + yield b'data: {"usage": {"prompt_tokens": 9, "completion_tokens": 1}}\n\n' + raise RuntimeError("connection reset") + class _Resp: + def __enter__(self): + return frames() + def __exit__(self, *exc): + return False + return _Resp() + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + u = Usage() + call = openai_compatible("m", usage=u, retries=0, stream=True) + with pytest.raises(Exception): + call("hi") + assert (u.prompt_tokens, u.completion_tokens) == (9, 1) + assert u.failures == 1 and u.failed_usage_attempts == 1 + + +def test_top_level_usage_and_junk_counts_are_ignored(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"usage": {"prompt_tokens": "12", "completion_tokens": -5}}) + _openai_error(monkeypatch, body) + u = Usage() + with pytest.raises(Exception): + openai_compatible("m", usage=u, retries=0)("hi") + assert u.total_tokens == 0 and u.failed_usage_attempts == 0 + + +def test_the_subprocess_and_sdk_paths_add_no_tokens(monkeypatch): + u = Usage() + with pytest.raises(AgentError): + cli_agent(["sh", "-c", "exit 3"], usage=u)("hi") + assert u.failures == 1 and u.total_tokens == 0 + + class _Messages: + def create(self, **kw): + raise RuntimeError("sdk boom") + class _Client: + messages = _Messages() + u2 = Usage() + with pytest.raises(RuntimeError): + claude("m", usage=u2, client=_Client(), retries=0)("hi") + assert u2.failures == 1 and u2.total_tokens == 0 + + +# --- additive fields and the report --------------------------------------- + +def test_an_old_usage_payload_without_the_new_fields_loads_with_defaults(): + old = {"calls": 2, "prompt_tokens": 3, "completion_tokens": 4, + "seconds": 1.0, "failures": 1, "failure_seconds": 0.5} + u = Usage(**json.loads(json.dumps(old))) # round-trips through JSON + assert (u.failed_usage_attempts, u.failed_usage_tokens) == (0, 0) + assert u.total_tokens == 7 + # An old fixture prints the same numbers it printed before the fields. + assert u.summary() == Usage(**old).summary() + + +def test_the_summary_separates_failures_that_reported_usage(): + u = Usage() + u.record(prompt_tokens=40, completion_tokens=60, failed=True, + reported_usage=True) + u.record(seconds=0.1, failed=True) + summary = u.summary() + assert "2 failed" in summary + assert "100" in summary and "1" in summary + assert u.failed_usage_tokens <= u.total_tokens + + +def test_a_failed_call_with_usage_moves_the_token_meter_earlier(): + """The tokens of a failed call are money spent: the cap must see them. + + Two runs of the same two-round shape; the first call fails at the provider. + With usage on the failure the ``max_tokens`` meter trips; without usage the + same run stays under the cap -- that is the whole difference (#1202). + """ + import warnings + + from agentdescent.evolution import Task, evolve + + def _run(failure: str): + u = Usage() + state = {"failed": False} + + class _Agent: + def solve(self, rendered, task): + if not state["failed"]: + state["failed"] = True + if failure == "usage": + u.record(prompt_tokens=1_000_000, failed=True, + reported_usage=True) + else: + u.record(failed=True) + raise RuntimeError("429 from the provider") + u.record(prompt_tokens=100, completion_tokens=200) + return task.id + + def propose(self, rendered, task, output, reward): + u.record(prompt_tokens=100, completion_tokens=200) + return None + + tasks = [Task(id=f"t{i}", prompt=f"task {i}") for i in range(8)] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = evolve(tasks, lambda t, o: 0.5, agent=_Agent(), rounds=2, + n_workers=1, max_tokens=5_000, usage=u, + asynchronous=False) + return result, u + + metered, usage_metered = _run("usage") + unmetered, usage_unmetered = _run("none") + assert usage_metered.failed_usage_attempts == 1 + assert usage_unmetered.failed_usage_attempts == 0 + assert metered.stop_reason == "max_tokens" + assert unmetered.stop_reason != "max_tokens" + + +def test_a_sink_records_the_failed_column_without_double_counting(monkeypatch): + """Spec runs hand the adapter a sink, so the column needs a sink path. + + ``add_tokens`` already added the tokens; the failure column must be bumped + without adding them twice, and the sink must not count a call. + """ + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"error": {"message": "boom"}, + "usage": {"prompt_tokens": 40, "completion_tokens": 60}}) + _openai_error(monkeypatch, body) + u = Usage() + call = openai_compatible("m", usage=u.token_sink(), retries=0) + with pytest.raises(Exception): + call("hi") + assert u.calls == 0, "the sink must not count a call" + assert u.total_tokens == 100 + assert u.failed_usage_attempts == 1 + assert u.failed_usage_tokens == 100 + # The sink path has no call counter behind it, so only the subset holds. + assert u.failed_usage_tokens <= u.total_tokens + + +def test_one_retried_call_can_report_several_usage_bearing_attempts(monkeypatch): + """On the sink path the column counts attempts, which may exceed calls.""" + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"usage": {"prompt_tokens": 10, "completion_tokens": 0}}) + _openai_error(monkeypatch, body, code=503) + u = Usage() + call = openai_compatible("m", usage=u.token_sink(), retries=2) + with pytest.raises(Exception): + call("hi") + assert u.calls == 0 + assert u.failed_usage_attempts >= 2 + assert u.failed_usage_tokens == 10 * u.failed_usage_attempts + assert u.failed_usage_tokens <= u.total_tokens + + +def test_the_saved_run_summary_carries_the_new_counters(tmp_path): + """The JSON summary lists them, so they are not visible only in summary(). + + The keys are additive: a reader that knows the old five still works, which + the second assertion pins. + """ + import warnings + + from agentdescent.evolution import Task, evolve + + u = Usage() + state = {"failed": False} + + class _Agent: + def solve(self, rendered, task): + if not state["failed"]: + state["failed"] = True + u.record(prompt_tokens=40, completion_tokens=60, failed=True, + reported_usage=True) + raise RuntimeError("429") + u.record(prompt_tokens=10, completion_tokens=20) + return task.id + + def propose(self, rendered, task, output, reward): + u.record(prompt_tokens=10, completion_tokens=20) + return None + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = evolve([Task(id=f"t{i}", prompt="p") for i in range(4)], + lambda t, o: 0.5, agent=_Agent(), rounds=1, n_workers=1, + usage=u, asynchronous=False) + out = tmp_path / "run.json" + result.save(str(out)) + usage = json.loads(out.read_text())["usage"] + assert usage["failed_usage_attempts"] == 1 + assert usage["failed_usage_tokens"] == 100 + assert usage["failed_usage_tokens"] <= (usage["prompt_tokens"] + + usage["completion_tokens"]) + # Additive: a reader of the old payload's five keys still finds them. + assert {"calls", "prompt_tokens", "completion_tokens", "seconds", + "failures"} <= set(usage)