From 1a0c45cec1b35b5b85bf204c00e9eff9c47d93f3 Mon Sep 17 00:00:00 2001 From: HugoXK Date: Sat, 21 Nov 2026 03:18:05 +0000 Subject: [PATCH 1/2] fix(agents): count the tokens a failed call already billed (#1202) A 4xx body, or a stream that dies after its last chunk, can carry usage numbers -- tokens the provider charged and reported. The adapter recorded the call's wall-clock and dropped the tokens, so the run's totals (and the max_tokens meter) under-reported what was spent. Both body-reading sites now read the body once, before record(), and parse counts from the untruncated text while detail stays cut at 400 bytes. The accumulation happens in the same record() call that adds to the totals, so the failed-with-usage column cannot drift from total_tokens. TokenSink mirrors the counters only, so a spec run's provider attempts are visible without counting anything twice. A failure with no usage records 0 tokens and is only a failed attempt; no estimation tightens a limit. --- agentdescent/agents.py | 148 ++++++++-- agentdescent/evolution.py | 7 +- changelog.d/1202-failed-usage-tokens.md | 7 + docs/api.md | 3 + tests/test_failed_call_usage.py | 364 ++++++++++++++++++++++++ 5 files changed, 511 insertions(+), 18 deletions(-) create mode 100644 changelog.d/1202-failed-usage-tokens.md create mode 100644 tests/test_failed_call_usage.py diff --git a/agentdescent/agents.py b/agentdescent/agents.py index acfe23b5..d471219a 100644 --- a/agentdescent/agents.py +++ b/agentdescent/agents.py @@ -76,6 +76,18 @@ class Usage: #: network outage mid-sweep and its 43-minute wall carried ~17 minutes of #: stall that had nothing to do with the architecture under test. failure_seconds: float = 0.0 + #: Failed **attempts** whose provider reported usage -- a 4xx body or a + #: stream that died after the last chunk can carry the tokens already billed. + #: Their tokens are in ``prompt_tokens``/``completion_tokens`` like any other + #: call (money already spent, so the ``max_tokens`` meter must see it); these + #: two only make that part of the bill visible. "Attempts", not calls, is the + #: honest unit: on the direct path each record is a call, but a spec run's + #: sink sees provider attempts and one actor call can be retried, so this may + #: exceed ``calls`` there. A failure that reported nothing adds to neither. + failed_usage_attempts: int = 0 + #: Tokens of the failures counted by ``failed_usage_attempts``; always a subset + #: of ``total_tokens`` (same ``record`` call, never recomputed). + failed_usage_tokens: int = 0 #: In-flight invocations, token -> phase, in :meth:`begin` order. Read #: through :attr:`pending` / :attr:`phase`. _inflight: Dict[int, Optional[str]] = field( @@ -153,10 +165,16 @@ def end(self, token: int) -> None: def record(self, *, prompt_tokens: int = 0, completion_tokens: int = 0, seconds: float = 0.0, failed: bool = False, - finishes: Optional[int] = None) -> None: + finishes: Optional[int] = None, + reported_usage: bool = False) -> None: """Account one finished call. ``finishes`` is the :meth:`begin` token of the invocation this record completes; without it, in-flight state - is left alone (an unpaired provider attempt is not the actor call).""" + is left alone (an unpaired provider attempt is not the actor call). + + ``reported_usage`` says the provider returned usage numbers for this + call, so a failed call's tokens are in the totals *and* in the + failed-with-usage column. Both are incremented here, in one pass, so + the two can never disagree.""" with self._lock: self.calls += 1 self.prompt_tokens += prompt_tokens @@ -165,6 +183,9 @@ def record(self, *, prompt_tokens: int = 0, completion_tokens: int = 0, if failed: self.failures += 1 self.failure_seconds += seconds + if reported_usage: + self.failed_usage_attempts += 1 + self.failed_usage_tokens += prompt_tokens + completion_tokens if finishes is not None: self._inflight.pop(finishes, None) self._notify() @@ -179,6 +200,20 @@ def add_tokens(self, prompt_tokens: int = 0, completion_tokens: int = 0) -> None self.completion_tokens += completion_tokens self._notify() + def note_failed_usage(self, prompt_tokens: int = 0, + completion_tokens: int = 0) -> None: + """Count a usage-bearing failed attempt without touching the totals. + + The sink path has already added the tokens with :meth:`add_tokens`, so + adding them again here would double the bill. Used by + :class:`TokenSink` for a failed provider attempt that reported usage + (#1202); on that path the count is usage-bearing **attempts**, not + actor calls, because one call can be retried several times.""" + with self._lock: + self.failed_usage_attempts += 1 + self.failed_usage_tokens += prompt_tokens + completion_tokens + self._notify() + def token_sink(self) -> "TokenSink": """A ``usage=`` for a model adapter that adds only its **tokens** here. @@ -200,20 +235,33 @@ def summary(self) -> str: f"{self.completion_tokens:,} completion tokens, " f"{self.seconds:.1f}s in the model" + (f", {self.failures} failed ({self.failure_seconds:.1f}s lost)" - if self.failures else "")) + if self.failures else "") + + (f", {self.failed_usage_attempts:,} of them reported " + f"{self.failed_usage_tokens:,} tokens" + if self.failed_usage_attempts else "")) class TokenSink: - """What :meth:`Usage.token_sink` returns: ``record`` keeps only the tokens.""" + """What :meth:`Usage.token_sink` returns: ``record`` keeps only the tokens. + + The engine counts calls and seconds itself on a spec run, so this object + only forwards tokens -- plus, for a failed attempt that reported usage, the + failed-with-usage column (:meth:`Usage.note_failed_usage`). That column is + then a count of usage-bearing provider **attempts**: one actor call can be + retried, so it may exceed ``calls`` -- unlike the direct path, where every + provider record is also a call.""" def __init__(self, target: Usage) -> None: self.target = target def record(self, *, prompt_tokens: int = 0, completion_tokens: int = 0, seconds: float = 0.0, failed: bool = False, - finishes: Optional[int] = None) -> None: + finishes: Optional[int] = None, + reported_usage: bool = False) -> None: if prompt_tokens or completion_tokens: self.target.add_tokens(prompt_tokens, completion_tokens) + if reported_usage and failed: + self.target.note_failed_usage(prompt_tokens, completion_tokens) def metered(completion: Completion, usage: Usage) -> Completion: @@ -765,7 +813,45 @@ def complete(prompt: str) -> str: return with_retries(complete, attempts=retries) if retries > 1 else complete -def _read_sse(response: Any) -> Dict[str, Any]: +def _usage_counts(payload: Any) -> Optional[Tuple[int, int]]: + """``(prompt, completion)`` tokens a provider reported, or ``None``. + + Accepts the two shapes an error body uses -- ``{"usage": {...}}`` and a + top-level mapping -- and both wire formats (``prompt_tokens`` / + ``completion_tokens``, ``input_tokens`` / ``output_tokens``). Only + non-negative ints count: a proxy's string, a boolean or a negative number + must not move a budget. A body that carries only ``total_tokens`` is *not* + split into halves -- that would be a guess -- so it is no evidence. + """ + if not isinstance(payload, dict): + return None + inner = payload.get("usage") + counts = inner if isinstance(inner, dict) else payload + + def _int(*names: str) -> Optional[int]: + for name in names: + value = counts.get(name) + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return None + + prompt = _int("prompt_tokens", "input_tokens") + completion = _int("completion_tokens", "output_tokens") + if prompt is None and completion is None: + return None + return (prompt or 0, completion or 0) + + +def _usage_from_body(body: str) -> Optional[Tuple[int, int]]: + """The usage an error body carries, parsed from the **whole** body.""" + try: + payload = json.loads(body) + except (TypeError, ValueError): + return None + return _usage_counts(payload) + + +def _read_sse(response: Any, seen: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Reassemble an OpenAI-shaped SSE stream into the response it stands for. Returns the same shape the non-streaming path parses -- ``choices[0].message`` @@ -793,6 +879,10 @@ def _read_sse(response: Any) -> Dict[str, Any]: continue if isinstance(chunk.get("usage"), dict): usage_payload = chunk["usage"] + if seen is not None: + # A stream that dies after this chunk still billed these tokens; + # the caller reads `seen` on the way out of its error path. + seen["usage"] = usage_payload for choice in chunk.get("choices") or (): delta = choice.get("delta") or {} piece = delta.get("content") @@ -932,21 +1022,32 @@ def complete(prompt: str) -> str: f"{base}/chat/completions", data=body, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}) t0 = time.time() + seen: Dict[str, Any] = {} try: with urllib.request.urlopen(req, timeout=timeout) as resp: - data = _read_sse(resp) if stream else json.load(resp) + data = _read_sse(resp, seen) if stream else json.load(resp) except urllib.error.HTTPError as e: # The body carries the only useful part -- "rate limit: retry in 12s", # "context length exceeded", "insufficient quota" -- and it lives on # e.read(), so re-raising bare collapses every 4xx to "HTTP Error 429: # Too Many Requests". `_git` and `_CliAgent` both surface the # underlying detail; this was the one provider path that did not. - if usage is not None: - usage.record(seconds=time.time() - t0, failed=True) + # + # Read before `record` (a body reads once) and outside the + # `usage is not None` guard: a caller with no Usage still gets the + # detail in its message. A body that carries usage is money already + # billed, so its counts go into the same `record` call (#1202). try: - detail = e.read().decode("utf-8", "replace").strip()[:400] + body_text = e.read().decode("utf-8", "replace") except Exception: # noqa: BLE001 - the body is best-effort - detail = "" + body_text = "" + detail = body_text.strip()[:400] + if usage is not None: + spent = _usage_from_body(body_text) + usage.record(prompt_tokens=spent[0] if spent else 0, + completion_tokens=spent[1] if spent else 0, + seconds=time.time() - t0, failed=True, + reported_usage=spent is not None) message = (f"{base} returned HTTP {e.code} for model {model!r}" + (f": {detail}" if detail else "")) if e.code in (429, 503): @@ -984,8 +1085,14 @@ def complete(prompt: str) -> str: raise ProviderConfigError(f"{message} -- {hint}") from e raise RuntimeError(message) from e except Exception as e: + # A stream can bill usage in a chunk and then die; `seen` holds the + # last usage the reader got before the transport failed. + spent = _usage_counts(seen.get("usage")) if seen else None if usage is not None: - usage.record(seconds=time.time() - t0, failed=True) + usage.record(prompt_tokens=spent[0] if spent else 0, + completion_tokens=spent[1] if spent else 0, + seconds=time.time() - t0, failed=True, + reported_usage=spent is not None) if _is_timeout(e): # A bare "The read operation timed out" names nothing (#467). raise RuntimeError(_timeout_message( @@ -1076,13 +1183,20 @@ def complete(prompt: str) -> str: data = json.load(resp) except urllib.error.HTTPError as e: # Same reasoning as `openai_compatible`: the body carries the only - # useful part of a 4xx and it lives on `e.read()`. - if usage is not None: - usage.record(seconds=time.time() - t0, failed=True) + # useful part of a 4xx and it lives on `e.read()`. Read it first, + # outside the usage guard, so a `usage=None` caller keeps its detail + # and a usage-bearing body lands in one `record` call (#1202). try: - detail = e.read().decode("utf-8", "replace").strip()[:400] + body_text = e.read().decode("utf-8", "replace") except Exception: # noqa: BLE001 - the body is best-effort - detail = "" + body_text = "" + detail = body_text.strip()[:400] + if usage is not None: + spent = _usage_from_body(body_text) + usage.record(prompt_tokens=spent[0] if spent else 0, + completion_tokens=spent[1] if spent else 0, + seconds=time.time() - t0, failed=True, + reported_usage=spent is not None) message = (f"{base} returned HTTP {e.code} for model {model!r}" + (f": {detail}" if detail else "")) if e.code in (429, 503): diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index 1a4cc459..e1c4a86c 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -1965,7 +1965,12 @@ def save(self, path: str) -> None: "prompt_tokens": self.usage.prompt_tokens, "completion_tokens": self.usage.completion_tokens, "seconds": self.usage.seconds, - "failures": self.usage.failures}, + "failures": self.usage.failures, + # Additive: the part of the bill that came from failed + # calls whose provider reported usage. Not a required + # key for readers; absent in older payloads. + "failed_usage_attempts": self.usage.failed_usage_attempts, + "failed_usage_tokens": self.usage.failed_usage_tokens}, "wallclock": self.wallclock, "rollouts": self.rollouts, "rollout_seconds": self.rollout_seconds, diff --git a/changelog.d/1202-failed-usage-tokens.md b/changelog.d/1202-failed-usage-tokens.md new file mode 100644 index 00000000..eef2221f --- /dev/null +++ b/changelog.d/1202-failed-usage-tokens.md @@ -0,0 +1,7 @@ +Fixed +A failed model call now records the tokens its provider reported -- a 4xx body, +or a stream that died after its last chunk -- so the run's totals and the +`max_tokens` meter no longer under-report what was spent. `Usage` gains +`failed_usage_attempts` and `failed_usage_tokens`, which report that part of +the bill; both are additive fields with defaults, and the accounting basis is +unchanged (#1202). diff --git a/docs/api.md b/docs/api.md index 3213b896..87206ad4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -879,6 +879,8 @@ Usage( seconds: float = 0.0, failures: int = 0, failure_seconds: float = 0.0, + failed_usage_attempts: int = 0, + failed_usage_tokens: int = 0, on_change: Optional[Callable[[], None]] = None, _lock: threading.Lock = ) -> None @@ -890,6 +892,7 @@ Usage( | `begin(phase: Optional[str] = None) -> int` | Mark an actor invocation as started; returns its token. | | `end(token: int) -> None` | Finish the invocation `begin` returned `token` for. | | `estimated_cost(per_1m_prompt: float, per_1m_completion: float) -> float` | Cost at the given per-million-token prices (both provider-specific). | +| `note_failed_usage(prompt_tokens: int = 0, completion_tokens: int = 0) -> None` | Count a usage-bearing failed attempt without touching the totals. | | `record(...)` | Account one finished call. `finishes` is the `begin` token of the invocation this record completes; without it, in-flight state is left alone (an unpaired provider attempt is not the actor call). | | `token_sink() -> 'TokenSink'` | A `usage=` for a model adapter that adds only its **tokens** here. | diff --git a/tests/test_failed_call_usage.py b/tests/test_failed_call_usage.py new file mode 100644 index 00000000..2426d44a --- /dev/null +++ b/tests/test_failed_call_usage.py @@ -0,0 +1,364 @@ +"""Tokens a provider charged on a *failed* call must not vanish (#1202). + +A failed call already records its wall-clock (``Usage.failure_seconds``) but not +the tokens the provider reported -- and a 4xx body or a stream that dies after +the last chunk can carry ``usage``. That is money already spent, so it lands in +the ordinary counters (and therefore in the ``max_tokens`` meter), while the +report keeps it visible as its own column: + +- ``failed_usage_attempts`` / ``failed_usage_tokens`` count only failures that came + with usage numbers; a failure without them records **0** tokens and is only a + failed call. +- The accumulation happens in the same ``record()`` call that adds the tokens, + so the failure column and the totals cannot drift apart. + +Both fields are additive with defaults: an old ``Usage`` payload that lacks them +loads unchanged. +""" + +import io +import json +import urllib.error + +import pytest + +from agentdescent.agents import ( + AgentError, Usage, claude, cli_agent, openai_compatible, +) + + +def _http_error(body, code=429, url="https://api.example/v1/chat/completions"): + return urllib.error.HTTPError(url, code, "err", {}, io.BytesIO(body.encode())) + + +def _openai_ok(monkeypatch, *, prompt=10, completion=20, calls=None): + """urlopen that answers 200 with a well-formed chat completion.""" + def urlopen(req, timeout=None): + if calls is not None: + calls.append(req) + body = {"choices": [{"message": {"content": "ok"}}], + "usage": {"prompt_tokens": prompt, "completion_tokens": completion}} + return io.BytesIO(json.dumps(body).encode()) + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + + +def _openai_error(monkeypatch, body, code=429): + def urlopen(req, timeout=None): + raise _http_error(body, code=code, url=req.full_url) + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + + +# --- the two new fields ---------------------------------------------------- + +def test_a_failed_call_with_reported_usage_counts_in_totals_and_in_its_own_column(): + u = Usage() + u.record(prompt_tokens=40, completion_tokens=60, seconds=1.0, + failed=True, reported_usage=True) + assert (u.prompt_tokens, u.completion_tokens) == (40, 60) + assert u.total_tokens == 100 + assert u.failures == 1 + assert u.failed_usage_attempts == 1 + assert u.failed_usage_tokens == 100 + + +def test_a_failed_call_without_usage_records_zero_tokens(): + u = Usage() + u.record(seconds=2.0, failed=True) + assert (u.prompt_tokens, u.completion_tokens, u.total_tokens) == (0, 0, 0) + assert u.failures == 1 + assert u.failed_usage_attempts == 0 + assert u.failed_usage_tokens == 0 + + +def test_a_successful_call_records_the_same_numbers_as_today(): + u = Usage() + u.record(prompt_tokens=10, completion_tokens=20, seconds=0.5) + assert u.total_tokens == 30 + assert u.failures == 0 and u.failed_usage_attempts == 0 + + +def test_the_failure_column_is_a_subset_of_the_totals_not_a_parallel_tally(): + u = Usage() + u.record(prompt_tokens=40, completion_tokens=60, failed=True, + reported_usage=True) + u.record(seconds=0.1, failed=True) # no usage: failure only + assert u.failures == 2 + assert u.failed_usage_attempts == 1 + assert u.failed_usage_tokens <= u.total_tokens == 100 + # Direct ``Usage`` path only: every provider record is also a call here. + # The sink path counts attempts instead, so this bound does not hold there. + assert u.failed_usage_attempts <= u.calls + + +# --- the two body-reading sites ------------------------------------------- + +def test_an_openai_error_body_with_usage_records_them_and_still_raises(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"error": {"message": "boom"}, + "usage": {"prompt_tokens": 7, "completion_tokens": 3}}) + _openai_error(monkeypatch, body, code=429) + u = Usage() + call = openai_compatible("m", usage=u, retries=0) + with pytest.raises(Exception) as raised: + call("hi") + assert "boom" in str(raised.value) + assert (u.prompt_tokens, u.completion_tokens) == (7, 3) + assert u.failures == 1 and u.failed_usage_attempts == 1 + assert u.failed_usage_tokens == 10 + + +def test_an_error_body_without_usage_is_only_a_failure(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + _openai_error(monkeypatch, json.dumps({"error": {"message": "rate limit"}})) + u = Usage() + call = openai_compatible("m", usage=u, retries=0) + with pytest.raises(Exception): + call("hi") + assert u.total_tokens == 0 + assert u.failures == 1 and u.failed_usage_attempts == 0 + + +def test_the_detail_stops_at_400_bytes_while_usage_parses_from_the_whole_body(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + padding = "x" * 600 + body = json.dumps({"error": {"message": padding}, + "usage": {"prompt_tokens": 5, "completion_tokens": 6}}) + _openai_error(monkeypatch, body) + u = Usage() + call = openai_compatible("m", usage=u, retries=0) + with pytest.raises(Exception) as raised: + call("hi") + message = str(raised.value) + assert padding[:400] not in message.split(": ", 1)[-1][:400] + "zzz" + assert len(message.split(": ", 1)[-1]) <= 400 + assert u.failed_usage_tokens == 11 + + +def test_an_anthropic_error_body_with_usage_records_input_and_output(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "k") + body = json.dumps({"type": "error", + "usage": {"input_tokens": 8, "output_tokens": 2}}) + def urlopen(req, timeout=None): + raise _http_error(body, code=500, url=req.full_url) + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + u = Usage() + from agentdescent.agents import anthropic_compatible + call = anthropic_compatible("m", usage=u, retries=0) + with pytest.raises(Exception): + call("hi") + assert (u.prompt_tokens, u.completion_tokens) == (8, 2) + assert u.failed_usage_attempts == 1 + + +def test_a_caller_without_usage_gets_the_same_error_text(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"error": {"message": "quota"}}) + _openai_error(monkeypatch, body, code=400) + calls, texts = [], [] + for usage in (None, Usage()): + calls.append(openai_compatible("m", usage=usage, retries=0)) + with pytest.raises(Exception) as raised: + calls[-1]("hi") + texts.append(str(raised.value)) + assert texts[0] == texts[1] + assert "quota" in texts[0] + + +def test_a_stream_that_dies_after_usage_still_counts_the_tokens(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + def urlopen(req, timeout=None): + def frames(): + yield b'data: {"usage": {"prompt_tokens": 9, "completion_tokens": 1}}\n\n' + raise RuntimeError("connection reset") + class _Resp: + def __enter__(self): + return frames() + def __exit__(self, *exc): + return False + return _Resp() + monkeypatch.setattr("agentdescent.agents.urllib.request.urlopen", urlopen) + u = Usage() + call = openai_compatible("m", usage=u, retries=0, stream=True) + with pytest.raises(Exception): + call("hi") + assert (u.prompt_tokens, u.completion_tokens) == (9, 1) + assert u.failures == 1 and u.failed_usage_attempts == 1 + + +def test_top_level_usage_and_junk_counts_are_ignored(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"usage": {"prompt_tokens": "12", "completion_tokens": -5}}) + _openai_error(monkeypatch, body) + u = Usage() + with pytest.raises(Exception): + openai_compatible("m", usage=u, retries=0)("hi") + assert u.total_tokens == 0 and u.failed_usage_attempts == 0 + + +def test_the_subprocess_and_sdk_paths_add_no_tokens(monkeypatch): + u = Usage() + with pytest.raises(AgentError): + cli_agent(["sh", "-c", "exit 3"], usage=u)("hi") + assert u.failures == 1 and u.total_tokens == 0 + + class _Messages: + def create(self, **kw): + raise RuntimeError("sdk boom") + class _Client: + messages = _Messages() + u2 = Usage() + with pytest.raises(RuntimeError): + claude("m", usage=u2, client=_Client(), retries=0)("hi") + assert u2.failures == 1 and u2.total_tokens == 0 + + +# --- additive fields and the report --------------------------------------- + +def test_an_old_usage_payload_without_the_new_fields_loads_with_defaults(): + old = {"calls": 2, "prompt_tokens": 3, "completion_tokens": 4, + "seconds": 1.0, "failures": 1, "failure_seconds": 0.5} + u = Usage(**json.loads(json.dumps(old))) # round-trips through JSON + assert (u.failed_usage_attempts, u.failed_usage_tokens) == (0, 0) + assert u.total_tokens == 7 + # An old fixture prints the same numbers it printed before the fields. + assert u.summary() == Usage(**old).summary() + + +def test_the_summary_separates_failures_that_reported_usage(): + u = Usage() + u.record(prompt_tokens=40, completion_tokens=60, failed=True, + reported_usage=True) + u.record(seconds=0.1, failed=True) + summary = u.summary() + assert "2 failed" in summary + assert "100" in summary and "1" in summary + assert u.failed_usage_tokens <= u.total_tokens + + +def test_a_failed_call_with_usage_moves_the_token_meter_earlier(): + """The tokens of a failed call are money spent: the cap must see them. + + Two runs of the same two-round shape; the first call fails at the provider. + With usage on the failure the ``max_tokens`` meter trips; without usage the + same run stays under the cap -- that is the whole difference (#1202). + """ + import warnings + + from agentdescent.evolution import Task, evolve + + def _run(failure: str): + u = Usage() + state = {"failed": False} + + class _Agent: + def solve(self, rendered, task): + if not state["failed"]: + state["failed"] = True + if failure == "usage": + u.record(prompt_tokens=1_000_000, failed=True, + reported_usage=True) + else: + u.record(failed=True) + raise RuntimeError("429 from the provider") + u.record(prompt_tokens=100, completion_tokens=200) + return task.id + + def propose(self, rendered, task, output, reward): + u.record(prompt_tokens=100, completion_tokens=200) + return None + + tasks = [Task(id=f"t{i}", prompt=f"task {i}") for i in range(8)] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = evolve(tasks, lambda t, o: 0.5, agent=_Agent(), rounds=2, + n_workers=1, max_tokens=5_000, usage=u, + asynchronous=False) + return result, u + + metered, usage_metered = _run("usage") + unmetered, usage_unmetered = _run("none") + assert usage_metered.failed_usage_attempts == 1 + assert usage_unmetered.failed_usage_attempts == 0 + assert metered.stop_reason == "max_tokens" + assert unmetered.stop_reason != "max_tokens" + + +def test_a_sink_records_the_failed_column_without_double_counting(monkeypatch): + """Spec runs hand the adapter a sink, so the column needs a sink path. + + ``add_tokens`` already added the tokens; the failure column must be bumped + without adding them twice, and the sink must not count a call. + """ + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"error": {"message": "boom"}, + "usage": {"prompt_tokens": 40, "completion_tokens": 60}}) + _openai_error(monkeypatch, body) + u = Usage() + call = openai_compatible("m", usage=u.token_sink(), retries=0) + with pytest.raises(Exception): + call("hi") + assert u.calls == 0, "the sink must not count a call" + assert u.total_tokens == 100 + assert u.failed_usage_attempts == 1 + assert u.failed_usage_tokens == 100 + # The sink path has no call counter behind it, so only the subset holds. + assert u.failed_usage_tokens <= u.total_tokens + + +def test_one_retried_call_can_report_several_usage_bearing_attempts(monkeypatch): + """On the sink path the column counts attempts, which may exceed calls.""" + monkeypatch.setenv("OPENAI_API_KEY", "k") + body = json.dumps({"usage": {"prompt_tokens": 10, "completion_tokens": 0}}) + _openai_error(monkeypatch, body, code=503) + u = Usage() + call = openai_compatible("m", usage=u.token_sink(), retries=2) + with pytest.raises(Exception): + call("hi") + assert u.calls == 0 + assert u.failed_usage_attempts >= 2 + assert u.failed_usage_tokens == 10 * u.failed_usage_attempts + assert u.failed_usage_tokens <= u.total_tokens + + +def test_the_saved_run_summary_carries_the_new_counters(tmp_path): + """The JSON summary lists them, so they are not visible only in summary(). + + The keys are additive: a reader that knows the old five still works, which + the second assertion pins. + """ + import warnings + + from agentdescent.evolution import Task, evolve + + u = Usage() + state = {"failed": False} + + class _Agent: + def solve(self, rendered, task): + if not state["failed"]: + state["failed"] = True + u.record(prompt_tokens=40, completion_tokens=60, failed=True, + reported_usage=True) + raise RuntimeError("429") + u.record(prompt_tokens=10, completion_tokens=20) + return task.id + + def propose(self, rendered, task, output, reward): + u.record(prompt_tokens=10, completion_tokens=20) + return None + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = evolve([Task(id=f"t{i}", prompt="p") for i in range(4)], + lambda t, o: 0.5, agent=_Agent(), rounds=1, n_workers=1, + usage=u, asynchronous=False) + out = tmp_path / "run.json" + result.save(str(out)) + usage = json.loads(out.read_text())["usage"] + assert usage["failed_usage_attempts"] == 1 + assert usage["failed_usage_tokens"] == 100 + assert usage["failed_usage_tokens"] <= (usage["prompt_tokens"] + + usage["completion_tokens"]) + # Additive: a reader of the old payload's five keys still finds them. + assert {"calls", "prompt_tokens", "completion_tokens", "seconds", + "failures"} <= set(usage) From 320b4a09c4f7a66ce6696195412e3dc2dd51789e Mon Sep 17 00:00:00 2001 From: HugoXK Date: Sat, 21 Nov 2026 03:40:39 +0000 Subject: [PATCH 2/2] fix(agents): import Tuple for the usage-body annotations (#1202) `_usage_counts` and `_usage_from_body` return `Optional[Tuple[int, int]]`, but `Tuple` was never imported: the typing line stopped at `Sequence, runtime_checkable`. With `from __future__ import annotations` the paste does not fail at import, so the tests and CI stayed green, and the gap only showed when the annotations were resolved -- `typing.get_type_hints` on either function raised `NameError: name 'Tuple' is not defined`. --- agentdescent/agents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentdescent/agents.py b/agentdescent/agents.py index d471219a..a8a8fbd2 100644 --- a/agentdescent/agents.py +++ b/agentdescent/agents.py @@ -28,7 +28,7 @@ import urllib.request from dataclasses import dataclass, field from typing import (Any, Callable, Dict, List, Mapping, Optional, Protocol, - Sequence, runtime_checkable) + Sequence, Tuple, runtime_checkable) from .evolvable import ContractError from ._subprocess import decode_output