diff --git a/app/nullius/judge/client.py b/app/nullius/judge/client.py index 496b4d6e..23e7c3d8 100644 --- a/app/nullius/judge/client.py +++ b/app/nullius/judge/client.py @@ -436,6 +436,7 @@ def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Calla body = {"model": model, "input": state, "questions": openai_questions(asked)} if openai else {"state": state, "model": model, "questions": asked} post = post or http_post status, data = 0, {} + started = time.monotonic() for attempt in range(retries + 1): status, data = post(url, body, key) if status in (429, 503, 529) and attempt < retries: @@ -459,6 +460,7 @@ def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Calla rec = {"questions": Q.VERSION, "judge": {"provider": provider, "model": str(data.get("model") or model)}, "asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated), "answers": answers, "usage": {"input_tokens": usage.get("input_tokens"), "output_tokens": usage.get("output_tokens"), + "seconds": round(time.monotonic() - started, 2), **({"cost_usd": float(usage["cost"])} if _number(usage.get("cost")) is not None else {})}} except (AttributeError, KeyError, TypeError, ValueError) as e: # a reply of another shape: said, never a crash of the batch raise JudgeError(502, f"the judge's reply could not be read: {e.__class__.__name__}") from e @@ -525,6 +527,7 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio "response_format": {"type": "json_object"}, **({"usage": {"include": True}} if judge.provider == "openrouter" else {})} post = post or http_post status, data = 0, {} + started = time.monotonic() for attempt in range(retries + 1): status, data = post(judge.url, body, judge.key, auth=providers.auth("messages", judge.key)) if messages else post(judge.url, body, judge.key) if status in (429, 502, 503, 529) and attempt < retries: @@ -567,6 +570,7 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio rec = {"questions": Q.VERSION, "judge": {"provider": judge.provider, "model": str(data.get("model") or judge.model)}, "asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated), "answers": answers, "usage": {"input_tokens": usage.get(tokens_in), "output_tokens": usage.get(tokens_out), + "seconds": round(time.monotonic() - started, 2), **({"cost_usd": usage["cost"]} if isinstance(usage.get("cost"), (int, float)) else {})}} if kind is not None: rec["states"] = {kind: {"chars": len(state), "truncated": bool(truncated), "input_tokens": usage.get(tokens_in), @@ -593,7 +597,7 @@ def merge(records) -> Optional[dict]: if rec.get("refusals"): merged["refusals"] = {**(merged.get("refusals") or {}), **rec["refusals"]} merged["asked_at"] = rec["asked_at"]; merged["judge"] = rec["judge"] - for k in ("input_tokens", "output_tokens", "cost_usd"): + for k in ("input_tokens", "output_tokens", "cost_usd", "seconds"): # the calls run one after another: their times add up if rec["usage"].get(k) is not None: merged["usage"][k] = (merged["usage"].get(k) or 0) + rec["usage"][k] return merged diff --git a/app/static/style.css b/app/static/style.css index 75de2ae6..cc25d710 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1818,3 +1818,7 @@ select.ui-select-native { display: none !important; } .tree-kids { box-sizing: border-box; } @media (prefers-reduced-motion: no-preference) { .shell { transition: grid-template-columns .24s ease; } .tree-caret .ic { transition: transform .24s ease; } } @media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .la-fold-btn::before, :root .disclosure-chevron::before, .tree-caret .ic, .side { transition: none; } } + +/* the judge's button under its badge, what its last call took beside it */ +.judge-actions { display: flex; align-items: center; gap: .75rem; flex-wrap: wrap; margin: .6rem 0 1rem; } +.judge-actions .header-form { margin: 0; } diff --git a/app/templates/traces/_judgement.html b/app/templates/traces/_judgement.html index 0c85e0b4..a60592a9 100644 --- a/app/templates/traces/_judgement.html +++ b/app/templates/traces/_judgement.html @@ -13,9 +13,12 @@
experimental
+ +
{% if j.keyed %}
{% else %}{{ 'LLM judge' if llm else 'Decision model' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %} + {% if j.record %}{% set u = j.record.usage or {} %}{% if u.get('seconds') is not none %}Judged in {{ u.seconds|secs }} · {% endif %}{{ j.record.judge.model }}{% if u.get('input_tokens') is not none %} · {{ ((u.input_tokens or 0) + (u.output_tokens or 0))|tokens }} tokens{% endif %}{% if u.get('cost_usd') is not none %} · {{ u.cost_usd|usd }}{% endif %}{% endif %}
{% if j.record and (j.record.refusals or j.earlier or j.record.truncated) %}{% set note = namespace(sep='') %}

{%- if j.record.refusals %}{{ j.record.refusals|length }} refused{% set note.sep = ' · ' %}{% endif -%} diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index bd717689..74ffb859 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -132,7 +132,7 @@ def test_the_record_shape_from_the_api_answer(self): self.assertEqual(rec["answers"]["task_completed"], {"noul": 0.93}) self.assertEqual(rec["answers"]["failure_cause"]["choice"], "rate_limit_or_timeout"); self.assertEqual(rec["answers"]["failure_cause"]["probabilities"]["rate_limit_or_timeout"], 0.81) self.assertEqual(rec["answers"]["transcript_quality"]["score"], 1.7); self.assertEqual(rec["answers"]["transcript_quality"]["legend"][2], "direct, correct, well explained") - self.assertNotIn("not_a_question", rec["answers"]); self.assertEqual(rec["usage"], {"input_tokens": 4821, "output_tokens": 96}) + self.assertNotIn("not_a_question", rec["answers"]); self.assertGreaterEqual(rec["usage"].pop("seconds"), 0); self.assertEqual(rec["usage"], {"input_tokens": 4821, "output_tokens": 96}) # the call timed, its wait included url, body, key = seen[0] self.assertEqual(url, client.URL); self.assertEqual(key, KEY); self.assertEqual(body["model"], "jev-latest"); self.assertEqual(body["state"], "the state") self.assertEqual(sorted(body["questions"]), ["failure_cause", "task_completed", "transcript_quality"]) diff --git a/app/tests/test_judge_openrouter.py b/app/tests/test_judge_openrouter.py index a66ba3d2..96de8b88 100644 --- a/app/tests/test_judge_openrouter.py +++ b/app/tests/test_judge_openrouter.py @@ -95,6 +95,7 @@ def test_one_call_asks_every_question_and_reads_each_answer_by_its_type(self): self.assertEqual(rec["answers"]["failure_cause"]["probabilities"], {"agent_error": 0.75, "environment_error": 0.25}) # normalised self.assertIsNone(rec["answers"]["failure_cause"]["confidence"]) # stated, not calibrated self.assertEqual(rec["answers"]["transcript_quality"]["legend"], Q.QUESTIONS["transcript_quality"]["criteria"]) + self.assertGreaterEqual(rec["usage"].pop("seconds"), 0) self.assertEqual(rec["usage"], {"input_tokens": 1200, "output_tokens": 80, "cost_usd": 0.0012}) self.assertEqual(rec["states"]["trial"]["input_tokens"], 1200) self.assertNotIn("sk-or-1", json.dumps(rec)) diff --git a/app/ui_tests/test_trial_head.py b/app/ui_tests/test_trial_head.py index f522471c..d1bdc793 100644 --- a/app/ui_tests/test_trial_head.py +++ b/app/ui_tests/test_trial_head.py @@ -161,6 +161,25 @@ def test_a_refusal_and_a_truncated_state_are_two_notes_and_a_clean_judgement_has self.assertEqual(self.notes("failed__x"), "1 refused · state truncated (trial)") self.assertIsNone(self.notes("plain__x")) + def test_the_last_call_says_its_time_model_tokens_and_cost_beside_the_button(self): + """After a judgement, beside the button: how long the calls took, the model, the tokens and + the cost, each only when the record has it.""" + rec = record("environment_error", 0.9) + store.save(self.ws.dir, "r-agent", "failed__x", dict(rec, usage={"input_tokens": 1200, "output_tokens": 34, + "seconds": 12.4, "cost_usd": 0.0031})) + store.save(self.ws.dir, "r-agent", "plain__x", rec) # an older record: no time, no cost + + def said(trial): + response = self.client.get(f"/trial/r-agent/{trial}") + self.addCleanup(response.close) + panel = response.get_data(as_text=True).split('id="judgement"', 1)[1] + row = re.search(r'

(.*?)
', panel, re.S).group(1) + self.assertIn("Judge again", row) if "(.*?)\s*$', row.strip(), re.S) + return text_of(found.group(1)) if found else None + self.assertEqual(said("failed__x"), "Judged in 12.40s · jev-latest · 1.2k tokens · $0.0031") + self.assertEqual(said("plain__x"), "jev-latest · 2 tokens") + class DurationTwins(unittest.TestCase): """A duration reads the same wherever it is drawn: the ``secs`` filter, a diff --git a/docs/settings/decision.md b/docs/settings/decision.md index 9c0098d2..6948d0b7 100644 --- a/docs/settings/decision.md +++ b/docs/settings/decision.md @@ -23,6 +23,7 @@ Decision asks a provider that lists a decision protocol | --- | --- | | A question the model refuses | `refusals` in the record, never an answer; the trial page counts them | | The provider's own cost (`usage.cost`) | `usage.cost_usd` | +| The calls' time | `usage.seconds`; the judge tab shows it beside the button | | HTTP 402 | Said as such: the key's credit is spent, or the model is not free | ## Configure