Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion app/nullius/judge/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Calla
body = {"model": model, "input": state, "questions": openai_questions(asked)} if openai else {"state": state, "model": model, "questions": asked}
post = post or http_post
status, data = 0, {}
started = time.monotonic()
for attempt in range(retries + 1):
status, data = post(url, body, key)
if status in (429, 503, 529) and attempt < retries:
Expand All @@ -459,6 +460,7 @@ def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Calla
rec = {"questions": Q.VERSION, "judge": {"provider": provider, "model": str(data.get("model") or model)},
"asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated),
"answers": answers, "usage": {"input_tokens": usage.get("input_tokens"), "output_tokens": usage.get("output_tokens"),
"seconds": round(time.monotonic() - started, 2),
**({"cost_usd": float(usage["cost"])} if _number(usage.get("cost")) is not None else {})}}
except (AttributeError, KeyError, TypeError, ValueError) as e: # a reply of another shape: said, never a crash of the batch
raise JudgeError(502, f"the judge's reply could not be read: {e.__class__.__name__}") from e
Expand Down Expand Up @@ -525,6 +527,7 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio
"response_format": {"type": "json_object"}, **({"usage": {"include": True}} if judge.provider == "openrouter" else {})}
post = post or http_post
status, data = 0, {}
started = time.monotonic()
for attempt in range(retries + 1):
status, data = post(judge.url, body, judge.key, auth=providers.auth("messages", judge.key)) if messages else post(judge.url, body, judge.key)
if status in (429, 502, 503, 529) and attempt < retries:
Expand Down Expand Up @@ -567,6 +570,7 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio
rec = {"questions": Q.VERSION, "judge": {"provider": judge.provider, "model": str(data.get("model") or judge.model)},
"asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated),
"answers": answers, "usage": {"input_tokens": usage.get(tokens_in), "output_tokens": usage.get(tokens_out),
"seconds": round(time.monotonic() - started, 2),
**({"cost_usd": usage["cost"]} if isinstance(usage.get("cost"), (int, float)) else {})}}
if kind is not None:
rec["states"] = {kind: {"chars": len(state), "truncated": bool(truncated), "input_tokens": usage.get(tokens_in),
Expand All @@ -593,7 +597,7 @@ def merge(records) -> Optional[dict]:
if rec.get("refusals"):
merged["refusals"] = {**(merged.get("refusals") or {}), **rec["refusals"]}
merged["asked_at"] = rec["asked_at"]; merged["judge"] = rec["judge"]
for k in ("input_tokens", "output_tokens", "cost_usd"):
for k in ("input_tokens", "output_tokens", "cost_usd", "seconds"): # the calls run one after another: their times add up
if rec["usage"].get(k) is not None:
merged["usage"][k] = (merged["usage"].get(k) or 0) + rec["usage"][k]
return merged
4 changes: 4 additions & 0 deletions app/static/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -1818,3 +1818,7 @@ select.ui-select-native { display: none !important; }
.tree-kids { box-sizing: border-box; }
@media (prefers-reduced-motion: no-preference) { .shell { transition: grid-template-columns .24s ease; } .tree-caret .ic { transition: transform .24s ease; } }
@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .la-fold-btn::before, :root .disclosure-chevron::before, .tree-caret .ic, .side { transition: none; } }

/* the judge's button under its badge, what its last call took beside it */
.judge-actions { display: flex; align-items: center; gap: .75rem; flex-wrap: wrap; margin: .6rem 0 1rem; }
.judge-actions .header-form { margin: 0; }
3 changes: 3 additions & 0 deletions app/templates/traces/_judgement.html
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@
<div>
<span class="badge" title="{% if j.record %}{{ j.record.judge.model }}{% if j.record.judge.provider not in ('typesafe.ai', 'typesafe') %} via {{ j.record.judge.provider|provider }}{% endif %} · {{ j.record.questions }} · asked {{ j.record.asked_at[:16]|replace('T', ' ') }}Z{% if j.record.usage and j.record.usage.input_tokens %} · {{ j.record.usage.input_tokens }} input tokens{% endif %}{% elif j.keyed %}Asks {{ j.judge.label }}{% else %}A judge model's answers, beside the recorded reward and outcome, which they never change{% endif %}">experimental</span>
</div>
</div>
<div class="judge-actions">
{% if j.keyed %}
<form method="post" action="{{ url_for('judge_trial_route', job_id=job.job_id, trial_name=trial.trial_name) }}" class="header-form"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="lane" value="{{ j.lane }}"><button class="btn sm {{ 'ghost' if j.judged }}" type="submit" title="One click, one judge call per state the questions read; the answers replace the ones shown">{{ 'Judge again' if j.judged else 'Judge' }}</button></form>
{% else %}<span class="small faint">{{ 'LLM judge' if llm else 'Decision model' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — <a href="{{ url_for('settings_page') }}#judge">set it in Settings</a></span><button class="btn sm ghost" type="button" disabled title="Not configured: Settings, Judge">Judge</button>{% endif %}
{% if j.record %}{% set u = j.record.usage or {} %}<span class="small muted judge-call">{% if u.get('seconds') is not none %}Judged in {{ u.seconds|secs }} · {% endif %}<span class="mono">{{ j.record.judge.model }}</span>{% if u.get('input_tokens') is not none %} · {{ ((u.input_tokens or 0) + (u.output_tokens or 0))|tokens }} tokens{% endif %}{% if u.get('cost_usd') is not none %} · {{ u.cost_usd|usd }}{% endif %}</span>{% endif %}
</div>
{% if j.record and (j.record.refusals or j.earlier or j.record.truncated) %}{% set note = namespace(sep='') %}<p class="small muted judgement-provenance">
{%- if j.record.refusals %}<span title="{% for q, why in j.record.refusals.items() %}{{ q }}: {{ why }}{{ '; ' if not loop.last }}{% endfor %}">{{ j.record.refusals|length }} refused</span>{% set note.sep = ' · ' %}{% endif -%}
Expand Down
2 changes: 1 addition & 1 deletion app/tests/test_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def test_the_record_shape_from_the_api_answer(self):
self.assertEqual(rec["answers"]["task_completed"], {"noul": 0.93})
self.assertEqual(rec["answers"]["failure_cause"]["choice"], "rate_limit_or_timeout"); self.assertEqual(rec["answers"]["failure_cause"]["probabilities"]["rate_limit_or_timeout"], 0.81)
self.assertEqual(rec["answers"]["transcript_quality"]["score"], 1.7); self.assertEqual(rec["answers"]["transcript_quality"]["legend"][2], "direct, correct, well explained")
self.assertNotIn("not_a_question", rec["answers"]); self.assertEqual(rec["usage"], {"input_tokens": 4821, "output_tokens": 96})
self.assertNotIn("not_a_question", rec["answers"]); self.assertGreaterEqual(rec["usage"].pop("seconds"), 0); self.assertEqual(rec["usage"], {"input_tokens": 4821, "output_tokens": 96}) # the call timed, its wait included
url, body, key = seen[0]
self.assertEqual(url, client.URL); self.assertEqual(key, KEY); self.assertEqual(body["model"], "jev-latest"); self.assertEqual(body["state"], "the state")
self.assertEqual(sorted(body["questions"]), ["failure_cause", "task_completed", "transcript_quality"])
Expand Down
1 change: 1 addition & 0 deletions app/tests/test_judge_openrouter.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ def test_one_call_asks_every_question_and_reads_each_answer_by_its_type(self):
self.assertEqual(rec["answers"]["failure_cause"]["probabilities"], {"agent_error": 0.75, "environment_error": 0.25}) # normalised
self.assertIsNone(rec["answers"]["failure_cause"]["confidence"]) # stated, not calibrated
self.assertEqual(rec["answers"]["transcript_quality"]["legend"], Q.QUESTIONS["transcript_quality"]["criteria"])
self.assertGreaterEqual(rec["usage"].pop("seconds"), 0)
self.assertEqual(rec["usage"], {"input_tokens": 1200, "output_tokens": 80, "cost_usd": 0.0012})
self.assertEqual(rec["states"]["trial"]["input_tokens"], 1200)
self.assertNotIn("sk-or-1", json.dumps(rec))
Expand Down
19 changes: 19 additions & 0 deletions app/ui_tests/test_trial_head.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,25 @@ def test_a_refusal_and_a_truncated_state_are_two_notes_and_a_clean_judgement_has
self.assertEqual(self.notes("failed__x"), "1 refused · state truncated (trial)")
self.assertIsNone(self.notes("plain__x"))

def test_the_last_call_says_its_time_model_tokens_and_cost_beside_the_button(self):
"""After a judgement, beside the button: how long the calls took, the model, the tokens and
the cost, each only when the record has it."""
rec = record("environment_error", 0.9)
store.save(self.ws.dir, "r-agent", "failed__x", dict(rec, usage={"input_tokens": 1200, "output_tokens": 34,
"seconds": 12.4, "cost_usd": 0.0031}))
store.save(self.ws.dir, "r-agent", "plain__x", rec) # an older record: no time, no cost

def said(trial):
response = self.client.get(f"/trial/r-agent/{trial}")
self.addCleanup(response.close)
panel = response.get_data(as_text=True).split('id="judgement"', 1)[1]
row = re.search(r'<div class="judge-actions">(.*?)</div>', panel, re.S).group(1)
self.assertIn("Judge again</button>", row) if "<form" in row else None
found = re.search(r'<span class="small muted judge-call">(.*?)</span>\s*$', row.strip(), re.S)
return text_of(found.group(1)) if found else None
self.assertEqual(said("failed__x"), "Judged in 12.40s · jev-latest · 1.2k tokens · $0.0031")
self.assertEqual(said("plain__x"), "jev-latest · 2 tokens")


class DurationTwins(unittest.TestCase):
"""A duration reads the same wherever it is drawn: the ``secs`` filter, a
Expand Down
1 change: 1 addition & 0 deletions docs/settings/decision.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ Decision asks a provider that lists a decision protocol
| --- | --- |
| A question the model refuses | `refusals` in the record, never an answer; the trial page counts them |
| The provider's own cost (`usage.cost`) | `usage.cost_usd` |
| The calls' time | `usage.seconds`; the judge tab shows it beside the button |
| HTTP 402 | Said as such: the key's credit is spent, or the model is not free |

## Configure
Expand Down
Loading