From 7d09e708a6a76fafe7c3703a2072b2d4b425b41c Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:34:23 +0100 Subject: [PATCH 01/21] refactor(web): share the table toolkit's word matching The run lists Compare adds search as the toolkit's tables do: every word, in any order and case. NulliusTable.matches is that test, and the toolbar's search now calls it. --- app/static/tables.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/static/tables.js b/app/static/tables.js index f5e72df6..806105f1 100644 --- a/app/static/tables.js +++ b/app/static/tables.js @@ -2,6 +2,12 @@ into (the macros of templates/_tables.html; their styles in style.css, "the table toolkit"), and the card toolkit's (templates/_cards.html). */ window.NulliusTable = { + /* Whether `text` holds every word of `query`, in any order and case. */ + matches(text, query) { + const t = (text ?? '').toLowerCase(); + return query.toLowerCase().split(/\s+/).every(w => t.includes(w)); + }, + /* A header with data-type sorts its column on a click or Enter; num sorts descending first. A cell's data-sort wins over its text. A header that carries data-sorted at load is the sort the rows came in. The sorted @@ -152,10 +158,7 @@ window.NulliusTable = { q.addEventListener('input', changed); q.addEventListener('keydown', e => { if (e.key === 'Escape' && q.value) { e.preventDefault(); q.value = ''; changed(); } }); } - const found = row => { - const text = (row.dataset.search ?? '').toLowerCase(); - return !q || q.value.toLowerCase().split(/\s+/).every(w => text.includes(w)); - }; + const found = row => !q || this.matches(row.dataset.search, q.value); const menus = [...(fmenu ? [[fbtn, fmenu, () => 1]] : []), ...[...tools.querySelectorAll('[data-dropdown]')].map(b => [b, b.parentNode.querySelector('.menu'), set => set.size])]; const mark = () => menus.forEach(([btn, menu, n]) => { From 63a879914a3c809dbd97b143c098f631d509cb3a Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:34:23 +0100 Subject: [PATCH 02/21] test(compare): pin every row and section the Compare page shows The summary rows, conditional ones included, the detail rows, the cohort switch, the cohort note, the judgement and both sections, so a redesign of the columns cannot lose one. --- app/ui_tests/test_compare_kept.py | 79 +++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 app/ui_tests/test_compare_kept.py diff --git a/app/ui_tests/test_compare_kept.py b/app/ui_tests/test_compare_kept.py new file mode 100644 index 00000000..10f76d10 --- /dev/null +++ b/app/ui_tests/test_compare_kept.py @@ -0,0 +1,79 @@ +"""Every row and section the Compare page shows, pinned by name: a redesign of its columns keeps them all.""" + +from contextlib import ExitStack +from dataclasses import replace +import re +import unittest +from unittest.mock import patch + +from tests.test_comparison import job, trial +from .test_routes import web +from nullius import workspace + +# whole runs: every summary row a scored run with categories, a runner and tokens shows +WHOLE = ["Agent", "Model", "Dataset", "Runner", "Outcome mix", "Trials", "Mean score", "Category 1", "Category 2", + "Solved", "Raw rate", "Adjusted rate", "Infra errors", "Broke tests", "Prompt size", "Fresh prompt", "Cached", + "Cached ratio", "Output tokens", "Cost (estimated)", "Run wall time", "Rerun"] +# matched tasks: no run wall time, which no subset of a run has +MATCHED = [label for label in WHOLE if label != "Run wall time"] +# a task's detail row +DETAIL = ["Outcome", "Score", "Fail → Pass", "Pass → Pass", "Agent time", "Env / Verify", "Steps", "Tool calls", "Failed calls", "Tool mix", "First edit", + "Edit → test cycles", "Redundant reads", "Retried failed calls", "Exploration per edit", "Prompt size", + "Fresh prompt", "Cached", "Cached ratio", "Output tokens", "Cost (estimated)", "Files edited", "Broke"] +SECTIONS = ["Side by side", "Task by task"] + + +def labels(body): + return [re.sub(r"<[^>]+>", "", m).strip() for m in re.findall(r']*>(.*?)', body, re.S)] + + +class KeptRows(unittest.TestCase): + def setUp(self): + self.stack = ExitStack() + self.addCleanup(self.stack.close) + ws = {name: workspace.Workspace(name, name.title(), [], "", "", "/unused", []) for name in ("active", "foreign")} + + def scored(task, groups, **kw): + n = sum(c for _, c in groups.values()) + return trial(task, outcome="scored", reward=None, score=sum(s for s, _ in groups.values()) / n, n_parts=n, + part_groups={g: list(v) for g, v in groups.items()}, parts_by="category", p2p_fail=1, + p2p_failures=["test_kept"], **kw) + left = replace(job([scored("conv-1", {"1": (3.0, 4), "2": (1.0, 4)}), scored("conv-2", {"1": (1.0, 4)})], + dataset="locomo10"), runner="upstream", harbor_command="harbor run") + right = replace(job([scored("conv-1", {"1": (2.0, 4), "2": (3.0, 4)}, trial_name="b")], dataset="locomo10"), + runner="upstream", agent_name="agent:B") + jobs = {"active": [left], "foreign": [right]} + self.stack.enter_context(patch.dict(web.shared.STATE, workspace=ws["active"], roots=[], job_roots=[], label="Active")) + self.stack.enter_context(patch.dict(web.app.config, TESTING=True)) + self.stack.enter_context(patch.object(workspace, "list_all", return_value=list(ws.values()))) + self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) + self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: ws.get(name))) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots: jobs[w.name])) + self.client = web.app.test_client() + + def get(self, path="/compare", **query): + response = self.client.get(path, query_string={"pick": ["active:run", "foreign:run"], **query}) + self.addCleanup(response.close) + self.assertEqual(response.status_code, 200) + return response.get_data(as_text=True) + + def test_every_row_and_section_is_kept(self): + whole, matched = self.get(cohort="whole"), self.get(cohort="matched") + self.assertEqual(labels(whole), WHOLE) + self.assertEqual(labels(matched), MATCHED) + for body in (whole, matched): + for section in SECTIONS: + self.assertIn(f"

{section}

", body) + # the cohort switch: its two links, their labels and cohort= values + switch = re.search(r'', body, re.S).group(1) + self.assertEqual(re.findall(r'cohort=(\w+)"[^>]*>([^<]+)', switch), [("whole", "Whole runs"), ("matched", "Matched tasks")]) + self.assertIn('class="cohort-note"', body) + self.assertIn("Open trials side by side", body) + self.assertIn('data-detail-url="/compare/detail/0?', body) + self.assertIn('id="judgement"', matched) + self.assertIn("Better of two", matched) + self.assertEqual(labels(self.get("/compare/detail/0", cohort="matched")), DETAIL) + + +if __name__ == "__main__": + unittest.main() From 57890d3fabcd9b818536b7558a09e3f681d4332c Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:47:35 +0100 Subject: [PATCH 03/21] feat(compare): compare runs in columns, each a workspace and a run The comparison shows from one run on, a column a run, the first the baseline. Each column header picks its workspace and its run: the run list is searchable, newest launch first, the baseline's dataset first, checks marked, a sharded run once. A workspace picked lists its runs in place from /compare/runs; a run picked opens the comparison with it. Add a run, remove and reorder are links, so the ordered pick= values stay the comparison and its cohort rides along. An empty column is pick=:. Under each summary number of columns 2..N, its difference from the baseline in the row's unit, coloured by the row's direction. The best is still marked; one column ranks nothing. A shard's id stands for its whole run: Compare folds a participant's shards as the Runs page does (runs.offered over _grouped and _fold), so the Runs page's job= links keep working. The task table, its detail rows and the per-task page follow the columns. /compare/pick redirects to /compare. --- app/nullius/compare/comparison.py | 5 +- app/static/compare/columns.js | 116 ++++++++++ app/static/compare/comparison.css | 56 +++-- app/static/style.css | 11 +- app/templates/base.html | 2 +- app/templates/compare/_detail.html | 29 +-- app/templates/compare/compare.html | 151 ++++++++----- app/templates/compare/compare_picker.html | 51 ----- app/ui_tests/test_compare_columns.py | 111 ++++++++++ app/ui_tests/test_comparison_routes.py | 162 ++++++++++++-- app/ui_tests/test_every_route.py | 8 +- app/ui_tests/test_launch_shards.py | 24 +++ app/web/compare.py | 249 ++++++++++++++-------- app/web/runs.py | 13 ++ 14 files changed, 737 insertions(+), 251 deletions(-) create mode 100644 app/static/compare/columns.js delete mode 100644 app/templates/compare/compare_picker.html create mode 100644 app/ui_tests/test_compare_columns.py diff --git a/app/nullius/compare/comparison.py b/app/nullius/compare/comparison.py index 0a0b69f8..604fda73 100644 --- a/app/nullius/compare/comparison.py +++ b/app/nullius/compare/comparison.py @@ -24,9 +24,10 @@ def value(self) -> str: return f"{self.workspace}:{self.job_id}" @classmethod - def parse(cls, value: str) -> "RunRef": + def parse(cls, value: str, empty: bool = False) -> "RunRef": + """``empty``: also a column with no run yet, ``:``.""" name, separator, job_id = value.partition(":") - if not separator or not job_id: + if not separator or not (job_id or empty): raise ValueError("A run selection must include its workspace and run id.") # Empty means the active source, for legacy selections without a saved # workspace. Named workspaces use the same slug alphabet as workspace.py. diff --git a/app/static/compare/columns.js b/app/static/compare/columns.js new file mode 100644 index 00000000..059578ad --- /dev/null +++ b/app/static/compare/columns.js @@ -0,0 +1,116 @@ +/* Compare's columns (templates/compare/compare.html): each header's workspace and run. The page's URL is the + comparison, its ordered pick= values: a run picked opens it with that run in its column; a workspace picked + lists its runs in place (/compare/runs). The run list searches as the table toolkit does. */ +window.NulliusCompare = { + /* This page's URL with column `k` holding `value` (`:`, `:` for none). */ + url(values, k, value) { + const u = new URL(location.href); + u.searchParams.delete('pick'); + u.searchParams.delete('job'); + values.forEach((v, i) => u.searchParams.append('pick', i === k ? value : v)); + return u.pathname + u.search; + }, + + /* `node` showing the run `r`: agent · model, a check or its shards marked, the date at the right, where it + ran from below; its id in the tooltip. */ + entry(node, r) { + const span = (cls, text) => { const s = document.createElement('span'); s.setAttribute('class', cls); s.textContent = text; return s; }; + const line = span('run-line', [r.agent, r.model].filter(Boolean).join(' · ')); + if (r.check) line.appendChild(span('chip-control', 'check')); + if (r.shards > 1) line.appendChild(span('run-shards', `${r.shards} shards`)); + node.textContent = ''; + [line, span('run-date', r.date), span('run-where', r.where)].forEach(s => node.appendChild(s)); + node.setAttribute('title', r.id); + }, + + /* The listbox `list` of `runs`, the `same` first (the baseline's dataset) and a labelled line after them, a + line between launches. `n` names the options. */ + fill(list, runs, same, n) { + const line = text => { + const li = document.createElement('li'); + li.setAttribute('role', 'separator'); + li.setAttribute('class', text ? 'run-sep labelled' : 'run-sep'); + li.textContent = text; + return li; + }; + list.textContent = ''; + runs.forEach((r, i) => { + if (i && i === same) list.appendChild(line('Other datasets')); + else if (i && r.launch !== runs[i - 1].launch) list.appendChild(line('')); + const li = document.createElement('li'); + li.setAttribute('role', 'option'); + li.setAttribute('id', `${n}-${i}`); + li.setAttribute('class', 'run-entry'); + li.dataset.value = r.value; + li.dataset.search = [r.agent, r.model, r.where, r.date, r.id, r.check ? 'check' : '', r.shards > 1 ? `${r.shards} shards` : ''].join(' '); + + this.entry(li, r); + list.appendChild(li); + }); + }, + + /* One column's header `th`: its workspace select, its run button and the searchable list it opens. Returns + open(), which opens the list. */ + column(th, cfg) { + const k = +th.dataset.column, ws = th.querySelector('[data-ws]'), btn = th.querySelector('[data-run-btn]'); + const menu = th.querySelector('[data-run-menu]'), q = menu.querySelector('input'), list = menu.querySelector('[role="listbox"]'); + const none = menu.querySelector('[data-none]'); + let at = -1; + const shown = () => [...list.querySelectorAll('[role="option"]')].filter(o => !o.hidden); + const mark = i => { + const os = shown(); + at = Math.max(Math.min(i, os.length - 1), os.length ? 0 : -1); + os.forEach((o, j) => o.setAttribute('aria-selected', String(j === at))); + if (at < 0) q.removeAttribute('aria-activedescendant'); + else { q.setAttribute('aria-activedescendant', os[at].id); os[at].scrollIntoView?.({block: 'nearest'}); } + }; + const search = () => { + list.querySelectorAll('[role="option"]').forEach(o => { o.hidden = !NulliusTable.matches(o.dataset.search, q.value); }); + list.querySelectorAll('[role="separator"]').forEach(s => { s.hidden = Boolean(q.value.trim()); }); + none.hidden = shown().length > 0; + mark(0); + }; + const show = data => { this.fill(list, data.runs, data.same, list.id); search(); }; + const pick = o => { location.href = this.url(cfg.values, k, o.dataset.value); }; + q.addEventListener('input', search); + q.addEventListener('keydown', e => { + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { e.preventDefault(); mark(at + (e.key === 'ArrowDown' ? 1 : -1)); } + else if (e.key === 'Enter' && shown()[at]) { e.preventDefault(); pick(shown()[at]); } + }); + list.addEventListener('click', e => { const o = e.target.closest('[role="option"]'); if (o) pick(o); }); + // fixed under its button (the table's scroll box would clip it), placed before it opens and takes the focus, + // so the focus never scrolls the table + const place = () => { + const b = btn.getBoundingClientRect(), w = Math.min(Math.max(b.width, 320), innerWidth - 16); + Object.assign(menu.style, {top: `${b.bottom + 4}px`, left: `${Math.max(8, Math.min(b.left, innerWidth - w - 8))}px`, width: `${w}px`}); + }; + btn.addEventListener('click', place); + NulliusTable.menu(btn, menu); + + addEventListener('resize', place); + document.addEventListener('scroll', e => { if (!menu.contains(e.target)) place(); }, true); + + ws.addEventListener('change', () => { + nulliusFetch(cfg.runs + '?' + new URLSearchParams({workspace: ws.value, dataset: cfg.dataset})) + .then(r => r.ok ? r.json() : Promise.reject(r.status)) + .then(data => { + show(data); + // an empty column keeps its workspace in the URL; one with a run keeps it until another is picked + if (!th.dataset.run) { cfg.values[k] = ws.value + ':'; history.replaceState(null, '', this.url(cfg.values, k, cfg.values[k])); } + if (menu.hidden) btn.click(); + }); + }); + show(cfg.lists[ws.value] || {runs: [], same: 0}); + const mine = (cfg.lists[ws.value]?.runs || []).find(r => r.value === cfg.values[k]); + if (mine) this.entry(btn.querySelector('[data-run-label]'), mine); + return () => btn.click(); + }, + + /* Every column of the page; the first without a run opens its list. */ + wire(cfg) { + const heads = [...document.querySelectorAll('[data-column]')]; + const opens = heads.map(th => this.column(th, cfg)); + const empty = heads.findIndex(th => !th.dataset.run); + if (empty >= 0) opens[empty](); + }, +}; diff --git a/app/static/compare/comparison.css b/app/static/compare/comparison.css index a9e8a52c..661b4bfa 100644 --- a/app/static/compare/comparison.css +++ b/app/static/compare/comparison.css @@ -12,20 +12,44 @@ .compare-excluded { color: var(--text-faint); font-size: .73rem; margin-top: .35rem; } .compare-attempts { display: block; color: var(--text-faint); font-size: .72rem; margin-top: .4rem; } .compare-task-dataset { display: block; color: var(--text-faint); font: .7rem var(--sans); margin: .3rem 0 0 1.5rem; } -.compare-picker { max-width: 1100px; } -.compare-pick-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1.4rem; } -.compare-pick-side { min-width: 0; display: flex; flex-direction: column; gap: .6rem; } -.compare-pick-side > label { color: var(--text-dim); font-size: .82rem; } -.compare-pick-side select { width: 100%; min-width: 0; padding: .75rem .85rem; border: 1px solid var(--border); border-radius: 7px; font: .88rem var(--sans); color: var(--text); background: var(--surface); } -.compare-pick-side option { color: var(--text); background: var(--surface); } -.compare-pick-caption { color: var(--text-faint); font-size: .8rem; margin: .2rem 0 0; } -.compare-pick-actions { display: flex; align-items: center; gap: 1rem; margin-top: 1.4rem; flex-wrap: wrap; } -.compare-pick-actions > label { display: inline-flex; gap: .45rem; align-items: center; color: var(--text-dim); font-size: .85rem; } -.compare-pick-step { color: var(--text-faint); font: .7rem var(--mono); letter-spacing: .08em; text-transform: uppercase; margin: 0 0 1.4rem; } @media (max-width: 640px) { - .compare-pick-grid { grid-template-columns: 1fr; gap: 1rem; } .cohort-tools { align-items: flex-start; flex-direction: column; } } +.compare-tools { flex-wrap: wrap; } +.compare-warn { + color: var(--patch_failed_tests); font-size: .86rem; margin: -.6rem 0 1.4rem; } + +/* A column's header: its tools, its workspace and its run; an empty column greyed, with no numbers. */ +.compare-col { vertical-align: top; text-transform: none; letter-spacing: 0; font-weight: 400; } +.col-head { display: flex; flex-direction: column; gap: .4rem; min-width: 0; } +.col-tools { display: flex; align-items: center; gap: .15rem; } +.col-tools .icon-btn { width: 26px; height: 26px; font-size: .9rem; text-decoration: none; } +.compare-baseline { margin-right: auto; color: var(--accent); font-size: .7rem; letter-spacing: .07em; text-transform: uppercase; } +.col-ws, .col-run { width: 100%; min-width: 0; padding: .45rem .6rem; border: 1px solid var(--border); border-radius: 7px; font: .82rem var(--sans); color: var(--text); background: var(--surface); text-align: left; } +.col-run { cursor: pointer; } +.col-run:hover, .col-ws:hover { border-color: var(--text-faint); } +.compare-col.empty .col-ws, .compare-col.empty .col-run { color: var(--text-dim); background: var(--surface-2); border-style: dashed; } +.compare-empty { background: color-mix(in srgb, var(--surface-2) 45%, transparent); } +/* a run as its list and its button show it */ +.run-entry { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: .1rem .6rem; min-width: 0; } +.run-line { overflow-wrap: anywhere; color: var(--text); } +.run-line .chip-control, .run-shards { margin-left: .4rem; } +.run-shards { color: var(--text-faint); font-size: .72rem; white-space: nowrap; } +.run-date { color: var(--text-faint); font-size: .72rem; white-space: nowrap; } +.run-where { grid-column: 1 / -1; color: var(--text-dim); font-size: .72rem; overflow-wrap: anywhere; } +.run-menu { position: fixed; z-index: 30; padding: .5rem; } + +.run-menu .st-search { margin: 0 0 .4rem; } +.run-list { list-style: none; margin: 0; padding: 0; max-height: min(60vh, 26rem); overflow-y: auto; } +.run-list [role="option"] { padding: .45rem .5rem; border-radius: 6px; cursor: pointer; font-size: .82rem; } +.run-list [role="option"]:hover, .run-list [aria-selected="true"] { background: var(--surface-2); } +.run-list [aria-selected="true"] { box-shadow: inset 2px 0 0 var(--accent); } +.run-sep { border-top: 1px solid var(--border-soft); margin: .25rem 0; } +.run-sep.labelled { padding: .4rem .5rem .1rem; color: var(--text-faint); font-size: .7rem; letter-spacing: .07em; text-transform: uppercase; } +/* a summary cell's difference from the baseline, in the row's unit */ +.delta { display: block; font-size: .72rem; font-weight: 400; color: var(--text-faint); } +.delta.ok { color: var(--resolved); } +.delta.bad { color: var(--bad); } /* One column contract for the overview, task rows and expanded metrics. */ .compare-table-wrap { @@ -78,13 +102,15 @@ } @media (max-width: 640px) { .compare-table-wrap { --compare-label-width: 14rem; } + /* the summary keeps room for a run column beside its pinned names */ + .compare-table-wrap .grid.compare:not(.tasks) { --compare-label-width: 8.5rem; min-width: calc(var(--compare-label-width) + var(--cols, 2) * 15rem); } /* Let a whole agent column enter the viewport when scrolling on a phone. */ - .compare-table-wrap .grid.compare th.rowhead, - .compare-table-wrap .grid.inner th.rowhead, - .compare-table-wrap .grid.compare.tasks > thead > tr > th:first-child, - .compare-table-wrap .grid.compare.tasks > tbody > .compare-task-row > td:first-child { position: static; } + .compare-table-wrap .grid.compare.tasks .sticky-col, + .compare-table-wrap .grid.inner .sticky-col { position: static; } } + + /* Side-by-side trial headings: section-title margins must not offset badges. */ .cmp-head .compare-trial-heading { display: flex; diff --git a/app/static/style.css b/app/static/style.css index 5e903da3..09e5397e 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -490,16 +490,9 @@ button.seg:hover { filter: brightness(1.25); } .grid.compare th.rowhead, .grid.inner th.rowhead { color: var(--text-dim); font-weight: 500; font-size: .78rem; text-transform: uppercase; letter-spacing: .07em; width: 11rem; text-align: left; - position: sticky; left: 0; z-index: 2; background: var(--bg); -} -.grid.compare.tasks td:first-child { - position: sticky; left: 0; z-index: 2; background: var(--bg); - box-shadow: 1px 0 0 var(--border-soft); -} -.grid.compare.tasks tbody tr:hover td:first-child { background: var(--surface); } -.grid.compare.tasks thead th:first-child { - position: sticky; left: 0; z-index: 3; background: var(--bg); width: 20rem; } +.grid.compare.tasks thead th:first-child { width: 20rem; } + /* the detail row spans the table, so its inner table must not re-scroll */ .detail-row > td { overflow-x: auto; } .grid.compare tbody tr:hover { background: var(--surface); } diff --git a/app/templates/base.html b/app/templates/base.html index f76cacf0..17bd4a09 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -162,7 +162,7 @@ +{% if jobs %}

Task by task

Tasks whose pass rate moved most are listed first. A blank cell means that run didn't include the task.

@@ -148,15 +181,18 @@

Task by task

{% endif %}
- +
{{ comparison_columns() }} - - {% for j in jobs %} - + {% for c in columns %} + {% set j = jobs[c.i] if c.i is not none else none %} + {% endfor %} {% if judgement %}{% endif %} @@ -167,7 +203,7 @@

Task by task

{% set task_index = loop.index0 %} {% set row = taskrow.trials %} - - {% for t in row %} + {% call(i) each() %}{% set t = row[i] %} - {% endfor %} + {% endcall %} {% if judgement %} - - {% else %}{% endfor %} + {% else %}{% endfor %}
Task + Task + {%- if j is not none %} {{ j.agent_label or j.job_id }} {%- if j.is_control %} control{% endif %} + {%- endif %} Better of two
+
{# One glyph, rotated by CSS on aria-expanded. #}
{% if not comparison.datasets_match or not comparison.datasets_known %}{{ 'Dataset not recorded' if taskrow.dataset.startswith('[unidentified:') else taskrow.dataset|dsname }}{% endif %}
{% if t %}
- + {{ t.outcome_label }} {% if t.score is not none %}{{ '%.2f'|format(t.score) }}{% endif %} @@ -194,16 +230,16 @@

Task by task

Total time{{ t.dur_total|secs }} {% set used_tokens = (t.n_input_tokens or 0) + (t.n_output_tokens or 0) %} - Tokens{{ used_tokens|tokens(jobs[loop.index0].kind) }} + Tokens{{ used_tokens|tokens(jobs[i].kind) }}
- {%- set tried = taskrow.tried[loop.index0] %} + {%- set tried = taskrow.tried[i] %} {% if tried.n > 1 %}{{ attempt_count(tried) }}{% endif %} {% else %} — {% endif %}
{% set jr = judgement.rows.get(taskrow.task) %} {% if jr and jr.earlier %}{{ {'A': jobs[0].agent_label or 'A', 'B': jobs[1].agent_label or 'B', 'same': 'same'}[jr.choice] if jr.choice in ('A', 'B', 'same') else jr.choice }} {{ '%.0f'|format((jr.p or 0) * 100) }}% earlier vocabulary @@ -224,24 +260,29 @@

Task by task

{# detail rows: the same metrics for each side, so differences line up #}
No tasks in this comparison cohort.
No tasks in this comparison cohort.
+{% endif %}