From 60b600d466e21392c955ba3ecf894329dcbf51e5 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 11:41:26 +0100 Subject: [PATCH 1/8] refactor(catalogue): build the benchmarks' listing once, in the backend recipe.benchmarks gives the catalogue's listed benchmarks with their datasets, tasks, machine needs and what they are run with (agents, or a kind of capability). The Benchmarks page draws its cards from it through catalogue.benchmark_cards, with the same output as before, so the builder's Benchmark step can draw the same cards. --- app/nullius/standard/recipe.py | 26 +++++++++++++++++-- app/tests/test_benchmark_listing.py | 24 ++++++++++++++++++ app/web/catalogue.py | 39 ++++++++++++++++++----------- 3 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 app/tests/test_benchmark_listing.py diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index ba289628..5e4ee4ea 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -108,18 +108,40 @@ def _shown(o: dict, v: str, values: dict) -> str: return (o.get("names") or {}).get(v) or v +def _capability_kind(b: dict) -> str: + """The kind of capability a benchmark runs (``memory``), read from the ports it needs; empty for an agent benchmark.""" + kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b.get("family") == "capability" else [] + return kinds[0] if kinds else "" + + +def benchmarks(cat) -> List[dict]: + """The benchmarks the Benchmarks page and the builder's Benchmark step list: each of the catalogue's listed + entries (``Catalog.listed_benchmarks``, whichever runner runs it) as ``b``, with its ``datasets``, its ``tasks`` + and their ``unit`` (``Catalog.tasks_of``), what some of its options need of a machine (``host``, None for nothing), + and what it is run with: ``family`` agent or capability (a benchmark only Harbor runs runs agents) and, for a + capability, its ``kind``.""" + defs = {d["key"]: d for d in definitions.load("benchmark")[0]} + out = [] + for b in cat.listed_benchmarks: + d = defs.get(b.key) or {} + tasks, unit = cat.tasks_of(b) + out.append({"b": b, "datasets": cat.datasets_of(b), "tasks": tasks, "unit": unit, + "host": sorted(d["host"]) if d.get("host") else None, + "family": d.get("family") or "agent", "kind": _capability_kind(d)}) + return out + + def _bench_page(b: dict) -> dict: """A benchmark as a page shows it. ``controls``: the ones it declares, each with who wrote it and where it comes from - what the builder offers beside the agents or capabilities. Each dataset carries the same controls, each with its ``proven`` record on that dataset (None: not proven), and ``proofs``, the units and leak checks the same way (``prove.status``).""" - kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b["family"] == "capability" else [] status = prove.status(b) controls = [{**{k: c[k] for k in ("key", "label", "source", "from", "expect")}, "must": prove.must_score(c["expect"])} for c in (definitions.control(b, w) for w in definitions.CONTROLS) if c] versioned = definitions.version_of(b) is not None - return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": kinds[0] if kinds else "", + return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": _capability_kind(b), "ref": definitions.ref_of(b) if "_version" in b else b["key"], "version": b["_version"] if versioned else "", # its versions, each by the ref a recipe names it with (the bare key: the default) diff --git a/app/tests/test_benchmark_listing.py b/app/tests/test_benchmark_listing.py new file mode 100644 index 00000000..e2e8af2e --- /dev/null +++ b/app/tests/test_benchmark_listing.py @@ -0,0 +1,24 @@ +"""The benchmarks the Benchmarks page and the builder's Benchmark step list (recipe.benchmarks): the catalogue's +listed entries, each with what it is run with.""" +from __future__ import annotations + +import unittest + +from nullius.catalogue import catalog +from nullius.standard import recipe + + +class Listing(unittest.TestCase): + def test_it_is_the_catalogues_listed_benchmarks_with_their_family(self): + cat = catalog.load(catalog.ROOT) + rows = recipe.benchmarks(cat) + self.assertEqual([r["b"].key for r in rows], [b.key for b in cat.listed_benchmarks]) + by = {r["b"].key: r for r in rows} + self.assertEqual((by["locomo"]["family"], by["locomo"]["kind"]), ("capability", "memory")) + self.assertEqual((by["gsm8k"]["family"], by["gsm8k"]["kind"]), ("agent", "")) + self.assertEqual((by["gsm8k"]["tasks"], by["gsm8k"]["unit"]), cat.tasks_of(cat.benchmark("gsm8k"))) + self.assertTrue(all(r["family"] in ("agent", "capability") for r in rows)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/catalogue.py b/app/web/catalogue.py index 4cf4f0e8..f7f735c7 100644 --- a/app/web/catalogue.py +++ b/app/web/catalogue.py @@ -24,29 +24,37 @@ @app.route("/benchmarks") def benchmarks_page(): - # the benchmarks of the standard some of whose options need a machine's setup (``host``): a bubble on their card - setup = {b["key"]: sorted(b["host"]) for b in standard_definitions.load("benchmark")[0] if b.get("host")} cat = shared._catalog_cached() + return render_template("catalogue/benchmarks.html", cat=cat, items=benchmark_cards(cat), choices=benchmark_choices(cat), + prompts=howto.prompts(), catalogue_dir=paths.catalogue_dir()) + + +RUNS_WITH = {"standard": ("Nullius", "the Nullius standard"), "harbor": ("Harbor", "Harbor")} +# what a benchmark is run with (``recipe.benchmarks``' family), as a card and its filter say it +FAMILIES = {"agent": "Agents", "capability": "Capabilities"} + + +def benchmark_cards(cat) -> list: + """The benchmarks as cards (templates/_cards.html), one per row of ``recipe.benchmarks``: what the Benchmarks + page and the builder's Benchmark step both draw.""" with open(MARKS, encoding="utf-8") as fh: sprite = fh.read() sheet = url_for("static", filename="catalogue/marks.svg") - items = [_card(cat, b, setup.get(b.key), sheet if f' dict: + """The values the benchmarks' cards are filtered by: their category, what runs them, their origin, what they run.""" + return {"category": [(c, c) for c in cat.categories], "runs": [(way, label) for way, (label, _) in RUNS_WITH.items()], + "origin": [(o, o.capitalize()) for o in catalog.ORIGINS], "with": list(FAMILIES.items())} -def _card(cat, b, needs, sheet) -> dict: - """A benchmark's card on the list (templates/_cards.html): ``needs`` what some of its options need installed, - ``sheet`` the sprite holding its own mark, none when it has none.""" - ds = cat.datasets_of(b) - tasks, unit = cat.tasks_of(b) - return {"href": url_for("benchmark_page", key=b.key), "shelf": b.origin, "keys": " ".join([b.key, *b.datasets]), +def _card(row, sheet) -> dict: + """A benchmark's card on the list (templates/_cards.html), from a row of ``recipe.benchmarks``: ``sheet`` the + sprite holding its own mark, none when it has none.""" + b, ds, tasks, unit, needs = row["b"], row["datasets"], row["tasks"], row["unit"], row["host"] + return {"key": b.key, "href": url_for("benchmark_page", key=b.key), "shelf": b.origin, "keys": " ".join([b.key, *b.datasets]), "search": " ".join([b.title, b.category, b.key, *(d.title for d in ds), *b.datasets]), "category": b.category, "origin": {"text": b.label, "class": b.origin, "value": b.origin}, "icon": sheet and f"{sheet}#bm-{b.key}", "title": b.title, "domain": b.domain, @@ -56,6 +64,7 @@ def _card(cat, b, needs, sheet) -> dict: "title": f"Some of its options need {', '.join(needs)} installed on the machine it runs on: see its page"}, "tasks": {"text": tasks, "label": unit.capitalize(), "value": tasks.replace(",", "").split("–")[-1]}, "datasets": {"text": len(ds), "label": "Datasets" if len(ds) != 1 else "Dataset"}, + "with": {"text": FAMILIES[row["family"]], "value": row["family"], "title": f"{row['kind'].capitalize()} capabilities" if row["kind"] else ""}, "scoring": b.scoring, "blocker": b.blocker[:1].upper() + b.blocker[1:]} From 8cbfa20cc3b59be6c25c90fadaa7253c7430d2af Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 11:41:27 +0100 Subject: [PATCH 2/8] feat(web): let a pick card be a single choice A pick with one: true is a single choice: choosing its card lets the others go and choosing it again keeps it; in a table its box is a radio. Toggle picks are unchanged. --- app/static/tables.js | 9 ++++++--- app/templates/_cards.html | 13 +++++++------ app/ui_tests/test_cards.py | 39 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/app/static/tables.js b/app/static/tables.js index 586fea11..53ac9231 100644 --- a/app/static/tables.js +++ b/app/static/tables.js @@ -285,7 +285,8 @@ window.NulliusTable = { }, /* The pick toggles under `root`, buttons (data-pick, aria-pressed) or - checkboxes (data-pick): a click, Space or Enter flips one and sends nullius-pick {value, pressed} from it; with + checkboxes (data-pick): a click, Space or Enter flips one and sends nullius-pick {value, pressed} from it. A single + choice, a button with data-pick-one or a radio, is pressed again rather than let go, and lets the others go; with `row`, a click elsewhere in the row (a selector) that holds one, but on a control of its own or at the end of a text selection, clicks it. */ picks(root, row) { @@ -296,8 +297,10 @@ window.NulliusTable = { } const b = e.target.closest('[data-pick]'); if (!b || b.disabled) return; - const pressed = b.type === 'checkbox' ? b.checked : b.getAttribute('aria-pressed') !== 'true'; // a box has ticked itself - if (b.type !== 'checkbox') b.setAttribute('aria-pressed', String(pressed)); + const box = b.type === 'checkbox' || b.type === 'radio', one = b.dataset.pickOne !== undefined; + const pressed = box ? b.checked : one || b.getAttribute('aria-pressed') !== 'true'; // a box has ticked itself + if (one) root.querySelectorAll('[data-pick-one][aria-pressed="true"]').forEach(x => x.setAttribute('aria-pressed', 'false')); + if (!box) b.setAttribute('aria-pressed', String(pressed)); b.dispatchEvent(new CustomEvent('nullius-pick', {bubbles: true, detail: {value: b.dataset.pick, pressed}})); }); }, diff --git a/app/templates/_cards.html b/app/templates/_cards.html index c695d4e9..3b372ab2 100644 --- a/app/templates/_cards.html +++ b/app/templates/_cards.html @@ -8,9 +8,10 @@ An item is a dict of display values by field key: a string, a dict {text, class, title, label, href, value, attrs} or a list of them, empty for none; a filter and Sort by read a value's `value`, else its text; an action's `attrs` are its button's own attributes, its `class` added to the button's, and a `delete` takes the same shape (a quiet - trash button). Plus `href` (its title's link), or `pick` {value, on, disabled}: the card is a toggle, its title a + trash button). Plus `href` (its title's link), or `pick` {value, on, disabled, one}: the card is a toggle, its title a button[data-pick][aria-pressed] that tables.js flips, sending `nullius-pick` {value, pressed} (a disabled one's - reason goes in its `alert`), and an enabled one's corner holds the check that marks it chosen; a title's `title` is the tooltip of its link or button; + reason goes in its `alert`), and an enabled one's corner holds the check that marks it chosen; `one`, a single + choice: choosing it lets the others go, choosing it again keeps it (a radio in a table); a title's `title` is the tooltip of its link or button; `id`, `search` (what the search reads), `keys`, and `unless` {field: other}: the item passes the field's filter whenever the other field's filter narrows and passes it. A title's `after` is markup in the flow of its name, after its last word (a control of its own), and its `class` goes on the title row. @@ -52,8 +53,8 @@ {# The items of a cards collection as a table (the table toolkit's, with its Columns menu): a column per field the table takes that some item fills (a filtered or 'num' one always), in the order of `cols`; the title pinned, its - icon before it and a link to `href`, the others as on the card; picks put a checkbox in a leftmost column of no - name (its header no tick-all: a pick is one at a time), the name then plain. A field `off` starts hidden. #} + icon before it and a link to `href`, the others as on the card; picks put a checkbox (a radio for `one`) in a leftmost + column of no name (its header no tick-all: a pick is one at a time), the name then plain. A field `off` starts hidden. #} {% macro rows(id, cols, items, search, choices, only, empty, noun, sort, default) %} {%- set icon = (cols|selectattr('card', 'equalto', 'icon')|list + [none])[0] %} {%- set icons = icon and items|selectattr(icon.key)|list|length > 0 %} @@ -64,7 +65,7 @@ {% call(r, c) table(id, shown, rs, attrs, search=search, columns=true, choices=choices, only=only, empty=empty, noun=noun, sort=sort, layout='table', default=default) %} {%- set v = r.item.get(c.key) %}{% set by %}{% call(f, x) each(r.item, [c]) %}{{ x.get('value', x.text) }} {% endcall %}{% endset %} {%- set pick = r.item.get('pick') %}{% set t = r.item.get((cols|selectattr('card', 'equalto', 'title')|first).key) %}{% set t = t if t is mapping else {'text': t} %} - {%- if c.key == '_pick' %}{% call cell(c, class='pick-col') %}{% endcall %} + {%- if c.key == '_pick' %}{% call cell(c, class='pick-col') %}{% endcall %} {%- else %}{% call cell(c, sort=by|trim) %}{% if c.card == 'title' %}{% set name = t.text %}{% if icons %}{{ mark(icon, r.item.get(icon.key), name) }}{% endif %} {%- if pick %}{{ name }}{% else %}{{ name }}{% endif %} {%- elif c.style == 'code' %}{{ v }} @@ -91,7 +92,7 @@ {%- set cls = [title.class, v.class]|select|join(' ') %}
{% if icons %}{{ mark(icon, item.get(icon.key), name) }}{% endif -%} {%- set pick = item.get('pick') %} - {%- set link %}{% if pick %} + {%- set link %}{% if pick %} {%- else %}{{ name }}{% endif %}{% endset %} {%- if v.after %}{{ link }} {{ v.after }}{% else %}{{ link }}{% endif %} {%- set corner %}{% if pick and not pick.disabled %}{% endif %}{% call(f, v) each(item, by.check) %}{% endcall %} diff --git a/app/ui_tests/test_cards.py b/app/ui_tests/test_cards.py index e574cb4d..d0c81b57 100644 --- a/app/ui_tests/test_cards.py +++ b/app/ui_tests/test_cards.py @@ -136,6 +136,11 @@ def test_a_pick_card_is_a_toggle_and_a_disabled_one_says_why(self): '
Lacks tools
', html) self.assertNotIn('Beta', self.render(cols, items)) + class Script(unittest.TestCase): def test_search_and_filter_hide_cards_and_their_empty_groups(self): @@ -235,6 +240,40 @@ def test_a_pick_flips_and_says_so_and_a_disabled_one_stays(self): """) self.assertEqual(got, [[["a", True], ["a", False], ["a", True]], "true", "false"]) # its pill too: the card is the toggle + def test_a_single_choice_lets_the_others_go_and_stays_chosen(self): + got = node(r""" +global.CustomEvent = class { constructor(type, o) { this.type = type; this.detail = o.detail; } }; +const picks = [], card = v => el('article', {'data-card': ''}, el('div', {}, el('button', {class: 'card-link', 'data-pick': v, 'data-pick-one': '', 'aria-pressed': 'false'}, v))); +const root = el('div', {id: 'c', 'data-cards': ''}, card('a'), card('b')); +root.querySelectorAll('[data-pick]').forEach(b => { b.dispatchEvent = ev => b.dispatch(ev.type, {detail: ev.detail}); }); +document.appendChild(el('main', {}, el('div', {id: 'c-tools'}), root, el('p', {'data-empty': 'c', hidden: true}), el('span', {'data-live': 'c', 'data-noun': 'cards'}))); +global.getSelection = () => ({isCollapsed: true}); +T.cards(root); +root.addEventListener('nullius-pick', e => picks.push([e.detail.value, e.detail.pressed])); +const a = root.querySelector('[data-pick="a"]'), b = root.querySelector('[data-pick="b"]'), state = () => [a, b].map(x => x.getAttribute('aria-pressed')).join(' '); +const out = []; +a.click(); out.push(state()); b.click(); out.push(state()); b.click(); out.push(state()); +process.stdout.write(JSON.stringify([picks, out])); +""") + self.assertEqual(got, [[["a", True], ["b", True], ["b", True]], ["true false", "false true", "false true"]]) + + def test_a_row_click_chooses_its_radio(self): + got = node(r""" +global.CustomEvent = class { constructor(type, o) { this.type = type; this.detail = o.detail; } }; +global.getSelection = () => ({isCollapsed: true}); +const row = v => el('tr', {}, el('td', {}, el('input', {type: 'radio', name: 'c-pick', 'data-pick': v})), el('td', {}, v)); +const table = el('table', {}, row('a'), row('b')); +table.querySelectorAll('[data-pick]').forEach(b => { b.dispatchEvent = ev => b.dispatch(ev.type, {detail: ev.detail}); b.type = 'radio'; }); +document.appendChild(table); +const picks = []; +T.picks(table, 'tr'); +table.addEventListener('nullius-pick', e => picks.push([e.detail.value, e.detail.pressed])); +const tds = table.querySelectorAll('td'), checked = () => table.querySelectorAll('[data-pick]').filter(b => b.checked).map(b => b.dataset.pick).join(''); +tds[1].dispatch('click', {button: 0}); const first = checked(); tds[3].dispatch('click', {button: 0}); tds[3].dispatch('click', {button: 0}); +process.stdout.write(JSON.stringify([picks, first, checked()])); +""") + self.assertEqual(got, [[["a", True], ["b", True], ["b", True]], "a", "b"]) + def test_a_card_passes_a_filter_its_unless_column_narrows_to(self): got = node(r""" const card = (id, from, runs, unless) => el('article', Object.assign({'data-card': '', id, 'data-f-from': from + '\n', 'data-f-runs': runs + '\n'}, unless ? {'data-unless-from': 'runs'} : {}), id); From cc97198fda210b4651cae5d96535270f26fc98c9 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 11:41:27 +0100 Subject: [PATCH 3/8] feat(builder): draw the Benchmark step with the Benchmarks page's cards, as cards or a table GET /new-run/benchmarks returns the step's grid: the Benchmarks page's cards as single-choice pick cards, or a table with ?view=table. The category pills stay; Run with (agents or capabilities) is a menu of its own beside Filter (Runner), Sort by and the cards/table toggle. The builder loads it once per view, switches the view in place and keeps it per viewer, and presses the recipe's benchmark, which stays in view under any filter, also after a recipe import. The step's own tiles, search and chips are gone. --- app/static/builder/command-builder.css | 36 ++--- app/static/builder/command-builder.js | 137 +++++++------------- app/templates/builder/benchmarks.html | 11 ++ app/ui_tests/test_builder_benchmarks.py | 71 ++++++++++ app/ui_tests/test_builder_where.py | 6 +- app/ui_tests/test_catalogue_routes.py | 4 +- app/ui_tests/test_data_provider_guidance.py | 2 +- app/web/builder.py | 21 +++ docs/builder.md | 2 +- 9 files changed, 168 insertions(+), 122 deletions(-) create mode 100644 app/templates/builder/benchmarks.html create mode 100644 app/ui_tests/test_builder_benchmarks.py diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index b54dcbc7..b294b73a 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -146,19 +146,7 @@ /* Consent to run without a sandbox. */ #nullius-builder .nb-consent { margin-top: 16px; padding: 12px 14px; border: 1px solid var(--nb-line); border-radius: 8px; background: var(--nb-surface); } #nullius-builder .nb-consent > span { display: flex; align-items: center; gap: 9px; color: var(--nb-text); font-size: 13px; } -/* The benchmark and agent steps: a search and filter chips above a grid of tiles, as the Benchmarks page draws its - cards. A tile: its mark and name, its tags, one line on what it is, then its facts; chosen, it takes the accent. */ -#nullius-builder .nb-pick { display: grid; gap: 16px; } -#nullius-builder .nb-pick-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; } -#nullius-builder .nb-pick-find { position: relative; flex: 1 1 240px; max-width: 360px; display: flex; align-items: center; } -#nullius-builder .nb-pick-find svg { position: absolute; left: 11px; width: 15px; height: 15px; color: var(--nb-muted); pointer-events: none; } -#nullius-builder .nb-pick-find input { width: 100%; min-height: 38px; padding: 8px 12px 8px 33px; font: inherit; font-size: 13px; color: var(--nb-text); background: var(--nb-surface); border: 1px solid var(--nb-line); border-radius: 8px; } -#nullius-builder .nb-pick-find input:focus { outline: none; border-color: var(--nb-blue); box-shadow: inset 0 0 0 1px var(--nb-blue); } -#nullius-builder .nb-pick-chips { display: flex; flex-wrap: wrap; gap: 6px; } -#nullius-builder .nb-pick-chips + .nb-pick-chips { padding-left: 14px; border-left: 1px solid var(--nb-line); } -#nullius-builder .nb-pick-chip { font: inherit; font-size: 12px; color: var(--nb-muted); background: transparent; border: 1px solid var(--nb-line); border-radius: 999px; padding: 6px 13px; min-height: 32px; cursor: pointer; } -#nullius-builder .nb-pick-chip:hover { color: var(--nb-text); border-color: var(--nb-muted); } -#nullius-builder .nb-pick-chip[aria-pressed=true] { color: var(--nb-bg); background: var(--nb-blue); border-color: var(--nb-blue); } +/* A choice as a tile: its mark and name, one line on what it is, then its facts; chosen, it takes the accent. */ #nullius-builder .nb-pick-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); gap: 12px; } #nullius-builder .nb-tile { position: relative; display: flex; flex-direction: column; gap: 9px; min-width: 0; text-align: left; padding: 15px 16px; border-radius: 10px; background: var(--nb-surface); border: 1px solid var(--nb-line); color: var(--nb-text); cursor: pointer; transition: border-color .12s ease, background .12s ease; } #nullius-builder .nb-tile[hidden] { display: none; } @@ -172,10 +160,6 @@ #nullius-builder .nb-tile-head strong { flex: 1; min-width: 0; font-size: 14.5px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } #nullius-builder .nb-tile-check { flex: none; width: 20px; height: 20px; border-radius: 50%; display: grid; place-items: center; font-size: 11px; color: var(--nb-bg); } #nullius-builder .nb-tile[aria-pressed=true] .nb-tile-check { background: var(--nb-blue); } -#nullius-builder .nb-tile-tags { display: flex; flex-wrap: wrap; gap: 5px; } -#nullius-builder .nb-tile-tags > span { font-size: 11px; font-weight: 500; letter-spacing: .02em; padding: 1px 7px; border-radius: 4px; color: var(--nb-muted); background: var(--nb-soft); } -#nullius-builder .nb-tile-tags .nb-tile-cat { color: var(--nb-blue); background: var(--nb-tint); } -#nullius-builder .nb-tile-tags .nb-tile-community { color: var(--origin-community, #b8741a); background: color-mix(in srgb, var(--origin-community, #b8741a) 16%, transparent); } #nullius-builder .nb-tile-line { font-size: 12.5px; line-height: 1.5; color: var(--nb-muted); display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } #nullius-builder .nb-tile-note { font-size: 11.5px; color: var(--nb-warn); } #nullius-builder .nb-tile-foot { margin-top: auto; padding-top: 9px; border-top: 1px solid var(--nb-line); display: flex; flex-wrap: wrap; align-items: center; gap: 5px 8px; font-size: 11.5px; color: var(--nb-muted); } @@ -185,8 +169,6 @@ #nullius-builder .nb-tick.ok { color: var(--nb-ok); } #nullius-builder .nb-tick.no { color: var(--nb-warn); } #nullius-builder .nb-tile-unfit { opacity: .5; cursor: not-allowed; background: transparent; } -#nullius-builder .nb-pick-none { color: var(--nb-muted); font-size: 13px; margin: 0; } -@media (max-width: 640px) { #nullius-builder .nb-pick-chips + .nb-pick-chips { padding-left: 0; border-left: 0; } #nullius-builder .nb-pick-find { max-width: none; } #nullius-builder .nb-pick-find input { font-size: 16px; } } /* The Where step's machine setup: each chosen machine, its state for the search's requirements, check and install. */ #nullius-builder .std-hostreq { padding: 16px; border: 1px solid var(--nb-line); border-radius: 8px; font-size: 13px; } #nullius-builder .std-hostreq-row { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } @@ -294,13 +276,13 @@ #nullius-builder .std-isolation small:empty { display: none; } #nullius-builder .std-boat-sizes .nb-tile { min-height: 60px; } #nullius-builder .std-boat-sizes .nb-tile-note { font-size: 13px; line-height: 1.4; } -/* the Agent step's grid: the card toolkit's (catalogue.css), fitted to the form */ +/* the Benchmark and Agent steps' grids: the card toolkit's (catalogue.css), fitted to the form */ #nullius-builder .nb-step { border-radius: 0; margin: 0; } /* catalogue.css has a .nb-step of its own */ -#nullius-builder #nb-agent-grid { margin-bottom: 20px; } -#nullius-builder #nb-agent-grid .bench-grid { grid-template-columns: repeat(auto-fill, minmax(min(100%, 230px), 1fr)); gap: 12px; grid-auto-rows: 1fr; } -#nullius-builder #nb-agent-grid .bench-card { padding: 16px; gap: 10px; } -#nullius-builder #nb-agent-grid .bench-card-title { font-size: 15px; } -#nullius-builder #nb-agent-grid .std-tag { white-space: nowrap; } -#nullius-builder #nb-agent-grid .card-link:focus-visible { outline: none; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) { margin-bottom: 20px; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .bench-grid { grid-template-columns: repeat(auto-fill, minmax(min(100%, 230px), 1fr)); gap: 12px; grid-auto-rows: 1fr; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .bench-card { padding: 16px; gap: 10px; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .bench-card-title { font-size: 15px; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .std-tag { white-space: nowrap; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .card-link:focus-visible { outline: none; } /* its outlined pills (Protocol, Proven) want more air than snug's text beside text; a phone keeps snug's, for the pinned name */ -@media (min-width: 641px) { #nullius-builder #nb-agent-grid .grid.snug :is(th, td):not(.pick-col) { padding-left: .6rem; padding-right: .6rem; } } /* the card's ring (catalogue.css), not a box round the name */ +@media (min-width: 641px) { #nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .grid.snug :is(th, td):not(.pick-col) { padding-left: .6rem; padding-right: .6rem; } } /* the card's ring (catalogue.css), not a box round the name */ diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index d174a40c..9b987d8a 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -215,20 +215,13 @@ }; // the version a standard recipe names (``key@version``), '' for the default const versionOf = rec => C.isStandard(rec) && String(rec.benchmark || '').includes('@') ? String(rec.benchmark).split('@')[1] : ''; - // Every benchmark of both runners in one list; picking one sets the runner that runs it (Nullius when both do). + // Every benchmark of both runners in one list, the Benchmarks page's (placeGrid, /new-run/benchmarks); picking one + // sets the runner that runs it (Nullius when both do). const benchmarkStep = rec => { - const st = C.standard(), sb = C.standardBase(rec), on = C.benchKey(rec), seen = new Set(), tiles = []; - const add = (key, title, fam, kind, summary) => { if (seen.has(key)) return; seen.add(key); const c = catOf(key); - tiles.push({attr: 'bench', key, on: key === on, title: c.title || title, - mark: ``, cat: c.category, origin: c.origin, - fam, famLabel: fam === 'agent' ? 'Agents' : 'Capabilities · ' + kind, tip: c.domain || summary || '', meta: tasksOf(key, key === on ? versionOf(rec) : ''), - find: [c.title || title, key, c.category, c.domain, kind].join(' ')}); }; - st.benchmarks.forEach(x => add(x.key, x.label, x.family, x.kind, x.summary)); - C.benchmarks().forEach(x => add(x.key, x.title, 'agent', '', x.domain)); - tiles.sort((a, b) => a.title.localeCompare(b.title, 'en', {sensitivity: 'base'})); + const sb = C.standardBase(rec); const versions = C.isStandard(rec) && sb && (sb.versions || []).length > 1 ? `` : ''; - return `

Choose a benchmark.

` + picker(tiles, [['agent', 'Agents'], ['capability', 'Capabilities']]) + versions; + return `

Choose a benchmark.

` + versions; }; // Put the recipe on a runner, keeping the one each runner had: the benchmark follows. const toRunner = (target, key) => { @@ -628,94 +621,66 @@ return `

${state ? `Replace the expired ${what}` : `Add ${provider ? 'a ' : ''}${what}`} in ${settings}${required ? '' : ' (some tasks may need it)'}.

`; }).join(''); } - // The benchmark step, both ways (Harbor's catalogue, the standard's benchmarks): a search, filter chips and a grid - // of tiles, as the Benchmarks page draws them. The filter hides tiles in place (applyPick), so typing keeps focus. - const picks = {}, pickOf = () => (picks[step] ||= {q: '', cat: '', fam: ''}); // each step its own search and filters - const catOf = key => (schema.benchmarks || []).find(b => b.key === key) || {}; - // its tasks; `version`: a version of the standard's that counts its datasets apart (the catalogue entry's ``versions``) - const tasksOf = (key, version) => { const total = catOf(key).task_count, own = ((catOf(key).versions || {})[version] || {}).datasets || {}; - if (Number.isInteger(total) && total > 0) return `${total.toLocaleString('en-US')} tasks`; - const t = schema.datasets.filter(d => d.benchmark === key && d.tasks).map(d => (own[d.key] || {}).tasks || d.tasks); - return !t.length ? '' : t.every(n => n === t[0]) && t.length > 1 ? `${t[0]} tasks each` : `${t.reduce((a, n) => a + n, 0)} tasks`; }; - const pickChip = (attr, v, label, on) => ``; - // one tile: {attr, key, on, title, mark, cat, origin, line, meta, foot, find, fam, data (more attributes), tip (its title)} - const tile = t => `'; // a machine to run on: this one or a host, its connection state filled in by Test connection const icon = name => ``; const machineTile = (attr, v, t, sub, on) => tile({attr, key: v, on, mark: icon(v === 'local' ? 'machine' : 'network'), title: t, tip: sub, foot: v !== 'local' ? `` : ''}); - const picker = (tiles, fams, noun = 'benchmark') => { - const pick = pickOf(), cats = [...new Set(tiles.map(t => t.cat).filter(Boolean))].sort(); - return `
` - + `` - + (cats.length > 1 ? `
${pickChip('pick-cat', '', 'All', !pick.cat)}${cats.map(c => pickChip('pick-cat', c, c, pick.cat === c)).join('')}
` : '') - + (fams ? `
${pickChip('pick-fam', '', 'Any', !pick.fam)}${fams.map(([v, l]) => pickChip('pick-fam', v, l, pick.fam === v)).join('')}
` : '') - + `
${tiles.map(tile).join('')}
`; - }; - function applyPick() { - const box = root.querySelector('.nb-pick'), pick = pickOf(); if (!box) return; - const words = pick.q.toLowerCase().split(/\s+/).filter(Boolean); - let shown = 0; - box.querySelectorAll('.nb-tile').forEach(t => { - const on = t.getAttribute('aria-pressed') === 'true'; - const fits = (!pick.cat || t.dataset.cat === pick.cat) && (!pick.fam || t.dataset.fam === pick.fam) && words.every(w => t.dataset.find.includes(w)); - t.hidden = !fits && !on; if (!t.hidden) shown++; - }); - box.querySelectorAll('[data-pick-cat]').forEach(b => b.setAttribute('aria-pressed', b.dataset.pickCat === pick.cat)); - box.querySelectorAll('[data-pick-fam]').forEach(b => b.setAttribute('aria-pressed', b.dataset.pickFam === pick.fam)); - box.querySelector('.nb-pick-none').hidden = shown > 0; - } - // The standard's Agent step grid: drawn by the server for the benchmark (web/builder.py, /new-run/agents) from the - // Agents page's entries, run by the card toolkit (tables.js); kept while the benchmark is the same, so a search or a - // filter survives each render, and its cards pressed as the recipe chooses (nullius-pick changes the recipe). - // Its view, cards or a table, is the viewer's: the toggle switches it in place, never leaving the page. - let agentView = 'cards'; - try { agentView = localStorage.getItem('nb-agent-view') === 'table' ? 'table' : 'cards'; } catch {} - const grid = {ref: '', node: null}; - function placeAgentGrid() { - const slot = $('#nb-agent-grid'), rec = r(); + // The grids drawn by the server and run by the card toolkit (tables.js): the Benchmark step's (/new-run/benchmarks, + // one choice) and the standard's Agent step's for its benchmark (/new-run/agents, from the Agents page's entries). + // Each is kept while what it draws is the same, so a search or a filter survives each render, and its cards are + // pressed as the recipe chooses (nullius-pick changes the recipe). Its view, cards or a table, is the viewer's: + // the toggle switches it in place, never leaving the page. + const grids = [ + {slot: 'nb-bench-grid', store: 'nb-bench-view', url: () => '/new-run/benchmarks?', chosen: rec => [C.benchKey(rec)]}, + {slot: 'nb-agent-grid', store: 'nb-agent-view', url: rec => '/new-run/agents?' + new URLSearchParams({benchmark: rec.benchmark}) + '&', chosen: rec => C.chosenOf(rec)}, + ]; + grids.forEach(g => { g.view = 'cards'; g.ref = ''; g.node = null; try { g.view = localStorage.getItem(g.store) === 'table' ? 'table' : 'cards'; } catch {} }); + const placeGrids = () => grids.forEach(placeGrid); + function placeGrid(g) { + const slot = $('#' + g.slot), rec = r(); if (!slot) return; - if (grid.ref !== rec.benchmark + ' ' + agentView) { grid.ref = rec.benchmark + ' ' + agentView; grid.node = null; loadAgentGrid(grid.ref); } - if (!grid.node) { slot.setAttribute('aria-busy', 'true'); slot.innerHTML = ''; return; } - slot.replaceWith(grid.node); - const chosen = C.chosenOf(rec); - grid.node.querySelectorAll('[data-pick]').forEach(b => { // a card's toggle, or a row's box + const ref = g.url(rec) + (g.view === 'table' ? 'view=table' : ''); + if (g.ref !== ref) { g.ref = ref; g.node = null; loadGrid(g, ref); } + if (!g.node) { slot.setAttribute('aria-busy', 'true'); slot.innerHTML = ''; return; } + slot.replaceWith(g.node); + const chosen = g.chosen(rec); + g.node.querySelectorAll('[data-pick]').forEach(b => { // a card's toggle, or a row's box const on = !b.disabled && chosen.includes(b.dataset.pick); - if (b.type === 'checkbox') b.checked = on; else b.setAttribute('aria-pressed', String(on)); + if (b.type === 'checkbox' || b.type === 'radio') b.checked = on; else b.setAttribute('aria-pressed', String(on)); }); // a choice made elsewhere (a recipe loaded) that the filters hide: the toolkit draws again, and it shows - if (grid.node.dataset.wired && [...grid.node.querySelectorAll('[data-pick]')].some(b => b.matches('[aria-pressed="true"], :checked') && b.closest('[data-card], tr').hidden)) - grid.node.querySelector('[data-search]')?.dispatchEvent(new Event('input')); - if (!grid.node.dataset.wired) { - grid.node.dataset.wired = '1'; - const cards = grid.node.querySelector('[data-cards]'), table = grid.node.querySelector('table[data-table]'); + if (g.node.dataset.wired && [...g.node.querySelectorAll('[data-pick]')].some(b => b.matches('[aria-pressed="true"], :checked') && b.closest('[data-card], tr').hidden)) + g.node.querySelector('[data-search]')?.dispatchEvent(new Event('input')); + if (!g.node.dataset.wired) { + g.node.dataset.wired = '1'; + const cards = g.node.querySelector('[data-cards]'), table = g.node.querySelector('table[data-table]'); if (cards) NulliusTable.cards(cards); if (table) { NulliusTable.sticky(table); NulliusTable.wire(table); } - grid.node.querySelectorAll('.seg-toggle a').forEach(a => a.addEventListener('click', e => { + g.node.querySelectorAll('.seg-toggle a').forEach(a => a.addEventListener('click', e => { e.preventDefault(); - agentView = new URL(a.href).searchParams.get('view') === 'table' ? 'table' : 'cards'; - try { localStorage.setItem('nb-agent-view', agentView); } catch {} - placeAgentGrid(); + g.view = new URL(a.href).searchParams.get('view') === 'table' ? 'table' : 'cards'; + try { localStorage.setItem(g.store, g.view); } catch {} + placeGrid(g); })); } } - async function loadAgentGrid(ref) { - const [benchmark, view] = ref.split(' '); - const res = await nulliusFetch('/new-run/agents?' + new URLSearchParams({benchmark, ...(view === 'table' ? {view} : {})})).catch(() => null); + async function loadGrid(g, ref) { + const res = await nulliusFetch(ref).catch(() => null); const html = res && res.ok ? await res.text().catch(() => null) : null; - if (grid.ref !== ref) return; // another benchmark was chosen while it loaded - if (html === null) { // said once; the next render asks again - grid.ref = ''; - const slot = $('#nb-agent-grid'); + if (g.ref !== ref) return; // another benchmark or view was chosen while it loaded + if (html === null) { // said once; the next render asks again + g.ref = ''; + const slot = $('#' + g.slot); if (slot) { slot.removeAttribute('aria-busy'); slot.innerHTML = '

Could not load the list. Change step and come back to try again.

'; } return; } - grid.node = Object.assign(document.createElement('div'), {id: 'nb-agent-grid', innerHTML: html}); - placeAgentGrid(); + g.node = Object.assign(document.createElement('div'), {id: g.slot, innerHTML: html}); + placeGrid(g); } function modelChoiceCards(rec) { const hasPaper = C.paperOffered(rec), on = rec.model_choice || 'any'; @@ -962,7 +927,7 @@ const rows=own?[...((runnerOut&&runnerOut.review)||[]),...(C.isStandard(rec)?[]:[[S.where,'Where',C.whereLabel(rec)]])]:C.review(rec); html='

Review the experiment.

'+rows.map(([i,t,v])=>`
${t}${esc(v)}
${button('Edit',`data-step="${i}"`)}
`).join('')+'
'+(schema.local?'
':'')+runPanel(); } - $('#nb-form').innerHTML=html+'

';applyPick();placeAgentGrid(); + $('#nb-form').innerHTML=html+'

';placeGrids(); if(step===REVIEW&&schema.local){drawPrereq();checkPrereq();} $('#nb-back').hidden=step===S.benchmark; $('#nb-next').textContent=step===REVIEW?'Export recipe JSON':stepName(steps[nextShown(step,1)],nextShown(step,1))+' →'; @@ -1124,8 +1089,6 @@ if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check',d.hostreqName);return;} if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install',d.hostreqName);return;} if(d.copyCode!==undefined){const t=b.closest('.std-code').querySelector('code').textContent;try{await navigator.clipboard.writeText(t);b.textContent='Copied';}catch{b.textContent='Select it';}setTimeout(()=>{b.textContent='Copy';},1500);return;} - if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;} - if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;} if(d.agent){const on=r().agents.includes(d.agent);r().agents=on?r().agents.filter(id=>id!==d.agent):[...r().agents,d.agent];renderForm();refreshPlan();return;} else if(d.href){if(!window.nulliusOpenSettings?.(d.href,b))location.href=d.href;return;} else if(b.id==='nb-add-custom-agent'){r().customAgents.push('');r().agents.push('custom-'+(r().customAgents.length-1));renderForm();refreshPlan();root.querySelector('[data-custom-agent="'+(r().customAgents.length-1)+'"]')?.focus();return;} @@ -1136,7 +1099,6 @@ else if(d.harborScan!==undefined){askScan(true);return;} else if(d.harborRefsFresh!==undefined){loadRefs(C.harborRepoUrl(r().harbor.repo),true);return;} else if(d.runner){toRunner(d.runner,C.benchKey(r()));} - else if(d.bench){toRunner(C.runnerFor(r(),d.bench),d.bench);} else if(d.stdBench){C.setStandardBenchmark(r(),d.stdBench);} else if(d.stdOther){const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(d.stdOther)?c.filter(k=>k!==d.stdOther):[...c,d.stdOther];delete rec.other;if(rec.options)delete rec.options[d.stdOther];} // a check else if(d.stdDataset){const rec=r(),c=C.datasetsOf(rec);rec.datasets=c.includes(d.stdDataset)?(c.length>1?c.filter(k=>k!==d.stdDataset):c):[...c,d.stdDataset];delete rec.dataset;} @@ -1177,12 +1139,12 @@ else return; render(d.step!==undefined||['nb-next','nb-back'].includes(b.id)); }); - // an agent, a capability or the setup picked or let go on the Agent step's grid; the focus stays on its card - root.addEventListener('nullius-pick',event=>{const k=event.detail.value,rec=r(),c=C.chosenOf(rec);rec.others=c.includes(k)?c.filter(x=>x!==k):[...c,k];delete rec.other;if(rec.options)delete rec.options[k];inPlace(()=>render());event.target.focus({preventScroll:true});}); + // a benchmark chosen on the Benchmark step's grid, or an agent, a capability or the setup picked or let go on the + // Agent step's; the page stays where it is and the focus on its card + root.addEventListener('nullius-pick',event=>{const k=event.detail.value;if(event.target.closest('#nb-bench-grid')){toRunner(C.runnerFor(r(),k),k);}else{const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(k)?c.filter(x=>x!==k):[...c,k];delete rec.other;if(rec.options)delete rec.options[k];}inPlace(()=>render());event.target.focus({preventScroll:true});}); root.addEventListener('input',event=>{ const el=event.target,d=el.dataset,rec=r(); if(el.id==='nb-recipe-title'){updateSavePreview();return;} - if(d.pickFind!==undefined){pickOf().q=el.value;applyPick();return;} if(d.servedBy!==undefined)return; // what is typed in a Served by filters its menu: a pick, or Enter, adds it (nb-pick) if(d.providerModel && rec.modelFormat !== 'provider-id'){for(const key of ['model','userModel','evalModel'])if(typeof rec[key]==='string')rec[key]=C.bareModel(rec[key]);rec.modelFormat='provider-id';} if(d.field){const p=d.field.startsWith('param:')?C.parameters().find(p=>p.key===d.field.slice(6)):null; @@ -1218,9 +1180,8 @@ const el=event.target,d=el.dataset; if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} - else if(d.stdVersion!==undefined){ // in place: only the card's count changes on this step, so nothing else is drawn again - C.setStandardBenchmark(r(),el.value);const key=C.benchKey(r()),meta=root.querySelector(`[data-bench="${CSS.escape(key)}"] .nb-tile-foot > span`); - if(meta)meta.textContent=tasksOf(key,versionOf(r()));render(false,false);} + else if(d.stdVersion!==undefined){ // in place: nothing else on this step changes + C.setStandardBenchmark(r(),el.value);render(false,false);} // an option at its default is left out: the recipe names only what was changed. A select's value can show or hide // the options that apply under it (`when`): the step is drawn again and the focus goes back to the select, or to the // menu button controls.js puts after it once this change is handled; a typed value is left as it is diff --git a/app/templates/builder/benchmarks.html b/app/templates/builder/benchmarks.html new file mode 100644 index 00000000..c70b442b --- /dev/null +++ b/app/templates/builder/benchmarks.html @@ -0,0 +1,11 @@ +{#- The builder's Benchmark step grid (web/builder.py, new_run_benchmarks): the Benchmarks page's cards + (web/catalogue.py, benchmark_cards) as single-choice pick cards of the card toolkit (_cards.html), or its rows + (`layout` 'table'), loaded by command-builder.js, which keeps the choice and switches the view in place. -#} +{% from '_cards.html' import cards %} +{#- the category a row of pills; what it is run with a menu of its own, beside Filter (origin, runner) -#} +{% set cols = [dict(key='icon', label='Mark', card='icon', table=false, class='bench-mark'), dict(key='title', label='Name', card='title', width='expand'), + dict(key='with', label='Run with', card='badge', class='std-tag', filter='dropdown', filter_icon='plug'), + dict(key='tasks', label='Tasks', card='stat', type='num'), + dict(key='category', label='Category', card='tag', class='cat-tag', filter='pills'), dict(key='origin', label='Origin', card='tag', class='origin-tag'), + dict(key='runs', label='Runner', class='run-tag'), dict(key='domain', label='Domain', off=true)] %} +{{ cards('nb-benchmarks', cols, items, search='Search benchmarks', choices=choices, sort=['title', 'tasks'], empty='No benchmark matches.', noun='benchmarks', layout=layout, default='cards') }} diff --git a/app/ui_tests/test_builder_benchmarks.py b/app/ui_tests/test_builder_benchmarks.py new file mode 100644 index 00000000..c20d5df7 --- /dev/null +++ b/app/ui_tests/test_builder_benchmarks.py @@ -0,0 +1,71 @@ +"""The builder's Benchmark step grid (/new-run/benchmarks): the Benchmarks page's cards as single-choice pick cards, +with what each benchmark is run with as a menu of its own.""" +import os +import re +import shutil +import tempfile +import unittest +from html import unescape +from unittest.mock import patch + +from .test_routes import web + + +class BenchmarkGrid(unittest.TestCase): + def setUp(self): + tmp = tempfile.mkdtemp(prefix="nullius-bench-grid-") + self.addCleanup(shutil.rmtree, tmp, True) + env = patch.dict(os.environ, {"NULLIUS_HOME": tmp, "HOSTS": "", "RUNS_DIR": os.path.join(tmp, "runs")}) + env.start(); self.addCleanup(env.stop) + os.environ.pop("NULLIUS_CATALOGUE", None) + self.client = web.app.test_client() + + def get(self, url): + r = self.client.get(url) + self.assertEqual(r.status_code, 200, url) + return r.get_data(as_text=True) + + def test_it_lists_the_benchmarks_pages_benchmarks_each_a_single_choice(self): + html = self.get("/new-run/benchmarks") + cards = {m.group(1): m.group(0) for m in re.finditer(r'
', html, re.S)} + listed = re.findall(r'Capabilities', cards["locomo"]) + self.assertIn('Agents', cards["swe-bench-verified"]) # Harbor runs agents + self.assertIn('
Tasks
1,319
', cards["gsm8k"]) + self.assertIn('Math', cards["gsm8k"]) + self.assertNotIn("Official", html) # only a community one says so + self.assertIn('title="Real-world questions', cards["gaia"]) # its domain, as its tooltip + + def test_its_controls_category_pills_run_with_beside_filter_sort_and_the_toggle(self): + html = self.get("/new-run/benchmarks") + self.assertIn('data-pills="category"', html) + self.assertIn('data-dropdown aria-haspopup="true" aria-expanded="false">Run with<', html) + self.assertLess(html.index("data-dropdown"), html.index('data-tools-btn="filter"')) + menu = unescape(html[html.index('
")[0] + self.assertIn('data-value="agent" checked> Agents', menu) + self.assertIn('data-value="capability" checked> Capabilities', menu) + self.assertIn('data-filter-menu="runs"', html[html.index('data-tools="filter"'):]) + self.assertIn('data-tools-btn="sort"', html) + self.assertIn('href="/new-run/benchmarks?view=table" aria-label="Table"', html) + + def test_its_table_picks_one_with_radios(self): + html = self.get("/new-run/benchmarks?view=table") + self.assertIn('', html) + self.assertIn('', html) + self.assertNotIn('type="checkbox" data-pick', html) + self.assertIn('#nb-benchmarks > * > tr > :nth-child(2) { width: auto;', html) # Name takes the spare width + self.assertEqual(re.findall(r'', html) self.assertNotIn('type="checkbox" data-pick', html) self.assertIn('#nb-benchmarks > * > tr > :nth-child(2) { width: auto;', html) # Name takes the spare width - self.assertEqual(re.findall(r'
', self.get("/new-run")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index de9f575c..1f2f85b5 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -22,7 +22,7 @@ def draw(self, docker, edit="", backends=("local", "docker"), boat=None, slots=2 "boatSizes": self.SIZES, "boatSlots": slots}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" + line("esc") + line("button") + cut(" const unitsOf = ", " // the first clause") + line("words") - + cut(" const tile = t =>", " const picker = ") + + cut(" const tile = t =>", " // The grids drawn by the server") + cut(" const consentBox = ", " async function hostReq(") + cut(" function benchOptions(", " // Only credentials") + "const out={};for(const b of C.standard().benchmarks)for(const backend of " + json.dumps(list(backends)) + "){" + "const r=C.createStandard(b.key);r.backend=backend;" + edit @@ -143,7 +143,7 @@ def test_an_option_shows_its_example_and_a_setup_its_label(self): line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" schema = dict(self.schema, hosts=[], standardHere={"docker": None, "licences": {}, "host": {"osworld2": {"docker": ["missing"]}}}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" - + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + cut(" function benchOptions(", " // Only credentials") + + line("esc") + line("button") + cut(" const tile = t =>", " // The grids drawn by the server") + cut(" function benchOptions(", " // Only credentials") + "const out={};for(const k of ['osworld2','gsm8k']){const r=C.createStandard(k);out[k]=benchOptions(r,C.standardBench(r),'shared')+benchOptions(r,C.standardBench(r),'setup')+benchOptions(r,C.standardBench(r));}" + "const r=C.createStandard('osworld2'),b=C.standardBench(r);r.datasets=['offline36'];out.offline=benchOptions(r,b,'dataset');r.datasets=['test_v2'];out.all=benchOptions(r,b,'dataset');" + "process.stdout.write(JSON.stringify(out));") @@ -158,7 +158,7 @@ def test_an_own_agents_settings_are_in_its_tab_and_the_agent_is_no_option(self): line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" schema = dict(self.schema, hosts=[], standardHere={"docker": None, "licences": {}, "host": {}}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" - + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + cut(" function benchOptions(", " // Only credentials") + + line("esc") + line("button") + cut(" const tile = t =>", " // The grids drawn by the server") + cut(" function benchOptions(", " // Only credentials") + "const r=C.createStandard('osworld1'),b=C.standardBench(r);r.others=['osworld1:m3','osworld1:claude'];" + "const own=k=>benchOptions(r,b,'own',C.ownAgentsOf(b).find(a=>a.key===k));" + "process.stdout.write(JSON.stringify({m3:own('osworld1:m3'),claude:own('osworld1:claude')," diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index 367c332c..e1e017f9 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -201,12 +201,12 @@ def test_an_anchor_from_the_one_page_list_goes_on_to_its_page(self): def test_every_listed_benchmark_has_its_mark(self): # one mark per benchmark, by key - its maker's logo or a line icon - in the sprite the builder's cards and these pages use marks = (Path(__file__).resolve().parents[1] / "static" / "catalogue" / "marks.svg").read_text(encoding="utf-8") - builder = (Path(__file__).resolve().parents[1] / "static" / "builder" / "command-builder.js").read_text(encoding="utf-8") - self.assertIn("/static/catalogue/marks.svg#bm-${esc(key)}", builder) + builder = self.client.get("/new-run/benchmarks").get_data(as_text=True) # the builder's cards are these for b in catalog.load().listed_benchmarks: with self.subTest(b.key): self.assertIn(' String(s); const button = (label, attrs) => ``; diff --git a/app/web/builder.py b/app/web/builder.py index b1188f6d..44c8ecf6 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -27,6 +27,7 @@ "rovodev-cli": ["ROVODEV_USER_API_TOKEN"]} from . import shared from . import execute +from . import catalogue @app.route("/new-run") @@ -61,6 +62,26 @@ def new_run(): return render_template("builder/new_run.html", builder_data=data) +@app.route("/new-run/benchmarks") +def new_run_benchmarks(): + """The Benchmark step's grid, as a fragment the builder puts in its page: the Benchmarks page's cards + (``catalogue.benchmark_cards``) as single-choice pick cards (templates/builder/benchmarks.html), or rows with + ?view=table; the builder presses the one its recipe names.""" + cat = shared._catalog_cached() + # a community benchmark says so; an official one says nothing + items = [dict(i, pick={"value": i["key"], "one": True}, id="pick-bench-" + i["key"], title={"text": i["title"], "title": i["domain"]}, + origin=i["origin"] if i["shelf"] != "official" else None) for i in catalogue.benchmark_cards(cat)] + every = catalogue.benchmark_choices(cat) + choices = {k: [(v, t) for v, t in every[k] if any(v in _values(i[k]) for i in items)] for k in ("category", "with", "runs")} + return render_template("builder/benchmarks.html", items=items, choices={k: v for k, v in choices.items() if len(v) > 1}, + layout="table" if request.args.get("view") == "table" else "cards") + + +def _values(v) -> list: + """A card field's filter values: a value's ``value``, else its text.""" + return [x.get("value", x.get("text")) if isinstance(x, dict) else x for x in (v if isinstance(v, list) else [v])] + + @app.route("/new-run/agents") def new_run_agents(): """The Agent step's grid for one benchmark of the standard (?benchmark=key or key@version), as a fragment the diff --git a/docs/builder.md b/docs/builder.md index d049cef3..dd2213b4 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -22,7 +22,7 @@ Then [read its results](results.md). | Stage | Decides | | --- | --- | -| Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`) | +| Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`). The cards are the Benchmarks page's, as cards or a table; one is chosen at a time. The category pills, *Run with* (agents or capabilities) and the Filter menu (Runner) narrow them, and the chosen one stays in view under any filter | | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A benchmark option its datasets need (`step: dataset`) is asked here, for the datasets it applies to. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. A benchmark with [agents of its own](standard.md#a-benchmarks-own-agents) (OSWorld's) lists them first, in place of its Original tile; several may be chosen, a job each, and a setting for one of them is in its tab. The cards are the Agents page's, each greyed with why it cannot run, and show as cards or a table. *Fit* starts on *Can run it*; the Filter menu (Protocol, From, Proves) narrows them further, and a chosen agent stays in view under any filter. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | From 63d746479623b6b7d892deba9fe4d8018bd3b27c Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 11:54:58 +0100 Subject: [PATCH 4/8] fix(web): wrap a card table's name between its words A name column that takes the spare width broke a long name inside a word on a phone (LongMemEval as "LongMemEv al"). The name now wraps only between words, and the table scrolls sideways instead. --- app/static/catalogue/catalogue.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/static/catalogue/catalogue.css b/app/static/catalogue/catalogue.css index c758a460..fe6c417d 100644 --- a/app/static/catalogue/catalogue.css +++ b/app/static/catalogue/catalogue.css @@ -59,7 +59,7 @@ tr:has([data-pick]:not(:disabled)) { cursor: pointer; } .bench-card-name > .icon-btn { vertical-align: middle; } .bench-mark { flex: none; width: 22px; height: 22px; color: var(--accent); } /* a table row's name (_cards.html rows): its icon, then its link */ -.row-name { display: inline-flex; align-items: center; gap: 8px; } +.row-name { display: inline-flex; align-items: center; gap: 8px; overflow-wrap: break-word; } /* a name wraps between its words, never inside one: an expand column breaks anywhere, the table scrolls instead */ .bench-card-domain { margin-top: -6px; font-size: 14px; line-height: 1.5; color: var(--text-dim); overflow-wrap: anywhere; } /* a long file name breaks rather than spills */ /* the stats in columns of one width, a column each, each its label then its value; on a card too narrow for one line (a phone), two balanced rows: 4 as 2+2, 5 as 3+2, 6 as 3+3 */ From e6f081552121910a7f267467422af29a91b1f0ab Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 13:45:47 +0100 Subject: [PATCH 5/8] feat(builder): count the chosen version's tasks on its benchmark card The benchmarks' listing carries each other version's tasks (OSWorld1's main pin: 369, its default tag: 368). The Benchmark step's card and row of a benchmark with versions hold them as data-tasks, and the builder shows the chosen benchmark's count for the version its recipe names, in place when the version changes, as the step's tiles did. The card toolkit takes data attributes for a card and its row. --- app/nullius/standard/recipe.py | 8 +++++--- app/static/builder/command-builder.js | 18 +++++++++++++++--- app/templates/_cards.html | 6 +++--- app/tests/test_benchmark_listing.py | 10 ++++++++++ app/ui_tests/test_builder_benchmarks.py | 10 ++++++++++ app/web/builder.py | 5 ++++- app/web/catalogue.py | 1 + 7 files changed, 48 insertions(+), 10 deletions(-) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 5e4ee4ea..7a88f170 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -118,8 +118,8 @@ def benchmarks(cat) -> List[dict]: """The benchmarks the Benchmarks page and the builder's Benchmark step list: each of the catalogue's listed entries (``Catalog.listed_benchmarks``, whichever runner runs it) as ``b``, with its ``datasets``, its ``tasks`` and their ``unit`` (``Catalog.tasks_of``), what some of its options need of a machine (``host``, None for nothing), - and what it is run with: ``family`` agent or capability (a benchmark only Harbor runs runs agents) and, for a - capability, its ``kind``.""" + what it is run with: ``family`` agent or capability (a benchmark only Harbor runs runs agents) and, for a + capability, its ``kind``; and ``versions``, another version's tasks and unit by version.""" defs = {d["key"]: d for d in definitions.load("benchmark")[0]} out = [] for b in cat.listed_benchmarks: @@ -127,7 +127,9 @@ def benchmarks(cat) -> List[dict]: tasks, unit = cat.tasks_of(b) out.append({"b": b, "datasets": cat.datasets_of(b), "tasks": tasks, "unit": unit, "host": sorted(d["host"]) if d.get("host") else None, - "family": d.get("family") or "agent", "kind": _capability_kind(d)}) + "family": d.get("family") or "agent", "kind": _capability_kind(d), + # another version's tasks, when its datasets count differently (``Catalog.in_version``) + "versions": {v: cat.tasks_of(b, cat.in_version(b, v)[1]) for v in b.versions}}) return out diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 9b987d8a..12f16f82 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -636,7 +636,7 @@ // pressed as the recipe chooses (nullius-pick changes the recipe). Its view, cards or a table, is the viewer's: // the toggle switches it in place, never leaving the page. const grids = [ - {slot: 'nb-bench-grid', store: 'nb-bench-view', url: () => '/new-run/benchmarks?', chosen: rec => [C.benchKey(rec)]}, + {slot: 'nb-bench-grid', store: 'nb-bench-view', url: () => '/new-run/benchmarks?', chosen: rec => [C.benchKey(rec)], after: benchTasks}, {slot: 'nb-agent-grid', store: 'nb-agent-view', url: rec => '/new-run/agents?' + new URLSearchParams({benchmark: rec.benchmark}) + '&', chosen: rec => C.chosenOf(rec)}, ]; grids.forEach(g => { g.view = 'cards'; g.ref = ''; g.node = null; try { g.view = localStorage.getItem(g.store) === 'table' ? 'table' : 'cards'; } catch {} }); @@ -656,6 +656,7 @@ // a choice made elsewhere (a recipe loaded) that the filters hide: the toolkit draws again, and it shows if (g.node.dataset.wired && [...g.node.querySelectorAll('[data-pick]')].some(b => b.matches('[aria-pressed="true"], :checked') && b.closest('[data-card], tr').hidden)) g.node.querySelector('[data-search]')?.dispatchEvent(new Event('input')); + g.after?.(); if (!g.node.dataset.wired) { g.node.dataset.wired = '1'; const cards = g.node.querySelector('[data-cards]'), table = g.node.querySelector('table[data-table]'); @@ -669,6 +670,17 @@ })); } } + // the Tasks of a benchmark with versions (its data-tasks, {version: {text, label}}, the default's under ''): the + // chosen benchmark's as the version the recipe names reads, any other's its default's + function benchTasks() { + const rec = r(), on = C.benchKey(rec), v = versionOf(rec); + root.querySelectorAll('#nb-bench-grid [data-tasks]').forEach(el => { + const by = JSON.parse(el.dataset.tasks), t = by[el.querySelector('[data-pick]')?.dataset.pick === on && by[v] ? v : '']; + const dd = el.querySelector('.bench-stats dd'), dt = el.querySelector('.bench-stats dt'), td = el.querySelector('td[data-col="tasks"] > span'); + if (dd) { dd.textContent = t.text; dt.textContent = t.label; } + if (td) td.textContent = t.text; + }); + } async function loadGrid(g, ref) { const res = await nulliusFetch(ref).catch(() => null); const html = res && res.ok ? await res.text().catch(() => null) : null; @@ -1180,8 +1192,8 @@ const el=event.target,d=el.dataset; if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} - else if(d.stdVersion!==undefined){ // in place: nothing else on this step changes - C.setStandardBenchmark(r(),el.value);render(false,false);} + else if(d.stdVersion!==undefined){ // in place: only the card's count changes on this step, so nothing else is drawn again + C.setStandardBenchmark(r(),el.value);benchTasks();render(false,false);} // an option at its default is left out: the recipe names only what was changed. A select's value can show or hide // the options that apply under it (`when`): the step is drawn again and the focus goes back to the select, or to the // menu button controls.js puts after it once this change is handled; a typed value is left as it is diff --git a/app/templates/_cards.html b/app/templates/_cards.html index 3b372ab2..662810ed 100644 --- a/app/templates/_cards.html +++ b/app/templates/_cards.html @@ -12,7 +12,7 @@ button[data-pick][aria-pressed] that tables.js flips, sending `nullius-pick` {value, pressed} (a disabled one's reason goes in its `alert`), and an enabled one's corner holds the check that marks it chosen; `one`, a single choice: choosing it lets the others go, choosing it again keeps it (a radio in a table); a title's `title` is the tooltip of its link or button; - `id`, `search` (what the search reads), `keys`, and `unless` {field: other}: the item passes the field's filter + `id`, `search` (what the search reads), `keys`, `data` {name: value} (data- attributes, on its row too), and `unless` {field: other}: the item passes the field's filter whenever the other field's filter narrows and passes it. A title's `after` is markup in the flow of its name, after its last word (a control of its own), and its `class` goes on the title row. A field's `class` goes on each of its values; `card_label` names a stat when its column header does not, and a @@ -59,7 +59,7 @@ {%- set icon = (cols|selectattr('card', 'equalto', 'icon')|list + [none])[0] %} {%- set icons = icon and items|selectattr(icon.key)|list|length > 0 %} {%- set shown = [dict(key='_pick', label='Chosen', blank=true, pick=true, sticky=true)] if items|selectattr('pick')|list else [] %}{% for c in cols if c.table is not sameas false and (c.key in choices or c.type == 'num' or items|selectattr(c.key)|list) %}{% set _ = shown.append(dict(c, sticky=c.card == 'title')) %}{% endfor %} -{%- set rs = [] %}{% for item in items %}{% set a %}{% if item.get('id') %} id="{{ item.id }}"{% endif %} data-search="{{ item.get('search', '') }}" +{%- set rs = [] %}{% for item in items %}{% set a %}{% if item.get('id') %} id="{{ item.id }}"{% endif %} data-search="{{ item.get('search', '') }}"{% for k, v in (item.get('data') or {}).items() %} data-{{ k }}="{{ v }}"{% endfor %} {%- for f in cols if f.key in choices %} data-f-{{ f.key }}="{% call(f, v) each(item, [f]) %}{{ v.get('value', v.text) }} {% endcall %}"{% endfor %}{% for k, by in (item.get('unless') or {}).items() %} data-unless-{{ k }}="{{ by }}"{% endfor %}{% endset %} {%- set _ = rs.append({'item': item, 'attrs': a}) %}{% endfor %} {% call(r, c) table(id, shown, rs, attrs, search=search, columns=true, choices=choices, only=only, empty=empty, noun=noun, sort=sort, layout='table', default=default) %} @@ -85,7 +85,7 @@ {%- set icons = icon and (every or items)|selectattr(icon.key)|list|length > 0 %}
{%- for item in items %} -
{%- set title = by.title[0] %}{% set v = item.get(title.key) %}{% set v = v if v is mapping else {'text': v} %}{% set name = v.text %} diff --git a/app/tests/test_benchmark_listing.py b/app/tests/test_benchmark_listing.py index e2e8af2e..3601a5c6 100644 --- a/app/tests/test_benchmark_listing.py +++ b/app/tests/test_benchmark_listing.py @@ -19,6 +19,16 @@ def test_it_is_the_catalogues_listed_benchmarks_with_their_family(self): self.assertEqual((by["gsm8k"]["tasks"], by["gsm8k"]["unit"]), cat.tasks_of(cat.benchmark("gsm8k"))) self.assertTrue(all(r["family"] in ("agent", "capability") for r in rows)) + def test_a_benchmark_with_versions_counts_each_ones_tasks(self): + cat = catalog.load(catalog.ROOT) + os1 = next(r for r in recipe.benchmarks(cat) if r["b"].key == "osworld1") + b = cat.benchmark("osworld1") + self.assertEqual((os1["tasks"], os1["unit"]), cat.tasks_of(b)) # the default version + self.assertEqual(set(os1["versions"]), set(b.versions)) + for v in b.versions: + self.assertEqual(os1["versions"][v], cat.tasks_of(b, cat.in_version(b, v)[1])) + self.assertEqual(next(r for r in recipe.benchmarks(cat) if r["b"].key == "gsm8k")["versions"], {}) + if __name__ == "__main__": unittest.main() diff --git a/app/ui_tests/test_builder_benchmarks.py b/app/ui_tests/test_builder_benchmarks.py index c20d5df7..f534e8ce 100644 --- a/app/ui_tests/test_builder_benchmarks.py +++ b/app/ui_tests/test_builder_benchmarks.py @@ -1,5 +1,6 @@ """The builder's Benchmark step grid (/new-run/benchmarks): the Benchmarks page's cards as single-choice pick cards, with what each benchmark is run with as a menu of its own.""" +import json import os import re import shutil @@ -40,6 +41,15 @@ def test_it_lists_the_benchmarks_pages_benchmarks_each_a_single_choice(self): self.assertNotIn("Official", html) # only a community one says so self.assertIn('title="Real-world questions', cards["gaia"]) # its domain, as its tooltip + def test_a_benchmark_with_versions_carries_each_ones_tasks(self): + html = self.get("/new-run/benchmarks") + card = re.search(r'
]*id="pick-bench-osworld1"[^>]*>', html).group(0) + tasks = json.loads(unescape(re.search(r'data-tasks="([^"]*)"', card).group(1))) + self.assertEqual(tasks[""]["text"], "368") # the default version's + self.assertEqual(tasks["044bc33"], {"text": "369", "label": "Tasks"}) + self.assertNotIn("data-tasks", re.search(r'
]*id="pick-bench-gsm8k"[^>]*>', html).group(0)) + self.assertRegex(self.get("/new-run/benchmarks?view=table"), r']*id="pick-bench-osworld1"[^>]*data-tasks=') # its row too + def test_its_controls_category_pills_run_with_beside_filter_sort_and_the_toggle(self): html = self.get("/new-run/benchmarks") self.assertIn('data-pills="category"', html) diff --git a/app/web/builder.py b/app/web/builder.py index 44c8ecf6..cee10a33 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -69,8 +69,11 @@ def new_run_benchmarks(): ?view=table; the builder presses the one its recipe names.""" cat = shared._catalog_cached() # a community benchmark says so; an official one says nothing + # a benchmark with versions carries each one's Tasks (the default's under ""), for the builder to show the chosen one's items = [dict(i, pick={"value": i["key"], "one": True}, id="pick-bench-" + i["key"], title={"text": i["title"], "title": i["domain"]}, - origin=i["origin"] if i["shelf"] != "official" else None) for i in catalogue.benchmark_cards(cat)] + origin=i["origin"] if i["shelf"] != "official" else None, + data={"tasks": json.dumps({"": i["tasks"], **i["versions"]})} if i["versions"] else None) + for i in catalogue.benchmark_cards(cat)] every = catalogue.benchmark_choices(cat) choices = {k: [(v, t) for v, t in every[k] if any(v in _values(i[k]) for i in items)] for k in ("category", "with", "runs")} return render_template("builder/benchmarks.html", items=items, choices={k: v for k, v in choices.items() if len(v) > 1}, diff --git a/app/web/catalogue.py b/app/web/catalogue.py index f7f735c7..8ee2bad1 100644 --- a/app/web/catalogue.py +++ b/app/web/catalogue.py @@ -63,6 +63,7 @@ def _card(row, sheet) -> dict: "needs": needs and {"text": "needs " + ", ".join(needs), "title": f"Some of its options need {', '.join(needs)} installed on the machine it runs on: see its page"}, "tasks": {"text": tasks, "label": unit.capitalize(), "value": tasks.replace(",", "").split("–")[-1]}, + "versions": {v: {"text": n, "label": u.capitalize()} for v, (n, u) in row["versions"].items()}, "datasets": {"text": len(ds), "label": "Datasets" if len(ds) != 1 else "Dataset"}, "with": {"text": FAMILIES[row["family"]], "value": row["family"], "title": f"{row['kind'].capitalize()} capabilities" if row["kind"] else ""}, "scoring": b.scoring, "blocker": b.blocker[:1].upper() + b.blocker[1:]} From b54a370fc4458c95424386b4bf1290a231c920e0 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 13:46:08 +0100 Subject: [PATCH 6/8] feat(builder): name the Benchmark step's agents or capabilities menu Evaluates, with a target icon "Run with" read oddly beside the runner, and the plug did not say it. The menu, its column and its field are now Evaluates (Agents, Capabilities), led by a new i-target icon in the sprite: three concentric circles in the sprite's line style. i-plug stays for Settings. --- app/templates/base.html | 1 + app/templates/builder/benchmarks.html | 4 ++-- app/ui_tests/test_builder_benchmarks.py | 12 ++++++------ app/web/builder.py | 2 +- app/web/catalogue.py | 6 +++--- docs/builder.md | 2 +- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/app/templates/base.html b/app/templates/base.html index eeadbdaa..77df93e9 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -72,6 +72,7 @@ + diff --git a/app/templates/builder/benchmarks.html b/app/templates/builder/benchmarks.html index c70b442b..50d6d2f6 100644 --- a/app/templates/builder/benchmarks.html +++ b/app/templates/builder/benchmarks.html @@ -2,9 +2,9 @@ (web/catalogue.py, benchmark_cards) as single-choice pick cards of the card toolkit (_cards.html), or its rows (`layout` 'table'), loaded by command-builder.js, which keeps the choice and switches the view in place. -#} {% from '_cards.html' import cards %} -{#- the category a row of pills; what it is run with a menu of its own, beside Filter (origin, runner) -#} +{#- the category a row of pills; what it evaluates (agents or capabilities) a menu of its own, beside Filter (runner) -#} {% set cols = [dict(key='icon', label='Mark', card='icon', table=false, class='bench-mark'), dict(key='title', label='Name', card='title', width='expand'), - dict(key='with', label='Run with', card='badge', class='std-tag', filter='dropdown', filter_icon='plug'), + dict(key='evaluates', label='Evaluates', card='badge', class='std-tag', filter='dropdown', filter_icon='target'), dict(key='tasks', label='Tasks', card='stat', type='num'), dict(key='category', label='Category', card='tag', class='cat-tag', filter='pills'), dict(key='origin', label='Origin', card='tag', class='origin-tag'), dict(key='runs', label='Runner', class='run-tag'), dict(key='domain', label='Domain', off=true)] %} diff --git a/app/ui_tests/test_builder_benchmarks.py b/app/ui_tests/test_builder_benchmarks.py index f534e8ce..22a2f3dd 100644 --- a/app/ui_tests/test_builder_benchmarks.py +++ b/app/ui_tests/test_builder_benchmarks.py @@ -1,5 +1,5 @@ """The builder's Benchmark step grid (/new-run/benchmarks): the Benchmarks page's cards as single-choice pick cards, -with what each benchmark is run with as a menu of its own.""" +with what each benchmark evaluates, agents or capabilities, as a menu of its own.""" import json import os import re @@ -33,7 +33,7 @@ def test_it_lists_the_benchmarks_pages_benchmarks_each_a_single_choice(self): self.assertEqual(sorted(cards), sorted(listed)) # one listing for k, c in cards.items(): self.assertIn(f'data-pick="{k}" data-pick-one aria-pressed="false"', c) - self.assertIn('data-f-with="capability "', cards["locomo"]) + self.assertIn('data-f-evaluates="capability "', cards["locomo"]) self.assertIn('Capabilities', cards["locomo"]) self.assertIn('Agents', cards["swe-bench-verified"]) # Harbor runs agents self.assertIn('
Tasks
1,319
', cards["gsm8k"]) @@ -53,9 +53,9 @@ def test_a_benchmark_with_versions_carries_each_ones_tasks(self): def test_its_controls_category_pills_run_with_beside_filter_sort_and_the_toggle(self): html = self.get("/new-run/benchmarks") self.assertIn('data-pills="category"', html) - self.assertIn('data-dropdown aria-haspopup="true" aria-expanded="false">Run with<', html) + self.assertIn('data-dropdown aria-haspopup="true" aria-expanded="false">Evaluates<', html) self.assertLess(html.index("data-dropdown"), html.index('data-tools-btn="filter"')) - menu = unescape(html[html.index('
")[0] + menu = unescape(html[html.index('
")[0] self.assertIn('data-value="agent" checked> Agents', menu) self.assertIn('data-value="capability" checked> Capabilities', menu) self.assertIn('data-filter-menu="runs"', html[html.index('data-tools="filter"'):]) @@ -68,12 +68,12 @@ def test_its_table_picks_one_with_radios(self): self.assertIn('
', self.get("/new-run")) diff --git a/app/web/builder.py b/app/web/builder.py index cee10a33..1ccc5926 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -75,7 +75,7 @@ def new_run_benchmarks(): data={"tasks": json.dumps({"": i["tasks"], **i["versions"]})} if i["versions"] else None) for i in catalogue.benchmark_cards(cat)] every = catalogue.benchmark_choices(cat) - choices = {k: [(v, t) for v, t in every[k] if any(v in _values(i[k]) for i in items)] for k in ("category", "with", "runs")} + choices = {k: [(v, t) for v, t in every[k] if any(v in _values(i[k]) for i in items)] for k in ("category", "evaluates", "runs")} return render_template("builder/benchmarks.html", items=items, choices={k: v for k, v in choices.items() if len(v) > 1}, layout="table" if request.args.get("view") == "table" else "cards") diff --git a/app/web/catalogue.py b/app/web/catalogue.py index 8ee2bad1..ea618fc3 100644 --- a/app/web/catalogue.py +++ b/app/web/catalogue.py @@ -30,7 +30,7 @@ def benchmarks_page(): RUNS_WITH = {"standard": ("Nullius", "the Nullius standard"), "harbor": ("Harbor", "Harbor")} -# what a benchmark is run with (``recipe.benchmarks``' family), as a card and its filter say it +# what a benchmark evaluates (``recipe.benchmarks``' family): its agents or its capabilities, as a card and its filter say it FAMILIES = {"agent": "Agents", "capability": "Capabilities"} @@ -47,7 +47,7 @@ def benchmark_cards(cat) -> list: def benchmark_choices(cat) -> dict: """The values the benchmarks' cards are filtered by: their category, what runs them, their origin, what they run.""" return {"category": [(c, c) for c in cat.categories], "runs": [(way, label) for way, (label, _) in RUNS_WITH.items()], - "origin": [(o, o.capitalize()) for o in catalog.ORIGINS], "with": list(FAMILIES.items())} + "origin": [(o, o.capitalize()) for o in catalog.ORIGINS], "evaluates": list(FAMILIES.items())} def _card(row, sheet) -> dict: @@ -65,7 +65,7 @@ def _card(row, sheet) -> dict: "tasks": {"text": tasks, "label": unit.capitalize(), "value": tasks.replace(",", "").split("–")[-1]}, "versions": {v: {"text": n, "label": u.capitalize()} for v, (n, u) in row["versions"].items()}, "datasets": {"text": len(ds), "label": "Datasets" if len(ds) != 1 else "Dataset"}, - "with": {"text": FAMILIES[row["family"]], "value": row["family"], "title": f"{row['kind'].capitalize()} capabilities" if row["kind"] else ""}, + "evaluates": {"text": FAMILIES[row["family"]], "value": row["family"], "title": f"{row['kind'].capitalize()} capabilities" if row["kind"] else ""}, "scoring": b.scoring, "blocker": b.blocker[:1].upper() + b.blocker[1:]} diff --git a/docs/builder.md b/docs/builder.md index dd2213b4..c6fe852c 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -22,7 +22,7 @@ Then [read its results](results.md). | Stage | Decides | | --- | --- | -| Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`). The cards are the Benchmarks page's, as cards or a table; one is chosen at a time. The category pills, *Run with* (agents or capabilities) and the Filter menu (Runner) narrow them, and the chosen one stays in view under any filter | +| Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`). The cards are the Benchmarks page's, as cards or a table; one is chosen at a time. The category pills, *Evaluates* (agents or capabilities) and the Filter menu (Runner) narrow them, and the chosen one stays in view under any filter | | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A benchmark option its datasets need (`step: dataset`) is asked here, for the datasets it applies to. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. A benchmark with [agents of its own](standard.md#a-benchmarks-own-agents) (OSWorld's) lists them first, in place of its Original tile; several may be chosen, a job each, and a setting for one of them is in its tab. The cards are the Agents page's, each greyed with why it cannot run, and show as cards or a table. *Fit* starts on *Can run it*; the Filter menu (Protocol, From, Proves) narrows them further, and a chosen agent stays in view under any filter. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | From f0707fab95afd262711ce1f8eafa76486fe84373 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 14:01:14 +0100 Subject: [PATCH 7/8] fix(web): keep a chosen pick card's tags and marks filled A chosen pick card takes --accent-soft as its background, so what is filled with it inside the card (a benchmark's category tag, an agent's initials) lost its fill, and in the dark theme an outlined tag (an agent's protocol) lost its outline. Inside a chosen card both are now a shade nearer the accent, so they stand out as on any card, on every pick card, in both themes. Unchosen cards do not change. --- app/static/catalogue/catalogue.css | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/static/catalogue/catalogue.css b/app/static/catalogue/catalogue.css index fe6c417d..3f918d95 100644 --- a/app/static/catalogue/catalogue.css +++ b/app/static/catalogue/catalogue.css @@ -32,7 +32,11 @@ button.card-link:disabled { cursor: not-allowed; } .table-wrap .grid tbody tr:has([data-pick]:disabled) > td { color: var(--text-faint); } tr:has([data-pick]:disabled) .row-name > :first-child { opacity: .55; } tr:has([data-pick]:not(:disabled)) { cursor: pointer; } -.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) { border-color: var(--accent); background: var(--accent-soft); } +.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) { border-color: var(--accent); background: var(--accent-soft); + --pick-fill: color-mix(in srgb, var(--accent) 16%, var(--accent-soft)); --pick-line: color-mix(in srgb, var(--accent) 40%, var(--border)); } +/* inside it, what is filled with --accent-soft (a category tag, a mark) or outlined with --border (a tag) stands out from + that background as it does on any card: a shade nearer the accent */ +.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) > * { --accent-soft: var(--pick-fill); --border: var(--pick-line); } .pick-check { flex: none; width: 20px; height: 20px; border-radius: 50%; display: grid; place-items: center; background: var(--accent); color: var(--btn-fg); visibility: hidden; } .pick-check .ic { width: 12px; height: 12px; stroke-width: 3; } .bench-card:has([data-pick][aria-pressed="true"]) .pick-check { visibility: visible; } From 6d1ef7b1b0f079d8f4d326658d51ebfbf212daf2 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 14:01:20 +0100 Subject: [PATCH 8/8] test(catalogue): pin Benchmark.versions in the field contract #261 added the versions field to the catalogue's Benchmark without refreshing the pin; this records it, so its removal fails the check. --- app/tests/field_contract.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tests/field_contract.json b/app/tests/field_contract.json index 9da9cce2..3fa0b57a 100644 --- a/app/tests/field_contract.json +++ b/app/tests/field_contract.json @@ -80,7 +80,8 @@ "timeouts", "title", "upstream_limitations", - "usable_routes" + "usable_routes", + "versions" ], "Catalog": [ "_folders",