diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index ba289628..7a88f170 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -108,18 +108,42 @@ def _shown(o: dict, v: str, values: dict) -> str: return (o.get("names") or {}).get(v) or v +def _capability_kind(b: dict) -> str: + """The kind of capability a benchmark runs (``memory``), read from the ports it needs; empty for an agent benchmark.""" + kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b.get("family") == "capability" else [] + return kinds[0] if kinds else "" + + +def benchmarks(cat) -> List[dict]: + """The benchmarks the Benchmarks page and the builder's Benchmark step list: each of the catalogue's listed + entries (``Catalog.listed_benchmarks``, whichever runner runs it) as ``b``, with its ``datasets``, its ``tasks`` + and their ``unit`` (``Catalog.tasks_of``), what some of its options need of a machine (``host``, None for nothing), + what it is run with: ``family`` agent or capability (a benchmark only Harbor runs runs agents) and, for a + capability, its ``kind``; and ``versions``, another version's tasks and unit by version.""" + defs = {d["key"]: d for d in definitions.load("benchmark")[0]} + out = [] + for b in cat.listed_benchmarks: + d = defs.get(b.key) or {} + tasks, unit = cat.tasks_of(b) + out.append({"b": b, "datasets": cat.datasets_of(b), "tasks": tasks, "unit": unit, + "host": sorted(d["host"]) if d.get("host") else None, + "family": d.get("family") or "agent", "kind": _capability_kind(d), + # another version's tasks, when its datasets count differently (``Catalog.in_version``) + "versions": {v: cat.tasks_of(b, cat.in_version(b, v)[1]) for v in b.versions}}) + return out + + def _bench_page(b: dict) -> dict: """A benchmark as a page shows it. ``controls``: the ones it declares, each with who wrote it and where it comes from - what the builder offers beside the agents or capabilities. Each dataset carries the same controls, each with its ``proven`` record on that dataset (None: not proven), and ``proofs``, the units and leak checks the same way (``prove.status``).""" - kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b["family"] == "capability" else [] status = prove.status(b) controls = [{**{k: c[k] for k in ("key", "label", "source", "from", "expect")}, "must": prove.must_score(c["expect"])} for c in (definitions.control(b, w) for w in definitions.CONTROLS) if c] versioned = definitions.version_of(b) is not None - return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": kinds[0] if kinds else "", + return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": _capability_kind(b), "ref": definitions.ref_of(b) if "_version" in b else b["key"], "version": b["_version"] if versioned else "", # its versions, each by the ref a recipe names it with (the bare key: the default) diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index b54dcbc7..b294b73a 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -146,19 +146,7 @@ /* Consent to run without a sandbox. */ #nullius-builder .nb-consent { margin-top: 16px; padding: 12px 14px; border: 1px solid var(--nb-line); border-radius: 8px; background: var(--nb-surface); } #nullius-builder .nb-consent > span { display: flex; align-items: center; gap: 9px; color: var(--nb-text); font-size: 13px; } -/* The benchmark and agent steps: a search and filter chips above a grid of tiles, as the Benchmarks page draws its - cards. A tile: its mark and name, its tags, one line on what it is, then its facts; chosen, it takes the accent. */ -#nullius-builder .nb-pick { display: grid; gap: 16px; } -#nullius-builder .nb-pick-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; } -#nullius-builder .nb-pick-find { position: relative; flex: 1 1 240px; max-width: 360px; display: flex; align-items: center; } -#nullius-builder .nb-pick-find svg { position: absolute; left: 11px; width: 15px; height: 15px; color: var(--nb-muted); pointer-events: none; } -#nullius-builder .nb-pick-find input { width: 100%; min-height: 38px; padding: 8px 12px 8px 33px; font: inherit; font-size: 13px; color: var(--nb-text); background: var(--nb-surface); border: 1px solid var(--nb-line); border-radius: 8px; } -#nullius-builder .nb-pick-find input:focus { outline: none; border-color: var(--nb-blue); box-shadow: inset 0 0 0 1px var(--nb-blue); } -#nullius-builder .nb-pick-chips { display: flex; flex-wrap: wrap; gap: 6px; } -#nullius-builder .nb-pick-chips + .nb-pick-chips { padding-left: 14px; border-left: 1px solid var(--nb-line); } -#nullius-builder .nb-pick-chip { font: inherit; font-size: 12px; color: var(--nb-muted); background: transparent; border: 1px solid var(--nb-line); border-radius: 999px; padding: 6px 13px; min-height: 32px; cursor: pointer; } -#nullius-builder .nb-pick-chip:hover { color: var(--nb-text); border-color: var(--nb-muted); } -#nullius-builder .nb-pick-chip[aria-pressed=true] { color: var(--nb-bg); background: var(--nb-blue); border-color: var(--nb-blue); } +/* A choice as a tile: its mark and name, one line on what it is, then its facts; chosen, it takes the accent. */ #nullius-builder .nb-pick-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); gap: 12px; } #nullius-builder .nb-tile { position: relative; display: flex; flex-direction: column; gap: 9px; min-width: 0; text-align: left; padding: 15px 16px; border-radius: 10px; background: var(--nb-surface); border: 1px solid var(--nb-line); color: var(--nb-text); cursor: pointer; transition: border-color .12s ease, background .12s ease; } #nullius-builder .nb-tile[hidden] { display: none; } @@ -172,10 +160,6 @@ #nullius-builder .nb-tile-head strong { flex: 1; min-width: 0; font-size: 14.5px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } #nullius-builder .nb-tile-check { flex: none; width: 20px; height: 20px; border-radius: 50%; display: grid; place-items: center; font-size: 11px; color: var(--nb-bg); } #nullius-builder .nb-tile[aria-pressed=true] .nb-tile-check { background: var(--nb-blue); } -#nullius-builder .nb-tile-tags { display: flex; flex-wrap: wrap; gap: 5px; } -#nullius-builder .nb-tile-tags > span { font-size: 11px; font-weight: 500; letter-spacing: .02em; padding: 1px 7px; border-radius: 4px; color: var(--nb-muted); background: var(--nb-soft); } -#nullius-builder .nb-tile-tags .nb-tile-cat { color: var(--nb-blue); background: var(--nb-tint); } -#nullius-builder .nb-tile-tags .nb-tile-community { color: var(--origin-community, #b8741a); background: color-mix(in srgb, var(--origin-community, #b8741a) 16%, transparent); } #nullius-builder .nb-tile-line { font-size: 12.5px; line-height: 1.5; color: var(--nb-muted); display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } #nullius-builder .nb-tile-note { font-size: 11.5px; color: var(--nb-warn); } #nullius-builder .nb-tile-foot { margin-top: auto; padding-top: 9px; border-top: 1px solid var(--nb-line); display: flex; flex-wrap: wrap; align-items: center; gap: 5px 8px; font-size: 11.5px; color: var(--nb-muted); } @@ -185,8 +169,6 @@ #nullius-builder .nb-tick.ok { color: var(--nb-ok); } #nullius-builder .nb-tick.no { color: var(--nb-warn); } #nullius-builder .nb-tile-unfit { opacity: .5; cursor: not-allowed; background: transparent; } -#nullius-builder .nb-pick-none { color: var(--nb-muted); font-size: 13px; margin: 0; } -@media (max-width: 640px) { #nullius-builder .nb-pick-chips + .nb-pick-chips { padding-left: 0; border-left: 0; } #nullius-builder .nb-pick-find { max-width: none; } #nullius-builder .nb-pick-find input { font-size: 16px; } } /* The Where step's machine setup: each chosen machine, its state for the search's requirements, check and install. */ #nullius-builder .std-hostreq { padding: 16px; border: 1px solid var(--nb-line); border-radius: 8px; font-size: 13px; } #nullius-builder .std-hostreq-row { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } @@ -294,13 +276,13 @@ #nullius-builder .std-isolation small:empty { display: none; } #nullius-builder .std-boat-sizes .nb-tile { min-height: 60px; } #nullius-builder .std-boat-sizes .nb-tile-note { font-size: 13px; line-height: 1.4; } -/* the Agent step's grid: the card toolkit's (catalogue.css), fitted to the form */ +/* the Benchmark and Agent steps' grids: the card toolkit's (catalogue.css), fitted to the form */ #nullius-builder .nb-step { border-radius: 0; margin: 0; } /* catalogue.css has a .nb-step of its own */ -#nullius-builder #nb-agent-grid { margin-bottom: 20px; } -#nullius-builder #nb-agent-grid .bench-grid { grid-template-columns: repeat(auto-fill, minmax(min(100%, 230px), 1fr)); gap: 12px; grid-auto-rows: 1fr; } -#nullius-builder #nb-agent-grid .bench-card { padding: 16px; gap: 10px; } -#nullius-builder #nb-agent-grid .bench-card-title { font-size: 15px; } -#nullius-builder #nb-agent-grid .std-tag { white-space: nowrap; } -#nullius-builder #nb-agent-grid .card-link:focus-visible { outline: none; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) { margin-bottom: 20px; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .bench-grid { grid-template-columns: repeat(auto-fill, minmax(min(100%, 230px), 1fr)); gap: 12px; grid-auto-rows: 1fr; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .bench-card { padding: 16px; gap: 10px; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .bench-card-title { font-size: 15px; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .std-tag { white-space: nowrap; } +#nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .card-link:focus-visible { outline: none; } /* its outlined pills (Protocol, Proven) want more air than snug's text beside text; a phone keeps snug's, for the pinned name */ -@media (min-width: 641px) { #nullius-builder #nb-agent-grid .grid.snug :is(th, td):not(.pick-col) { padding-left: .6rem; padding-right: .6rem; } } /* the card's ring (catalogue.css), not a box round the name */ +@media (min-width: 641px) { #nullius-builder :is(#nb-bench-grid, #nb-agent-grid) .grid.snug :is(th, td):not(.pick-col) { padding-left: .6rem; padding-right: .6rem; } } /* the card's ring (catalogue.css), not a box round the name */ diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index d174a40c..12f16f82 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -215,20 +215,13 @@ }; // the version a standard recipe names (``key@version``), '' for the default const versionOf = rec => C.isStandard(rec) && String(rec.benchmark || '').includes('@') ? String(rec.benchmark).split('@')[1] : ''; - // Every benchmark of both runners in one list; picking one sets the runner that runs it (Nullius when both do). + // Every benchmark of both runners in one list, the Benchmarks page's (placeGrid, /new-run/benchmarks); picking one + // sets the runner that runs it (Nullius when both do). const benchmarkStep = rec => { - const st = C.standard(), sb = C.standardBase(rec), on = C.benchKey(rec), seen = new Set(), tiles = []; - const add = (key, title, fam, kind, summary) => { if (seen.has(key)) return; seen.add(key); const c = catOf(key); - tiles.push({attr: 'bench', key, on: key === on, title: c.title || title, - mark: ``, cat: c.category, origin: c.origin, - fam, famLabel: fam === 'agent' ? 'Agents' : 'Capabilities · ' + kind, tip: c.domain || summary || '', meta: tasksOf(key, key === on ? versionOf(rec) : ''), - find: [c.title || title, key, c.category, c.domain, kind].join(' ')}); }; - st.benchmarks.forEach(x => add(x.key, x.label, x.family, x.kind, x.summary)); - C.benchmarks().forEach(x => add(x.key, x.title, 'agent', '', x.domain)); - tiles.sort((a, b) => a.title.localeCompare(b.title, 'en', {sensitivity: 'base'})); + const sb = C.standardBase(rec); const versions = C.isStandard(rec) && sb && (sb.versions || []).length > 1 ? `` : ''; - return `

Choose a benchmark.

` + picker(tiles, [['agent', 'Agents'], ['capability', 'Capabilities']]) + versions; + return `

Choose a benchmark.

` + versions; }; // Put the recipe on a runner, keeping the one each runner had: the benchmark follows. const toRunner = (target, key) => { @@ -628,94 +621,78 @@ return `

${state ? `Replace the expired ${what}` : `Add ${provider ? 'a ' : ''}${what}`} in ${settings}${required ? '' : ' (some tasks may need it)'}.

`; }).join(''); } - // The benchmark step, both ways (Harbor's catalogue, the standard's benchmarks): a search, filter chips and a grid - // of tiles, as the Benchmarks page draws them. The filter hides tiles in place (applyPick), so typing keeps focus. - const picks = {}, pickOf = () => (picks[step] ||= {q: '', cat: '', fam: ''}); // each step its own search and filters - const catOf = key => (schema.benchmarks || []).find(b => b.key === key) || {}; - // its tasks; `version`: a version of the standard's that counts its datasets apart (the catalogue entry's ``versions``) - const tasksOf = (key, version) => { const total = catOf(key).task_count, own = ((catOf(key).versions || {})[version] || {}).datasets || {}; - if (Number.isInteger(total) && total > 0) return `${total.toLocaleString('en-US')} tasks`; - const t = schema.datasets.filter(d => d.benchmark === key && d.tasks).map(d => (own[d.key] || {}).tasks || d.tasks); - return !t.length ? '' : t.every(n => n === t[0]) && t.length > 1 ? `${t[0]} tasks each` : `${t.reduce((a, n) => a + n, 0)} tasks`; }; - const pickChip = (attr, v, label, on) => ``; - // one tile: {attr, key, on, title, mark, cat, origin, line, meta, foot, find, fam, data (more attributes), tip (its title)} - const tile = t => `'; // a machine to run on: this one or a host, its connection state filled in by Test connection const icon = name => ``; const machineTile = (attr, v, t, sub, on) => tile({attr, key: v, on, mark: icon(v === 'local' ? 'machine' : 'network'), title: t, tip: sub, foot: v !== 'local' ? `` : ''}); - const picker = (tiles, fams, noun = 'benchmark') => { - const pick = pickOf(), cats = [...new Set(tiles.map(t => t.cat).filter(Boolean))].sort(); - return `
` - + `` - + (cats.length > 1 ? `
${pickChip('pick-cat', '', 'All', !pick.cat)}${cats.map(c => pickChip('pick-cat', c, c, pick.cat === c)).join('')}
` : '') - + (fams ? `
${pickChip('pick-fam', '', 'Any', !pick.fam)}${fams.map(([v, l]) => pickChip('pick-fam', v, l, pick.fam === v)).join('')}
` : '') - + `
${tiles.map(tile).join('')}
`; - }; - function applyPick() { - const box = root.querySelector('.nb-pick'), pick = pickOf(); if (!box) return; - const words = pick.q.toLowerCase().split(/\s+/).filter(Boolean); - let shown = 0; - box.querySelectorAll('.nb-tile').forEach(t => { - const on = t.getAttribute('aria-pressed') === 'true'; - const fits = (!pick.cat || t.dataset.cat === pick.cat) && (!pick.fam || t.dataset.fam === pick.fam) && words.every(w => t.dataset.find.includes(w)); - t.hidden = !fits && !on; if (!t.hidden) shown++; - }); - box.querySelectorAll('[data-pick-cat]').forEach(b => b.setAttribute('aria-pressed', b.dataset.pickCat === pick.cat)); - box.querySelectorAll('[data-pick-fam]').forEach(b => b.setAttribute('aria-pressed', b.dataset.pickFam === pick.fam)); - box.querySelector('.nb-pick-none').hidden = shown > 0; - } - // The standard's Agent step grid: drawn by the server for the benchmark (web/builder.py, /new-run/agents) from the - // Agents page's entries, run by the card toolkit (tables.js); kept while the benchmark is the same, so a search or a - // filter survives each render, and its cards pressed as the recipe chooses (nullius-pick changes the recipe). - // Its view, cards or a table, is the viewer's: the toggle switches it in place, never leaving the page. - let agentView = 'cards'; - try { agentView = localStorage.getItem('nb-agent-view') === 'table' ? 'table' : 'cards'; } catch {} - const grid = {ref: '', node: null}; - function placeAgentGrid() { - const slot = $('#nb-agent-grid'), rec = r(); + // The grids drawn by the server and run by the card toolkit (tables.js): the Benchmark step's (/new-run/benchmarks, + // one choice) and the standard's Agent step's for its benchmark (/new-run/agents, from the Agents page's entries). + // Each is kept while what it draws is the same, so a search or a filter survives each render, and its cards are + // pressed as the recipe chooses (nullius-pick changes the recipe). Its view, cards or a table, is the viewer's: + // the toggle switches it in place, never leaving the page. + const grids = [ + {slot: 'nb-bench-grid', store: 'nb-bench-view', url: () => '/new-run/benchmarks?', chosen: rec => [C.benchKey(rec)], after: benchTasks}, + {slot: 'nb-agent-grid', store: 'nb-agent-view', url: rec => '/new-run/agents?' + new URLSearchParams({benchmark: rec.benchmark}) + '&', chosen: rec => C.chosenOf(rec)}, + ]; + grids.forEach(g => { g.view = 'cards'; g.ref = ''; g.node = null; try { g.view = localStorage.getItem(g.store) === 'table' ? 'table' : 'cards'; } catch {} }); + const placeGrids = () => grids.forEach(placeGrid); + function placeGrid(g) { + const slot = $('#' + g.slot), rec = r(); if (!slot) return; - if (grid.ref !== rec.benchmark + ' ' + agentView) { grid.ref = rec.benchmark + ' ' + agentView; grid.node = null; loadAgentGrid(grid.ref); } - if (!grid.node) { slot.setAttribute('aria-busy', 'true'); slot.innerHTML = ''; return; } - slot.replaceWith(grid.node); - const chosen = C.chosenOf(rec); - grid.node.querySelectorAll('[data-pick]').forEach(b => { // a card's toggle, or a row's box + const ref = g.url(rec) + (g.view === 'table' ? 'view=table' : ''); + if (g.ref !== ref) { g.ref = ref; g.node = null; loadGrid(g, ref); } + if (!g.node) { slot.setAttribute('aria-busy', 'true'); slot.innerHTML = ''; return; } + slot.replaceWith(g.node); + const chosen = g.chosen(rec); + g.node.querySelectorAll('[data-pick]').forEach(b => { // a card's toggle, or a row's box const on = !b.disabled && chosen.includes(b.dataset.pick); - if (b.type === 'checkbox') b.checked = on; else b.setAttribute('aria-pressed', String(on)); + if (b.type === 'checkbox' || b.type === 'radio') b.checked = on; else b.setAttribute('aria-pressed', String(on)); }); // a choice made elsewhere (a recipe loaded) that the filters hide: the toolkit draws again, and it shows - if (grid.node.dataset.wired && [...grid.node.querySelectorAll('[data-pick]')].some(b => b.matches('[aria-pressed="true"], :checked') && b.closest('[data-card], tr').hidden)) - grid.node.querySelector('[data-search]')?.dispatchEvent(new Event('input')); - if (!grid.node.dataset.wired) { - grid.node.dataset.wired = '1'; - const cards = grid.node.querySelector('[data-cards]'), table = grid.node.querySelector('table[data-table]'); + if (g.node.dataset.wired && [...g.node.querySelectorAll('[data-pick]')].some(b => b.matches('[aria-pressed="true"], :checked') && b.closest('[data-card], tr').hidden)) + g.node.querySelector('[data-search]')?.dispatchEvent(new Event('input')); + g.after?.(); + if (!g.node.dataset.wired) { + g.node.dataset.wired = '1'; + const cards = g.node.querySelector('[data-cards]'), table = g.node.querySelector('table[data-table]'); if (cards) NulliusTable.cards(cards); if (table) { NulliusTable.sticky(table); NulliusTable.wire(table); } - grid.node.querySelectorAll('.seg-toggle a').forEach(a => a.addEventListener('click', e => { + g.node.querySelectorAll('.seg-toggle a').forEach(a => a.addEventListener('click', e => { e.preventDefault(); - agentView = new URL(a.href).searchParams.get('view') === 'table' ? 'table' : 'cards'; - try { localStorage.setItem('nb-agent-view', agentView); } catch {} - placeAgentGrid(); + g.view = new URL(a.href).searchParams.get('view') === 'table' ? 'table' : 'cards'; + try { localStorage.setItem(g.store, g.view); } catch {} + placeGrid(g); })); } } - async function loadAgentGrid(ref) { - const [benchmark, view] = ref.split(' '); - const res = await nulliusFetch('/new-run/agents?' + new URLSearchParams({benchmark, ...(view === 'table' ? {view} : {})})).catch(() => null); + // the Tasks of a benchmark with versions (its data-tasks, {version: {text, label}}, the default's under ''): the + // chosen benchmark's as the version the recipe names reads, any other's its default's + function benchTasks() { + const rec = r(), on = C.benchKey(rec), v = versionOf(rec); + root.querySelectorAll('#nb-bench-grid [data-tasks]').forEach(el => { + const by = JSON.parse(el.dataset.tasks), t = by[el.querySelector('[data-pick]')?.dataset.pick === on && by[v] ? v : '']; + const dd = el.querySelector('.bench-stats dd'), dt = el.querySelector('.bench-stats dt'), td = el.querySelector('td[data-col="tasks"] > span'); + if (dd) { dd.textContent = t.text; dt.textContent = t.label; } + if (td) td.textContent = t.text; + }); + } + async function loadGrid(g, ref) { + const res = await nulliusFetch(ref).catch(() => null); const html = res && res.ok ? await res.text().catch(() => null) : null; - if (grid.ref !== ref) return; // another benchmark was chosen while it loaded - if (html === null) { // said once; the next render asks again - grid.ref = ''; - const slot = $('#nb-agent-grid'); + if (g.ref !== ref) return; // another benchmark or view was chosen while it loaded + if (html === null) { // said once; the next render asks again + g.ref = ''; + const slot = $('#' + g.slot); if (slot) { slot.removeAttribute('aria-busy'); slot.innerHTML = '

Could not load the list. Change step and come back to try again.

'; } return; } - grid.node = Object.assign(document.createElement('div'), {id: 'nb-agent-grid', innerHTML: html}); - placeAgentGrid(); + g.node = Object.assign(document.createElement('div'), {id: g.slot, innerHTML: html}); + placeGrid(g); } function modelChoiceCards(rec) { const hasPaper = C.paperOffered(rec), on = rec.model_choice || 'any'; @@ -962,7 +939,7 @@ const rows=own?[...((runnerOut&&runnerOut.review)||[]),...(C.isStandard(rec)?[]:[[S.where,'Where',C.whereLabel(rec)]])]:C.review(rec); html='

Review the experiment.

'+rows.map(([i,t,v])=>`
${t}${esc(v)}
${button('Edit',`data-step="${i}"`)}
`).join('')+'
'+(schema.local?'
':'')+runPanel(); } - $('#nb-form').innerHTML=html+'

';applyPick();placeAgentGrid(); + $('#nb-form').innerHTML=html+'

';placeGrids(); if(step===REVIEW&&schema.local){drawPrereq();checkPrereq();} $('#nb-back').hidden=step===S.benchmark; $('#nb-next').textContent=step===REVIEW?'Export recipe JSON':stepName(steps[nextShown(step,1)],nextShown(step,1))+' →'; @@ -1124,8 +1101,6 @@ if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check',d.hostreqName);return;} if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install',d.hostreqName);return;} if(d.copyCode!==undefined){const t=b.closest('.std-code').querySelector('code').textContent;try{await navigator.clipboard.writeText(t);b.textContent='Copied';}catch{b.textContent='Select it';}setTimeout(()=>{b.textContent='Copy';},1500);return;} - if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;} - if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;} if(d.agent){const on=r().agents.includes(d.agent);r().agents=on?r().agents.filter(id=>id!==d.agent):[...r().agents,d.agent];renderForm();refreshPlan();return;} else if(d.href){if(!window.nulliusOpenSettings?.(d.href,b))location.href=d.href;return;} else if(b.id==='nb-add-custom-agent'){r().customAgents.push('');r().agents.push('custom-'+(r().customAgents.length-1));renderForm();refreshPlan();root.querySelector('[data-custom-agent="'+(r().customAgents.length-1)+'"]')?.focus();return;} @@ -1136,7 +1111,6 @@ else if(d.harborScan!==undefined){askScan(true);return;} else if(d.harborRefsFresh!==undefined){loadRefs(C.harborRepoUrl(r().harbor.repo),true);return;} else if(d.runner){toRunner(d.runner,C.benchKey(r()));} - else if(d.bench){toRunner(C.runnerFor(r(),d.bench),d.bench);} else if(d.stdBench){C.setStandardBenchmark(r(),d.stdBench);} else if(d.stdOther){const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(d.stdOther)?c.filter(k=>k!==d.stdOther):[...c,d.stdOther];delete rec.other;if(rec.options)delete rec.options[d.stdOther];} // a check else if(d.stdDataset){const rec=r(),c=C.datasetsOf(rec);rec.datasets=c.includes(d.stdDataset)?(c.length>1?c.filter(k=>k!==d.stdDataset):c):[...c,d.stdDataset];delete rec.dataset;} @@ -1177,12 +1151,12 @@ else return; render(d.step!==undefined||['nb-next','nb-back'].includes(b.id)); }); - // an agent, a capability or the setup picked or let go on the Agent step's grid; the focus stays on its card - root.addEventListener('nullius-pick',event=>{const k=event.detail.value,rec=r(),c=C.chosenOf(rec);rec.others=c.includes(k)?c.filter(x=>x!==k):[...c,k];delete rec.other;if(rec.options)delete rec.options[k];inPlace(()=>render());event.target.focus({preventScroll:true});}); + // a benchmark chosen on the Benchmark step's grid, or an agent, a capability or the setup picked or let go on the + // Agent step's; the page stays where it is and the focus on its card + root.addEventListener('nullius-pick',event=>{const k=event.detail.value;if(event.target.closest('#nb-bench-grid')){toRunner(C.runnerFor(r(),k),k);}else{const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(k)?c.filter(x=>x!==k):[...c,k];delete rec.other;if(rec.options)delete rec.options[k];}inPlace(()=>render());event.target.focus({preventScroll:true});}); root.addEventListener('input',event=>{ const el=event.target,d=el.dataset,rec=r(); if(el.id==='nb-recipe-title'){updateSavePreview();return;} - if(d.pickFind!==undefined){pickOf().q=el.value;applyPick();return;} if(d.servedBy!==undefined)return; // what is typed in a Served by filters its menu: a pick, or Enter, adds it (nb-pick) if(d.providerModel && rec.modelFormat !== 'provider-id'){for(const key of ['model','userModel','evalModel'])if(typeof rec[key]==='string')rec[key]=C.bareModel(rec[key]);rec.modelFormat='provider-id';} if(d.field){const p=d.field.startsWith('param:')?C.parameters().find(p=>p.key===d.field.slice(6)):null; @@ -1219,8 +1193,7 @@ if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} else if(d.stdVersion!==undefined){ // in place: only the card's count changes on this step, so nothing else is drawn again - C.setStandardBenchmark(r(),el.value);const key=C.benchKey(r()),meta=root.querySelector(`[data-bench="${CSS.escape(key)}"] .nb-tile-foot > span`); - if(meta)meta.textContent=tasksOf(key,versionOf(r()));render(false,false);} + C.setStandardBenchmark(r(),el.value);benchTasks();render(false,false);} // an option at its default is left out: the recipe names only what was changed. A select's value can show or hide // the options that apply under it (`when`): the step is drawn again and the focus goes back to the select, or to the // menu button controls.js puts after it once this change is handled; a typed value is left as it is diff --git a/app/static/catalogue/catalogue.css b/app/static/catalogue/catalogue.css index c758a460..3f918d95 100644 --- a/app/static/catalogue/catalogue.css +++ b/app/static/catalogue/catalogue.css @@ -32,7 +32,11 @@ button.card-link:disabled { cursor: not-allowed; } .table-wrap .grid tbody tr:has([data-pick]:disabled) > td { color: var(--text-faint); } tr:has([data-pick]:disabled) .row-name > :first-child { opacity: .55; } tr:has([data-pick]:not(:disabled)) { cursor: pointer; } -.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) { border-color: var(--accent); background: var(--accent-soft); } +.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) { border-color: var(--accent); background: var(--accent-soft); + --pick-fill: color-mix(in srgb, var(--accent) 16%, var(--accent-soft)); --pick-line: color-mix(in srgb, var(--accent) 40%, var(--border)); } +/* inside it, what is filled with --accent-soft (a category tag, a mark) or outlined with --border (a tag) stands out from + that background as it does on any card: a shade nearer the accent */ +.bench-card:has(> .bench-card-title [data-pick][aria-pressed="true"]) > * { --accent-soft: var(--pick-fill); --border: var(--pick-line); } .pick-check { flex: none; width: 20px; height: 20px; border-radius: 50%; display: grid; place-items: center; background: var(--accent); color: var(--btn-fg); visibility: hidden; } .pick-check .ic { width: 12px; height: 12px; stroke-width: 3; } .bench-card:has([data-pick][aria-pressed="true"]) .pick-check { visibility: visible; } @@ -59,7 +63,7 @@ tr:has([data-pick]:not(:disabled)) { cursor: pointer; } .bench-card-name > .icon-btn { vertical-align: middle; } .bench-mark { flex: none; width: 22px; height: 22px; color: var(--accent); } /* a table row's name (_cards.html rows): its icon, then its link */ -.row-name { display: inline-flex; align-items: center; gap: 8px; } +.row-name { display: inline-flex; align-items: center; gap: 8px; overflow-wrap: break-word; } /* a name wraps between its words, never inside one: an expand column breaks anywhere, the table scrolls instead */ .bench-card-domain { margin-top: -6px; font-size: 14px; line-height: 1.5; color: var(--text-dim); overflow-wrap: anywhere; } /* a long file name breaks rather than spills */ /* the stats in columns of one width, a column each, each its label then its value; on a card too narrow for one line (a phone), two balanced rows: 4 as 2+2, 5 as 3+2, 6 as 3+3 */ diff --git a/app/static/tables.js b/app/static/tables.js index 586fea11..53ac9231 100644 --- a/app/static/tables.js +++ b/app/static/tables.js @@ -285,7 +285,8 @@ window.NulliusTable = { }, /* The pick toggles under `root`, buttons (data-pick, aria-pressed) or - checkboxes (data-pick): a click, Space or Enter flips one and sends nullius-pick {value, pressed} from it; with + checkboxes (data-pick): a click, Space or Enter flips one and sends nullius-pick {value, pressed} from it. A single + choice, a button with data-pick-one or a radio, is pressed again rather than let go, and lets the others go; with `row`, a click elsewhere in the row (a selector) that holds one, but on a control of its own or at the end of a text selection, clicks it. */ picks(root, row) { @@ -296,8 +297,10 @@ window.NulliusTable = { } const b = e.target.closest('[data-pick]'); if (!b || b.disabled) return; - const pressed = b.type === 'checkbox' ? b.checked : b.getAttribute('aria-pressed') !== 'true'; // a box has ticked itself - if (b.type !== 'checkbox') b.setAttribute('aria-pressed', String(pressed)); + const box = b.type === 'checkbox' || b.type === 'radio', one = b.dataset.pickOne !== undefined; + const pressed = box ? b.checked : one || b.getAttribute('aria-pressed') !== 'true'; // a box has ticked itself + if (one) root.querySelectorAll('[data-pick-one][aria-pressed="true"]').forEach(x => x.setAttribute('aria-pressed', 'false')); + if (!box) b.setAttribute('aria-pressed', String(pressed)); b.dispatchEvent(new CustomEvent('nullius-pick', {bubbles: true, detail: {value: b.dataset.pick, pressed}})); }); }, diff --git a/app/templates/_cards.html b/app/templates/_cards.html index c695d4e9..662810ed 100644 --- a/app/templates/_cards.html +++ b/app/templates/_cards.html @@ -8,10 +8,11 @@ An item is a dict of display values by field key: a string, a dict {text, class, title, label, href, value, attrs} or a list of them, empty for none; a filter and Sort by read a value's `value`, else its text; an action's `attrs` are its button's own attributes, its `class` added to the button's, and a `delete` takes the same shape (a quiet - trash button). Plus `href` (its title's link), or `pick` {value, on, disabled}: the card is a toggle, its title a + trash button). Plus `href` (its title's link), or `pick` {value, on, disabled, one}: the card is a toggle, its title a button[data-pick][aria-pressed] that tables.js flips, sending `nullius-pick` {value, pressed} (a disabled one's - reason goes in its `alert`), and an enabled one's corner holds the check that marks it chosen; a title's `title` is the tooltip of its link or button; - `id`, `search` (what the search reads), `keys`, and `unless` {field: other}: the item passes the field's filter + reason goes in its `alert`), and an enabled one's corner holds the check that marks it chosen; `one`, a single + choice: choosing it lets the others go, choosing it again keeps it (a radio in a table); a title's `title` is the tooltip of its link or button; + `id`, `search` (what the search reads), `keys`, `data` {name: value} (data- attributes, on its row too), and `unless` {field: other}: the item passes the field's filter whenever the other field's filter narrows and passes it. A title's `after` is markup in the flow of its name, after its last word (a control of its own), and its `class` goes on the title row. A field's `class` goes on each of its values; `card_label` names a stat when its column header does not, and a @@ -52,19 +53,19 @@ {# The items of a cards collection as a table (the table toolkit's, with its Columns menu): a column per field the table takes that some item fills (a filtered or 'num' one always), in the order of `cols`; the title pinned, its - icon before it and a link to `href`, the others as on the card; picks put a checkbox in a leftmost column of no - name (its header no tick-all: a pick is one at a time), the name then plain. A field `off` starts hidden. #} + icon before it and a link to `href`, the others as on the card; picks put a checkbox (a radio for `one`) in a leftmost + column of no name (its header no tick-all: a pick is one at a time), the name then plain. A field `off` starts hidden. #} {% macro rows(id, cols, items, search, choices, only, empty, noun, sort, default) %} {%- set icon = (cols|selectattr('card', 'equalto', 'icon')|list + [none])[0] %} {%- set icons = icon and items|selectattr(icon.key)|list|length > 0 %} {%- set shown = [dict(key='_pick', label='Chosen', blank=true, pick=true, sticky=true)] if items|selectattr('pick')|list else [] %}{% for c in cols if c.table is not sameas false and (c.key in choices or c.type == 'num' or items|selectattr(c.key)|list) %}{% set _ = shown.append(dict(c, sticky=c.card == 'title')) %}{% endfor %} -{%- set rs = [] %}{% for item in items %}{% set a %}{% if item.get('id') %} id="{{ item.id }}"{% endif %} data-search="{{ item.get('search', '') }}" +{%- set rs = [] %}{% for item in items %}{% set a %}{% if item.get('id') %} id="{{ item.id }}"{% endif %} data-search="{{ item.get('search', '') }}"{% for k, v in (item.get('data') or {}).items() %} data-{{ k }}="{{ v }}"{% endfor %} {%- for f in cols if f.key in choices %} data-f-{{ f.key }}="{% call(f, v) each(item, [f]) %}{{ v.get('value', v.text) }} {% endcall %}"{% endfor %}{% for k, by in (item.get('unless') or {}).items() %} data-unless-{{ k }}="{{ by }}"{% endfor %}{% endset %} {%- set _ = rs.append({'item': item, 'attrs': a}) %}{% endfor %} {% call(r, c) table(id, shown, rs, attrs, search=search, columns=true, choices=choices, only=only, empty=empty, noun=noun, sort=sort, layout='table', default=default) %} {%- set v = r.item.get(c.key) %}{% set by %}{% call(f, x) each(r.item, [c]) %}{{ x.get('value', x.text) }} {% endcall %}{% endset %} {%- set pick = r.item.get('pick') %}{% set t = r.item.get((cols|selectattr('card', 'equalto', 'title')|first).key) %}{% set t = t if t is mapping else {'text': t} %} - {%- if c.key == '_pick' %}{% call cell(c, class='pick-col') %}{% endcall %} + {%- if c.key == '_pick' %}{% call cell(c, class='pick-col') %}{% endcall %} {%- else %}{% call cell(c, sort=by|trim) %}{% if c.card == 'title' %}{% set name = t.text %}{% if icons %}{{ mark(icon, r.item.get(icon.key), name) }}{% endif %} {%- if pick %}{{ name }}{% else %}{{ name }}{% endif %} {%- elif c.style == 'code' %}{{ v }} @@ -84,14 +85,14 @@ {%- set icons = icon and (every or items)|selectattr(icon.key)|list|length > 0 %}
{%- for item in items %} -
{%- set title = by.title[0] %}{% set v = item.get(title.key) %}{% set v = v if v is mapping else {'text': v} %}{% set name = v.text %} {%- set cls = [title.class, v.class]|select|join(' ') %}
{% if icons %}{{ mark(icon, item.get(icon.key), name) }}{% endif -%} {%- set pick = item.get('pick') %} - {%- set link %}{% if pick %} + {%- set link %}{% if pick %} {%- else %}{{ name }}{% endif %}{% endset %} {%- if v.after %}{{ link }} {{ v.after }}{% else %}{{ link }}{% endif %} {%- set corner %}{% if pick and not pick.disabled %}{% endif %}{% call(f, v) each(item, by.check) %}{% endcall %} diff --git a/app/templates/base.html b/app/templates/base.html index eeadbdaa..77df93e9 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -72,6 +72,7 @@ + diff --git a/app/templates/builder/benchmarks.html b/app/templates/builder/benchmarks.html new file mode 100644 index 00000000..50d6d2f6 --- /dev/null +++ b/app/templates/builder/benchmarks.html @@ -0,0 +1,11 @@ +{#- The builder's Benchmark step grid (web/builder.py, new_run_benchmarks): the Benchmarks page's cards + (web/catalogue.py, benchmark_cards) as single-choice pick cards of the card toolkit (_cards.html), or its rows + (`layout` 'table'), loaded by command-builder.js, which keeps the choice and switches the view in place. -#} +{% from '_cards.html' import cards %} +{#- the category a row of pills; what it evaluates (agents or capabilities) a menu of its own, beside Filter (runner) -#} +{% set cols = [dict(key='icon', label='Mark', card='icon', table=false, class='bench-mark'), dict(key='title', label='Name', card='title', width='expand'), + dict(key='evaluates', label='Evaluates', card='badge', class='std-tag', filter='dropdown', filter_icon='target'), + dict(key='tasks', label='Tasks', card='stat', type='num'), + dict(key='category', label='Category', card='tag', class='cat-tag', filter='pills'), dict(key='origin', label='Origin', card='tag', class='origin-tag'), + dict(key='runs', label='Runner', class='run-tag'), dict(key='domain', label='Domain', off=true)] %} +{{ cards('nb-benchmarks', cols, items, search='Search benchmarks', choices=choices, sort=['title', 'tasks'], empty='No benchmark matches.', noun='benchmarks', layout=layout, default='cards') }} diff --git a/app/tests/field_contract.json b/app/tests/field_contract.json index 9da9cce2..3fa0b57a 100644 --- a/app/tests/field_contract.json +++ b/app/tests/field_contract.json @@ -80,7 +80,8 @@ "timeouts", "title", "upstream_limitations", - "usable_routes" + "usable_routes", + "versions" ], "Catalog": [ "_folders", diff --git a/app/tests/test_benchmark_listing.py b/app/tests/test_benchmark_listing.py new file mode 100644 index 00000000..3601a5c6 --- /dev/null +++ b/app/tests/test_benchmark_listing.py @@ -0,0 +1,34 @@ +"""The benchmarks the Benchmarks page and the builder's Benchmark step list (recipe.benchmarks): the catalogue's +listed entries, each with what it is run with.""" +from __future__ import annotations + +import unittest + +from nullius.catalogue import catalog +from nullius.standard import recipe + + +class Listing(unittest.TestCase): + def test_it_is_the_catalogues_listed_benchmarks_with_their_family(self): + cat = catalog.load(catalog.ROOT) + rows = recipe.benchmarks(cat) + self.assertEqual([r["b"].key for r in rows], [b.key for b in cat.listed_benchmarks]) + by = {r["b"].key: r for r in rows} + self.assertEqual((by["locomo"]["family"], by["locomo"]["kind"]), ("capability", "memory")) + self.assertEqual((by["gsm8k"]["family"], by["gsm8k"]["kind"]), ("agent", "")) + self.assertEqual((by["gsm8k"]["tasks"], by["gsm8k"]["unit"]), cat.tasks_of(cat.benchmark("gsm8k"))) + self.assertTrue(all(r["family"] in ("agent", "capability") for r in rows)) + + def test_a_benchmark_with_versions_counts_each_ones_tasks(self): + cat = catalog.load(catalog.ROOT) + os1 = next(r for r in recipe.benchmarks(cat) if r["b"].key == "osworld1") + b = cat.benchmark("osworld1") + self.assertEqual((os1["tasks"], os1["unit"]), cat.tasks_of(b)) # the default version + self.assertEqual(set(os1["versions"]), set(b.versions)) + for v in b.versions: + self.assertEqual(os1["versions"][v], cat.tasks_of(b, cat.in_version(b, v)[1])) + self.assertEqual(next(r for r in recipe.benchmarks(cat) if r["b"].key == "gsm8k")["versions"], {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_builder_benchmarks.py b/app/ui_tests/test_builder_benchmarks.py new file mode 100644 index 00000000..22a2f3dd --- /dev/null +++ b/app/ui_tests/test_builder_benchmarks.py @@ -0,0 +1,81 @@ +"""The builder's Benchmark step grid (/new-run/benchmarks): the Benchmarks page's cards as single-choice pick cards, +with what each benchmark evaluates, agents or capabilities, as a menu of its own.""" +import json +import os +import re +import shutil +import tempfile +import unittest +from html import unescape +from unittest.mock import patch + +from .test_routes import web + + +class BenchmarkGrid(unittest.TestCase): + def setUp(self): + tmp = tempfile.mkdtemp(prefix="nullius-bench-grid-") + self.addCleanup(shutil.rmtree, tmp, True) + env = patch.dict(os.environ, {"NULLIUS_HOME": tmp, "HOSTS": "", "RUNS_DIR": os.path.join(tmp, "runs")}) + env.start(); self.addCleanup(env.stop) + os.environ.pop("NULLIUS_CATALOGUE", None) + self.client = web.app.test_client() + + def get(self, url): + r = self.client.get(url) + self.assertEqual(r.status_code, 200, url) + return r.get_data(as_text=True) + + def test_it_lists_the_benchmarks_pages_benchmarks_each_a_single_choice(self): + html = self.get("/new-run/benchmarks") + cards = {m.group(1): m.group(0) for m in re.finditer(r'
', html, re.S)} + listed = re.findall(r'Capabilities', cards["locomo"]) + self.assertIn('Agents', cards["swe-bench-verified"]) # Harbor runs agents + self.assertIn('
Tasks
1,319
', cards["gsm8k"]) + self.assertIn('Math', cards["gsm8k"]) + self.assertNotIn("Official", html) # only a community one says so + self.assertIn('title="Real-world questions', cards["gaia"]) # its domain, as its tooltip + + def test_a_benchmark_with_versions_carries_each_ones_tasks(self): + html = self.get("/new-run/benchmarks") + card = re.search(r'
]*id="pick-bench-osworld1"[^>]*>', html).group(0) + tasks = json.loads(unescape(re.search(r'data-tasks="([^"]*)"', card).group(1))) + self.assertEqual(tasks[""]["text"], "368") # the default version's + self.assertEqual(tasks["044bc33"], {"text": "369", "label": "Tasks"}) + self.assertNotIn("data-tasks", re.search(r'
]*id="pick-bench-gsm8k"[^>]*>', html).group(0)) + self.assertRegex(self.get("/new-run/benchmarks?view=table"), r']*id="pick-bench-osworld1"[^>]*data-tasks=') # its row too + + def test_its_controls_category_pills_run_with_beside_filter_sort_and_the_toggle(self): + html = self.get("/new-run/benchmarks") + self.assertIn('data-pills="category"', html) + self.assertIn('data-dropdown aria-haspopup="true" aria-expanded="false">Evaluates<', html) + self.assertLess(html.index("data-dropdown"), html.index('data-tools-btn="filter"')) + menu = unescape(html[html.index('
")[0] + self.assertIn('data-value="agent" checked> Agents', menu) + self.assertIn('data-value="capability" checked> Capabilities', menu) + self.assertIn('data-filter-menu="runs"', html[html.index('data-tools="filter"'):]) + self.assertIn('data-tools-btn="sort"', html) + self.assertIn('href="/new-run/benchmarks?view=table" aria-label="Table"', html) + + def test_its_table_picks_one_with_radios(self): + html = self.get("/new-run/benchmarks?view=table") + self.assertIn('', html) + self.assertIn('', html) + self.assertNotIn('type="checkbox" data-pick', html) + self.assertIn('#nb-benchmarks > * > tr > :nth-child(2) { width: auto;', html) # Name takes the spare width + self.assertEqual(re.findall(r'
', self.get("/new-run")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index de9f575c..1f2f85b5 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -22,7 +22,7 @@ def draw(self, docker, edit="", backends=("local", "docker"), boat=None, slots=2 "boatSizes": self.SIZES, "boatSlots": slots}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" + line("esc") + line("button") + cut(" const unitsOf = ", " // the first clause") + line("words") - + cut(" const tile = t =>", " const picker = ") + + cut(" const tile = t =>", " // The grids drawn by the server") + cut(" const consentBox = ", " async function hostReq(") + cut(" function benchOptions(", " // Only credentials") + "const out={};for(const b of C.standard().benchmarks)for(const backend of " + json.dumps(list(backends)) + "){" + "const r=C.createStandard(b.key);r.backend=backend;" + edit @@ -143,7 +143,7 @@ def test_an_option_shows_its_example_and_a_setup_its_label(self): line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" schema = dict(self.schema, hosts=[], standardHere={"docker": None, "licences": {}, "host": {"osworld2": {"docker": ["missing"]}}}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" - + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + cut(" function benchOptions(", " // Only credentials") + + line("esc") + line("button") + cut(" const tile = t =>", " // The grids drawn by the server") + cut(" function benchOptions(", " // Only credentials") + "const out={};for(const k of ['osworld2','gsm8k']){const r=C.createStandard(k);out[k]=benchOptions(r,C.standardBench(r),'shared')+benchOptions(r,C.standardBench(r),'setup')+benchOptions(r,C.standardBench(r));}" + "const r=C.createStandard('osworld2'),b=C.standardBench(r);r.datasets=['offline36'];out.offline=benchOptions(r,b,'dataset');r.datasets=['test_v2'];out.all=benchOptions(r,b,'dataset');" + "process.stdout.write(JSON.stringify(out));") @@ -158,7 +158,7 @@ def test_an_own_agents_settings_are_in_its_tab_and_the_agent_is_no_option(self): line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" schema = dict(self.schema, hosts=[], standardHere={"docker": None, "licences": {}, "host": {}}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" - + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + cut(" function benchOptions(", " // Only credentials") + + line("esc") + line("button") + cut(" const tile = t =>", " // The grids drawn by the server") + cut(" function benchOptions(", " // Only credentials") + "const r=C.createStandard('osworld1'),b=C.standardBench(r);r.others=['osworld1:m3','osworld1:claude'];" + "const own=k=>benchOptions(r,b,'own',C.ownAgentsOf(b).find(a=>a.key===k));" + "process.stdout.write(JSON.stringify({m3:own('osworld1:m3'),claude:own('osworld1:claude')," diff --git a/app/ui_tests/test_cards.py b/app/ui_tests/test_cards.py index e574cb4d..d0c81b57 100644 --- a/app/ui_tests/test_cards.py +++ b/app/ui_tests/test_cards.py @@ -136,6 +136,11 @@ def test_a_pick_card_is_a_toggle_and_a_disabled_one_says_why(self): '
Lacks tools
', html) self.assertNotIn('Beta', self.render(cols, items)) + class Script(unittest.TestCase): def test_search_and_filter_hide_cards_and_their_empty_groups(self): @@ -235,6 +240,40 @@ def test_a_pick_flips_and_says_so_and_a_disabled_one_stays(self): """) self.assertEqual(got, [[["a", True], ["a", False], ["a", True]], "true", "false"]) # its pill too: the card is the toggle + def test_a_single_choice_lets_the_others_go_and_stays_chosen(self): + got = node(r""" +global.CustomEvent = class { constructor(type, o) { this.type = type; this.detail = o.detail; } }; +const picks = [], card = v => el('article', {'data-card': ''}, el('div', {}, el('button', {class: 'card-link', 'data-pick': v, 'data-pick-one': '', 'aria-pressed': 'false'}, v))); +const root = el('div', {id: 'c', 'data-cards': ''}, card('a'), card('b')); +root.querySelectorAll('[data-pick]').forEach(b => { b.dispatchEvent = ev => b.dispatch(ev.type, {detail: ev.detail}); }); +document.appendChild(el('main', {}, el('div', {id: 'c-tools'}), root, el('p', {'data-empty': 'c', hidden: true}), el('span', {'data-live': 'c', 'data-noun': 'cards'}))); +global.getSelection = () => ({isCollapsed: true}); +T.cards(root); +root.addEventListener('nullius-pick', e => picks.push([e.detail.value, e.detail.pressed])); +const a = root.querySelector('[data-pick="a"]'), b = root.querySelector('[data-pick="b"]'), state = () => [a, b].map(x => x.getAttribute('aria-pressed')).join(' '); +const out = []; +a.click(); out.push(state()); b.click(); out.push(state()); b.click(); out.push(state()); +process.stdout.write(JSON.stringify([picks, out])); +""") + self.assertEqual(got, [[["a", True], ["b", True], ["b", True]], ["true false", "false true", "false true"]]) + + def test_a_row_click_chooses_its_radio(self): + got = node(r""" +global.CustomEvent = class { constructor(type, o) { this.type = type; this.detail = o.detail; } }; +global.getSelection = () => ({isCollapsed: true}); +const row = v => el('tr', {}, el('td', {}, el('input', {type: 'radio', name: 'c-pick', 'data-pick': v})), el('td', {}, v)); +const table = el('table', {}, row('a'), row('b')); +table.querySelectorAll('[data-pick]').forEach(b => { b.dispatchEvent = ev => b.dispatch(ev.type, {detail: ev.detail}); b.type = 'radio'; }); +document.appendChild(table); +const picks = []; +T.picks(table, 'tr'); +table.addEventListener('nullius-pick', e => picks.push([e.detail.value, e.detail.pressed])); +const tds = table.querySelectorAll('td'), checked = () => table.querySelectorAll('[data-pick]').filter(b => b.checked).map(b => b.dataset.pick).join(''); +tds[1].dispatch('click', {button: 0}); const first = checked(); tds[3].dispatch('click', {button: 0}); tds[3].dispatch('click', {button: 0}); +process.stdout.write(JSON.stringify([picks, first, checked()])); +""") + self.assertEqual(got, [[["a", True], ["b", True], ["b", True]], "a", "b"]) + def test_a_card_passes_a_filter_its_unless_column_narrows_to(self): got = node(r""" const card = (id, from, runs, unless) => el('article', Object.assign({'data-card': '', id, 'data-f-from': from + '\n', 'data-f-runs': runs + '\n'}, unless ? {'data-unless-from': 'runs'} : {}), id); diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index 367c332c..e1e017f9 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -201,12 +201,12 @@ def test_an_anchor_from_the_one_page_list_goes_on_to_its_page(self): def test_every_listed_benchmark_has_its_mark(self): # one mark per benchmark, by key - its maker's logo or a line icon - in the sprite the builder's cards and these pages use marks = (Path(__file__).resolve().parents[1] / "static" / "catalogue" / "marks.svg").read_text(encoding="utf-8") - builder = (Path(__file__).resolve().parents[1] / "static" / "builder" / "command-builder.js").read_text(encoding="utf-8") - self.assertIn("/static/catalogue/marks.svg#bm-${esc(key)}", builder) + builder = self.client.get("/new-run/benchmarks").get_data(as_text=True) # the builder's cards are these for b in catalog.load().listed_benchmarks: with self.subTest(b.key): self.assertIn(' String(s); const button = (label, attrs) => ``; diff --git a/app/web/builder.py b/app/web/builder.py index b1188f6d..1ccc5926 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -27,6 +27,7 @@ "rovodev-cli": ["ROVODEV_USER_API_TOKEN"]} from . import shared from . import execute +from . import catalogue @app.route("/new-run") @@ -61,6 +62,29 @@ def new_run(): return render_template("builder/new_run.html", builder_data=data) +@app.route("/new-run/benchmarks") +def new_run_benchmarks(): + """The Benchmark step's grid, as a fragment the builder puts in its page: the Benchmarks page's cards + (``catalogue.benchmark_cards``) as single-choice pick cards (templates/builder/benchmarks.html), or rows with + ?view=table; the builder presses the one its recipe names.""" + cat = shared._catalog_cached() + # a community benchmark says so; an official one says nothing + # a benchmark with versions carries each one's Tasks (the default's under ""), for the builder to show the chosen one's + items = [dict(i, pick={"value": i["key"], "one": True}, id="pick-bench-" + i["key"], title={"text": i["title"], "title": i["domain"]}, + origin=i["origin"] if i["shelf"] != "official" else None, + data={"tasks": json.dumps({"": i["tasks"], **i["versions"]})} if i["versions"] else None) + for i in catalogue.benchmark_cards(cat)] + every = catalogue.benchmark_choices(cat) + choices = {k: [(v, t) for v, t in every[k] if any(v in _values(i[k]) for i in items)] for k in ("category", "evaluates", "runs")} + return render_template("builder/benchmarks.html", items=items, choices={k: v for k, v in choices.items() if len(v) > 1}, + layout="table" if request.args.get("view") == "table" else "cards") + + +def _values(v) -> list: + """A card field's filter values: a value's ``value``, else its text.""" + return [x.get("value", x.get("text")) if isinstance(x, dict) else x for x in (v if isinstance(v, list) else [v])] + + @app.route("/new-run/agents") def new_run_agents(): """The Agent step's grid for one benchmark of the standard (?benchmark=key or key@version), as a fragment the diff --git a/app/web/catalogue.py b/app/web/catalogue.py index 4cf4f0e8..ea618fc3 100644 --- a/app/web/catalogue.py +++ b/app/web/catalogue.py @@ -24,29 +24,37 @@ @app.route("/benchmarks") def benchmarks_page(): - # the benchmarks of the standard some of whose options need a machine's setup (``host``): a bubble on their card - setup = {b["key"]: sorted(b["host"]) for b in standard_definitions.load("benchmark")[0] if b.get("host")} cat = shared._catalog_cached() + return render_template("catalogue/benchmarks.html", cat=cat, items=benchmark_cards(cat), choices=benchmark_choices(cat), + prompts=howto.prompts(), catalogue_dir=paths.catalogue_dir()) + + +RUNS_WITH = {"standard": ("Nullius", "the Nullius standard"), "harbor": ("Harbor", "Harbor")} +# what a benchmark evaluates (``recipe.benchmarks``' family): its agents or its capabilities, as a card and its filter say it +FAMILIES = {"agent": "Agents", "capability": "Capabilities"} + + +def benchmark_cards(cat) -> list: + """The benchmarks as cards (templates/_cards.html), one per row of ``recipe.benchmarks``: what the Benchmarks + page and the builder's Benchmark step both draw.""" with open(MARKS, encoding="utf-8") as fh: sprite = fh.read() sheet = url_for("static", filename="catalogue/marks.svg") - items = [_card(cat, b, setup.get(b.key), sheet if f' dict: + """The values the benchmarks' cards are filtered by: their category, what runs them, their origin, what they run.""" + return {"category": [(c, c) for c in cat.categories], "runs": [(way, label) for way, (label, _) in RUNS_WITH.items()], + "origin": [(o, o.capitalize()) for o in catalog.ORIGINS], "evaluates": list(FAMILIES.items())} -def _card(cat, b, needs, sheet) -> dict: - """A benchmark's card on the list (templates/_cards.html): ``needs`` what some of its options need installed, - ``sheet`` the sprite holding its own mark, none when it has none.""" - ds = cat.datasets_of(b) - tasks, unit = cat.tasks_of(b) - return {"href": url_for("benchmark_page", key=b.key), "shelf": b.origin, "keys": " ".join([b.key, *b.datasets]), +def _card(row, sheet) -> dict: + """A benchmark's card on the list (templates/_cards.html), from a row of ``recipe.benchmarks``: ``sheet`` the + sprite holding its own mark, none when it has none.""" + b, ds, tasks, unit, needs = row["b"], row["datasets"], row["tasks"], row["unit"], row["host"] + return {"key": b.key, "href": url_for("benchmark_page", key=b.key), "shelf": b.origin, "keys": " ".join([b.key, *b.datasets]), "search": " ".join([b.title, b.category, b.key, *(d.title for d in ds), *b.datasets]), "category": b.category, "origin": {"text": b.label, "class": b.origin, "value": b.origin}, "icon": sheet and f"{sheet}#bm-{b.key}", "title": b.title, "domain": b.domain, @@ -55,7 +63,9 @@ def _card(cat, b, needs, sheet) -> dict: "needs": needs and {"text": "needs " + ", ".join(needs), "title": f"Some of its options need {', '.join(needs)} installed on the machine it runs on: see its page"}, "tasks": {"text": tasks, "label": unit.capitalize(), "value": tasks.replace(",", "").split("–")[-1]}, + "versions": {v: {"text": n, "label": u.capitalize()} for v, (n, u) in row["versions"].items()}, "datasets": {"text": len(ds), "label": "Datasets" if len(ds) != 1 else "Dataset"}, + "evaluates": {"text": FAMILIES[row["family"]], "value": row["family"], "title": f"{row['kind'].capitalize()} capabilities" if row["kind"] else ""}, "scoring": b.scoring, "blocker": b.blocker[:1].upper() + b.blocker[1:]} diff --git a/docs/builder.md b/docs/builder.md index d049cef3..c6fe852c 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -22,7 +22,7 @@ Then [read its results](results.md). | Stage | Decides | | --- | --- | -| Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`) | +| Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`). The cards are the Benchmarks page's, as cards or a table; one is chosen at a time. The category pills, *Evaluates* (agents or capabilities) and the Filter menu (Runner) narrow them, and the chosen one stays in view under any filter | | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A benchmark option its datasets need (`step: dataset`) is asked here, for the datasets it applies to. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. A benchmark with [agents of its own](standard.md#a-benchmarks-own-agents) (OSWorld's) lists them first, in place of its Original tile; several may be chosen, a job each, and a setting for one of them is in its tab. The cards are the Agents page's, each greyed with why it cannot run, and show as cards or a table. *Fit* starts on *Can run it*; the Filter menu (Protocol, From, Proves) narrows them further, and a chosen agent stays in view under any filter. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab |