diff --git a/AGENTS.md b/AGENTS.md index 6ba4ab10..3c241962 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,7 +78,7 @@ and its resources. The source layout stays under `app/`. `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), `paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request and before a capped run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `constraints` (what Nullius may use on each machine: one table, the container and systemd scope probes, the warnings), `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine says is done, and where each step is done), `gitrefs` (a repository's branches and tags, - by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository, and the Hugging Face token of Data providers), `sandbox` (Docker's state, and the Docker Hub login Data providers uses), `charts` (categorical charts, bars, dots and a scatter, and their style registry: Compare's Chart view) and `redact` at the top. + by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository, and the Hugging Face token of Data providers), `sandbox` (the Docker Hub login Data providers uses), `charts` (categorical charts, bars, dots and a scatter, and their style registry: Compare's Chart view) and `redact` at the top. - `app/templates/`, `app/static/` — the interface, grouped by the same capability names (`runs/`, `traces/`, `builder/`, …; `base.html`, `style.css` and the fonts at the top), including the command builder's JavaScript, which owns the generated recipes. `app/templates/_tables.html` and diff --git a/app/nullius/sandbox.py b/app/nullius/sandbox.py index 399bbe28..e2de12a5 100644 --- a/app/nullius/sandbox.py +++ b/app/nullius/sandbox.py @@ -1,10 +1,6 @@ -"""The Settings page's Sandbox: the container backend's engine on this machine, -docker or podman (``container.DOCKER``), and its Docker Hub login under Data -providers. +"""The container engine's Docker Hub login, under Data providers: docker or +podman (``container.DOCKER``). -Two things, each something a user acts on: - -- **Engine**: whether it answers, and its name and version (``container.unusable``). - **Registry login**: Docker Hub limits anonymous pulls, which stops a benchmark with an image per unit part-way. The username and an access token go to ``docker login --password-stdin`` (``podman login ... docker.io``) @@ -19,7 +15,6 @@ import json import os import re -import shutil import subprocess from typing import List, Optional @@ -31,37 +26,6 @@ _TOKEN_MAX = 4096 -def docker_state() -> dict: - """{ok, version} when the engine answers - its name and version, ``podman - 6.1.4`` - else {ok: False, error}; with ``also``, the other engine when both - are installed (``_other``), so the page says why this one is in use.""" - why = container.unusable() - state = {"ok": False, "error": why} if why else None - if state is None: - r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False) - state = {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"} - also = _other() - return dict(state, also=also) if also else state - - -def _other() -> Optional[dict]: - """The engine not in use, when it is installed too and is not the same program - (podman's ``docker``): {name, ok, said}, ``said`` its version or why it does not answer.""" - other = "podman" if container.DOCKER == "docker" else "docker" - path = shutil.which(other) - if not path or (other == "docker" and container._runs_podman(path)): - return None - try: - r = subprocess.run([other, "version", "--format", "{{.Server.Version}}"], stdin=subprocess.DEVNULL, - capture_output=True, text=True, timeout=10) - except (OSError, subprocess.SubprocessError) as e: - return {"name": other, "ok": False, "said": str(e)[:200]} - if r.returncode == 0 and r.stdout.strip(): - return {"name": other, "ok": True, "said": f"{other} {r.stdout.strip()}"} - lines = (r.stderr or r.stdout).strip().splitlines() - return {"name": other, "ok": False, "said": lines[-1][:200] if lines else f"exit {r.returncode}"} - - def _config_dir() -> str: return os.environ.get("DOCKER_CONFIG") or os.path.join(os.path.expanduser("~"), ".docker") diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index edb2b133..56d34b4f 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -392,9 +392,10 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, the run sends it as.""" d = defs or _all() errors, notes = [], [] - # each problem once: several picks of one kind (a benchmark's own agents) can each find it - add = lambda step, message: None if {"step": STEP[step], "message": message} in errors \ - else errors.append({"step": STEP[step], "message": message}) + # each problem once; shown: the step already says it elsewhere, so only Review lists it + def add(step, message, shown=False): + if not any(e["step"] == STEP[step] and e["message"] == message for e in errors): + errors.append({"step": STEP[step], "message": message, **({"shown": True} if shown else {})}) out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": "", "served_models": {}} name = str(recipe.get("name") or "") @@ -652,7 +653,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, elif backend == "boat" and targets != ["local"]: add("where", "Hosts and Boat sandboxes do not mix: choose Hosts, or Sandboxes alone.") elif backend == "boat" and boat_unusable is not None and (why := boat_unusable(True)): - add("where", f"Boat cannot run: {why}.") + add("where", f"Boat cannot run: {why}.", shown=not boat.key()) elif backend == "boat" and boat_size_problem is not None and str(recipe.get("boat_size") or boat.SIZE) in boat.SIZES: why = boat_size_problem(str(recipe.get("boat_size") or boat.SIZE)) if why: @@ -676,7 +677,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, elsewhere = [definitions.setup_label(b, "original") if p.get("setup") else p["o"]["label"] for p in picks if not whole and (p["control"] or p.get("setup") or (computer and side == "agent"))] if backend == "boat" and elsewhere: - add("where", f"{', '.join(elsewhere)} cannot run in a Boat VM yet: choose Hosts.") + add("where", f"{', '.join(elsewhere)} cannot run in a Boat VM: choose Hosts.") # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]) paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 3f34f999..7ab09aba 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -281,11 +281,14 @@ #nullius-builder .std-machines button[aria-pressed=true] { background: var(--nb-tint); } #nullius-builder .std-machines button:focus-visible { outline: 2px solid var(--nb-blue); outline-offset: 2px; } #nullius-builder .std-machines button:disabled { opacity: .45; cursor: not-allowed; } -#nullius-builder .std-boat-card { display: flex; flex-direction: column; align-items: flex-start; gap: 6px; width: fit-content; max-width: 100%; min-width: 190px; margin-top: 12px; padding: 12px 14px; border: 1px solid var(--nb-blue); border-radius: 10px; background: var(--nb-tint); } -#nullius-builder .std-boat-card strong { font-size: 15px; font-weight: 500; } +#nullius-builder .std-boat-card { display: flex; flex-direction: column; align-items: flex-start; gap: 6px; width: fit-content; max-width: 100%; min-width: 190px; margin-top: 20px; padding: 12px 14px; border: 1px solid var(--nb-blue); border-radius: 10px; background: var(--nb-tint); } +#nullius-builder .std-boat-card strong { display: flex; align-items: center; gap: 8px; font-size: 15px; font-weight: 500; } +#nullius-builder .std-boat-logo { width: 20px; height: 20px; flex: none; } #nullius-builder .std-boat-card > span { font-size: 13px; color: var(--nb-muted); } -#nullius-builder .std-boat-slots { border: 1px solid var(--nb-line); border-radius: 999px; padding: 2px 8px; } -#nullius-builder .std-hosts { margin-top: 12px; grid-template-columns: repeat(auto-fill, minmax(190px, 240px)); } +#nullius-builder .std-boat-card.off { border-color: var(--nb-line); background: transparent; cursor: not-allowed; } +#nullius-builder .std-boat-card.off > :not(.std-boat-key) { opacity: .5; } +#nullius-builder .std-boat-card.off .std-boat-key { cursor: pointer; } +#nullius-builder .std-hosts { margin-top: 20px; grid-template-columns: repeat(auto-fill, minmax(190px, 240px)); } #nullius-builder .std-isolation { padding: 12px 14px; border: 1px solid var(--nb-line); border-radius: 10px; background: var(--nb-soft); } #nullius-builder .std-isolation small { display: block; margin-top: 4px; font-size: 13px; color: var(--nb-muted); } #nullius-builder .std-isolation small:empty { display: none; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 87a5dec3..12cc62af 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -109,6 +109,8 @@ // An unsupported draft opens nothing; its rejection is shown as a toast. else { let d = null; try { d = C.unpackDraft(localStorage.getItem(draftKey), query); } catch {} if (d && d.recipe) { recipe = d.recipe; step = d.step; tab = d.tab; savedId = d.saved; } else if (d && d.gone) opened = d.gone; } const saveDraft = () => { try { localStorage.setItem(draftKey, C.packDraft({recipe, step, tab, saved: savedId})); } catch {} }; + // Settings closed: reload from the draft to read what changed + document.addEventListener('close', e => { if (e.target.id === 'settings-modal') { saveDraft(); location.replace(location.pathname); } }, true); let compiled, pendingDirs, dirsSequence = 0; const r = () => recipe; const store = recipeStore({request: (...a) => nulliusFetch(...a), csrf: schema.csrf, storage: (() => { try { return localStorage; } catch { return {length: 0}; } })()}); @@ -480,9 +482,6 @@ const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy // a VM benchmark's provider of one value goes on the fixed line; several are its cards (benchOptions) const provider = vms && Object.values(b.options || {}).find(o => o.step === 'where' && o.values && o.values.length === 1 && Object.values(o.needs || {}).some(n => n && n.vm)); - // what a sandbox cannot run (recipe.py): a setup or a check runs on a machine; the computer port's socket is on this one - // a benchmark booting VMs of its own runs each unit whole in a VM, with a relay container in each VM (boat_units) - const outside = !sandboxes ? [] : selected.filter(o => !vms && (o.setup || o.control || ((b.needs || []).some(p => p.startsWith('agent.computer/')) && b.family === 'agent'))).map(o => o.label); let sizes = ''; if (sandboxes && !here.boat) { const offered = boatSizes(rec, b), fit = offered.find(z => !z.why); @@ -493,8 +492,8 @@ disabled: !!z.why, why: z.why, note: z.note || z.spec, title: z.label})).join('')}`; } return '

Where does it run?

' - + `
${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boat || '')}
` - + (sandboxes ? `
BoatOne VM per ${esc(noun)}${here.boatSlots ? `${here.boatSlots} at once on this account` : ''}${here.boat ? `${esc(here.boat)}` : ''}
` + sizes + + `
${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boatKey ? here.boat || '' : '')}
` + + (sandboxes ? `
Boat${here.boat ? (here.boatKey ? `${esc(here.boat)}` : button('Add API Key', 'data-href="/settings#sandbox-boat-key"', 'nb-inlinebutton std-boat-key')) : ''}
` + sizes : `
${machine('local', 'This machine', 'Runs here')}${H.map(h => machine(h.entry, h.label, 'Over SSH')).join('')}
` + (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '')) + '

Isolation

' @@ -506,7 +505,6 @@ : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + '') + (b ? benchOptions(rec, b, 'where') : '') + (agentRun && backend === 'local' ? consentBox(rec) : '') - + (outside.length ? `

${esc(outside.join(', '))} cannot run in a Boat VM yet: choose Hosts.

` : '') + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - ${sandboxes ? 'in a Boat VM' : 'on local processes'} their units are recorded as not run.

` : '') + (sandboxes ? '' : hostSetup(rec)); } @@ -974,7 +972,7 @@ const out=runnerOut||{errors:[],notes:[]}; compiled={errors:out.errors||[],notes:out.notes||[]}; }else{compiled=C.compile(rec);} - $('#nb-localerrors').textContent=compiled.errors.filter(e=>e.step===step).map(e=>e.message).join(' '); + $('#nb-localerrors').textContent=compiled.errors.filter(e=>e.step===step&&!e.shown).map(e=>e.message).join(' '); const errors=$('#nb-errors'); if(errors){ errors.hidden=!compiled.errors.length; @@ -1120,7 +1118,7 @@ if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;} if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;} if(d.agent){const on=r().agents.includes(d.agent);r().agents=on?r().agents.filter(id=>id!==d.agent):[...r().agents,d.agent];renderForm();refreshPlan();return;} - else if(d.href){location.href=d.href;return;} + else if(d.href){if(!window.nulliusOpenSettings?.(d.href,b))location.href=d.href;return;} else if(b.id==='nb-add-custom-agent'){r().customAgents.push('');r().agents.push('custom-'+(r().customAgents.length-1));renderForm();refreshPlan();root.querySelector('[data-custom-agent="'+(r().customAgents.length-1)+'"]')?.focus();return;} else if(d.removeCustomAgent!==undefined){const i=Number(d.removeCustomAgent),rec=r(),kept=rec.agents.filter(id=>id.startsWith('custom-')&&id!=='custom-'+i).map(id=>Number(id.slice(7)));rec.customAgents.splice(i,1);rec.agents=rec.agents.filter(id=>!id.startsWith('custom-')).concat(kept.map(k=>'custom-'+(k>i?k-1:k)));renderForm();refreshPlan();return;} // The Harbor panel (Runner step): a release or a repository, a ref of it, installed and asked for its agents. diff --git a/app/static/settings/modal.js b/app/static/settings/modal.js index 9df3d5b5..936c0189 100644 --- a/app/static/settings/modal.js +++ b/app/static/settings/modal.js @@ -72,13 +72,10 @@ proxy.remove(); }); }); - document.addEventListener('click', event => { - const link = event.target.closest('a[href]'); - if (!link || event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey || link.target || link.download) return; - const url = new URL(link.href, location.href); - if (url.origin !== location.origin || url.pathname !== '/settings' || location.pathname === '/settings') return; - event.preventDefault(); - opener = link; + function open(href, from) { + const url = new URL(href, location.href); + if (url.origin !== location.origin || url.pathname !== '/settings' || location.pathname === '/settings') return false; + opener = from; baseline = ''; closing = false; closeButton.disabled = false; @@ -87,5 +84,12 @@ frame.src = url.href; modal.showModal(); document.getElementById('settings-modal-title').focus(); + return true; + } + window.nulliusOpenSettings = open; + document.addEventListener('click', event => { + const link = event.target.closest('a[href]'); + if (!link || event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey || link.target || link.download) return; + if (open(link.href, link)) event.preventDefault(); }); })(); diff --git a/app/static/settings/sandbox.js b/app/static/settings/sandbox.js index b8b9caab..d656bcb9 100644 --- a/app/static/settings/sandbox.js +++ b/app/static/settings/sandbox.js @@ -1,4 +1,4 @@ -// The Sandbox section: docker's state and whether a Boat key is saved, asked once when the section is first shown. +// The Sandbox section: whether a Boat key is saved, asked once when the section is first shown. (() => { const section = document.getElementById('sandbox'); if (!section) return; @@ -26,13 +26,7 @@ const load = async () => { asked = true; let j; - try { j = await (await nulliusFetch(section.dataset.sandbox)).json(); } catch (err) { j = {docker: {ok: false, error: String(err)}, boat: {set: false, env: false}}; } - const docker = $('[data-docker-state]'); - docker.textContent = j.docker.ok ? '✓ ' + j.docker.version : '✕ ' + j.docker.error; - docker.className = 'st-row-value ' + (j.docker.ok ? 'ok' : 'bad'); - const also = j.docker.also, other = $('[data-docker-also]'); - other.hidden = !also; - if (also) other.lastElementChild.textContent = also.ok ? also.said + ' is installed too' : also.name + ' is installed and does not answer: ' + also.said; + try { j = await (await nulliusFetch(section.dataset.sandbox)).json(); } catch (err) { j = {boat: {set: false, env: false}}; } boat = j.boat || {set: false, env: false}; render(); }; @@ -56,7 +50,10 @@ section.querySelectorAll('button').forEach(b => { b.disabled = false; }); } }; - if (edit) edit.addEventListener('click', () => { editing = true; render(); form.querySelector('input').focus(); }); + const open = () => { editing = true; render(); form.querySelector('input').focus(); }; + if (edit) edit.addEventListener('click', open); + const fromHash = () => { if (form && location.hash === '#sandbox-boat-key') open(); }; + window.addEventListener('hashchange', fromHash); if (cancel) cancel.addEventListener('click', () => { form.reset(); editing = false; render(); edit.focus(); }); if (form) form.addEventListener('submit', e => { e.preventDefault(); change('save'); }); if (remove) remove.addEventListener('click', () => asking(removeAsk.hidden)); @@ -68,4 +65,5 @@ const ask = () => { if (!asked && !section.hidden) load(); }; new MutationObserver(ask).observe(section, {attributes: true, attributeFilter: ['hidden']}); ask(); + fromHash(); })(); diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index c1e2f980..b83f2ca7 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -86,7 +86,7 @@
Execution
{{ section_link('hosts', 'Hosts', 'server', 'ssh remote machines server hosts') }} - {{ section_link('sandbox', 'Sandbox', 'kit', 'docker container boat vm') }} + {{ section_link('sandbox', 'Sandbox', 'kit', 'boat vm') }}
Application
@@ -391,12 +391,8 @@

Judges

{% endfor %}{% endfor %}
{% endif %} {% include 'settings/data_providers.html' %} - {# the container backend's engine on this machine, outside the form: asked when the section opens (sandbox.js) #}

Sandbox

Where agents run, apart from this machine.

-
-
EngineChecking…
-

Boat

Checking…

{% if writable %}{% endif %}
{% if writable %} diff --git a/app/tests/test_sandbox.py b/app/tests/test_sandbox.py index 8342bb2c..de4771c8 100644 --- a/app/tests/test_sandbox.py +++ b/app/tests/test_sandbox.py @@ -52,12 +52,6 @@ def test_podman_logs_in_to_docker_hub_by_name_and_says_who(self): with mock.patch.object(container, "_docker", side_effect=container.Unavailable("podman is not installed on this machine")): self.assertIsNone(sandbox.login_user()) # the page still answers - def test_the_engine_is_named_with_its_version(self): - with mock.patch.object(container, "DOCKER", "podman"), mock.patch.object(container, "unusable", return_value=None), \ - mock.patch.object(container, "_docker", return_value=done(out="6.1.4\n")), \ - mock.patch.object(sandbox.shutil, "which", return_value=None): # no other engine: the machine's own is not asked - self.assertEqual(sandbox.docker_state(), {"ok": True, "version": "podman 6.1.4"}) - def test_a_bad_username_or_empty_token_asks_docker_nothing(self): with mock.patch.object(container, "_docker", lambda *a, **kw: self.fail("docker asked")): self.assertFalse(sandbox.login("", "t")["ok"]) diff --git a/app/tests/test_standard_boat.py b/app/tests/test_standard_boat.py index 404ff0fd..cf106a93 100644 --- a/app/tests/test_standard_boat.py +++ b/app/tests/test_standard_boat.py @@ -590,7 +590,7 @@ def test_a_benchmark_booting_its_own_vms_wants_a_size_that_holds_them_and_never_ small = errors(backend="boat", boat_size="small") self.assertIn("A Small VM is too small for OSWorld2, whose units boot a 4 GB VM: choose a larger size.", small) # Model-calling setups and controls use a relay container inside each whole-task VM. - self.assertNotIn("Original cannot run in a Boat VM yet: choose Hosts.", small) + self.assertNotIn("Original cannot run in a Boat VM: choose Hosts.", small) self.assertEqual(errors(backend="boat", boat_size="default", others=["original", "nop"]), []) self.assertFalse([m for m in errors(backend="boat", boat_size="default") if "too small" in m]) self.assertIn("OSWorld2 starts each unit's VM on the machine: choose local processes.", errors(backend="container")) @@ -602,6 +602,10 @@ def test_without_its_key_the_where_step_says_so(self): out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda here=True: "no Boat API key: set it in Settings → Sandbox") self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) self.assertIn("Settings → Sandbox", out["errors"][0]["message"]) + for saved, shown in ((None, True), ("k", False)): + with mock.patch.object(boat, "key", return_value=saved): + out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda here=True: "missing") + self.assertEqual(out["errors"][0].get("shown", False), shown) def test_a_conformance_check_on_boat_says_its_backend(self): agent = definitions.load("agent", ROOT)[0][0] diff --git a/app/tests/test_standard_container.py b/app/tests/test_standard_container.py index bb301697..bd6628a6 100644 --- a/app/tests/test_standard_container.py +++ b/app/tests/test_standard_container.py @@ -84,20 +84,6 @@ def test_with_both_on_the_path_the_one_that_answers_is_the_engine(self): with self.subTest(want=want): self.assertEqual(self.engine(), want) - def test_the_sandbox_says_the_other_engine_when_both_are_installed(self): - from nullius import sandbox - def other(engine): - with mock.patch.dict(os.environ, {"PATH": self.tmp}), mock.patch.object(container, "DOCKER", engine): - return sandbox._other() - self.program("podman", "#!/bin/sh\necho 5.4.2\n") - self.assertIsNone(other("podman")) # podman alone: nothing else to say - self.program("docker", "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock' >&2\nexit 1\n") - self.assertEqual(other("podman"), {"name": "docker", "ok": False, - "said": "Cannot connect to the Docker daemon at unix:///var/run/docker.sock"}) - self.assertEqual(other("docker"), {"name": "podman", "ok": True, "said": "podman 5.4.2"}) - self.program("docker", "#!/bin/sh\nexec podman \"$@\"\n") # podman-docker's docker: the same program - self.assertIsNone(other("podman")) - def test_the_host_probe_and_the_run_there_pick_the_same_engine(self): from nullius import hosts for tool in ("sh", "sed", "grep", "hostname"): diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 0be840b6..5c4a35d0 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -612,7 +612,7 @@ def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_ {"key": "large", "label": "Large", "spec": "", "memory_gb": None, "why": None}, {"key": "xlarge", "label": "XLarge", "spec": "", "memory_gb": None, "why": None}] self.assertEqual(schema["standardHere"], {"docker": None, "boat": "no Boat API key: set it in Settings → Sandbox", - "boatSizes": sizes, "boatSlots": None, "licences": {}, "host": {}}) + "boatSizes": sizes, "boatSlots": None, "boatKey": False, "licences": {}, "host": {}}) def test_the_builder_offers_the_boat_sizes_and_vms_at_once_its_account_says(self): from nullius.standard import boat diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 366c7bfa..de9f575c 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -39,7 +39,7 @@ def test_hosts_or_sandboxes_never_both_and_a_sandbox_is_its_own_isolation(self): self.assertIn('aria-pressed="true"', card(html, "boat")) self.assertIn('aria-pressed="false"', card(html, "hosts")) self.assertIn(">Sandboxes", html) - self.assertIn("2 at once on this account", html) # the account's VMs at once, from its API + self.assertNotIn("at once", html) self.assertNotIn("data-std-where", html) # no host to add to a sandbox run self.assertNotIn("data-std-backend", html) # nothing to choose: the VM is the sandbox self.assertNotIn("not sandboxed", html.lower()) @@ -54,11 +54,18 @@ def test_hosts_or_sandboxes_never_both_and_a_sandbox_is_its_own_isolation(self): self.assertIn("The agent alone in each problem's VM", ordinary) self.assertIn("Answers and keys stay on this machine", ordinary) self.assertIn('class="std-boat-card"', ordinary) + self.assertIn('class="std-boat-card off"', self.draw(None, backends=("boat",), boat="no Boat API key")["gsm8k boat"]) + self.assertIn('src="/static/brand/boat.svg"', ordinary) + self.assertIn('src="/static/brand/boat-dark.svg"', ordinary) self.assertNotIn("shards, merged", ordinary) - self.assertNotIn("at once", self.draw(None, backends=("boat",), slots=None)["gsm8k boat"]) # the API said nothing: no number gone = self.draw(None, backends=("boat",), boat="no Boat API key: set it in Settings → Sandbox")["gsm8k boat"] - self.assertIn('disabled aria-disabled="true" title="no Boat API key: set it in Settings → Sandbox"', card(gone, "boat")) + self.assertNotIn("disabled", card(gone, "boat")) self.assertNotIn("data-std-boat-size", gone) + self.assertIn('class="nb-inlinebutton std-boat-key" data-href="/settings#sandbox-boat-key">Add API Key', gone) + missing = self.draw(None, "schema.standardHere.boatKey=true;", backends=("boat",), boat="missing on this machine: ssh")["gsm8k boat"] + self.assertIn('missing on this machine: ssh', missing) + self.assertNotIn("std-boat-key", missing) + self.assertIn('disabled aria-disabled="true" title="missing on this machine: ssh"', card(missing, "boat")) def test_a_benchmark_booting_its_own_vms_asks_no_isolation_and_refuses_a_size_too_small_for_its_vm(self): for key in ("osworld1", "osworld2"): diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index 135ae26e..e2bb92db 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -1108,10 +1108,10 @@ def test_the_sandbox_is_asked_when_shown_and_a_login_goes_to_docker_only(self): self.assertIn('

Docker Hub

', providers) self.assertIn('

Hugging Face

', providers) self.assertIn('name="token" type="password"', providers) - with patch.object(sandbox, "docker_state", lambda: {"ok": True, "version": "29.0"}), \ - patch.object(sandbox, "login_user", lambda: self.fail("asked")): + self.assertNotIn("data-docker-state", section) + with patch.object(sandbox, "login_user", lambda: self.fail("asked")): r = self.client.get("/settings/sandbox"); self.addCleanup(r.close) - self.assertEqual(r.json, {"docker": {"ok": True, "version": "29.0"}, "boat": {"set": False, "env": False}}) + self.assertEqual(r.json, {"boat": {"set": False, "env": False}}) seen = [] headers = {"Origin": "http://localhost", "X-CSRF-Token": self.token()} with patch.object(sandbox, "login", lambda u, t: seen.append((u, t)) or {"ok": True, "user": u}): @@ -1140,8 +1140,7 @@ def test_the_boat_key_is_saved_from_this_machines_browser_and_never_echoed(self) self.assertNotIn("boat_synthetic", response.get_data(as_text=True)) self.assertEqual(secrets.value(boat.STORE, boat.NAME), b"boat_synthetic") self.assertEqual(boat.key({}), "boat_synthetic") - with patch.object(sandbox, "docker_state", lambda: {"ok": True, "version": "29.0"}): - state = self.client.get("/settings/sandbox").json + state = self.client.get("/settings/sandbox").json self.assertEqual(state["boat"], {"set": True, "env": False}) self.assertNotIn("boat_synthetic", json.dumps(state)) self.assertIsNone(self.file()) diff --git a/app/web/builder.py b/app/web/builder.py index 39aea929..a9d45775 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -174,7 +174,7 @@ def _builder_data() -> dict: data["standardHere"] = {"docker": execute.docker_problem(), "boat": boat_why, "boatSizes": [{"key": k, "label": z["label"], "spec": boat.spec(k), "memory_gb": z.get("memory_gb"), "why": types.get(k)} for k, z in boat.SIZES.items() if not types or k in types], - "boatSlots": account.get("slots"), "licences": _licence_states(), "host": host} + "boatSlots": account.get("slots"), "boatKey": bool(boat.key()), "licences": _licence_states(), "host": host} data["csrf"] = shared._csrf_token() data["local"] = shared_local_client() data["recipesWhy"] = WRITE_HERE diff --git a/app/web/settings.py b/app/web/settings.py index b74eb2ba..7df0ebb2 100644 --- a/app/web/settings.py +++ b/app/web/settings.py @@ -249,9 +249,8 @@ def settings_host_enrol(): @app.route("/settings/sandbox") def settings_sandbox(): - """The Sandbox section's state, asked when the section opens: docker's, and whether a boat key is - set (never the key).""" - return jsonify(docker=sandbox.docker_state(), boat={"set": bool(secrets.status(boat.STORE).get(boat.NAME, {}).get("set")), + """The Sandbox section's state, asked when the section opens: whether a boat key is set (never the key).""" + return jsonify(boat={"set": bool(secrets.status(boat.STORE).get(boat.NAME, {}).get("set")), "env": bool(os.environ.get(boat.NAME))}) diff --git a/docs/settings/sandbox.md b/docs/settings/sandbox.md index fd375adf..5c58244a 100644 --- a/docs/settings/sandbox.md +++ b/docs/settings/sandbox.md @@ -1,13 +1,11 @@ # Sandbox -Where agents run apart from this machine: Docker or Podman here, and Boat VMs. Under **Execution**. +Where agents run apart from this machine: Boat VMs. Under **Execution**. | Card | Does | | --- | --- | -| Engine | Docker or Podman: whether it answers, its name and version; why not when it does not | | Boat | The [Boat backend](../standard.md#the-boat-backend)'s API key, never shown again. `BOAT_API_KEY` wins | | Rule | | | --- | --- | -| Hosts | This machine only; a host keeps its own login and images | -| Code | `app/nullius/sandbox.py`, `app/nullius/standard/boat.py` | +| Code | `app/nullius/standard/boat.py` |