From 20f8bdfacbd29feafb09f2c4e50035e4e7449813 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:52:34 +0100 Subject: [PATCH 1/5] refactor(builder): drop the workspace name from the builder's data Only the Saved recipes dialog read it, for a label that said recipes belong to a workspace. They do not. --- app/nullius/recipes/launch.py | 4 ++-- app/web/builder.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/nullius/recipes/launch.py b/app/nullius/recipes/launch.py index 0c74ef47..d0182f0b 100644 --- a/app/nullius/recipes/launch.py +++ b/app/nullius/recipes/launch.py @@ -55,14 +55,14 @@ def parameter(key, label, group, kind, default=None, *, help="", **extra): } -def payload(cat, jobs, workspace_name=""): +def payload(cat, jobs): from nullius import providers from nullius.standard import recipe as standard models = sorted({j.model_name for j in jobs if j.model_name}) dated = [j for j in jobs if j.model_name and j.started_at] latest = max(dated, key=lambda j: j.started_at.timestamp()).model_name if dated else "" return { - "version": 1, "workspace": workspace_name or "unassigned", "agents": AGENTS, + "version": 1, "agents": AGENTS, # Harbor and the Nullius standard: the page offers choices, the core and standard/recipe.py compile them. "runners": [standard.RUNNER, providers.HARBOR], # why a saved recipe of a runner no longer offered does not open (a benchmark's own scripts: its Original setup) diff --git a/app/web/builder.py b/app/web/builder.py index f2edc8dc..85424807 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -29,8 +29,7 @@ @app.route("/new-run") def new_run(): """Build a Harbor recipe; never execute it from this route.""" - active = shared.STATE.get("workspace") - data = command_builder.payload(catalog.load(), shared._load_jobs(), active.name if active else "") + data = command_builder.payload(catalog.load(), shared._load_jobs()) # Execution sets these credential variables from Settings. data["credentialVars"] = {"key": "API_TOKEN", "url": "API_URL"} # Where a recipe can run: this machine, or the hosts named in Settings. From dc9974327abefe9c18acca8c8620d1e2309ede70 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:52:34 +0100 Subject: [PATCH 2/5] feat(builder): keep saved recipes as files in the profile's data directory One owner-only JSON file per recipe under recipes/ in the user data directory, written whole and listed latest first. The id is random, so two names never share a file, and names are unique. A save updates the recipe it names by id, which renames it, or else the recipe of the same name. An import adds only names not saved yet: strict, one bad recipe refuses them all, or lenient, the good ones are added and each bad one is returned with its reason. nullius paths lists the directory. --- app/nullius/paths.py | 6 +- app/nullius/recipes/store.py | 141 +++++++++++++++++++++++++++++++++ app/tests/test_recipe_store.py | 127 +++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 app/nullius/recipes/store.py create mode 100644 app/tests/test_recipe_store.py diff --git a/app/nullius/paths.py b/app/nullius/paths.py index 4377a553..2f3ddb0b 100644 --- a/app/nullius/paths.py +++ b/app/nullius/paths.py @@ -66,6 +66,10 @@ def runs_dir(env=None) -> str: return env.get("RUNS_DIR") or os.path.join(data_dir(env), "runs") +def recipes_dir(env=None) -> str: + return os.path.join(data_dir(env), "recipes") + + def secrets_dir(env=None) -> str: env = os.environ if env is None else env return env.get("SECRETS_DIR") or os.path.join(config_dir(env), "secrets") @@ -76,7 +80,7 @@ def main(argv=None) -> int: parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) values = {"settings": settings_file(), "secrets": secrets_dir(), "prices": prices_file(), "catalogue": catalogue_dir(), - "workspaces": workspaces_dir(), "runs": runs_dir(), "cache": cache_dir()} + "workspaces": workspaces_dir(), "runs": runs_dir(), "recipes": recipes_dir(), "cache": cache_dir()} if args.json: print(json.dumps(values, indent=2)) else: diff --git a/app/nullius/recipes/store.py b/app/nullius/recipes/store.py new file mode 100644 index 00000000..a445be5b --- /dev/null +++ b/app/nullius/recipes/store.py @@ -0,0 +1,141 @@ +"""Saved builder recipes: one owner-only JSON file per recipe under the +profile's data directory (``recipes/.json``). + +A file holds ``id``, ``name`` (the title the builder shows), ``recipe`` (the +recipe as the builder saves it) and ``saved`` (UTC, ISO 8601). The id is +random, so two names can never share a file; names are unique instead. A save +names the recipe it updates by id (a rename keeps the file), or else updates +the recipe of the same name. Standard library only. +""" +from __future__ import annotations + +import json +import os +import re +import secrets +import tempfile +import time +from typing import Dict, Iterable, List, Optional + +from nullius import paths + +ID = re.compile(r"^[0-9a-f]{16}$") +_HARNESS = re.compile(r"^[a-z0-9][a-z0-9-]{0,31}$") +_NAME = re.compile(r"^[a-z0-9][a-z0-9._-]{0,71}$") +MAX_BYTES = 200_000 # as the builder's other recipe routes +MAX_NAME = 200 + + +def root(env=None) -> str: + return paths.recipes_dir(env) + + +def _path(recipe_id: str) -> str: + if not isinstance(recipe_id, str) or not ID.match(recipe_id): + raise ValueError(f"{recipe_id!r} is not a recipe id") + return os.path.join(root(), recipe_id + ".json") + + +def entry(recipe) -> Dict: + """The stored form of one recipe, without its id, or ValueError saying what is wrong with it.""" + if not isinstance(recipe, dict) or recipe.get("version") != 1: + raise ValueError("a recipe is a JSON object with \"version\": 1") + harness, name = recipe.get("harness") or "harbor", recipe.get("name") + if not isinstance(harness, str) or not _HARNESS.match(harness): + raise ValueError(f"{harness!r} is not a runner") + if not isinstance(name, str) or not _NAME.match(name): + raise ValueError(f"{name!r} is not a recipe name (lowercase letters, digits, '.', '_', '-')") + if len(json.dumps(recipe)) > MAX_BYTES: + raise ValueError(f"recipe {name!r} is larger than {MAX_BYTES // 1000} kB") + title = recipe.get("title") + title = title.strip() if isinstance(title, str) and title.strip() else name + if len(title) > MAX_NAME: + raise ValueError(f"a recipe's name is at most {MAX_NAME} characters") + return {"name": title, "saved": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "recipe": recipe} + + +def _write(item: Dict) -> None: + """Written whole under a temporary 0600 name beside it, then renamed over the old one.""" + path = _path(item["id"]) + os.makedirs(root(), mode=0o700, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=root()) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump(item, fh, indent=2, ensure_ascii=False) + fh.write("\n") + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def listing() -> List[Dict]: + """Every saved recipe, the latest saved first; an unreadable file is left out.""" + out = [] + try: + names = os.listdir(root()) + except OSError: + return [] + for f in names: + if not f.endswith(".json") or not ID.match(f[:-5]): + continue + try: + with open(os.path.join(root(), f), encoding="utf-8") as fh: + got = json.load(fh) + except (OSError, ValueError): + continue + if isinstance(got, dict) and got.get("id") == f[:-5] and isinstance(got.get("recipe"), dict): + out.append(got) + return sorted(out, key=lambda x: (str(x.get("saved", "")), x["id"]), reverse=True) + + +def save(recipe, recipe_id: Optional[str] = None) -> Dict: + """Update the recipe ``recipe_id`` (renamed when its name changed), or else the + recipe of the same name, or else add one. A name another recipe holds is refused.""" + item = entry(recipe) + held = recipe_id if recipe_id and os.path.exists(_path(recipe_id)) else None + same = next((x["id"] for x in listing() if x.get("name") == item["name"]), None) + if held and same and same != held: + raise ValueError(f"another saved recipe is named {item['name']!r}") + item = {"id": held or same or secrets.token_hex(8), **item} + _write(item) + return item + + +def delete(recipe_id: str) -> bool: + try: + os.unlink(_path(recipe_id)) + return True + except FileNotFoundError: + return False + + +def import_(items: Iterable, lenient: bool = False) -> Dict[str, list]: + """Add each recipe whose name is not saved yet; never replace one. An item is a + recipe, or ``{"recipe": ...}`` as the builder's earlier browser list held them. + Strict: one bad item refuses them all, before anything is written. Lenient: the + good ones are added and each bad one is returned with its reason.""" + items = list(items) + checked, rejected = [], [] + for n, item in enumerate(items): + recipe = item.get("recipe") if isinstance(item, dict) and "recipe" in item else item + try: + checked.append(entry(recipe)) + except ValueError as e: + if not lenient: + raise ValueError(f"recipe {n + 1} of {len(items)}: {e}") from None + rejected.append({"index": n, "reason": str(e)}) + taken = {x.get("name") for x in listing()} + added, skipped = [], [] + for item in checked: + if item["name"] in taken: + skipped.append(item["name"]) + continue + item = {"id": secrets.token_hex(8), **item} + _write(item) + taken.add(item["name"]) + added.append(item["id"]) + return {"added": added, "skipped": skipped, "rejected": rejected} diff --git a/app/tests/test_recipe_store.py b/app/tests/test_recipe_store.py new file mode 100644 index 00000000..f3cc8bac --- /dev/null +++ b/app/tests/test_recipe_store.py @@ -0,0 +1,127 @@ +"""Saved builder recipes on the server (``nullius.recipes.store``): one 0600 +file per recipe under the profile's data directory, written whole, listed +latest first, a random id that cannot name another path, unique names, and a +rename that keeps the recipe's file.""" +from __future__ import annotations + +import json +import os +import shutil +import stat +import tempfile +import time +import unittest +from unittest import mock + +from nullius import paths +from nullius.recipes import store + + +def recipe(title, name=None, **kw): + return dict({"version": 1, "harness": "harbor", "name": name or "driver-vs-ref", "title": title}, **kw) + + +class Store(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + patch = mock.patch.dict(os.environ, {"NULLIUS_HOME": self.tmp}) + patch.start() + self.addCleanup(patch.stop) + + def names(self): + return sorted(x["name"] for x in store.listing()) + + def test_a_saved_recipe_reads_back_as_saved_in_its_own_owner_only_file(self): + item = store.save(recipe("First try")) + self.assertRegex(item["id"], r"^[0-9a-f]{16}$") + self.assertEqual(item["name"], "First try") + path = os.path.join(self.tmp, "data", "recipes", item["id"] + ".json") + self.assertEqual((store.root(), paths.recipes_dir()), (os.path.dirname(path), os.path.dirname(path))) + with open(path) as f: + self.assertEqual(json.load(f), item) + self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600) + # the same name again updates it + again = store.save(recipe("First try", limit=3)) + self.assertEqual(again["id"], item["id"]) + self.assertEqual([(x["id"], x["recipe"]["limit"]) for x in store.listing()], [(item["id"], 3)]) + self.assertTrue(store.delete(item["id"])) + self.assertFalse(store.delete(item["id"])) + self.assertEqual(store.listing(), []) + + def test_names_that_slug_alike_are_different_recipes(self): + long = "a long name " * 13 # past the 72-character slug and the old 128-character id + titles = ["Driver vs Ref", "driver vs ref", "Driver-vs-Ref", "Driver vs Réf", "Driver vs Ref!", long + "one", long + "two"] + ids = {store.save(recipe(t))["id"] for t in titles} + self.assertEqual(len(ids), len(titles)) + self.assertEqual(self.names(), sorted(titles)) + with self.assertRaisesRegex(ValueError, "at most 200"): + store.save(recipe("x" * 201)) + + def test_a_rename_keeps_the_file_and_never_takes_another_recipes_name(self): + a, b = store.save(recipe("A")), store.save(recipe("B")) + renamed = store.save(recipe("A2", limit=5), a["id"]) + self.assertEqual((renamed["id"], renamed["name"]), (a["id"], "A2")) + self.assertEqual(self.names(), ["A2", "B"]) + with self.assertRaisesRegex(ValueError, "another saved recipe is named 'B'"): + store.save(recipe("B"), a["id"]) + self.assertEqual(self.names(), ["A2", "B"]) + # an id no longer saved (deleted elsewhere): saved by its name as a new recipe + store.delete(b["id"]) + back = store.save(recipe("B"), b["id"]) + self.assertNotEqual(back["id"], b["id"]) + self.assertEqual(self.names(), ["A2", "B"]) + + def test_the_latest_saved_comes_first(self): + with mock.patch("time.gmtime", return_value=time.gmtime(1_000)): + old = store.save(recipe("old")) + with mock.patch("time.gmtime", return_value=time.gmtime(2_000)): + new = store.save(recipe("new")) + self.assertEqual([x["id"] for x in store.listing()], [new["id"], old["id"]]) + + def test_an_id_or_a_recipe_that_could_name_another_path_is_refused(self): + for bad in ("../settings", "a/b", ".hidden", "", "harbor-x", "0123456789ABCDEF", None): + with self.subTest(bad=bad), self.assertRaises(ValueError): + store.delete(bad) + for bad, rid in ((recipe("x", "../up"), None), (recipe("x", "a/b"), None), (recipe("x", harness="../x"), None), + (recipe("x", version=2), None), ([], None), ("x", None), (None, None), + (recipe("x", blob="x" * store.MAX_BYTES), None), (recipe("x"), "../settings")): + with self.subTest(bad=str(bad)[:40], rid=rid), self.assertRaises(ValueError): + store.save(bad, rid) + self.assertFalse(os.path.exists(store.root())) + + def test_a_failed_write_leaves_the_old_file_and_no_temporary_one(self): + item = store.save(recipe("kept", limit=1)) + with mock.patch("json.dump", side_effect=OSError("disk full")), self.assertRaises(OSError): + store.save(recipe("kept", limit=2)) + self.assertEqual(os.listdir(store.root()), [item["id"] + ".json"]) + self.assertEqual(store.listing()[0]["recipe"]["limit"], 1) + + def test_a_strict_import_adds_new_names_skips_saved_ones_and_writes_nothing_when_one_is_bad(self): + store.save(recipe("there", limit=1)) + # the builder's earlier browser list: {id, updated, recipe} + out = store.import_([{"id": "harbor:there", "updated": "1/2/2026", "recipe": recipe("there", limit=9)}, + {"id": "harbor:new", "recipe": recipe("new")}, recipe("bare"), recipe("new")]) + self.assertEqual((len(out["added"]), out["skipped"], out["rejected"]), (2, ["there", "new"], [])) + self.assertEqual({x["name"]: x["recipe"].get("limit") for x in store.listing()}, {"there": 1, "new": None, "bare": None}) + with self.assertRaisesRegex(ValueError, "recipe 2 of 2"): + store.import_([recipe("fine"), {"recipe": recipe("x", "../x")}]) + self.assertNotIn("fine", self.names()) + + def test_a_lenient_import_adds_the_good_ones_and_names_each_bad_one(self): + out = store.import_([recipe("good"), {"recipe": recipe("x", "../x")}, "garbage", recipe("other")], lenient=True) + self.assertEqual(len(out["added"]), 2) + self.assertEqual([x["index"] for x in out["rejected"]], [1, 2]) + self.assertIn("not a recipe name", out["rejected"][0]["reason"]) + self.assertEqual(self.names(), ["good", "other"]) + + def test_a_stray_or_unreadable_file_is_not_listed(self): + good = store.save(recipe("good")) + for name, text in (("a" * 16 + ".json", "{"), ("notes.txt", "{}"), ("b" * 16 + ".json", json.dumps({"id": "x", "recipe": {}}))): + with open(os.path.join(store.root(), name), "w") as f: + f.write(text) + self.assertEqual([x["id"] for x in store.listing()], [good["id"]]) + + +if __name__ == "__main__": + unittest.main() From 67544613d2ac57897d15dee3bf31e325747a80be Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:52:34 +0100 Subject: [PATCH 3/5] feat(builder): list, save, delete and import saved recipes over HTTP Anyone who can open the builder reads the list. A write needs the form token and a browser on the application host, as Run and Settings do. The builder page greys Save recipe for any other browser, with the reason as its tooltip. An import is strict unless the request asks for the lenient one. --- app/static/builder/command-builder.css | 1 + app/templates/builder/new_run.html | 2 +- app/ui_tests/test_recipe_routes.py | 125 +++++++++++++++++++++++++ app/web/builder.py | 53 +++++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 app/ui_tests/test_recipe_routes.py diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 0e86bdab..9ef09621 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -29,6 +29,7 @@ #nullius-builder .nb-savedrow > span { min-width: 0; overflow-wrap: anywhere; } #nullius-builder .nb-savedrow small { display: block; } #nullius-builder .nb-button[hidden] { display: none; } +#nullius-builder .nb-button[aria-disabled=true] { opacity: .55; cursor: default; border-color: var(--nb-line); } /* it cannot here: greyed, its reason the tooltip */ #nullius-builder .nb-field input { scroll-margin-top: 100px; } @media(max-width:520px) { #nullius-builder .nb-headingactions { gap: 8px; } #nullius-builder .nb-errorlist { padding: 10px; } #nullius-builder .nb-field input { font-size: 16px; } } diff --git a/app/templates/builder/new_run.html b/app/templates/builder/new_run.html index 39075d68..6289014a 100644 --- a/app/templates/builder/new_run.html +++ b/app/templates/builder/new_run.html @@ -9,7 +9,7 @@
- +
diff --git a/app/ui_tests/test_recipe_routes.py b/app/ui_tests/test_recipe_routes.py new file mode 100644 index 00000000..6ac36836 --- /dev/null +++ b/app/ui_tests/test_recipe_routes.py @@ -0,0 +1,125 @@ +"""Saved recipes over HTTP (/new-run/recipes): open to read, written only from +a browser on this machine with the form token; and the builder page greys +Save recipe for any other browser.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +from nullius.recipes import store +from .test_routes import web + +REMOTE = {"REMOTE_ADDR": "10.0.0.2"} + + +def recipe(title, name="r", **kw): + return dict({"version": 1, "harness": "harbor", "name": name, "title": title}, **kw) + + +class RecipeRoutes(unittest.TestCase): + def setUp(self): + self.stack = ExitStack(); self.addCleanup(self.stack.close) + self.tmp = tempfile.mkdtemp(prefix="nullius-recipes-"); self.addCleanup(shutil.rmtree, self.tmp, True) + self.stack.enter_context(patch.dict(os.environ, {"NULLIUS_HOME": self.tmp})) + self.stack.enter_context(patch.dict(web.app.config, TESTING=True)) + self.client = web.app.test_client() + r = self.client.get("/new-run"); self.addCleanup(r.close) + with self.client.session_transaction() as s: + self.headers = {"Origin": "http://localhost", "X-CSRF-Token": s["csrf"]} + + def send(self, method, url, body=None, headers=None, **kw): + r = self.client.open(url, method=method, data=json.dumps(body), content_type="application/json", + headers=self.headers if headers is None else headers, **kw) + self.addCleanup(r.close); return r + + def listed(self): + r = self.client.get("/new-run/recipes"); self.addCleanup(r.close) + return sorted((x["name"], x["recipe"].get("limit")) for x in r.get_json()["recipes"]) + + def test_save_update_rename_list_and_delete(self): + self.assertEqual(self.listed(), []) + j = self.send("POST", "/new-run/recipes", {"recipe": recipe("a", limit=1)}).get_json() + self.assertTrue(j["ok"]) + rid = j["recipe"]["id"] + self.send("POST", "/new-run/recipes", {"recipe": recipe("a", limit=2)}) + self.assertEqual(self.listed(), [("a", 2)]) + self.assertEqual(self.send("POST", "/new-run/recipes", {"recipe": recipe("b"), "id": rid}).get_json()["recipe"]["id"], rid) + self.assertEqual(self.listed(), [("b", None)]) + self.assertTrue(self.send("DELETE", f"/new-run/recipes/{rid}").get_json()["ok"]) + self.assertFalse(self.send("DELETE", f"/new-run/recipes/{rid}").get_json()["ok"]) + self.assertEqual(self.listed(), []) + + def test_a_rename_onto_another_recipes_name_is_refused(self): + a = self.send("POST", "/new-run/recipes", {"recipe": recipe("a")}).get_json()["recipe"]["id"] + self.send("POST", "/new-run/recipes", {"recipe": recipe("b")}) + r = self.send("POST", "/new-run/recipes", {"recipe": recipe("b"), "id": a}) + self.assertEqual(r.status_code, 400) + self.assertIn("another saved recipe is named 'b'", r.get_json()["error"]) + self.assertEqual(self.listed(), [("a", None), ("b", None)]) + + def test_import_takes_one_or_several_and_skips_saved_names(self): + store.save(recipe("there", limit=1)) + j = self.send("POST", "/new-run/recipes/import", {"recipes": [{"id": "harbor:there", "recipe": recipe("there", limit=5)}, + {"id": "harbor:b", "recipe": recipe("b")}]}).get_json() + self.assertEqual((j["ok"], len(j["added"]), j["skipped"], j["rejected"]), (True, 1, ["there"], [])) + self.assertEqual(len(self.send("POST", "/new-run/recipes/import", recipe("c")).get_json()["added"]), 1) + self.assertEqual(len(self.send("POST", "/new-run/recipes/import", [recipe("d")]).get_json()["added"]), 1) + self.assertEqual(dict(self.listed())["there"], 1) + + def test_a_strict_import_refuses_bad_input_with_its_reason_and_writes_nothing(self): + for body in ([recipe("ok"), {"recipe": recipe("x", "../x")}], {"recipes": ["x"]}, None, {"version": 2}, + {"recipes": [recipe("ok"), "x"], "lenient": "yes"}): + with self.subTest(body=body): + r = self.send("POST", "/new-run/recipes/import", body) + self.assertEqual(r.status_code, 400) + self.assertTrue(r.get_json()["error"].startswith("Not imported: ")) + r = self.send("POST", "/new-run/recipes", {"recipe": recipe("n", "Bad Name")}) + self.assertEqual(r.status_code, 400) + self.assertIn("not a recipe name", r.get_json()["error"]) + self.assertEqual(self.send("DELETE", "/new-run/recipes/..%2Fsettings").status_code, 404) # no such route + self.assertEqual(self.send("DELETE", "/new-run/recipes/..settings").status_code, 400) + self.assertEqual(self.listed(), []) + + def test_a_lenient_import_adds_the_good_ones_and_returns_the_bad_with_reasons(self): + j = self.send("POST", "/new-run/recipes/import", {"recipes": [recipe("ok"), {"recipe": recipe("x", "../x")}, "x"], "lenient": True}).get_json() + self.assertEqual((j["ok"], len(j["added"]), [x["index"] for x in j["rejected"]]), (True, 1, [1, 2])) + self.assertTrue(all(x["reason"] for x in j["rejected"])) + self.assertEqual(self.listed(), [("ok", None)]) + + def test_a_write_needs_the_token_and_a_browser_on_this_machine_and_a_read_does_not(self): + kept = store.save(recipe("kept"))["id"] + writes = (("POST", "/new-run/recipes", {"recipe": recipe("n")}), ("POST", "/new-run/recipes/import", [recipe("n")]), + ("DELETE", f"/new-run/recipes/{kept}", None)) + for method, url, body in writes: + with self.subTest(url=url, method=method): + self.assertEqual(self.send(method, url, body, headers={"Origin": "http://localhost"}).status_code, 403) + self.assertEqual(self.send(method, url, body, headers={**self.headers, "X-Forwarded-For": "10.0.0.2"}).status_code, 403) + r = self.send(method, url, body, environ_base=REMOTE) + self.assertEqual(r.status_code, 403) + self.assertIn(web.builder.WRITE_HERE, r.get_data(as_text=True)) + self.assertEqual(self.listed(), [("kept", None)]) + r = self.client.get("/new-run/recipes", environ_base=REMOTE); self.addCleanup(r.close) + self.assertEqual((r.status_code, len(r.get_json()["recipes"])), (200, 1)) + + def test_the_builder_greys_save_for_a_browser_on_another_machine(self): + def save_button(**kw): + r = self.client.get("/new-run", **kw); self.addCleanup(r.close) + html = r.get_data(as_text=True) + return next(line for line in html.splitlines() if 'id="nb-save"' in line).strip() + why = web.builder.WRITE_HERE + self.assertEqual(save_button(), '') + self.assertEqual(save_button(environ_base=REMOTE), f'') + r = self.client.get("/new-run", environ_base=REMOTE); self.addCleanup(r.close) + html = r.get_data(as_text=True) + schema = json.loads(html.split('")[0]) + self.assertEqual((schema["local"], schema["recipesWhy"]), (False, why)) + self.assertNotIn("workspace", schema) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/builder.py b/app/web/builder.py index 85424807..e2b33bfe 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -13,6 +13,7 @@ from nullius.harbor import source as harbor_source from nullius.catalogue import catalog from nullius.recipes import launch as command_builder +from nullius.recipes import store as recipe_store from nullius.standard import boat, definitions, hostreq from nullius.standard import recipe as standard_recipe @@ -60,6 +61,7 @@ def new_run(): "boatSpecs": {k: v for k, v in boat.SIZES.items() if v}, "licences": _licence_states(), "host": host} data["csrf"] = shared._csrf_token() data["local"] = shared_local_client() + data["recipesWhy"] = WRITE_HERE # ?run=: the recipe a run of this application was started from, # opened as it was. ?like=, the job page's "New run like # this": that recipe when the run kept one, else what the job recorded - @@ -235,6 +237,57 @@ def _licence_states() -> dict: return out +# Saved recipes (nullius/recipes/store.py): anyone who can open the builder reads them; a browser on this machine +# alone writes them, with the form token. +WRITE_HERE = "Saved recipes change from a browser on the application host only." + + +def _recipe_write(): + shared._require_csrf() + if not shared_local_client(): + abort(403, description=WRITE_HERE) + if request.content_length and request.content_length > 10 * recipe_store.MAX_BYTES: + abort(413) + return request.get_json(silent=True) + + +@app.route("/new-run/recipes") +def recipes_list(): + return jsonify(recipes=recipe_store.listing()) + + +@app.route("/new-run/recipes", methods=["POST"]) +def recipes_save(): + body = _recipe_write() or {} + try: + return jsonify(ok=True, recipe=recipe_store.save(body.get("recipe"), body.get("id"))) + except ValueError as e: + return jsonify(ok=False, error=str(e)), 400 + + +@app.route("/new-run/recipes/", methods=["DELETE"]) +def recipes_delete(recipe_id): + _recipe_write() + try: + return jsonify(ok=recipe_store.delete(recipe_id)) + except ValueError as e: + return jsonify(ok=False, error=str(e)), 400 + + +@app.route("/new-run/recipes/import", methods=["POST"]) +def recipes_import(): + """One recipe or a list, or {"recipes": [...], "lenient": true}: each one added unless its name is saved + already. Lenient (the builder's move of its earlier browser list) adds the good ones and names the bad.""" + body = _recipe_write() + wrapped = isinstance(body, dict) and "recipes" in body + items = body["recipes"] if wrapped else body + items = items if isinstance(items, list) else [items] + try: + return jsonify(ok=True, **recipe_store.import_(items, lenient=wrapped and body.get("lenient") is True)) + except ValueError as e: + return jsonify(ok=False, error="Not imported: " + str(e)), 400 + + def _kept_recipe(job_id: str) -> bool: ex = execution.for_job(job_id) return bool(ex and ex.plan.recipe) From fc382833533f4fd6012477185bf4e8c4c106bbaf Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Tue, 17 Nov 2026 18:52:34 +0100 Subject: [PATCH 4/5] feat(builder): read and write the Saved recipes dialog through the server The dialog lists, saves, updates, renames and deletes through /new-run/recipes. The draft keeps the id of the saved recipe the page holds, so a save after a reload still updates it. A browser on another machine sees the list and can load and export a recipe; Delete is greyed with the reason. On the application host the builder moves the list an earlier version kept in the browser to the server once, with the lenient import. It keeps the recipes the server refused under nullius.recipes.v1.rejected, says once how many, and never sends them again. When the move fails, the list stays for the next visit. --- AGENTS.md | 4 +- app/static/builder/command-builder-core.js | 4 +- app/static/builder/command-builder.js | 90 ++++++++++++------- .../test_command_builder_execution.py | 7 +- app/ui_tests/test_recipe_store_script.py | 90 +++++++++++++++++++ docs/builder.md | 2 +- docs/concepts.md | 2 +- docs/getting-started.md | 4 +- docs/settings/README.md | 1 + 9 files changed, 161 insertions(+), 43 deletions(-) create mode 100644 app/ui_tests/test_recipe_store_script.py diff --git a/AGENTS.md b/AGENTS.md index febcc43a..d3c69efc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,7 @@ and its resources. The source layout stays under `app/`. a Harbor benchmark added from the Benchmarks page's form; `citation`, an entry's BibTeX read into the paper its page shows; `storage`, whole-folder user overlays and restoring bundled entries; `sources`, where each dataset's data comes from for each way it runs), `recipes/` - (`launch`, the builder's vocabulary), `providers` (the model provider registry: protocols, presets and each provider's record, read by every consumer; and the builder's steps), + (`launch`, the builder's vocabulary; `store`, its saved recipes), `providers` (the model provider registry: protocols, presets and each provider's record, read by every consumer; and the builder's steps), `harbor/` (`source`: the Harbor a recipe pins - built, kept, scanned for its agents; `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), @@ -116,7 +116,7 @@ a recipe without a pin), and the viewer never imports it. `nullius paths` lists persistent locations. On Linux, settings, the model price list and benchmark licences live under `~/.config/nullius/`, user catalogue entries, workspaces, -runs and standard installs under `~/.local/share/nullius/`, and caches under +runs, saved builder recipes and standard installs under `~/.local/share/nullius/`, and caches under `~/.cache/nullius/`. XDG variables move those roots; `NULLIUS_HOME` selects a separate profile with `config/`, `data/` and `cache/`. Existing storage overrides still apply. Settings saves owner-only JSON; process environment values win. diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index cb849255..cc605a60 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -829,7 +829,7 @@ // opened for something else - a prefill from a run, or a run's own recipe. const OPENED_FOR = ['run','like','dataset','model','agent','n_tasks','standard','other']; function packDraft(state) { - return JSON.stringify({version: 1, at: new Date().toISOString(), recipe: state.recipe, step: state.step, steps: STEPS.length, first: 'benchmark', tab: state.tab}); + return JSON.stringify({version: 1, at: new Date().toISOString(), recipe: state.recipe, step: state.step, steps: STEPS.length, first: 'benchmark', tab: state.tab, saved: state.saved || ''}); } function unpackDraft(text, query) { if (!text || Object.keys(query || {}).some(k => OPENED_FOR.includes(k))) return null; @@ -842,7 +842,7 @@ if (Number.isInteger(d.step) && had < STEPS.length && step >= STEP.setup) step += 1; if (d.first !== 'benchmark' && (step === 0 || step === 1)) step = 1 - step; // a draft from when the runner came first return {recipe: fromJSON(d.recipe), step: Math.min(step, STEP.review), tab: typeof d.tab === 'string' ? d.tab : 'select', - at: String(d.at || '')}; + at: String(d.at || ''), saved: typeof d.saved === 'string' ? d.saved : ''}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } return {STANDARD_PHASES,STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 73dd6d8b..2071898a 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -51,7 +51,42 @@ } return {field, sync, acceptCompile}; } - if (typeof module === 'object' && module.exports) { module.exports = {servedRows}; return; } + // Saved recipes are the server's (/new-run/recipes, nullius/recipes/store.py); a browser on its host alone writes them. + // move() sends what this browser kept before (nullius.recipes.v1, and the older nullius.recipes.v1. lists) + // to the lenient import, which skips names already saved, and removes those keys once the server answered. The + // recipes it refused are kept under REJECTED, never sent again; move() returns how many. On a failure the keys stay + // for the next visit. `request` is nulliusFetch; the tests run this under Node with stand-ins. + function recipeStore({request, csrf, storage}) { + const call = async (url, method, body) => { + const res = await request(url, method ? {method, headers: {'Content-Type': 'application/json', 'X-CSRF-Token': csrf}, body: JSON.stringify(body || {})} : {}); + const j = await res.json().catch(() => ({})); + if (!res.ok || j.ok === false) throw Error(j.error || 'HTTP ' + res.status); + return j; + }; + const OLD = 'nullius.recipes.v1', REJECTED = OLD + '.rejected'; + const read = k => { try { const l = JSON.parse(storage.getItem(k)); return Array.isArray(l) ? l : [l]; } catch { return [storage.getItem(k)]; } }; + return { + list: async () => (await call('/new-run/recipes')).recipes, + save: async (recipe, id) => (await call('/new-run/recipes', 'POST', {recipe, id})).recipe, + remove: id => call('/new-run/recipes/' + encodeURIComponent(id), 'DELETE'), + async move() { + const keys = Array.from({length: storage.length}, (_, i) => storage.key(i)).filter(k => k !== REJECTED && (k === OLD || k.startsWith(OLD + '.'))); + const recipes = keys.flatMap(read); + if (!keys.length) return 0; + const bad = recipes.length ? (await call('/new-run/recipes/import', 'POST', {recipes, lenient: true})).rejected.map(x => recipes[x.index]) : []; + if (bad.length) storage.setItem(REJECTED, JSON.stringify([...(storage.getItem(REJECTED) ? read(REJECTED) : []), ...bad])); + keys.forEach(k => storage.removeItem(k)); + return bad.length; + }, + }; + } + // The Saved recipes dialog's list. A browser on another machine gets Delete greyed, `why` its tooltip. + function savedList({saved, local, why, esc, button, state}) { + const del = id => local ? `data-delete="${esc(id)}"` : `aria-disabled="true" title="${esc(why)}"`; + return saved.length ? saved.map(s => `
${esc(s.name)}${esc(s.recipe.harness)} · ${esc(new Date(s.saved).toLocaleDateString())}${state(s.recipe)}` + + `
${button('Load', `data-load="${esc(s.id)}"`)} ${button('Delete', del(s.id))}
`).join('') : '

No saved recipes.

'; + } + if (typeof module === 'object' && module.exports) { module.exports = {servedRows, recipeStore, savedList}; return; } const root = document.getElementById('nullius-builder'); if (!root) return; const schema = JSON.parse(document.getElementById('nb-schema').textContent); @@ -74,15 +109,16 @@ // A run's own recipe (?run=) or a prefill from a run's fields comes first. // A draft comes back silently: the page simply is as it was left. const draftKey = 'nullius.recipe-draft.v1'; - let opened = ''; + let opened = '', savedId = ''; // savedId: the saved recipe this page holds, which a save updates (renames) if (schema.loadRecipe) { try { recipe = C.fromJSON(schema.loadRecipe); step = REVIEW; opened = 'Opened the recipe of run “' + schema.loadRun + '”. Run gives it a new name.'; } catch (e) { opened = 'This run’s recipe cannot be read: ' + e.message; } } else if (schema.loadLike) { const o = C.fromJob(schema.loadLike); recipe = o.recipe; step = REVIEW; opened = o.notes.join(' '); } // An unsupported draft opens nothing; its rejection is shown as a toast. - else { let d = null; try { d = C.unpackDraft(localStorage.getItem(draftKey), query); } catch {} if (d && d.recipe) { recipe = d.recipe; step = d.step; tab = d.tab; } else if (d && d.gone) opened = d.gone; } - const saveDraft = () => { try { localStorage.setItem(draftKey, C.packDraft({recipe, step, tab})); } catch {} }; + else { let d = null; try { d = C.unpackDraft(localStorage.getItem(draftKey), query); } catch {} if (d && d.recipe) { recipe = d.recipe; step = d.step; tab = d.tab; savedId = d.saved; } else if (d && d.gone) opened = d.gone; } + const saveDraft = () => { try { localStorage.setItem(draftKey, C.packDraft({recipe, step, tab, saved: savedId})); } catch {} }; let compiled, pendingDirs, dirsSequence = 0; const r = () => recipe; - const storageKey = 'nullius.recipes.v1'; // one list for this browser; a run may open another workspace and the recipes stay + const store = recipeStore({request: (...a) => nulliusFetch(...a), csrf: schema.csrf, storage: (() => { try { return localStorage; } catch { return {length: 0}; } })()}); + let saved = []; // A message is a toast at the bottom right: a new one replaces it, its bar runs down and it goes, or its × closes it. // Shown again on each message, so it lands above a dialog opened since. const status = text => { @@ -893,44 +929,33 @@ renderForm();refreshPlan();moveUnderline();saveDraft();harborFollow(); if(focus){const heading=$('#nb-form h2');if(heading){heading.tabIndex=-1;heading.focus({preventScroll:true});heading.scrollIntoView({block:'nearest'});}} } - function readSaved(){ - try{ - const ok=list=>Array.isArray(list)?list.filter(x=>x&&x.recipe&&typeof x.id==='string'):[]; - let list=ok(JSON.parse(localStorage.getItem(storageKey)||'[]')); - // Merge workspace-scoped recipe lists into this browser's shared list. - for(const k of Object.keys(localStorage))if(k.startsWith(storageKey+'.')){const old=ok(JSON.parse(localStorage.getItem(k)||'[]'));list=list.concat(old.filter(o=>!list.some(x=>x.id===o.id)));localStorage.removeItem(k);localStorage.setItem(storageKey,JSON.stringify(list));} - return list; - }catch{return [];} - } - const workspaceLabel = () => schema.workspace && schema.workspace!=='unassigned' ? 'Workspace: '+schema.workspace : 'Personal recipes · no workspace selected'; + async function loadSaved(){try{saved=await store.list();}catch(e){status('Saved recipes cannot be read: '+e.message);}if($('#nb-saved').open)renderSaved();} const recipeSlug = title => title.normalize('NFKD').replace(/[\u0300-\u036f]/g,'').toLowerCase().replace(/[^a-z0-9._-]+/g,'-').replace(/^[^a-z0-9]+|[-.]+$/g,'').slice(0,72) || 'recipe'; function savedState(recipe){try{return C.validate(C.fromJSON(recipe)).errors.length?' · Draft':'';}catch{return ' · Needs review';}} function renderSaved(){ - const saved=readSaved(); - $('#nb-saved-list').innerHTML='

'+esc(workspaceLabel())+' · stored in this browser

'+(saved.length?saved.map(s=>`
${esc(s.recipe.title||s.recipe.name)}${esc(s.recipe.harness)} · ${esc(s.updated)}${savedState(s.recipe)}
${button('Load',`data-load="${esc(s.id)}"`)} ${button('Delete',`data-delete="${esc(s.id)}"`)}
`).join(''):'

Save your first recipe to find it here.

'); + $('#nb-saved-list').innerHTML=savedList({saved,local:schema.local,why:schema.recipesWhy,esc,button,state:savedState}); } function download(text,filename,type){const url=URL.createObjectURL(new Blob([text],{type})),link=document.createElement('a');link.href=url;link.download=filename;link.click();setTimeout(()=>URL.revokeObjectURL(url),1000);} function updateSavePreview(){ - const title=$('#nb-recipe-title').value.trim(),name=recipeSlug(title),existing=readSaved().find(s=>s.id===(r().harness||'harbor')+':'+name); - $('#nb-recipe-title').setCustomValidity(title?'':'Enter a name for this recipe.'); - $('#nb-save-submit').textContent=existing?'Update recipe':'Save recipe'; - $('#nb-save-message').textContent=existing?'A recipe with this name exists. Saving will update it.':workspaceLabel()+'. You can save unfinished recipes as drafts.'; + // as the store decides: the recipe this page holds is updated (renamed), else the one of this name; a name another holds is refused + const title=$('#nb-recipe-title').value.trim(),held=saved.find(s=>s.id===savedId),other=saved.find(s=>s.name===title&&s!==held); + $('#nb-recipe-title').setCustomValidity(!title?'Enter a name for this recipe.':held&&other?'Another saved recipe has this name.':''); + $('#nb-save-submit').textContent=held||other?'Update recipe':'Save recipe'; + $('#nb-save-message').textContent=held&&other?'Another saved recipe has this name.':held&&held.name!==title?'Saving renames “'+held.name+'”.':other?'A recipe with this name exists. Saving will update it.':''; } function openSave(){ $('#nb-recipe-title').value=r().title||r().name.replace(/-/g,' '); updateSavePreview();$('#nb-save-dialog').showModal();$('#nb-recipe-title').focus();$('#nb-recipe-title').select(); } - function save(){ + async function save(){ const title=$('#nb-recipe-title').value.trim();if(!title){$('#nb-recipe-title').focus();return;} const candidate={...C.copy(r()),title,name:recipeSlug(title)}; if(C.validate(candidate).errors.some(e=>/environment variable name|credentials; do not put/.test(e.message))){$('#nb-save-message').textContent='Use environment variable names for credentials before saving. Their values belong in your shell.';return;} try{ - const list=readSaved(),id=(candidate.harness||'harbor')+':'+candidate.name; // the harness prefix earlier versions wrote; a runner's key for its recipes - const saved={id,updated:new Date().toLocaleDateString(),recipe:candidate}; - localStorage.setItem(storageKey,JSON.stringify([saved,...list.filter(s=>s.id!==id)].slice(0,50))); + const item=await store.save(candidate,savedId);saved=[item,...saved.filter(s=>s.id!==item.id)];savedId=item.id; recipe=candidate;$('#nb-save-dialog').close();render();renderSaved(); status('Saved “'+title+'”'+(compiled.errors.length?' as a draft.':'.')); - }catch{ $('#nb-save-message').textContent='Browser storage is unavailable. Close this dialog and export the recipe as JSON instead.'; } + }catch(e){ $('#nb-save-message').textContent='Not saved: '+e.message; } } root.addEventListener('submit',event=>{if(event.target.id==='nb-save-form'){event.preventDefault();save();}}); // Running from here: the runner script is posted with the target; the page follows the log until the run ends. @@ -1055,15 +1080,15 @@ return; } else if(d.remove!==undefined)r().custom.splice(Number(d.remove),1); - else if(d.load){try{const saved=readSaved().find(s=>s.id===d.load);recipe=C.fromJSON(saved.recipe);step=REVIEW;$('#nb-saved').close();status('Loaded “'+(recipe.title||recipe.name)+'”.');}catch(error){status('This saved recipe cannot be read: '+error.message);return;}} - else if(d.delete){try{localStorage.setItem(storageKey,JSON.stringify(readSaved().filter(s=>s.id!==d.delete)));renderSaved();status('Saved recipe deleted.');}catch{status('Browser storage is unavailable.');}return;} + else if(d.load){try{recipe=C.fromJSON(saved.find(s=>s.id===d.load).recipe);savedId=d.load;step=REVIEW;$('#nb-saved').close();status('Loaded “'+(recipe.title||recipe.name)+'”.');}catch(error){status('This saved recipe cannot be read: '+error.message);return;}} + else if(d.delete){try{await store.remove(d.delete);saved=saved.filter(s=>s.id!==d.delete);if(savedId===d.delete)savedId='';renderSaved();status('Saved recipe deleted.');}catch(e){status('Not deleted: '+e.message);}return;} else if(b.id==='nb-add'){r().custom.push({scope:'agent',key:'',type:'string',value:'',...(d.addAgent?{agent:d.addAgent}:{})});} else if(b.id==='nb-next'){if(step===REVIEW){if(!compiled.errors.length)download(JSON.stringify(r(),null,2)+'\n',r().name+'.json','application/json');return;}step=nextShown(step,1);} else if(b.id==='nb-back')step=nextShown(step,-1); - else if(b.id==='nb-save'){openSave();return;} - else if(b.id==='nb-startover'){try{localStorage.removeItem(draftKey);}catch{}recipe=C.prefill({});runnerOut=null;runnerKey='';step=S.benchmark;tab='select';status('Started over from the defaults.');} + else if(b.id==='nb-save'){if(schema.local)openSave();return;} + else if(b.id==='nb-startover'){try{localStorage.removeItem(draftKey);}catch{}recipe=C.prefill({});savedId='';runnerOut=null;runnerKey='';step=S.benchmark;tab='select';status('Started over from the defaults.');} else if(b.id==='nb-save-cancel'){$('#nb-save-dialog').close();return;} - else if(b.id==='nb-saved-toggle'){renderSaved();$('#nb-saved').showModal();return;} + else if(b.id==='nb-saved-toggle'){renderSaved();$('#nb-saved').showModal();loadSaved();return;} else if(b.id==='nb-saved-close'){$('#nb-saved').close();return;} else if(b.id==='nb-import'){$('#nb-import-file').click();return;} else if(b.id==='nb-refresh'){await refreshRegistry();return;} @@ -1124,10 +1149,11 @@ else if(d.harborChoice!==undefined){const rec=r(),v=el.value;if(v==='path')rec.harborDev='';else if(v==='dev')rec.harborDev=rec.harborDev||'~/harbor-dev/.venv/bin/harbor';renderForm();refreshPlan();if(v==='dev')root.querySelector('[data-field="harborDev"]')?.select();} else if(el.id==='nb-import-file'){ const file=el.files[0];if(!file)return; - try{if(file.size>100000)throw Error('Recipe file is too large.');recipe=C.fromJSON(await file.text());step=REVIEW;render(true);status('Imported “'+(recipe.title||recipe.name)+'”. Review the recipe before copying commands.');}catch(error){status('Could not import recipe: '+error.message);}el.value=''; + try{if(file.size>100000)throw Error('Recipe file is too large.');recipe=C.fromJSON(await file.text());savedId='';step=REVIEW;render(true);status('Imported “'+(recipe.title||recipe.name)+'”. Review the recipe before copying commands.');}catch(error){status('Could not import recipe: '+error.message);}el.value=''; } }); { const dl=document.getElementById('nb-models'); if(dl){const seen=new Set(); for(const o of [...dl.options]){const v=C.bareModel(o.value); if(!v||seen.has(v)){o.remove();continue;} seen.add(v); o.value=v;}} } render(); status(opened||''); + (async()=>{if(schema.local)try{const bad=await store.move();if(bad)status(bad+' old recipe'+(bad>1?'s':'')+' could not be moved.');}catch(e){status('Recipes saved in this browser were not moved to Nullius: '+e.message+'. Trying again next time.');}loadSaved();})(); })(); diff --git a/app/ui_tests/test_command_builder_execution.py b/app/ui_tests/test_command_builder_execution.py index b18bf2e0..088d4537 100644 --- a/app/ui_tests/test_command_builder_execution.py +++ b/app/ui_tests/test_command_builder_execution.py @@ -164,10 +164,11 @@ class DraftBetweenPages(CompilerCase): Opening a run's recipe or an explicit prefill takes precedence.""" def test_a_draft_packs_and_comes_back_unless_the_page_was_opened_for_something_else(self): - out = self.js('const r=C.create("harbor");r.tasks=9;r.title="mine";const text=C.packDraft({recipe:r,step:4,tab:"tau3-llm-agent"});' + out = self.js('const r=C.create("harbor");r.tasks=9;r.title="mine";const text=C.packDraft({recipe:r,step:4,tab:"tau3-llm-agent",saved:"0123456789abcdef"});' 'const back=C.unpackDraft(text,{});const pre=C.unpackDraft(text,{dataset:"x"});const run=C.unpackDraft(text,{run:"j"});const bad=C.unpackDraft("{nope",{});const none=C.unpackDraft(null,{});' - 'process.stdout.write(JSON.stringify([typeof text,back.recipe.tasks,back.recipe.title,back.step,back.tab,typeof back.at,pre,run,bad,none]));') - self.assertEqual(out, ['string', 9, 'mine', 4, 'tau3-llm-agent', 'string', None, None, None, None]) + 'process.stdout.write(JSON.stringify([typeof text,back.recipe.tasks,back.recipe.title,back.step,back.tab,typeof back.at,pre,run,bad,none,back.saved]));') + # `saved`: the saved recipe the page holds, so a save after a reload still updates (renames) it + self.assertEqual(out, ['string', 9, 'mine', 4, 'tau3-llm-agent', 'string', None, None, None, None, '0123456789abcdef']) if __name__ == "__main__": diff --git a/app/ui_tests/test_recipe_store_script.py b/app/ui_tests/test_recipe_store_script.py new file mode 100644 index 00000000..36e5ecb1 --- /dev/null +++ b/app/ui_tests/test_recipe_store_script.py @@ -0,0 +1,90 @@ +"""The builder's saved-recipe code in command-builder.js under Node, with a +stand-in fetch and storage: recipeStore (the list through the API, and the +one-time move of the list an earlier version kept in the browser) and +savedList (the dialog's rows, Delete greyed for a browser on another machine).""" +from __future__ import annotations + +import json +import shutil +import subprocess +import unittest +from pathlib import Path + +BUILDER = Path(__file__).resolve().parents[1] / "static" / "builder" / "command-builder.js" +OLD = {"nullius.recipes.v1": json.dumps([{"id": "harbor:a", "recipe": {"name": "a"}}]), + "nullius.recipes.v1.ws": json.dumps([{"id": "harbor:b", "recipe": {"name": "b"}}, {"broken": 1}]), + "nullius.recipes.v1.bad": "{", "nullius.recipe-draft.v1": "{\"keep\":1}", "other": "x"} +SENT = [{"id": "harbor:a", "recipe": {"name": "a"}}, {"id": "harbor:b", "recipe": {"name": "b"}}, {"broken": 1}, "{"] + + +class RecipeStoreScript(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.node = shutil.which("node") + if not cls.node: + raise unittest.SkipTest("Node is required for the builder's script") + + SETUP = ("const {recipeStore,savedList}=require(" + json.dumps(str(BUILDER)) + ");" + "const sent=[];let answer=()=>({ok:true,recipes:[]}),status=200;" + "const fetch=async(url,o={})=>{sent.push([o.method||'GET',url,o.body?JSON.parse(o.body):null,(o.headers||{})['X-CSRF-Token']||null]);" + "const j=answer(url,o);return {ok:status<400,status,json:async()=>j};};" + "const data=new Map(Object.entries(SEED));" + "const storage={get length(){return data.size;},key:i=>[...data.keys()][i],getItem:k=>data.has(k)?data.get(k):null," + "setItem:(k,v)=>data.set(k,String(v)),removeItem:k=>data.delete(k)};" + "const s=recipeStore({request:fetch,csrf:'tok',storage});") + + def run_js(self, seed, code): + program = "const SEED=" + json.dumps(seed) + ";" + self.SETUP + "(async()=>{" + code + "})().catch(e=>{process.stdout.write(JSON.stringify({thrown:e.message}));});" + out = subprocess.run([self.node, "-"], input=program, capture_output=True, text=True, check=True).stdout + return json.loads(out) + + def test_the_dialog_lists_saves_and_deletes_through_the_api(self): + out = self.run_js({}, "answer=(u,o)=>o.method==='POST'?{ok:true,recipe:{id:'0123456789abcdef'}}:{ok:true,recipes:[{id:'0123456789abcdef'}]};" + "const got=[await s.list(),await s.save({name:'a'},'0123456789abcdef')];await s.remove('0123456789abcdef');" + "status=403;answer=()=>({error:'Saved recipes change from a browser on the application host only.'});" + "try{await s.save({name:'b'});}catch(e){got.push(e.message);}" + "process.stdout.write(JSON.stringify({got,sent}));") + self.assertEqual(out["got"], [[{"id": "0123456789abcdef"}], {"id": "0123456789abcdef"}, "Saved recipes change from a browser on the application host only."]) + self.assertEqual([s[:2] for s in out["sent"]], [["GET", "/new-run/recipes"], ["POST", "/new-run/recipes"], + ["DELETE", "/new-run/recipes/0123456789abcdef"], ["POST", "/new-run/recipes"]]) + self.assertEqual((out["sent"][1][2], out["sent"][1][3]), ({"recipe": {"name": "a"}, "id": "0123456789abcdef"}, "tok")) + + def test_the_earlier_list_is_sent_once_leniently_and_its_keys_removed_after_the_server_answered(self): + out = self.run_js(OLD, "answer=()=>({ok:true,added:['x'],skipped:['a'],rejected:[]});const bad=await s.move();" + "const again=await s.move();process.stdout.write(JSON.stringify({bad,again,sent,left:[...data.keys()]}));") + self.assertEqual(out["sent"], [["POST", "/new-run/recipes/import", {"recipes": SENT, "lenient": True}, "tok"]]) + self.assertEqual((out["bad"], out["again"], out["left"]), (0, 0, ["nullius.recipe-draft.v1", "other"])) + + def test_the_recipes_the_server_refused_are_kept_apart_and_never_sent_again(self): + out = self.run_js({**OLD, "nullius.recipes.v1.rejected": json.dumps(["earlier"])}, + "answer=()=>({ok:true,added:['x','y'],skipped:[],rejected:[{index:2,reason:'r'},{index:3,reason:'r'}]});" + "const bad=await s.move();const again=await s.move();" + "process.stdout.write(JSON.stringify({bad,again,n:sent.length,sent:sent[0][2].recipes,left:[...data.keys()],kept:JSON.parse(data.get('nullius.recipes.v1.rejected'))}));") + self.assertEqual((out["bad"], out["again"], out["n"]), (2, 0, 1)) + self.assertEqual(out["sent"], SENT) # the key of earlier refusals is not sent + self.assertEqual(sorted(out["left"]), ["nullius.recipe-draft.v1", "nullius.recipes.v1.rejected", "other"]) + self.assertEqual(out["kept"], ["earlier", {"broken": 1}, "{"]) + + def test_a_failed_move_keeps_the_keys_for_the_next_visit(self): + out = self.run_js(OLD, "status=500;answer=()=>({});let err;try{await s.move();}catch(e){err=e.message;}" + "const kept=[...data.keys()];status=200;answer=()=>({ok:true,added:[],skipped:[],rejected:[]});await s.move();" + "process.stdout.write(JSON.stringify({err,kept,left:[...data.keys()],n:sent.length}));") + self.assertEqual(out["err"], "HTTP 500") + self.assertEqual(len(out["kept"]), 5) + self.assertEqual((out["left"], out["n"]), (["nullius.recipe-draft.v1", "other"], 2)) + + def test_delete_is_greyed_with_the_reason_for_a_browser_on_another_machine(self): + out = self.run_js({}, "const esc=x=>String(x).replace(/\"/g,'"'),button=(t,a)=>``,state=()=>'';" + "const saved=[{id:'0123456789abcdef',name:'n',saved:'2026-11-17T00:00:00Z',recipe:{harness:'harbor'}}];" + "const row=local=>savedList({saved,local,why:'Only here.',esc,button,state});" + "process.stdout.write(JSON.stringify([row(true),row(false),savedList({saved:[],local:false,why:'',esc,button,state})]));") + here, there, empty = out + self.assertIn('', here) + self.assertIn('', there) + self.assertNotIn("data-delete", there) + self.assertIn('data-load="0123456789abcdef"', there) # Load still works there + self.assertEqual(empty, '

No saved recipes.

') + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/builder.md b/docs/builder.md index 032d0ae5..1b169e73 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -89,7 +89,7 @@ a declared model list keeps those published choices. | What | Where | When | | --- | --- | --- | | Draft | The browser, one per browser | Saved on every change; *Start over* forgets it | -| Saved recipes | The browser | Save, Load, Delete, import and export as JSON | +| Saved recipes | `recipes/.json` under the user data directory, one file each | Any browser lists and loads them. Save and Delete work only from a browser on the application host. Saving a name that exists updates that recipe, and saving a loaded recipe under a new name renames it. Import and export as JSON. A list kept in the browser by an earlier version moves here when the builder opens on the host. Recipes the server refuses stay in the browser under `nullius.recipes.v1.rejected` | | A run's recipe | `runs//run.json` | Job page, *New run like this*: the same recipe, at Review | | A job's record | Its `config.json` and `lock.json` | *New run like this* without a kept recipe: rebuilt, what it cannot carry named | diff --git a/docs/concepts.md b/docs/concepts.md index bbccc76e..16f3eb67 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -46,7 +46,7 @@ Importing results needs no runner. | Object | Holds | Where | | --- | --- | --- | -| Recipe | Runner, benchmark, dataset, agents, models, limits, machines | The builder; saved in the browser or exported as JSON | +| Recipe | Runner, benchmark, dataset, agents, models, limits, machines | The builder; saved in the user data directory or exported as JSON | | Launch | The runs one press of **Run** starts | Runs, *By launch*; every launch, in a workspace or not, on **Launches** | | Run | One agent, or one check (`nop`, `oracle`), on the launch's tasks. Harbor calls it a job | Runs, one row | | Trial | One task attempted by one run | A run's page | diff --git a/docs/getting-started.md b/docs/getting-started.md index d05ed65e..58e5f467 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -96,8 +96,8 @@ dependency; benchmark and agent environments stay separate. Save provider credentials in **Settings**. The owner-only `settings.json` lives in the user configuration directory, and environment variables override it. -`nullius paths` lists settings, licences, user catalogue, workspaces, runs and -cache paths without showing credentials. Linux defaults use XDG directories; +`nullius paths` lists settings, licences, user catalogue, workspaces, runs, saved +recipes and cache paths without showing credentials. Linux defaults use XDG directories; see [Settings](settings/README.md#storage). For a separate profile, use the same `NULLIUS_HOME` for each command: diff --git a/docs/settings/README.md b/docs/settings/README.md index 4b5f4e4f..c3090efa 100644 --- a/docs/settings/README.md +++ b/docs/settings/README.md @@ -84,6 +84,7 @@ directory and installation: | User catalogue | `~/.local/share/nullius/catalogue/` | | Workspaces | `~/.local/share/nullius/workspaces/` | | Runs started from Review | `~/.local/share/nullius/runs/` | +| Saved recipes | `~/.local/share/nullius/recipes/` | | Provider test results | `~/.local/share/nullius/provider-tests.json` | | Standard installs and data | `~/.local/share/nullius/standard/` | | Caches | `~/.cache/nullius/` | From f41597e8c153a1df0772877d324c62a508412128 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 10:45:58 +0100 Subject: [PATCH 5/5] fix(builder): keep and count the old recipes whose name is already saved The move to the server skipped them, then removed them from the browser. One with the same name but other settings was lost. They are now kept beside the refused ones, and the toast counts both. --- app/nullius/recipes/store.py | 9 +++++---- app/static/builder/command-builder.js | 13 +++++++------ app/tests/test_recipe_store.py | 2 +- app/ui_tests/test_recipe_routes.py | 2 +- app/ui_tests/test_recipe_store_script.py | 7 ++++--- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/nullius/recipes/store.py b/app/nullius/recipes/store.py index a445be5b..2bcbd145 100644 --- a/app/nullius/recipes/store.py +++ b/app/nullius/recipes/store.py @@ -117,22 +117,23 @@ def import_(items: Iterable, lenient: bool = False) -> Dict[str, list]: """Add each recipe whose name is not saved yet; never replace one. An item is a recipe, or ``{"recipe": ...}`` as the builder's earlier browser list held them. Strict: one bad item refuses them all, before anything is written. Lenient: the - good ones are added and each bad one is returned with its reason.""" + good ones are added and each bad one is returned with its reason. A skipped one is + returned with its index and name.""" items = list(items) checked, rejected = [], [] for n, item in enumerate(items): recipe = item.get("recipe") if isinstance(item, dict) and "recipe" in item else item try: - checked.append(entry(recipe)) + checked.append((n, entry(recipe))) except ValueError as e: if not lenient: raise ValueError(f"recipe {n + 1} of {len(items)}: {e}") from None rejected.append({"index": n, "reason": str(e)}) taken = {x.get("name") for x in listing()} added, skipped = [], [] - for item in checked: + for n, item in checked: if item["name"] in taken: - skipped.append(item["name"]) + skipped.append({"index": n, "name": item["name"]}) continue item = {"id": secrets.token_hex(8), **item} _write(item) diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 2071898a..43154614 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -54,8 +54,8 @@ // Saved recipes are the server's (/new-run/recipes, nullius/recipes/store.py); a browser on its host alone writes them. // move() sends what this browser kept before (nullius.recipes.v1, and the older nullius.recipes.v1. lists) // to the lenient import, which skips names already saved, and removes those keys once the server answered. The - // recipes it refused are kept under REJECTED, never sent again; move() returns how many. On a failure the keys stay - // for the next visit. `request` is nulliusFetch; the tests run this under Node with stand-ins. + // recipes it refused or skipped are kept under REJECTED, never sent again; move() returns how many of each. On a + // failure the keys stay for the next visit. `request` is nulliusFetch; the tests run this under Node with stand-ins. function recipeStore({request, csrf, storage}) { const call = async (url, method, body) => { const res = await request(url, method ? {method, headers: {'Content-Type': 'application/json', 'X-CSRF-Token': csrf}, body: JSON.stringify(body || {})} : {}); @@ -72,11 +72,12 @@ async move() { const keys = Array.from({length: storage.length}, (_, i) => storage.key(i)).filter(k => k !== REJECTED && (k === OLD || k.startsWith(OLD + '.'))); const recipes = keys.flatMap(read); - if (!keys.length) return 0; - const bad = recipes.length ? (await call('/new-run/recipes/import', 'POST', {recipes, lenient: true})).rejected.map(x => recipes[x.index]) : []; + if (!keys.length) return {refused: 0, taken: 0}; + const out = recipes.length ? await call('/new-run/recipes/import', 'POST', {recipes, lenient: true}) : {rejected: [], skipped: []}; + const bad = [...out.rejected, ...out.skipped].map(x => recipes[x.index]); if (bad.length) storage.setItem(REJECTED, JSON.stringify([...(storage.getItem(REJECTED) ? read(REJECTED) : []), ...bad])); keys.forEach(k => storage.removeItem(k)); - return bad.length; + return {refused: out.rejected.length, taken: out.skipped.length}; }, }; } @@ -1155,5 +1156,5 @@ { const dl=document.getElementById('nb-models'); if(dl){const seen=new Set(); for(const o of [...dl.options]){const v=C.bareModel(o.value); if(!v||seen.has(v)){o.remove();continue;} seen.add(v); o.value=v;}} } render(); status(opened||''); - (async()=>{if(schema.local)try{const bad=await store.move();if(bad)status(bad+' old recipe'+(bad>1?'s':'')+' could not be moved.');}catch(e){status('Recipes saved in this browser were not moved to Nullius: '+e.message+'. Trying again next time.');}loadSaved();})(); + (async()=>{if(schema.local)try{const {refused,taken}=await store.move(),n=(k,w)=>k+' old recipe'+(k>1?'s':'')+w;const say=[refused&&n(refused,' could not be moved.'),taken&&n(taken,' not moved: '+(taken>1?'their names are':'its name is')+' already saved.')].filter(Boolean);if(say.length)status(say.join(' '));}catch(e){status('Recipes saved in this browser were not moved to Nullius: '+e.message+'. Trying again next time.');}loadSaved();})(); })(); diff --git a/app/tests/test_recipe_store.py b/app/tests/test_recipe_store.py index f3cc8bac..ce570587 100644 --- a/app/tests/test_recipe_store.py +++ b/app/tests/test_recipe_store.py @@ -102,7 +102,7 @@ def test_a_strict_import_adds_new_names_skips_saved_ones_and_writes_nothing_when # the builder's earlier browser list: {id, updated, recipe} out = store.import_([{"id": "harbor:there", "updated": "1/2/2026", "recipe": recipe("there", limit=9)}, {"id": "harbor:new", "recipe": recipe("new")}, recipe("bare"), recipe("new")]) - self.assertEqual((len(out["added"]), out["skipped"], out["rejected"]), (2, ["there", "new"], [])) + self.assertEqual((len(out["added"]), out["skipped"], out["rejected"]), (2, [{"index": 0, "name": "there"}, {"index": 3, "name": "new"}], [])) self.assertEqual({x["name"]: x["recipe"].get("limit") for x in store.listing()}, {"there": 1, "new": None, "bare": None}) with self.assertRaisesRegex(ValueError, "recipe 2 of 2"): store.import_([recipe("fine"), {"recipe": recipe("x", "../x")}]) diff --git a/app/ui_tests/test_recipe_routes.py b/app/ui_tests/test_recipe_routes.py index 6ac36836..c76c1591 100644 --- a/app/ui_tests/test_recipe_routes.py +++ b/app/ui_tests/test_recipe_routes.py @@ -66,7 +66,7 @@ def test_import_takes_one_or_several_and_skips_saved_names(self): store.save(recipe("there", limit=1)) j = self.send("POST", "/new-run/recipes/import", {"recipes": [{"id": "harbor:there", "recipe": recipe("there", limit=5)}, {"id": "harbor:b", "recipe": recipe("b")}]}).get_json() - self.assertEqual((j["ok"], len(j["added"]), j["skipped"], j["rejected"]), (True, 1, ["there"], [])) + self.assertEqual((j["ok"], len(j["added"]), j["skipped"], j["rejected"]), (True, 1, [{"index": 0, "name": "there"}], [])) self.assertEqual(len(self.send("POST", "/new-run/recipes/import", recipe("c")).get_json()["added"]), 1) self.assertEqual(len(self.send("POST", "/new-run/recipes/import", [recipe("d")]).get_json()["added"]), 1) self.assertEqual(dict(self.listed())["there"], 1) diff --git a/app/ui_tests/test_recipe_store_script.py b/app/ui_tests/test_recipe_store_script.py index 36e5ecb1..e96da01f 100644 --- a/app/ui_tests/test_recipe_store_script.py +++ b/app/ui_tests/test_recipe_store_script.py @@ -50,17 +50,18 @@ def test_the_dialog_lists_saves_and_deletes_through_the_api(self): self.assertEqual((out["sent"][1][2], out["sent"][1][3]), ({"recipe": {"name": "a"}, "id": "0123456789abcdef"}, "tok")) def test_the_earlier_list_is_sent_once_leniently_and_its_keys_removed_after_the_server_answered(self): - out = self.run_js(OLD, "answer=()=>({ok:true,added:['x'],skipped:['a'],rejected:[]});const bad=await s.move();" + out = self.run_js(OLD, "answer=()=>({ok:true,added:['x'],skipped:[{index:0,name:'a'}],rejected:[]});const bad=await s.move();" "const again=await s.move();process.stdout.write(JSON.stringify({bad,again,sent,left:[...data.keys()]}));") self.assertEqual(out["sent"], [["POST", "/new-run/recipes/import", {"recipes": SENT, "lenient": True}, "tok"]]) - self.assertEqual((out["bad"], out["again"], out["left"]), (0, 0, ["nullius.recipe-draft.v1", "other"])) + self.assertEqual((out["bad"], out["again"]), ({"refused": 0, "taken": 1}, {"refused": 0, "taken": 0})) + self.assertEqual(sorted(out["left"]), ["nullius.recipe-draft.v1", "nullius.recipes.v1.rejected", "other"]) # a name already saved stays in this browser def test_the_recipes_the_server_refused_are_kept_apart_and_never_sent_again(self): out = self.run_js({**OLD, "nullius.recipes.v1.rejected": json.dumps(["earlier"])}, "answer=()=>({ok:true,added:['x','y'],skipped:[],rejected:[{index:2,reason:'r'},{index:3,reason:'r'}]});" "const bad=await s.move();const again=await s.move();" "process.stdout.write(JSON.stringify({bad,again,n:sent.length,sent:sent[0][2].recipes,left:[...data.keys()],kept:JSON.parse(data.get('nullius.recipes.v1.rejected'))}));") - self.assertEqual((out["bad"], out["again"], out["n"]), (2, 0, 1)) + self.assertEqual((out["bad"], out["again"], out["n"]), ({"refused": 2, "taken": 0}, {"refused": 0, "taken": 0}, 1)) self.assertEqual(out["sent"], SENT) # the key of earlier refusals is not sent self.assertEqual(sorted(out["left"]), ["nullius.recipe-draft.v1", "nullius.recipes.v1.rejected", "other"]) self.assertEqual(out["kept"], ["earlier", {"broken": 1}, "{"])