Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ and its resources. The source layout stays under `app/`.
a Harbor benchmark added from the Benchmarks page's form; `citation`, an entry's
BibTeX read into the paper its page shows; `storage`, whole-folder user overlays and restoring bundled entries; `sources`, where each dataset's data comes
from for each way it runs), `recipes/`
(`launch`, the builder's vocabulary), `providers` (the model provider registry: protocols, presets and each provider's record, read by every consumer; and the builder's steps),
(`launch`, the builder's vocabulary; `store`, its saved recipes), `providers` (the model provider registry: protocols, presets and each provider's record, read by every consumer; and the builder's steps),
`harbor/` (`source`: the Harbor a recipe pins - built, kept, scanned for its agents;
`feed`: the live feed any Harbor writes, from its own trial hooks),
`importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`),
Expand Down Expand Up @@ -116,7 +116,7 @@ a recipe without a pin), and the viewer never imports it.

`nullius paths` lists persistent locations. On Linux, settings, the model price
list and benchmark licences live under `~/.config/nullius/`, user catalogue entries, workspaces,
runs and standard installs under `~/.local/share/nullius/`, and caches under
runs, saved builder recipes and standard installs under `~/.local/share/nullius/`, and caches under
`~/.cache/nullius/`. XDG variables move those roots; `NULLIUS_HOME` selects a
separate profile with `config/`, `data/` and `cache/`. Existing storage overrides
still apply. Settings saves owner-only JSON; process environment values win.
Expand Down
6 changes: 5 additions & 1 deletion app/nullius/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ def runs_dir(env=None) -> str:
return env.get("RUNS_DIR") or os.path.join(data_dir(env), "runs")


def recipes_dir(env=None) -> str:
return os.path.join(data_dir(env), "recipes")


def secrets_dir(env=None) -> str:
env = os.environ if env is None else env
return env.get("SECRETS_DIR") or os.path.join(config_dir(env), "secrets")
Expand All @@ -76,7 +80,7 @@ def main(argv=None) -> int:
parser.add_argument("--json", action="store_true")
args = parser.parse_args(argv)
values = {"settings": settings_file(), "secrets": secrets_dir(), "prices": prices_file(), "catalogue": catalogue_dir(),
"workspaces": workspaces_dir(), "runs": runs_dir(), "cache": cache_dir()}
"workspaces": workspaces_dir(), "runs": runs_dir(), "recipes": recipes_dir(), "cache": cache_dir()}
if args.json:
print(json.dumps(values, indent=2))
else:
Expand Down
4 changes: 2 additions & 2 deletions app/nullius/recipes/launch.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,14 @@ def parameter(key, label, group, kind, default=None, *, help="", **extra):
}


def payload(cat, jobs, workspace_name=""):
def payload(cat, jobs):
from nullius import providers
from nullius.standard import recipe as standard
models = sorted({j.model_name for j in jobs if j.model_name})
dated = [j for j in jobs if j.model_name and j.started_at]
latest = max(dated, key=lambda j: j.started_at.timestamp()).model_name if dated else ""
return {
"version": 1, "workspace": workspace_name or "unassigned", "agents": AGENTS,
"version": 1, "agents": AGENTS,
# Harbor and the Nullius standard: the page offers choices, the core and standard/recipe.py compile them.
"runners": [standard.RUNNER, providers.HARBOR],
# why a saved recipe of a runner no longer offered does not open (a benchmark's own scripts: its Original setup)
Expand Down
142 changes: 142 additions & 0 deletions app/nullius/recipes/store.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
"""Saved builder recipes: one owner-only JSON file per recipe under the
profile's data directory (``recipes/<id>.json``).

A file holds ``id``, ``name`` (the title the builder shows), ``recipe`` (the
recipe as the builder saves it) and ``saved`` (UTC, ISO 8601). The id is
random, so two names can never share a file; names are unique instead. A save
names the recipe it updates by id (a rename keeps the file), or else updates
the recipe of the same name. Standard library only.
"""
from __future__ import annotations

import json
import os
import re
import secrets
import tempfile
import time
from typing import Dict, Iterable, List, Optional

from nullius import paths

ID = re.compile(r"^[0-9a-f]{16}$")
_HARNESS = re.compile(r"^[a-z0-9][a-z0-9-]{0,31}$")
_NAME = re.compile(r"^[a-z0-9][a-z0-9._-]{0,71}$")
MAX_BYTES = 200_000 # as the builder's other recipe routes
MAX_NAME = 200


def root(env=None) -> str:
return paths.recipes_dir(env)


def _path(recipe_id: str) -> str:
if not isinstance(recipe_id, str) or not ID.match(recipe_id):
raise ValueError(f"{recipe_id!r} is not a recipe id")
return os.path.join(root(), recipe_id + ".json")


def entry(recipe) -> Dict:
"""The stored form of one recipe, without its id, or ValueError saying what is wrong with it."""
if not isinstance(recipe, dict) or recipe.get("version") != 1:
raise ValueError("a recipe is a JSON object with \"version\": 1")
harness, name = recipe.get("harness") or "harbor", recipe.get("name")
if not isinstance(harness, str) or not _HARNESS.match(harness):
raise ValueError(f"{harness!r} is not a runner")
if not isinstance(name, str) or not _NAME.match(name):
raise ValueError(f"{name!r} is not a recipe name (lowercase letters, digits, '.', '_', '-')")
if len(json.dumps(recipe)) > MAX_BYTES:
raise ValueError(f"recipe {name!r} is larger than {MAX_BYTES // 1000} kB")
title = recipe.get("title")
title = title.strip() if isinstance(title, str) and title.strip() else name
if len(title) > MAX_NAME:
raise ValueError(f"a recipe's name is at most {MAX_NAME} characters")
return {"name": title, "saved": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "recipe": recipe}


def _write(item: Dict) -> None:
"""Written whole under a temporary 0600 name beside it, then renamed over the old one."""
path = _path(item["id"])
os.makedirs(root(), mode=0o700, exist_ok=True)
fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=root())
try:
with os.fdopen(fd, "w", encoding="utf-8") as fh:
json.dump(item, fh, indent=2, ensure_ascii=False)
fh.write("\n")
os.replace(tmp, path)
except BaseException:
try:
os.unlink(tmp)
except OSError:
pass
raise


def listing() -> List[Dict]:
"""Every saved recipe, the latest saved first; an unreadable file is left out."""
out = []
try:
names = os.listdir(root())
except OSError:
return []
for f in names:
if not f.endswith(".json") or not ID.match(f[:-5]):
continue
try:
with open(os.path.join(root(), f), encoding="utf-8") as fh:
got = json.load(fh)
except (OSError, ValueError):
continue
if isinstance(got, dict) and got.get("id") == f[:-5] and isinstance(got.get("recipe"), dict):
out.append(got)
return sorted(out, key=lambda x: (str(x.get("saved", "")), x["id"]), reverse=True)


def save(recipe, recipe_id: Optional[str] = None) -> Dict:
"""Update the recipe ``recipe_id`` (renamed when its name changed), or else the
recipe of the same name, or else add one. A name another recipe holds is refused."""
item = entry(recipe)
held = recipe_id if recipe_id and os.path.exists(_path(recipe_id)) else None
same = next((x["id"] for x in listing() if x.get("name") == item["name"]), None)
if held and same and same != held:
raise ValueError(f"another saved recipe is named {item['name']!r}")
item = {"id": held or same or secrets.token_hex(8), **item}
_write(item)
return item


def delete(recipe_id: str) -> bool:
try:
os.unlink(_path(recipe_id))
return True
except FileNotFoundError:
return False


def import_(items: Iterable, lenient: bool = False) -> Dict[str, list]:
"""Add each recipe whose name is not saved yet; never replace one. An item is a
recipe, or ``{"recipe": ...}`` as the builder's earlier browser list held them.
Strict: one bad item refuses them all, before anything is written. Lenient: the
good ones are added and each bad one is returned with its reason. A skipped one is
returned with its index and name."""
items = list(items)
checked, rejected = [], []
for n, item in enumerate(items):
recipe = item.get("recipe") if isinstance(item, dict) and "recipe" in item else item
try:
checked.append((n, entry(recipe)))
except ValueError as e:
if not lenient:
raise ValueError(f"recipe {n + 1} of {len(items)}: {e}") from None
rejected.append({"index": n, "reason": str(e)})
taken = {x.get("name") for x in listing()}
added, skipped = [], []
for n, item in checked:
if item["name"] in taken:
skipped.append({"index": n, "name": item["name"]})
continue
item = {"id": secrets.token_hex(8), **item}
_write(item)
taken.add(item["name"])
added.append(item["id"])
return {"added": added, "skipped": skipped, "rejected": rejected}
4 changes: 2 additions & 2 deletions app/static/builder/command-builder-core.js
Original file line number Diff line number Diff line change
Expand Up @@ -829,7 +829,7 @@
// opened for something else - a prefill from a run, or a run's own recipe.
const OPENED_FOR = ['run','like','dataset','model','agent','n_tasks','standard','other'];
function packDraft(state) {
return JSON.stringify({version: 1, at: new Date().toISOString(), recipe: state.recipe, step: state.step, steps: STEPS.length, first: 'benchmark', tab: state.tab});
return JSON.stringify({version: 1, at: new Date().toISOString(), recipe: state.recipe, step: state.step, steps: STEPS.length, first: 'benchmark', tab: state.tab, saved: state.saved || ''});
}
function unpackDraft(text, query) {
if (!text || Object.keys(query || {}).some(k => OPENED_FOR.includes(k))) return null;
Expand All @@ -842,7 +842,7 @@
if (Number.isInteger(d.step) && had < STEPS.length && step >= STEP.setup) step += 1;
if (d.first !== 'benchmark' && (step === 0 || step === 1)) step = 1 - step; // a draft from when the runner came first
return {recipe: fromJSON(d.recipe), step: Math.min(step, STEP.review), tab: typeof d.tab === 'string' ? d.tab : 'select',
at: String(d.at || '')};
at: String(d.at || ''), saved: typeof d.saved === 'string' ? d.saved : ''};
} catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; }
}
return {STANDARD_PHASES,STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById};
Expand Down
1 change: 1 addition & 0 deletions app/static/builder/command-builder.css

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading