Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions app/nullius/catalogue/benchmarks/osworld1/standard.json
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,20 @@
"original",
"model"
],
"names": {
"m3": "M3 Agent",
"prompt": "PromptAgent",
"claude": "Claude computer use",
"gpt54": "GPT-5.4",
"openai_cua": "OpenAI CUA",
"qwen": "Qwen",
"os_symphony": "OS-Symphony",
"evocua": "EvoCUA",
"autoglm": "AutoGLM",
"autoglm_v": "AutoGLM-V",
"mano": "Mano",
"muse_spark": "Muse Spark"
},
"about": {
"m3": "MiniMax's M3Agent: run_multienv_m3.py, MiniMax-M3 over Anthropic's Messages API, 100 steps",
"prompt": "OSWorld's PromptAgent: run_multienv.py, GPT-4o on screenshots over Chat Completions, 15 steps",
Expand Down Expand Up @@ -205,6 +219,7 @@
},
"vm_provider": {
"label": "VM provider",
"step": "where",
"summary": "--provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own.",
"values": [
"docker"
Expand Down
13 changes: 11 additions & 2 deletions app/nullius/standard/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--attempts K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS ...] [--max-turns N] [--max-calls N] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N]
PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N]
PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...]
PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID]
PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force]
PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed
Expand Down Expand Up @@ -636,11 +636,16 @@ def cmd_fidelity(a):

def cmd_prove(a):
b = _bench(a.benchmark)
jobs = options(a.job)
for c in jobs:
if c not in definitions.CONTROLS:
sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}")
held, spill = licences(b, from_run=a.from_run)
try:
got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model,
stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch,
env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images)
env=run_env(a.provider) if a.from_run else None, licences=held,
bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images)
except run.Refused as e:
print("refused:", e)
return 2
Expand Down Expand Up @@ -765,6 +770,10 @@ def main(argv=None):
v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)")
v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only")
v.add_argument("--launch", help="the launch id the control jobs of every shard record")
v.add_argument("--bench-option", action="append", metavar="NAME=VALUE",
help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs")
v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR",
help="a finished control job over the whole dataset, checked against its bound instead of running the control")
v.add_argument("--remove-images", action="store_true",
help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)")
g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job")
Expand Down
10 changes: 7 additions & 3 deletions app/nullius/standard/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi
model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(),
key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None,
ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None,
max_calls: Optional[int] = None) -> List[str]:
timeout: Optional[int] = None, max_calls: Optional[int] = None) -> List[str]:
"""``docker run`` for a unit's relay: on the default bridge (its way to the
provider), its port published on 127.0.0.1 only. The key is never given:
``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with
Expand Down Expand Up @@ -764,6 +764,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi
a += ["--served-by", f"{m}={','.join(slugs)}"]
if key_header != "authorization":
a += ["--key-header", key_header]
if timeout:
a += ["--timeout", str(timeout)]
return a


Expand Down Expand Up @@ -1025,11 +1027,12 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str,
side_models=(), key_header: str = "authorization", servers: Iterable[str] = (),
doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None,
ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True,
serving: Optional[Dict[str, List[str]]] = None, max_calls: Optional[int] = None):
serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None, max_calls: Optional[int] = None):
self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app
self.max_calls = max_calls # the run's cap on its agent's model calls, when it has one
self.ledger = ledger # the file of the run's spending cap, when it has one
self.serving = dict(serving or {}) # by model, the serving providers that may answer it
self.timeout = timeout
self.servers = list(servers) # licence servers, host:port each: a forwarder apiece
self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece
self.side_models, self.key_header = tuple(side_models), key_header
Expand All @@ -1055,7 +1058,8 @@ def start(self) -> "Unit":
argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir,
key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes,
prices=self.prices, side_models=self.side_models, key_header=self.key_header,
hosts=self.hosts, ledger=self.ledger, serving=self.serving, max_calls=self.max_calls)
hosts=self.hosts, ledger=self.ledger, serving=self.serving,
timeout=self.timeout, max_calls=self.max_calls)
self.started.append(self.names["relay"])
_docker(argv[1:])
_docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]])
Expand Down
42 changes: 36 additions & 6 deletions app/nullius/standard/definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ def _bench_options(d: dict) -> List[str]:
``embeddings`` model called through the run's relay, ``host`` requirements or ``files`` by name - and
``when`` it applies: another option's values (a retriever's, only with a retrieval). ``limit``:
a bound the upstream has of its own (its steps, its errors), offered with the run's limits; ``by``:
its default under each value of another option (an agent's own steps), for a default of ``default``."""
its default under each value of another option (an agent's own steps), for a default of ``default``. ``names``: a name for some of its values, shown in their place. ``step``: ``where`` for a choice of where the work runs, ``dataset`` for one its datasets need, the builder step that asks it."""
opts = d.get("options")
if opts is None:
return []
Expand All @@ -490,6 +490,8 @@ def _bench_options(d: dict) -> List[str]:
bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability)")
if "limit" in o and (not isinstance(o["limit"], bool) or "pattern" not in o):
bad.append(f"{w}.limit: true for a bound the upstream has of its own, a number its pattern allows")
if o.get("step", "dataset") != "dataset" and (o["step"] != "where" or "values" not in o) or "step" in o and o.get("limit"):
bad.append(f"{w}.step: where, a choice of where the work runs among its values, or dataset")
by = o.get("by")
if by is not None and not (isinstance(by, dict) and len(by) == 1 and all(
k != name and k in opts and isinstance(m, dict) and set(m) == set(opts[k].get("values") or [None])
Expand All @@ -501,6 +503,10 @@ def _bench_options(d: dict) -> List[str]:
about = o.get("about") or {}
if not isinstance(about, dict) or any(option_problem(o, k) is not None or not isinstance(v, str) for k, v in about.items()):
bad.append(f"{w}.about: a line for some of its values")
names = o.get("names", {})
if not isinstance(names, dict) or any(k not in (o.get("values") or []) or not isinstance(v, str) or not v.strip() for k, v in names.items()) \
or len(set(names.values())) != len(names):
bad.append(f"{w}.names: a name for some of its values")
needs = o.get("needs") or {}
if not isinstance(needs, dict):
bad.append(f"{w}.needs: value -> {{embeddings, reranker, host, files}}")
Expand Down Expand Up @@ -1044,6 +1050,25 @@ def control(bench: dict, which: str) -> Optional[dict]:
"_kind": "control", "_ref": which, "_here": bench.get("_here", "")}


def comparable(orig: dict, model: str) -> bool:
"""Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do."""
c = orig.get("comparable", True)
return model in c if isinstance(c, list) else bool(c)


def upstream_name(orig: dict, model: str, provider: str) -> str:
"""``model`` by the name the original's upstream knows, when given as ``provider`` lists it."""
pre = (orig.get("providers") or {}).get(provider, "")
names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []}
return model if model in (orig.get("models") or []) else names.get(model, model)


def paper_models(orig: dict) -> List[str]:
"""The original's models whose runs compare with the paper's."""
c = orig.get("comparable", True)
return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c]


def _leaderboard(d: dict) -> List[str]:
"""``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the
upstream's leaderboard fixes besides the ranked model, from its own submission
Expand Down Expand Up @@ -1131,8 +1156,11 @@ def _setups(s) -> List[str]:
bad.append(f"{w}.turn_seconds: a whole number of seconds")
if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()):
bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase")
if "comparable" in x and not isinstance(x["comparable"], bool):
bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only")
c = x.get("comparable", True)
if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c)
or x.get("default") not in c):
bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; "
"or those of its models whose runs compare with the paper's, its default among them")
else:
allowed |= {"runs_as"}
orig = s.get("original") if isinstance(s.get("original"), dict) else None
Expand Down Expand Up @@ -1224,7 +1252,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple
model = model or orig["default"]
if model not in orig["models"]:
return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): "
f"{model} is not one - choose the model setup for another")
f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else ""))
if provider and provider not in orig["providers"]:
return None, (f"the paper's models are not served by {provider}: the original runs on "
+ " or ".join(orig["providers"]))
Expand All @@ -1240,7 +1268,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple
asked, sent = x["runs_as"], model
units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"}
return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}",
"summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True),
"summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model),
"turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which,
"_here": bench.get("_here", "")}, ""

Expand Down Expand Up @@ -1322,14 +1350,16 @@ def problems(d: dict, kind: str) -> List[str]:
if not isinstance(lim, dict):
bad.append("limits: an object")
else:
for f in ("turn_seconds", "turns"):
for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"):
v = lim.get(f)
if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000):
bad.append(f"limits.{f}: a whole number from 1")
a = lim.get("attempts")
if a is not None and (not isinstance(a, str) or not a.strip()):
bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), "
"in a sentence - the driver is then sent attempt with setup, turn and answer")
if "gated" in d and not isinstance(d["gated"], bool):
bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept")
s = d.get("score") or {}
if s.get("kind") not in SCORES or not s.get("says"):
bad.append(f"score: {{kind: one of {SCORES}, says}}")
Expand Down
4 changes: 2 additions & 2 deletions app/nullius/standard/fidelity.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa
# records side by side, with the fields fidelity reads.
r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model,
side_models=(agent.get("model") or {}).get("side_models") or (),
key_header=r.key_header, fields=True).start()
key_header=r.key_header, fields=True, timeout=r.call_seconds).start()
order = [(s, n) for n in range(1, passes + 1) for s in SIDES]
got: Dict[str, List[dict]] = {s: [] for s in SIDES}
solved: Dict[str, List[str]] = {}
Expand Down Expand Up @@ -230,7 +230,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k
except (run._Ended, jsonl.Failed) as e:
r.say(f" {tag} {unit}: {e}")
try:
score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600)
score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds)
except jsonl.Failed:
score = {}
if os.path.isdir(os.path.join(base, "agent")):
Expand Down
Loading
Loading