From fa8fa81c8cf4dc05342d3e65f211c6965543db3e Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 17:21:29 +0100 Subject: [PATCH 1/8] feat(standard): list a benchmark's own agents as agents of the catalogue OSWorld 1 and 2 kept their upstream's agent scripts as a benchmark option, written three times (options.agent, agents.json, setups.original.models). Each agents.json entry is now an agent definition that starts with a new, optional start kind of nullius.agent/v1, start.driver: the upstream's own value and the script, argv and env its driver runs. Where an agent comes from and a benchmark agent's only_for are derived from where it sits. The loader rebuilds options.agent's values, names, about and default from the file, so drivers, --bench-option agent=... and kept run scripts see the option as before; problems() keeps the entries' models within the Original setup's. match.fit binds a benchmark's own agent to its benchmark and checks no port. The Agents page lists them behind a From filter whose default leaves them out; their card shows the benchmark and its published models. --- .../catalogue/benchmarks/osworld1/agents.json | 698 ++++++++++-------- .../catalogue/benchmarks/osworld1/driver.py | 3 + .../benchmarks/osworld1/standard.json | 45 +- .../catalogue/benchmarks/osworld2/agents.json | 118 +-- .../benchmarks/osworld2/benchmark.json | 2 +- .../catalogue/benchmarks/osworld2/driver.py | 3 + .../benchmarks/osworld2/standard.json | 14 +- app/nullius/standard/definitions.py | 135 ++++ app/nullius/standard/match.py | 11 +- app/nullius/standard/recipe.py | 3 +- app/templates/_cards.html | 4 +- app/templates/_tables.html | 10 +- app/templates/agents/standard.html | 8 +- app/tests/catalogue_docs.py | 3 +- app/tests/test_benchmark_agents.py | 119 +++ app/tests/test_osworld2_judge.py | 5 +- app/tests/test_osworld2_muse.py | 7 +- app/ui_tests/test_agents.py | 19 + app/web/agents.py | 39 +- docs/agents/add-manually.md | 4 + docs/benchmarks/supported/osworld2.md | 2 +- docs/standard.md | 26 + 22 files changed, 837 insertions(+), 441 deletions(-) create mode 100644 app/tests/test_benchmark_agents.py diff --git a/app/nullius/catalogue/benchmarks/osworld1/agents.json b/app/nullius/catalogue/benchmarks/osworld1/agents.json index 65179a64..c48abc7e 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/agents.json +++ b/app/nullius/catalogue/benchmarks/osworld1/agents.json @@ -1,5 +1,5 @@ { - "about": "The upstream's agents the driver runs, one per value of the agent option: each its own script at the tested commit, as published, on one task. argv is what the script's own documented invocation passes (its README, docstring or scripts/bash wrapper), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the task's meta file, the unit's results); env points the script's model client at the unit's relay. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1, {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them. aws: the script imports the upstream's AWS manager whatever the provider, which needs these variables set; no AWS call is made on docker.", + "about": "The upstream's agents the driver runs, each an agent of the catalogue (nullius.agent/v1, start.driver), its upstream value the agent option's: each its own script at the tested commit, as published, on one task. argv is what the script's own documented invocation passes (its README, docstring or scripts/bash wrapper), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the task's meta file, the unit's results); env points the script's model client at the unit's relay. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1, {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them. aws: the script imports the upstream's AWS manager whatever the provider, which needs these variables set; no AWS call is made on docker.", "common": [ "--provider_name", "docker", @@ -22,371 +22,467 @@ }, "agents": { "m3": { - "script": "scripts/python/run_multienv_m3.py", + "key": "m3", + "label": "M3 Agent", + "summary": "MiniMax's M3Agent: run_multienv_m3.py, MiniMax-M3 over Anthropic's Messages API, 100 steps", "models": [ "MiniMax-M3" ], - "argv": [ - "--observation_type", - "screenshot", - "--max_steps", - "100", - "--model", - "{model}" - ], - "env": { - "ANTHROPIC_BASE_URL": "{base}", - "ANTHROPIC_API_KEY": "{key}" + "start": { + "driver": { + "agent": "m3", + "script": "scripts/python/run_multienv_m3.py", + "argv": [ + "--observation_type", + "screenshot", + "--max_steps", + "100", + "--model", + "{model}" + ], + "env": { + "ANTHROPIC_BASE_URL": "{base}", + "ANTHROPIC_API_KEY": "{key}" + } + } } }, "prompt": { - "script": "scripts/python/run_multienv.py", + "key": "prompt", + "label": "PromptAgent", + "summary": "OSWorld's PromptAgent: run_multienv.py, GPT-4o on screenshots over Chat Completions, 15 steps", "models": [ "gpt-4o" ], - "argv": [ - "--observation_type", - "screenshot", - "--model", - "{model}", - "--sleep_after_execution", - "3", - "--max_steps", - "15", - "--client_password", - "password" - ], - "env": { - "OPENAI_BASE_URL": "{base_url}", - "OPENAI_API_KEY": "{key}" + "start": { + "driver": { + "agent": "prompt", + "script": "scripts/python/run_multienv.py", + "argv": [ + "--observation_type", + "screenshot", + "--model", + "{model}", + "--sleep_after_execution", + "3", + "--max_steps", + "15", + "--client_password", + "password" + ], + "env": { + "OPENAI_BASE_URL": "{base_url}", + "OPENAI_API_KEY": "{key}" + } + } } }, "claude": { - "script": "scripts/python/run_multienv_claude.py", + "key": "claude", + "label": "Claude computer use", + "summary": "Anthropic's computer-use agent: run_multienv_claude.py, Claude with the computer-use beta tool, 50 steps", "models": [ "claude-sonnet-4-5-20250929", "claude-opus-4-5", "claude-opus-4-6", "claude-opus-4-7" ], - "argv": [ - "--observation_type", - "screenshot", - "--action_space", - "claude_computer_use", - "--model", - "{model}", - "--max_steps", - "50", - "--client_password", - "password" - ], - "env": { - "ANTHROPIC_BASE_URL": "{base}", - "ANTHROPIC_API_KEY": "{key}" - }, - "aws": true + "start": { + "driver": { + "agent": "claude", + "script": "scripts/python/run_multienv_claude.py", + "argv": [ + "--observation_type", + "screenshot", + "--action_space", + "claude_computer_use", + "--model", + "{model}", + "--max_steps", + "50", + "--client_password", + "password" + ], + "env": { + "ANTHROPIC_BASE_URL": "{base}", + "ANTHROPIC_API_KEY": "{key}" + }, + "aws": true + } + } }, "gpt54": { - "script": "scripts/python/run_multienv_gpt54.py", + "key": "gpt54", + "label": "GPT-5.4", + "summary": "OpenAI's GPT-5.4 computer use: run_multienv_gpt54.py, the Responses API's computer tool, 50 steps", "models": [ "gpt-5.4" ], - "argv": [ - "--observation_type", - "screenshot", - "--model", - "{model}", - "--max_steps", - "50", - "--client_password", - "password", - "--sleep_after_execution", - "3", - "--log_level", - "DEBUG" - ], - "env": { - "OPENAI_BASE_URL": "{base_url}", - "OPENAI_API_KEY": "{key}" + "start": { + "driver": { + "agent": "gpt54", + "script": "scripts/python/run_multienv_gpt54.py", + "argv": [ + "--observation_type", + "screenshot", + "--model", + "{model}", + "--max_steps", + "50", + "--client_password", + "password", + "--sleep_after_execution", + "3", + "--log_level", + "DEBUG" + ], + "env": { + "OPENAI_BASE_URL": "{base_url}", + "OPENAI_API_KEY": "{key}" + } + } } }, - "openai_cua": { - "script": "scripts/python/run_multienv_openaicua.py", + "openai-cua": { + "key": "openai-cua", + "label": "OpenAI CUA", + "summary": "OpenAI's CUA: run_multienv_openaicua.py, computer-use-preview over the Responses API, 50 steps", "models": [ "computer-use-preview" ], - "argv": [ - "--observation_type", - "screenshot", - "--model", - "{model}", - "--max_steps", - "50", - "--client_password", - "password" - ], - "env": { - "OPENAI_BASE_URL": "{base_url}", - "OPENAI_API_KEY_CUA": "{key}" - }, - "aws": true + "start": { + "driver": { + "agent": "openai_cua", + "script": "scripts/python/run_multienv_openaicua.py", + "argv": [ + "--observation_type", + "screenshot", + "--model", + "{model}", + "--max_steps", + "50", + "--client_password", + "password" + ], + "env": { + "OPENAI_BASE_URL": "{base_url}", + "OPENAI_API_KEY_CUA": "{key}" + }, + "aws": true + } + } }, "qwen": { - "script": "scripts/python/run_multienv_qwen.py", + "key": "qwen", + "label": "Qwen", + "summary": "Qwen-CUA: run_multienv_qwen.py, qwen3.7-plus over Chat Completions, 50 steps", "models": [ "qwen3.7-plus" ], - "argv": [ - "--model", - "{model}", - "--base_url", - "{base_url}", - "--api_key", - "{key}", - "--no-enable_proxy" - ], - "env": {} + "start": { + "driver": { + "agent": "qwen", + "script": "scripts/python/run_multienv_qwen.py", + "argv": [ + "--model", + "{model}", + "--base_url", + "{base_url}", + "--api_key", + "{key}", + "--no-enable_proxy" + ], + "env": {} + } + } }, - "os_symphony": { - "script": "scripts/python/run_multienv_os_symphony.py", + "os-symphony": { + "key": "os-symphony", + "label": "OS-Symphony", + "summary": "OS-Symphony: run_multienv_os_symphony.py, GPT-5 roles with a UI-TARS-1.5-7B grounder and a search VM, 50 steps", "models": [ "gpt-5" ], - "argv": [ - "--client_password", - "password", - "--max_steps", - "50", - "--benchmark", - "osworld", - "--domain", - "all", - "--tool_config", - "{up}/mm_agents/os_symphony/tool/all_tool_config.yaml", - "--orchestrator_provider", - "openai", - "--orchestrator_model", - "{model}", - "--orchestrator_url", - "{base_url}", - "--orchestrator_api_key", - "{key}", - "--orchestrator_temperature", - "0.1", - "--orchestrator_keep_first_image", - "--max_trajectory_length", - "8", - "--grounder_provider", - "vllm", - "--grounder_model", - "UI-TARS-1.5-7B", - "--grounder_api_key", - "{key}", - "--grounder_url", - "{base_url}", - "--grounding_smart_resize", - "--grounding_width", - "1920", - "--grounding_height", - "1080", - "--coder_provider", - "openai", - "--coder_model", - "{model}", - "--coder_url", - "{base_url}", - "--coder_api_key", - "{key}", - "--coder_temperature", - "0.1", - "--coder_budget", - "20", - "--memoryer_provider", - "openai", - "--memoryer_model", - "{model}", - "--memoryer_url", - "{base_url}", - "--memoryer_api_key", - "{key}", - "--memoryer_temperature", - "0.1", - "--memoryer_max_images", - "8", - "--searcher_provider", - "openai", - "--searcher_model", - "{model}", - "--searcher_url", - "{base_url}", - "--searcher_api_key", - "{key}", - "--searcher_temperature", - "0.1", - "--searcher_type", - "vlm", - "--searcher_engine", - "google", - "--searcher_budget", - "20", - "--searcher_screen_width", - "1920", - "--searcher_screen_height", - "1080", - "--sleep_after_execution", - "3", - "--exp_name", - "nullius", - "--enable_reflection" - ], - "env": {} + "start": { + "driver": { + "agent": "os_symphony", + "script": "scripts/python/run_multienv_os_symphony.py", + "argv": [ + "--client_password", + "password", + "--max_steps", + "50", + "--benchmark", + "osworld", + "--domain", + "all", + "--tool_config", + "{up}/mm_agents/os_symphony/tool/all_tool_config.yaml", + "--orchestrator_provider", + "openai", + "--orchestrator_model", + "{model}", + "--orchestrator_url", + "{base_url}", + "--orchestrator_api_key", + "{key}", + "--orchestrator_temperature", + "0.1", + "--orchestrator_keep_first_image", + "--max_trajectory_length", + "8", + "--grounder_provider", + "vllm", + "--grounder_model", + "UI-TARS-1.5-7B", + "--grounder_api_key", + "{key}", + "--grounder_url", + "{base_url}", + "--grounding_smart_resize", + "--grounding_width", + "1920", + "--grounding_height", + "1080", + "--coder_provider", + "openai", + "--coder_model", + "{model}", + "--coder_url", + "{base_url}", + "--coder_api_key", + "{key}", + "--coder_temperature", + "0.1", + "--coder_budget", + "20", + "--memoryer_provider", + "openai", + "--memoryer_model", + "{model}", + "--memoryer_url", + "{base_url}", + "--memoryer_api_key", + "{key}", + "--memoryer_temperature", + "0.1", + "--memoryer_max_images", + "8", + "--searcher_provider", + "openai", + "--searcher_model", + "{model}", + "--searcher_url", + "{base_url}", + "--searcher_api_key", + "{key}", + "--searcher_temperature", + "0.1", + "--searcher_type", + "vlm", + "--searcher_engine", + "google", + "--searcher_budget", + "20", + "--searcher_screen_width", + "1920", + "--searcher_screen_height", + "1080", + "--sleep_after_execution", + "3", + "--exp_name", + "nullius", + "--enable_reflection" + ], + "env": {} + } + } }, "evocua": { - "script": "scripts/python/run_multienv_evocua.py", + "key": "evocua", + "label": "EvoCUA", + "summary": "EvoCUA: run_multienv_evocua.py, the EvoCUA-S2 or S1 checkpoint on a model server, 50 steps", "models": [ "EvoCUA-S2", "EvoCUA-S1" ], - "argv": [ - "--observation_type", - "screenshot", - "--model", - "{model}", - "--max_steps", - "50" - ], - "model_argv": { - "EvoCUA-S2": [ - "--temperature", - "0.01", - "--max_history_turns", - "4", - "--coordinate_type", - "relative", - "--resize_factor", - "32", - "--prompt_style", - "S2" - ], - "EvoCUA-S1": [ - "--max_history_turns", - "3", - "--coordinate_type", - "qwen25", - "--max_tokens", - "10240", - "--resize_factor", - "28", - "--prompt_style", - "S1" - ] - }, - "env": { - "OPENAI_BASE_URL": "{base_url}", - "OPENAI_API_KEY": "{key}" + "start": { + "driver": { + "agent": "evocua", + "script": "scripts/python/run_multienv_evocua.py", + "argv": [ + "--observation_type", + "screenshot", + "--model", + "{model}", + "--max_steps", + "50" + ], + "model_argv": { + "EvoCUA-S2": [ + "--temperature", + "0.01", + "--max_history_turns", + "4", + "--coordinate_type", + "relative", + "--resize_factor", + "32", + "--prompt_style", + "S2" + ], + "EvoCUA-S1": [ + "--max_history_turns", + "3", + "--coordinate_type", + "qwen25", + "--max_tokens", + "10240", + "--resize_factor", + "28", + "--prompt_style", + "S1" + ] + }, + "env": { + "OPENAI_BASE_URL": "{base_url}", + "OPENAI_API_KEY": "{key}" + } + } } }, "autoglm": { - "script": "scripts/python/run_autoglm.py", + "key": "autoglm", + "label": "AutoGLM", + "summary": "AutoGLM-OS: run_autoglm.py, autoglm-os on the accessibility tree, on a model server, 50 steps", "models": [ "autoglm-os" ], - "drop": [ - "--num_envs", - "--headless" - ], - "argv": [ - "--model", - "{model}" - ], - "env": { - "OPENAI_BASE_URL": "{base_url}", - "OPENAI_API_KEY": "{key}" + "start": { + "driver": { + "agent": "autoglm", + "script": "scripts/python/run_autoglm.py", + "drop": [ + "--num_envs", + "--headless" + ], + "argv": [ + "--model", + "{model}" + ], + "env": { + "OPENAI_BASE_URL": "{base_url}", + "OPENAI_API_KEY": "{key}" + } + } } }, - "autoglm_v": { - "script": "scripts/python/run_autoglm_v.py", + "autoglm-v": { + "key": "autoglm-v", + "label": "AutoGLM-V", + "summary": "AutoGLM-OS with vision: run_autoglm_v.py, autoglm-os on screenshots, on a model server, 50 steps", "models": [ "autoglm-os" ], - "drop": [ - "--num_envs", - "--headless", - "--test_config_base_dir" - ], - "argv": [ - "--test_config_base_dir", - "{examples}/examples", - "--model", - "{model}" - ], - "env": { - "OPENAI_BASE_URL": "{base_url}", - "OPENAI_API_KEY": "{key}" + "start": { + "driver": { + "agent": "autoglm_v", + "script": "scripts/python/run_autoglm_v.py", + "drop": [ + "--num_envs", + "--headless", + "--test_config_base_dir" + ], + "argv": [ + "--test_config_base_dir", + "{examples}/examples", + "--model", + "{model}" + ], + "env": { + "OPENAI_BASE_URL": "{base_url}", + "OPENAI_API_KEY": "{key}" + } + } } }, "mano": { - "script": "scripts/python/run_multienv_mano.py", + "key": "mano", + "label": "Mano", + "summary": "Mano: run_multienv_mano.py, the mano model on a model server, 15 steps", "models": [ "mano" ], - "argv": [ - "--model", - "{model}" - ], - "env": { - "MANO_API_URL": "{base_url}", - "MANO_API_KEY": "{key}" + "start": { + "driver": { + "agent": "mano", + "script": "scripts/python/run_multienv_mano.py", + "argv": [ + "--model", + "{model}" + ], + "env": { + "MANO_API_URL": "{base_url}", + "MANO_API_KEY": "{key}" + } + } } }, - "muse_spark": { - "script": "scripts/python/run_multienv_muse_spark.py", + "muse-spark": { + "key": "muse-spark", + "label": "Muse Spark", + "summary": "Meta's Muse Spark: run_multienv_muse_spark.py, muse-spark-1.1 over the Responses API, 100 steps", "models": [ "muse-spark-1.1" ], - "argv": [ - "--observation_type", - "screenshot", - "--model", - "{model}", - "--base_url", - "{base_url}", - "--max_steps", - "100", - "--client_password", - "password", - "--sleep_after_execution", - "3", - "--reasoning_effort", - "high", - "--max_tokens", - "131072", - "--request_timeout", - "3200", - "--max_retries", - "6", - "--max_seq_len", - "131072", - "--output_clamp_margin", - "2048", - "--min_output_tokens", - "512", - "--max_recent_media", - "10", - "--min_media_removal_threshold", - "20", - "--max_batched_actions", - "20", - "--tool_name", - "computer.computer", - "--stop_tool_name", - "computer.stop", - "--log_level", - "INFO" - ], - "env": { - "MUSE_SPARK_API_KEY": "{key}" + "start": { + "driver": { + "agent": "muse_spark", + "script": "scripts/python/run_multienv_muse_spark.py", + "argv": [ + "--observation_type", + "screenshot", + "--model", + "{model}", + "--base_url", + "{base_url}", + "--max_steps", + "100", + "--client_password", + "password", + "--sleep_after_execution", + "3", + "--reasoning_effort", + "high", + "--max_tokens", + "131072", + "--request_timeout", + "3200", + "--max_retries", + "6", + "--max_seq_len", + "131072", + "--output_clamp_margin", + "2048", + "--min_output_tokens", + "512", + "--max_recent_media", + "10", + "--min_media_removal_threshold", + "20", + "--max_batched_actions", + "20", + "--tool_name", + "computer.computer", + "--stop_tool_name", + "computer.stop", + "--log_level", + "INFO" + ], + "env": { + "MUSE_SPARK_API_KEY": "{key}" + } + } } } } diff --git a/app/nullius/catalogue/benchmarks/osworld1/driver.py b/app/nullius/catalogue/benchmarks/osworld1/driver.py index e38b979e..ea361a01 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld1/driver.py @@ -106,6 +106,9 @@ os.path.join(HERE, "runner.py")][len(_ARGS):])[:3] with open(AGENTS_FILE, encoding="utf-8") as _f: AGENTS = json.load(_f) +# each entry an agent of the catalogue (start.driver): the driver's spec by its upstream value, the agent option's +AGENTS["agents"] = {e["start"]["driver"]["agent"]: dict(e["start"]["driver"], models=e["models"]) + for e in AGENTS["agents"].values()} VM =os.path.join(UP, "docker_vm_data", "Ubuntu.qcow2") # where the upstream's DockerVMManager puts it RESULTS = "results" # VM starts on one machine, one after another: the upstream's docker provider creates each container under diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index a1d5a253..87b5c989 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -136,53 +136,10 @@ "agent": { "label": "Agent", "summary": "Which of OSWorld's own agents runs: each its script as published, with its own prompts, action format and step limit.", - "values": [ - "m3", - "prompt", - "claude", - "gpt54", - "openai_cua", - "qwen", - "os_symphony", - "evocua", - "autoglm", - "autoglm_v", - "mano", - "muse_spark" - ], - "default": "m3", "setups": [ "original", "model" - ], - "names": { - "m3": "M3 Agent", - "prompt": "PromptAgent", - "claude": "Claude computer use", - "gpt54": "GPT-5.4", - "openai_cua": "OpenAI CUA", - "qwen": "Qwen", - "os_symphony": "OS-Symphony", - "evocua": "EvoCUA", - "autoglm": "AutoGLM", - "autoglm_v": "AutoGLM-V", - "mano": "Mano", - "muse_spark": "Muse Spark" - }, - "about": { - "m3": "MiniMax's M3Agent: run_multienv_m3.py, MiniMax-M3 over Anthropic's Messages API, 100 steps", - "prompt": "OSWorld's PromptAgent: run_multienv.py, GPT-4o on screenshots over Chat Completions, 15 steps", - "claude": "Anthropic's computer-use agent: run_multienv_claude.py, Claude with the computer-use beta tool, 50 steps", - "gpt54": "OpenAI's GPT-5.4 computer use: run_multienv_gpt54.py, the Responses API's computer tool, 50 steps", - "openai_cua": "OpenAI's CUA: run_multienv_openaicua.py, computer-use-preview over the Responses API, 50 steps", - "qwen": "Qwen-CUA: run_multienv_qwen.py, qwen3.7-plus over Chat Completions, 50 steps", - "os_symphony": "OS-Symphony: run_multienv_os_symphony.py, GPT-5 roles with a UI-TARS-1.5-7B grounder and a search VM, 50 steps", - "evocua": "EvoCUA: run_multienv_evocua.py, the EvoCUA-S2 or S1 checkpoint on a model server, 50 steps", - "autoglm": "AutoGLM-OS: run_autoglm.py, autoglm-os on the accessibility tree, on a model server, 50 steps", - "autoglm_v": "AutoGLM-OS with vision: run_autoglm_v.py, autoglm-os on screenshots, on a model server, 50 steps", - "mano": "Mano: run_multienv_mano.py, the mano model on a model server, 15 steps", - "muse_spark": "Meta's Muse Spark: run_multienv_muse_spark.py, muse-spark-1.1 over the Responses API, 100 steps" - } + ] }, "thinking": { "summary": "M3_THINKING_MODE: adaptive, as run_multienv_m3.py's own documented invocation exports it (the model decides whether to think; temperature 1, no top_p), or off, the agent's code default when the variable is unset.", diff --git a/app/nullius/catalogue/benchmarks/osworld2/agents.json b/app/nullius/catalogue/benchmarks/osworld2/agents.json index c558dfbe..fd023168 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/agents.json +++ b/app/nullius/catalogue/benchmarks/osworld2/agents.json @@ -1,5 +1,5 @@ { - "about": "The upstream's agents the driver runs, by the agent option: each its own script at the tested commit, as published, on one task. argv is what its scripts/bash wrapper passes (run_multienv_m3.sh, run_multienv_muse_spark.sh), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the image's password as the wrapper passes it, the task's meta file, the unit's results) and less the aws provider's --region and --use_public_ip and the wrapper's one --specific_task_id. M3 also gets the relay's key as --api_key, since its script reads MINIMAX_ANTHROPIC_BASE_URL and MINIMAX_API_KEY before ANTHROPIC_*, and a .env file in the upstream (each script calls load_dotenv) could set them. Muse Spark gets the reasoning effort and output budget of its published run's args.json (max, 262144) in place of its wrapper's (high, 8192), and MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT, since its count of input tokens before each call asks the endpoint for a path OpenRouter does not serve. env points each script's model client at the unit's relay too, and its simulated user's, asked by the name the documented invocation gives it (OSWORLD_USER_SIM_MODEL), which the Original setup sends on as a side model. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1 (the Anthropic client adds /v1/messages), {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples. --max_steps is each wrapper's own, unless the max_steps option sets another. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them.", + "about": "The upstream's agents the driver runs, each an agent of the catalogue (nullius.agent/v1, start.driver), its upstream value the agent option's: each its own script at the tested commit, as published, on one task. argv is what its scripts/bash wrapper passes (run_multienv_m3.sh, run_multienv_muse_spark.sh), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the image's password as the wrapper passes it, the task's meta file, the unit's results) and less the aws provider's --region and --use_public_ip and the wrapper's one --specific_task_id. M3 also gets the relay's key as --api_key, since its script reads MINIMAX_ANTHROPIC_BASE_URL and MINIMAX_API_KEY before ANTHROPIC_*, and a .env file in the upstream (each script calls load_dotenv) could set them. Muse Spark gets the reasoning effort and output budget of its published run's args.json (max, 262144) in place of its wrapper's (high, 8192), and MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT, since its count of input tokens before each call asks the endpoint for a path OpenRouter does not serve. env points each script's model client at the unit's relay too, and its simulated user's, asked by the name the documented invocation gives it (OSWORLD_USER_SIM_MODEL), which the Original setup sends on as a side model. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1 (the Anthropic client adds /v1/messages), {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples. --max_steps is each wrapper's own, unless the max_steps option sets another. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them.", "common": [ "--provider_name", "docker", @@ -19,64 +19,80 @@ ], "agents": { "m3": { - "script": "scripts/python/run_multienv_m3.py", + "key": "m3", + "label": "MiniMax M3", + "summary": "MiniMax's M3 agent: run_multienv_m3.py, MiniMax-M3 over Anthropic's Messages API, as in the paper, 100 steps", "models": [ "MiniMax-M3" ], - "argv": [ - "--action_space", - "pyautogui", - "--observation_type", - "screenshot", - "--max_tokens", - "8192", - "--max_steps", - "100", - "--model", - "{model}", - "--base_url", - "{base}", - "--api_key", - "{key}" - ], - "env": { - "ANTHROPIC_BASE_URL": "{base}", - "ANTHROPIC_API_KEY": "{key}", - "OSWORLD_USER_SIM_PROVIDER": "anthropic", - "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", - "OSWORLD_USER_SIM_BASE_URL": "{base}", - "OSWORLD_USER_SIM_API_KEY": "{key}" + "start": { + "driver": { + "agent": "m3", + "script": "scripts/python/run_multienv_m3.py", + "argv": [ + "--action_space", + "pyautogui", + "--observation_type", + "screenshot", + "--max_tokens", + "8192", + "--max_steps", + "100", + "--model", + "{model}", + "--base_url", + "{base}", + "--api_key", + "{key}" + ], + "env": { + "ANTHROPIC_BASE_URL": "{base}", + "ANTHROPIC_API_KEY": "{key}", + "OSWORLD_USER_SIM_PROVIDER": "anthropic", + "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "{base}", + "OSWORLD_USER_SIM_API_KEY": "{key}" + } + } } }, - "muse_spark": { - "script": "scripts/python/run_multienv_muse_spark.py", + "muse-spark": { + "key": "muse-spark", + "label": "Muse Spark", + "summary": "Meta's Muse Spark: run_multienv_muse_spark.py, muse-spark-1.3 over the Responses API, 500 steps", "models": [ "muse-spark-1.3" ], - "argv": [ - "--action_space", - "pyautogui", - "--observation_type", - "screenshot", - "--max_tokens", - "262144", - "--reasoning_effort", - "max", - "--max_steps", - "500", - "--model", - "{model}", - "--base_url", - "{base_url}", - "--save_model_eval_raw_info" - ], - "env": { - "MUSE_SPARK_API_KEY": "{key}", - "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1", - "OSWORLD_USER_SIM_PROVIDER": "anthropic", - "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", - "OSWORLD_USER_SIM_BASE_URL": "{base}", - "OSWORLD_USER_SIM_API_KEY": "{key}" + "start": { + "driver": { + "agent": "muse_spark", + "script": "scripts/python/run_multienv_muse_spark.py", + "argv": [ + "--action_space", + "pyautogui", + "--observation_type", + "screenshot", + "--max_tokens", + "262144", + "--reasoning_effort", + "max", + "--max_steps", + "500", + "--model", + "{model}", + "--base_url", + "{base_url}", + "--save_model_eval_raw_info" + ], + "env": { + "MUSE_SPARK_API_KEY": "{key}", + "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1", + "OSWORLD_USER_SIM_PROVIDER": "anthropic", + "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "{base}", + "OSWORLD_USER_SIM_API_KEY": "{key}" + } + } } } } diff --git a/app/nullius/catalogue/benchmarks/osworld2/benchmark.json b/app/nullius/catalogue/benchmarks/osworld2/benchmark.json index 683dbe76..5ad19c7c 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/benchmark.json +++ b/app/nullius/catalogue/benchmarks/osworld2/benchmark.json @@ -19,7 +19,7 @@ "Tasks 103 and 104 are not supported: they stay in the datasets, and an agent's unit ends in an error.", "A slow or loaded machine, or a run across 16:00 UTC (the VM's midnight), can change scores.", "The null control scores above 0 on 015, 076, 090 and 092, and on 077 in some runs.", - "Only OSWorld-V2's own M3 and Muse Spark run it; no agent from the Agents page yet.", + "Only OSWorld-V2's own M3 and Muse Spark run it; no other agent yet.", "Fetch the gated tasks and prepare the machine first: see OSWorld2 machine setup in the docs." ], "todo": [ diff --git a/app/nullius/catalogue/benchmarks/osworld2/driver.py b/app/nullius/catalogue/benchmarks/osworld2/driver.py index 183d2c8f..02177167 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld2/driver.py @@ -60,6 +60,9 @@ websites.SITE = SITE with open(AGENTS_FILE, encoding="utf-8") as _f: AGENTS = json.load(_f) +# each entry an agent of the catalogue (start.driver): the driver's spec by its upstream value, the agent option's +AGENTS["agents"] = {e["start"]["driver"]["agent"]: dict(e["start"]["driver"], models=e["models"]) + for e in AGENTS["agents"].values()} PASSWORD = AGENTS["common"][AGENTS["common"].index("--client_password") + 1] EXAMPLES = os.path.join(UP, "evaluation_examples") VM = os.path.join(UP, "docker_vm_data", "osworld-v2-ubuntu-x86.qcow2") diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json index 8b953451..f9eef9f8 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -121,21 +121,9 @@ "agent": { "label": "Agent", "summary": "Which of OSWorld-V2's own agents runs: each its script as published, with its own prompts, tools and history, asking the model it is published with.", - "values": [ - "m3", - "muse_spark" - ], - "default": "m3", "setups": [ "original" - ], - "names": { - "m3": "MiniMax M3", - "muse_spark": "Muse Spark" - }, - "about": { - "m3": "as in the paper" - } + ] }, "max_steps": { "label": "Steps per task", diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index c2af34c1..5c4826cb 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -71,6 +71,17 @@ vendor's own, filled into its templates as ``{opt_}`` (:func:`options_of`). An agent may name its own non-interactive command (``fidelity.command``), which ``fidelity`` drives beside ACP to show both send the model the same requests. + +A benchmark may bring its **own agents**: ``agents.json`` beside its +``standard.json``, keyed JSON whose ``agents`` are agent definitions that start +with ``start.driver`` - the upstream's own value (``agent``) and what its driver +runs (``script``, ``argv``, ``env``) in the benchmark's Original setup - each +with the ``models`` it is published with. Such an agent speaks no port: no +``offers``, no ``proven``. Where an agent comes from (``from``: ``bundled``, +``yours``, ``benchmark``) and a benchmark agent's ``only_for`` (its benchmark) +are derived from where it sits, never written; :func:`own_agents` lists them, +each named ``:``, and the loader rebuilds the benchmark's +``options.agent`` (its values, names, about and default) from them. """ from __future__ import annotations @@ -1300,6 +1311,120 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple "_here": bench.get("_here", "")}, "" +AGENTS_FILE = "agents.json" +FROM = {"bundled": "Bundled", "yours": "Yours", "benchmark": "A benchmark's own"} + + +def _driver_agent(d: dict) -> List[str]: + """An agent of a benchmark's ``agents.json``: ``start.driver``, what the benchmark's driver runs; the + models it is published with; no port, so no ``offers`` and no ``proven``; its benchmark in ``only_for``.""" + st, bad = d["start"], [] + if "acp" in st or "command" in st: + bad.append("start: acp, command or driver, one of them") + drv = st["driver"] + if not isinstance(drv, dict): + return bad + ["start.driver: {agent, script, argv, env}, what the benchmark's driver runs"] + if not isinstance(drv.get("agent"), str) or not _OPTION_VALUE.match(drv["agent"]): + bad.append("start.driver.agent: the upstream's own name for it, a plain word") + if not isinstance(drv.get("script"), str) or not _RELATIVE.match(drv["script"]): + bad.append("start.driver.script: its script, a path in the upstream") + if not isinstance(drv.get("argv", []), list) or not all(isinstance(w, str) for w in drv.get("argv", [])): + bad.append("start.driver.argv: the words its documented invocation passes") + env = drv.get("env", {}) + if not isinstance(env, dict) or not all(_ENV_NAME.match(str(k)) and isinstance(v, str) for k, v in env.items()): + bad.append("start.driver.env: variable names and templates") + if d.get("offers") or d.get("proven"): + bad.append("offers, proven: none - a benchmark's own agent speaks no port, its driver runs it") + only = d.get("only_for") + if not isinstance(only, list) or len(only) != 1: + bad.append(f"start.driver: a benchmark's own agent, in its folder's {AGENTS_FILE}") + ms = d.get("models") + if not isinstance(ms, list) or not ms or len(set(map(str, ms))) != len(ms) \ + or not all(isinstance(m, str) and _MODEL_NAME.match(m) for m in ms): + bad.append("models: the models it is published with, each once, as its code names them") + return bad + + +def _own_entry(bench: dict, key: str, e: dict) -> dict: + """One entry of a benchmark's ``agents.json`` as an agent definition: where it comes from and the one + benchmark it runs derived from its folder.""" + return dict(e, schema=KINDS["agent"]["schema"], only_for=[bench.get("key")], benchmark=bench.get("key"), + benchmark_label=bench.get("label"), _kind="agent", _here=bench.get("_here", ""), + _ref=f"{bench.get('key')}:{key}", _version="", _versions=[""], _default="", **{"from": "benchmark"}) + + +def own_agents(bench: dict) -> List[dict]: + """A benchmark's own agents (its ``agents.json``), each an agent definition named ``:``.""" + rows = bench.get("_agents") + return [_own_entry(bench, k, e) for k, e in rows.items() if isinstance(e, dict)] if isinstance(rows, dict) else [] + + +def benchmark_agents(benches: List[dict]) -> List[dict]: + """Every benchmark's own agents, once: those of each benchmark's default version.""" + return [a for b in benches if b.get("_version") == b.get("_default") for a in own_agents(b)] + + +def own_agent(bench: dict, ref: str) -> Optional[dict]: + """The benchmark's own agent ``ref`` names (``:``), or None.""" + return next((a for a in own_agents(bench) if a["_ref"] == ref), None) + + +def _with_agents(d: dict, here: str) -> dict: + """A benchmark whose folder holds ``agents.json``: its entries (``_agents``) and ``options.agent`` + rebuilt from them - the upstream values in order, their labels and summaries, the first the default.""" + path = os.path.join(here, AGENTS_FILE) + if not isinstance(d, dict) or not os.path.isfile(path): + return d + with open(path) as f: + raw = json.load(f) + rows = raw.get("agents") if isinstance(raw, dict) else None + d = dict(d, _agents=rows) + opt = (d.get("options") or {}).get("agent") if isinstance(d.get("options"), dict) else None + if not isinstance(opt, dict) or not isinstance(rows, dict): + return d + written = [k for k in ("values", "names", "about", "default") if k in opt] + if written: + return dict(d, _agents_written=written) + es = [e for e in rows.values() if isinstance(e, dict)] + ups = [((e.get("start") or {}).get("driver") or {}).get("agent") if isinstance(e.get("start"), dict) else None for e in es] + opt = dict(opt, values=ups, names={u: e.get("label") for u, e in zip(ups, es)}, + about={u: e.get("summary") for u, e in zip(ups, es)}, default=ups[0] if ups else None) + return dict(d, options=dict(d["options"], agent=opt)) + + +def _own_agents(d: dict) -> List[str]: + """A benchmark's ``agents.json``: each entry an agent (:func:`_driver_agent`) keyed by its key, each + upstream value once, each model among the Original setup's; ``options.agent`` with its label, summary + and setups only, the rest rebuilt from the file.""" + if "_agents" not in d: + return [] + rows, w = d["_agents"], AGENTS_FILE + if not isinstance(rows, dict) or not rows: + return [f"{w}: agents, key -> an agent (nullius.agent/v1, start.driver)"] + bad = [f"options.agent.{k}: from {w}, not written here (label, summary and setups only)" for k in d.get("_agents_written") or []] + if not isinstance((d.get("options") or {}).get("agent"), dict): + bad.append(f"options.agent: {{label, summary, setups}}, its values from {w}") + orig = (d.get("setups") or {}).get("original") or {} + seen = set() + for k, e in rows.items(): + at = f"{w}: agents.{k}" + if not isinstance(e, dict): + bad.append(f"{at}: an object") + continue + if "only_for" in e and e["only_for"] != [d.get("key")]: + bad.append(f"{at}.only_for: its own benchmark ({d.get('key')}) alone, or left out") + if e.get("key") != k: + bad.append(f"{at}.key: {k}, its name in agents") + bad += [f"{at}: {p}" for p in problems(_own_entry(d, k, dict(e, only_for=[d.get("key")])), "agent")] + up = ((e.get("start") or {}).get("driver") or {}).get("agent") if isinstance(e.get("start"), dict) else None + if up in seen: + bad.append(f"{at}.start.driver.agent: {up} is another entry's") + seen.add(up) + if isinstance(orig.get("models"), list) and isinstance(e.get("models"), list): + bad += [f"{at}.models: {m} is not one of setups.original.models" for m in e["models"] if m not in orig["models"]] + return bad + + def problems(d: dict, kind: str) -> List[str]: """What is wrong with definition ``d`` of ``kind``; empty when it may be offered.""" kind = names.kind(kind) @@ -1413,6 +1538,7 @@ def problems(d: dict, kind: str) -> List[str]: bad += _controls(d.get("controls")) bad += _leaderboard(d) bad += _setups(d.get("setups")) + bad += _own_agents(d) bad += _environments(d.get("environments"), d.get("_here")) envs = d.get("environments") if isinstance(d.get("environments"), dict) else {} if d.get("family") == "capability": @@ -1423,6 +1549,8 @@ def problems(d: dict, kind: str) -> List[str]: bad.append(f"datasets[{i}].environment: one of environments ({', '.join(envs) or 'none declared'})") if d.get("proven") is not None and not isinstance(d.get("proven"), dict): bad.append("proven: an object, per dataset, written by cli prove") + elif kind == "agent" and "driver" in (d.get("start") or {}): + bad += _driver_agent(d) elif kind == "agent": st = d.get("start") or {} if ("acp" in st) == ("command" in st): @@ -1489,6 +1617,8 @@ def _read(path: str, kind: str, key: str) -> Tuple[Optional[dict], List[str]]: try: with open(path) as f: d = names.definition(json.load(f)) + if kind == "benchmark": + d = _with_agents(d, os.path.dirname(path)) except (OSError, ValueError, TypeError, AttributeError) as e: return None, [f"{kind} {key}: {e}"] try: @@ -1498,9 +1628,14 @@ def _read(path: str, kind: str, key: str) -> Tuple[Optional[dict], List[str]]: bad = [f"{kind} {key}: {p}" for p in found] if isinstance(d, dict) and d.get("key") != key: bad.append(f"{kind} {key}: key {d.get('key')!r} is not its folder's name") + if kind == "agent" and isinstance(d, dict) and "driver" in (d.get("start") or {}): + bad.append(f"{kind} {key}: start.driver: only an entry of a benchmark's {AGENTS_FILE}") if bad: return None, bad d = dict(d, _here=os.path.dirname(path), _kind=kind) + if kind == "agent": # where it comes from: the user's catalogue, or the bundle + user = os.path.realpath(paths.catalogue_dir()) + d["from"] = "yours" if os.path.realpath(path).startswith(user + os.sep) else "bundled" return d, [] diff --git a/app/nullius/standard/match.py b/app/nullius/standard/match.py index a1c875b1..2396b85e 100644 --- a/app/nullius/standard/match.py +++ b/app/nullius/standard/match.py @@ -5,14 +5,16 @@ every port the benchmark needs - and, for a port whose kit has skills, has shown each skill the benchmark relies on (``needs_skills``). An agent with ``only_for`` - a benchmark's own reference agent - fits those benchmarks -alone, whatever it has proven. The reasons are sentences a page can show -beside a greyed-out choice. +alone, whatever it has proven; a benchmark's own agent (``definitions.own_agents``) +is bound the same way to its benchmark, and speaks no port to check. The +reasons are sentences a page can show beside a greyed-out choice. """ from __future__ import annotations from typing import Dict, List from nullius.standard import ports +from nullius.standard import definitions from nullius.standard.definitions import CONTROLS, port_skills, ref_of, skills_shown @@ -34,7 +36,11 @@ def fit(bench: dict, other: dict) -> List[str]: name = f"{other['label']} {version}" if "@" in str(other.get("_ref") or "") else other["label"] only = other.get("only_for") or [] if only and bench.get("key") not in only: + if other.get("benchmark"): # a benchmark's own agent (its agents.json) + return [f"{name} belongs to {other.get('benchmark_label') or other['benchmark']}"] return [f"{name} is the reference agent of {', '.join(only)}, and runs no other benchmark"] + if "driver" in (other.get("start") or {}): # its benchmark's driver runs it: no port to prove + return [] why = [] for p in bench.get("needs") or []: on = str((proven.get(p) or {}).get("version") or "") @@ -58,6 +64,7 @@ def table(benches: List[dict], agents: List[dict], capabilities: List[dict]) -> ``oracle``, which always fit. A benchmark is keyed as an agent is: its key for its default version, ``key@version`` for another.""" out = {} + agents = agents + definitions.benchmark_agents(benches) # a benchmark's own agents beside the catalogue's for b in benches: others = agents if b.get("family") == "agent" else capabilities k = ref_of(b) if "_version" in b else b["key"] diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 4afbeeb3..868739dc 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -176,7 +176,8 @@ def protocol(o: dict) -> str: return "no agent" if o["_kind"] == "capability": return "JSON lines" - return "ACP" if "acp" in (o.get("start") or {}) else "a command" + st = o.get("start") or {} + return "ACP" if "acp" in st else "its own script" if "driver" in st else "a command" def skills_of(port: str, record: Optional[dict]) -> List[dict]: diff --git a/app/templates/_cards.html b/app/templates/_cards.html index 3114aede..aef7dc2c 100644 --- a/app/templates/_cards.html +++ b/app/templates/_cards.html @@ -94,8 +94,8 @@
{{ badges }}
{% endif %} {%- call(f, v) each(item, by.note) %}
{% if f.icon %}{% endif %}{{ v.text }}
{% endcall %} - {%- if by.stat %} -
{% for f in by.stat %}{% set v = item.get(f.key) %}{% set v = v if v is mapping else {'text': v} -%} + {%- set stats = by.stat|rejectattr('key', 'in', item.get('hide', []))|list %}{% if stats %} +
{% for f in stats %}{% set v = item.get(f.key) %}{% set v = v if v is mapping else {'text': v} -%}
{{ v.label or f.card_label or f.label }}
{{ '—' if v.text is none or v.text == '' else v.text }}
{% endfor %}
{% endif %} {%- call(f, v) each(item, by.alert) %}
{{ v.text }}
{% endcall %} diff --git a/app/templates/_tables.html b/app/templates/_tables.html index 5ead8bab..5bb42046 100644 --- a/app/templates/_tables.html +++ b/app/templates/_tables.html @@ -66,8 +66,8 @@ {# A table's controls row (static/tables.js `toolbar`) over the columns `cols`; each feature on only when asked. `search`: the search box's name and placeholder, over each row's data-search text; `columns`: the - Columns menu; `choices`: column -> its (value, label) pairs to filter by; `only`: column -> the one value chosen - at load; then the Sort by menu, unless `sort` is false; a list of keys + Columns menu; `choices`: column -> its (value, label) pairs to filter by; `only`: column -> the value chosen + at load, or a list of them; then the Sort by menu, unless `sort` is false; a list of keys offers those columns only. `views` and `actions`: macros writing a page's own buttons, each any number of them, run by the page: views (how the table looks) before Columns, actions (on the rows or the selection) after Sort by. `layout` 'table' or 'cards', the one shown: a toggle of links to the @@ -121,7 +121,7 @@ {% for c in by.pills %}
{{ icon(c) }} - {%- for value, text in choices[c.key] %}{% endfor %} + {%- for value, text in choices[c.key] %}{% endfor %}
{% endfor %} @@ -134,13 +134,13 @@ {#- A column's `filter_icon`, if it has one. #} {% macro icon(c) %}{% if c.filter_icon %}{% endif %}{% endmacro %} -{#- A column's choices as checkboxes, in the Filter menu or a dropdown: all ticked, or only its `only` value. #} +{#- A column's choices as checkboxes, in the Filter menu or a dropdown: all ticked, or only its `only` values. #} {# `legend=false` in a dropdown, whose button already names the category #} {% macro values(c, choices, only, legend=true) %}
{% if legend %}{{ c.label }}{% endif %} - {% for value, text in choices %}{% endfor %} + {% for value, text in choices %}{% endfor %}
{% endmacro %} diff --git a/app/templates/agents/standard.html b/app/templates/agents/standard.html index 3e72259f..c6908df1 100644 --- a/app/templates/agents/standard.html +++ b/app/templates/agents/standard.html @@ -34,14 +34,16 @@ {% if problems %}
Not offered: a definition with problems
    {% for p in problems %}
  • {{ p }}
  • {% endfor %}
{% endif %} {% if not items %}

No {{ kind }} yet.

{% endif %} - {#- A card's fields (web/agents.py `_card`); an agent's protocol and the ports it proves as dropdowns. #} + {#- A card's fields (web/agents.py `_card`); an agent's protocol, the ports it proves and where it comes from as filters. #} {% set cols = [dict(key='icon', label='Logo', card='icon', table=false, class='std-mark small'), dict(key='title', label='Name', card='title'), - dict(key='kind', label='Kind', card='tag', class='cat-tag'), dict(key='protocol', label='Protocol', card='tag', class='std-tag'), + dict(key='kind', label='Kind', card='tag', class='cat-tag'), dict(key='benchmark', label='Benchmark', card='tag', class='cat-tag'), + dict(key='protocol', label='Protocol', card='tag', class='std-tag'), dict(key='uses', label='Uses', card='tag', class='std-tag'), + dict(key='from', label='From'), dict(key='models', label='Models', card='note'), dict(key='proves', label='Proves', class='std-tag', off=true), dict(key='example', label='Example', card='tag', class='std-tag', off=true), dict(key='summary', label='Summary', card='text', class='std-clamp', width='expand', off=true), dict(key='ports', label='Ports proven', card='stat', type='num'), dict(key='runs', label='Benchmarks it runs', card='stat', type='num'), dict(key='installs', label='Installs', card='stat', class='word'), dict(key='version', label='Version', card='meta')] %} - {% if items %}{{ cards(plural|lower, cols, items, search='Name, protocol, summary', choices=choices, sort=['title', 'ports', 'runs'], empty='No ' ~ kind ~ ' matches.', noun=plural|lower, layout='table' if request.args.get('view') == 'table' else 'cards', default='cards') }}{% endif %} + {% if items %}{{ cards(plural|lower, cols, items, search='Name, protocol, summary', choices=choices, only=only, sort=['title', 'ports', 'runs'], empty='No ' ~ kind ~ ' matches.', noun=plural|lower, layout='table' if request.args.get('view') == 'table' else 'cards', default='cards') }}{% endif %} {% endblock %} {% block scripts %}{% endblock %} diff --git a/app/tests/catalogue_docs.py b/app/tests/catalogue_docs.py index 7bef9284..de1742af 100644 --- a/app/tests/catalogue_docs.py +++ b/app/tests/catalogue_docs.py @@ -461,7 +461,8 @@ def _setups(v: dict) -> str: fixed = original.get("models") or v.get("simulator") or (v.get("score") or {}).get("judge") model_choice = ", ".join((["Like the paper"] if fixed else []) + (["Like the leaderboard"] if v.get("leaderboard") else []) + ["Any of your choice"]) model_choice = " or ".join(model_choice.rsplit(", ", 1)) - participant = original.get("reference_agent") or ("Original memory" if v.get("family") == "capability" else "No agent") + participant = original.get("reference_agent") or ("its own agents" if definitions.own_agents(v) + else "Original memory" if v.get("family") == "capability" else "No agent") return f"Original · {participant} · {model_choice}" diff --git a/app/tests/test_benchmark_agents.py b/app/tests/test_benchmark_agents.py new file mode 100644 index 00000000..a7c3dfac --- /dev/null +++ b/app/tests/test_benchmark_agents.py @@ -0,0 +1,119 @@ +"""A benchmark's own agents (its agents.json): agents of the catalogue that start with start.driver, bound to +their benchmark by where they sit, and the benchmark's agent option rebuilt from them.""" +from __future__ import annotations + +import copy +import json +import os +import shutil +import tempfile +import unittest + +from nullius.standard import definitions, match + +BENCHES = os.path.join(definitions.CATALOGUE, "benchmarks") + + +def bench(key="osworld1"): + return definitions.get("benchmark", key, root=definitions.CATALOGUE) + + +class Loaded(unittest.TestCase): + def test_each_entry_is_an_agent_named_by_its_benchmark_and_bound_to_it(self): + b = bench() + own = definitions.own_agents(b) + self.assertEqual(len(own), 12) + cua = definitions.own_agent(b, "osworld1:openai-cua") + self.assertEqual((cua["key"], cua["from"], cua["only_for"], cua["benchmark"]), ("openai-cua", "benchmark", ["osworld1"], "osworld1")) + self.assertEqual(cua["start"]["driver"]["agent"], "openai_cua") + self.assertEqual(definitions.problems(cua, "agent"), []) + with open(os.path.join(BENCHES, "osworld1", "agents.json")) as f: + self.assertTrue(all("from" not in e and "only_for" not in e for e in json.load(f)["agents"].values())) + + def test_the_agent_option_is_rebuilt_from_the_file(self): + with open(os.path.join(BENCHES, "osworld1", "standard.json")) as f: + written = json.load(f)["options"]["agent"] + self.assertEqual(set(written), {"label", "summary", "setups"}) + opt = bench()["options"]["agent"] + own = definitions.own_agents(bench()) + self.assertEqual(opt["values"], [a["start"]["driver"]["agent"] for a in own]) + self.assertEqual(opt["default"], "m3") + self.assertEqual(opt["names"]["openai_cua"], "OpenAI CUA") + self.assertEqual(definitions.bench_options_of(bench(), "test_small", {"agent": "openai_cua"}, "original")[0]["agent"], "openai_cua") + + def test_every_published_model_is_the_original_setups(self): + for key in ("osworld1", "osworld2"): + b = bench(key) + self.assertEqual(definitions.problems(b, "benchmark"), []) + self.assertEqual({m for a in definitions.own_agents(b) for m in a["models"]}, set(b["setups"]["original"]["models"])) + + def test_the_catalogues_agents_say_where_they_come_from(self): + tmp = tempfile.mkdtemp(prefix="nullius-from-") + self.addCleanup(shutil.rmtree, tmp, True) + shutil.copytree(os.path.join(definitions.CATALOGUE, "agents", "codex"), os.path.join(tmp, "agents", "codex")) + old = os.environ.get("NULLIUS_CATALOGUE") + os.environ["NULLIUS_CATALOGUE"] = tmp + self.addCleanup(lambda: os.environ.pop("NULLIUS_CATALOGUE") if old is None else os.environ.update(NULLIUS_CATALOGUE=old)) + got = {a["key"]: a["from"] for a in definitions.load("agent")[0]} + self.assertEqual((got["codex"], got["goose"]), ("yours", "bundled")) + + +class Refused(unittest.TestCase): + def entry(self, change): + b = copy.deepcopy(bench()) + change(b["_agents"]["openai-cua"], b) + return definitions.problems(b, "benchmark") + + def test_a_driver_agent_speaks_no_port(self): + bad = self.entry(lambda e, b: e.update(offers=["agent.computer/v1"], proven={"agent.computer/v1": {}})) + self.assertIn("agents.json: agents.openai-cua: offers, proven: none - a benchmark's own agent speaks no port, its driver runs it", bad) + + def test_an_entry_says_what_it_is_and_what_it_is_published_with(self): + for field, said in (("label", "label: say it"), ("summary", "summary: say it"), + ("models", "models: the models it is published with, each once, as its code names them")): + with self.subTest(field): + self.assertIn(f"agents.json: agents.openai-cua: {said}", self.entry(lambda e, b: e.pop(field))) + self.assertIn("agents.json: agents.openai-cua: start.driver.script: its script, a path in the upstream", + self.entry(lambda e, b: e["start"]["driver"].pop("script"))) + + def test_a_model_the_original_setup_does_not_take(self): + self.assertIn("agents.json: agents.openai-cua.models: gpt-9 is not one of setups.original.models", + self.entry(lambda e, b: e["models"].append("gpt-9"))) + + def test_only_for_names_its_own_benchmark_or_is_left_out(self): + self.assertEqual(self.entry(lambda e, b: e.update(only_for=["osworld1"])), []) + self.assertIn("agents.json: agents.openai-cua.only_for: its own benchmark (osworld1) alone, or left out", + self.entry(lambda e, b: e.update(only_for=["osworld2"]))) + + def test_the_option_values_are_not_written_beside_the_file(self): + with open(os.path.join(BENCHES, "osworld1", "standard.json")) as f: + d = json.load(f) + d["options"]["agent"]["values"] = ["m3"] + d = definitions._with_agents(d, os.path.join(BENCHES, "osworld1")) + self.assertIn("options.agent.values: from agents.json, not written here (label, summary and setups only)", + definitions.problems(d, "benchmark")) + + def test_a_driver_start_outside_a_benchmark_folder(self): + tmp = tempfile.mkdtemp(prefix="nullius-driver-") + self.addCleanup(shutil.rmtree, tmp, True) + os.makedirs(os.path.join(tmp, "agents", "cua")) + e = dict(bench()["_agents"]["openai-cua"], key="cua", schema="nullius.agent/v1", only_for=["osworld1"]) + with open(os.path.join(tmp, "agents", "cua", "agent.json"), "w") as f: + json.dump(e, f) + got, bad = definitions.load("agent", tmp) + self.assertEqual(got, []) + self.assertIn("agent cua: start.driver: only an entry of a benchmark's agents.json", bad) + + +class Fits(unittest.TestCase): + def test_its_own_benchmark_alone_with_no_port_reason(self): + benches = definitions.load("benchmark", definitions.CATALOGUE)[0] + t = match.table(benches, [], []) + self.assertEqual(t["osworld1"]["osworld1:openai-cua"], []) + self.assertEqual(t["osworld2"]["osworld1:openai-cua"], ["OpenAI CUA belongs to OSWorld1"]) + self.assertEqual(t["gsm8k"]["osworld2:muse-spark"], ["Muse Spark belongs to OSWorld2"]) + self.assertNotIn("osworld1:openai-cua", t.get("locomo", {})) # a capability benchmark takes none + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_judge.py b/app/tests/test_osworld2_judge.py index a0386f18..d2270068 100644 --- a/app/tests/test_osworld2_judge.py +++ b/app/tests/test_osworld2_judge.py @@ -43,8 +43,7 @@ def load_driver(up): def bench(): - with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: - return json.load(f) + return definitions.get("benchmark", "osworld2", root=definitions.CATALOGUE) class Started(Exception): @@ -445,7 +444,7 @@ def test_the_judge_and_the_simulated_user_are_sonnet_as_openrouter_names_it(self self.assertEqual(definitions.problems(b, "benchmark"), []) self.assertEqual(providers.served_as(b["score"]["judge"]["model"], "openrouter"), SONNET) with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: - asked = json.load(f)["agents"]["m3"]["env"]["OSWORLD_USER_SIM_MODEL"] + asked = json.load(f)["agents"]["m3"]["start"]["driver"]["env"]["OSWORLD_USER_SIM_MODEL"] other, why = definitions.setup(b, "original", "MiniMax-M3", "openrouter") self.assertEqual(why, "") self.assertEqual(other["sent"], "minimax/minimax-m3") diff --git a/app/tests/test_osworld2_muse.py b/app/tests/test_osworld2_muse.py index 4373c4e3..aa2d6f01 100644 --- a/app/tests/test_osworld2_muse.py +++ b/app/tests/test_osworld2_muse.py @@ -36,8 +36,7 @@ def load_driver(up): def bench(): - with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: - return json.load(f) + return definitions.get("benchmark", "osworld2", root=definitions.CATALOGUE) class Started(Exception): @@ -175,7 +174,7 @@ def test_the_relay_waits_longer_than_muse_spark(self): b = bench() self.assertEqual(b["limits"]["call_seconds"], 3300) with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: - self.assertNotIn("--request_timeout", json.load(f)["agents"]["muse_spark"]["argv"]) + self.assertNotIn("--request_timeout", json.load(f)["agents"]["muse-spark"]["start"]["driver"]["argv"]) for bad in (0, -1, 1.5, True, "3300"): b["limits"]["call_seconds"] = bad with self.subTest(bad): @@ -394,7 +393,7 @@ def test_every_flag_passed_is_one_the_runners_parser_has(self): for a in n.args if isinstance(a, ast.Constant) and str(a.value).startswith("--")} with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: agents = json.load(f) - passed = {w for w in agents["common"] + agents["agents"]["muse_spark"]["argv"] if w.startswith("--")} + passed = {w for w in agents["common"] + agents["agents"]["muse-spark"]["start"]["driver"]["argv"] if w.startswith("--")} self.assertEqual(passed - known, set()) diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 75d06cce..f48576cd 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -178,6 +178,25 @@ def test_agents_is_a_card_per_agent_that_opens_its_page(self): self.assertNotIn("agent.task/v1", card[card.index(">"):card.index("")]) # the detail is on its page self.assertEqual(self.get("/agents/nope").status_code, 404) + def test_a_benchmarks_own_agents_are_listed_behind_the_from_filter(self): + with patch.object(definitions, "load", _LOAD): + html = self.get("/agents").get_data(as_text=True) + table = self.get("/agents?view=table").get_data(as_text=True) + menu = html[html.index('data-filter-menu="from"'):] + menu = menu[:menu.index("")] + self.assertIn('data-value="bundled" checked> Bundled', menu) + self.assertIn('data-value="benchmark"> A benchmark's own', menu) # one click away, off by default + card = html[html.index('id="osworld1-openai-cua"'):] + card = card[:card.index("")] + self.assertIn('data-f-from="benchmark "', card) + self.assertIn('href="/benchmarks/osworld1"', card) + self.assertIn('OSWorld1', card) + self.assertIn('
computer-use-preview
', card) + self.assertIn("Computer use", card) + self.assertNotIn("Ports proven", card) + self.assertIn('id="osworld2-muse-spark"', table) + self.assertEqual(self.get("/agents/openai-cua").status_code, 404) # its benchmark's page is its page + def test_agents_sort_and_filter_by_protocol_and_the_ports_they_prove(self): html = self.get("/agents").get_data(as_text=True) menu = html[html.index('data-tools="sort"'):] diff --git a/app/web/agents.py b/app/web/agents.py index a3a4f1f0..67dfaece 100644 --- a/app/web/agents.py +++ b/app/web/agents.py @@ -52,15 +52,20 @@ def _install_words(o: dict) -> list: return out -def _entries(kind: str, every: bool = False) -> dict: +def _entries(kind: str, every: bool = False, own: bool = False) -> dict: + """Its entries; ``own``: and the benchmarks' own agents (``definitions.own_agents``), which have no page.""" others, bad = definitions.load(kind, every=every) - benches = [b for b in definitions.load("benchmark")[0] if b["family"] == kind] + loaded = definitions.load("benchmark")[0] + benches = [b for b in loaded if b["family"] == kind] + if own and kind == "agent": + others = others + definitions.benchmark_agents(loaded) jobs = {j.job_id for j in shared._jobs_for_request()} out = [] for o in others: - cand = set(conform.candidates(o)) + driver = "driver" in (o.get("start") or {}) # a benchmark's own: its driver runs it, it speaks no port + cand = set() if driver else set(conform.candidates(o)) rows = [] - for p, spec in ports.PORTS.items(): + for p, spec in ports.PORTS.items() if not driver else (): if ports.family(p) != kind or (kind == "capability" and p.split(".", 1)[1].split("/")[0] != o.get("kind")): continue rec = (o.get("proven") or {}).get(p) if p in (o.get("offers") or []) else None @@ -88,10 +93,10 @@ def _entries(kind: str, every: bool = False) -> dict: def _page(kind: str): example = os.path.join(definitions.CATALOGUE, "agents", "reference") if kind == "agent" else \ os.path.join(definitions.CATALOGUE, "capabilities", "keyword-memory") - found = _entries(kind) + found = _entries(kind, own=True) return render_template("agents/standard.html", kind=kind, how=howto.steps(kind), prompts=howto.prompts(), catalogue_dir=paths.catalogue_dir(), example_dir=example, problems=found["problems"], - items=[_card(kind, e) for e in found["entries"]], choices=_choices(kind, found["entries"])) + items=[_card(kind, e) for e in found["entries"]], choices=_choices(kind, found["entries"]), only=ONLY.get(kind, {})) def _choices(kind: str, entries: list) -> dict: @@ -99,13 +104,29 @@ def _choices(kind: str, entries: list) -> dict: if kind != "agent": return {} proven = {p["port"] for e in entries for p in e["ports"] if p["state"] == "proven"} + have = {e["d"].get("from") for e in entries} return {"protocol": [(x, x[:1].upper() + x[1:]) for x in dict.fromkeys(e["protocol"] for e in entries)], - "proves": [(p, spec.get("label") or p) for p, spec in ports.PORTS.items() if p in proven]} + "proves": [(p, spec.get("label") or p) for p, spec in ports.PORTS.items() if p in proven], + "from": [(k, v) for k, v in definitions.FROM.items() if k in have]} + + +# The From filter's default: the catalogue's own agents; a benchmark's own are one click away. +ONLY = {"agent": {"from": ["bundled", "yours"]}} def _card(kind: str, e: dict) -> dict: - """An agent's or a capability's card on its list (templates/_cards.html).""" + """An agent's or a capability's card on its list (templates/_cards.html). A benchmark's own agent's + opens its benchmark's page and shows its published models in place of ports, installs and versions.""" d = e["d"] + if d.get("from") == "benchmark": + computer = any(p.startswith("agent.computer/") for b in definitions.load("benchmark")[0] + if b["key"] == d["benchmark"] for p in b.get("needs") or []) + return {"href": url_for("benchmark_page", key=d["benchmark"]), "id": d["_ref"].replace(":", "-"), + "search": " ".join([d["label"], d["key"], d["benchmark_label"], e["protocol"], d.get("summary", "")]), + "protocol": e["protocol"], "icon": agent_mark_url(d["key"]), "title": d["label"], "summary": d.get("summary"), + "from": {"text": definitions.FROM["benchmark"], "value": "benchmark"}, "benchmark": d["benchmark_label"], + "uses": "Computer use" if computer else None, "models": ", ".join(d.get("models") or []), + "hide": ["ports", "runs", "installs"]} version, n = (d.get("upstream") or {}).get("version"), len(d.get("_versions") or ()) return {"href": url_for("agent_page" if kind == "agent" else "feature_page", key=d["key"]), "id": d["key"], "search": " ".join([d["label"], d["key"], d.get("kind", ""), e["protocol"], d.get("summary", "")]), @@ -113,7 +134,7 @@ def _card(kind: str, e: dict) -> dict: "example": {"text": "example", "title": "Nullius's own, to read or copy - not an agent to compare"} if d.get("example") else None, "version": ("v" + str(version) if version else "") + (f" · {n} versions" if n > 1 else ""), "icon": agent_mark_url(d["key"]) if kind == "agent" else None, "title": d["label"], - "summary": d.get("summary"), + "summary": d.get("summary"), "from": {"text": definitions.FROM[d["from"]], "value": d["from"]} if d.get("from") else None, "proves": [{"text": p["label"], "value": p["port"]} for p in e["ports"] if p["state"] == "proven"], "ports": "{}/{}".format(sum(p["state"] == "proven" for p in e["ports"]), len(e["ports"])), "runs": "{}/{}".format(sum(not f["why"] for f in e["fits"]), len(e["fits"])), diff --git a/docs/agents/add-manually.md b/docs/agents/add-manually.md index 93b54324..663c42ad 100644 --- a/docs/agents/add-manually.md +++ b/docs/agents/add-manually.md @@ -129,6 +129,10 @@ A benchmark's own reference agent, made for that benchmark rather than for any task, names it in `only_for` (`"only_for": ["terminal-bench"]` for Terminus 2): it is offered for those benchmarks alone, whatever ports it has proven. +A benchmark whose upstream ships its own agent scripts keeps them in its folder's +`agents.json` instead, each started by the benchmark's driver (`start.driver`): +see [A benchmark's own agents](../standard.md#a-benchmarks-own-agents). + ## If it does not work | Symptom | Check | diff --git a/docs/benchmarks/supported/osworld2.md b/docs/benchmarks/supported/osworld2.md index 40f6c951..23baf8ae 100644 --- a/docs/benchmarks/supported/osworld2.md +++ b/docs/benchmarks/supported/osworld2.md @@ -62,7 +62,7 @@ Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a - Tasks 103 and 104 are not supported: they stay in the datasets, and an agent's unit ends in an error. - A slow or loaded machine, or a run across 16:00 UTC (the VM's midnight), can change scores. - The null control scores above 0 on 015, 076, 090 and 092, and on 077 in some runs. -- Only OSWorld-V2's own M3 and Muse Spark run it; no agent from the Agents page yet. +- Only OSWorld-V2's own M3 and Muse Spark run it; no other agent yet. - Fetch the gated tasks and prepare the machine first: see OSWorld2 machine setup in the docs. ## Cite diff --git a/docs/standard.md b/docs/standard.md index ecc1221d..da12fd7e 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -634,6 +634,32 @@ already accepts arbitrary models needs only the `original` method. | Attempts | `cli run --attempts K` runs each unit K times; `result.json` carries [`pass_hat_k`](field-guide.md#attempts) | | Retries | `cli run --retries N` runs a failed attempt again: [which failures](cli/run.md#retries) | +### A benchmark's own agents + +A benchmark whose upstream ships several agent scripts (OSWorld's) keeps them in +`agents.json` beside its `standard.json`. Each entry of its `agents` is an agent +definition (`nullius.agent/v1`) that starts with `start.driver`: the upstream's +own name for it (`agent`) and what the driver runs in the Original setup +(`script`, `argv`, `env`, and whatever else the driver reads), with the `models` +it is published with. It speaks no port, so it has no `offers` or `proven`. +`common` and `aws` beside `agents` stay driver data. + +```json title="agents.json · one entry" +"openai-cua": { + "key": "openai-cua", "label": "OpenAI CUA", "summary": "...", + "models": ["computer-use-preview"], + "start": {"driver": {"agent": "openai_cua", "script": "scripts/python/run_multienv_openaicua.py", "argv": ["..."], "env": {"...": "..."}}} +} +``` + +| Rule | Detail | +| --- | --- | +| Named | `:`, e.g. `osworld1:openai-cua`; the key follows the key rule (lower case, digits, dashes) | +| Derived, never written | Where it comes from (`from`: `benchmark`; a catalogue agent's is `bundled` or `yours`) and `only_for`, its benchmark. A written `only_for` names that benchmark alone | +| `options.agent` | `standard.json` writes its `label`, `summary` and `setups`; the loader rebuilds `values` (each entry's `start.driver.agent`, in order), `names`, `about` and `default` (the first), so `when`, `by`, `--bench-option agent=...` and `NULLIUS_OPTION_AGENT` read as before | +| Checked | Each entry's `models` are among `setups.original.models`; each upstream name once | +| Offered | On the Agents page behind its From filter, and in the builder beside the catalogue's agents; another benchmark greys it with the reason | + ### Every setting, inventoried A benchmark's folder holds `upstream-settings.json`: every setting of every evaluation entry point of its upstream. From c30bf4389e15c2b7774a78b82b32e360d21ae7d0 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 17:38:30 +0100 Subject: [PATCH 2/8] feat(builder): pick a benchmark's own agents in the shared agent selector OSWorld's agents were one select in its Original setup's tab, one value for the whole recipe. They are now tiles of the Agent step, first, in place of the benchmark's Original tile; several may be chosen, a job each, and a setting that names one of them (M3's thinking) is in that agent's tab. Other benchmarks' own agents are greyed with the benchmark they belong to, and a filter hides them. A pick : compiles through the Original setup with the upstream's own value as its agent option: one pick's command is the one the agent option gave. Like the paper names the agent a model is not published with. A recipe from before (original with bench_options.agent) is read as that agent by standardFromJSON and by the server's compile. --- .claude/skills/import-benchmark/SKILL.md | 3 +- .claude/skills/import-standard/SKILL.md | 7 ++- .claude/skills/remove-integration/SKILL.md | 3 + .../benchmarks/osworld1/standard.json | 3 +- .../benchmarks/osworld2/standard.json | 1 - app/nullius/standard/recipe.py | 63 +++++++++++++++---- app/static/builder/command-builder-core.js | 37 ++++++++--- app/static/builder/command-builder.js | 48 ++++++++------ app/templates/agents/standard.html | 4 +- app/tests/test_benchmark_agents.py | 52 ++++++++++++++- app/tests/test_computer_merge.py | 9 ++- app/tests/test_standard_bench_options.py | 5 +- app/tests/test_standard_driver_hooks.py | 2 +- app/tests/test_standard_setups.py | 3 +- app/ui_tests/test_builder_where.py | 17 +++++ app/ui_tests/test_command_builder_state.py | 28 ++++++++- docs/benchmarks/supported/osworld1.md | 2 +- docs/benchmarks/supported/osworld2.md | 2 +- docs/builder.md | 4 +- docs/field-guide.md | 2 +- 20 files changed, 235 insertions(+), 60 deletions(-) diff --git a/.claude/skills/import-benchmark/SKILL.md b/.claude/skills/import-benchmark/SKILL.md index 63dca9d6..d99bf1a0 100644 --- a/.claude/skills/import-benchmark/SKILL.md +++ b/.claude/skills/import-benchmark/SKILL.md @@ -217,5 +217,6 @@ detailed prerequisites and single/multiple datasets should remain readable. The catalogue presents one Original procedure. Put a published Harbor agent in `benchmark.json.published.agent` so the builder promotes it to the reference group. For a standard benchmark, follow import-standard's `reference_agent` -and paper-model declarations. Keep participant choice on Agent/Memory and +and paper-model declarations, or its own agents (`agents.json`, `start.driver`) +when the upstream ships several agent scripts. Keep participant choice on Agent/Memory and model filtering on Model; do not add separate Original/your-model setup rows. diff --git a/.claude/skills/import-standard/SKILL.md b/.claude/skills/import-standard/SKILL.md index bde64b48..ef986716 100644 --- a/.claude/skills/import-standard/SKILL.md +++ b/.claude/skills/import-standard/SKILL.md @@ -447,7 +447,12 @@ upstream rule as a Nullius defect. Show one **Original** procedure. Model select **Like the paper** filters documented models; **Any of your choice** uses the provider catalogue. Do not invent paper models. Declare `reference_agent` on `setups.original` when upstream runs a named agent; otherwise the builder shows -**No agent · Original**, or **Original memory** for a memory benchmark. The +**No agent · Original**, or **Original memory** for a memory benchmark. An +upstream that ships several agent scripts brings them as its own agents: +`agents.json` beside `standard.json`, each entry an agent with `start.driver` +and its published `models` (docs/standard.md, "A benchmark's own agents"), +`options.agent` written with its label, summary and setups only; recipes then +name one as `:`. The `original` / `model` driver methods preserve upstream model aliases; they are not separate user-facing setups. Recipes select `others: ["original"]`, one `model`, and `model_choice: "paper" | "any"`. diff --git a/.claude/skills/remove-integration/SKILL.md b/.claude/skills/remove-integration/SKILL.md index 115a759e..72e07fe3 100644 --- a/.claude/skills/remove-integration/SKILL.md +++ b/.claude/skills/remove-integration/SKILL.md @@ -21,6 +21,9 @@ the user's requested scope, including logs and stored data when requested. - Catalogue entries live under `app/nullius/catalogue/`. Capabilities can also have leftover folders under the legacy `features/` directory. Remove the whole entry folder: an empty folder can still be selected by the loader. +- A benchmark's own agents are entries of its folder's `agents.json`: they go + with the benchmark. Removing one agent alone is removing its entry, and its + saved recipes (`:`) then name an agent the compile refuses. - For requested runtime cleanup, locate the application's actual profile, catalogue, runs, workspaces and standard installs through its settings and path APIs. With `uv tool`, use the installed tool's interpreter; the shell's diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 87b5c989..d2845333 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -124,8 +124,7 @@ "turn_seconds": 7200, "side_models": [ "UI-TARS-1.5-7B" - ], - "reference_agent": "OSWorld's agents" + ] }, "model": { "summary": "The same agent with any model in its own model's place: the upstream's code asks for the agent's first published model, with its prompt and action format, and the relay sends the model chosen, every side call included. Its scores compare with other runs of this setup.", diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json index f9eef9f8..7a04bc97 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -92,7 +92,6 @@ "setups": { "original": { "summary": "OSWorld-V2's own agent, M3 or Muse Spark, as published, on a Docker VM.", - "reference_agent": "OSWorld-V2 agents", "models": [ "MiniMax-M3", "muse-spark-1.3" diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 868739dc..d6c58c06 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -195,7 +195,10 @@ def skills_of(port: str, record: Optional[dict]) -> List[dict]: def _other_page(o: dict) -> dict: - return {"key": o["_ref"], "base": o["key"], "label": definitions.title(o), "summary": o["summary"], "kind": o["_kind"], + # a benchmark's own agent: its benchmark, the models it is published with and the upstream's name for it + own = {"benchmark": o["benchmark"], "benchmark_label": o["benchmark_label"], "models": list(o.get("models") or []), "agent": o["start"]["driver"]["agent"]} \ + if "driver" in (o.get("start") or {}) else {} + return {**own, "key": o["_ref"], "base": o["key"], "label": definitions.title(o), "summary": o["summary"], "kind": o["_kind"], "featureKind": o.get("kind", ""), "protocol": protocol(o), "apis": list((o.get("model") or {}).get("apis") or ["chat"]), "version": str((o.get("upstream") or {}).get("version") or ""), "default": o.get("_version") == o.get("_default"), "offers": list(o.get("offers") or []), @@ -218,7 +221,7 @@ def choices() -> dict: defaults = [b for b in d["benchmark"] if b.get("_version") == b.get("_default")] return {"benchmarks": [_bench_page(b) for b in defaults], # an example (the reference agent, the example memory) is there to read or copy, never to choose - "agents": [_other_page(o) for o in d["agent"] if not o.get("example")], + "agents": [_other_page(o) for o in d["agent"] + definitions.benchmark_agents(d["benchmark"]) if not o.get("example")], "capabilities": [_other_page(o) for o in d["capability"] if not o.get("example")], "fits": match.table(d["benchmark"], d["agent"], d["capability"]), "ports": {p: s["summary"] for p, s in ports.PORTS.items()}, @@ -262,6 +265,20 @@ def chosen_of(recipe: dict) -> List[str]: return list(dict.fromkeys(str(x) for x in picked if isinstance(x, str) and x)) +def with_own_agents(recipe: dict, bench: dict) -> dict: + """A recipe from before a benchmark's own agents were agents (``definitions.own_agents``): its + ``original`` with ``bench_options.agent`` is that agent, ``:``, by the upstream's name for + it; without the option, the first; a name none has is kept, ``:``, for compile to say.""" + own, refs = definitions.own_agents(bench), chosen_of(recipe) + if not own or "original" not in refs: + return recipe + opts = dict(recipe["bench_options"]) if isinstance(recipe.get("bench_options"), dict) else {} + v = opts.pop("agent", None) + a = next((x for x in own if x["start"]["driver"]["agent"] == v), None) if v else own[0] + ref = a["_ref"] if a else f"{bench['key']}:{v}" + return dict(recipe, others=[ref if r == "original" else r for r in refs], bench_options=opts) + + def options_for(recipe: dict, ref: str, several: bool): """One choice's options: ``options`` is {ref: {name: word}}; a recipe from before, or one with a single choice, may give that choice's {name: word} directly.""" @@ -332,6 +349,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("benchmark", "Choose a benchmark of the standard.") return out b = dict(b, label=definitions.title(b)) + recipe = with_own_agents(recipe, b) # one dataset or several (``datasets``; a recipe from before names one, ``dataset``): each its own jobs wanted = recipe.get("datasets") if isinstance(recipe.get("datasets"), list) else ([recipe["dataset"]] if recipe.get("dataset") else []) wanted = list(dict.fromkeys(str(x) for x in wanted if isinstance(x, str))) @@ -361,7 +379,13 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if ref == "model": add("agent", "Choose Original, then choose the model on the Model step.") continue - if ref == "original": + own = definitions.own_agent(b, ref) if ":" in ref else None # one of the benchmark's own agents + if ":" in ref and own is None: + other = definitions.find(definitions.benchmark_agents(d["benchmark"]), ref) + add("agent", f"{other['label']} cannot run {b['label']}: " + "; ".join(match.fit(b, other)) + "." if other + else f"Choose an agent for {b['label']}: {ref} is not one of its own.") + continue + if ref == "original" or own: orig = (b.get("setups") or {}).get("original") or {} picked = str(recipe.get("model") or "").strip() if not picked: @@ -370,11 +394,15 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original" if method == "original": picked = definitions.upstream_name(orig, picked, provider_) + if own and method == "original" and picked not in own["models"]: + add("model", f"{own['label']} is published with {', '.join(own['models'])}: choose one of them" + + (", or Any of your choice for another." if "model" in (b.get("setups") or {}) else ".")) + continue o, why_not = definitions.setup(b, method, picked, provider_) if o is None: add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".") continue - picks.append({"o": o, "control": False, "setup": True, "given": {}, "opts": {}, "model": picked or o["asked"]}) + picks.append({"o": o, "control": False, "setup": True, "given": {}, "opts": {}, "model": picked or o["asked"], "agent": own}) continue if control: o = definitions.control(b, ref) @@ -409,20 +437,25 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if chosen_opts.get(k, v) != v: add("dataset", f"Like the leaderboard sets {(b.get('options') or {})[k].get('label') or k.replace('_', ' ')} to {v}: choose Any of your choice for {chosen_opts[k]}.") chosen_opts = {**(lb.get("bench_options") or {}), **chosen_opts} + if definitions.own_agents(b): # the agent is each pick's own, never the recipe's + chosen_opts.pop("agent", None) + agent_of = lambda p: {"agent": p["agent"]["start"]["driver"]["agent"]} if p.get("agent") else {} per_ds, needs = {}, {"embeddings": [], "reranker": [], "host": []} # each participant's scope and kind: an option an agent's run alone takes (setups: agent) is an agent's - scopes = list(dict.fromkeys((p["o"]["key"], None) if p.get("setup") else ("yours", p["o"].get("_kind")) - for p in picks)) or [("yours", None)] + scopes = list(dict.fromkeys((p["o"]["key"], None, agent_of(p).get("agent")) if p.get("setup") else ("yours", p["o"].get("_kind"), None) + for p in picks)) or [("yours", None, None)] for x in dss: # Original's retrievers have machine requirements too; collect the # options of every participant that the compiled commands will run. vals, bad = {}, [] - for scope, kind in scopes: - chosen, problems = definitions.bench_options_of(b, x["id"], chosen_opts, scope, kind=kind) + for scope, kind, agent in scopes: + chosen, problems = definitions.bench_options_of(b, x["id"], dict(chosen_opts, **({"agent": agent} if agent else {})), scope, kind=kind) vals.update(chosen) bad.extend(problems) for why_not in dict.fromkeys(bad): add("dataset", why_not[0].upper() + why_not[1:] + ".") + if len({a for _, _, a in scopes if a}) > 1: + vals["agent"] = None # several of its own agents: a default by agent is each one's per_ds[x["id"]] = vals n = definitions.option_needs(b, vals) needs["embeddings"] += [m for m in n["embeddings"] if m not in needs["embeddings"]] @@ -665,7 +698,7 @@ def pinned(p: dict) -> Dict[str, List[str]]: pairs = [(x, p) for x in dss for p in picks] def word_of(d: dict, p: dict) -> str: - return "-".join(([d["id"]] if many else []) + ([definitions.job_word(p["o"])] if several else [])) + return "-".join(([d["id"]] if many else []) + ([definitions.job_word(p.get("agent") or p["o"])] if several else [])) def launch_of(d: dict, p: dict) -> str: return "-".join(filter(None, [name, word_of(d, p)])) @@ -675,7 +708,7 @@ def job_of(d: dict, p: dict, index: int) -> str: def options_for_pick(d: dict, p: dict) -> dict: """The benchmark's options for this dataset and this pick: a setup's own, or those of the port.""" - return definitions.bench_options_of(b, d["id"], chosen_opts, p["o"]["key"] if p.get("setup") else "yours", + return definitions.bench_options_of(b, d["id"], dict(chosen_opts, **agent_of(p)), p["o"]["key"] if p.get("setup") else "yours", kind=None if p.get("setup") else p["o"].get("_kind"))[0] def command_for(d: dict, p: dict, index: int) -> str: @@ -719,7 +752,7 @@ def script_for(d: dict, p: dict, command: str, index: int) -> str: command = command_for(d, p, i) commands.append(command) parts.append({"id": word_of(d, p) or definitions.job_word(p["o"]), - "label": " · ".join(([d.get("label") or d["id"]] if many else []) + [p["o"]["label"]]) + (f" · shard {i}/{of}" if of > 1 else ""), + "label": " · ".join(([d.get("label") or d["id"]] if many else []) + [(p.get("agent") or p["o"])["label"]]) + (f" · shard {i}/{of}" if of > 1 else ""), "script": script_for(d, p, command, i)}) # a shard's launch id, so its run record joins the launch before its job is written (runs.launches) jobs.append({"name": job_of(d, p, i), "agent": p["o"]["_ref"], "dataset": definitions.ref_of(b), @@ -763,6 +796,8 @@ def row(p: dict) -> list: o = p["o"] if p["control"]: return [STEP["agent"], "Check", f"{o['label']} ({o['key']}) · {o['source']} · {o['from']}"] + if p.get("agent"): + return [STEP["agent"], "Participant", f"{p['agent']['label']} · {b['label']}'s own"] if p.get("setup"): original = (b.get("setups") or {}).get("original") or {} label = original.get("reference_agent") or ("No agent · Original" if side == "agent" else "Original " + (_bench_page(b)["kind"] or "capability")) @@ -775,7 +810,11 @@ def row(p: dict) -> list: ", ".join(f"{x.get('label') or x['id']}" + (f" · {x['units']} {nouns}" if x.get("units") else "") for x in dss)], *[[STEP["limits" if (b.get("options") or {})[k].get("limit") else "dataset" if (b.get("options") or {})[k].get("step") else "agent"], (b.get("options") or {})[k].get("label") or k.replace("_", " ").capitalize(), _shown((b.get("options") or {})[k], v, per_ds[next(x["id"] for x in dss if per_ds[x["id"]].get(k) == v)]) + ("" if k in chosen_opts else " (its default)") + f" · {', '.join(x['id'] for x in dss if per_ds[x['id']].get(k) == v)}"] - for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items() if (b.get("options") or {})[k].get("step") != "where"], + for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items() + if (b.get("options") or {})[k].get("step") != "where" and not (k == "agent" and definitions.own_agents(b)) + and not ((b.get("options") or {})[k].get("when") or {}).get("agent") + or "agent" in ((b.get("options") or {})[k].get("when") or {}) and any( # one of its own agents' settings: that agent's + p.get("agent") and p["agent"]["start"]["driver"]["agent"] in b["options"][k]["when"]["agent"] for p in picks)], *[row(p) for p in picks], *[[STEP["agent"], "Options" + (f" of {p['o']['label']}" if several else ""), ", ".join(f"{k} {v}" + ("" if k in p["given"] else " (its default)") for k, v in p["opts"].items())] diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 554b4986..a76008d3 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -35,16 +35,24 @@ // Every agent (or capability) a benchmark of the standard could take, each with the reasons it does not fit: none, it fits. // Then the controls it declares (nop, oracle): no agent, no model, always runnable - each with its source // (official, adapter, nullius) and, per dataset, its proof. + // A benchmark's own agents (recipe.py, _other_page: its `benchmark`), each run by its driver as its Original setup. + const ownAgentsOf = b => b ? standard().agents.filter(a => a.benchmark === b.key) : []; function standardOthers(r) { const b = standardBench(r); if (!b) return []; const why = standard().fits[r.benchmark] || standard().fits[b.key] || {}; const controls = (b.controls || []).map(c => ({key: c.key, label: c.label, summary: c.from, protocol: 'no agent', source: c.source, version: '', offers: [], proven: {}, options: {}, control: true, why: []})); - // its setups (recipe.py, _bench_page): the benchmark as published, run by its own code - beside the agents or capabilities - const setups = (b.setups || []).map(x => ({...x, protocol: 'its own code', version: '', offers: [], proven: {}, options: {}, setup: true, why: []})); - return (b.family === 'agent' ? standard().agents : standard().capabilities).map(o => ({...o, why: why[o.key] || []})).concat(setups, controls); - } + // its setups (recipe.py, _bench_page): the benchmark as published, run by its own code - beside the agents or capabilities; + // a benchmark with agents of its own runs them in its Original setup instead, first among the agents + const own = ownAgentsOf(b).map(o => ({...o, why: why[o.key] || [], setup: true, own: true})); + const setups = own.length ? [] : (b.setups || []).map(x => ({...x, protocol: 'its own code', version: '', offers: [], proven: {}, options: {}, setup: true, why: []})); + const rest = b.family === 'agent' ? standard().agents.filter(o => o.benchmark !== b.key) : standard().capabilities; + return own.concat(rest.map(o => ({...o, why: why[o.key] || [], elsewhere: !!o.benchmark})), setups, controls); + } + // the chosen ones of a benchmark's own agents, and what each choice runs as: its own agent, its Original setup + const ownChosen = r => ownAgentsOf(standardBench(r)).filter(a => chosenOf(r).includes(a.key)); + const runsOf = r => { const own = ownAgentsOf(standardBench(r)); return [...new Set(chosenOf(r).map(k => own.some(a => a.key === k) ? 'original' : k))]; }; // A control that calls no model at all - on a benchmark with no judge - needs no key to run. // The agents, capabilities and controls a standard recipe runs, each its own job of the one run. const chosenOf = r => Array.isArray(r.others) ? r.others : (r.other ? [r.other] : []); @@ -56,7 +64,13 @@ // A step this recipe shows: every one but Setup. const stepShown = (r, i) => i !== STEP.setup; // Every participant except an unjudged control uses the one selected model. - const paperModels = r => ((standardBench(r) || {}).setups || []).find(x => x.key === 'original') || {}; + // Like the paper's models: the Original setup's, those of the chosen own agents when the benchmark has agents of its own + function paperModels(r) { + const o = ((standardBench(r) || {}).setups || []).find(x => x.key === 'original') || {}, own = ownChosen(r); + if (!own.length || !o.models) return o; + const keep = m => own.some(a => a.models.includes(m)); + return {...o, models: o.models.filter(keep), sent: Object.fromEntries(Object.entries(o.sent || {}).map(([p, ms]) => [p, Object.fromEntries(Object.entries(ms).filter(([m]) => keep(m)))]))}; + } // Like the paper keeps what the paper fixed: its models when it lists them, its judge and its user simulator. const paperListed = r => !!(paperModels(r).models || []).length; const paperOffered = r => { const b = standardBench(r) || {}; return paperListed(r) || !!(b.judge || b.simulator); }; @@ -154,7 +168,7 @@ r.datasets = kept.length ? kept : [b.datasets[0].id]; delete r.dataset; const fitting = standardOthers(r).filter(o => !o.why.length); - const keys = fitting.map(o => o.key), first = fitting.find(o => o.setup && o.key === 'original') + const keys = fitting.map(o => o.key), first = fitting.find(o => o.own) || fitting.find(o => o.setup && o.key === 'original') || fitting.find(o => !o.control && !o.setup); // Keep explicit choices that still fit; otherwise prefer the benchmark's reference. r.others = other && keys.includes(other) ? [other] : chosenOf(r).filter(k => keys.includes(k)); @@ -162,6 +176,7 @@ if (!r.others.length && first) r.others = [first.key]; r.options = Object.fromEntries(Object.entries(r.options || {}).filter(([k, v]) => r.others.includes(k) && v && typeof v === 'object')); // options are each agent's own if (r.bench_options) r.bench_options = ownOptions(b, r.bench_options); + if (r.bench_options && ownAgentsOf(b).length) delete r.bench_options.agent; // each own agent is its own choice if (was !== b || r.name === 'standard-experiment') { r.name = b.key + '-experiment'; r.title = b.label + ' experiment'; } return r; } @@ -178,6 +193,14 @@ if (['small', 'default', 'large', 'xlarge'].includes(raw.boat_size)) r.boat_size = raw.boat_size; if (raw.bench_options && typeof raw.bench_options === 'object' && !Array.isArray(raw.bench_options)) // the benchmark's own options, a word each r.bench_options = ownOptions(standardBench(r), Object.fromEntries(Object.entries(raw.bench_options).filter(([, v]) => typeof v === 'string'))); + // a recipe from before a benchmark's own agents were agents (recipe.py, with_own_agents): its Original with the agent + // option is that agent; without the option the first; a name none has kept, for the server to say + const own = ownAgentsOf(standardBench(r)); + if (own.length && r.others.includes('original')) { + const v = typeof (raw.bench_options || {}).agent === 'string' ? raw.bench_options.agent : '', a = v ? own.find(x => x.agent === v) : own[0]; + r.others = [...new Set(r.others.map(k => k === 'original' ? (a ? a.key : standardBench(r).key + ':' + v) : k))]; + } + if (own.length && r.bench_options) delete r.bench_options.agent; // Options per choice, {ref: {name: word}}; a single agent also accepts flat {name: word} options. if (raw.options && typeof raw.options === 'object' && !Array.isArray(raw.options)) { const words = o => Object.fromEntries(Object.entries(o).filter(([, v]) => typeof v === 'string')); @@ -845,5 +868,5 @@ at: String(d.at || ''), saved: typeof d.saved === 'string' ? d.saved : ''}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STANDARD_PHASES,STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STANDARD_PHASES,STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,ownAgentsOf,ownChosen,runsOf,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 83831868..375cc930 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -427,8 +427,8 @@ // ones that have not proved the ports it needs shown greyed, with the reasons - a model and a scope. The server // compiles the choices (POST /new-run/runner, nullius/standard/recipe.py). // what a unit is: the benchmark's, or a setup's own when only setups run (a conversation, where the paper batches its questions) - const unitsOf = rec => { const b = C.standardBench(rec) || {}, own = (b.setups || []).filter(x => C.chosenOf(rec).includes(x.key)); - return own.length && own.length === C.chosenOf(rec).length ? own[0].units : b.units || {noun: 'unit', nouns: 'units'}; }; + const unitsOf = rec => { const b = C.standardBench(rec) || {}, runs = C.runsOf(rec), own = (b.setups || []).filter(x => runs.includes(x.key)); + return own.length && own.length === runs.length ? own[0].units : b.units || {noun: 'unit', nouns: 'units'}; }; // the first clause of a long description, for a line; the whole goes in its title // settings as sub-tabs, one a chosen way of running that has any: [key, label, html]; the open one kept while it is there let setTab = ''; @@ -576,14 +576,19 @@ // A benchmark's own options (a search, a mode), for the chosen datasets they apply to, and what a value needs of this // machine. Those an agent's or capability's run takes ('yours') on the Agent step, with the agents; those only a setup // takes (a retriever, a top-k of the paper's pipeline) in that setup's tab, once it is chosen. - function benchOptions(rec, b, scope = 'yours') { + // A benchmark with agents of its own: its agent option is each one's own choice, and an option whose `when` names + // agents is in the tab of each of them (scope 'own', `agent` the one); else the options of `scope`. + function benchOptions(rec, b, scope = 'yours', agent = null) { const local = ((schema.standardHere || {}).host || {})[b.key] || {}; - const chosen = C.chosenOf(rec); + const chosen = C.runsOf(rec), owned = C.ownAgentsOf(b).length > 0; + const given = {...(rec.bench_options || {}), ...(agent ? {agent: agent.agent} : {})}; return Object.entries(b.options || {}).filter(([, o]) => !o.limit && o.step === (['where', 'dataset'].includes(scope) ? scope : '')) // a bound of the upstream's own is with the limits + .filter(([n, o]) => !owned || (n !== 'agent' && (scope === 'own') === !!(o.when || {}).agent)) .filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) // one that applies only under another's values (a top-k, only with a retrieval) is left out otherwise - .filter(([, o]) => Object.entries(o.when || {}).every(([k, vs]) => vs.includes((rec.bench_options || {})[k] ?? (b.options[k] || {}).default))) + .filter(([, o]) => Object.entries(o.when || {}).every(([k, vs]) => vs.includes(given[k] ?? (b.options[k] || {}).default))) .filter(([, o]) => { + if (scope === 'own') return true; const on = o.setups || ['yours']; if (o.step) return on.some(s => s === 'yours' ? chosen.some(k => !(b.setups || []).some(x => x.key === k)) : chosen.includes(s)); const shared = on.includes('yours') && on.includes('original'); @@ -641,7 +646,7 @@ return !t.length ? '' : t.every(n => n === t[0]) && t.length > 1 ? `${t[0]} tasks each` : `${t.reduce((a, n) => a + n, 0)} tasks`; }; const pickChip = (attr, v, label, on) => ``; // one tile: {attr, key, on, title, mark, cat, origin, line, meta, foot, find, fam, data (more attributes), tip (its title)} - const tile = t => `