From 21d3e15bd2247a4c44803f3e4360713cc93dc613 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 13:27:11 +0000 Subject: [PATCH 01/33] Add swarm agent_type: 8-member react swarms with a shared-memory channel --- apn/solver.py | 106 ++++++++++++++++++++++++++++++++----------------- apn/task.py | 8 ++++ pyproject.toml | 3 +- uv.lock | 63 +++++++++++++---------------- 4 files changed, 107 insertions(+), 73 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index d7783a2..7bddc46 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -9,6 +9,7 @@ AgentContinue, AgentState, AgentSubmit, + agent, as_solver, deepagent, general, @@ -24,6 +25,7 @@ from inspect_ai.tool import Tool, ToolDef, ToolResult, ToolSource, text_editor, tool from inspect_ai.util import sandbox, store from inspect_boltons.tools import resources +from inspect_swarm import member, shared_memory, swarm, swarm_on_continue, swarm_tools from apn.checker import Claim from apn.layout import ENTRY_PATH @@ -112,7 +114,54 @@ async def execute(claim: Claim) -> ToolResult: return execute -AgentType = Literal["deep", "react"] +AgentType = Literal["deep", "react", "swarm"] + + +def _attempts(gated: bool) -> AgentAttempts: + # An effectively infinite max_attempts on the built-in Inspect react() and deepagent() + # solvers is used to implement "gated submission" (Inspect allows retries if the scorer + # returns INCORRECT). This is a bit of a hack: it would be clearer to implement this + # inside the solver. However, doing it like this lets us benefit automatically from + # upstream improvements in the built-in solvers; in particular, Inspect maintainers will + # keep these up to date with internal Inspect changes. + max_attempts = 99_999_999 if gated else 1 + return AgentAttempts( + attempts=max_attempts, incorrect_message=gated_incorrect_message + ) + + +def _prover_tools() -> list[Tool]: + return [ + text_editor(), + bash(timeout=300), + resources(), + ] + + +def _prover_submit() -> AgentSubmit: + # Name it distinctly from any subagents' "submit" tool and keep the + # call in the message history. keep_in_messages=True stops the loop + # from folding the tool's return into the assistant message; the + # distinct name means the main loop's submission scan can never match + # a subagent's "submit" (no early-termination collision). + return AgentSubmit(tool=submit(), name="submit_proof", keep_in_messages=True) + + +# The default continue message is very generic ("proceed to the next step"), this one +# might be better at avoiding doom loops. +CONTINUE_MESSAGE = "Continue working on the problem." + + +@agent +def prover_member(gated: bool) -> Agent: + """A swarm member: the react prover loop plus the swarm's channel tools.""" + return react( + tools=[*_prover_tools(), swarm_tools()], + attempts=_attempts(gated), + submit=_prover_submit(), + on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), + compaction=CompactionSummary(threshold=300_000), + ) def build_agent( @@ -163,6 +212,7 @@ def lean_prover( gated: bool, literature: bool, util_module: str, + swarm_size: int = 8, ) -> Solver: """ Args: @@ -172,6 +222,7 @@ def lean_prover( util_module: The dataset pin's FC util module (``apn.dataset.fc_profile(...).util_module``), named in the prompt's import-integrity rule. + swarm_size: Number of members when ``agent_type`` is ``"swarm"``. """ async def solve(state: TaskState, generate: Generate) -> TaskState: @@ -179,42 +230,25 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: sketch = state.metadata["sketch"] await sandbox().write_file(ENTRY_PATH, sketch) - tools = [ - text_editor(), - bash(timeout=300), - resources(), - ] - - # An effectively infinite max_attempts on the built-in Inspect react() and deepagent() - # solvers is used to implement "gated submission" (Inspect allows retries if the scorer - # returns INCORRECT). This is a bit of a hack: it would be clearer to implement this - # inside the solver. However, doing it like this lets us benefit automatically from - # upstream improvements in the built-in solvers; in particular, Inspect maintainers will - # keep these up to date with internal Inspect changes. - if gated: - max_attempts = 99_999_999 + if agent_type == "swarm": + agent = swarm( + members=member( + prover_member, + name="prover", + args={"gated": gated}, + count=swarm_size, + ), + channels=[shared_memory()], + ) else: - max_attempts = 1 - - agent = build_agent( - agent_type, - tools=tools, - attempts=AgentAttempts( - attempts=max_attempts, incorrect_message=gated_incorrect_message - ), - # Name it distinctly from any subagents' "submit" tool and keep the - # call in the message history. keep_in_messages=True stops the loop - # from folding the tool's return into the assistant message; the - # distinct name means the main loop's submission scan can never match - # a subagent's "submit" (no early-termination collision). - submit=AgentSubmit( - tool=submit(), name="submit_proof", keep_in_messages=True - ), - # The default continue message is very generic ("proceed to the next step"), this one - # might be better at avoiding doom loops. - on_continue=continue_unless_looping("Continue working on the problem."), - compaction=CompactionSummary(threshold=300_000), - ) + agent = build_agent( + agent_type, + tools=_prover_tools(), + attempts=_attempts(gated), + submit=_prover_submit(), + on_continue=continue_unless_looping(CONTINUE_MESSAGE), + compaction=CompactionSummary(threshold=300_000), + ) state.messages = [ ChatMessageUser( content=user_prompt(ENTRY_PATH, state.token_limit, literature, util_module), diff --git a/apn/task.py b/apn/task.py index 59d9087..6c48197 100644 --- a/apn/task.py +++ b/apn/task.py @@ -254,6 +254,7 @@ def apn_oeis( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", + swarm_size: int = 8, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Formal Conjectures autoformalized OEIS conjectures (492 samples). @@ -271,6 +272,7 @@ def apn_oeis( gated=gated, literature=literature, agent_type=agent_type, + swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), scorer=proof_scorer(SandboxComparator()), @@ -284,6 +286,7 @@ def apn_fc100open( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", + swarm_size: int = 8, sandbox_backend: SandboxBackend | None = None, ) -> Task: name_list = load_subset(FC100_DIR, subset) if subset is not None else None @@ -294,6 +297,7 @@ def apn_fc100open( gated=gated, literature=literature, agent_type=agent_type, + swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), scorer=proof_scorer(SandboxComparator()), @@ -307,6 +311,7 @@ def apn_erdos( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", + swarm_size: int = 8, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Bloom statement selection of Erdős problems.""" @@ -318,6 +323,7 @@ def apn_erdos( gated=gated, literature=literature, agent_type=agent_type, + swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), scorer=proof_scorer(SandboxComparator()), @@ -331,6 +337,7 @@ def apn_personal_corresp( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", + swarm_size: int = 8, sandbox_backend: SandboxBackend | None = None, ) -> Task: """Open conjectures sent to us in personal correspondence, formalized by @@ -345,6 +352,7 @@ def apn_personal_corresp( gated=gated, literature=literature, agent_type=agent_type, + swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), scorer=proof_scorer(SandboxComparator()), diff --git a/pyproject.toml b/pyproject.toml index c11c1ad..0fa3584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ license-files = ["LICENSE"] requires-python = ">=3.13,<3.14" dependencies = [ "inspect-boltons @ git+https://github.com/tadamcz/inspect_boltons@35b794e845f1748947a4f18efdfcb31dd0cf746a", + "inspect-swarm>=0.1.0,<0.2.0", ] [build-system] @@ -31,7 +32,7 @@ apn = "apn._registry" dev = [ "anthropic>=0.105.2", "hawk[cli]", - "inspect-ai>=0.3.263", + "inspect-ai>=0.3.283", "mypy>=2.1.0", "openai>=2.38.0", "pytest>=8.0", diff --git a/uv.lock b/uv.lock index 3babc27..e0443c3 100644 --- a/uv.lock +++ b/uv.lock @@ -22,19 +22,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/61/0a2186c8ca53cf464767d4c4fb7fd9e23f48ce4547c2fa0d1c83a4d52410/agent_client_protocol-0.12.1-py3-none-any.whl", hash = "sha256:aaf3cc301ed87d9e2c4ccb91adfe1cd58784355f6ce7e513f50e7ca2391862ef", size = 85929, upload-time = "2026-08-16T14:08:21.307Z" }, ] -[[package]] -name = "aioboto3" -version = "15.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiobotocore", extra = ["boto3"] }, - { name = "aiofiles" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/01/92e9ab00f36e2899315f49eefcd5b4685fbb19016c7f19a9edf06da80bb0/aioboto3-15.5.0.tar.gz", hash = "sha256:ea8d8787d315594842fbfcf2c4dce3bac2ad61be275bc8584b2ce9a3402a6979", size = 255069, upload-time = "2025-10-30T13:37:16.122Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/3e/e8f5b665bca646d43b916763c901e00a07e40f7746c9128bdc912a089424/aioboto3-15.5.0-py3-none-any.whl", hash = "sha256:cc880c4d6a8481dd7e05da89f41c384dbd841454fc1998ae25ca9c39201437a6", size = 35913, upload-time = "2025-10-30T13:37:14.549Z" }, -] - [[package]] name = "aiobotocore" version = "2.25.1" @@ -53,20 +40,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/2a/d275ec4ce5cd0096665043995a7d76f5d0524853c76a3d04656de49f8808/aiobotocore-2.25.1-py3-none-any.whl", hash = "sha256:eb6daebe3cbef5b39a0bb2a97cffbe9c7cb46b2fcc399ad141f369f3c2134b1f", size = 86039, upload-time = "2025-10-28T22:33:19.949Z" }, ] -[package.optional-dependencies] -boto3 = [ - { name = "boto3" }, -] - -[[package]] -name = "aiofiles" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, -] - [[package]] name = "aiohappyeyeballs" version = "2.6.2" @@ -187,6 +160,7 @@ version = "0.1.11rc11" source = { editable = "." } dependencies = [ { name = "inspect-boltons" }, + { name = "inspect-swarm" }, ] [package.dev-dependencies] @@ -203,14 +177,17 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a" }] +requires-dist = [ + { name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a" }, + { name = "inspect-swarm", specifier = ">=0.1.0,<0.2.0" }, +] [package.metadata.requires-dev] dev = [ { name = "anthropic", specifier = ">=0.105.2" }, { name = "environs" }, { name = "hawk", extras = ["cli"], git = "https://github.com/METR/hawk?subdirectory=hawk&tag=v3.3.1" }, - { name = "inspect-ai", specifier = ">=0.3.263" }, + { name = "inspect-ai", specifier = ">=0.3.283" }, { name = "mypy", specifier = ">=2.1.0" }, { name = "openai", specifier = ">=2.38.0" }, { name = "pytest", specifier = ">=8.0" }, @@ -632,11 +609,11 @@ wheels = [ [[package]] name = "inspect-ai" -version = "0.3.263" +version = "0.3.285" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "agent-client-protocol" }, - { name = "aioboto3" }, + { name = "aiobotocore" }, { name = "anyio" }, { name = "beautifulsoup4" }, { name = "boto3" }, @@ -676,9 +653,9 @@ dependencies = [ { name = "zipp" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/03/3bd61990e197e6e0a68ebd7f1496d0b80af6c1966eea0e48fcfc1ed3b793/inspect_ai-0.3.263.tar.gz", hash = "sha256:54553ca8bfe711853414b49d962e492a60fd4cac8df935be47287a4b719f92b1", size = 35796833, upload-time = "2026-09-04T01:38:04.104Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/e0/595bf39b253826aaa8fd886e44ca41fc5fc9730e20aad044b8d5f927d32d/inspect_ai-0.3.285.tar.gz", hash = "sha256:78b927ddb6bf959f00cb9baffc5081eb4872067961000463696a92134c8d2025", size = 36578076, upload-time = "2026-10-25T14:29:12.556Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/c0/c12e505462462d5cc2b3fa98b6b2142c9589ebc0308144498f6e9de0c70c/inspect_ai-0.3.263-py3-none-any.whl", hash = "sha256:503e7ff509d77fcdf65989027a19fea966fb668deb88a234468c3290964fd86a", size = 34294681, upload-time = "2026-09-04T01:37:58.013Z" }, + { url = "https://files.pythonhosted.org/packages/b7/1f/ee409d639d66453f864c719c3d0d544cd234b3b2a0deef730c4cd4d55c32/inspect_ai-0.3.285-py3-none-any.whl", hash = "sha256:4658e36773e9d78116a69a3eb066f47698aa7ea9fc1c0146927a86f02b455aa6", size = 35251997, upload-time = "2026-10-25T14:29:08.272Z" }, ] [[package]] @@ -690,6 +667,20 @@ dependencies = [ { name = "tabulate" }, ] +[[package]] +name = "inspect-swarm" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "inspect-ai" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/7a/47f9fb7dd58b29715b2985b5398c53be6cc09b3321b73ada98ab273b02f0/inspect_swarm-0.1.0.tar.gz", hash = "sha256:78e75588583f6a09716973589fe1dbb9bebacf0f2b6a0cd6244ae0e31d7b9048", size = 88033, upload-time = "2026-10-23T13:23:48.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/0f/b86e986fb631688286690571dfd2d85e1ed76b49dadd74809b7e061dd12b/inspect_swarm-0.1.0-py3-none-any.whl", hash = "sha256:e4600ae6709673cd33048b50728a626376d5f943df9e79e2918d6b330b99299d", size = 101415, upload-time = "2026-10-23T13:23:46.65Z" }, +] + [[package]] name = "jaraco-classes" version = "3.4.0" @@ -1085,11 +1076,11 @@ wheels = [ [[package]] name = "nest-asyncio2" -version = "1.7.2" +version = "1.7.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b4/73/731debf26e27e0a0323d7bda270dc2f634b398e38f040a09da1f4351d0aa/nest_asyncio2-1.7.2.tar.gz", hash = "sha256:1921d70b92cc4612c374928d081552efb59b83d91b2b789d935c665fa01729a8", size = 14743, upload-time = "2026-02-13T00:34:04.386Z" } +sdist = { url = "https://files.pythonhosted.org/packages/32/2c/70e3b6bb2189cfd260280b7e1042bd622949a258663f95e887fada4ac1e6/nest_asyncio2-1.7.4.tar.gz", hash = "sha256:995b1409d926ec55e0876ea39a69bffdd8852cf1db3652c177e8c8ffdbcaf3cc", size = 16891, upload-time = "2026-10-07T22:31:50.507Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/3c/3179b85b0e1c3659f0369940200cd6d0fa900e6cefcc7ea0bc6dd0e29ffb/nest_asyncio2-1.7.2-py3-none-any.whl", hash = "sha256:f5dfa702f3f81f6a03857e9a19e2ba578c0946a4ad417b4c50a24d7ba641fe01", size = 7843, upload-time = "2026-02-13T00:34:02.691Z" }, + { url = "https://files.pythonhosted.org/packages/09/ed/559d2915dce85794bc0c36064043da170da50cd98746dcbddd85d5932017/nest_asyncio2-1.7.4-py3-none-any.whl", hash = "sha256:86e77b52c3d1496b294af3c944b44213b9b82a12a5ee590c0acc4ca4692ef6c7", size = 8543, upload-time = "2026-10-07T22:31:49.245Z" }, ] [[package]] From 9eb7e720cf078ea3d6d1a375fd4124147ff1a451 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 13:27:17 +0000 Subject: [PATCH 02/33] Add erdos74-swarm8 eval-set config --- configs/erdos74-swarm8.yaml | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 configs/erdos74-swarm8.yaml diff --git a/configs/erdos74-swarm8.yaml b/configs/erdos74-swarm8.yaml new file mode 100644 index 0000000..eba58f7 --- /dev/null +++ b/configs/erdos74-swarm8.yaml @@ -0,0 +1,60 @@ +# Schema: https://github.com/METR/hawk/blob/main/hawk/api/EvalSetConfig.schema.json +name: erdos74-swarm8 + +tasks: + - package: git+ssh://git@github.com/epoch-research/LeanOpenProblems.git@21d3e15bd2247a4c44803f3e4360713cc93dc613 + name: apn + items: + - name: apn_erdos + args: + subset: bloom_selection + agent_type: swarm + swarm_size: 8 + literature: true + sample_ids: + - Erdos74.erdos_74 +models: + - package: openai + name: openai + items: + - name: gpt-6-astra + args: + config: + reasoning_effort: "max" + max_retries: 7 + max_connections: 10 + +cost_limit: 9600.0 +working_limit: 432_000 # 5 * 24 * 60 * 60 + +# $ per 1M tokens; keys must match Inspect's resolved / names. +model_cost_config: + openai/gpt-6-astra: + input: 10.0 + output: 50.0 + input_cache_read: 1.0 + input_cache_write: 12.5 + +secrets: + - name: LEAN_OPEN_PROBLEMS_IMAGE_NAME + description: "The ECR repo containing the LeanOpenProblems sandbox images" + +runner: + memory: 200Gi + storage_request: 410Gi + storage_limit: 410Gi + environment: + INSPECT_LOG_CONDENSE: "true" + # inspect-swarm 0.1.0 was published 2026-10-23; the smoke config's older + # cutoff would exclude it. + UV_EXCLUDE_NEWER: "2026-10-24T00:00:00Z" + +packages: + - inspect-ai==0.3.283 + - semver>=3.0.0,<3.1.0 + +checkpoint: + enabled: true + trigger: + type: time + every: 43200 # 12 * 60 * 60 From 8b58fb2f0e36f5e7cf44b90dbd902a7d5e243639 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 13:28:16 +0000 Subject: [PATCH 03/33] Point erdos74-swarm8 at the swarm branch head --- configs/erdos74-swarm8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/erdos74-swarm8.yaml b/configs/erdos74-swarm8.yaml index eba58f7..caa710b 100644 --- a/configs/erdos74-swarm8.yaml +++ b/configs/erdos74-swarm8.yaml @@ -2,7 +2,7 @@ name: erdos74-swarm8 tasks: - - package: git+ssh://git@github.com/epoch-research/LeanOpenProblems.git@21d3e15bd2247a4c44803f3e4360713cc93dc613 + - package: git+ssh://git@github.com/epoch-research/LeanOpenProblems.git@swarm name: apn items: - name: apn_erdos From 9e35c37da8e93ee878a6c2ea4136ce2c655350dc Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 13:42:46 +0000 Subject: [PATCH 04/33] Give each swarm member its own sandbox; split the budget equally Each member works in its own agent sandbox (prover-1..N) and reaches the shared comparator by name; the shared-memory board stays the only channel. The submit tool records the claim per member (keyed by the member's tool state scope), an in-member gate check scores that member's own sandbox, and the final score checks every member's sandbox: CORRECT if any passes. Concurrent checks on one comparator sandbox are now serialized: each check resets the comparator's shared staging directories, so two members checking at once would clobber each other's inputs. Swap the erdos74-swarm8 config for erdos74-swarm4: 4 members, $400 cap, no checkpointing (inspect_swarm does not support resuming a swarm). --- apn/checker.py | 13 +- apn/scorer.py | 138 ++++++++++++------ apn/solver.py | 48 ++++-- apn/task.py | 106 +++++++++----- ...rdos74-swarm8.yaml => erdos74-swarm4.yaml} | 16 +- tests/test_checker.py | 130 ++++++++++++++++- 6 files changed, 347 insertions(+), 104 deletions(-) rename configs/{erdos74-swarm8.yaml => erdos74-swarm4.yaml} (78%) diff --git a/apn/checker.py b/apn/checker.py index e713dc1..60cb5c4 100644 --- a/apn/checker.py +++ b/apn/checker.py @@ -6,8 +6,10 @@ from dataclasses import dataclass from pathlib import PurePosixPath from typing import Literal, Protocol, runtime_checkable +from weakref import WeakKeyDictionary -from inspect_ai.util import OutputLimitExceededError, sandbox +import anyio +from inspect_ai.util import OutputLimitExceededError, SandboxEnvironment, sandbox from apn.layout import ENTRY_MODULE, ENTRY_PATH, ENTRY_REL, PROJECT, SUBMISSION_DIR @@ -230,6 +232,9 @@ def comparator_config(decl: str, claim: Claim) -> str: ) +_CHECK_LOCKS: WeakKeyDictionary[SandboxEnvironment, anyio.Lock] = WeakKeyDictionary() + + class SandboxComparator: """Runs Lean FRO's Comparator against the trusted ``comparator`` sandbox. @@ -277,6 +282,12 @@ async def check( except InvalidSubmission as exc: return CheckOutcome(ok=False, stage="entry_missing", detail=str(exc)) + async with _CHECK_LOCKS.setdefault(sb, anyio.Lock()): + return await self._run(sb, spec, staged, decl, claim) + + async def _run( + self, sb: SandboxEnvironment, spec: str, staged: bytes, decl: str, claim: Claim + ) -> CheckOutcome: # Trusted filesystem reset (a reference step: failure raises). It # restores a pristine .lake -- the only path the untrusted build can # write under its landrun sandbox; it does not terminate processes a diff --git a/apn/scorer.py b/apn/scorer.py index 63225e2..644c60c 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -13,7 +13,14 @@ stderr, ) from inspect_ai.solver import TaskState -from inspect_ai.util import OutputLimitExceededError, sandbox, store +from inspect_ai.util import ( + OutputLimitExceededError, + SandboxEnvironment, + sandbox, + store, + tool_state_instance, + tool_state_scope, +) from apn.checker import Claim, ProofChecker from apn.filetree import build_tree_from_tar, read_submission_tar @@ -23,59 +30,108 @@ # Where the submit tool records the agent's declared claim (see apn.solver). CLAIM_STORE_KEY = "submission_claim" +SWARM_RESULT_KEY = "inspect_swarm:result" + + +def claim_store_key() -> str: + instance = tool_state_instance(None) + return CLAIM_STORE_KEY if instance is None else f"{CLAIM_STORE_KEY}:{instance}" + @scorer(metrics=[accuracy(), stderr()]) -def proof_scorer(checker: ProofChecker) -> Scorer: +def proof_scorer(checker: ProofChecker, swarm: bool = False) -> Scorer: """Score a sample by checking the agent's ``Submission/`` module tree (entry module ``Spec.lean``) with the configured proof checker (Comparator in - production).""" + production). + + With ``swarm``, a check inside a member scores that member's own sandbox, + and the final score after the swarm checks every member's sandbox: CORRECT + if any member's submission passes.""" async def score(state: TaskState, target: Target) -> Score: - # Per-attempt attempt index, kept in the sample store (the react/deepagent - # attempt_count is not reachable from here). Increments even when - # max_attempts=1, so a single-attempt sample still tags attempt-1. - attempt = store().get("_score_call_idx", 0) + 1 - store().set("_score_call_idx", attempt) - - try: - tar = await read_submission_tar(sandbox()) - except OutputLimitExceededError as exc: - return Score( - value=INCORRECT, - explanation=str(exc), - metadata={"stage": "submission_oversize", "verifier_output": None}, - ) - - _write_submission_sidecar(state, attempt, tar) - _record_submission_tree(state, tar) - - # The claim the agent declared on its submit call. A sample scored - # without one (e.g. it hit its limits before ever submitting) defaults - # to "proof" -- deterministic, and such submissions reject anyway. - claim: Claim = store().get(CLAIM_STORE_KEY, "proof") - # The target theorem's fully qualified name (== the sample id except - # where the manifest overrides it; see apn.dataset.build_dataset). - decl = state.metadata["decl_name"] - - spec = state.metadata["sketch"] - outcome = await checker.check(spec, tar, decl=decl, claim=claim) - return Score( - value=CORRECT if outcome.ok else INCORRECT, - explanation=outcome.detail, - metadata={ - "stage": outcome.stage, - "claim": claim, - "verifier_output": outcome.detail, - }, + if swarm and tool_state_instance(None) is None: + return await _score_swarm(state, checker) + return await _score_sandbox( + state, checker, sandbox(), claim_store_key(), "submission_contents" ) return score -def _record_submission_tree(state: TaskState, tar: bytes) -> None: +async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: + record = state.store.get(SWARM_RESULT_KEY) + scores: dict[str, Score] = {} + for m in record["members"]: + with tool_state_scope(m["name"]): + claim_key = claim_store_key() + scores[m["name"]] = await _score_sandbox( + state, + checker, + sandbox(m["sandbox"]), + claim_key, + f"submission_contents_{m['name']}", + ) + passed = [name for name, s in scores.items() if s.value == CORRECT] + return Score( + value=CORRECT if passed else INCORRECT, + explanation=f"passed: {', '.join(passed)}" if passed else "no member passed", + metadata={ + "passed": passed, + "members": {name: s.metadata for name, s in scores.items()}, + }, + ) + + +async def _score_sandbox( + state: TaskState, + checker: ProofChecker, + sb: SandboxEnvironment, + claim_key: str, + tree_key: str, +) -> Score: + # Per-attempt attempt index, kept in the sample store (the react/deepagent + # attempt_count is not reachable from here). Increments even when + # max_attempts=1, so a single-attempt sample still tags attempt-1. + attempt = store().get("_score_call_idx", 0) + 1 + store().set("_score_call_idx", attempt) + + try: + tar = await read_submission_tar(sb) + except OutputLimitExceededError as exc: + return Score( + value=INCORRECT, + explanation=str(exc), + metadata={"stage": "submission_oversize", "verifier_output": None}, + ) + + _write_submission_sidecar(state, attempt, tar) + _record_submission_tree(state, tar, tree_key) + + # The claim the agent declared on its submit call. A sample scored + # without one (e.g. it hit its limits before ever submitting) defaults + # to "proof" -- deterministic, and such submissions reject anyway. + claim: Claim = store().get(claim_key, "proof") + # The target theorem's fully qualified name (== the sample id except + # where the manifest overrides it; see apn.dataset.build_dataset). + decl = state.metadata["decl_name"] + + spec = state.metadata["sketch"] + outcome = await checker.check(spec, tar, decl=decl, claim=claim) + return Score( + value=CORRECT if outcome.ok else INCORRECT, + explanation=outcome.detail, + metadata={ + "stage": outcome.stage, + "claim": claim, + "verifier_output": outcome.detail, + }, + ) + + +def _record_submission_tree(state: TaskState, tar: bytes, key: str) -> None: """Set the agent's ``Submission/`` directory as a display tree on sample metadata.""" try: - state.metadata["submission_contents"] = build_tree_from_tar(tar) + state.metadata[key] = build_tree_from_tar(tar) except Exception: logger.warning( "Failed to build the Submission/ display tree from the scored tar; " diff --git a/apn/solver.py b/apn/solver.py index 7bddc46..1357295 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -25,13 +25,20 @@ from inspect_ai.tool import Tool, ToolDef, ToolResult, ToolSource, text_editor, tool from inspect_ai.util import sandbox, store from inspect_boltons.tools import resources -from inspect_swarm import member, shared_memory, swarm, swarm_on_continue, swarm_tools +from inspect_swarm import ( + Budget, + member, + shared_memory, + swarm, + swarm_on_continue, + swarm_tools, +) from apn.checker import Claim from apn.layout import ENTRY_PATH from apn.limits import continue_unless_looping from apn.prompts import user_prompt -from apn.scorer import CLAIM_STORE_KEY +from apn.scorer import claim_store_key from apn.tools import bash logger = logging.getLogger(__name__) @@ -108,7 +115,7 @@ async def execute(claim: Claim) -> ToolResult: """ if claim not in ("proof", "disproof"): raise ValueError(f'claim must be "proof" or "disproof", got {claim!r}') - store().set(CLAIM_STORE_KEY, claim) + store().set(claim_store_key(), claim) return "Submitted." return execute @@ -152,6 +159,13 @@ def _prover_submit() -> AgentSubmit: CONTINUE_MESSAGE = "Continue working on the problem." +COMPARATOR_SANDBOX = "comparator" + + +def swarm_member_names(swarm_size: int) -> list[str]: + return [f"prover-{i}" for i in range(1, swarm_size + 1)] + + @agent def prover_member(gated: bool) -> Agent: """A swarm member: the react prover loop plus the swarm's channel tools.""" @@ -212,7 +226,7 @@ def lean_prover( gated: bool, literature: bool, util_module: str, - swarm_size: int = 8, + swarm_size: int = 4, ) -> Solver: """ Args: @@ -222,25 +236,35 @@ def lean_prover( util_module: The dataset pin's FC util module (``apn.dataset.fc_profile(...).util_module``), named in the prompt's import-integrity rule. - swarm_size: Number of members when ``agent_type`` is ``"swarm"``. + swarm_size: Number of members when ``agent_type`` is ``"swarm"``. Each + member works in its own sandbox (see ``swarm_member_names``) and + shares the comparator; the swarm's budget is split equally. """ async def solve(state: TaskState, generate: Generate) -> TaskState: _warn_if_ignored_formalizations(state) sketch = state.metadata["sketch"] - await sandbox().write_file(ENTRY_PATH, sketch) if agent_type == "swarm": + names = swarm_member_names(swarm_size) + for name in names: + await sandbox(name).write_file(ENTRY_PATH, sketch) agent = swarm( - members=member( - prover_member, - name="prover", - args={"gated": gated}, - count=swarm_size, - ), + members=[ + member( + prover_member, + name=name, + args={"gated": gated}, + sandbox=name, + visible_sandboxes=[COMPARATOR_SANDBOX], + ) + for name in names + ], channels=[shared_memory()], + budget=Budget(split="equal"), ) else: + await sandbox().write_file(ENTRY_PATH, sketch) agent = build_agent( agent_type, tools=_prover_tools(), diff --git a/apn/task.py b/apn/task.py index 6c48197..ef8e9be 100644 --- a/apn/task.py +++ b/apn/task.py @@ -4,13 +4,13 @@ import re import tempfile from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, Sequence import yaml from inspect_ai import Task, task from apn import __version__ -from apn.solver import AgentType, lean_prover +from apn.solver import AgentType, lean_prover, swarm_member_names from apn.checker import SandboxComparator from apn.dataset import ( ERDOS_DIR, @@ -109,7 +109,11 @@ def _build_section(target: str, fc_commit: str) -> dict[str, Any]: } -def get_compose_file_content(fc_commit: str, literature: bool = False) -> str: +def get_compose_file_content( + fc_commit: str, + literature: bool = False, + agent_services: Sequence[str] = ("default",), +) -> str: """The docker-backend sandbox config (local runs, CI tests). Two services: the agent's workspace, and the trusted `comparator` verifier. @@ -123,13 +127,16 @@ def get_compose_file_content(fc_commit: str, literature: bool = False) -> str: agent_kind = _agent_image_kind(literature) compose: dict[str, Any] = { "services": { - "default": { - "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image(agent_kind, fc_commit)}", - "build": _build_section(agent_kind, fc_commit), - "init": True, - "entrypoint": list(SANDBOX_COMMAND), - "mem_limit": f"{AGENT_MEMORY_GIB}g", - "network_mode": "none", + **{ + name: { + "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image(agent_kind, fc_commit)}", + "build": _build_section(agent_kind, fc_commit), + "init": True, + "entrypoint": list(SANDBOX_COMMAND), + "mem_limit": f"{AGENT_MEMORY_GIB}g", + "network_mode": "none", + } + for name in agent_services }, "comparator": { "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image('comparator', fc_commit)}", @@ -148,7 +155,11 @@ def get_compose_file_content(fc_commit: str, literature: bool = False) -> str: COMPARATOR_EPHEMERAL_STORAGE_GIB = 8 -def get_values_file_content(fc_commit: str, literature: bool = False) -> str: +def get_values_file_content( + fc_commit: str, + literature: bool = False, + agent_services: Sequence[str] = ("default",), +) -> str: """The k8s/Hawk-backend sandbox config: chart-native agent-env values. Written directly in the Helm chart's vocabulary. k8s_sandbox could @@ -183,14 +194,19 @@ def resources(memory_gib: int, ephemeral_storage_gib: int) -> dict[str, Any]: values: dict[str, Any] = { "services": { - "default": { - "image": f"{repository}:{get_identifier_for_image(agent_kind, fc_commit)}", - # Without this the chart defaults to a bare `tail -f /dev/null` - # as PID 1, which never reaps orphans (see SANDBOX_COMMAND). - "command": list(SANDBOX_COMMAND), - "networkIsolated": True, - "dnsRecord": True, - "resources": resources(AGENT_MEMORY_GIB, AGENT_EPHEMERAL_STORAGE_GIB), + **{ + name: { + "image": f"{repository}:{get_identifier_for_image(agent_kind, fc_commit)}", + # Without this the chart defaults to a bare `tail -f /dev/null` + # as PID 1, which never reaps orphans (see SANDBOX_COMMAND). + "command": list(SANDBOX_COMMAND), + "networkIsolated": True, + "dnsRecord": True, + "resources": resources( + AGENT_MEMORY_GIB, AGENT_EPHEMERAL_STORAGE_GIB + ), + } + for name in agent_services }, "comparator": { "image": f"{repository}:{get_identifier_for_image('comparator', fc_commit)}", @@ -208,7 +224,10 @@ def resources(memory_gib: int, ephemeral_storage_gib: int) -> dict[str, Any]: def get_sandbox_config( - fc_commit: str, literature: bool, backend: SandboxBackend | None + fc_commit: str, + literature: bool, + backend: SandboxBackend | None, + agent_services: Sequence[str] = ("default",), ) -> tuple[SandboxBackend, str]: """The Inspect ``sandbox`` spec ``(type, config-file path)`` for a backend (``None``: see :func:`resolve_sandbox_backend`). @@ -216,7 +235,9 @@ def get_sandbox_config( Each backend gets its own generated artifact (``compose.yaml`` for docker, ``values.yaml`` for k8s -- k8s_sandbox treats any config file NOT named ``*compose.yaml``/``*compose.yml`` as chart values). Files are isolated in - per-(version, FC pin, variant) subdirs so they don't clobber each other. + per-(version, FC pin, variant, agent services) subdirs so they don't + clobber each other. ``agent_services`` names one agent sandbox each (a + swarm's members); all share the one comparator. """ backend = resolve_sandbox_backend(backend) variant = "corpus" if literature else "closed-book" @@ -225,14 +246,15 @@ def get_sandbox_config( / _docker_tag_component(__version__) / f"fc_{fc_commit[:12]}" / variant + / "+".join(agent_services) ) directory.mkdir(parents=True, exist_ok=True) if backend == "docker": path = directory / "compose.yaml" - content = get_compose_file_content(fc_commit, literature) + content = get_compose_file_content(fc_commit, literature, agent_services) elif backend == "k8s": path = directory / "values.yaml" - content = get_values_file_content(fc_commit, literature) + content = get_values_file_content(fc_commit, literature, agent_services) else: raise ValueError( f"Unknown sandbox_backend {backend!r}; expected 'docker' or 'k8s'." @@ -248,13 +270,19 @@ def get_compose_file(fc_commit: str, literature: bool = False) -> Path: return Path(get_sandbox_config(fc_commit, literature, "docker")[1]) +def _agent_services(agent_type: AgentType, swarm_size: int) -> list[str]: + if agent_type == "swarm": + return swarm_member_names(swarm_size) + return ["default"] + + @task def apn_oeis( subset: str | None = None, gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 8, + swarm_size: int = 4, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Formal Conjectures autoformalized OEIS conjectures (492 samples). @@ -275,8 +303,10 @@ def apn_oeis( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator()), - sandbox=get_sandbox_config(pin, literature, sandbox_backend), + scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + sandbox=get_sandbox_config( + pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + ), ) @@ -286,7 +316,7 @@ def apn_fc100open( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 8, + swarm_size: int = 4, sandbox_backend: SandboxBackend | None = None, ) -> Task: name_list = load_subset(FC100_DIR, subset) if subset is not None else None @@ -300,8 +330,10 @@ def apn_fc100open( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator()), - sandbox=get_sandbox_config(pin, literature, sandbox_backend), + scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + sandbox=get_sandbox_config( + pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + ), ) @@ -311,7 +343,7 @@ def apn_erdos( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 8, + swarm_size: int = 4, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Bloom statement selection of Erdős problems.""" @@ -326,8 +358,10 @@ def apn_erdos( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator()), - sandbox=get_sandbox_config(pin, literature, sandbox_backend), + scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + sandbox=get_sandbox_config( + pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + ), ) @@ -337,7 +371,7 @@ def apn_personal_corresp( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 8, + swarm_size: int = 4, sandbox_backend: SandboxBackend | None = None, ) -> Task: """Open conjectures sent to us in personal correspondence, formalized by @@ -355,6 +389,8 @@ def apn_personal_corresp( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator()), - sandbox=get_sandbox_config(pin, literature, sandbox_backend), + scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + sandbox=get_sandbox_config( + pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + ), ) diff --git a/configs/erdos74-swarm8.yaml b/configs/erdos74-swarm4.yaml similarity index 78% rename from configs/erdos74-swarm8.yaml rename to configs/erdos74-swarm4.yaml index caa710b..9f6fba0 100644 --- a/configs/erdos74-swarm8.yaml +++ b/configs/erdos74-swarm4.yaml @@ -1,5 +1,5 @@ # Schema: https://github.com/METR/hawk/blob/main/hawk/api/EvalSetConfig.schema.json -name: erdos74-swarm8 +name: erdos74-swarm4 tasks: - package: git+ssh://git@github.com/epoch-research/LeanOpenProblems.git@swarm @@ -9,7 +9,7 @@ tasks: args: subset: bloom_selection agent_type: swarm - swarm_size: 8 + swarm_size: 4 literature: true sample_ids: - Erdos74.erdos_74 @@ -24,7 +24,7 @@ models: max_retries: 7 max_connections: 10 -cost_limit: 9600.0 +cost_limit: 400.0 working_limit: 432_000 # 5 * 24 * 60 * 60 # $ per 1M tokens; keys must match Inspect's resolved / names. @@ -41,20 +41,10 @@ secrets: runner: memory: 200Gi - storage_request: 410Gi - storage_limit: 410Gi environment: INSPECT_LOG_CONDENSE: "true" - # inspect-swarm 0.1.0 was published 2026-10-23; the smoke config's older - # cutoff would exclude it. UV_EXCLUDE_NEWER: "2026-10-24T00:00:00Z" packages: - inspect-ai==0.3.283 - semver>=3.0.0,<3.1.0 - -checkpoint: - enabled: true - trigger: - type: time - every: 43200 # 12 * 60 * 60 diff --git a/tests/test_checker.py b/tests/test_checker.py index 381ac1c..dec52dd 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -28,11 +28,12 @@ from types import SimpleNamespace from typing import cast +import anyio import pytest from inspect_ai.model import ModelName from inspect_ai.scorer import CORRECT, INCORRECT, Score, Target from inspect_ai.solver import TaskState -from inspect_ai.util import ExecResult, OutputLimitExceededError +from inspect_ai.util import ExecResult, OutputLimitExceededError, tool_state_scope import apn.checker as checker_mod import apn.scorer as scorer_mod @@ -51,7 +52,12 @@ module_path, sanitize_submission, ) -from apn.scorer import CLAIM_STORE_KEY, proof_scorer +from apn.scorer import ( + CLAIM_STORE_KEY, + SWARM_RESULT_KEY, + claim_store_key, + proof_scorer, +) SPEC = "import Mathlib\ntheorem tgt : True := by sorry\ntheorem tgt.disproof : ¬ (type_of% @tgt) := sorry\n" @@ -709,3 +715,123 @@ async def test_scorer_writes_attempt_sidecar( sidecars = list((tmp_path / "artifacts").rglob("attempt-*.tar")) assert len(sidecars) == 1 assert sidecars[0].read_bytes() == tar + + +async def test_concurrent_checks_on_one_comparator_are_serialized( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class SlowSandbox(ScriptedSandbox): + async def exec(self, cmd: list[str], **kwargs: object) -> ExecResult[str]: + await anyio.sleep(0.01) + return await super().exec(cmd, **kwargs) + + sb = SlowSandbox(comparator=_ok(_ACCEPT_OUT)) + monkeypatch.setattr(checker_mod, "sandbox", lambda *a, **k: sb) + checker = SandboxComparator() + + async def check() -> None: + await checker.check(SPEC, SUBMISSION_TAR, decl="tgt", claim="proof") + + async with anyio.create_task_group() as tg: + tg.start_soon(check) + tg.start_soon(check) + + one_check = [ + ("exec", [RESET_SCRIPT]), + ("exec", STAGING_RESET_CMD), + ("write", CHALLENGE_PATH), + ("write", SUBMISSION_TAR_PATH), + ("write", CONFIG_PATH), + ("exec", UNPACK_CMD), + ("exec", COMPARATOR_CMD), + ] + assert sb.events == one_check + one_check + + +def test_claim_store_key_is_per_tool_state_scope() -> None: + assert claim_store_key() == CLAIM_STORE_KEY + with tool_state_scope("prover-1"): + key_1 = claim_store_key() + with tool_state_scope("prover-2"): + key_2 = claim_store_key() + assert len({CLAIM_STORE_KEY, key_1, key_2}) == 3 + + +class TarChecker: + def __init__(self, passing: bytes) -> None: + self._passing = passing + self.calls: list[tuple[bytes, str]] = [] + + async def check( + self, spec: str, submission_tar: bytes, decl: str, claim: str + ) -> CheckOutcome: + self.calls.append((submission_tar, claim)) + ok = submission_tar == self._passing + return CheckOutcome(ok=ok, stage="stub", detail="stub outcome") + + +def _swarm_fixture( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[TaskState, dict[str, bytes], FakeStore]: + tars = { + name: _tar_of({"./Spec.lean": f"{name} work"}) + for name in ("prover-1", "prover-2", "prover-3") + } + store = FakeStore() + for name, claim in (("prover-1", "proof"), ("prover-2", "disproof")): + with tool_state_scope(name): + store.set(claim_store_key(), claim) + state = _state(store) + state.store.set( + SWARM_RESULT_KEY, + {"members": [{"name": name, "sandbox": name} for name in tars]}, + ) + sandboxes = {name: FakeSandbox(tar) for name, tar in tars.items()} + + def fake_sandbox(name: str | None = None) -> FakeSandbox: + return sandboxes["prover-2" if name is None else name] + + monkeypatch.setattr(scorer_mod, "sandbox", fake_sandbox) + monkeypatch.setattr(scorer_mod, "store", lambda: store) + return state, tars, store + + +async def test_swarm_final_score_checks_every_member_sandbox( + monkeypatch: pytest.MonkeyPatch, +) -> None: + state, tars, _ = _swarm_fixture(monkeypatch) + checker = TarChecker(passing=tars["prover-2"]) + score = await proof_scorer(checker, swarm=True)(state, Target("")) + assert score is not None + assert score.value == CORRECT + assert score.metadata is not None + assert score.metadata["passed"] == ["prover-2"] + assert checker.calls == [ + (tars["prover-1"], "proof"), + (tars["prover-2"], "disproof"), + (tars["prover-3"], "proof"), + ] + assert {"submission_contents_prover-1", "submission_contents_prover-3"} <= set( + state.metadata + ) + + +async def test_swarm_final_score_incorrect_when_no_member_passes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + state, _, _ = _swarm_fixture(monkeypatch) + score = await proof_scorer(TarChecker(passing=b""), swarm=True)(state, Target("")) + assert score is not None + assert score.value == INCORRECT + + +async def test_swarm_score_inside_a_member_checks_only_its_own_sandbox( + monkeypatch: pytest.MonkeyPatch, +) -> None: + state, tars, _ = _swarm_fixture(monkeypatch) + checker = TarChecker(passing=tars["prover-2"]) + with tool_state_scope("prover-2"): + score = await proof_scorer(checker, swarm=True)(state, Target("")) + assert score is not None + assert score.value == CORRECT + assert checker.calls == [(tars["prover-2"], "disproof")] From 3a5ef91ad7b265f503ac99fb14e7a342d3ac8d83 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 13:51:24 +0000 Subject: [PATCH 05/33] Give each swarm member its own comparator instead of locking a shared one Each member prover-N now gets its own comparator-prover-N sandbox, the only one it can see, so concurrent checks never share a comparator and the lock goes. The checker picks the comparator for the caller's member (from its tool state scope), and final scoring runs each member's check inside that member's scope so it resolves the same comparator and claim. Single-agent runs keep the single default + comparator pair. --- apn/checker.py | 39 ++++++------ apn/scorer.py | 30 ++++------ apn/solver.py | 11 ++-- apn/task.py | 135 ++++++++++++++++++++---------------------- tests/test_checker.py | 35 ++++------- 5 files changed, 110 insertions(+), 140 deletions(-) diff --git a/apn/checker.py b/apn/checker.py index 60cb5c4..68131e5 100644 --- a/apn/checker.py +++ b/apn/checker.py @@ -6,13 +6,25 @@ from dataclasses import dataclass from pathlib import PurePosixPath from typing import Literal, Protocol, runtime_checkable -from weakref import WeakKeyDictionary -import anyio -from inspect_ai.util import OutputLimitExceededError, SandboxEnvironment, sandbox +from inspect_ai.util import OutputLimitExceededError, sandbox, tool_state_instance from apn.layout import ENTRY_MODULE, ENTRY_PATH, ENTRY_REL, PROJECT, SUBMISSION_DIR +COMPARATOR_SANDBOX = "comparator" + + +def current_member() -> str | None: + """The swarm member whose tool state scope the caller runs in, if any.""" + instance = tool_state_instance(None) + return None if instance is None else instance.removeprefix("@") + + +def comparator_sandbox_name(member: str | None) -> str: + """The comparator sandbox that checks ``member``'s submissions: one per + swarm member, or the sample's single comparator outside a swarm.""" + return COMPARATOR_SANDBOX if member is None else f"{COMPARATOR_SANDBOX}-{member}" + # Paths inside the trusted `comparator` sandbox (the comparator stage of # apn/lean/Dockerfile). The challenge is staged into the lake project's `run/` # scratch lib; the agent's submission tree is staged at `Submission/` -- the @@ -232,11 +244,9 @@ def comparator_config(decl: str, claim: Claim) -> str: ) -_CHECK_LOCKS: WeakKeyDictionary[SandboxEnvironment, anyio.Lock] = WeakKeyDictionary() - - class SandboxComparator: - """Runs Lean FRO's Comparator against the trusted ``comparator`` sandbox. + """Runs Lean FRO's Comparator against the trusted comparator sandbox + (the caller's, per :func:`comparator_sandbox_name`). Per check (comparator-migration-plan.md §3.2): reset the sandbox's ``.lake`` to the image's pristine tree, recreate the ``run/`` and @@ -261,18 +271,13 @@ class SandboxComparator: Comparator transcript at scoring time. """ - def __init__( - self, - sandbox_name: str | None = "comparator", - timeout: int = 60 * 60, - ) -> None: - self._sandbox_name = sandbox_name + def __init__(self, timeout: int = 60 * 60) -> None: self._timeout = timeout async def check( self, spec: str, submission_tar: bytes, decl: str, claim: Claim ) -> CheckOutcome: - sb = sandbox(self._sandbox_name) + sb = sandbox(comparator_sandbox_name(current_member())) # Host-side: re-serialize the agent's tar as a trusted archive of its # Lean modules (a verdict if that is impossible; the sandbox is never @@ -282,12 +287,6 @@ async def check( except InvalidSubmission as exc: return CheckOutcome(ok=False, stage="entry_missing", detail=str(exc)) - async with _CHECK_LOCKS.setdefault(sb, anyio.Lock()): - return await self._run(sb, spec, staged, decl, claim) - - async def _run( - self, sb: SandboxEnvironment, spec: str, staged: bytes, decl: str, claim: Claim - ) -> CheckOutcome: # Trusted filesystem reset (a reference step: failure raises). It # restores a pristine .lake -- the only path the untrusted build can # write under its landrun sandbox; it does not terminate processes a diff --git a/apn/scorer.py b/apn/scorer.py index 644c60c..002a55c 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -18,11 +18,10 @@ SandboxEnvironment, sandbox, store, - tool_state_instance, tool_state_scope, ) -from apn.checker import Claim, ProofChecker +from apn.checker import Claim, ProofChecker, current_member from apn.filetree import build_tree_from_tar, read_submission_tar logger = logging.getLogger(__name__) @@ -34,8 +33,8 @@ def claim_store_key() -> str: - instance = tool_state_instance(None) - return CLAIM_STORE_KEY if instance is None else f"{CLAIM_STORE_KEY}:{instance}" + member = current_member() + return CLAIM_STORE_KEY if member is None else f"{CLAIM_STORE_KEY}:{member}" @scorer(metrics=[accuracy(), stderr()]) @@ -49,11 +48,9 @@ def proof_scorer(checker: ProofChecker, swarm: bool = False) -> Scorer: if any member's submission passes.""" async def score(state: TaskState, target: Target) -> Score: - if swarm and tool_state_instance(None) is None: + if swarm and current_member() is None: return await _score_swarm(state, checker) - return await _score_sandbox( - state, checker, sandbox(), claim_store_key(), "submission_contents" - ) + return await _score_sandbox(state, checker, sandbox(), "submission_contents") return score @@ -63,14 +60,12 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: scores: dict[str, Score] = {} for m in record["members"]: with tool_state_scope(m["name"]): - claim_key = claim_store_key() - scores[m["name"]] = await _score_sandbox( - state, - checker, - sandbox(m["sandbox"]), - claim_key, - f"submission_contents_{m['name']}", - ) + scores[m["name"]] = await _score_sandbox( + state, + checker, + sandbox(m["sandbox"]), + f"submission_contents_{m['name']}", + ) passed = [name for name, s in scores.items() if s.value == CORRECT] return Score( value=CORRECT if passed else INCORRECT, @@ -86,7 +81,6 @@ async def _score_sandbox( state: TaskState, checker: ProofChecker, sb: SandboxEnvironment, - claim_key: str, tree_key: str, ) -> Score: # Per-attempt attempt index, kept in the sample store (the react/deepagent @@ -110,7 +104,7 @@ async def _score_sandbox( # The claim the agent declared on its submit call. A sample scored # without one (e.g. it hit its limits before ever submitting) defaults # to "proof" -- deterministic, and such submissions reject anyway. - claim: Claim = store().get(claim_key, "proof") + claim: Claim = store().get(claim_store_key(), "proof") # The target theorem's fully qualified name (== the sample id except # where the manifest overrides it; see apn.dataset.build_dataset). decl = state.metadata["decl_name"] diff --git a/apn/solver.py b/apn/solver.py index 1357295..82d9203 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -34,7 +34,7 @@ swarm_tools, ) -from apn.checker import Claim +from apn.checker import Claim, comparator_sandbox_name from apn.layout import ENTRY_PATH from apn.limits import continue_unless_looping from apn.prompts import user_prompt @@ -159,9 +159,6 @@ def _prover_submit() -> AgentSubmit: CONTINUE_MESSAGE = "Continue working on the problem." -COMPARATOR_SANDBOX = "comparator" - - def swarm_member_names(swarm_size: int) -> list[str]: return [f"prover-{i}" for i in range(1, swarm_size + 1)] @@ -237,8 +234,8 @@ def lean_prover( (``apn.dataset.fc_profile(...).util_module``), named in the prompt's import-integrity rule. swarm_size: Number of members when ``agent_type`` is ``"swarm"``. Each - member works in its own sandbox (see ``swarm_member_names``) and - shares the comparator; the swarm's budget is split equally. + member works in its own sandbox (see ``swarm_member_names``) with + its own comparator; the swarm's budget is split equally. """ async def solve(state: TaskState, generate: Generate) -> TaskState: @@ -256,7 +253,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: name=name, args={"gated": gated}, sandbox=name, - visible_sandboxes=[COMPARATOR_SANDBOX], + visible_sandboxes=[comparator_sandbox_name(name)], ) for name in names ], diff --git a/apn/task.py b/apn/task.py index ef8e9be..dd57da9 100644 --- a/apn/task.py +++ b/apn/task.py @@ -11,7 +11,7 @@ from apn import __version__ from apn.solver import AgentType, lean_prover, swarm_member_names -from apn.checker import SandboxComparator +from apn.checker import SandboxComparator, comparator_sandbox_name from apn.dataset import ( ERDOS_DIR, FC100_DIR, @@ -109,14 +109,23 @@ def _build_section(target: str, fc_commit: str) -> dict[str, Any]: } +def _sandbox_pairs(members: Sequence[str] | None) -> list[tuple[str, str]]: + """(agent service, comparator service) names: ``default`` and the single + comparator, or one pair per swarm member.""" + if members is None: + return [("default", comparator_sandbox_name(None))] + return [(member, comparator_sandbox_name(member)) for member in members] + + def get_compose_file_content( fc_commit: str, literature: bool = False, - agent_services: Sequence[str] = ("default",), + members: Sequence[str] | None = None, ) -> str: """The docker-backend sandbox config (local runs, CI tests). - Two services: the agent's workspace, and the trusted `comparator` verifier. + Two services per agent (see :func:`_sandbox_pairs`): the agent's workspace, + and the trusted `comparator` verifier. The comparator container needs no filesystem hardening of its own (comparator-migration-plan.md §3.1): comparator's landrun (Landlock) sandbox confines the untrusted solution build to writes in `.lake`, every @@ -125,30 +134,25 @@ def get_compose_file_content( reset-dotlake.sh restores a pristine `.lake`. """ agent_kind = _agent_image_kind(literature) - compose: dict[str, Any] = { - "services": { - **{ - name: { - "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image(agent_kind, fc_commit)}", - "build": _build_section(agent_kind, fc_commit), - "init": True, - "entrypoint": list(SANDBOX_COMMAND), - "mem_limit": f"{AGENT_MEMORY_GIB}g", - "network_mode": "none", - } - for name in agent_services - }, - "comparator": { - "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image('comparator', fc_commit)}", - "build": _build_section("comparator", fc_commit), - "init": True, - "entrypoint": list(SANDBOX_COMMAND), - "mem_limit": f"{COMPARATOR_MEMORY_GIB}g", - "network_mode": "none", - }, + services: dict[str, Any] = {} + for agent, comparator in _sandbox_pairs(members): + services[agent] = { + "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image(agent_kind, fc_commit)}", + "build": _build_section(agent_kind, fc_commit), + "init": True, + "entrypoint": list(SANDBOX_COMMAND), + "mem_limit": f"{AGENT_MEMORY_GIB}g", + "network_mode": "none", } - } - return yaml.safe_dump(compose, sort_keys=False) + services[comparator] = { + "image": f"{IMAGE_REPOSITORY}:{get_identifier_for_image('comparator', fc_commit)}", + "build": _build_section("comparator", fc_commit), + "init": True, + "entrypoint": list(SANDBOX_COMMAND), + "mem_limit": f"{COMPARATOR_MEMORY_GIB}g", + "network_mode": "none", + } + return yaml.safe_dump({"services": services}, sort_keys=False) AGENT_EPHEMERAL_STORAGE_GIB = 20 @@ -158,7 +162,7 @@ def get_compose_file_content( def get_values_file_content( fc_commit: str, literature: bool = False, - agent_services: Sequence[str] = ("default",), + members: Sequence[str] | None = None, ) -> str: """The k8s/Hawk-backend sandbox config: chart-native agent-env values. @@ -192,42 +196,35 @@ def resources(memory_gib: int, ephemeral_storage_gib: int) -> dict[str, Any]: "requests": {"ephemeral-storage": f"{ephemeral_storage_gib}Gi"}, } - values: dict[str, Any] = { - "services": { - **{ - name: { - "image": f"{repository}:{get_identifier_for_image(agent_kind, fc_commit)}", - # Without this the chart defaults to a bare `tail -f /dev/null` - # as PID 1, which never reaps orphans (see SANDBOX_COMMAND). - "command": list(SANDBOX_COMMAND), - "networkIsolated": True, - "dnsRecord": True, - "resources": resources( - AGENT_MEMORY_GIB, AGENT_EPHEMERAL_STORAGE_GIB - ), - } - for name in agent_services - }, - "comparator": { - "image": f"{repository}:{get_identifier_for_image('comparator', fc_commit)}", - "command": list(SANDBOX_COMMAND), - "runtimeClassName": "CLUSTER_DEFAULT", - "networkIsolated": True, - "dnsRecord": True, - "resources": resources( - COMPARATOR_MEMORY_GIB, COMPARATOR_EPHEMERAL_STORAGE_GIB - ), - }, + services: dict[str, Any] = {} + for agent, comparator in _sandbox_pairs(members): + services[agent] = { + "image": f"{repository}:{get_identifier_for_image(agent_kind, fc_commit)}", + # Without this the chart defaults to a bare `tail -f /dev/null` + # as PID 1, which never reaps orphans (see SANDBOX_COMMAND). + "command": list(SANDBOX_COMMAND), + "networkIsolated": True, + "dnsRecord": True, + "resources": resources(AGENT_MEMORY_GIB, AGENT_EPHEMERAL_STORAGE_GIB), } - } - return yaml.safe_dump(values, sort_keys=False) + services[comparator] = { + "image": f"{repository}:{get_identifier_for_image('comparator', fc_commit)}", + "command": list(SANDBOX_COMMAND), + "runtimeClassName": "CLUSTER_DEFAULT", + "networkIsolated": True, + "dnsRecord": True, + "resources": resources( + COMPARATOR_MEMORY_GIB, COMPARATOR_EPHEMERAL_STORAGE_GIB + ), + } + return yaml.safe_dump({"services": services}, sort_keys=False) def get_sandbox_config( fc_commit: str, literature: bool, backend: SandboxBackend | None, - agent_services: Sequence[str] = ("default",), + members: Sequence[str] | None = None, ) -> tuple[SandboxBackend, str]: """The Inspect ``sandbox`` spec ``(type, config-file path)`` for a backend (``None``: see :func:`resolve_sandbox_backend`). @@ -235,9 +232,9 @@ def get_sandbox_config( Each backend gets its own generated artifact (``compose.yaml`` for docker, ``values.yaml`` for k8s -- k8s_sandbox treats any config file NOT named ``*compose.yaml``/``*compose.yml`` as chart values). Files are isolated in - per-(version, FC pin, variant, agent services) subdirs so they don't - clobber each other. ``agent_services`` names one agent sandbox each (a - swarm's members); all share the one comparator. + per-(version, FC pin, variant, swarm members) subdirs so they don't + clobber each other. ``members``, a swarm's member names, gives each member + its own agent and comparator sandboxes. """ backend = resolve_sandbox_backend(backend) variant = "corpus" if literature else "closed-book" @@ -246,15 +243,15 @@ def get_sandbox_config( / _docker_tag_component(__version__) / f"fc_{fc_commit[:12]}" / variant - / "+".join(agent_services) + / ("single-agent" if members is None else "+".join(members)) ) directory.mkdir(parents=True, exist_ok=True) if backend == "docker": path = directory / "compose.yaml" - content = get_compose_file_content(fc_commit, literature, agent_services) + content = get_compose_file_content(fc_commit, literature, members) elif backend == "k8s": path = directory / "values.yaml" - content = get_values_file_content(fc_commit, literature, agent_services) + content = get_values_file_content(fc_commit, literature, members) else: raise ValueError( f"Unknown sandbox_backend {backend!r}; expected 'docker' or 'k8s'." @@ -270,10 +267,8 @@ def get_compose_file(fc_commit: str, literature: bool = False) -> Path: return Path(get_sandbox_config(fc_commit, literature, "docker")[1]) -def _agent_services(agent_type: AgentType, swarm_size: int) -> list[str]: - if agent_type == "swarm": - return swarm_member_names(swarm_size) - return ["default"] +def _swarm_members(agent_type: AgentType, swarm_size: int) -> list[str] | None: + return swarm_member_names(swarm_size) if agent_type == "swarm" else None @task @@ -305,7 +300,7 @@ def apn_oeis( ), scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) ), ) @@ -332,7 +327,7 @@ def apn_fc100open( ), scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) ), ) @@ -360,7 +355,7 @@ def apn_erdos( ), scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) ), ) @@ -391,6 +386,6 @@ def apn_personal_corresp( ), scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _agent_services(agent_type, swarm_size) + pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) ), ) diff --git a/tests/test_checker.py b/tests/test_checker.py index dec52dd..9d5e7c6 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -28,7 +28,6 @@ from types import SimpleNamespace from typing import cast -import anyio import pytest from inspect_ai.model import ModelName from inspect_ai.scorer import CORRECT, INCORRECT, Score, Target @@ -717,35 +716,21 @@ async def test_scorer_writes_attempt_sidecar( assert sidecars[0].read_bytes() == tar -async def test_concurrent_checks_on_one_comparator_are_serialized( +async def test_check_uses_the_member_s_own_comparator( monkeypatch: pytest.MonkeyPatch, ) -> None: - class SlowSandbox(ScriptedSandbox): - async def exec(self, cmd: list[str], **kwargs: object) -> ExecResult[str]: - await anyio.sleep(0.01) - return await super().exec(cmd, **kwargs) + requested: list[str | None] = [] - sb = SlowSandbox(comparator=_ok(_ACCEPT_OUT)) - monkeypatch.setattr(checker_mod, "sandbox", lambda *a, **k: sb) - checker = SandboxComparator() + def fake_sandbox(name: str | None = None) -> ScriptedSandbox: + requested.append(name) + return ScriptedSandbox(comparator=_ok(_ACCEPT_OUT)) - async def check() -> None: + monkeypatch.setattr(checker_mod, "sandbox", fake_sandbox) + checker = SandboxComparator() + await checker.check(SPEC, SUBMISSION_TAR, decl="tgt", claim="proof") + with tool_state_scope("prover-1"): await checker.check(SPEC, SUBMISSION_TAR, decl="tgt", claim="proof") - - async with anyio.create_task_group() as tg: - tg.start_soon(check) - tg.start_soon(check) - - one_check = [ - ("exec", [RESET_SCRIPT]), - ("exec", STAGING_RESET_CMD), - ("write", CHALLENGE_PATH), - ("write", SUBMISSION_TAR_PATH), - ("write", CONFIG_PATH), - ("exec", UNPACK_CMD), - ("exec", COMPARATOR_CMD), - ] - assert sb.events == one_check + one_check + assert requested == ["comparator", "comparator-prover-1"] def test_claim_store_key_is_per_tool_state_scope() -> None: From c933fe293cc90a9c53d046e042f240c679cd509a Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:01:23 +0000 Subject: [PATCH 06/33] Pin inspect-boltons with the swarm-aware resources tool Swarm members' resources tool now shows their own share of the swarm's cost and token caps instead of the sample's limits. --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0fa3584..2b8bc17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ license = "MIT AND Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.13,<3.14" dependencies = [ - "inspect-boltons @ git+https://github.com/tadamcz/inspect_boltons@35b794e845f1748947a4f18efdfcb31dd0cf746a", + "inspect-boltons @ git+https://github.com/tadamcz/inspect_boltons@0d1af2c6a19395212b72914e150d23a0513f9d3d", "inspect-swarm>=0.1.0,<0.2.0", ] diff --git a/uv.lock b/uv.lock index e0443c3..4832d42 100644 --- a/uv.lock +++ b/uv.lock @@ -178,7 +178,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a" }, + { name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=0d1af2c6a19395212b72914e150d23a0513f9d3d" }, { name = "inspect-swarm", specifier = ">=0.1.0,<0.2.0" }, ] @@ -661,7 +661,7 @@ wheels = [ [[package]] name = "inspect-boltons" version = "0.1.0" -source = { git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a#35b794e845f1748947a4f18efdfcb31dd0cf746a" } +source = { git = "https://github.com/tadamcz/inspect_boltons?rev=0d1af2c6a19395212b72914e150d23a0513f9d3d#0d1af2c6a19395212b72914e150d23a0513f9d3d" } dependencies = [ { name = "inspect-ai" }, { name = "tabulate" }, From 386a5c31809e18bd181208eb7982e4b02ceebe88 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:09:19 +0000 Subject: [PATCH 07/33] Revert "Pin inspect-boltons with the swarm-aware resources tool" This reverts commit c933fe293cc90a9c53d046e042f240c679cd509a. --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2b8bc17..0fa3584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ license = "MIT AND Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.13,<3.14" dependencies = [ - "inspect-boltons @ git+https://github.com/tadamcz/inspect_boltons@0d1af2c6a19395212b72914e150d23a0513f9d3d", + "inspect-boltons @ git+https://github.com/tadamcz/inspect_boltons@35b794e845f1748947a4f18efdfcb31dd0cf746a", "inspect-swarm>=0.1.0,<0.2.0", ] diff --git a/uv.lock b/uv.lock index 4832d42..e0443c3 100644 --- a/uv.lock +++ b/uv.lock @@ -178,7 +178,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=0d1af2c6a19395212b72914e150d23a0513f9d3d" }, + { name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a" }, { name = "inspect-swarm", specifier = ">=0.1.0,<0.2.0" }, ] @@ -661,7 +661,7 @@ wheels = [ [[package]] name = "inspect-boltons" version = "0.1.0" -source = { git = "https://github.com/tadamcz/inspect_boltons?rev=0d1af2c6a19395212b72914e150d23a0513f9d3d#0d1af2c6a19395212b72914e150d23a0513f9d3d" } +source = { git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a#35b794e845f1748947a4f18efdfcb31dd0cf746a" } dependencies = [ { name = "inspect-ai" }, { name = "tabulate" }, From 7104fd3ff0658ffa1f481f57dc68696463328c15 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:09:51 +0000 Subject: [PATCH 08/33] Give swarm members swarm_resources as their resources tool The boltons resources tool reads sample_limits(), which shows the sample's cost limit rather than the member's share of the swarm's cap. Swarm members get inspect_swarm's swarm_resources() instead, under the name the prompt uses (resources); single agents keep the boltons tool. --- apn/solver.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 82d9203..b7248b0 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -31,6 +31,7 @@ shared_memory, swarm, swarm_on_continue, + swarm_resources, swarm_tools, ) @@ -137,11 +138,11 @@ def _attempts(gated: bool) -> AgentAttempts: ) -def _prover_tools() -> list[Tool]: +def _prover_tools(budget: Tool) -> list[Tool]: return [ text_editor(), bash(timeout=300), - resources(), + budget, ] @@ -165,9 +166,13 @@ def swarm_member_names(swarm_size: int) -> list[str]: @agent def prover_member(gated: bool) -> Agent: - """A swarm member: the react prover loop plus the swarm's channel tools.""" + """A swarm member: the react prover loop plus the swarm's channel tools, + with the swarm's budget (the member's own share) as its ``resources`` tool.""" return react( - tools=[*_prover_tools(), swarm_tools()], + tools=[ + *_prover_tools(ToolDef(swarm_resources(), name="resources").as_tool()), + swarm_tools(), + ], attempts=_attempts(gated), submit=_prover_submit(), on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), @@ -264,7 +269,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: await sandbox().write_file(ENTRY_PATH, sketch) agent = build_agent( agent_type, - tools=_prover_tools(), + tools=_prover_tools(resources()), attempts=_attempts(gated), submit=_prover_submit(), on_continue=continue_unless_looping(CONTINUE_MESSAGE), From 5c9c2c6e0524519c5314384a1379d2a63f1299d2 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:34:19 +0000 Subject: [PATCH 09/33] Switch swarms on with swarm_size, not agent_type: swarm agent_type is back to the loop alone (react or deep), and a swarm's members use it too. A positive swarm_size runs a swarm of that many members; None, zero or negative runs a single agent. --- apn/solver.py | 58 ++++++++++++++++++++----------------- apn/task.py | 36 +++++++++++++---------- configs/erdos74-swarm4.yaml | 2 +- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index b7248b0..52d4c09 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -122,7 +122,7 @@ async def execute(claim: Claim) -> ToolResult: return execute -AgentType = Literal["deep", "react", "swarm"] +AgentType = Literal["deep", "react"] def _attempts(gated: bool) -> AgentAttempts: @@ -160,26 +160,14 @@ def _prover_submit() -> AgentSubmit: CONTINUE_MESSAGE = "Continue working on the problem." -def swarm_member_names(swarm_size: int) -> list[str]: +def swarm_member_names(swarm_size: int | None) -> list[str] | None: + """The swarm's member names, or ``None`` for no swarm (``swarm_size`` unset, + zero or negative).""" + if swarm_size is None or swarm_size <= 0: + return None return [f"prover-{i}" for i in range(1, swarm_size + 1)] -@agent -def prover_member(gated: bool) -> Agent: - """A swarm member: the react prover loop plus the swarm's channel tools, - with the swarm's budget (the member's own share) as its ``resources`` tool.""" - return react( - tools=[ - *_prover_tools(ToolDef(swarm_resources(), name="resources").as_tool()), - swarm_tools(), - ], - attempts=_attempts(gated), - submit=_prover_submit(), - on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), - compaction=CompactionSummary(threshold=300_000), - ) - - def build_agent( agent_type: AgentType, *, @@ -222,33 +210,51 @@ def build_agent( ) +@agent +def prover_member(agent_type: AgentType, gated: bool) -> Agent: + """A swarm member: the prover loop plus the swarm's channel tools, with the + swarm's budget (the member's own share) as its ``resources`` tool.""" + return build_agent( + agent_type, + tools=[ + *_prover_tools(ToolDef(swarm_resources(), name="resources").as_tool()), + swarm_tools(), + ], + attempts=_attempts(gated), + submit=_prover_submit(), + on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), + compaction=CompactionSummary(threshold=300_000), + ) + + @solver def lean_prover( agent_type: AgentType, gated: bool, literature: bool, util_module: str, - swarm_size: int = 4, + swarm_size: int | None = None, ) -> Solver: """ Args: gated: Gated submission (retry until correct or token/time limit). literature: Run with the offline arXiv corpus. - agent_type: Which agent loop to run. + agent_type: Which agent loop to run (each member's, in a swarm). util_module: The dataset pin's FC util module (``apn.dataset.fc_profile(...).util_module``), named in the prompt's import-integrity rule. - swarm_size: Number of members when ``agent_type`` is ``"swarm"``. Each - member works in its own sandbox (see ``swarm_member_names``) with - its own comparator; the swarm's budget is split equally. + swarm_size: Run a swarm of this many members; ``None``, zero or + negative runs a single agent. Each member works in its own sandbox + (see ``swarm_member_names``) with its own comparator; the swarm's + budget is split equally. """ async def solve(state: TaskState, generate: Generate) -> TaskState: _warn_if_ignored_formalizations(state) sketch = state.metadata["sketch"] - if agent_type == "swarm": - names = swarm_member_names(swarm_size) + names = swarm_member_names(swarm_size) + if names is not None: for name in names: await sandbox(name).write_file(ENTRY_PATH, sketch) agent = swarm( @@ -256,7 +262,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: member( prover_member, name=name, - args={"gated": gated}, + args={"agent_type": agent_type, "gated": gated}, sandbox=name, visible_sandboxes=[comparator_sandbox_name(name)], ) diff --git a/apn/task.py b/apn/task.py index dd57da9..3de6b9b 100644 --- a/apn/task.py +++ b/apn/task.py @@ -267,17 +267,13 @@ def get_compose_file(fc_commit: str, literature: bool = False) -> Path: return Path(get_sandbox_config(fc_commit, literature, "docker")[1]) -def _swarm_members(agent_type: AgentType, swarm_size: int) -> list[str] | None: - return swarm_member_names(swarm_size) if agent_type == "swarm" else None - - @task def apn_oeis( subset: str | None = None, gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 4, + swarm_size: int | None = None, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Formal Conjectures autoformalized OEIS conjectures (492 samples). @@ -298,9 +294,11 @@ def apn_oeis( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + scorer=proof_scorer( + SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None + ), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) + pin, literature, sandbox_backend, swarm_member_names(swarm_size) ), ) @@ -311,7 +309,7 @@ def apn_fc100open( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 4, + swarm_size: int | None = None, sandbox_backend: SandboxBackend | None = None, ) -> Task: name_list = load_subset(FC100_DIR, subset) if subset is not None else None @@ -325,9 +323,11 @@ def apn_fc100open( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + scorer=proof_scorer( + SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None + ), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) + pin, literature, sandbox_backend, swarm_member_names(swarm_size) ), ) @@ -338,7 +338,7 @@ def apn_erdos( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 4, + swarm_size: int | None = None, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Bloom statement selection of Erdős problems.""" @@ -353,9 +353,11 @@ def apn_erdos( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + scorer=proof_scorer( + SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None + ), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) + pin, literature, sandbox_backend, swarm_member_names(swarm_size) ), ) @@ -366,7 +368,7 @@ def apn_personal_corresp( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - swarm_size: int = 4, + swarm_size: int | None = None, sandbox_backend: SandboxBackend | None = None, ) -> Task: """Open conjectures sent to us in personal correspondence, formalized by @@ -384,8 +386,10 @@ def apn_personal_corresp( swarm_size=swarm_size, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer(SandboxComparator(), swarm=agent_type == "swarm"), + scorer=proof_scorer( + SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None + ), sandbox=get_sandbox_config( - pin, literature, sandbox_backend, _swarm_members(agent_type, swarm_size) + pin, literature, sandbox_backend, swarm_member_names(swarm_size) ), ) diff --git a/configs/erdos74-swarm4.yaml b/configs/erdos74-swarm4.yaml index 9f6fba0..37ed726 100644 --- a/configs/erdos74-swarm4.yaml +++ b/configs/erdos74-swarm4.yaml @@ -8,7 +8,7 @@ tasks: - name: apn_erdos args: subset: bloom_selection - agent_type: swarm + agent_type: react swarm_size: 4 literature: true sample_ids: From 8574dadaac7db7f4e510a123a4a55df3fda27be3 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:36:58 +0000 Subject: [PATCH 10/33] Pass swarm_resources to swarm members under its own name --- apn/solver.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 52d4c09..598f2d1 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -212,14 +212,11 @@ def build_agent( @agent def prover_member(agent_type: AgentType, gated: bool) -> Agent: - """A swarm member: the prover loop plus the swarm's channel tools, with the - swarm's budget (the member's own share) as its ``resources`` tool.""" + """A swarm member: the prover loop plus the swarm's channel tools, with + ``swarm_resources`` (the member's own share of the budget) as its budget tool.""" return build_agent( agent_type, - tools=[ - *_prover_tools(ToolDef(swarm_resources(), name="resources").as_tool()), - swarm_tools(), - ], + tools=[*_prover_tools(swarm_resources()), swarm_tools()], attempts=_attempts(gated), submit=_prover_submit(), on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), From 9336b237b24465c2b32268d2471d65a2280211d1 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:45:52 +0000 Subject: [PATCH 11/33] Build single agents and swarm members with one prover builder swarm_tools() gives no tools and swarm_on_continue() defers to its inner hook outside a swarm, so one builder serves both; only the budget tool differs (resources, or swarm_resources for a member's own share). --- apn/solver.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 598f2d1..6d87fcd 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -138,14 +138,6 @@ def _attempts(gated: bool) -> AgentAttempts: ) -def _prover_tools(budget: Tool) -> list[Tool]: - return [ - text_editor(), - bash(timeout=300), - budget, - ] - - def _prover_submit() -> AgentSubmit: # Name it distinctly from any subagents' "submit" tool and keep the # call in the message history. keep_in_messages=True stops the loop @@ -211,12 +203,20 @@ def build_agent( @agent -def prover_member(agent_type: AgentType, gated: bool) -> Agent: - """A swarm member: the prover loop plus the swarm's channel tools, with - ``swarm_resources`` (the member's own share of the budget) as its budget tool.""" +def prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: + """The prover loop, for a single agent or a swarm member. + + ``swarm_tools()`` and ``swarm_on_continue()`` do nothing outside a swarm. A + member's budget tool is ``swarm_resources`` (its own share of the budget), + since ``resources`` shows the sample's limits.""" return build_agent( agent_type, - tools=[*_prover_tools(swarm_resources()), swarm_tools()], + tools=[ + text_editor(), + bash(timeout=300), + swarm_resources() if swarm else resources(), + swarm_tools(), + ], attempts=_attempts(gated), submit=_prover_submit(), on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), @@ -257,9 +257,9 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: agent = swarm( members=[ member( - prover_member, + prover, name=name, - args={"agent_type": agent_type, "gated": gated}, + args={"agent_type": agent_type, "gated": gated, "swarm": True}, sandbox=name, visible_sandboxes=[comparator_sandbox_name(name)], ) @@ -270,14 +270,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: ) else: await sandbox().write_file(ENTRY_PATH, sketch) - agent = build_agent( - agent_type, - tools=_prover_tools(resources()), - attempts=_attempts(gated), - submit=_prover_submit(), - on_continue=continue_unless_looping(CONTINUE_MESSAGE), - compaction=CompactionSummary(threshold=300_000), - ) + agent = prover(agent_type, gated) state.messages = [ ChatMessageUser( content=user_prompt(ENTRY_PATH, state.token_limit, literature, util_module), From ab2dded433c921bc6d94b0d7e86e7b3dfcf5e78e Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:47:01 +0000 Subject: [PATCH 12/33] rename to clarify private --- apn/solver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 6d87fcd..4bca2e3 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -203,7 +203,7 @@ def build_agent( @agent -def prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: +def _prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: """The prover loop, for a single agent or a swarm member. ``swarm_tools()`` and ``swarm_on_continue()`` do nothing outside a swarm. A @@ -257,7 +257,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: agent = swarm( members=[ member( - prover, + _prover, name=name, args={"agent_type": agent_type, "gated": gated, "swarm": True}, sandbox=name, @@ -270,7 +270,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: ) else: await sandbox().write_file(ENTRY_PATH, sketch) - agent = prover(agent_type, gated) + agent = _prover(agent_type, gated) state.messages = [ ChatMessageUser( content=user_prompt(ENTRY_PATH, state.token_limit, literature, util_module), From 1abcde0ba241305b220461492635461d5a957d76 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:48:18 +0000 Subject: [PATCH 13/33] Add TODO to make the boltons resources tool swarm-aware --- apn/solver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apn/solver.py b/apn/solver.py index 4bca2e3..feaadd7 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -214,6 +214,9 @@ def _prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: tools=[ text_editor(), bash(timeout=300), + # TODO: make inspect_boltons' resources() report a swarm member's own + # budget via swarm_budget() (started on tadamcz/inspect_boltons branch + # swarm-aware-resources), then use it for everyone and drop `swarm`. swarm_resources() if swarm else resources(), swarm_tools(), ], From e34f2e6ea17212f33b38d9a87a76794216e7ed0c Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:49:32 +0000 Subject: [PATCH 14/33] word --- apn/solver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index feaadd7..81349aa 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -215,8 +215,8 @@ def _prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: text_editor(), bash(timeout=300), # TODO: make inspect_boltons' resources() report a swarm member's own - # budget via swarm_budget() (started on tadamcz/inspect_boltons branch - # swarm-aware-resources), then use it for everyone and drop `swarm`. + # budget via swarm_budget() (started on tadamcz/inspect_boltons branch + # swarm-aware-resources), then use it for everyone and drop this helper function. swarm_resources() if swarm else resources(), swarm_tools(), ], From c95e0f99862db0009702e61ebff3e4ecffe3bdeb Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:54:28 +0000 Subject: [PATCH 15/33] Name swarm sandbox config dirs by swarm size Joining every member's name made a path component too long for the filesystem at 32 members, so the task failed to load. --- apn/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apn/task.py b/apn/task.py index 3de6b9b..2b21168 100644 --- a/apn/task.py +++ b/apn/task.py @@ -232,7 +232,7 @@ def get_sandbox_config( Each backend gets its own generated artifact (``compose.yaml`` for docker, ``values.yaml`` for k8s -- k8s_sandbox treats any config file NOT named ``*compose.yaml``/``*compose.yml`` as chart values). Files are isolated in - per-(version, FC pin, variant, swarm members) subdirs so they don't + per-(version, FC pin, variant, swarm size) subdirs so they don't clobber each other. ``members``, a swarm's member names, gives each member its own agent and comparator sandboxes. """ @@ -243,7 +243,7 @@ def get_sandbox_config( / _docker_tag_component(__version__) / f"fc_{fc_commit[:12]}" / variant - / ("single-agent" if members is None else "+".join(members)) + / ("single-agent" if members is None else f"swarm-{len(members)}") ) directory.mkdir(parents=True, exist_ok=True) if backend == "docker": From 8c1b05b06ad2f334aec07bb5527985f24ed0c175 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:56:49 +0000 Subject: [PATCH 16/33] Use time_limit, not working_limit, in the swarm example config Working time stalls when many concurrent members wait, so it suits a swarm poorly; the swarm also turns a time limit into a soft-stopping cap. --- configs/erdos74-swarm4.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/configs/erdos74-swarm4.yaml b/configs/erdos74-swarm4.yaml index 37ed726..4ed8d08 100644 --- a/configs/erdos74-swarm4.yaml +++ b/configs/erdos74-swarm4.yaml @@ -25,7 +25,15 @@ models: max_connections: 10 cost_limit: 400.0 -working_limit: 432_000 # 5 * 24 * 60 * 60 +# Swarms use time_limit, not working_limit. Working time is wall-clock time +# minus the periods when any task waits (for a connection slot, a retry or a +# rate limit), with overlapping waits counted once, so with many concurrent +# members it can stall far behind the wall clock: working_limit is poorly +# suited to a swarm. The swarm also turns a time limit into a cap that stops +# members gently at 95% of it. A time limit does charge members for that +# waiting, so avoid contention when running a swarm: give max_connections room +# for every member running at once across samples, and mind rate limits. +time_limit: 432_000 # 5 * 24 * 60 * 60 # $ per 1M tokens; keys must match Inspect's resolved / names. model_cost_config: From b2850d661e90716ebe9fdbb24d2df7f0041e37a9 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 14:59:35 +0000 Subject: [PATCH 17/33] Use deepagent members in the swarm example config --- configs/erdos74-swarm4.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/erdos74-swarm4.yaml b/configs/erdos74-swarm4.yaml index 4ed8d08..18a8fbe 100644 --- a/configs/erdos74-swarm4.yaml +++ b/configs/erdos74-swarm4.yaml @@ -8,7 +8,7 @@ tasks: - name: apn_erdos args: subset: bloom_selection - agent_type: react + agent_type: deep swarm_size: 4 literature: true sample_ids: From 5323e532898138ae0209278cb57ef147a356846a Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 18:29:45 +0000 Subject: [PATCH 18/33] Explain a swarm's score member by member The whole-sample score now states the swarm's size and which members' work passes, then one line per member: verdict, comparator stage, claim, and whether the claim was the default and whether the member submitted. The scorer also stops storing the default "proof" claim: Store.get with a default writes it, so members that never submitted looked as if they had claimed "proof". --- apn/scorer.py | 46 ++++++++++++++++++++++++++++++++++++++----- tests/test_checker.py | 30 +++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/apn/scorer.py b/apn/scorer.py index 002a55c..c267f60 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -58,18 +58,29 @@ async def score(state: TaskState, target: Target) -> Score: async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: record = state.store.get(SWARM_RESULT_KEY) scores: dict[str, Score] = {} + stages: dict[str, str] = {} + claims: dict[str, str] = {} for m in record["members"]: - with tool_state_scope(m["name"]): - scores[m["name"]] = await _score_sandbox( + name = m["name"] + with tool_state_scope(name): + recorded = store().get(claim_store_key()) + scores[name] = await _score_sandbox( state, checker, sandbox(m["sandbox"]), - f"submission_contents_{m['name']}", + f"submission_contents_{name}", ) + metadata = scores[name].metadata + assert metadata is not None + stages[name] = metadata["stage"] + notes = ["submitted" if m["submitted"] else "not submitted"] + if recorded is None: + notes.insert(0, "default") + claims[name] = f"claim {recorded or 'proof'} ({', '.join(notes)})" passed = [name for name, s in scores.items() if s.value == CORRECT] return Score( value=CORRECT if passed else INCORRECT, - explanation=f"passed: {', '.join(passed)}" if passed else "no member passed", + explanation=_swarm_explanation(scores, stages, claims, passed), metadata={ "passed": passed, "members": {name: s.metadata for name, s in scores.items()}, @@ -77,6 +88,30 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: ) +def _swarm_explanation( + scores: dict[str, Score], + stages: dict[str, str], + claims: dict[str, str], + passed: list[str], +) -> str: + if not passed: + summary = f"Swarm of {len(scores)}: no member's work passes." + elif len(passed) == 1: + summary = f"Swarm of {len(scores)}: 1 member's work passes ({passed[0]})." + else: + summary = ( + f"Swarm of {len(scores)}: {len(passed)} members' work passes " + f"({', '.join(passed)})." + ) + width = max(len(stage) for stage in stages.values()) + lines = [ + f"{name}: {'CORRECT' if s.value == CORRECT else 'INCORRECT':<9} " + f"{stages[name]:<{width}} {claims[name]}" + for name, s in scores.items() + ] + return "\n".join([summary, *lines]) + + async def _score_sandbox( state: TaskState, checker: ProofChecker, @@ -104,7 +139,8 @@ async def _score_sandbox( # The claim the agent declared on its submit call. A sample scored # without one (e.g. it hit its limits before ever submitting) defaults # to "proof" -- deterministic, and such submissions reject anyway. - claim: Claim = store().get(claim_store_key(), "proof") + recorded: Claim | None = store().get(claim_store_key()) + claim: Claim = "proof" if recorded is None else recorded # The target theorem's fully qualified name (== the sample id except # where the manifest overrides it; see apn.dataset.build_dataset). decl = state.metadata["decl_name"] diff --git a/tests/test_checker.py b/tests/test_checker.py index 9d5e7c6..893f58c 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -33,6 +33,7 @@ from inspect_ai.scorer import CORRECT, INCORRECT, Score, Target from inspect_ai.solver import TaskState from inspect_ai.util import ExecResult, OutputLimitExceededError, tool_state_scope +from inspect_ai.util._store import Store import apn.checker as checker_mod import apn.scorer as scorer_mod @@ -769,7 +770,12 @@ def _swarm_fixture( state = _state(store) state.store.set( SWARM_RESULT_KEY, - {"members": [{"name": name, "sandbox": name} for name in tars]}, + { + "members": [ + {"name": name, "sandbox": name, "submitted": name == "prover-2"} + for name in tars + ] + }, ) sandboxes = {name: FakeSandbox(tar) for name, tar in tars.items()} @@ -791,6 +797,12 @@ async def test_swarm_final_score_checks_every_member_sandbox( assert score.value == CORRECT assert score.metadata is not None assert score.metadata["passed"] == ["prover-2"] + assert score.explanation == ( + "Swarm of 3: 1 member's work passes (prover-2).\n" + "prover-1: INCORRECT stub claim proof (not submitted)\n" + "prover-2: CORRECT stub claim disproof (submitted)\n" + "prover-3: INCORRECT stub claim proof (default, not submitted)" + ) assert checker.calls == [ (tars["prover-1"], "proof"), (tars["prover-2"], "disproof"), @@ -808,6 +820,8 @@ async def test_swarm_final_score_incorrect_when_no_member_passes( score = await proof_scorer(TarChecker(passing=b""), swarm=True)(state, Target("")) assert score is not None assert score.value == INCORRECT + assert score.explanation is not None + assert score.explanation.splitlines()[0] == "Swarm of 3: no member's work passes." async def test_swarm_score_inside_a_member_checks_only_its_own_sandbox( @@ -820,3 +834,17 @@ async def test_swarm_score_inside_a_member_checks_only_its_own_sandbox( assert score is not None assert score.value == CORRECT assert checker.calls == [(tars["prover-2"], "disproof")] + + +async def test_scorer_does_not_record_a_default_claim( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = Store() + monkeypatch.setattr( + scorer_mod, "sandbox", lambda *a, **k: FakeSandbox(_tar_of({"./Spec.lean": "x"})) + ) + monkeypatch.setattr(scorer_mod, "store", lambda: store) + checker = StubChecker(False) + await proof_scorer(checker)(_state(FakeStore()), Target("")) + assert checker.calls[0][3] == "proof" + assert CLAIM_STORE_KEY not in store.keys() From a8d727705a9551ea292bb84ea0f57e2ee4d135f6 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 18:34:33 +0000 Subject: [PATCH 19/33] Say whether each swarm member called submit_proof in the score explanation --- apn/scorer.py | 9 +++++---- tests/test_checker.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apn/scorer.py b/apn/scorer.py index c267f60..5dd320a 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -73,10 +73,11 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: metadata = scores[name].metadata assert metadata is not None stages[name] = metadata["stage"] - notes = ["submitted" if m["submitted"] else "not submitted"] - if recorded is None: - notes.insert(0, "default") - claims[name] = f"claim {recorded or 'proof'} ({', '.join(notes)})" + claims[name] = ( + 'did not call submit_proof(), defaulted to claim="proof"' + if recorded is None + else f'called submit_proof(claim="{recorded}")' + ) passed = [name for name, s in scores.items() if s.value == CORRECT] return Score( value=CORRECT if passed else INCORRECT, diff --git a/tests/test_checker.py b/tests/test_checker.py index 893f58c..ee039ea 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -799,9 +799,9 @@ async def test_swarm_final_score_checks_every_member_sandbox( assert score.metadata["passed"] == ["prover-2"] assert score.explanation == ( "Swarm of 3: 1 member's work passes (prover-2).\n" - "prover-1: INCORRECT stub claim proof (not submitted)\n" - "prover-2: CORRECT stub claim disproof (submitted)\n" - "prover-3: INCORRECT stub claim proof (default, not submitted)" + 'prover-1: INCORRECT stub called submit_proof(claim="proof")\n' + 'prover-2: CORRECT stub called submit_proof(claim="disproof")\n' + 'prover-3: INCORRECT stub did not call submit_proof(), defaulted to claim="proof"' ) assert checker.calls == [ (tars["prover-1"], "proof"), From fb0ba05cef62cd4ed16878fc4db5136f5a0b3423 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 18:35:18 +0000 Subject: [PATCH 20/33] Drop the singular special case from the swarm score summary --- apn/scorer.py | 8 +++----- tests/test_checker.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/apn/scorer.py b/apn/scorer.py index 5dd320a..dcb4c24 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -95,15 +95,13 @@ def _swarm_explanation( claims: dict[str, str], passed: list[str], ) -> str: - if not passed: - summary = f"Swarm of {len(scores)}: no member's work passes." - elif len(passed) == 1: - summary = f"Swarm of {len(scores)}: 1 member's work passes ({passed[0]})." - else: + if passed: summary = ( f"Swarm of {len(scores)}: {len(passed)} members' work passes " f"({', '.join(passed)})." ) + else: + summary = f"Swarm of {len(scores)}: no member's work passes." width = max(len(stage) for stage in stages.values()) lines = [ f"{name}: {'CORRECT' if s.value == CORRECT else 'INCORRECT':<9} " diff --git a/tests/test_checker.py b/tests/test_checker.py index ee039ea..8316f2f 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -798,7 +798,7 @@ async def test_swarm_final_score_checks_every_member_sandbox( assert score.metadata is not None assert score.metadata["passed"] == ["prover-2"] assert score.explanation == ( - "Swarm of 3: 1 member's work passes (prover-2).\n" + "Swarm of 3: 1 members' work passes (prover-2).\n" 'prover-1: INCORRECT stub called submit_proof(claim="proof")\n' 'prover-2: CORRECT stub called submit_proof(claim="disproof")\n' 'prover-3: INCORRECT stub did not call submit_proof(), defaulted to claim="proof"' From d8afec0b84aeb41b64d1525c84747b240569e01a Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 18:37:56 +0000 Subject: [PATCH 21/33] Show the swarm score explanation as a markdown table One row per member: result, claim, stage, and the last line of the checker's output, which says why a member's work failed. --- apn/scorer.py | 47 ++++++++++++++++++++++--------------------- tests/test_checker.py | 13 ++++++++---- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/apn/scorer.py b/apn/scorer.py index dcb4c24..71ab6f9 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -58,8 +58,7 @@ async def score(state: TaskState, target: Target) -> Score: async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: record = state.store.get(SWARM_RESULT_KEY) scores: dict[str, Score] = {} - stages: dict[str, str] = {} - claims: dict[str, str] = {} + rows: list[list[str]] = [] for m in record["members"]: name = m["name"] with tool_state_scope(name): @@ -70,18 +69,23 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: sandbox(m["sandbox"]), f"submission_contents_{name}", ) - metadata = scores[name].metadata - assert metadata is not None - stages[name] = metadata["stage"] - claims[name] = ( - 'did not call submit_proof(), defaulted to claim="proof"' + score = scores[name] + assert score.metadata is not None and score.explanation is not None + claim = ( + 'did not call `submit_proof()`, defaulted to `claim="proof"`' if recorded is None - else f'called submit_proof(claim="{recorded}")' + else f'called `submit_proof(claim="{recorded}")`' ) + check = next( + (line for line in reversed(score.explanation.splitlines()) if line.strip()), + "", + ) + result = "CORRECT" if score.value == CORRECT else "INCORRECT" + rows.append([name, result, claim, score.metadata["stage"], check]) passed = [name for name, s in scores.items() if s.value == CORRECT] return Score( value=CORRECT if passed else INCORRECT, - explanation=_swarm_explanation(scores, stages, claims, passed), + explanation=_swarm_explanation(len(scores), passed, rows), metadata={ "passed": passed, "members": {name: s.metadata for name, s in scores.items()}, @@ -89,26 +93,23 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: ) -def _swarm_explanation( - scores: dict[str, Score], - stages: dict[str, str], - claims: dict[str, str], - passed: list[str], -) -> str: +def _swarm_explanation(size: int, passed: list[str], rows: list[list[str]]) -> str: if passed: summary = ( - f"Swarm of {len(scores)}: {len(passed)} members' work passes " + f"Swarm of {size}: {len(passed)} members' work passes " f"({', '.join(passed)})." ) else: - summary = f"Swarm of {len(scores)}: no member's work passes." - width = max(len(stage) for stage in stages.values()) - lines = [ - f"{name}: {'CORRECT' if s.value == CORRECT else 'INCORRECT':<9} " - f"{stages[name]:<{width}} {claims[name]}" - for name, s in scores.items() + summary = f"Swarm of {size}: no member's work passes." + table = [ + "| Member | Result | Claim | Stage | Checker |", + "| --- | --- | --- | --- | --- |", + *( + "| " + " | ".join(cell.replace("|", "\\|") for cell in row) + " |" + for row in rows + ), ] - return "\n".join([summary, *lines]) + return "\n".join([summary, "", *table]) async def _score_sandbox( diff --git a/tests/test_checker.py b/tests/test_checker.py index 8316f2f..92ca630 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -753,7 +753,8 @@ async def check( ) -> CheckOutcome: self.calls.append((submission_tar, claim)) ok = submission_tar == self._passing - return CheckOutcome(ok=ok, stage="stub", detail="stub outcome") + last = "Your solution is okay!" if ok else "uncaught exception: a | b" + return CheckOutcome(ok=ok, stage="comparator", detail=f"Building\n{last}\n") def _swarm_fixture( @@ -799,9 +800,13 @@ async def test_swarm_final_score_checks_every_member_sandbox( assert score.metadata["passed"] == ["prover-2"] assert score.explanation == ( "Swarm of 3: 1 members' work passes (prover-2).\n" - 'prover-1: INCORRECT stub called submit_proof(claim="proof")\n' - 'prover-2: CORRECT stub called submit_proof(claim="disproof")\n' - 'prover-3: INCORRECT stub did not call submit_proof(), defaulted to claim="proof"' + "\n" + "| Member | Result | Claim | Stage | Checker |\n" + "| --- | --- | --- | --- | --- |\n" + '| prover-1 | INCORRECT | called `submit_proof(claim="proof")` | comparator | uncaught exception: a \\| b |\n' + '| prover-2 | CORRECT | called `submit_proof(claim="disproof")` | comparator | Your solution is okay! |\n' + "| prover-3 | INCORRECT | did not call `submit_proof()`, defaulted to " + '`claim="proof"` | comparator | uncaught exception: a \\| b |' ) assert checker.calls == [ (tars["prover-1"], "proof"), From 839a63cda77ce11811db80405f08f2f038569b86 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 19:16:02 +0000 Subject: [PATCH 22/33] Show how each swarm member ended in the score explanation Adds the member's swarm status (submitted, stopped, share, cancelled, ...) as a column, and the swarm's stop reason, plus what ended it from outside when anything did, to the summary line. --- apn/scorer.py | 19 ++++++++++++++----- tests/test_checker.py | 28 ++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/apn/scorer.py b/apn/scorer.py index 71ab6f9..1d5f5b7 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import Any from inspect_ai.scorer import ( CORRECT, @@ -30,6 +31,7 @@ CLAIM_STORE_KEY = "submission_claim" SWARM_RESULT_KEY = "inspect_swarm:result" +SWARM_LEDGER_KEY = "inspect_swarm:ledger" def claim_store_key() -> str: @@ -81,11 +83,13 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: "", ) result = "CORRECT" if score.value == CORRECT else "INCORRECT" - rows.append([name, result, claim, score.metadata["stage"], check]) + rows.append([name, m["status"], result, claim, score.metadata["stage"], check]) passed = [name for name, s in scores.items() if s.value == CORRECT] return Score( value=CORRECT if passed else INCORRECT, - explanation=_swarm_explanation(len(scores), passed, rows), + explanation=_swarm_explanation( + len(scores), passed, state.store.get(SWARM_LEDGER_KEY), rows + ), metadata={ "passed": passed, "members": {name: s.metadata for name, s in scores.items()}, @@ -93,7 +97,9 @@ async def _score_swarm(state: TaskState, checker: ProofChecker) -> Score: ) -def _swarm_explanation(size: int, passed: list[str], rows: list[list[str]]) -> str: +def _swarm_explanation( + size: int, passed: list[str], ledger: dict[str, Any], rows: list[list[str]] +) -> str: if passed: summary = ( f"Swarm of {size}: {len(passed)} members' work passes " @@ -101,9 +107,12 @@ def _swarm_explanation(size: int, passed: list[str], rows: list[list[str]]) -> s ) else: summary = f"Swarm of {size}: no member's work passes." + summary += f" Stop reason: {ledger['stop_reason']}." + if ledger["ended_by"] is not None: + summary += f" Ended by: {ledger['ended_by']}." table = [ - "| Member | Result | Claim | Stage | Checker |", - "| --- | --- | --- | --- | --- |", + "| Member | Status | Result | Claim | Stage | Checker |", + "| --- | --- | --- | --- | --- | --- |", *( "| " + " | ".join(cell.replace("|", "\\|") for cell in row) + " |" for row in rows diff --git a/tests/test_checker.py b/tests/test_checker.py index 92ca630..0037ea1 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -54,6 +54,7 @@ ) from apn.scorer import ( CLAIM_STORE_KEY, + SWARM_LEDGER_KEY, SWARM_RESULT_KEY, claim_store_key, proof_scorer, @@ -773,11 +774,16 @@ def _swarm_fixture( SWARM_RESULT_KEY, { "members": [ - {"name": name, "sandbox": name, "submitted": name == "prover-2"} + { + "name": name, + "sandbox": name, + "status": "submitted" if name == "prover-2" else "stopped", + } for name in tars ] }, ) + state.store.set(SWARM_LEDGER_KEY, {"stop_reason": "submitted", "ended_by": None}) sandboxes = {name: FakeSandbox(tar) for name, tar in tars.items()} def fake_sandbox(name: str | None = None) -> FakeSandbox: @@ -799,13 +805,13 @@ async def test_swarm_final_score_checks_every_member_sandbox( assert score.metadata is not None assert score.metadata["passed"] == ["prover-2"] assert score.explanation == ( - "Swarm of 3: 1 members' work passes (prover-2).\n" + "Swarm of 3: 1 members' work passes (prover-2). Stop reason: submitted.\n" "\n" - "| Member | Result | Claim | Stage | Checker |\n" - "| --- | --- | --- | --- | --- |\n" - '| prover-1 | INCORRECT | called `submit_proof(claim="proof")` | comparator | uncaught exception: a \\| b |\n' - '| prover-2 | CORRECT | called `submit_proof(claim="disproof")` | comparator | Your solution is okay! |\n' - "| prover-3 | INCORRECT | did not call `submit_proof()`, defaulted to " + "| Member | Status | Result | Claim | Stage | Checker |\n" + "| --- | --- | --- | --- | --- | --- |\n" + '| prover-1 | stopped | INCORRECT | called `submit_proof(claim="proof")` | comparator | uncaught exception: a \\| b |\n' + '| prover-2 | submitted | CORRECT | called `submit_proof(claim="disproof")` | comparator | Your solution is okay! |\n' + "| prover-3 | stopped | INCORRECT | did not call `submit_proof()`, defaulted to " '`claim="proof"` | comparator | uncaught exception: a \\| b |' ) assert checker.calls == [ @@ -822,11 +828,17 @@ async def test_swarm_final_score_incorrect_when_no_member_passes( monkeypatch: pytest.MonkeyPatch, ) -> None: state, _, _ = _swarm_fixture(monkeypatch) + state.store.set( + SWARM_LEDGER_KEY, {"stop_reason": "swarm_cap", "ended_by": "sample_limit:cost"} + ) score = await proof_scorer(TarChecker(passing=b""), swarm=True)(state, Target("")) assert score is not None assert score.value == INCORRECT assert score.explanation is not None - assert score.explanation.splitlines()[0] == "Swarm of 3: no member's work passes." + assert score.explanation.splitlines()[0] == ( + "Swarm of 3: no member's work passes. Stop reason: swarm_cap. " + "Ended by: sample_limit:cost." + ) async def test_swarm_score_inside_a_member_checks_only_its_own_sandbox( From ebe62a83319e70d596eb3357f293921f05519838 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 19:25:19 +0000 Subject: [PATCH 23/33] set shared_memory max sizes --- apn/solver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apn/solver.py b/apn/solver.py index 81349aa..9e1c0e1 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -268,7 +268,13 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: ) for name in names ], - channels=[shared_memory()], + channels=[shared_memory( + # TODO: remove these if inspect_swarm changes their defaults to be more sensible + max_files=10_000, + max_file_bytes=1_000_000, + max_file_lines=100_000, + max_board_bytes=400_000_000, + )], budget=Budget(split="equal"), ) else: From 9559a8968dc1230ed03376b476191d599b418482 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 19:29:46 +0000 Subject: [PATCH 24/33] Set shared_memory limits at inspect_swarm's caps max_file_bytes (262,144) and max_board_bytes (16,777,216) were above the bounds shared_memory() accepts, so building the swarm raised ValueError. --- apn/solver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 9e1c0e1..f1ebf2c 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -271,9 +271,9 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: channels=[shared_memory( # TODO: remove these if inspect_swarm changes their defaults to be more sensible max_files=10_000, - max_file_bytes=1_000_000, + max_file_bytes=262_144, max_file_lines=100_000, - max_board_bytes=400_000_000, + max_board_bytes=16_777_216, )], budget=Budget(split="equal"), ) From e436d9073939a2d214265b5d9b77224b7d011835 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 19:30:03 +0000 Subject: [PATCH 25/33] Add TODO to raise shared_memory limits once inspect_swarm drops its caps --- apn/solver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apn/solver.py b/apn/solver.py index f1ebf2c..b4f8603 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -270,6 +270,8 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: ], channels=[shared_memory( # TODO: remove these if inspect_swarm changes their defaults to be more sensible + # TODO: raise max_file_bytes and max_board_bytes (set at inspect_swarm's + # current caps) once the upstream PR removing the caps is released max_files=10_000, max_file_bytes=262_144, max_file_lines=100_000, From 1e87782146843f0bb7392ff7d6590fd63153f769 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 26 Oct 2026 21:24:38 +0000 Subject: [PATCH 26/33] Use poll delivery for swarm members Members no longer get notices injected between turns; they learn of board changes from the status line ending each shared_memory result and from swarm_wait. --- apn/solver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apn/solver.py b/apn/solver.py index b4f8603..74c8bc1 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -265,6 +265,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: args={"agent_type": agent_type, "gated": gated, "swarm": True}, sandbox=name, visible_sandboxes=[comparator_sandbox_name(name)], + delivery="poll", ) for name in names ], From 846dfe6e5a5e26fc9cd390f27c9cf53605640d2c Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 15:41:34 +0000 Subject: [PATCH 27/33] Upgrade inspect_swarm to 0.2.0 0.2.0 removes swarm notices, swarm_on_continue() and member(delivery=...), so drop them (members already used poll delivery). It also removes the shared_memory caps, so restore the intended max_file_bytes and max_board_bytes. --- apn/solver.py | 16 ++++++---------- pyproject.toml | 2 +- uv.lock | 8 ++++---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 74c8bc1..fe545fd 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -30,7 +30,6 @@ member, shared_memory, swarm, - swarm_on_continue, swarm_resources, swarm_tools, ) @@ -206,9 +205,9 @@ def build_agent( def _prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: """The prover loop, for a single agent or a swarm member. - ``swarm_tools()`` and ``swarm_on_continue()`` do nothing outside a swarm. A - member's budget tool is ``swarm_resources`` (its own share of the budget), - since ``resources`` shows the sample's limits.""" + ``swarm_tools()`` does nothing outside a swarm. A member's budget tool is + ``swarm_resources`` (its own share of the budget), since ``resources`` + shows the sample's limits.""" return build_agent( agent_type, tools=[ @@ -222,7 +221,7 @@ def _prover(agent_type: AgentType, gated: bool, swarm: bool = False) -> Agent: ], attempts=_attempts(gated), submit=_prover_submit(), - on_continue=swarm_on_continue(continue_unless_looping(CONTINUE_MESSAGE)), + on_continue=continue_unless_looping(CONTINUE_MESSAGE), compaction=CompactionSummary(threshold=300_000), ) @@ -265,18 +264,15 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: args={"agent_type": agent_type, "gated": gated, "swarm": True}, sandbox=name, visible_sandboxes=[comparator_sandbox_name(name)], - delivery="poll", ) for name in names ], channels=[shared_memory( # TODO: remove these if inspect_swarm changes their defaults to be more sensible - # TODO: raise max_file_bytes and max_board_bytes (set at inspect_swarm's - # current caps) once the upstream PR removing the caps is released max_files=10_000, - max_file_bytes=262_144, + max_file_bytes=1_000_000, max_file_lines=100_000, - max_board_bytes=16_777_216, + max_board_bytes=400_000_000, )], budget=Budget(split="equal"), ) diff --git a/pyproject.toml b/pyproject.toml index 0fa3584..e6d141d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license-files = ["LICENSE"] requires-python = ">=3.13,<3.14" dependencies = [ "inspect-boltons @ git+https://github.com/tadamcz/inspect_boltons@35b794e845f1748947a4f18efdfcb31dd0cf746a", - "inspect-swarm>=0.1.0,<0.2.0", + "inspect-swarm>=0.2.0,<0.3.0", ] [build-system] diff --git a/uv.lock b/uv.lock index e0443c3..eac932e 100644 --- a/uv.lock +++ b/uv.lock @@ -179,7 +179,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "inspect-boltons", git = "https://github.com/tadamcz/inspect_boltons?rev=35b794e845f1748947a4f18efdfcb31dd0cf746a" }, - { name = "inspect-swarm", specifier = ">=0.1.0,<0.2.0" }, + { name = "inspect-swarm", specifier = ">=0.2.0,<0.3.0" }, ] [package.metadata.requires-dev] @@ -669,16 +669,16 @@ dependencies = [ [[package]] name = "inspect-swarm" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "inspect-ai" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/7a/47f9fb7dd58b29715b2985b5398c53be6cc09b3321b73ada98ab273b02f0/inspect_swarm-0.1.0.tar.gz", hash = "sha256:78e75588583f6a09716973589fe1dbb9bebacf0f2b6a0cd6244ae0e31d7b9048", size = 88033, upload-time = "2026-10-23T13:23:48.01Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/21/46b5b1fcd993f0bc2dcf3a66a3b47bdec02932e15eb22a65dbfefa70d7f5/inspect_swarm-0.2.0.tar.gz", hash = "sha256:ef86c4ba67d3e1547650c3ec154323ed1c5f82130758ba9c95a95c886c04817b", size = 84228, upload-time = "2026-10-27T17:05:43.433Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/0f/b86e986fb631688286690571dfd2d85e1ed76b49dadd74809b7e061dd12b/inspect_swarm-0.1.0-py3-none-any.whl", hash = "sha256:e4600ae6709673cd33048b50728a626376d5f943df9e79e2918d6b330b99299d", size = 101415, upload-time = "2026-10-23T13:23:46.65Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3b/d5eb921541f4e52fae990a60716b2d128ce9efc2facae6de17df0df43db0/inspect_swarm-0.2.0-py3-none-any.whl", hash = "sha256:e11f61870418fe78050e3ea6d45fb7093dc3d3cd55c42d49dbd93c59a158864b", size = 96924, upload-time = "2026-10-27T17:05:42.238Z" }, ] [[package]] From 4d5de022e953b1db1128b444906ccb699002ad7f Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 15:51:24 +0000 Subject: [PATCH 28/33] Hold back 1% of the sample limits for the swarm, not 5% --- apn/solver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apn/solver.py b/apn/solver.py index fe545fd..6a5ef71 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -274,7 +274,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: max_file_lines=100_000, max_board_bytes=400_000_000, )], - budget=Budget(split="equal"), + budget=Budget(split="equal", reserve=0.01), ) else: await sandbox().write_file(ENTRY_PATH, sketch) From 35ad0f49db36e1a85443ea40ada2205320d145f9 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 16:01:50 +0000 Subject: [PATCH 29/33] Keep swarm members' submission sidecars apart Each member's scored tars go under artifacts/// with its own attempt counter, so the artifacts show whose work each tar is. --- apn/scorer.py | 19 ++++++++++++++----- tests/test_checker.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/apn/scorer.py b/apn/scorer.py index 1d5f5b7..40f2859 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -34,9 +34,13 @@ SWARM_LEDGER_KEY = "inspect_swarm:ledger" -def claim_store_key() -> str: +def _member_key(key: str) -> str: member = current_member() - return CLAIM_STORE_KEY if member is None else f"{CLAIM_STORE_KEY}:{member}" + return key if member is None else f"{key}:{member}" + + +def claim_store_key() -> str: + return _member_key(CLAIM_STORE_KEY) @scorer(metrics=[accuracy(), stderr()]) @@ -130,8 +134,9 @@ async def _score_sandbox( # Per-attempt attempt index, kept in the sample store (the react/deepagent # attempt_count is not reachable from here). Increments even when # max_attempts=1, so a single-attempt sample still tags attempt-1. - attempt = store().get("_score_call_idx", 0) + 1 - store().set("_score_call_idx", attempt) + attempt_key = _member_key("_score_call_idx") + attempt = store().get(attempt_key, 0) + 1 + store().set(attempt_key, attempt) try: tar = await read_submission_tar(sb) @@ -180,7 +185,8 @@ def _record_submission_tree(state: TaskState, tar: bytes, key: str) -> None: def _write_submission_sidecar(state: TaskState, attempt: int, tar: bytes) -> None: - """Write the scored ``Submission/`` tar to ``artifacts//attempt-N.tar``.""" + """Write the scored ``Submission/`` tar to ``artifacts//attempt-N.tar``, + or ``artifacts///attempt-N.tar`` for a swarm member.""" try: # private Inspect API -- no public way to get the log path from a scorer. from inspect_ai.log._samples import sample_active @@ -191,6 +197,9 @@ def _write_submission_sidecar(state: TaskState, attempt: int, tar: bytes) -> Non logger.warning("Could not get active sample; skipping submission sidecar") return sidecar_dir = UPath(active.log_location).parent / "artifacts" / state.uuid + member = current_member() + if member is not None: + sidecar_dir = sidecar_dir / member sidecar_dir.mkdir(parents=True, exist_ok=True) # Zero-pad the attempt index (PortBench's sidecar convention) so the # files sort lexicographically in attempt order. diff --git a/tests/test_checker.py b/tests/test_checker.py index 0037ea1..c8ce0ce 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -865,3 +865,31 @@ async def test_scorer_does_not_record_a_default_claim( await proof_scorer(checker)(_state(FakeStore()), Target("")) assert checker.calls[0][3] == "proof" assert CLAIM_STORE_KEY not in store.keys() + + +async def test_swarm_sidecars_are_per_member( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import inspect_ai.log._samples as samples_mod + + monkeypatch.setattr( + samples_mod, + "sample_active", + lambda: SimpleNamespace(log_location=str(tmp_path / "run.eval")), + ) + state, tars, _ = _swarm_fixture(monkeypatch) + checker = TarChecker(passing=tars["prover-2"]) + with tool_state_scope("prover-2"): + await proof_scorer(checker, swarm=True)(state, Target("")) + await proof_scorer(checker, swarm=True)(state, Target("")) + root = tmp_path / "artifacts" / state.uuid + sidecars = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("attempt-*.tar") + } + assert sidecars == { + "prover-1/attempt-00001.tar": tars["prover-1"], + "prover-2/attempt-00001.tar": tars["prover-2"], + "prover-2/attempt-00002.tar": tars["prover-2"], + "prover-3/attempt-00001.tar": tars["prover-3"], + } From 6e1669976a9cdce9856a56472f31a48c051cde37 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 16:02:25 +0000 Subject: [PATCH 30/33] Give the swarm example config 24 hours of wall time --- configs/erdos74-swarm4.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/erdos74-swarm4.yaml b/configs/erdos74-swarm4.yaml index 18a8fbe..0499baf 100644 --- a/configs/erdos74-swarm4.yaml +++ b/configs/erdos74-swarm4.yaml @@ -33,7 +33,7 @@ cost_limit: 400.0 # members gently at 95% of it. A time limit does charge members for that # waiting, so avoid contention when running a swarm: give max_connections room # for every member running at once across samples, and mind rate limits. -time_limit: 432_000 # 5 * 24 * 60 * 60 +time_limit: 86_400 # 24 * 60 * 60 # $ per 1M tokens; keys must match Inspect's resolved / names. model_cost_config: From c4b00bf14e7c93c361ccd4fa0105482a8aa64db9 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 16:03:04 +0000 Subject: [PATCH 31/33] Drop the reserve from the swarm example config's comment --- configs/erdos74-swarm4.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/configs/erdos74-swarm4.yaml b/configs/erdos74-swarm4.yaml index 0499baf..fff8459 100644 --- a/configs/erdos74-swarm4.yaml +++ b/configs/erdos74-swarm4.yaml @@ -29,10 +29,9 @@ cost_limit: 400.0 # minus the periods when any task waits (for a connection slot, a retry or a # rate limit), with overlapping waits counted once, so with many concurrent # members it can stall far behind the wall clock: working_limit is poorly -# suited to a swarm. The swarm also turns a time limit into a cap that stops -# members gently at 95% of it. A time limit does charge members for that -# waiting, so avoid contention when running a swarm: give max_connections room -# for every member running at once across samples, and mind rate limits. +# suited to a swarm. A time limit does charge members for that waiting, so +# avoid contention when running a swarm: give max_connections room for every +# member running at once across samples, and mind rate limits. time_limit: 86_400 # 24 * 60 * 60 # $ per 1M tokens; keys must match Inspect's resolved / names. From 9fa39045f21285e9e35e3ea310e804094172b205 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 16:06:02 +0000 Subject: [PATCH 32/33] Let the swarm example config install inspect-swarm 0.2.0 Its UV_EXCLUDE_NEWER predated the release, so the package could not resolve. --- configs/erdos74-swarm4.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/erdos74-swarm4.yaml b/configs/erdos74-swarm4.yaml index fff8459..c79a52b 100644 --- a/configs/erdos74-swarm4.yaml +++ b/configs/erdos74-swarm4.yaml @@ -50,7 +50,9 @@ runner: memory: 200Gi environment: INSPECT_LOG_CONDENSE: "true" - UV_EXCLUDE_NEWER: "2026-10-24T00:00:00Z" + # TODO: remove UV_EXCLUDE_NEWER once no longer needed (it circumvents the + # security cooldown). It admits inspect-swarm 0.2.0, released 2026-10-27. + UV_EXCLUDE_NEWER: "2026-10-28T00:00:00Z" packages: - inspect-ai==0.3.283 From 9affb773b605ecc89cc313f1d201877a80028a01 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Wed, 28 Oct 2026 16:28:14 +0000 Subject: [PATCH 33/33] Note that swarm members share the submission_contents tree --- apn/scorer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apn/scorer.py b/apn/scorer.py index 40f2859..b346a61 100644 --- a/apn/scorer.py +++ b/apn/scorer.py @@ -56,6 +56,7 @@ def proof_scorer(checker: ProofChecker, swarm: bool = False) -> Scorer: async def score(state: TaskState, target: Target) -> Score: if swarm and current_member() is None: return await _score_swarm(state, checker) + # Known limitation: in a swarm, every member writes this same key, so the last one wins. return await _score_sandbox(state, checker, sandbox(), "submission_contents") return score