Add 11 search and retrieval benchmark tasks

tasks/budgeted-bimetric-retrieval/.gitignore

@@ -0,0 +1,4 @@
+/data/
+/models/
+__pycache__/
+*.py[cod]

tasks/budgeted-bimetric-retrieval/README.md

@@ -0,0 +1,83 @@
+# budgeted-bimetric-retrieval
+
+Improve retrieval over 250,000 NQ passages with 20 controlled expensive distances and two seconds per query.
+
+The complete task contract is in [raw-instruction.md](raw-instruction.md).
+The coding agent receives [instruction.md](instruction.md),
+[available resources](environment/docs/available_resources.md), and
+[environment instructions](environment/docs/environment.md).
+Those documents specify the submission interface, allowed inputs, scoring,
+failure behavior and public self-check commands.
+
+## Restore inputs
+
+Run from the repository root using the repository's existing Python environment:
+
+```bash
+python scripts/download_assets.py --task budgeted-bimetric-retrieval
+python scripts/download_assets.py --task budgeted-bimetric-retrieval --verify-only
+```
+
+[assets.json](assets.json) fixes file sizes, hashes and source revisions or bundled
+restoration paths. Keep verifier inputs outside agent mounts. Model cards,
+licenses and data-use restrictions are described in the resource document.
+
+## Run
+
+The task allocates 16 CPU cores, 32 GiB RAM and
+64 GiB storage. Agent development has
+7200 seconds. Final-stage limits and accounting are
+specified in the task contract; the verifier service timeout also includes its
+trusted setup and checks.
+
+This task uses no GPU.
+
+On the Linux Docker host, `/dev/fuse` and `mkfs.ext4` must be available.
+Prepare fresh bounded storage for each run, then set `BIMETRIC_CPUSET` to
+16 available logical CPU IDs on a supported CPU. Use a unique `RUN_ID`:
+
+```bash
+python tasks/budgeted-bimetric-retrieval/tools/prepare_storage.py --run-id RUN_ID
+export BIMETRIC_STORAGE_ROOT="$PWD/jobs/.task-storage/budgeted-bimetric-retrieval/RUN_ID"
+```
+
+Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or
+authorized process environment, following the repository README.
+
+The independent verifier audit also requires `VERIFIER_OPENAI_API_KEY` and
+`VERIFIER_OPENAI_BASE_URL=https://openrouter.ai/api/v1` in that local configuration.
+It uses `deepseek/deepseek-v4.1-flash`; only verifier egress to `openrouter.ai`
+is allowed. These credentials are never passed to submitted commands.
+The verifier protects the transferred trajectory, records the raw numerical
+score, terminates candidate processes, and publishes reward only after a
+healthy passing integrity audit. See `audit-status.json` for audit health.
+
+Invoke the standard entrypoint directly:
+
+```bash
+bash scripts/run_task.sh --task budgeted-bimetric-retrieval --agent codex --openrouter \
+  --model openai/gpt-6-luna --reasoning-effort max \
+  --output "jobs/budgeted-bimetric-retrieval-codex-openrouter-luna-max-RUN_ID"
+```
+
+Inspect the resulting agent trajectory, submission and verifier report under the
+job directory. The score alone does not distinguish an invalid submission from
+an infrastructure error.
+
+After every container/process for this run has exited, unmount its storage:
+
+```bash
+python tasks/budgeted-bimetric-retrieval/tools/prepare_storage.py --run-id RUN_ID --unmount
+```
+
+## Validate the package
+
+```bash
+python scripts/check_tasks.py tasks/budgeted-bimetric-retrieval
+python scripts/download_assets.py --task budgeted-bimetric-retrieval --verify-only
+```
+
+Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective
+directories as build contexts. Run the public helper and verifier with the task's
+resource controls and mounts. Tests that exercise process isolation or GPU
+enforcement belong in disposable task containers, not the shared host.

tasks/budgeted-bimetric-retrieval/assets.json

@@ -0,0 +1,83 @@
+{
+  "schema_version": 1,
+  "files": [
+    {
+      "path": "data/public/cheap.f32",
+      "size_bytes": 384000000,
+      "sha256": "91775ac5391c7d8ef43d4f441573dff081c54a83531beeaf27e3d3a7198d7f49",
+      "mode": "0644",
+      "source": {
+        "repo_id": "search-swe/Search-SWE",
+        "repo_type": "dataset",
+        "revision": "3635ca4269c1a1122c2da50187d0db42f3f73006",
+        "filename": "tasks/budgeted-bimetric-retrieval/v020/public/cheap.f32"
+      }
+    },
+    {
+      "path": "data/public/config.json",
+      "size_bytes": 1367,
+      "sha256": "1f21c2da82aa07edb72a33e8f15eed3cf90d73297d252e079428cb8cefb92c44",
+      "mode": "0644",
+      "source": {
+        "local_path": "environment/config/public.json"
+      }
+    },
+    {
+      "path": "data/public/corpus.parquet",
+      "size_bytes": 46972114,
+      "sha256": "94fdc82caa239bef860e5e4717e93e7e5947ada50bdc3c7cbac129d293cec1d5",
+      "mode": "0644",
+      "source": {
+        "repo_id": "search-swe/Search-SWE",
+        "repo_type": "dataset",
+        "revision": "3635ca4269c1a1122c2da50187d0db42f3f73006",
+        "filename": "tasks/budgeted-bimetric-retrieval/v020/public/corpus.parquet"
+      }
+    },
+    {
+      "path": "data/public/dev-qrels.tsv",
+      "size_bytes": 11267,
+      "sha256": "a341442961897df119aa6fd42acd8fda1bd5c5883e0b910bd9802c3302cb1482",
+      "mode": "0644",
+      "source": {
+        "repo_id": "search-swe/Search-SWE",
+        "repo_type": "dataset",
+        "revision": "3635ca4269c1a1122c2da50187d0db42f3f73006",
+        "filename": "tasks/budgeted-bimetric-retrieval/v020/public/dev-qrels.tsv"
+      }
+    },
+    {
+      "path": "data/public/dev-queries.jsonl",
+      "size_bytes": 4269697,
+      "sha256": "21f2c63024a655d30be5e7aedf05213fdaf5fc833e159b0012ab198544b250e9",
+      "mode": "0644",
+      "source": {
+        "repo_id": "search-swe/Search-SWE",
+        "repo_type": "dataset",
+        "revision": "3635ca4269c1a1122c2da50187d0db42f3f73006",
+        "filename": "tasks/budgeted-bimetric-retrieval/v020/public/dev-queries.jsonl"
+      }
+    },
+    {
+      "path": "data/verifier/dev-queries.f32",
+      "size_bytes": 8192000,
+      "sha256": "d62d3f92ce26c11e5fe0e68926706f597ebfb19c35bb3153d77b4b16bbf78dd8",
+      "mode": "0600",
+      "source": {
+        "local_path": "tests/bundled-assets/data/verifier/dev-queries.f32"
+      }
+    },
+    {
+      "path": "data/verifier/expensive.f32",
+      "size_bytes": 4096000000,
+      "sha256": "9cc73e1ad727526f8eaf7a3b53ebfc70ff23db2dd32540b3750baa6673a2ecb1",
+      "mode": "0600",
+      "source": {
+        "repo_id": "search-swe/Search-SWE",
+        "repo_type": "dataset",
+        "revision": "3635ca4269c1a1122c2da50187d0db42f3f73006",
+        "filename": "tasks/budgeted-bimetric-retrieval/v020/verifier/expensive.f32"
+      }
+    }
+  ]
+}

tasks/budgeted-bimetric-retrieval/environment/.dockerignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/

tasks/budgeted-bimetric-retrieval/environment/Dockerfile

@@ -0,0 +1,26 @@
+FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0
+
+ARG SEARCH_SWE_CODEX_VERSION=0.157.1
+ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283
+ARG SEARCH_SWE_PI_VERSION=0.87.1
+RUN npm install --global --ignore-scripts \
+        --registry=https://registry.npmmirror.com \
+        "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \
+        "@earendil-works/pi-coding-agent@${SEARCH_SWE_PI_VERSION}" \
+    && npm install --global \
+        --registry=https://registry.npmmirror.com \
+        "@anthropic-ai/claude-code@${SEARCH_SWE_CLAUDE_CODE_VERSION}" \
+    && codex --version | grep -Fx "codex-cli ${SEARCH_SWE_CODEX_VERSION}" \
+    && test "$(pi --version)" = "${SEARCH_SWE_PI_VERSION}" \
+    && test "$(claude --version)" = "${SEARCH_SWE_CLAUDE_CODE_VERSION} (Claude Code)"
+RUN claude --help | grep -F "(low, medium, high, xhigh, max)" >/dev/null
+
+COPY tools /task/tools
+COPY baseline /task/baseline
+RUN useradd --create-home --uid 10001 --user-group submission \
+    && mkdir -p /app /task/public /task/docs /run/oracle \
+    && chmod -R a+rX /task/tools /task/baseline \
+    && chmod 755 /task/baseline/search.sh
+RUN /opt/conda/bin/python -c 'import faiss,numpy,pyarrow,psutil; import ctypes; ctypes.CDLL("libseccomp.so.2")'
+ENV HOME=/home/submission
+CMD ["sh", "-c", "sleep infinity"]

tasks/budgeted-bimetric-retrieval/environment/baseline/search.py

@@ -0,0 +1,87 @@
+"""Public simple baselines. Parameters are frozen using development queries only."""
+import argparse
+import json
+from pathlib import Path
+import re
+import sqlite3
+import sys
+import faiss
+import numpy as np
+import pyarrow.parquet as pq
+
+p = argparse.ArgumentParser()
+p.add_argument('--data-dir', type=Path, required=True)
+p.add_argument('--index-dir', type=Path, required=True)
+p.add_argument('--mode', choices=['build', 'load'], required=True)
+p.add_argument('--method', choices=['vector', 'direct', 'hybrid'], default='vector')
+p.add_argument('--budget', type=int)
+p.add_argument('--lexical-weight', type=float, default=.5)
+a = p.parse_args()
+config = json.loads((a.data_dir/'config.json').read_text())
+if a.budget is None:
+    a.budget = config['score_budget']
+n, d = config['documents'], config['cheap_dimension']
+vectors = np.memmap(a.data_dir/'cheap.f32', dtype='<f4', mode='r', shape=(n, d))
+index = faiss.IndexFlatIP(d)
+index.add(vectors)
+del vectors
+a.index_dir.mkdir(parents=True, exist_ok=True)
+db = None
+if a.method == 'hybrid':
+    db = sqlite3.connect(a.index_dir/'lexical.sqlite')
+    if a.mode == 'build':
+        db.execute('DROP TABLE IF EXISTS docs')
+        db.execute('CREATE VIRTUAL TABLE docs USING fts5(title, text, content="", tokenize="porter unicode61")')
+        offset = 0
+        for batch in pq.ParquetFile(a.data_dir/'corpus.parquet').iter_batches(batch_size=8192):
+            rows = [(offset+i+1, item['title'], item['text']) for i, item in enumerate(batch.to_pylist())]
+            db.executemany('INSERT INTO docs(rowid,title,text) VALUES (?,?,?)', rows)
+            offset += len(rows)
+        db.commit()
+
+
+def emit(message):
+    print(json.dumps(message, separators=(',', ':')), flush=True)
+
+
+stop = set('a an and are as at be by for from how in is it of on or that the to was were what when where which who why with'.split())
+emit({'op': 'ready'})
+for line in sys.stdin:
+    query = json.loads(line)
+    if query.get('op') != 'query':
+        raise ValueError('Expected a query')
+    count = max(1000, a.budget) if a.method == 'hybrid' else max(10, a.budget)
+    _, candidates = index.search(np.array([query['vector']], dtype='f4'), count)
+    candidates = candidates[0].tolist()
+    if a.method == 'hybrid':
+        terms = [x for x in re.findall(r'\w+', query['text'].lower()) if x not in stop]
+        expression = ' OR '.join('"'+x.replace('"','""')+'"' for x in terms)
+        # A fixed work cap makes fallback independent of transient machine load.
+        # Training-only timing calibration leaves room within the 2 s query gate.
+        lexical_steps = [0]
+        def progress():
+            lexical_steps[0] += 1000
+            return lexical_steps[0] >= 10000000
+        db.set_progress_handler(progress, 1000)
+        try:
+            lexical = [r[0]-1 for r in db.execute('SELECT rowid FROM docs WHERE docs MATCH ? ORDER BY rank LIMIT 1000', (expression,))] if expression else []
+        except sqlite3.OperationalError as exc:
+            if 'interrupted' not in str(exc): raise
+            lexical = []
+            print('lexical work limit: using dense candidates', file=sys.stderr, flush=True)
+        finally:
+            db.set_progress_handler(None, 0)
+        fused = {}
+        for weight, ranking in [(1-a.lexical_weight, candidates), (a.lexical_weight, lexical)]:
+            for rank, row in enumerate(ranking): fused[row] = fused.get(row, 0) + weight/(61+rank)
+        candidates = sorted(fused, key=lambda row: (-fused[row], row))[:a.budget]
+    elif a.method == 'direct':
+        emit({'op': 'result', 'token': query['token'], 'doc_ids': [f'doc{x}' for x in candidates[:10]]})
+        continue
+    else:
+        candidates = candidates[:a.budget]
+    ids = [f'doc{x}' for x in candidates]
+    emit({'op': 'score', 'token': query['token'], 'doc_ids': ids})
+    scores = json.loads(sys.stdin.readline())
+    ranked = sorted(zip(scores['distances'], ids), key=lambda item: (item[0], int(item[1][3:])))
+    emit({'op': 'result', 'token': query['token'], 'doc_ids': [item[1] for item in ranked[:10]]})

tasks/budgeted-bimetric-retrieval/environment/baseline/search.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /opt/conda/bin/python /task/baseline/search.py "$@"

tasks/budgeted-bimetric-retrieval/environment/config/public.json

@@ -0,0 +1,39 @@
+{
+  "documents": 250000,
+  "cheap_dimension": 384,
+  "expensive_dimension": 4096,
+  "dtype": "little-endian float32",
+  "unit_normalized": true,
+  "document_ids": "row i -> doc{i}; source_id records original BEIR ID",
+  "distance": "Euclidean L2 from stored float32 vectors; float64 difference and accumulation",
+  "cheap_model": {
+    "repo": "TaylorAI/bge-micro-v2",
+    "revision": "3edf6d7de0faa426b09780416fe61009f26ae589",
+    "max_tokens": 512,
+    "forward_dtype": "float32",
+    "pooling": "mean",
+    "query_prefix": ""
+  },
+  "expensive_model": {
+    "repo": "Salesforce/SFR-Embedding-Mistral",
+    "revision": "33a60fe7b5e00b9414981280c571e8a4f9f42f1b",
+    "max_tokens": 4096,
+    "forward_dtype": "bfloat16",
+    "pooling": "lasttoken",
+    "query_prefix": "Instruct: Given a question, retrieve Wikipedia passages that answer the question\nQuery: "
+  },
+  "document_input": "title + one space + text",
+  "public_queries": 500,
+  "test_queries": 3452,
+  "development_seed": 20261017,
+  "score_budget": 20,
+  "query_seconds": 2,
+  "fixed_asset_bytes": 4625555597,
+  "grading_references": {
+    "B": 0.8460412084949013,
+    "U": 0.9117533972383719
+  },
+  "source_documents": 2681468,
+  "corpus_seed": 20261021,
+  "corpus_selection": "All dev/test qrel documents plus seeded uniform sampling without replacement; sorted source rows; sequential remapping"
+}