diff --git a/.agents/skills/create-searchswe-task/references/task-authoring.md b/.agents/skills/create-searchswe-task/references/task-authoring.md index 9bf0a7b..9c5a44c 100644 --- a/.agents/skills/create-searchswe-task/references/task-authoring.md +++ b/.agents/skills/create-searchswe-task/references/task-authoring.md @@ -113,6 +113,17 @@ above; do not copy its task-specific values. When measuring improvement against a supplied starting system/model, document that baseline and the comparison criterion. +Use the task title as the README's level-one heading, followed by these +level-two headings in order: `Task Metadata`, `Change Log`, +`Difficulty Explanation`, `Verification Explanation`, and `References`. +Metadata covers identity, authors, inputs, metric and resource budgets. +Explain engineering difficulty with any measured evidence, and document scoring, +limits, setup and validation under verification; use level-three headings as needed. +List the papers, datasets or methods actually referenced by the task, or `None`. +New tasks start at `1.0.0` with a dated `Added initial task` changelog entry. +Keep the README and `task.version` aligned as the task evolves; retain original +versions and fingerprints in historical validation reports. + Search-SWE release packages use a separate verifier: ```toml @@ -160,6 +171,13 @@ CPU Compose files must not contain an NVIDIA reservation. Keep the image, shared image, stop and ask the maintainer before introducing a third base image. Treat `docker/README.md` as the source of truth if published versions change. +For a local GPU trial, use the shared launcher's optional +`--gpu-device `. It carries one resolved UUID into both +phases using per-run Compose copies. Keep the package's default `count: 1` +requests and document any task-specific GPU capacity requirement in its README. +Install the target checkout's pinned coding-agent CLIs in the agent Dockerfile; +the current versions are defined in `scripts/harness_versions.py`. + Pin task-specific package and system dependencies. Do not use `latest`, clone a moving branch during a build, copy a host virtual environment, or depend on a developer's local image/cache. The Docker build context is its containing diff --git a/.agents/skills/create-searchswe-task/references/validation.md b/.agents/skills/create-searchswe-task/references/validation.md index 93d9f5c..97c1720 100644 --- a/.agents/skills/create-searchswe-task/references/validation.md +++ b/.agents/skills/create-searchswe-task/references/validation.md @@ -101,7 +101,10 @@ GPU images target linux/amd64 with CUDA 13.0 userspace. GPU execution requires a compatible NVIDIA host driver and Container Toolkit. With the documented Harbor 0.22.0 Docker backend, `gpus = 1` alone fails preflight. The repository launcher passes `--override-gpus 0`, while **both Compose overlays still request -the real GPU**. For direct Harbor runs on that backend use the same override; +the real GPU**. Add `--gpu-device ` to the shared launcher +for explicit selection in both phases. Check the resulting container device +requests and the per-phase configuration under the trial's `gpu/` directory. +For direct Harbor runs on that backend use the same override; do not falsify task metadata to work around it. Recheck this behavior if Harbor changes. CPU tasks do not need the override or NVIDIA reservation. diff --git a/.agents/skills/create-searchswe-task/scripts/scaffold_task.py b/.agents/skills/create-searchswe-task/scripts/scaffold_task.py index e6a6d9f..b8f7c28 100755 --- a/.agents/skills/create-searchswe-task/scripts/scaffold_task.py +++ b/.agents/skills/create-searchswe-task/scripts/scaffold_task.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +from datetime import date import json from pathlib import Path import re @@ -74,11 +75,35 @@ def files_for(task_name: str, hardware: str) -> dict[str, str]: f""" # {task_name} - **Task:** `{task_name}` + ## Task Metadata - Replace this scaffold text with an author-facing overview of the - objective, dataset/model provenance and license, metric, resource - needs, and local validation procedure. + | Field | Value | + | --- | --- | + | Task | `{task_name}` | + | Version | 1.0.0 | + | Hardware | {hardware.upper()} | + + Describe the objective, authors, inputs, metric and resource budgets + consistently with task.toml. Link dataset/model provenance and licenses. + + ## Change Log + + | Date | Version | Description | + | --- | --- | --- | + | {date.today().isoformat()} | 1.0.0 | Added initial task | + + ## Difficulty Explanation + + Explain the engineering challenges and any measured difficulty evidence. + + ## Verification Explanation + + Describe the submission interface, scoring, limits, credential setup + and local validation commands. Record actual model-trial feedback. + + ## References + + None. """ ), "assets.json": json.dumps( @@ -147,7 +172,7 @@ def files_for(task_name: str, hardware: str) -> dict[str, str]: [task] name = "search-swe/{task_name}" - version = "0.1.0" + version = "1.0.0" description = "Replace with the task's observable objective." authors = [{{ name = "Search-SWE" }}] keywords = ["information-retrieval", "search"] diff --git a/.agents/skills/maintain-searchswe-task/references/task-audit.md b/.agents/skills/maintain-searchswe-task/references/task-audit.md index 7876969..777a77e 100644 --- a/.agents/skills/maintain-searchswe-task/references/task-audit.md +++ b/.agents/skills/maintain-searchswe-task/references/task-audit.md @@ -10,6 +10,7 @@ workflow/tool modifications need independent scrutiny, not implicit acceptance. | Repository precedent | For each new task, the contributor names one or two closest reviewed `tasks/` packages already merged into the base branch or states that none is close, and explains matching dimensions, structural reuse and intentional differences. Independently inspect the closest analogue by objective, grading/judge shape, resources, hardware and data/artifact layout. Current contracts and validators override legacy examples. Reject unexplained copying of task-specific IDs/authors, datasets or HF pins, thresholds/baselines, allowlists, budgets, licenses/provenance or hidden evaluation design. Merely existing on main is not proof that a pattern remains valid. | | Instructions | Compare `raw-instruction.md` with the final `instruction.md`: the agent-facing version starts from a concrete work need and preserves the goal, starting state, input/output schema and paths, executable interface, artifacts, thresholds and constraints. Both match every graded requirement. No secret-only correctness requirement or hidden solution hints. | | Objective/resources | Clear goal, starting state and measured outcomes. Supply a meaningful baseline when grading improvement. CPU default, GPU only for execution. Both Dockerfiles, both Compose reservations and `gpus` agree. Visible tools/API/model docs match injected resources and budgets. Pin dependencies/images; inspect build contexts. | +| Task README | One task title followed by `Task Metadata`, `Change Log`, `Difficulty Explanation`, `Verification Explanation`, and `References`. Current version matches `task.toml`; new tasks start at `1.0.0` with a dated `Added initial task` entry. Difficulty and validation claims match evidence. References identify existing paper/data/method sources, or `None`. Historical trial versions and fingerprints remain accurate. | | Solvability | Real known-good submission through the same separate verifier/artifact-transfer interface, expected vs observed reward and an authorized coding-agent trial. A scaffold's always-zero test, empty manifest, dry-run or static pass is not a working benchmark. No public author-only solution without approval. | | Model feedback | Trajectory, score and judge health inspected; task-setting changes explained, or a reason recorded for retaining the setting. Substantive task/scoring changes have relevant checks and a new model trial within the authorized budget, tied to the final evaluated revision. Missing evidence remains explicit; a valid low score alone is not a reason to weaken the task. | | Grader isolation | Separate verifier; hidden tests/labels/references absent from agent mounts/images. Submitted code actually runs unprivileged with timeout, restricted environment, no grader/reward writes or judge keys. Root in a separate container can still tamper with grading. Reward initializes to zero, numeric finite [0,1], failures propagate, cleanup/logs work. | @@ -61,6 +62,9 @@ trusted temporary base `main` service/image, then build both contexts only when authorized. Run known-good and negative verifier cases in disposable environments. For the documented Harbor 0.22.0 GPU backend, the launcher uses `--override-gpus 0` while **both** Compose overlays still request the GPU; recheck on version changes. +For explicit selection, use `--gpu-device ` with the trusted +launcher. Verify that the generated phase configurations expose the same UUID +and retain task mounts and network controls; package defaults use `count: 1`. The launcher does not support `--agent oracle`; a known-good implementation must be tested separately through the real verifier, in an author-only copy. diff --git a/.env.example b/.env.example index 30f5bad..a584745 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,7 @@ AGENT_MODEL= DEEPSEEK_API_KEY= ZAI_API_KEY= -# Optional Pi default: off, minimal, low, medium, high, or xhigh. +# Optional Pi default: off, minimal, low, medium, high, xhigh, or max. PI_THINKING= # Codex agent only: fill both endpoint fields when using --agent codex. diff --git a/README.md b/README.md index 9d5949c..4e4e25d 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ it. #### Claude Code and the official Anthropic API -Claude Code 2.1.283 is preinstalled in every task image. Set an Anthropic model +Claude Code 2.1.309 is preinstalled in every task image. Set an Anthropic model available to your API account and the dedicated coding-agent key: ```dotenv @@ -217,6 +217,17 @@ GPU, network-policy, and custom-provider options. +For a GPU task, add `--gpu-device 0` (host index or full GPU UUID) to select the +same device for the agent and verifier. For example, after preparing its inputs +and credentials: + +```bash +bash scripts/run_task.sh --task code-embedding-finetuning --agent codex --gpu-device 0 +``` + +Omit the option to use the default single-GPU allocation. See +[GPU device selection](docs/evaluation.md#gpu-device-selection) for details. + ## 📚 Documentation | Document | Contents | diff --git a/README_zh.md b/README_zh.md index ce9d8d8..61f65ef 100644 --- a/README_zh.md +++ b/README_zh.md @@ -157,7 +157,7 @@ bash scripts/run_task.sh --task reasoning-query-rewriting --agent codex \ #### Claude Code 和 Anthropic 官方 API -所有任务镜像都预装 Claude Code 2.1.283。填写 API 账户可用的 Anthropic 模型 +所有任务镜像都预装 Claude Code 2.1.309。填写 API 账户可用的 Anthropic 模型 和独立的编码智能体密钥： ```dotenv @@ -201,6 +201,15 @@ bash scripts/run_task.sh --task reasoning-query-rewriting --agent claude-code -- +GPU 任务可以增加 `--gpu-device 0`，使用宿主机编号或完整 GPU UUID，让 agent +和 verifier 使用同一设备。例如，准备好该任务的数据和凭证后运行： + +```bash +bash scripts/run_task.sh --task code-embedding-finetuning --agent codex --gpu-device 0 +``` + +省略该参数时使用默认单卡分配。详见[GPU 设备选择](docs/evaluation.md#gpu-device-selection)。 + ## 📚 文档索引 | 文档 | 内容 | diff --git a/docs/contributing.md b/docs/contributing.md index 6b3c712..9cf43e6 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -28,6 +28,14 @@ prefix and `all` are reserved. Check `tasks/` on the current base and active PRs for name collisions. Set `task.name` to `search-swe/`. A PR may contain multiple uniquely named tasks. +Use the task title as the README's level-one heading and these level-two sections +in order: `Task Metadata`, `Change Log`, `Difficulty Explanation`, +`Verification Explanation`, and `References`. Include dated versions and concise +changes in the changelog; new tasks start at `1.0.0` with `Added initial task`. +Keep the current README version aligned with `task.toml`. Describe difficulty, +scoring and runtime validation, and list existing paper/data/method references +or `None`. Setup commands and detailed explanations can use level-three headings. + Describe the task's goal, starting state and measured outcomes. CPU is the default; choose `--hardware gpu` for actual GPU execution. Repeat `--author` for coauthors. Use actual authors in `task.toml`, not only “Search-SWE”, @@ -68,6 +76,9 @@ Use `--task ` for downloads and local trials. The downloader's `--task-path tasks/` also accepts a canonical repository-relative path. Jobs default to `jobs/`; alternate asset output roots contain `/`. The launcher reads the package's local assets. +For GPU trials, add `--gpu-device ` to select the same device +for the agent and verifier. Choose an available GPU with the task's required +capacity and a separate output directory for each simultaneous run. Follow the skill's [validation layers](../.agents/skills/create-searchswe-task/references/validation.md): Compose rendering, builds, verifier known-good and negative cases, and an diff --git a/docs/evaluation.md b/docs/evaluation.md index 74c8f2a..73af1d4 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -141,7 +141,7 @@ native provider configuration and ignores the direct mode's `AGENT_CODEX_CONFIG` ### Claude Code with the official Anthropic API -Claude Code is pinned to version 2.1.283 and preinstalled in every task image. +Claude Code is pinned to version 2.1.309 and preinstalled in every task image. Set a model available to your Anthropic API account and its dedicated coding-agent key: @@ -160,7 +160,7 @@ bash scripts/run_task.sh \ --dry-run ``` -Claude Code 2.1.283 accepts `low`, `medium`, `high`, `xhigh`, and `max` effort. +Claude Code 2.1.309 accepts `low`, `medium`, `high`, `xhigh`, and `max` effort. Use `AGENT_REASONING_EFFORT` as a local default or `--reasoning-effort` for an explicit run. The launcher maps `AGENT_ANTHROPIC_API_KEY` to the agent-only `ANTHROPIC_API_KEY`, permits only `api.anthropic.com` during restricted Agent @@ -174,7 +174,7 @@ enabled by adding host environment variables. ### Pi native providers -Pi is pinned to version 0.87.1 by the launcher. It currently accepts these +Pi is pinned to version 1.4.0 by the launcher. It currently accepts these verified provider/model combinations: | Model | Required variable | @@ -198,7 +198,7 @@ bash scripts/run_task.sh \ ``` For Z.AI, set `ZAI_API_KEY` and use `zai/glm-5.3-flash`. The accepted thinking -levels are `off`, `minimal`, `low`, `medium`, `high`, and `xhigh`. Use +levels are `off`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`. Use `PI_THINKING` as a local default or `--thinking` for an explicit run. Pi's native provider key does not replace verifier or task-resource credentials. @@ -221,13 +221,18 @@ VERIFIER_OPENAI_BASE_URL=https://openrouter.ai/api/v1 VERIFIER_OPENAI_API_KEY=YOUR_OPENROUTER_KEY ``` -The verifier images pin `harbor-rewardkit==0.2.0` with its Codex CLI 0.147.0 +The verifier images pin `harbor-rewardkit==0.2.0` with Codex CLI 0.164.0 preloaded, and every RewardKit judge is fixed to `deepseek/deepseek-v4.1-flash`. `--model` changes the coding-agent model; it does not change this judge model. The trajectory-judge endpoint is `https://openrouter.ai/api/v1`, checked by the backend and permitted by the verifier allowlists. A relay on another hostname is rejected rather than silently broadening verifier egress. +The task-local backend selects Codex `0.164.0` through RewardKit 0.2.0's +versioned runtime/cache pin. The verifier image preinstalls that exact version +for both root and task-specific audit users. This integration is checked against +RewardKit 0.2.0; a RewardKit upgrade requires a fresh compatibility check. + RewardKit 0.2.0 creates a fresh temporary `CODEX_HOME` for each agent judge, so setting only a host `CODEX_HOME` or `OPENAI_BASE_URL` does not configure that Codex process. Search-SWE therefore registers a `deepseek-codex` backend that @@ -280,12 +285,42 @@ Tasks with `environment.gpus = 1` require one NVIDIA GPU. Their images use CUDA a compatible NVIDIA driver and Container Toolkit. Both task and verifier Compose overlays request the real GPU. +### GPU device selection + +Add `--gpu-device` to the usual launcher command to select one host GPU by index +or full UUID. The launcher resolves the device once and uses the same UUID for +the agent and separate verifier: + +```bash +nvidia-smi --query-gpu=index,uuid,name,memory.used --format=csv +bash scripts/run_task.sh --task code-embedding-finetuning --agent codex \ + --gpu-device 2 --output jobs/code-embedding-finetuning-gpu2 +``` + +Choose an available device that meets the task's hardware requirements. Selection +is explicit; coordinate concurrent users of the host. For simultaneous runs, +select different GPUs and distinct output directories. Omit the option for the +default single-GPU allocation. CPU tasks accept their usual commands. + +Each explicitly selected run stores its generated phase configurations under +`gpu/` in the trial directory. The task files keep their default `count: 1` +reservation; the generated copies replace it with one `device_ids` entry and +retain the original mounts. Run from the local NVIDIA Docker host. Container +CUDA device 0 refers to the single exposed device, regardless of its host index. +`--dry-run` previews the selection without checking GPU availability. + +### Harbor integration + Harbor 0.22.0's Docker backend rejects `gpus = 1` during its own preflight and does not translate that field into the Compose GPU request. The shared launcher therefore supplies `--override-gpus 0` automatically while leaving the truthful task metadata and Compose reservations intact. Direct Harbor invocations using `--env scripts.harbor_environments:PhaseScopedDocker` need the same override with that Harbor version. Recheck this workaround when upgrading Harbor. +Direct Harbor calls can pass `--environment-kwarg gpu_device=GPU-` to +`scripts.harbor_environments:PhaseScopedDocker`. Its environment kwargs are +retained when Harbor creates the separate verifier; extra agent-only Compose +overlays are not a replacement for this shared selection. ## Runtime network enforcement diff --git a/docs/network-policy.md b/docs/network-policy.md index d44c3c8..ca304d2 100644 --- a/docs/network-policy.md +++ b/docs/network-policy.md @@ -114,7 +114,7 @@ The launcher selects exactly one coding-model hostname: the hostname in `AGENT_O `api.anthropic.com` host for direct Claude Code. With `--openrouter`, Codex and Claude Code use `openrouter.ai`. The launcher passes the selected host through Harbor's `--allow-agent-host`, which augments only the `agent.run()` phase. -Codex 0.157.1, Pi 0.87.1, and Claude Code 2.1.283 are preinstalled in every +Codex 0.164.0, Pi 1.4.0, and Claude Code 2.1.309 are preinstalled in every agent image, so agent setup does not need package-registry or general internet access. @@ -144,7 +144,7 @@ invoking Harbor directly on a non-public task, add the matching hostname: PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" harbor run --path tasks/TASK_NAME \ --env scripts.harbor_environments:PhaseScopedDocker \ --agent scripts.harbor_agents:PreinstalledCodex --model MODEL_ID \ - --ak version=0.157.1 \ + --ak version=0.164.0 \ --allow-agent-host MODEL_API_HOST ``` @@ -156,7 +156,7 @@ PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" harbor run --path tasks/TASK_NAME \ --env scripts.harbor_environments:PhaseScopedDocker \ --agent scripts.harbor_agents:PreinstalledClaudeCode \ --model ANTHROPIC_MODEL_ID \ - --ak version=2.1.283 \ + --ak version=2.1.309 \ --ae 'ANTHROPIC_API_KEY=${AGENT_ANTHROPIC_API_KEY}' \ --allow-agent-host api.anthropic.com ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index ffc5ff8..3a0bf16 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -100,7 +100,7 @@ the run as successful. ## Next steps - Read the [full evaluation guide](evaluation.md) before selecting another task - or agent. It documents Codex, Pi, and the pinned Claude Code 2.1.283 launcher, + or agent. It documents Codex, Pi, and the pinned Claude Code 2.1.309 launcher, and explains how to read each task's own credential, hardware and network configuration. - Read the selected task's `README.md` and `instruction.md` for its resource diff --git a/scripts/gpu_device.py b/scripts/gpu_device.py new file mode 100644 index 0000000..164267d --- /dev/null +++ b/scripts/gpu_device.py @@ -0,0 +1,50 @@ +"""Select one host GPU and specialize a task's per-run Compose configuration.""" + +import copy +import re +import subprocess + + +GPU_UUID = re.compile(r"GPU-[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}\Z") + + +def validate_gpu_device(value): + if not (re.fullmatch(r"[0-9]+", value) or GPU_UUID.fullmatch(value)): + raise ValueError("--gpu-device must be one host GPU index or full GPU UUID") + return value + + +def resolve_gpu_device(value): + """Resolve against the local host once, before either phase is launched.""" + validate_gpu_device(value) + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=index,uuid", "--format=csv,noheader,nounits"], + check=True, capture_output=True, text=True, timeout=15, + ) + except (OSError, subprocess.SubprocessError) as error: + raise ValueError("Cannot list host GPUs; check nvidia-smi and the NVIDIA driver") from error + for line in result.stdout.splitlines(): + fields = [field.strip() for field in line.split(",")] + if len(fields) != 2 or not GPU_UUID.fullmatch(fields[1]): + raise ValueError("nvidia-smi returned an invalid GPU inventory") + index, identity = fields + if value.lower() == identity.lower() or (value.isdigit() and int(value) == int(index)): + return identity + raise ValueError(f"GPU {value!r} is not present on this host") + + +def select_compose_gpu(document, identity): + """Replace the existing single-card request without changing task mounts.""" + if not GPU_UUID.fullmatch(identity): + raise ValueError("GPU selection requires a resolved GPU UUID; use scripts/run_task.sh --gpu-device") + selected = copy.deepcopy(document) + devices = (selected.get("services", {}).get("main", {}).get("deploy", {}) + .get("resources", {}).get("reservations", {}).get("devices", [])) + if (len(devices) != 1 or devices[0].get("driver") != "nvidia" + or "gpu" not in devices[0].get("capabilities", []) + or devices[0].get("count") != 1 or "device_ids" in devices[0]): + raise ValueError("Explicit GPU selection requires a task Compose request for one NVIDIA GPU (count: 1)") + devices[0].pop("count") + devices[0]["device_ids"] = [identity] + return selected diff --git a/scripts/harbor_agents.py b/scripts/harbor_agents.py index e748375..4661525 100644 --- a/scripts/harbor_agents.py +++ b/scripts/harbor_agents.py @@ -3,6 +3,7 @@ from harbor.agents.installed.codex import Codex from harbor.agents.installed.claude_code import ClaudeCode from harbor.agents.installed.pi import Pi +from dataclasses import replace from harbor.environments.base import BaseEnvironment import json import shlex @@ -11,11 +12,7 @@ from harbor.models.trajectories.trajectory import Trajectory from scripts.pi_trajectory import convert_events - - -CODEX_VERSION = "0.157.1" -CLAUDE_CODE_VERSION = "2.1.283" -PI_VERSION = "0.87.1" +from scripts.harness_versions import CODEX_VERSION, CLAUDE_CODE_VERSION, PI_VERSION, PI_THINKING_LEVELS async def _require_version( @@ -105,6 +102,9 @@ def _resolve_auth_env(self): class PreinstalledPi(Pi): """Run the pinned Pi CLI without runtime package downloads.""" + CLI_FLAGS = [replace(flag, choices=list(PI_THINKING_LEVELS)) + if flag.kwarg == "thinking" else flag for flag in Pi.CLI_FLAGS] + async def run(self, instruction, environment, context): try: await super().run(instruction, environment, context) diff --git a/scripts/harbor_environments.py b/scripts/harbor_environments.py index b49e4be..8ff0838 100644 --- a/scripts/harbor_environments.py +++ b/scripts/harbor_environments.py @@ -20,6 +20,7 @@ from scripts.egress.config import DEFAULT_IMAGE, direct_config, exact_hosts, load_config, require_supported_harbor from scripts.egress.ownership import ProjectLock, PrivateDirectory, owner_root from environments.egress.gateway import LEASE_SECONDS +from scripts.gpu_device import GPU_UUID, select_compose_gpu # Containerd creation/removal can exceed a minute on busy hosts. These bounded @@ -53,7 +54,36 @@ def gateway_image_info(reference): return docker_json("image", "inspect", reference)[0] -class PhaseScopedDocker(DockerEnvironment): +class GpuDocker(DockerEnvironment): + """Use a per-phase copy of the task Compose file for explicit GPU selection.""" + + def __init__(self, *args, gpu_device=None, **kwargs): + if gpu_device is not None and not GPU_UUID.fullmatch(gpu_device): + raise ValueError("gpu_device must be a resolved UUID; use scripts/run_task.sh --gpu-device") + self.gpu_device = gpu_device + super().__init__(*args, **kwargs) + + @property + def _docker_compose_paths(self): + paths = super()._docker_compose_paths + if not getattr(self, "gpu_device", None): + return paths + source = self._environment_docker_compose_path + if source not in paths: + raise ValueError("The GPU task must supply environment/tests docker-compose.yaml") + document = select_compose_gpu(yaml.safe_load(source.read_text()), self.gpu_device) + # Harbor keeps --project-directory at the original environment directory, + # so relative bind mounts/build paths retain their original meaning. + directory = self.trial_paths.trial_dir / "gpu" + directory.mkdir(parents=True, exist_ok=True) + destination = directory / f"{_sanitize_docker_compose_project_name(self.session_id)}.json" + content = json.dumps(document, indent=2) + "\n" + if not destination.exists() or destination.read_text() != content: + destination.write_text(content) + return [destination if path == source else path for path in paths] + + +class PhaseScopedDocker(GpuDocker): """Linux/rootful/local, exact-host, phase-scoped direct or proxy egress.""" def __init__(self, *args, egress_config: str | None = None, egress_image: str = DEFAULT_IMAGE, diff --git a/scripts/harness_versions.py b/scripts/harness_versions.py new file mode 100644 index 0000000..dae2c7f --- /dev/null +++ b/scripts/harness_versions.py @@ -0,0 +1,6 @@ +"""Pinned coding-agent CLIs installed in every task's agent image.""" + +CODEX_VERSION = "0.164.0" +CLAUDE_CODE_VERSION = "2.1.309" +PI_VERSION = "1.4.0" +PI_THINKING_LEVELS = ("off", "minimal", "low", "medium", "high", "xhigh", "max") diff --git a/scripts/run_task.py b/scripts/run_task.py index 015a2c8..ca7f621 100755 --- a/scripts/run_task.py +++ b/scripts/run_task.py @@ -15,16 +15,18 @@ from .docker_dns import parse_servers from .task_paths import select_task, task_key from .download_assets import destination_path, read_manifest, relative_path + from .gpu_device import resolve_gpu_device, validate_gpu_device + from .harness_versions import CODEX_VERSION, CLAUDE_CODE_VERSION, PI_VERSION, PI_THINKING_LEVELS else: from docker_dns import parse_servers from task_paths import select_task, task_key from download_assets import destination_path, read_manifest, relative_path + from gpu_device import resolve_gpu_device, validate_gpu_device + from harness_versions import CODEX_VERSION, CLAUDE_CODE_VERSION, PI_VERSION, PI_THINKING_LEVELS REPO = Path(__file__).resolve().parents[1] TASKS = tuple(sorted(path.parent.name for path in (REPO / "tasks").glob("*/task.toml"))) -CODEX_VERSION = "0.157.1" -CLAUDE_CODE_VERSION = "2.1.283" CLAUDE_CODE_EFFORT_LEVELS = ("low", "medium", "high", "xhigh", "max") ANTHROPIC_HOST = "api.anthropic.com" OPENROUTER_HOST = "openrouter.ai" @@ -42,8 +44,6 @@ "CLAUDE_CODE_USE_VERTEX", "AWS_BEARER_TOKEN_BEDROCK", ) -PI_VERSION = "0.87.1" -PI_THINKING_LEVELS = ("off", "minimal", "low", "medium", "high", "xhigh") PI_MODEL_KEYS = { "deepseek/deepseek-flash": "DEEPSEEK_API_KEY", "zai/glm-5.3-flash": "ZAI_API_KEY", @@ -124,6 +124,7 @@ def main(): parser.add_argument("--egress-config", type=Path, help="Direct/proxy gateway JSON; overrides EGRESS_CONFIG. Without it, restricted tasks use the direct gateway.") parser.add_argument("--egress-image", help="Direct gateway image (pulled if missing); overrides EGRESS_IMAGE. Cannot be combined with EGRESS_CONFIG.") parser.add_argument("--output", type=Path, help="Job output directory; relative to the current directory") + parser.add_argument("--gpu-device", help="Host GPU index or full GPU UUID, shared by the agent and verifier") parser.add_argument("--dry-run", action="store_true", help="Print the command with variable references; do not launch") args = parser.parse_args() @@ -188,6 +189,13 @@ def main(): except ValueError as error: parser.error(str(error)) task_config = tomllib.loads((task / "task.toml").read_text()) + if args.gpu_device is not None: + if task_config.get("environment", {}).get("gpus", 0) != 1: + parser.error("--gpu-device requires a single-GPU task") + try: + validate_gpu_device(args.gpu_device) + except ValueError as error: + parser.error(str(error)) verifier_env = task_config.get("verifier", {}).get("env", {}) output = args.output.resolve() if args.output else REPO / "jobs" / task_key(REPO, task) command = [ @@ -353,6 +361,11 @@ def main(): if dns_servers: command.extend(["--environment-kwarg", "egress_dns=" + ",".join(dns_servers)]) + if args.gpu_device is not None: + if not use_gateway: + command[command.index("--env") + 1] = "scripts.harbor_environments:GpuDocker" + command.extend(["--environment-kwarg", f"gpu_device={args.gpu_device}"]) + # This launcher uses explicit API keys, without consulting a host auth.json. env.pop("CODEX_AUTH_JSON_PATH", None) env.pop("CODEX_FORCE_AUTH_JSON", None) @@ -390,6 +403,14 @@ def main(): if dns_servers and not use_gateway: parser.error("CONTAINER_DNS requires a restricted task or an explicit direct EGRESS_CONFIG") + if args.gpu_device is not None: + try: + identity = resolve_gpu_device(args.gpu_device) + except ValueError as error: + parser.error(str(error)) + command[command.index(f"gpu_device={args.gpu_device}")] = f"gpu_device={identity}" + print(f"GPU for agent and verifier: {identity}", flush=True) + # Harbor is a console script: changing cwd alone does not put this checkout # on its interpreter's search path for scripts.harbor_agents. existing_pythonpath = env.get("PYTHONPATH") diff --git a/scripts/tests/test_gpu_device.py b/scripts/tests/test_gpu_device.py new file mode 100644 index 0000000..f69736c --- /dev/null +++ b/scripts/tests/test_gpu_device.py @@ -0,0 +1,121 @@ +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import PropertyMock, patch + +import yaml + +from scripts.gpu_device import resolve_gpu_device, select_compose_gpu, validate_gpu_device +from scripts.harbor_environments import DockerEnvironment, GpuDocker, PhaseScopedDocker + + +REPO = Path(__file__).resolve().parents[2] +GPU_A = "GPU-aaaaaaaa-1111-2222-3333-444444444444" +GPU_B = "GPU-bbbbbbbb-1111-2222-3333-444444444444" + + +class GpuSelection(unittest.TestCase): + def test_single_index_and_uuid_only(self): + for value in ("0", "2", GPU_A): + self.assertEqual(validate_gpu_device(value), value) + for value in ("", "all", "-1", "0,1", "GPU-short", " 0", "${GPU}"): + with self.subTest(value=value), self.assertRaises(ValueError): + validate_gpu_device(value) + + def test_resolves_host_index_and_uuid(self): + inventory = SimpleNamespace(stdout=f"0, {GPU_A}\n2, {GPU_B}\n") + with patch("scripts.gpu_device.subprocess.run", return_value=inventory): + self.assertEqual(resolve_gpu_device("2"), GPU_B) + self.assertEqual(resolve_gpu_device(GPU_A), GPU_A) + with self.assertRaisesRegex(ValueError, "not present"): + resolve_gpu_device("1") + with patch("scripts.gpu_device.subprocess.run", side_effect=FileNotFoundError): + with self.assertRaisesRegex(ValueError, "nvidia-smi"): + resolve_gpu_device("0") + + def test_all_gpu_task_phases_only_replace_the_device_request(self): + import tomllib + for task in sorted((REPO / "tasks").glob("*/task.toml")): + if not tomllib.loads(task.read_text())["environment"]["gpus"]: + continue + for phase in ("environment", "tests"): + with self.subTest(task=task.parent.name, phase=phase): + source = yaml.safe_load((task.parent / phase / "docker-compose.yaml").read_text()) + original = copy.deepcopy(source) + selected = select_compose_gpu(source, GPU_B) + request = selected["services"]["main"]["deploy"]["resources"]["reservations"]["devices"][0] + self.assertEqual(request["device_ids"], [GPU_B]) + self.assertNotIn("count", request) + request.pop("device_ids") + request["count"] = 1 + self.assertEqual(selected, original) + self.assertEqual(source, original) + + def test_rejects_missing_ambiguous_or_multigpu_requests(self): + for devices in ([], [{"count": 2}], [{"driver": "nvidia", "count": 1, + "capabilities": ["gpu"], "device_ids": [GPU_A]}]): + source = {"services": {"main": {"deploy": {"resources": {"reservations": {"devices": devices}}}}}} + with self.assertRaises(ValueError): + select_compose_gpu(source, GPU_A) + + def test_phase_copies_are_independent_and_leave_source_intact(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for phase in ("environment", "tests"): + env_dir = root / phase + env_dir.mkdir() + source = env_dir / "docker-compose.yaml" + source.write_text((REPO / "tasks/code-embedding-finetuning" / phase / "docker-compose.yaml").read_text()) + before = source.read_bytes() + for cls in (GpuDocker, PhaseScopedDocker): + for run, identity in (("one", GPU_A), ("two", GPU_B)): + env = object.__new__(cls) + env.environment_dir = env_dir + env.session_id = f"{run}-{phase}" + env.gpu_device = identity + env.trial_paths = SimpleNamespace(trial_dir=root / run) + env._gateway_overlay = root / "gateway.json" + with patch.object(DockerEnvironment, "_docker_compose_paths", new_callable=PropertyMock, + return_value=[root / "base.yaml", source]): + paths = env._docker_compose_paths + selected = json.loads(paths[1].read_text()) + self.assertEqual(selected["services"]["main"]["deploy"]["resources"]["reservations"] + ["devices"][0]["device_ids"], [identity]) + if cls is PhaseScopedDocker: + self.assertEqual(paths[-1], env._gateway_overlay) + env.gpu_device = None + self.assertEqual(env._docker_compose_paths[1], source) + self.assertEqual(source.read_bytes(), before) + + def test_launcher_preview_and_cpu_rejection(self): + with tempfile.NamedTemporaryFile() as env_file: + env = {"PATH": os.environ["PATH"], "AGENT_OPENAI_BASE_URL": "https://api.example.org/v1"} + def launch(task, *options): + return subprocess.run( + [sys.executable, str(REPO / "scripts/run_task.py"), "--env-file", env_file.name, + "--task", task, "--model", "fixture", "--dry-run", *options], + env=env, capture_output=True, text=True, + ) + selected = launch("code-embedding-finetuning", "--gpu-device", "2") + self.assertEqual(selected.returncode, 0, selected.stderr) + self.assertIn("gpu_device=2", selected.stdout) + self.assertIn("PhaseScopedDocker", selected.stdout) + self.assertIn("--override-gpus 0", selected.stdout) + default = launch("code-embedding-finetuning") + self.assertEqual(default.returncode, 0, default.stderr) + self.assertNotIn("gpu_device=", default.stdout) + cpu = launch("reasoning-query-rewriting", "--gpu-device", "0") + self.assertNotEqual(cpu.returncode, 0) + self.assertIn("single-GPU task", cpu.stderr) + invalid = launch("code-embedding-finetuning", "--gpu-device", "0,1") + self.assertNotEqual(invalid.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_network_policies.py b/scripts/tests/test_network_policies.py index 4f77710..7eec3fa 100644 --- a/scripts/tests/test_network_policies.py +++ b/scripts/tests/test_network_policies.py @@ -101,9 +101,9 @@ def test_agent_images_pin_all_preinstalled_clis(self): text, ) self.assertIn("@earendil-works/pi-coding-agent@${SEARCH_SWE_PI_VERSION}", text) - self.assertIn("SEARCH_SWE_CODEX_VERSION=0.157.1", text) - self.assertIn("SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283", text) - self.assertIn("SEARCH_SWE_PI_VERSION=0.87.1", text) + self.assertIn("SEARCH_SWE_CODEX_VERSION=0.164.0", text) + self.assertIn("SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309", text) + self.assertIn("SEARCH_SWE_PI_VERSION=1.4.0", text) claude_install = text.split("&& npm install --global", 1)[1].split( "&& codex --version", 1 )[0] @@ -133,7 +133,7 @@ def test_claude_effort_levels_are_supported_by_the_harbor_adapter(self): agent = PreinstalledClaudeCode( logs_dir=Path(directory), model_name="fixture", - version="2.1.283", + version="2.1.309", reasoning_effort=effort, ) self.assertIn(f"--effort {effort}", agent.build_cli_flags()) @@ -149,14 +149,14 @@ async def exec(self, command, **kwargs): return SimpleNamespace(return_code=0, stdout=self.output) cases = ( - (PreinstalledCodex, "0.157.1", "codex-cli 0.157.1\n", "codex --version"), + (PreinstalledCodex, "0.164.0", "codex-cli 0.164.0\n", "codex --version"), ( PreinstalledClaudeCode, - "2.1.283", - "2.1.283 (Claude Code)\n", + "2.1.309", + "2.1.309 (Claude Code)\n", "claude --version", ), - (PreinstalledPi, "0.87.1", "0.87.1\n", "pi --version"), + (PreinstalledPi, "1.4.0", "1.4.0\n", "pi --version"), ) for agent_class, version, output, version_command in cases: with self.subTest(agent=agent_class.__name__): @@ -197,20 +197,20 @@ async def exec(self, command, **kwargs): version="2.1.282", ) with self.assertRaisesRegex( - RuntimeError, "Search-SWE requires Claude Code 2.1.283" + RuntimeError, "Search-SWE requires Claude Code 2.1.309" ): asyncio.run( - wrong_request.install(FakeEnvironment(0, "2.1.283 (Claude Code)")) + wrong_request.install(FakeEnvironment(0, "2.1.309 (Claude Code)")) ) wrong_installed_cli = PreinstalledClaudeCode( logs_dir=Path(directory), model_name="fixture", - version="2.1.283", + version="2.1.309", ) with self.assertRaisesRegex( RuntimeError, - "must provide Claude Code 2.1.283; found 2.1.282", + "must provide Claude Code 2.1.309; found 2.1.282", ): asyncio.run( wrong_installed_cli.install( @@ -221,7 +221,7 @@ async def exec(self, command, **kwargs): missing_cli = PreinstalledClaudeCode( logs_dir=Path(directory), model_name="fixture", - version="2.1.283", + version="2.1.309", ) with self.assertRaisesRegex(RuntimeError, "no usable CLI"): asyncio.run(missing_cli.install(FakeEnvironment(127, ""))) @@ -322,7 +322,7 @@ def test_claude_code_preview_uses_pinned_cli_and_namespaced_key(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn(secret, result.stdout + result.stderr) argv = self.argv(result) - self.assertIn("version=2.1.283", self.flag_values(argv, "--ak")) + self.assertIn("version=2.1.309", self.flag_values(argv, "--ak")) self.assertIn("reasoning_effort=xhigh", self.flag_values(argv, "--ak")) self.assertEqual( self.flag_values(argv, "--ae"), diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py index 16f79a7..886832e 100644 --- a/scripts/tests/test_release.py +++ b/scripts/tests/test_release.py @@ -353,7 +353,7 @@ def test_launcher_builds_supported_pi_commands(self): "scripts.harbor_agents:PreinstalledPi", ) self.assertEqual(argv[argv.index("-m") + 1], model) - self.assertIn("version=0.87.1", self.flag_values(argv, "--ak")) + self.assertIn("version=1.4.0", self.flag_values(argv, "--ak")) self.assertIn("thinking=xhigh", self.flag_values(argv, "--ak")) self.assertFalse(any(value.startswith(("reasoning_effort=", "config=")) for value in self.flag_values(argv, "--ak"))) @@ -392,7 +392,7 @@ def test_launcher_builds_pinned_claude_code_command(self): "scripts.harbor_agents:PreinstalledClaudeCode", ) self.assertEqual(argv[argv.index("-m") + 1], "claude-sonnet-4-6") - self.assertIn("version=2.1.283", self.flag_values(argv, "--ak")) + self.assertIn("version=2.1.309", self.flag_values(argv, "--ak")) self.assertIn("reasoning_effort=max", self.flag_values(argv, "--ak")) self.assertEqual( self.flag_values(argv, "--ae"), @@ -440,7 +440,7 @@ def test_pi_defaults_version_and_validates_environment_thinking(self): capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) argv = shlex.split(result.stdout.splitlines()[-1]) - self.assertIn("version=0.87.1", self.flag_values(argv, "--ak")) + self.assertIn("version=1.4.0", self.flag_values(argv, "--ak")) self.assertFalse(any(value.startswith("thinking=") for value in self.flag_values(argv, "--ak"))) env = self.launcher_env() diff --git a/scripts/tests/test_rewardkit_deepseek.py b/scripts/tests/test_rewardkit_deepseek.py index 2b0cac3..8f7d00e 100644 --- a/scripts/tests/test_rewardkit_deepseek.py +++ b/scripts/tests/test_rewardkit_deepseek.py @@ -1,4 +1,5 @@ import asyncio +import ast import importlib.metadata import importlib.util import json @@ -37,7 +38,7 @@ def test_every_rewardkit_task_uses_one_fixed_version_and_model(self): dockerfile = (tests / "Dockerfile").read_text() self.assertIn('"harbor-rewardkit==0.2.0"', dockerfile) self.assertIn('"litellm==1.97.0"', dockerfile) - self.assertIn("REWARDKIT_CODEX_VERSION=0.147.0", dockerfile) + self.assertIn("REWARDKIT_CODEX_VERSION=0.164.0", dockerfile) self.assertIn("@openai/codex@${REWARDKIT_CODEX_VERSION}", dockerfile) self.assertIn("registry.npmmirror.com", dockerfile) self.assertNotIn("harbor-rewardkit==0.1", dockerfile) @@ -57,6 +58,12 @@ def test_every_rewardkit_task_uses_one_fixed_version_and_model(self): self.assertNotIn("CODEX_HOME=", script) wrapper = (tests / "rewardkit_deepseek.py").read_bytes() + tree = ast.parse(wrapper) + codex_version = next(node.value.value for node in tree.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "CODEX_VERSION" + for target in node.targets)) + self.assertIn(f"REWARDKIT_CODEX_VERSION={codex_version}", dockerfile) if wrapper_bytes is None: wrapper_bytes = wrapper self.assertEqual(wrapper, wrapper_bytes) @@ -165,6 +172,10 @@ def test_generated_codex_config_is_valid_and_contains_no_key(self): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + from rewardkit import agents + self.assertEqual(agents._CODEX_VERSION, module.CODEX_VERSION) + self.assertEqual(agents._codex_cache_dir().name, module.CODEX_VERSION) + with mock.patch.dict( os.environ, {"OPENAI_BASE_URL": "https://openrouter.ai/api/v1///"}, diff --git a/tasks/agentic-search-vectorless/README.md b/tasks/agentic-search-vectorless/README.md index 0b5ae76..51444d4 100644 --- a/tasks/agentic-search-vectorless/README.md +++ b/tasks/agentic-search-vectorless/README.md @@ -1,22 +1,41 @@ # Agentic Search — Vectorless +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `agentic-search-vectorless` | +| Version | 1.0.0 | +| Authors | Jianlyu | +| Primary metric | GoldRecall@5 | +| Compute | 32 CPUs; 128 GiB RAM; 200 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 19200 s | +| Dataset | BrowseComp-Plus | +| Hidden queries | 20 | + Build retrieval over the 100,195-document BrowseComp-Plus corpus without embeddings, BM25 or external retrieval/reranking. No starter, index or entry points are supplied, and neither the language nor retrieval architecture is prescribed. Each query may make at most 20 independent retrieval rounds. Twenty public questions and TREC labels support development; twenty hidden questions are scored with mean Gold Recall@5 after a mandatory compliance audit. [raw-instruction.md](raw-instruction.md) is the structured specification. [instruction.md](instruction.md) is the complete prose request sent to the agent. The structured source is excluded from the agent's Docker context and mounts. -## Environment and verifier +## Change Log -The CPU shared image provides 32 CPUs, 128 GiB RAM and 200 GiB storage. Agent time is 7,200 seconds. The verifier discards the transferred `/app/index`, builds once as UID/GID 65534 and runs up to five query invocations sharing the fresh index and any optional build-started service. Build has a 600-second cap. Build, query execution and deterministic scoring share a 15,300-second wall-clock allowance, with no individual query timeout. The verifier has 19,200 seconds including a separate audit of up to 3,600 seconds. Any execution/output failure invalidates the entire submission. +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | -Submitted processes receive an explicit environment allowlist and the task `OPENROUTER_API_KEY`, never judge credentials. The task-local `tests/trajectory_gate.py` orchestrates retrieval and the mandatory audit. After retrieval, submission processes stop before auditing. The judge uses RewardKit 0.2.0, LiteLLM 1.97.0 and Codex 0.147.0 with `deepseek/deepseek-v4.1-flash` at `https://openrouter.ai/api/v1`. Configure host `VERIFIER_OPENAI_BASE_URL` and `VERIFIER_OPENAI_API_KEY` locally; the latter is a separate OpenRouter judge key. Missing trajectory, bad output, judge failure/timeout or a detected violation leaves zero reward. Algorithm and round-limit auditing is not a formal guarantee of detecting every violation. +## Difficulty Explanation -## Sources and structural references +Retrieval must combine evidence across a large corpus within 20 retrieval rounds per query. The resource policy excludes embeddings, BM25 and external retrieval services, so the system needs an effective corpus navigation and evidence selection strategy. -The corpus/public split has the exact current `tasks/agentic-search/assets.json` hashes and named paths, pinned to official dataset revision `6713b1eb7474e0768e906a0151c76af92f0cfcdd`. No source bytes were changed or republished. See [SOURCES.md](SOURCES.md) for provenance and usage terms. Published inputs may be reused at their original paths. The hidden split remains runtime-isolated but visible to repository readers; it is not a secret evaluation set. +## Verification Explanation -Read-only references: `tasks/agentic-search` for corpus layout, build/query lifecycle and deterministic Recall scoring, and `tasks/long-pdf-evidence` for the current trajectory-judge backend. Intentional differences: no ReAct/BM25 starter, no embeddings or BM25, no external retrieval/reranking, and a broader audit of the embedding/BM25 bans and other task constraints; the reviewed task now also has a trajectory audit. The allowed Qwen generation models remain those in this package's resource document. +### Environment and verifier -## Validation +The CPU shared image provides 32 CPUs, 128 GiB RAM and 200 GiB storage. Agent time is 7,200 seconds. The verifier discards the transferred `/app/index`, builds once as UID/GID 65534 and runs up to five query invocations sharing the fresh index and any optional build-started service. Build has a 600-second cap. Build, query execution and deterministic scoring share a 15,300-second wall-clock allowance, with no individual query timeout. The verifier has 19,200 seconds including a separate audit of up to 3,600 seconds. Any execution/output failure invalidates the entire submission. + +Submitted processes receive an explicit environment allowlist and the task `OPENROUTER_API_KEY`, never judge credentials. The task-local `tests/trajectory_gate.py` orchestrates retrieval and the mandatory audit. After retrieval, submission processes stop before auditing. The judge uses RewardKit 0.2.0, LiteLLM 1.97.0 and Codex 0.164.0 with `deepseek/deepseek-v4.1-flash` at `https://openrouter.ai/api/v1`. Configure host `VERIFIER_OPENAI_BASE_URL` and `VERIFIER_OPENAI_API_KEY` locally; the latter is a separate OpenRouter judge key. Missing trajectory, bad output, judge failure/timeout or a detected violation leaves zero reward. Algorithm and round-limit auditing is not a formal guarantee of detecting every violation. + +### Validation From the repository root, restore inputs with `python scripts/download_assets.py --task agentic-search-vectorless`. Public upstream and published official HF inputs are permitted with immutable pins, exact filenames, sizes and hashes. Cross-task official asset reuse is supported from Search-SWE `ac7dcc1`; no duplicate publication is needed. @@ -26,7 +45,7 @@ The October 15 Codex + OpenRouter `openai/gpt-6-luna`, effort `max`, trial produ The job is `jobs/evaluations/agentic-search-vectorless/codex/openai/gpt-6-luna/max/attempt-1/2026-10-15__14-05-14/`. Before the PR #17 fixes, all 28 recorded package file hashes matched the evaluated revision, apart from documentation changes. On October 17, against `ac7dcc1`, merge-ready checks, all 3 asset hashes, 22 task tests and the 174-test shared suite passed. These are historical results; v0.2.2 subsequently clarifies the audit's permitted query-input interface. Public and hidden query IDs and gold document IDs are disjoint. The author review is `contributing/dev/contribution-readiness/2026-10-17/REVIEW.md`. -### PR #17 audit clarification (v0.2.2) +#### PR #17 audit clarification (v0.2.2) The audit explicitly permits reading the query file passed to `run.sh --queries` during formal query execution. Obtaining hidden queries outside that interface, @@ -40,7 +59,7 @@ checks, release checks, and the **174-test shared suite**. The new rubric regression covers both the legitimate input exception and the retained bans. These checks are separate from the historical zero-score coding-model trial. -### Real-corpus known-good verification (October 17) +#### Real-corpus known-good verification (October 17) An author-only reference submission was frozen after public-only calibration (public Gold Recall@5 **0.115833**, 6/20 questions with hits). It reuses the earlier @@ -73,3 +92,20 @@ provenance checks and results are retained under `contributing/dev/agentic-search-vectorless/2026-10-17-pr17/`, outside the public task package. The query implementation SHA-256 is `a6beacc387cc65d219cee356563164e95e32275ccc48b48589c073d8511dea1d`. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task agentic-search-vectorless +bash scripts/run_task.sh --task agentic-search-vectorless --model YOUR_AGENT_MODEL +``` + +## References + +### Sources and structural references + +The corpus/public split has the exact current `tasks/agentic-search/assets.json` hashes and named paths, pinned to official dataset revision `6713b1eb7474e0768e906a0151c76af92f0cfcdd`. No source bytes were changed or republished. See [SOURCES.md](SOURCES.md) for provenance and usage terms. Published inputs may be reused at their original paths. The hidden split remains runtime-isolated but visible to repository readers; it is not a secret evaluation set. + +Read-only references: `tasks/agentic-search` for corpus layout, build/query lifecycle and deterministic Recall scoring, and `tasks/long-pdf-evidence` for the current trajectory-judge backend. Intentional differences: no ReAct/BM25 starter, no embeddings or BM25, no external retrieval/reranking, and a broader audit of the embedding/BM25 bans and other task constraints; the reviewed task now also has a trajectory audit. The allowed Qwen generation models remain those in this package's resource document. diff --git a/tasks/agentic-search-vectorless/environment/Dockerfile b/tasks/agentic-search-vectorless/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/agentic-search-vectorless/environment/Dockerfile +++ b/tasks/agentic-search-vectorless/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/agentic-search-vectorless/task.toml b/tasks/agentic-search-vectorless/task.toml index bf02a80..c1366e9 100644 --- a/tasks/agentic-search-vectorless/task.toml +++ b/tasks/agentic-search-vectorless/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/agentic-search-vectorless" -version = "0.2.2" +version = "1.0.0" description = "Implement an agentic retrieval system without embeddings or BM25." authors = [{ name = "Jianlyu" }] keywords = ["BrowseComp-Plus", "information-retrieval", "multi-constraint-retrieval"] diff --git a/tasks/agentic-search-vectorless/tests/Dockerfile b/tasks/agentic-search-vectorless/tests/Dockerfile index cae98f9..46726d0 100644 --- a/tasks/agentic-search-vectorless/tests/Dockerfile +++ b/tasks/agentic-search-vectorless/tests/Dockerfile @@ -7,7 +7,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --timeout 120 --retries 10 --resume-retries 10 \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/agentic-search-vectorless/tests/rewardkit_deepseek.py b/tasks/agentic-search-vectorless/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/agentic-search-vectorless/tests/rewardkit_deepseek.py +++ b/tasks/agentic-search-vectorless/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/agentic-search/README.md b/tasks/agentic-search/README.md index 59deaa7..5495eff 100644 --- a/tasks/agentic-search/README.md +++ b/tasks/agentic-search/README.md @@ -1,10 +1,21 @@ # Agentic Search -Improve a ReAct search system's document coverage on BrowseComp-Plus questions. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `agentic-search` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | GoldRecall@5 | +| Compute | 32 CPUs; 128 GiB RAM; 200 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 17100 s | +| Dataset | BrowseComp-Plus | +| Hidden queries | 20 | -**Task:** `agentic-search` · **Metric:** Gold Recall@5 +Improve a ReAct search system's document coverage on BrowseComp-Plus questions. -## Overview +### Overview BrowseComp-Plus questions often require combining several constraints. A search system may need to decompose a question, inspect intermediate results, @@ -16,16 +27,9 @@ working within a bounded number of searches. Evaluation measures the documents ultimately retrieved, not the fluency of an answer or the appearance of a trace. The current repository identifier is `agentic-search`. -## What This Task Tests +### Task Setup -- Turning compound information needs into effective search actions. -- Using observations to improve later retrieval and final document selection. -- Balancing document coverage with a finite retrieval-round budget. -- Serving concurrent questions through a shared index and search service. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -44,7 +48,7 @@ This stratified subset has a theoretical Gold Recall@5 ceiling of 1.0. It is not an estimate over the full dataset's natural question distribution. Answer labels are not used. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The corpus, five-document output, and resource limits are fixed. Indexing, document coverage, query planning, retrieval, observations, and selection may @@ -57,7 +61,7 @@ Each question may use at most **20 retrieval rounds**; independent searches in a batch count separately. A submission-generated trace is diagnostic, not trusted proof of compliance. -### Environment and Resource Limits +#### Environment and Resource Limits The CPU Python 3.12 environment provides 32 CPUs, 128 GiB memory, 200 GiB storage, and no GPU. The agent has two hours; the Harbor verifier has 17,100 seconds. @@ -68,7 +72,22 @@ there is **no individual query timeout**. A recorded local run of the offline lexical baseline completed a full-corpus build in about 38 seconds and achieved public Gold Recall@5 of `0.1741666667`. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Turning compound information needs into effective search actions. +- Using observations to improve later retrieval and final document selection. +- Balancing document coverage with a finite retrieval-round budget. +- Serving concurrent questions through a shared index and search service. + +## Verification Explanation + +### Submission Contract The deliverable contains `/app/build.sh`, `/app/run.sh`, and the implementation. The verifier rebuilds once and launches one query process per question with up @@ -79,23 +98,23 @@ Each result contains a query ID and exactly five distinct corpus IDs in `doc_ids This differs from the scored `results` objects used by several other tasks. See [instruction.md](instruction.md) for the exact contract. -## Evaluation +### Evaluation -### Search Quality +#### Search Quality For each query, Gold Recall@5 is the number of retrieved gold documents divided by its gold count. Reward is the arithmetic mean across 20 queries, in `[0, 1]`. All five positions are weighted equally; this is neither answer accuracy nor a rank-discounted metric. The evaluation report also gives a 0–100 score. -### Correctness and Resource Gates +#### Correctness and Resource Gates Failed execution, missing or extra query outputs, unknown or duplicate document IDs, or a result count other than five invalidate the whole submission. The build, shared execution budget, concurrency contract, and search-round limit remain task requirements. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward Gold Recall@5 remains the deterministic quality metric. Every run also applies `tests/jailbreak_judge/codex.toml` to the transferred agent trajectory and @@ -110,7 +129,7 @@ The evaluator writes `task-score.txt` and `task-score.json` with raw recall. includes the original 15,300-second execution budget and judge overhead. Previous deterministic baseline results do not validate the new judge integration. -## Running This Task +### Running This Task Run task-local regression tests without API calls: @@ -159,7 +178,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -171,3 +190,7 @@ assets, credentials, or hardware. | [Verifier](tests/) | Execution, output validation, and scoring implementation | | [Starter notes](environment/starter/README.md) | ReAct loop, lexical fallback, and service design | | [Resource policy](environment/docs/available_resources.md) | Allowed models and API operations | + +## References + +None. diff --git a/tasks/agentic-search/environment/Dockerfile b/tasks/agentic-search/environment/Dockerfile index 70484bf..9fab4d6 100644 --- a/tasks/agentic-search/environment/Dockerfile +++ b/tasks/agentic-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/agentic-search/task.toml b/tasks/agentic-search/task.toml index f0becf9..a26fe2c 100644 --- a/tasks/agentic-search/task.toml +++ b/tasks/agentic-search/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/agentic-search" -version = "0.6.0" +version = "1.0.0" description = "Optimize a ReAct retrieval starter to return five documents per query, scored by Gold Recall@5." authors = [{ name = "Search-SWE" }] keywords = ["BrowseComp-Plus", "information-retrieval", "multi-constraint-retrieval"] diff --git a/tasks/agentic-search/tests/Dockerfile b/tasks/agentic-search/tests/Dockerfile index cae98f9..46726d0 100644 --- a/tasks/agentic-search/tests/Dockerfile +++ b/tasks/agentic-search/tests/Dockerfile @@ -7,7 +7,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --timeout 120 --retries 10 --resume-retries 10 \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/agentic-search/tests/rewardkit_deepseek.py b/tasks/agentic-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/agentic-search/tests/rewardkit_deepseek.py +++ b/tasks/agentic-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/biomedical-graph-retrieval/README.md b/tasks/biomedical-graph-retrieval/README.md index c892090..6732dbe 100644 --- a/tasks/biomedical-graph-retrieval/README.md +++ b/tasks/biomedical-graph-retrieval/README.md @@ -1,22 +1,47 @@ # Biomedical Graph Retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `biomedical-graph-retrieval` | +| Version | 1.0.0 | +| Authors | Cooki-e | +| Primary metric | ExactSetAccuracy | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 5400 s | +| Dataset | Complete fixed biomedical graph with authored, verified questions | +| Public queries | 8 | +| Hidden queries | 20 | Author: Cooki-e. Build a reusable search system over the supplied knowledge graph and return complete sets of internal node IDs. No training or fine-tuning. The local package includes the corpus, 8 public validation questions and 20 -private test questions. Version 0.2.2 retains the original three public examples -and adds five independently verified examples: target-set intersection and +private test questions. The public examples include target-set intersection and exclusion, distinct support counting, ordinary shortest paths, exclusive-target pathways, and ordered protein-pair counting. The graph and private split are unchanged. Public and private query anchors are disjoint; the public set does not cover every private query structure. +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Queries require complete answer sets from a graph, including intersections, exclusions, counts and paths. A single missing or extra node fails exact-set scoring, and the system must finish within each query deadline. + +## Verification Explanation + +### Scoring and recorded validation + The completed v0.2.1 Codex/OpenRouter `openai/gpt-6-luna`, `max` trial scored 0/20 exact sets with valid execution, no query timeouts and a passing trajectory -audit. That trial used the previous three-example public set; it is not a -v0.2.2 model result. [VALIDATION.md](VALIDATION.md) records the public-data update, +audit. That trial used the three-example public set recorded in its original +revision. [VALIDATION.md](VALIDATION.md) records the public-data update, local checks, historical run and finalizer repair separately. Source attribution, licensing records and construction evidence remain in the authoring workspace; source dataset names are omitted from this package. @@ -48,3 +73,11 @@ python scripts/run_task.py --task biomedical-graph-retrieval --agent pi \ The last command previews a launch and does not call a model. Completed runtime evidence, revision coverage and remaining release steps are in [VALIDATION.md](VALIDATION.md). + +The executable contract is in [instruction.md](instruction.md); [raw-instruction.md](raw-instruction.md) records the structured specification. See [tests/](tests/) for the separate verifier and [task.toml](task.toml) for phase budgets. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +## References + +None. diff --git a/tasks/biomedical-graph-retrieval/environment/Dockerfile b/tasks/biomedical-graph-retrieval/environment/Dockerfile index a4d8eac..2e0f6c3 100644 --- a/tasks/biomedical-graph-retrieval/environment/Dockerfile +++ b/tasks/biomedical-graph-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/biomedical-graph-retrieval/task.toml b/tasks/biomedical-graph-retrieval/task.toml index 4cb8844..0637518 100644 --- a/tasks/biomedical-graph-retrieval/task.toml +++ b/tasks/biomedical-graph-retrieval/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/biomedical-graph-retrieval" -version = "0.2.2" +version = "1.0.0" description = "Build a graph retrieval engine that returns complete node-ID answer sets under a per-query deadline." authors = [{ name = "Cooki-e" }] keywords = ["information-retrieval", "knowledge-graph", "biomedical", "exact-set-accuracy"] diff --git a/tasks/biomedical-graph-retrieval/tests/Dockerfile b/tasks/biomedical-graph-retrieval/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/biomedical-graph-retrieval/tests/Dockerfile +++ b/tasks/biomedical-graph-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/biomedical-graph-retrieval/tests/rewardkit_deepseek.py b/tasks/biomedical-graph-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/biomedical-graph-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/biomedical-graph-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/budgeted-bimetric-retrieval/README.md b/tasks/budgeted-bimetric-retrieval/README.md index dc66fd6..35921f4 100644 --- a/tasks/budgeted-bimetric-retrieval/README.md +++ b/tasks/budgeted-bimetric-retrieval/README.md @@ -1,4 +1,15 @@ -# budgeted-bimetric-retrieval +# Budgeted Bimetric Retrieval + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `budgeted-bimetric-retrieval` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | baseline-normalized mean nDCG@10 | +| Compute | 16 CPUs; 32 GiB RAM; 64 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 16200 s | Improve retrieval over 250,000 NQ passages with 20 controlled expensive distances and two seconds per query. @@ -9,7 +20,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +A large candidate corpus must be searched with only 20 expensive distance calls and two seconds per query. Candidate generation and allocation of the limited oracle calls must improve ranking over the frozen baseline. + +## Verification Explanation + +The verifier averages nDCG@10 over all 3,452 hidden queries, including failed queries as zero, then normalizes improvement between the frozen baseline and full-corpus expensive-vector reference. Per-query oracle-call and time limits are enforced; build, resource and integrity failures give zero final reward. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -22,7 +47,7 @@ python scripts/download_assets.py --task budgeted-bimetric-retrieval --verify-on restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 32 GiB RAM and 64 GiB storage. Agent development has @@ -70,7 +95,7 @@ After every container/process for this run has exited, unmount its storage: python tasks/budgeted-bimetric-retrieval/tools/prepare_storage.py --run-id RUN_ID --unmount ``` -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/budgeted-bimetric-retrieval @@ -81,3 +106,7 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +## References + +None. diff --git a/tasks/budgeted-bimetric-retrieval/environment/Dockerfile b/tasks/budgeted-bimetric-retrieval/environment/Dockerfile index ce6ebd3..cf8f163 100644 --- a/tasks/budgeted-bimetric-retrieval/environment/Dockerfile +++ b/tasks/budgeted-bimetric-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/budgeted-bimetric-retrieval/task.toml b/tasks/budgeted-bimetric-retrieval/task.toml index 36f3df9..8a24db1 100644 --- a/tasks/budgeted-bimetric-retrieval/task.toml +++ b/tasks/budgeted-bimetric-retrieval/task.toml @@ -3,7 +3,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/budgeted-bimetric-retrieval" -version = "0.2.2" +version = "1.0.0" description = "Improve retrieval over 250,000 NQ passages with 20 controlled expensive distances and two seconds per query." authors = [{name = "jianlyu"}] keywords = ["retrieval", "bimetric", "nq", "query-budget", "cpu"] diff --git a/tasks/budgeted-bimetric-retrieval/tests/Dockerfile b/tasks/budgeted-bimetric-retrieval/tests/Dockerfile index a1dc9b9..5fdbf10 100644 --- a/tasks/budgeted-bimetric-retrieval/tests/Dockerfile +++ b/tasks/budgeted-bimetric-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/budgeted-bimetric-retrieval/tests/rewardkit_deepseek.py b/tasks/budgeted-bimetric-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/budgeted-bimetric-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/budgeted-bimetric-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/budgeted-candidate-reranking/README.md b/tasks/budgeted-candidate-reranking/README.md index 7a7f624..0c3e3cd 100644 --- a/tasks/budgeted-candidate-reranking/README.md +++ b/tasks/budgeted-candidate-reranking/README.md @@ -1,4 +1,16 @@ -# budgeted-candidate-reranking +# Budgeted Candidate Reranking + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `budgeted-candidate-reranking` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | nDCG@10 improvement normalized by fixed-candidate headroom | +| Compute | 16 CPUs; 32 GiB RAM; 40 GiB storage; 1 GPU | +| Agent / verifier time limits | 7200 s / 9000 s | +| Dataset | MS MARCO passage; all 43 judged TREC DL2019 public queries and all 54 judged DL2020 test queries | Improve fixed BM25 Top-100 passage reranking with a metered, frozen local Qwen3-1.7B model. @@ -9,7 +21,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The candidate pool and local model are fixed. The system must allocate a small weighted token budget across candidate comparisons and improve ranking while respecting strategy and inference deadlines. + +## Verification Explanation + +The verifier measures mean linear-gain nDCG@10 over all 54 hidden queries and normalizes improvement between a frozen baseline and the fixed-candidate oracle. Candidate pools, full-qrels normalization and model/token budgets remain fixed. A healthy passing trajectory audit gates the numerical reward. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -22,7 +48,7 @@ python scripts/download_assets.py --task budgeted-candidate-reranking --verify-o restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 32 GiB RAM and 40 GiB storage. Agent development has @@ -30,10 +56,8 @@ The task allocates 16 CPU cores, 32 GiB RAM and specified in the task contract; the verifier service timeout also includes its trusted setup and checks. -Both phases require one A100 40GB GPU. Reserve an idle device and configure -both Docker phases to expose that same device explicitly. Verify the actual -container mapping before running; a host CUDA_VISIBLE_DEVICES value alone is -insufficient. Release the reservation after all run containers/processes exit. +Both phases require one A100 40GB GPU. Use `--gpu-device` to select the same +host GPU for both phases. Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or authorized process environment, following the repository README. @@ -49,7 +73,7 @@ healthy passing integrity audit. See `audit-status.json` for audit health. Invoke the standard entrypoint directly: ```bash -bash scripts/run_task.sh --task budgeted-candidate-reranking --agent codex --openrouter \ +bash scripts/run_task.sh --task budgeted-candidate-reranking --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max \ --output "jobs/budgeted-candidate-reranking-codex-openrouter-luna-max-RUN_ID" ``` @@ -58,7 +82,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/budgeted-candidate-reranking @@ -69,3 +93,9 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +None. diff --git a/tasks/budgeted-candidate-reranking/environment/Dockerfile b/tasks/budgeted-candidate-reranking/environment/Dockerfile index 27f135e..0530e4d 100644 --- a/tasks/budgeted-candidate-reranking/environment/Dockerfile +++ b/tasks/budgeted-candidate-reranking/environment/Dockerfile @@ -6,9 +6,9 @@ RUN useradd --create-home --uid 10001 --user-group submission \ && mkdir -p /app /task/data /task/models /task/runtime /task/docs /work \ && chmod 0700 /task/models -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/budgeted-candidate-reranking/environment/docs/environment.md b/tasks/budgeted-candidate-reranking/environment/docs/environment.md index ef0e302..562c56f 100644 --- a/tasks/budgeted-candidate-reranking/environment/docs/environment.md +++ b/tasks/budgeted-candidate-reranking/environment/docs/environment.md @@ -1,6 +1,6 @@ # Runtime -Shared image: `docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0` (Linux amd64, CUDA 13.0). Python `/opt/conda/bin/python`, PyTorch 2.9.0+cu130, Transformers 5.15.1, Accelerate 1.15.0, psutil 7.2.2, compilers and standard retrieval packages are installed; task addition is `nvidia-ml-py==13.580.82`. The coding CLI is Codex 0.157.1. No strategy is installed under `/app`; optional replaceable baselines are under `/task/baselines`, described in `available_resources.md`. +Shared image: `docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0` (Linux amd64, CUDA 13.0). Python `/opt/conda/bin/python`, PyTorch 2.9.0+cu130, Transformers 5.15.1, Accelerate 1.15.0, psutil 7.2.2, compilers and standard retrieval packages are installed; task addition is `nvidia-ml-py==13.580.82`. The coding CLI is Codex 0.164.0. No strategy is installed under `/app`; optional replaceable baselines are under `/task/baselines`, described in `available_resources.md`. Resources: 16 CPUs, 32 GiB aggregate RAM, 40 GiB task storage including 4,079,434,577 fixed model bytes, one A100-SXM4-40GB with a 24 GiB total memory allowance and 22 GiB trusted allocator cap. The agent has 7200 seconds; final verification has 9000 seconds including the independent audit. Network is disabled for task code. The coding agent's own provider connection is managed by the harness separately. No submission APIs or judge credentials are supplied. diff --git a/tasks/budgeted-candidate-reranking/task.toml b/tasks/budgeted-candidate-reranking/task.toml index 59de6f9..80e744c 100644 --- a/tasks/budgeted-candidate-reranking/task.toml +++ b/tasks/budgeted-candidate-reranking/task.toml @@ -3,7 +3,7 @@ artifacts = ["/tmp/search-swe-submission.tar", "/logs/agent/trajectory.json"] [task] name = "search-swe/budgeted-candidate-reranking" -version = "0.2.2" +version = "1.0.0" description = "Improve fixed BM25 Top-100 passage reranking with a metered, frozen local Qwen3-1.7B model." authors = [{ name = "jianlyu" }] keywords = ["information-retrieval", "reranking", "inference-budget", "gpu"] diff --git a/tasks/budgeted-candidate-reranking/tests/Dockerfile b/tasks/budgeted-candidate-reranking/tests/Dockerfile index 2da95a1..c7e6906 100644 --- a/tasks/budgeted-candidate-reranking/tests/Dockerfile +++ b/tasks/budgeted-candidate-reranking/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/budgeted-candidate-reranking/tests/rewardkit_deepseek.py b/tasks/budgeted-candidate-reranking/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/budgeted-candidate-reranking/tests/rewardkit_deepseek.py +++ b/tasks/budgeted-candidate-reranking/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/chunqiu-temporal-retrieval/README.md b/tasks/chunqiu-temporal-retrieval/README.md index 9e2e29b..20ee982 100644 --- a/tasks/chunqiu-temporal-retrieval/README.md +++ b/tasks/chunqiu-temporal-retrieval/README.md @@ -1,15 +1,25 @@ # ChunQiu Temporal Retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. - -Author: **Bingyu**. Local task package, version **0.1.0**. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `chunqiu-temporal-retrieval` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Mean binary nDCG@10 | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 3000 s | +| Dataset | ChunQiuTR, audited event-only derivative | +| Public queries | 0 | +| Hidden queries | 778 | Retrieve original event records for reign-based dates and time windows in Classical Chinese annals. Return up to ten records from the complete corpus. The submission may use the supplied local model and the documented APIs. No starter implementation or author baseline is included in the agent files. -## Data and audit +### Data and audit Source: [official ChunQiuTR](https://github.com/xbdxwyh/ChunQiuTR/tree/392d6ec8d62de8256b3b45578f7b4484c22042ce), commit `392d6ec8d62de8256b3b45578f7b4484c22042ce`. @@ -54,7 +64,19 @@ records can still require difficult contextual temporal inference. See [LICENSE.data.md](LICENSE.data.md) for source and model notices. Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. -## Runtime and scoring +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Questions combine Classical Chinese event descriptions with reign-based dates and temporal windows. Retrieval must resolve these constraints against original records while preserving relevance under the fixed runtime limits. + +## Verification Explanation + +### Runtime and scoring One A100 40GB, 16 CPUs, 32 GiB RAM, 100 GiB storage; development is six hours. Both images use the repository GPU image. `build.sh --corpus --index-dir` has @@ -81,7 +103,7 @@ mounted read-only. Public examples are not mounted in verification. Hidden labels, grader code, audit credentials, trajectories and reward logs remain root-private. Query order and request IDs are refreshed at verification time. -## Structural references and validation +### Structural references and validation Reviewed base: `46197a43f8cbb364e72b6d63d5db873ae7d2cac8`. `tasks/reasoning-query-rewriting` supplies the full-corpus CLI and separate @@ -104,3 +126,20 @@ python scripts/check_tasks.py tasks/chunqiu-temporal-retrieval ``` Public data pins are present. See RUN_RESULTS.md for current trial evidence and VALIDATION.md for historical author-stage checks. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task chunqiu-temporal-retrieval +bash scripts/run_task.sh --task chunqiu-temporal-retrieval --gpu-device 0 --model YOUR_AGENT_MODEL +``` + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Task reference](https://aclanthology.org/2026.findings-acl.612/). diff --git a/tasks/chunqiu-temporal-retrieval/environment/Dockerfile b/tasks/chunqiu-temporal-retrieval/environment/Dockerfile index 3e8e43d..969f3a3 100644 --- a/tasks/chunqiu-temporal-retrieval/environment/Dockerfile +++ b/tasks/chunqiu-temporal-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/chunqiu-temporal-retrieval/task.toml b/tasks/chunqiu-temporal-retrieval/task.toml index 97b9237..9a571b3 100644 --- a/tasks/chunqiu-temporal-retrieval/task.toml +++ b/tasks/chunqiu-temporal-retrieval/task.toml @@ -4,7 +4,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/chunqiu-temporal-retrieval" -version = "0.1.0" +version = "1.0.0" description = "Retrieve original event records for reign-based dates and time windows in Classical Chinese annals." authors = [{ name = "Bingyu" }] keywords = ["information-retrieval", "temporal-reasoning", "classical-chinese", "gpu"] diff --git a/tasks/chunqiu-temporal-retrieval/tests/Dockerfile b/tasks/chunqiu-temporal-retrieval/tests/Dockerfile index d28ab66..61bbb89 100644 --- a/tasks/chunqiu-temporal-retrieval/tests/Dockerfile +++ b/tasks/chunqiu-temporal-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/chunqiu-temporal-retrieval/tests/rewardkit_deepseek.py b/tasks/chunqiu-temporal-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/chunqiu-temporal-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/chunqiu-temporal-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/code-context-retrieval/README.md b/tasks/code-context-retrieval/README.md index cf99dd5..b8ae912 100644 --- a/tasks/code-context-retrieval/README.md +++ b/tasks/code-context-retrieval/README.md @@ -1,12 +1,22 @@ # Code context retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. - -Author: **Bingyu**. Version: **0.1.0**. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `code-context-retrieval` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Mean core-line F1 | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 7800 s | +| Dataset | SWE-Explore, original reading-context labels | +| Public queries | 10 | +| Hidden queries | 20 | Build a reusable search engine that returns the source regions needed to understand and address an issue in a fixed repository snapshot. The target is the original SWE-Explore reading context, including relevant implementation, helpers and tests. It is not exact bug-line localization and does not require generating a patch. -## Data and interface +### Data and interface The shared `data/repos/` directory contains 51,920 complete eligible text files from 30 pinned snapshots across 22 repositories, totaling 324,317,309 bytes. Each snapshot has an anonymous `repo_` root directory; internal relative paths, file bytes and line breaks are preserved. Each query specifies exactly one snapshot to search. Repository names and source-commit mappings are not supplied as runtime metadata; source text and issue content are unchanged and may still identify their projects. @@ -16,15 +26,15 @@ Corpus eligibility matches the screening experiment: regular nonsymlink UTF-8 fi The solver supplies executable `/app/build.sh` and `/app/run.sh` with the commands in `instruction.md`. Its `/app` artifacts transfer to a fresh verifier. Public corpus and validation data are separate read-only mounts; only the corpus is mounted in the verifier. Private queries and labels are copied into the verifier image under root-only `/tests`. -## Metric and limits +## Change Log -A result contains up to five source regions and 500 distinct physical lines. Per-query line F1 is `2 * intersection / (predicted_lines + gold_lines)`, matching `(path, line)` in the requested snapshot. Overlaps are deduplicated; ordering has no effect. Every selected original gold fits the budget and has an attainable oracle score of 1. The reward is the mean over all 20 hidden questions, with no baseline rescaling. Precision and recall are diagnostics. +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | -Invalid or failed query outputs and timeouts score zero for that query. Build failure scores zero overall. A confirmed integrity violation invalidates the submission. Scoring is deterministic; no subjective answer judge is used. The API model allowlist is a task policy; the line scorer alone does not certify model/provider compliance. - -The task has 16 CPU cores, 32 GiB RAM, 100 GiB storage, no GPU and a two-hour development budget. Index construction and service readiness have 3,600 seconds. Each query invocation has 180 seconds including API waits and output writing, concurrency one. Both images use the standard CPU image. The resource documents match `agentic-search`; only its documented OpenRouter/Jina retrieval APIs and four Qwen generation models are allowed. Corpus or answer lookup on external sites is not permitted. +## Difficulty Explanation -## Difficulty evidence +### Difficulty evidence One packaged CPU/API Codex/OpenRouter `openai/gpt-6-luna`, `max` trial produced mean line F1 and reward **0.17146875932337885**. The coding agent reached its @@ -52,7 +62,30 @@ Author-side screening evaluated the same selected issues with a common five-regi This is a deliberately selected challenge subset, not a benchmark-wide estimate. Multi-turn exploration did not reliably beat reranking. The validation split is easier, especially for whole-file expansion. Original reading-context labels do not enumerate every useful alternative solution context. Difficulty is supported against the tested local implementations, not every coding agent. The per-question maximum over fixed methods was used only for screening and is not reported as an executable baseline. -## Sources and reproduction +## Verification Explanation + +### Metric and limits + +A result contains up to five source regions and 500 distinct physical lines. Per-query line F1 is `2 * intersection / (predicted_lines + gold_lines)`, matching `(path, line)` in the requested snapshot. Overlaps are deduplicated; ordering has no effect. Every selected original gold fits the budget and has an attainable oracle score of 1. The reward is the mean over all 20 hidden questions, with no baseline rescaling. Precision and recall are diagnostics. + +Invalid or failed query outputs and timeouts score zero for that query. Build failure scores zero overall. A confirmed integrity violation invalidates the submission. Scoring is deterministic; no subjective answer judge is used. The API model allowlist is a task policy; the line scorer alone does not certify model/provider compliance. + +The task has 16 CPU cores, 32 GiB RAM, 100 GiB storage, no GPU and a two-hour development budget. Index construction and service readiness have 3,600 seconds. Each query invocation has 180 seconds including API waits and output writing, concurrency one. Both images use the standard CPU image. The resource documents match `agentic-search`; only its documented OpenRouter/Jina retrieval APIs and four Qwen generation models are allowed. Corpus or answer lookup on external sites is not permitted. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task code-context-retrieval +bash scripts/run_task.sh --task code-context-retrieval --model YOUR_AGENT_MODEL +``` + +## References + +### Sources and reproduction [SWE-Explore paper](https://arxiv.org/abs/2606.07297), [official dataset](https://huggingface.co/datasets/SWE-Explore-Bench/SWE-Explore-Bench), and [official implementation](https://github.com/Qiushao-E/SWE-Explore-Bench). diff --git a/tasks/code-context-retrieval/environment/Dockerfile b/tasks/code-context-retrieval/environment/Dockerfile index a4d8eac..2e0f6c3 100644 --- a/tasks/code-context-retrieval/environment/Dockerfile +++ b/tasks/code-context-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/code-context-retrieval/task.toml b/tasks/code-context-retrieval/task.toml index a83293c..3139439 100644 --- a/tasks/code-context-retrieval/task.toml +++ b/tasks/code-context-retrieval/task.toml @@ -4,7 +4,7 @@ artifacts = ["/app"] [task] name = "search-swe/code-context-retrieval" -version = "0.1.0" +version = "1.0.0" description = "Retrieve source regions needed to understand and address an issue in a fixed repository snapshot." authors = [{ name = "Bingyu" }] keywords = ["code-search", "context-retrieval", "repository-exploration"] diff --git a/tasks/code-embedding-finetuning/README.md b/tasks/code-embedding-finetuning/README.md index 2420887..4904fc7 100644 --- a/tasks/code-embedding-finetuning/README.md +++ b/tasks/code-embedding-finetuning/README.md @@ -1,10 +1,22 @@ # Code Embedding Fine-tuning -Adapt a fixed embedding backbone to code retrieval and measure the resulting quality gain. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `code-embedding-finetuning` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | held-out Accuracy@1 improvement | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 43200 s / 5100 s | +| Dataset | code-retrieval | +| Public queries | 10 | +| Hidden queries | 200 | -**Task:** `code-embedding-finetuning` · **Metric:** Held-out Accuracy@1 improvement +Adapt a fixed embedding backbone to code retrieval and measure the resulting quality gain. -## Overview +### Overview A general embedding model may not organize natural-language requests and code documents in the way a code-search application needs. This task asks a coding @@ -15,16 +27,9 @@ The verifier applies the same embedding and exact-search procedure to the original backbone and the submission. This isolates improvement in the learned representation from changes to indexing, candidate selection, or serving code. -## What This Task Tests +### Task Setup -- Designing a useful domain adaptation process from limited supervised data. -- Choosing training objectives and sampling strategies that generalize. -- Preserving model compatibility while updating its parameters. -- Producing a reproducible, self-contained offline checkpoint. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -38,7 +43,7 @@ The hidden evaluation contains 200 queries. Public data and the backbone are mounted read-only under `/task/data` and `/task/models`; hidden questions and labels are excluded from the agent environment. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes Training choices and model weights may change. The backbone architecture, model class, structural configuration, parameter names, shapes, and count must @@ -48,7 +53,7 @@ Development uses the supplied offline data and packages. External checkpoints, datasets, labels, and answer mappings are not permitted; see the [resource policy](environment/docs/available_resources.md). -### Environment and Resource Limits +#### Environment and Resource Limits The runtime uses Python 3.12 with CUDA-enabled PyTorch. It is configured for 8 CPUs, 32 GiB memory, and 100 GiB storage. The environment and verifier @@ -62,7 +67,22 @@ followed by a separately bounded integrity check, within an 85-minute Harbor verifier phase. The submission training environment is offline; the coding agent and private judge have their own API configuration. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Designing a useful domain adaptation process from limited supervised data. +- Choosing training objectives and sampling strategies that generalize. +- Preserving model compatibility while updating its parameters. +- Producing a reproducible, self-contained offline checkpoint. + +## Verification Explanation + +### Submission Contract The artifact is a complete checkpoint at `/app/submission/model`, including the files needed by standard Transformers tokenizer and model loaders with @@ -72,9 +92,9 @@ There is no submitted retrieval CLI to optimize here: the verifier loads the checkpoint and executes its own fixed search procedure. Full compatibility requirements are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Accuracy@1 measures whether the highest-ranked code document is relevant. The grader evaluates both checkpoints on identical hidden queries, using @@ -88,20 +108,20 @@ max(0, submitted Accuracy@1 - original-backbone Accuracy@1) This is an absolute gain on the 0–1 accuracy scale, not a percentage-relative improvement and not the submitted accuracy by itself. -### Correctness and Resource Gates +#### Correctness and Resource Gates The checkpoint must load offline and match the backbone's structural signature. Invalid models, failed evaluation, and timeouts receive zero. Training-side validation results do not replace the verifier's independent comparison. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A trajectory audit gates the nonnegative quality gain. A failed audit or invalid evaluation produces zero, regardless of the apparent training result. The audit uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0 with verifier-only DeepSeek endpoint and key settings. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -115,7 +135,7 @@ The asset downloader restores both training data and the fixed backbone. ```bash python scripts/download_assets.py --task code-embedding-finetuning -bash scripts/run_task.sh --task code-embedding-finetuning --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task code-embedding-finetuning --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex, also supports Pi and Claude Code, and @@ -124,7 +144,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -136,3 +156,9 @@ assets, credentials, or hardware. | [Verifier](tests/) | Execution, output validation, and scoring implementation | | [Training starter](environment/starter/train.sh) | Initial training entry point | | [Resource policy](environment/docs/available_resources.md) | Offline development restrictions | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +None. diff --git a/tasks/code-embedding-finetuning/environment/Dockerfile b/tasks/code-embedding-finetuning/environment/Dockerfile index 86dfb8d..b83749c 100644 --- a/tasks/code-embedding-finetuning/environment/Dockerfile +++ b/tasks/code-embedding-finetuning/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/code-embedding-finetuning/task.toml b/tasks/code-embedding-finetuning/task.toml index 1b5dbfc..3b2f932 100644 --- a/tasks/code-embedding-finetuning/task.toml +++ b/tasks/code-embedding-finetuning/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/code-embedding-finetuning" -version = "0.1" +version = "1.0.0" description = "Improve a fixed-backbone code-retrieval embedder on supplied training data." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "code-retrieval", "embedding", "fine-tuning", "gpu"] diff --git a/tasks/code-embedding-finetuning/tests/Dockerfile b/tasks/code-embedding-finetuning/tests/Dockerfile index 00af6bd..7adb911 100644 --- a/tasks/code-embedding-finetuning/tests/Dockerfile +++ b/tasks/code-embedding-finetuning/tests/Dockerfile @@ -9,7 +9,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --timeout 120 --retries 10 --resume-retries 10 \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/code-embedding-finetuning/tests/rewardkit_deepseek.py b/tasks/code-embedding-finetuning/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/code-embedding-finetuning/tests/rewardkit_deepseek.py +++ b/tasks/code-embedding-finetuning/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/code-reranker-finetuning/README.md b/tasks/code-reranker-finetuning/README.md index 7b88d99..77101ed 100644 --- a/tasks/code-reranker-finetuning/README.md +++ b/tasks/code-reranker-finetuning/README.md @@ -1,10 +1,23 @@ # Code Reranker Fine-Tuning +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `code-reranker-finetuning` | +| Version | 1.0.0 | +| Authors | Jianlyu | +| Primary metric | Accuracy@5 improvement normalized by fixed-candidate headroom | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 10800 s | +| Dataset | code-retrieval | +| Hidden queries | 200 | + Fine-tune the supplied `BAAI/bge-reranker-base` weights while preserving architecture, configuration and tokenizer. The fixed first stage combines local BM25 Top-200 with local `BAAI/bge-base-en-v1.5` embedding Top-200 through equal-weight RRF (`k=60`), retaining the whole deduplicated union of 200–400 candidates. Original and submitted rerankers use exactly the same lists. Reward measures mean Accuracy@5 gain normalized by fixed-candidate headroom over all 200 hidden questions. [raw-instruction.md](raw-instruction.md) preserves the structured source, and [instruction.md](instruction.md) is the complete agent request. Only the prose instruction reaches the agent; the structured source is outside the agent Docker context and mounts. -## Inputs and environment +### Inputs and environment The corpus has 8,765 code documents and train has 5,000 records with one positive and no supplied negatives. There is no public validation split; training records may be held out locally. An editable starter samples corpus negatives, uses two-candidate cross-entropy and exports unchanged config/tokenizer bytes with safetensors. Only `/app/output/model` is transferred, and the verifier loads it offline with remote code disabled and safetensors required. Both scoring passes load the tokenizer from the trusted `/task/models/bge-reranker-base`, so additional submitted tokenizer metadata cannot change inference. The four required tokenizer files must still match the backbone byte for byte. Submitted scripts are not executed. @@ -12,23 +25,29 @@ Resources are one GPU, 8 CPUs, 32 GiB RAM and 100 GiB storage; agent time is 21, For baseline mean Accuracy@5 B, submitted mean S and candidate-hit fraction C across all hidden questions, reward is `clamp((S-B)/(C-B), 0, 1)` when C>B and zero otherwise. Every first-stage miss remains in all denominators. B and C are recalculated, not fixed to author measurements. Neural inference uses full question/document pairs, 512-token joint truncation and batch size 16, resolving equal logits by RRF order. -## Sources and structural references +## Change Log -Corpus and train reuse the exact hashes and named paths from the current formal `tasks/code-embedding-finetuning/assets.json`, pinned to official dataset revision `6713b1eb7474e0768e906a0151c76af92f0cfcdd`. The reranker retains its original `BAAI/bge-reranker-base` model revision `2cfc18c9415c912f9d8155881c133215df768a70`. The local embedding model is pinned to `BAAI/bge-base-en-v1.5` revision `a5beb1e3e68b9ab74eb54cfd186867f64f240e1a`; all eleven model/tokenizer/metadata files were downloaded and SHA-256 verified. No data/model bytes were republished. See [SOURCES.md](SOURCES.md) for provenance and usage terms. Published inputs may be reused at their original paths. The reused hidden split is runtime-isolated but visible in the repository and has a known contamination risk. +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | -Read-only references: `tasks/code-embedding-finetuning` for GPU training, corpus provenance, fixed-backbone model delivery and offline loading; `tasks/long-document-reranking` for candidate reranking and deterministic quality scoring. This task optimizes reranker weights with local embedding plus BM25 candidates, has no public validation split, and keeps its headroom-normalized reward instead of the reference metrics. +## Difficulty Explanation -## Historical author measurement +Only the reranker weights can change; retrieval candidates, architecture and tokenizer remain fixed. Training supplies positives without negatives, so the agent must learn useful distinctions that generalize to hidden queries. + +## Verification Explanation + +### Historical author measurement An earlier author-only local A100 run substituted local BGE embeddings for the API: original reranker Accuracy@5 was 1.5% (3/200), candidate ceiling 23.5% (47/200), headroom 22.0 percentage points, with 76,788 scored pairs and 350–399 candidates per query (mean 383.94). RRF before reranking scored 2.5% (5/200). The ceiling is lower than the historical embedding-only Top-500 ceiling of 28.0%. These historical results do not validate the current local inference implementation or demonstrate trained-checkpoint improvement. Retrieval parameters were not changed using those hidden scores. Historical outputs remain in author development evidence, outside the agent and public data. -## Validation +### Validation From the repository root, restore inputs with `python scripts/download_assets.py --task code-reranker-finetuning`. Public upstream and published official HF inputs are permitted with immutable pins, exact filenames, sizes and hashes. Cross-task official asset reuse is supported from Search-SWE `ac7dcc1`; no duplicate publication is needed. Use `python scripts/check_tasks.py tasks/code-reranker-finetuning`, `python scripts/download_assets.py --task code-reranker-finetuning --verify-only`, and `python -B -m unittest discover -s tasks/code-reranker-finetuning/tests -p 'test_*.py' -q`. Launch through `scripts/run_task.sh --task code-reranker-finetuning`, which preserves real Compose GPU reservations with the documented Harbor override. -### Historical 6-hour coding-model evaluation (v0.3.1) +#### Historical 6-hour coding-model evaluation (v0.3.1) The October 17 trial evaluated **v0.3.1 with the current 6-hour budget**, using Codex 0.157.1 + OpenRouter `openai/gpt-6-luna`, effort `max`, on one A100 40GB. It completed normally, without agent timeout, trial errors or retries. Agent execution took **10,901.70 seconds (3 hours 2 minutes)** and separate verification **1,398.98 seconds (23 minutes 19 seconds)**. @@ -38,11 +57,11 @@ The run used the README's `bash scripts/run_task.sh` entry directly in the right On October 17, against `ac7dcc1`, merge-ready and release checks, all 19 asset hashes, 25 task tests and the 174-test shared suite passed. The 5,000 training queries and 200 hidden query texts have no exact overlap. The earlier offline encoder smoke check also passed. The consolidated author review is `contributing/dev/contribution-readiness/2026-10-17/REVIEW.md`. -### Earlier 12-hour evaluation +#### Earlier 12-hour evaluation The October 15–16 trial used v0.3.0 with a 12-hour budget. It took 33,677.75 seconds in the agent phase and scored **0.2954545455** (submitted Accuracy@5 **0.08**). Its job is `jobs/evaluations/code-reranker-finetuning/codex/openai/gpt-6-luna/max/attempt-1/2026-10-15__14-05-36/`. A mistakenly started duplicate attempt 2 was cancelled. These records remain historical; the completed 6-hour trial above supplies the evidence for the current training budget. -### PR #17 tokenizer isolation fix (v0.3.2) +#### PR #17 tokenizer isolation fix (v0.3.2) Both model loads now obtain their tokenizer exclusively from the trusted backbone. The submitted directory is still used for model weights, and the four @@ -76,3 +95,13 @@ files retain portable one-GPU reservations. No new six-hour coding-model trial was run: the existing trained checkpoint and unchanged agent instructions are reused, while the corrected tokenizer path is covered by the new regressions and this complete regrade. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +### Sources and structural references + +Corpus and train reuse the exact hashes and named paths from the current formal `tasks/code-embedding-finetuning/assets.json`, pinned to official dataset revision `6713b1eb7474e0768e906a0151c76af92f0cfcdd`. The reranker retains its original `BAAI/bge-reranker-base` model revision `2cfc18c9415c912f9d8155881c133215df768a70`. The local embedding model is pinned to `BAAI/bge-base-en-v1.5` revision `a5beb1e3e68b9ab74eb54cfd186867f64f240e1a`; all eleven model/tokenizer/metadata files were downloaded and SHA-256 verified. No data/model bytes were republished. See [SOURCES.md](SOURCES.md) for provenance and usage terms. Published inputs may be reused at their original paths. The reused hidden split is runtime-isolated but visible in the repository and has a known contamination risk. + +Read-only references: `tasks/code-embedding-finetuning` for GPU training, corpus provenance, fixed-backbone model delivery and offline loading; `tasks/long-document-reranking` for candidate reranking and deterministic quality scoring. This task optimizes reranker weights with local embedding plus BM25 candidates, has no public validation split, and keeps its headroom-normalized reward instead of the reference metrics. diff --git a/tasks/code-reranker-finetuning/environment/Dockerfile b/tasks/code-reranker-finetuning/environment/Dockerfile index 49223db..1b26d4d 100644 --- a/tasks/code-reranker-finetuning/environment/Dockerfile +++ b/tasks/code-reranker-finetuning/environment/Dockerfile @@ -2,9 +2,9 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 ENV HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/code-reranker-finetuning/task.toml b/tasks/code-reranker-finetuning/task.toml index 40413c7..e5d70e0 100644 --- a/tasks/code-reranker-finetuning/task.toml +++ b/tasks/code-reranker-finetuning/task.toml @@ -6,7 +6,7 @@ artifacts = [ [task] name = "search-swe/code-reranker-finetuning" -version = "0.3.2" +version = "1.0.0" authors = [{ name = "Jianlyu" }] description = "Fine-tune a fixed BGE reranker against frozen BM25 and local BGE candidates fused by RRF." keywords = ["information-retrieval", "code-retrieval", "reranking", "fine-tuning", "gpu"] diff --git a/tasks/composed-image-search/README.md b/tasks/composed-image-search/README.md index a6cb5e3..40febd7 100644 --- a/tasks/composed-image-search/README.md +++ b/tasks/composed-image-search/README.md @@ -1,10 +1,22 @@ # Composed Image Search -Find matching gallery photographs from a reference image and a requested modification. +## Task Metadata -**Task:** `composed-image-search` · **Metric:** ComposedSourceQuality +| Field | Value | +| --- | --- | +| Task | `composed-image-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | ComposedSourceQuality | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 7200 s / 36000 s | +| Dataset | CIRCO and CIRR original relative captions | +| Public queries | 5 | +| Hidden queries | 4392 | -## Overview +Find matching gallery photographs from a reference image and a requested modification. + +### Overview A user can describe how an existing photograph should change without naming every detail that should stay the same. The system must combine the reference @@ -13,15 +25,9 @@ image and modification request, then rank photographs from a shared gallery. The collection combines two sources with different relevance annotations: CIRCO has multiple matching images per request, while CIRR supplies one target. -## What This Task Tests - -- Combining visual context with a natural-language modification. -- Preserving unchanged content while finding the requested difference. -- Indexing a large image gallery within GPU and execution budgets. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -44,7 +50,7 @@ contributes 4,181 original captions and 2,297 images. Image IDs from the were matched to the official captions. Requests and relevance targets were not generated for this task. CIRR requests have no shared-concept annotation. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The gallery, requests and output interface are fixed. Index design, visual representations, candidate selection and ranking are implementation choices. @@ -52,13 +58,27 @@ No precomputed gallery or query vectors and no starter are supplied. Optional OpenRouter and Jina calls follow the [resource policy](environment/docs/available_resources.md). Additional query-dependent visual inspection is limited to 100 candidates. -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has two hours; index construction has 3,600 seconds. Two query calls each have an independent 14,400-second allowance. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Combining visual context with a natural-language modification. +- Preserving unchanged content while finding the requested difference. +- Indexing a large image gallery within GPU and execution budgets. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting files under `/app`. Build once from the gallery, then accept repeated query files. @@ -72,9 +92,9 @@ temporary paths supplied to each stage. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality For CIRCO, quality is the mean of AP@5, AP@10, AP@25 and AP@50. For CIRR, it is the mean of Recall@1, Recall@5, Recall@10 and Recall@50. @@ -85,7 +105,7 @@ The source metrics retain their original positive labels. The shared gallery and equal weighting of the two sources are this task's adaptation; unlabeled alternatives are not treated as newly certified negatives. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.26812065144613684` and integer score anchor `a = 20` on the 0–100 scale: @@ -100,7 +120,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -109,7 +129,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -119,7 +139,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -135,7 +155,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task composed-image-search -bash scripts/run_task.sh --task composed-image-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task composed-image-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -143,7 +163,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/composed-image-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -156,3 +176,9 @@ credentials or hardware. Results are written under `jobs/composed-image-search/` | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/composed-image-search/environment/Dockerfile b/tasks/composed-image-search/environment/Dockerfile index 090e39c..487638b 100644 --- a/tasks/composed-image-search/environment/Dockerfile +++ b/tasks/composed-image-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/composed-image-search/task.toml b/tasks/composed-image-search/task.toml index 5e6a1d6..98fcedf 100644 --- a/tasks/composed-image-search/task.toml +++ b/tasks/composed-image-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/composed-image-search" -version = "0.5.0" +version = "1.0.0" description = "Build image search from a reference photo and a requested modification." authors = [{ name = "Haoran Jin" }] keywords = ["image-retrieval", "composition", "multimodal"] diff --git a/tasks/composed-image-search/tests/Dockerfile b/tasks/composed-image-search/tests/Dockerfile index ff89a9a..e1bd391 100644 --- a/tasks/composed-image-search/tests/Dockerfile +++ b/tasks/composed-image-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,5 +30,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/composed-image-search/tests/rewardkit_deepseek.py b/tasks/composed-image-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/composed-image-search/tests/rewardkit_deepseek.py +++ b/tasks/composed-image-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/constrained-molecular-search/README.md b/tasks/constrained-molecular-search/README.md index eec0183..db80592 100644 --- a/tasks/constrained-molecular-search/README.md +++ b/tasks/constrained-molecular-search/README.md @@ -1,4 +1,15 @@ -# constrained-molecular-search +# Constrained Molecular Search + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `constrained-molecular-search` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | Macro mean Recall@10000; public proportional-budget Recall@279 | +| Compute | 16 CPUs; 32 GiB RAM; 64 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 6300 s | Recover the truly feasible top 10,000 compounds from a complete molecular library under a two-second query deadline. @@ -9,7 +20,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The retriever must recover the true feasible top compounds, combining molecular constraints with ranking over the complete library. A two-second query deadline makes candidate pruning and exact feasibility checks interact. + +## Verification Explanation + +The verifier checks the returned compounds against the true feasible Top-10,000 from the complete library and computes macro mean Recall@10000. Preparation and query limits are enforced separately. The final numerical reward also requires a healthy passing integrity audit. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -22,7 +47,7 @@ python scripts/download_assets.py --task constrained-molecular-search --verify-o restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 32 GiB RAM and 64 GiB storage. Agent development has @@ -30,10 +55,8 @@ The task allocates 16 CPU cores, 32 GiB RAM and specified in the task contract; the verifier service timeout also includes its trusted setup and checks. -Both phases require one A100 40GB GPU. Reserve an idle device and configure -both Docker phases to expose that same device explicitly. Verify the actual -container mapping before running; a host CUDA_VISIBLE_DEVICES value alone is -insufficient. Release the reservation after all run containers/processes exit. +Both phases require one A100 40GB GPU. Use `--gpu-device` to select the same +host GPU for both phases. Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or authorized process environment, following the repository README. @@ -49,7 +72,7 @@ deterministic report; missing or unhealthy audits cannot produce a positive rewa Invoke the standard entrypoint directly: ```bash -bash scripts/run_task.sh --task constrained-molecular-search --agent codex --openrouter \ +bash scripts/run_task.sh --task constrained-molecular-search --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max \ --output "jobs/constrained-molecular-search-codex-openrouter-luna-max-RUN_ID" ``` @@ -58,7 +81,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/constrained-molecular-search @@ -69,3 +92,9 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Task reference](https://proceedings.mlr.press/v306/pedawi26a.html). diff --git a/tasks/constrained-molecular-search/environment/Dockerfile b/tasks/constrained-molecular-search/environment/Dockerfile index 84305c0..6ec386a 100644 --- a/tasks/constrained-molecular-search/environment/Dockerfile +++ b/tasks/constrained-molecular-search/environment/Dockerfile @@ -3,9 +3,9 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple \ "rdkit==2024.3.5" "duckdb==1.3.2" "tensorboard==2.20.0" "lightgbm==4.6.0" RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple "nvidia-ml-py==12.560.30" -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN --mount=type=cache,id=constrained-molecular-npm,target=/root/.npm npm install --global --ignore-scripts --no-audit --no-fund --prefer-offline \ --registry=https://registry.npmjs.org \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/constrained-molecular-search/environment/docs/environment.md b/tasks/constrained-molecular-search/environment/docs/environment.md index 5cbe16d..ee3b953 100644 --- a/tasks/constrained-molecular-search/environment/docs/environment.md +++ b/tasks/constrained-molecular-search/environment/docs/environment.md @@ -5,7 +5,7 @@ Python 3.12, PyTorch 2.9.0+cu130, CUDA 13.0 and the common Search-SWE packages. Use /opt/conda/bin/python and /opt/conda/bin/pip. Task additions are rdkit 2024.3.5, duckdb 1.3.2, tensorboard 2.20.0, lightgbm 4.6.0 and nvidia-ml-py 12.560.30. The agent image includes the repository's pinned -Codex CLI 0.157.1, Claude Code 2.1.283 and Pi 0.87.1. The base supplies Git, +Codex CLI 0.164.0, Claude Code 2.1.309 and Pi 1.4.0. The base supplies Git, C++ tools and nvcc. APEX and its GPU top-k extension are built from bundled pinned sources. diff --git a/tasks/constrained-molecular-search/task.toml b/tasks/constrained-molecular-search/task.toml index e13938c..b11d30f 100644 --- a/tasks/constrained-molecular-search/task.toml +++ b/tasks/constrained-molecular-search/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/constrained-molecular-search" -version = "0.3.3" +version = "1.0.0" description = "Recover the truly feasible top 10,000 compounds from a complete molecular library under a two-second query deadline." authors = [{ name = "jianlyu" }] keywords = ["molecular-search", "constraints", "combinatorial-library", "gpu", "top-k"] diff --git a/tasks/constrained-molecular-search/tests/Dockerfile b/tasks/constrained-molecular-search/tests/Dockerfile index c80bf73..e3af8a2 100644 --- a/tasks/constrained-molecular-search/tests/Dockerfile +++ b/tasks/constrained-molecular-search/tests/Dockerfile @@ -31,7 +31,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/constrained-molecular-search/tests/docs/environment.md b/tasks/constrained-molecular-search/tests/docs/environment.md index 5cbe16d..ee3b953 100644 --- a/tasks/constrained-molecular-search/tests/docs/environment.md +++ b/tasks/constrained-molecular-search/tests/docs/environment.md @@ -5,7 +5,7 @@ Python 3.12, PyTorch 2.9.0+cu130, CUDA 13.0 and the common Search-SWE packages. Use /opt/conda/bin/python and /opt/conda/bin/pip. Task additions are rdkit 2024.3.5, duckdb 1.3.2, tensorboard 2.20.0, lightgbm 4.6.0 and nvidia-ml-py 12.560.30. The agent image includes the repository's pinned -Codex CLI 0.157.1, Claude Code 2.1.283 and Pi 0.87.1. The base supplies Git, +Codex CLI 0.164.0, Claude Code 2.1.309 and Pi 1.4.0. The base supplies Git, C++ tools and nvcc. APEX and its GPU top-k extension are built from bundled pinned sources. diff --git a/tasks/constrained-molecular-search/tests/rewardkit_deepseek.py b/tasks/constrained-molecular-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/constrained-molecular-search/tests/rewardkit_deepseek.py +++ b/tasks/constrained-molecular-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/decision-model-training/README.md b/tasks/decision-model-training/README.md index 49700c1..26b7227 100644 --- a/tasks/decision-model-training/README.md +++ b/tasks/decision-model-training/README.md @@ -1,8 +1,19 @@ # Decision Model Training -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata -Author: Bingyu. Version: 0.1.0. Published task package; the saved coding-agent result is recorded in RUN_RESULTS.md. +| Field | Value | +| --- | --- | +| Task | `decision-model-training` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Mean exact accuracy across seven equally sized datasets | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 480 s | +| Dataset | OpenJev mixture and held-out suite | +| Hidden queries | 4200 | + +Author: Bingyu. Published task package; the saved coding-agent result is recorded in RUN_RESULTS.md. Build an offline Qwen3-0.6B-based system for categorical choices, Boolean judgments, and ordinal ratings. This task follows the user's revised OpenJev @@ -27,30 +38,19 @@ seven dataset accuracies, equal to overall accuracy because each has 600 rows. The verifier runs submitted code under UID 10001, protects labels and reward files, and stops all submission-owned processes before reading results. -## Sources and preparation +## Change Log -- [OpenJev training mixture](https://huggingface.co/datasets/s1lv3rj1nx/openjev-mixture/tree/e540d97fc37e92b493ceeb536bdb7a01e366bdc3): 323,466 original rows. -- [OpenJev held-out suite](https://huggingface.co/datasets/s1lv3rj1nx/openjev-heldout/tree/fc5fe530405de8de02e983adbc5b10795618979c): AG News, Banking77, Civil Comments toxicity, CLINC OOS, HelpSteer helpfulness, MASSIVE intent, SST5. -- [Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B/tree/c1899de289a04d12100db370d81485cdf75e47ca): exact files verified against the user-supplied local model; Apache-2.0. -- [OpenJev implementation](https://github.com/S1LV3RJ1NX/openjev/tree/91847b4c678f7d9330d8f3475e24c72c7fef73c2): author-side format and contamination checks; not provided as a solution. +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | -Sample without replacement within every source task configuration, seed -20261014. Proportional largest-remainder allocation yields exactly 80,000 rows, -with 8–742 examples per configuration. Before sampling, exclude two training -rows whose normalized state occurs in the hidden suite. The original 4,200 -hidden states, questions, option descriptions/order and typed labels are -preserved. Flatten per-example option overrides into each example's question -schema. Neither new labels nor synthetic questions are generated. The upstream -alias guard and normalized-state overlap check pass. No separate public -validation split or baseline implementation is supplied. +## Difficulty Explanation -OpenJev's assembly is Apache-2.0; constituent datasets retain their own terms. -Its held-out card explicitly flags unclear redistribution rights for AG News -and SST5. The training mixture also does not supply a complete license audit of -all underlying datasets. This package makes no claim that every row is cleared for republication. -Unresolved constituent terms remain recorded for maintainer review. +The model must generalize across categorical, Boolean and ordinal decision tasks using a fixed example mixture. The held-out suite measures exact decisions across seven equally weighted datasets under offline execution limits. -## Local validation and author records +## Verification Explanation + +### Local validation and author records Author scripts, original downloads, sampling lineage, GPU experiments and all known-answer fixtures are in @@ -69,3 +69,41 @@ bash -n tasks/decision-model-training/tests/test.sh ``` Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task decision-model-training +bash scripts/run_task.sh --task decision-model-training --gpu-device 0 --model YOUR_AGENT_MODEL +``` + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +### Sources and preparation + +- [OpenJev training mixture](https://huggingface.co/datasets/s1lv3rj1nx/openjev-mixture/tree/e540d97fc37e92b493ceeb536bdb7a01e366bdc3): 323,466 original rows. +- [OpenJev held-out suite](https://huggingface.co/datasets/s1lv3rj1nx/openjev-heldout/tree/fc5fe530405de8de02e983adbc5b10795618979c): AG News, Banking77, Civil Comments toxicity, CLINC OOS, HelpSteer helpfulness, MASSIVE intent, SST5. +- [Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B/tree/c1899de289a04d12100db370d81485cdf75e47ca): exact files verified against the user-supplied local model; Apache-2.0. +- [OpenJev implementation](https://github.com/S1LV3RJ1NX/openjev/tree/91847b4c678f7d9330d8f3475e24c72c7fef73c2): author-side format and contamination checks; not provided as a solution. + +Sample without replacement within every source task configuration, seed +20261014. Proportional largest-remainder allocation yields exactly 80,000 rows, +with 8–742 examples per configuration. Before sampling, exclude two training +rows whose normalized state occurs in the hidden suite. The original 4,200 +hidden states, questions, option descriptions/order and typed labels are +preserved. Flatten per-example option overrides into each example's question +schema. Neither new labels nor synthetic questions are generated. The upstream +alias guard and normalized-state overlap check pass. No separate public +validation split or baseline implementation is supplied. + +OpenJev's assembly is Apache-2.0; constituent datasets retain their own terms. +Its held-out card explicitly flags unclear redistribution rights for AG News +and SST5. The training mixture also does not supply a complete license audit of +all underlying datasets. This package makes no claim that every row is cleared for republication. +Unresolved constituent terms remain recorded for maintainer review. diff --git a/tasks/decision-model-training/environment/Dockerfile b/tasks/decision-model-training/environment/Dockerfile index a517a8b..52ebdd9 100644 --- a/tasks/decision-model-training/environment/Dockerfile +++ b/tasks/decision-model-training/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/decision-model-training/task.toml b/tasks/decision-model-training/task.toml index 6cf4e37..32cc3c6 100644 --- a/tasks/decision-model-training/task.toml +++ b/tasks/decision-model-training/task.toml @@ -4,7 +4,7 @@ artifacts = ["/app/submission"] [task] name = "search-swe/decision-model-training" -version = "0.1.0" +version = "1.0.0" description = "Build an offline decision model that generalizes across categorical, Boolean and ordinal tasks." authors = [{ name = "Bingyu" }] keywords = ["decision-model", "multi-task-learning", "classification", "gpu", "offline"] diff --git a/tasks/deletion-resilient-vector-search/README.md b/tasks/deletion-resilient-vector-search/README.md index 64b5c18..b2b6f74 100644 --- a/tasks/deletion-resilient-vector-search/README.md +++ b/tasks/deletion-resilient-vector-search/README.md @@ -1,4 +1,15 @@ -# deletion-resilient-vector-search +# Deletion Resilient Vector Search + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `deletion-resilient-vector-search` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | normalized log-error reduction * (0.8 + 0.2 memory efficiency) | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 86400 s | Maintain graph-vector search quality and reclaim memory through 100 deletion stages on SIFT1M and GIST1M. @@ -9,7 +20,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Repeated deletions affect graph connectivity, recall and memory use. The implementation must retain search quality while reclaiming memory across 100 deletion stages on two vector collections. + +## Verification Explanation + +The verifier replays 100 deletion stages on each dataset, measuring Recall@10 and process memory. Reward combines normalized logarithmic error reduction over the frozen tombstone reference with memory efficiency. Build, mutation, query and resource limits apply throughout the sequence. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -25,7 +50,7 @@ python tasks/deletion-resilient-vector-search/tools/prepare_truth.py --dataset g restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 64 GiB RAM and 100 GiB storage. Agent development has @@ -68,7 +93,7 @@ After every container/process for this run has exited, unmount its storage: python tasks/deletion-resilient-vector-search/tools/prepare_storage.py --run-id RUN_ID --unmount ``` -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/deletion-resilient-vector-search @@ -79,3 +104,7 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +## References + +None. diff --git a/tasks/deletion-resilient-vector-search/environment/Dockerfile b/tasks/deletion-resilient-vector-search/environment/Dockerfile index f4c6d6f..25e712c 100644 --- a/tasks/deletion-resilient-vector-search/environment/Dockerfile +++ b/tasks/deletion-resilient-vector-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/deletion-resilient-vector-search/task.toml b/tasks/deletion-resilient-vector-search/task.toml index 7926f5b..ec456cd 100644 --- a/tasks/deletion-resilient-vector-search/task.toml +++ b/tasks/deletion-resilient-vector-search/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/deletion-resilient-vector-search" -version = "0.1.6" +version = "1.0.0" description = "Maintain graph-vector search quality and reclaim memory through 100 deletion stages on SIFT1M and GIST1M." authors = [{ name = "jianlyu" }] keywords = ["vector-search", "dynamic-index", "deletion", "hnsw", "recall", "memory", "cpu"] diff --git a/tasks/deletion-resilient-vector-search/tests/Dockerfile b/tasks/deletion-resilient-vector-search/tests/Dockerfile index 37d1bc2..16bbb5d 100644 --- a/tasks/deletion-resilient-vector-search/tests/Dockerfile +++ b/tasks/deletion-resilient-vector-search/tests/Dockerfile @@ -17,7 +17,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/deletion-resilient-vector-search/tests/rewardkit_deepseek.py b/tasks/deletion-resilient-vector-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/deletion-resilient-vector-search/tests/rewardkit_deepseek.py +++ b/tasks/deletion-resilient-vector-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/document-page-search/README.md b/tasks/document-page-search/README.md index 1594461..e922f0b 100644 --- a/tasks/document-page-search/README.md +++ b/tasks/document-page-search/README.md @@ -1,10 +1,22 @@ # Multilingual Visual Document Page Search -Rank document pages for questions across several languages and document domains. +## Task Metadata -**Task:** `document-page-search` · **Metric:** DomainMacroNDCG@10 +| Field | Value | +| --- | --- | +| Task | `document-page-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | DomainMacroNDCG@10 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 7200 s / 19200 s | +| Dataset | ViDoRe V3, eight public domains | +| Public queries | 30 | +| Hidden queries | 14484 | -## Overview +Rank document pages for questions across several languages and document domains. + +### Overview Searching visual documents requires finding evidence in page images, tables and text, including when the question and document use different languages. @@ -13,15 +25,9 @@ This task indexes complete page collections and returns a ranked page list. Each request names a domain. Search covers the full supplied page collection for that domain; the target document and page are not given. -## What This Task Tests - -- Representing page text, images and layout for retrieval. -- Handling cross-language questions over a fixed page collection. -- Balancing indexing cost and ranking quality across eight domains. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -42,7 +48,7 @@ The five public question families give 30 input rows. The other 2,414 families give 14,484 held-out rows. Translations of one question stay in the same split. All page collections are shared by both splits. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The collection, question domain, relevance labels and ranked-output interface are fixed. Parsing, representations, index design and ranking are implementation @@ -50,13 +56,27 @@ choices. Page and query embeddings are not supplied; the local ColQwen2.5 resources are optional. OpenRouter and Jina access follows the [resource policy](environment/docs/available_resources.md). No starter is provided. -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has two hours. Index construction has 10,800 seconds; each of two query calls has 2,700 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Representing page text, images and layout for retrieval. +- Handling cross-language questions over a fixed page collection. +- Balancing indexing cost and ranking quality across eight domains. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with their supporting files under `/app`. Each question produces up to ten distinct page IDs from @@ -70,15 +90,15 @@ the paths supplied to the command. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Compute nDCG@10 with the original graded relevance labels and linear gains. Average query scores within each domain, then give the eight domains equal weight. Missing or invalid query results remain in the denominator. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.5362634643282924` and integer score anchor `a = 20` on the 0–100 scale: @@ -93,7 +113,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -102,7 +122,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -112,7 +132,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -128,7 +148,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task document-page-search -bash scripts/run_task.sh --task document-page-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task document-page-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -136,7 +156,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/document-page-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -149,3 +169,9 @@ credentials or hardware. Results are written under `jobs/document-page-search/`. | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/document-page-search/environment/Dockerfile b/tasks/document-page-search/environment/Dockerfile index 62cbf34..c76aa5b 100644 --- a/tasks/document-page-search/environment/Dockerfile +++ b/tasks/document-page-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/document-page-search/task.toml b/tasks/document-page-search/task.toml index 32c8eae..dd79d3c 100644 --- a/tasks/document-page-search/task.toml +++ b/tasks/document-page-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/document-page-search" -version = "0.5.0" +version = "1.0.0" description = "Build a multilingual page index over complete visual document collections." authors = [{ name = "Haoran Jin" }] keywords = ["visual-documents", "page-retrieval", "multilingual"] diff --git a/tasks/document-page-search/tests/Dockerfile b/tasks/document-page-search/tests/Dockerfile index edd538e..9b7207f 100644 --- a/tasks/document-page-search/tests/Dockerfile +++ b/tasks/document-page-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,8 +30,8 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple "colpali-engine==0.3.18" "peft==0.19.1" RUN apt-get update && apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng tesseract-ocr-fra tesseract-ocr-deu && rm -rf /var/lib/apt/lists/* diff --git a/tasks/document-page-search/tests/rewardkit_deepseek.py b/tasks/document-page-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/document-page-search/tests/rewardkit_deepseek.py +++ b/tasks/document-page-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/evolving-personal-memory/README.md b/tasks/evolving-personal-memory/README.md index ebf0ebf..6ee39e9 100644 --- a/tasks/evolving-personal-memory/README.md +++ b/tasks/evolving-personal-memory/README.md @@ -1,10 +1,21 @@ # Personal-History Search and Answering -Retrieve original conversation records and answer questions about a person's history at a specified cutoff. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `evolving-personal-memory` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | FAMA | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 21600 s | +| Dataset | Memora | +| Hidden queries | 595 | -**Task:** `evolving-personal-memory` · **Metric:** FAMA +Retrieve original conversation records and answer questions about a person's history at a specified cutoff. -## Overview +### Overview Personal facts can change across conversations. A reusable search system must identify records relevant to a question and its session cutoff, then @@ -14,16 +25,9 @@ Retrieval and answering are separate stages. The verifier loads complete original records for the IDs returned by search; the answer process has no access to the full history or index. -## What This Task Tests +### Task Setup -- Retrieving useful records from long personal histories. -- Respecting the requested person and session cutoff. -- Organizing an answer from a bounded set of complete records. -- Separating retrieval resources from per-request answer execution. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -44,7 +48,7 @@ Source criteria were deduplicated within each question by criterion text, expected decision and type. No new questions or criteria were generated. All original task types and time horizons remain in the combined data. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes Conversation records, question cutoffs and the search/answer interfaces are fixed. Indexing, retrieval and answer organization are implementation choices; @@ -54,7 +58,7 @@ for the requested person, with session IDs no later than the cutoff. Only records selected by search become answer-stage inputs. Optional APIs and stage-specific access follow the [resource policy](environment/docs/available_resources.md). -### Environment and Resource Limits +#### Environment and Resource Limits The CPU environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and no GPU. Development has two hours; the verifier has six hours. @@ -65,7 +69,22 @@ concurrently and a shared 7,200-second answer-stage budget. Records are passed in full. A 350-record request can be approximately 1.08 MB, and may require more than one permitted model call. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Retrieving useful records from long personal histories. +- Respecting the requested person and session cutoff. +- Organizing an answer from a bounded set of complete records. +- Separating retrieval resources from per-request answer execution. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh`, `/app/search.sh` and `/app/answer.sh` with their supporting files. Answer-side code and configuration live under @@ -82,9 +101,9 @@ hidden criteria and judge credentials are inaccessible. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Three independent judge configurations vote on each deduplicated criterion. MPA measures passed memory criteria, FAA measures passed forgetting criteria, @@ -98,7 +117,7 @@ Final quality is mean FAMA over 595 held-out questions. Retrieval IDs assemble t separate retrieval-quality metric. Record IDs are not appended to judged answer text. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.3712215103709065` and integer score anchor `a = 20` on the 0–100 scale: @@ -113,7 +132,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates Executable behavior, eligible record IDs, answer format, query coverage and stage limits are checked. Build and search stop before answer execution. @@ -121,7 +140,7 @@ Each answer process tree and workspace are removed before its role is reused. Transport or malformed-response failures in trusted judging invalidate the measurement and must not be interpreted as model quality. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -132,7 +151,7 @@ Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. Answer judges, where used, are distinct from trajectory auditing. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -160,7 +179,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/evolving-personal-memory/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -173,3 +192,7 @@ credentials or hardware. Results are written under `jobs/evolving-personal-memor | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/evolving-personal-memory/environment/Dockerfile b/tasks/evolving-personal-memory/environment/Dockerfile index ef60552..e14b799 100644 --- a/tasks/evolving-personal-memory/environment/Dockerfile +++ b/tasks/evolving-personal-memory/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/evolving-personal-memory/task.toml b/tasks/evolving-personal-memory/task.toml index 5823ba0..ba0d25c 100644 --- a/tasks/evolving-personal-memory/task.toml +++ b/tasks/evolving-personal-memory/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/evolving-personal-memory" -version = "1.1.0" +version = "1.0.0" description = "Build a personal-memory search and answering system over long conversation histories." authors = [{ name = "Haoran Jin" }] keywords = ["memory", "retrieval", "conversation", "question-answering"] diff --git a/tasks/evolving-personal-memory/tests/Dockerfile b/tasks/evolving-personal-memory/tests/Dockerfile index f5b3b33..2440690 100644 --- a/tasks/evolving-personal-memory/tests/Dockerfile +++ b/tasks/evolving-personal-memory/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -38,8 +38,8 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex RUN install -d -o root -g root -m 0700 /private-corpus \ && install -d -m 0755 /opt/answer \ diff --git a/tasks/evolving-personal-memory/tests/rewardkit_deepseek.py b/tasks/evolving-personal-memory/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/evolving-personal-memory/tests/rewardkit_deepseek.py +++ b/tasks/evolving-personal-memory/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/filtered-ann-retrieval/README.md b/tasks/filtered-ann-retrieval/README.md index fd3e8c2..113efeb 100644 --- a/tasks/filtered-ann-retrieval/README.md +++ b/tasks/filtered-ann-retrieval/README.md @@ -1,14 +1,26 @@ # Filtered ANN retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. - -Author: **bingyu**. Version: **0.11.1**. Migrated from +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `filtered-ann-retrieval` | +| Version | 1.0.0 | +| Authors | bingyu | +| Primary metric | Mean per-query exact filtered Top-10 correctness and latency | +| Compute | 8 CPUs; 16 GiB RAM; 64 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 5400 s | +| Dataset | NeurIPS 2023 Big-ANN Filter Track / YFCC-10M | +| Public queries | 10 | +| Hidden queries | 25 | + +Author: **bingyu**. version: **1.0.0**. Migrated from `search_swe_bingyu/task-submissions/bingyu/1-x-1` version 0.8.1. Build a persistent Faiss ANN backend returning exact nearest neighbors subject to conjunctions of metadata tags. CPU only; no embedding model or retrieval API. -## Data and provenance +### Data and provenance The corpus has 10,000,000 192-dimensional uint8 vectors and CSR metadata with 200,386 tag columns. The public split has ten queries (five retained and five @@ -30,7 +42,19 @@ Runtime `data/` is ignored by Git. The agent receives corpus, public examples an docs. Hidden queries, labels and the grader are copied into the separate verifier image. Reference implementations remain outside the task package. -## Interface and scoring +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Filtering changes the eligible neighbor set for each query. Exact filtered Top-10 results must be returned within 15 ms, requiring an index and execution strategy that handle both filter selectivity and vector similarity. + +## Verification Explanation + +### Interface and scoring See [instruction.md](instruction.md) and its structured source [raw-instruction.md](raw-instruction.md). @@ -54,7 +78,7 @@ Timing includes request transmission and response JSON decoding. Service startup is outside this timer with no independent timeout; the entire retrieval phase has a 3,900-second supervisor deadline. -## Runtime and migration +### Runtime and migration 8 CPUs, 16 GiB RAM, 64 GiB storage, no GPU; implementation time 120 minutes. Both images use `search-swe-base:cpu-py3.12-1.0.0`. Agent and submitted search code @@ -88,14 +112,14 @@ boundaries above. Version 0.11.0 changes the query split to ten public and 25 hidden cases and updates the scoring denominator. It selects five relatively easier public and twenty harder hidden cases from 27 repeatedly measured broad filter candidates, using reference-runtime medians as a difficulty proxy. -Version 0.11.1 also clears and verifies System V IPC at each execution boundary, +The verifier clears and verifies System V IPC at each execution boundary, using the owner-or-creator cleanup pattern in the reviewed `memory-constrained-dense-retrieval/tests/grader.py` as a structural precedent. The instruction, submission interfaces, corpus and time/resource limits are unchanged. Structural precedents are `memory-constrained-dense-retrieval` and `sparse-retrieval-pruning` for offline CPU retrieval and protected evaluation. -## Validation +### Validation See [VALIDATION.md](VALIDATION.md) for executed checks and unrun layers. Detailed migration reports live outside the package in @@ -105,5 +129,20 @@ The 0.11.0 split has local dataset, grader and saved-program replay checks; it has not received a fresh coding-model run or live judge call. Reference runs measure raw retrieval; without the required audit, final reward remains zero. The public assets are published at the immutable revision recorded above. -Version 0.11.1 has 22 passing grader regression tests and real subprocess IPC +The recorded v0.11.1 validation has 22 passing grader regression tests and real subprocess IPC recovery checks; no fresh coding-model trial or live judge call was run for it. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task filtered-ann-retrieval +bash scripts/run_task.sh --task filtered-ann-retrieval --model YOUR_AGENT_MODEL +``` + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/filtered-ann-retrieval/environment/Dockerfile b/tasks/filtered-ann-retrieval/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/filtered-ann-retrieval/environment/Dockerfile +++ b/tasks/filtered-ann-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/filtered-ann-retrieval/task.toml b/tasks/filtered-ann-retrieval/task.toml index cfb4c52..666b611 100644 --- a/tasks/filtered-ann-retrieval/task.toml +++ b/tasks/filtered-ann-retrieval/task.toml @@ -3,7 +3,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/filtered-ann-retrieval" -version = "0.11.1" +version = "1.0.0" description = "Build a persistent Faiss filtered ANN backend from scratch; score each of 25 hidden exact filtered Top-10 queries independently within 15 ms, then average." authors = [{ name = "bingyu" }] keywords = ["search", "faiss", "filtered-ann", "metadata", "yfcc", "indexing"] diff --git a/tasks/filtered-ann-retrieval/tests/Dockerfile b/tasks/filtered-ann-retrieval/tests/Dockerfile index bb21ca5..f73aa72 100644 --- a/tasks/filtered-ann-retrieval/tests/Dockerfile +++ b/tasks/filtered-ann-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/filtered-ann-retrieval/tests/rewardkit_deepseek.py b/tasks/filtered-ann-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/filtered-ann-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/filtered-ann-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/financial-report-qa/README.md b/tasks/financial-report-qa/README.md index cf3c34e..ddbb7c0 100644 --- a/tasks/financial-report-qa/README.md +++ b/tasks/financial-report-qa/README.md @@ -1,10 +1,22 @@ # Financial-Report Evidence Retrieval -Find the original report passages supporting financial questions over a shared annual-report collection. +## Task Metadata -**Task:** `financial-report-qa` · **Metric:** EvidenceRecall@5 +| Field | Value | +| --- | --- | +| Task | `financial-report-qa` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | EvidenceRecall@5 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 12000 s | +| Dataset | DocFinQA full reports aligned with original FinQA gold_inds | +| Public queries | 5 | +| Hidden queries | 4236 | -## Overview +Find the original report passages supporting financial questions over a shared annual-report collection. + +### Overview An analyst knows what figures need checking but may not know which report contains them. The system must search a fixed collection and return the exact @@ -13,15 +25,9 @@ original passages needed to verify each financial question. The task evaluates evidence retrieval. It does not ask the submitted system to calculate or generate a final financial answer. -## What This Task Tests - -- Searching across complete reports without a target document ID. -- Preserving original report identities and character positions. -- Covering all required evidence within a short passage budget. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -43,7 +49,7 @@ All 885 reports remain public, including reports without retained evidence. The 6,610 aligned support facts comprise seven public and 6,603 private units. Equivalent repeated passages are alternative locations for the same fact. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The reports, question/evidence split, passage interface and execution limits are fixed. Document processing, index structure, retrieval and evidence @@ -51,13 +57,27 @@ selection are implementation choices. No starter is supplied. Optional OpenRouter and Jina services follow the [resource policy](environment/docs/available_resources.md). -### Environment and Resource Limits +#### Environment and Resource Limits The CPU environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and no GPU. Development has two hours. Index construction has 1,800 seconds; each of two query calls has 3,600 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Searching across complete reports without a target document ID. +- Preserving original report identities and character positions. +- Covering all required evidence within a short passage budget. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` plus their supporting files under `/app`. Queries contain only a query ID and question. Each output @@ -71,9 +91,9 @@ writable. Generated data use the supplied index, output and temporary paths. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality For each question, EvidenceRecall@5 is the number of covered reference evidence units divided by the number of required units. Average these recalls @@ -88,7 +108,7 @@ Evidence locations are aligned to complete FinQA support text and table context, with checks on numeric values and formatting. They are not inferred by searching for the expected final answer number. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.6831800740440966` and integer score anchor `a = 20` on the 0–100 scale: @@ -103,7 +123,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query IDs, span count, report IDs, Unicode offsets and the 512-character limit. @@ -112,7 +132,7 @@ increase recall. Completed valid JSONL rows remain usable after a query timeout; missing or invalid query results score zero. A non-timeout query-command failure invalidates that call. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -122,7 +142,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -146,7 +166,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/financial-report-qa/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -159,3 +179,7 @@ credentials or hardware. Results are written under `jobs/financial-report-qa/`. | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/financial-report-qa/environment/Dockerfile b/tasks/financial-report-qa/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/financial-report-qa/environment/Dockerfile +++ b/tasks/financial-report-qa/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/financial-report-qa/task.toml b/tasks/financial-report-qa/task.toml index 058226a..f00d317 100644 --- a/tasks/financial-report-qa/task.toml +++ b/tasks/financial-report-qa/task.toml @@ -3,7 +3,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/financial-report-qa" -version = "1.1.0" +version = "1.0.0" description = "Retrieve supporting evidence and its original positions across a shared library of complete financial reports." authors = [{ name = "Haoran Jin" }] keywords = ["financial-reports", "retrieval", "evidence-localization"] diff --git a/tasks/financial-report-qa/tests/Dockerfile b/tasks/financial-report-qa/tests/Dockerfile index f9bd398..cdf7647 100644 --- a/tasks/financial-report-qa/tests/Dockerfile +++ b/tasks/financial-report-qa/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,5 +30,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/financial-report-qa/tests/rewardkit_deepseek.py b/tasks/financial-report-qa/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/financial-report-qa/tests/rewardkit_deepseek.py +++ b/tasks/financial-report-qa/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/icsi-meeting-memory/README.md b/tasks/icsi-meeting-memory/README.md index eef6d50..c1db8a1 100644 --- a/tasks/icsi-meeting-memory/README.md +++ b/tasks/icsi-meeting-memory/README.md @@ -1,10 +1,21 @@ # Compact Conversational Memory Question Answering -Build a compact memory and a retrieval-augmented system that answers questions about meeting transcripts. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `icsi-meeting-memory` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | EvidenceGroundedAnswerAccuracy | +| Compute | 8 CPUs; 8 GiB RAM; 8 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 5400 s | +| Dataset | ICSI Meeting Corpus | +| Hidden queries | 143 | -**Task:** `icsi-meeting-memory` · **Metric:** EvidenceGroundedAnswerAccuracy +Build a compact memory and a retrieval-augmented system that answers questions about meeting transcripts. -## Overview +### Overview The complete meeting history is available during development, but the submitted system must answer from a finished memory occupying at most five percent of the @@ -16,16 +27,9 @@ It contains 53,600 utterances and 719,788 whitespace-delimited words. The task evaluates both the evidence preserved in memory and the answers supported by that evidence. -## What This Task Tests +### Task Setup -- Preserving useful conversational information within a fixed memory budget. -- Retrieving relevant evidence from the prepared memory. -- Answering from the retrieved notes without consulting the original history. -- Rebuilding and running the pipeline under separate stage permissions. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -51,7 +55,7 @@ received independent human certification. The history is pinned in [assets.json](assets.json). The public examples are restored from bundled `asset-metadata/validation/` files. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The history, submission interfaces, memory budget and execution limits are fixed. Memory structure, information selection, indexing, retrieval and @@ -61,7 +65,7 @@ Local computation and Jina embedding and reranking APIs are available for memory preparation and retrieval. Only the answerer may use the four allowed OpenRouter generation models, subject to the [resource policy](environment/docs/available_resources.md). -### Environment and Resource Limits +#### Environment and Resource Limits The CPU Python 3.12 environment provides eight CPU cores, 8 GiB RAM, 8 GiB storage and no GPU. The agent has two hours; the verifier has 90 minutes. @@ -78,7 +82,22 @@ count toward this limit. The three scripts contain corpus-independent code and are outside the memory-file budget; the index is built from that memory during evaluation. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Preserving useful conversational information within a fixed memory budget. +- Retrieving relevant evidence from the prepared memory. +- Answering from the retrieved notes without consulting the original history. +- Rebuilding and running the pipeline under separate stage permissions. + +## Verification Explanation + +### Submission Contract Deliver exactly `/app/memory.json` and executable `/app/build.sh`, `/app/run.sh` and `/app/answer.sh`. Each script is self-contained. @@ -97,9 +116,9 @@ See [instruction.md](instruction.md) for the full interface and Only the rewritten instruction and the mounted data and resource documentation are exposed to the agent. -## Evaluation +### Evaluation -### Search Quality +#### Search Quality For each question, credit requires both an evidence hit and a correct answer. A hit means that the retrieved notes faithfully preserve at least one relevant @@ -113,7 +132,7 @@ EvidenceGroundedAnswerAccuracy is the mean of these joint binary outcomes over Evidence and answer accuracy are also reported separately. Semantic grading remains subject to model variability. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.06293706293706294` and integer score anchor `a = 20` on the 0–100 scale: @@ -128,7 +147,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The four files, executable permissions, JSON syntax and size, index build, script completion and output formats must satisfy the contract. A failed @@ -136,7 +155,7 @@ submitted process or invalid output invalidates the submission. Empty retrieval is an evidence miss. A malformed judge response or model-service failure invalidates the measurement rather than establishing model quality. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward The evidence and answer judges are separate from the trajectory audit. The audit checks task compliance and can set the entire reward to zero. @@ -151,7 +170,7 @@ environment filters credential variables, and an inherited Landlock policy limits filesystem access to read-only audit inputs and runtime files, with writes confined to audit state and output directories. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for model @@ -176,7 +195,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect command construction without starting an evaluation; it does not validate assets, credentials or API access. Use a fresh output directory for each run. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -190,3 +209,7 @@ assets, credentials or API access. Use a fresh output directory for each run. | [Resource policy](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [Corpus license](ICSI_LICENSE.html) | Original terms and attribution | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/icsi-meeting-memory/environment/Dockerfile b/tasks/icsi-meeting-memory/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/icsi-meeting-memory/environment/Dockerfile +++ b/tasks/icsi-meeting-memory/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/icsi-meeting-memory/task.toml b/tasks/icsi-meeting-memory/task.toml index 094aa92..931d29b 100644 --- a/tasks/icsi-meeting-memory/task.toml +++ b/tasks/icsi-meeting-memory/task.toml @@ -10,7 +10,7 @@ artifacts = [ [task] name = "search-swe/icsi-meeting-memory" -version = "0.6.0" +version = "1.0.0" description = "Build a compact conversational memory, a retriever, and a grounded answerer over long meeting transcripts." authors = [{ name = "Haoran Jin" }] keywords = ["memory", "retrieval", "conversation", "coreference", "question-answering"] diff --git a/tasks/icsi-meeting-memory/tests/Dockerfile b/tasks/icsi-meeting-memory/tests/Dockerfile index f9bd398..cdf7647 100644 --- a/tasks/icsi-meeting-memory/tests/Dockerfile +++ b/tasks/icsi-meeting-memory/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,5 +30,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/icsi-meeting-memory/tests/rewardkit_deepseek.py b/tasks/icsi-meeting-memory/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/icsi-meeting-memory/tests/rewardkit_deepseek.py +++ b/tasks/icsi-meeting-memory/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/icsi-meeting-memory/tests/test_audit_isolation.py b/tasks/icsi-meeting-memory/tests/test_audit_isolation.py index c8f95a1..76c9775 100644 --- a/tasks/icsi-meeting-memory/tests/test_audit_isolation.py +++ b/tasks/icsi-meeting-memory/tests/test_audit_isolation.py @@ -57,7 +57,7 @@ from pathlib import Path if "--version" in sys.argv: - print("codex-cli 0.147.0") + print("codex-cli 0.164.0") raise SystemExit(0) assert os.geteuid() == 10002 @@ -180,7 +180,7 @@ def test_production_launcher_and_inherited_tool_boundary(self): tmp.mkdir(mode=0o700, exist_ok=True) os.chown(tmp, 10002, 10002) - binary = Path("/home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex") + binary = Path("/home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex") self.assertTrue(binary.is_symlink(), "requires the verifier's pinned Codex link") link_target = os.readlink(binary) binary.unlink() diff --git a/tasks/incremental-vector-search/README.md b/tasks/incremental-vector-search/README.md index c9c2643..20fca01 100644 --- a/tasks/incremental-vector-search/README.md +++ b/tasks/incremental-vector-search/README.md @@ -1,4 +1,15 @@ -# incremental-vector-search +# Incremental Vector Search + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `incremental-vector-search` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | Time-gated logarithmic missed-neighbor reduction over public reference | +| Compute | 16 CPUs; 32 GiB RAM; 32 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 28800 s | Build an incremental vector index and maximize recall across five retrieval sizes under per-stage time budgets. @@ -9,7 +20,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The index must incorporate successive data additions and retain recall across five retrieval sizes. Index maintenance and query execution share strict stage budgets, so improvements must hold across the full sequence. + +## Verification Explanation + +The verifier replays incremental insertions and queries at five retrieval sizes. It counts missing exact neighbors, computes logarithmic error reduction over the fixed public reference, and aggregates the two collections. Each collection must pass its build, insertion, query and resource gates. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -23,7 +48,7 @@ python tasks/incremental-vector-search/prepare_data.py restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 32 GiB RAM and 32 GiB storage. Agent development has @@ -51,7 +76,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/incremental-vector-search @@ -62,3 +87,7 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +## References + +None. diff --git a/tasks/incremental-vector-search/environment/Dockerfile b/tasks/incremental-vector-search/environment/Dockerfile index 4d4bc27..8134a83 100644 --- a/tasks/incremental-vector-search/environment/Dockerfile +++ b/tasks/incremental-vector-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/incremental-vector-search/task.toml b/tasks/incremental-vector-search/task.toml index 476e36c..c81a912 100644 --- a/tasks/incremental-vector-search/task.toml +++ b/tasks/incremental-vector-search/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/incremental-vector-search" -version = "0.10.2" +version = "1.0.0" description = "Build an incremental vector index and maximize recall across five retrieval sizes under per-stage time budgets." authors = [{ name = "jianlyu" }] keywords = ["vector-search", "incremental-indexing", "approximate-nearest-neighbors", "cpu"] diff --git a/tasks/incremental-vector-search/tests/Dockerfile b/tasks/incremental-vector-search/tests/Dockerfile index cb942f1..4827570 100644 --- a/tasks/incremental-vector-search/tests/Dockerfile +++ b/tasks/incremental-vector-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/incremental-vector-search/tests/rewardkit_deepseek.py b/tasks/incremental-vector-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/incremental-vector-search/tests/rewardkit_deepseek.py +++ b/tasks/incremental-vector-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/instruction-following-embedding/README.md b/tasks/instruction-following-embedding/README.md index 5b1b322..2da3b78 100644 --- a/tasks/instruction-following-embedding/README.md +++ b/tasks/instruction-following-embedding/README.md @@ -1,4 +1,16 @@ -# instruction-following-embedding +# Instruction Following Embedding + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `instruction-following-embedding` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | Harmonic mean of three positive subset p-MRR values (otherwise zero), gated by six nDCG@10 checks | +| Compute | 16 CPUs; 32 GiB RAM; 32 GiB storage; 1 GPU | +| Agent / verifier time limits | 7200 s / 7200 s | +| Dataset | FollowIR Robust04/Core17/News21; topic-disjoint 20-group development and 84-group test partition | Adapt a compact E5-large-v2 query adapter within a 512KiB deployment budget while retaining the fixed base document space. @@ -9,7 +21,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +A query-side adapter must follow retrieval instructions while retaining the fixed document embedding space. The 512 KiB deployment budget, three-subset harmonic mean and six quality gates require improvements that transfer across subsets. + +## Verification Explanation + +The verifier measures instruction-sensitive p-MRR on three subsets and uses their harmonic mean when all three are positive. Six nDCG@10 retention gates require at least 90% of the fixed relevance reference. Reward normalizes the resulting metric between frozen anchors after model, resource and integrity checks. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -24,7 +50,7 @@ python tasks/instruction-following-embedding/prepare_inputs.py --verify-only restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 32 GiB RAM and 32 GiB storage. Agent development has @@ -32,10 +58,8 @@ The task allocates 16 CPU cores, 32 GiB RAM and specified in the task contract; the verifier service timeout also includes its trusted setup and checks. -Both phases require one A100 40GB GPU. Reserve an idle device and configure -both Docker phases to expose that same device explicitly. Verify the actual -container mapping before running; a host CUDA_VISIBLE_DEVICES value alone is -insufficient. Release the reservation after all run containers/processes exit. +Both phases require one A100 40GB GPU. Use `--gpu-device` to select the same +host GPU for both phases. Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or authorized process environment, following the repository README. @@ -46,7 +70,7 @@ are not submission inputs. Invoke the standard entrypoint directly: ```bash -bash scripts/run_task.sh --task instruction-following-embedding --agent codex --openrouter \ +bash scripts/run_task.sh --task instruction-following-embedding --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max \ --output "jobs/instruction-following-embedding-codex-openrouter-luna-max-RUN_ID" ``` @@ -55,7 +79,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/instruction-following-embedding @@ -66,3 +90,9 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +None. diff --git a/tasks/instruction-following-embedding/environment/Dockerfile b/tasks/instruction-following-embedding/environment/Dockerfile index daf51e5..7382e57 100644 --- a/tasks/instruction-following-embedding/environment/Dockerfile +++ b/tasks/instruction-following-embedding/environment/Dockerfile @@ -3,9 +3,9 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple "nvidia-ml-py==13.580.82" "peft==0.20.0" RUN useradd --create-home --uid 10001 --user-group submission \ && mkdir -p /app /task/corpus /task/models /task/runtime /task/public /task/docs -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/instruction-following-embedding/task.toml b/tasks/instruction-following-embedding/task.toml index 9304b41..aa1a967 100644 --- a/tasks/instruction-following-embedding/task.toml +++ b/tasks/instruction-following-embedding/task.toml @@ -3,7 +3,7 @@ artifacts = ["/var/tmp/search-swe-submission.tar", "/logs/agent/trajectory.json" [task] name = "search-swe/instruction-following-embedding" -version = "0.7.1" +version = "1.0.0" description = "Adapt a compact E5-large-v2 query adapter within a 512KiB deployment budget while retaining the fixed base document space." authors = [{ name = "jianlyu" }] keywords = ["information-retrieval", "instruction-following", "embedding", "fine-tuning", "gpu"] diff --git a/tasks/instruction-following-embedding/tests/Dockerfile b/tasks/instruction-following-embedding/tests/Dockerfile index 78e6c8e..8e07889 100644 --- a/tasks/instruction-following-embedding/tests/Dockerfile +++ b/tasks/instruction-following-embedding/tests/Dockerfile @@ -3,7 +3,7 @@ ENV LITELLM_LOCAL_MODEL_COST_MAP=true RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --index-url https://pypi.tuna.tsinghua.edu.cn/simple \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" "nvidia-ml-py==13.580.82" "peft==0.20.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/instruction-following-embedding/tests/rewardkit_deepseek.py b/tasks/instruction-following-embedding/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/instruction-following-embedding/tests/rewardkit_deepseek.py +++ b/tasks/instruction-following-embedding/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/join-aware-table-retrieval/README.md b/tasks/join-aware-table-retrieval/README.md index 69b8588..5502276 100644 --- a/tasks/join-aware-table-retrieval/README.md +++ b/tasks/join-aware-table-retrieval/README.md @@ -1,23 +1,38 @@ # Join-aware table retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `join-aware-table-retrieval` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Complete table-and-join coverage at 5 tables and 6 joins | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 4800 s | +| Hidden queries | 10 | Build a CPU search engine over 1,056 tables in 206 opaque database groups. Given a natural-language question without a database ID, return exactly five tables and at most six public column-level equality relationships. A question scores one only when an acceptable table-and-join plan is completely covered. The task score is the mean over all hidden questions. Author: **Bingyu**. -## Current revision +### Corpus and runtime -Version **0.6.0** contains **10 public validation questions and 10 hidden test questions**. Each scored response has **20 seconds of wall-clock time, including API calls**. A timeout scores zero for that question and leaves every question in the denominator. The verifier may restart the search service for subsequent questions. +The catalog contains 935 public relationships and 2,926 sample rows, at most three per table. Original semantic table and column names remain visible; table and database identities use persistent opaque IDs. The catalog is byte-identical to versions 0.4.0 and 0.5.0: SHA-256 `d2231b44cf71f2e618fbe8bbe2b3a3664220154f079fb4f96f564c6fa25a6ac4`. No catalog descriptions, rows or joins were edited to fit the new questions. Earlier uniform description and compound-primary-key repairs remain in place. Multiple `primary_key: true` columns jointly form a compound key. -| Split | Questions | Database groups | 3-table plans | 4-table plans | 5-table plans | 2 / 3 / 4 / 5 / 6 required equalities | -| --- | ---: | ---: | ---: | ---: | ---: | --- | -| Validation | 10 | 9 | 0 | 0 | 10 | 0 / 0 / 4 / 2 / 4 | -| Test | 10 | 7 | 1 | 1 | 8 | 1 / 0 / 3 / 2 / 4 | +Both images use `docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0`. Resources remain 16 CPUs, 32 GiB RAM and no GPU. Build/readiness/scored-response limits are 1,200/30/20 seconds, with an 8 GiB index. Coding and whole-verifier budgets remain 7,200 and 4,800 seconds respectively. Queries are sent individually; time cannot be pooled across questions. -All 20 table-and-join structures are distinct. Database and schema-family overlap is allowed; this is a performance-conditioned challenge, not a database-held-out benchmark. Public examples require multiple semantic roles, bridge tables or composite relationships. Full databases, author SQL, identity mappings, selection scripts and baselines are not supplied to the coding agent. +The agent starts with an empty `/app/submission`; only that directory transfers to a fresh verifier. The verifier builds a new index and runs submitted commands as UID 10001. Hidden questions, labels and reward files remain protected. One synthetic warm-up precedes scored work. This revision fixes an earlier recovery bug: a warm-up failure is recorded, the process is restarted, and the actual query still receives its own full deadline instead of losing a point before it was sent. -Every selected question has an executable, nonempty SQL witness and a manual review of its requested fields, table roles and public relationships. Checks cover declared columns, positive scoring, missing tables, missing equalities, wrong-role edges and alias-aware equivalent edge replacements. These checks support the labels; they are not an exhaustive proof of every possible SQL equivalence. Repeated aliases share a base table ID, while distinct required column equalities remain separate. +The CPU/API resource documents remain byte-identical to `tasks/agentic-search`. OpenRouter/Jina retrieval APIs and the four listed Qwen generation models remain available during indexing and queries. The archived newer engine uses Qwen 3.5 for query-time selection; Luna is the coding-agent model, not the retrieval model used on every query. + +## Change Log -## Difficulty evidence and its limits +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +### Difficulty evidence and its limits Selection uses the two unchanged programs produced in the earlier Codex/OpenRouter `openai/gpt-6-luna`, `max` trials, plus three fixed lexical baselines. Every test question has a recorded wrong response with available API output in **both** archived programs. API failures and timeouts alone do not qualify. The older program additionally needs a timestamp-aligned returned selector dictionary to count as a confirmed API response. @@ -37,7 +52,7 @@ The subset scores above were used for selection and are **not an independent est The fresh program used one Jina table-reranking request followed by local five-table subset and join-graph search, with a local fallback. It made no Qwen generation calls at query time. Its final public development self-tests scored 10/10 both online and offline; the checker was reviewed, but raw predictions were not saved and the development script did not enforce the wall-clock deadline. These are development scores, not an independent validation replay. -The fresh program solved `shipment_header_and_item_orders`; the earlier success on `municipal_occupancy` remains in the evidence. Consequently **eight of ten** have never been observed solved across all three programs, and the fresh program failed nine. This ten-question challenge has measured difficulty, not a claim that every question defeats every implementation. The full result is in `task_results/join-aware-table-retrieval/codex-openrouter-gpt-6-luna-max-v060-20261014-232013/REPORT.txt`, with per-query records in `query-results.json`. The requested model and effort were verified from the actual session. No score-based rerun occurred. This README was updated after evaluation; questions, labels, corpus, instructions and runtime files remain unchanged from the frozen trial. +The fresh program solved `shipment_header_and_item_orders`; the earlier success on `municipal_occupancy` remains in the evidence. Consequently **eight of ten** have never been observed solved across all three programs, and the fresh program failed nine. This ten-question challenge has measured difficulty, not a claim that every question defeats every implementation. The full result is in `task_results/join-aware-table-retrieval/codex-openrouter-gpt-6-luna-max-v060-20261014-232013/REPORT.txt`, with per-query records in `query-results.json`. The requested model and effort were verified from the actual session. No score-based rerun occurred. The original evaluated revision and fingerprints are retained in the validation records. [VALIDATION.md](VALIDATION.md) consolidates the trial ID, UTC timestamps, instruction fingerprint, configured budgets, raw/final scores and original @@ -46,26 +61,31 @@ non-README files in the frozen run manifest still match. This existing trial is reused without rerunning the coding agent. No model judge is configured for this task; the HTTP-observability limitation above is retained. -## Earlier validation sets +## Verification Explanation -| Validation set and measurement | Old archived program | New archived program | -| --- | ---: | ---: | -| Original 10 questions, current 20-second replay | 8/10 | 0/10 | -| Previous 20 questions, earlier 60-second measurements | 13/20 | 20/20 | +### Evaluation split and query limits -The original-10 replay had API warnings on four old-program questions and two new-program questions, plus three new-program timeouts. Its repeated question structures are not counted as independent difficult cases. The earlier old-program 20-question measurement had seven API warnings. The newer 20/20 consists of coding-agent development predictions subsequently rescored with the formal scorer; it was not an independent held-out replay. Different deadlines and provenance prevent treating these rows as a controlled score comparison. Detailed evidence is in `hard-v060/historical-validation-comparison.json`. +The task contains **10 public validation questions and 10 hidden test questions**. Each scored response has **20 seconds of wall-clock time, including API calls**. A timeout scores zero for that question and leaves every question in the denominator. The verifier may restart the search service for subsequent questions. + +| Split | Questions | Database groups | 3-table plans | 4-table plans | 5-table plans | 2 / 3 / 4 / 5 / 6 required equalities | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Validation | 10 | 9 | 0 | 0 | 10 | 0 / 0 / 4 / 2 / 4 | +| Test | 10 | 7 | 1 | 1 | 8 | 1 / 0 / 3 / 2 / 4 | -## Corpus and runtime +All 20 table-and-join structures are distinct. Database and schema-family overlap is allowed; this is a performance-conditioned challenge, not a database-held-out benchmark. Public examples require multiple semantic roles, bridge tables or composite relationships. Full databases, author SQL, identity mappings, selection scripts and baselines are not supplied to the coding agent. -The catalog contains 935 public relationships and 2,926 sample rows, at most three per table. Original semantic table and column names remain visible; table and database identities use persistent opaque IDs. The catalog is byte-identical to versions 0.4.0 and 0.5.0: SHA-256 `d2231b44cf71f2e618fbe8bbe2b3a3664220154f079fb4f96f564c6fa25a6ac4`. No catalog descriptions, rows or joins were edited to fit the new questions. Earlier uniform description and compound-primary-key repairs remain in place. Multiple `primary_key: true` columns jointly form a compound key. +Every selected question has an executable, nonempty SQL witness and a manual review of its requested fields, table roles and public relationships. Checks cover declared columns, positive scoring, missing tables, missing equalities, wrong-role edges and alias-aware equivalent edge replacements. These checks support the labels; they are not an exhaustive proof of every possible SQL equivalence. Repeated aliases share a base table ID, while distinct required column equalities remain separate. -Both images use `docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0`. Resources remain 16 CPUs, 32 GiB RAM and no GPU. Build/readiness/scored-response limits are 1,200/30/20 seconds, with an 8 GiB index. Coding and whole-verifier budgets remain 7,200 and 4,800 seconds respectively. Queries are sent individually; time cannot be pooled across questions. +### Earlier validation sets -The agent starts with an empty `/app/submission`; only that directory transfers to a fresh verifier. The verifier builds a new index and runs submitted commands as UID 10001. Hidden questions, labels and reward files remain protected. One synthetic warm-up precedes scored work. This revision fixes an earlier recovery bug: a warm-up failure is recorded, the process is restarted, and the actual query still receives its own full deadline instead of losing a point before it was sent. +| Validation set and measurement | Old archived program | New archived program | +| --- | ---: | ---: | +| Original 10 questions, current 20-second replay | 8/10 | 0/10 | +| Previous 20 questions, earlier 60-second measurements | 13/20 | 20/20 | -The CPU/API resource documents remain byte-identical to `tasks/agentic-search`. OpenRouter/Jina retrieval APIs and the four listed Qwen generation models remain available during indexing and queries. The archived newer engine uses Qwen 3.5 for query-time selection; Luna is the coding-agent model, not the retrieval model used on every query. +The original-10 replay had API warnings on four old-program questions and two new-program questions, plus three new-program timeouts. Its repeated question structures are not counted as independent difficult cases. The earlier old-program 20-question measurement had seven API warnings. The newer 20/20 consists of coding-agent development predictions subsequently rescored with the formal scorer; it was not an independent held-out replay. Different deadlines and provenance prevent treating these rows as a controlled score comparison. Detailed evidence is in `hard-v060/historical-validation-comparison.json`. -## Author records and reproduction +### Author records and reproduction The frozen source of the split is `task_development/join-aware-table-retrieval/hard-v060/selection.json`. `combined-evidence.json` retains successes, healthy errors, infrastructure failures, deadlines and source paths. `selected-confirmation.json` contains the selected retrospective subset; `selected-baselines.json` contains all lexical outputs. The complete pre-change package and its historical README are preserved in `package-build/revisions/pre-hard10-0.5.0/task/`. @@ -88,6 +108,21 @@ python scripts/check_tasks.py tasks/join-aware-table-retrieval Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. -## Attribution +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task join-aware-table-retrieval +bash scripts/run_task.sh --task join-aware-table-retrieval --model YOUR_AGENT_MODEL +``` + +## References + +### Attribution Source: [Spider 1.0](https://yale-lily.github.io/spider), by Tao Yu and collaborators, under [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). The captured official archive has 205,800,266 bytes and SHA-256 `00636695dabed6b5f4b8328a16b13e069a2f16591d5efcce57660669c85b121b`. Attribution and adaptation details are in `LICENSE.data.md`. + +- [Task reference](https://arxiv.org/abs/1809.08887). diff --git a/tasks/join-aware-table-retrieval/environment/Dockerfile b/tasks/join-aware-table-retrieval/environment/Dockerfile index 2d4c7b1..0c0c29c 100644 --- a/tasks/join-aware-table-retrieval/environment/Dockerfile +++ b/tasks/join-aware-table-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/join-aware-table-retrieval/task.toml b/tasks/join-aware-table-retrieval/task.toml index a9d10ff..7335dda 100644 --- a/tasks/join-aware-table-retrieval/task.toml +++ b/tasks/join-aware-table-retrieval/task.toml @@ -6,7 +6,7 @@ artifacts = [ [task] name = "search-swe/join-aware-table-retrieval" -version = "0.6.0" +version = "1.0.0" description = "Build a catalog search engine that retrieves all tables and column-level joins needed to answer an unseen question." authors = [{ name = "Bingyu" }] keywords = ["information-retrieval", "table-retrieval", "join-search"] diff --git a/tasks/khmer-embedding-training/README.md b/tasks/khmer-embedding-training/README.md index 0ce1e07..9e68204 100644 --- a/tasks/khmer-embedding-training/README.md +++ b/tasks/khmer-embedding-training/README.md @@ -1,10 +1,22 @@ # Khmer-to-English Embedding Retrieval -Adapt a supplied embedding backbone and retrieve English documents for Khmer questions. +## Task Metadata -**Task:** `khmer-embedding-training` · **Metric:** Recall@100 +| Field | Value | +| --- | --- | +| Task | `khmer-embedding-training` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | Recall@100 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 48000 s | +| Dataset | MKQA Khmer / BEIR NQ | +| Public queries | 3 | +| Hidden queries | 1000 | -## Overview +Adapt a supplied embedding backbone and retrieve English documents for Khmer questions. + +### Overview Cross-language retrieval must connect a Khmer question to an English document even when they share no words. This task supplies a large English @@ -14,15 +26,9 @@ Development can use an additional local generative model. That model is absent from evaluation, where the trained embedding checkpoint and submitted search pipeline must operate offline. -## What This Task Tests - -- Adapting cross-language representations with limited labeled examples. -- Building a reusable index over 2,681,468 documents. -- Packaging an offline checkpoint and deterministic ranked retrieval. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -45,7 +51,7 @@ generative checkpoint is Qwen/Qwen3.5-9B. Upstream model revisions and restored files are recorded in [assets.json](assets.json). No extra labeled training questions are supplied. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The embedding backbone architecture, parameter names and shapes, configuration and tokenizer are fixed. Its weights are trainable. Training-data preparation, @@ -55,14 +61,28 @@ No starter is supplied, and no external API or network resource is available. The optional generative model is a development resource. Neither it nor an already built corpus index is transferred as a runtime retrieval resource. -### Environment and Resource Limits +#### Environment and Resource Limits Development has six hours with eight CPU cores, 32 GiB RAM, 100 GiB storage and one A100 40 GiB GPU. The verifier provides the same CPU, RAM and GPU resources without the development-only generative model. Index construction has 43,200 seconds; each of two query calls has 1,800 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Adapting cross-language representations with limited labeled examples. +- Building a reusable index over 2,681,468 documents. +- Packaging an offline checkpoint and deterministic ranked retrieval. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh`, `/app/run.sh` and the trained checkpoint under `/app/submission/model/`, including `model.safetensors`, `config.json` @@ -78,15 +98,15 @@ its parent is writable. After construction, retrieval sees a read-only index. The full commands and output contract are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Recall@100 uses the source answer-token hit rule: a question succeeds when at least one of the first 100 retrieved documents contains an accepted answer. Recall@20 is diagnostic. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.595` and integer score anchor `a = 15` on the 0–100 scale: @@ -101,7 +121,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks local checkpoint loading, preserved architecture and tokenizer, a successful fresh build, complete query coverage, 100 valid unique @@ -109,7 +129,7 @@ positions per query, finite sorted scores and time limits. Hidden questions, answer aliases, the generative model and judge credentials are unavailable to submitted build and retrieval processes. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -119,7 +139,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -134,7 +154,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task khmer-embedding-training -bash scripts/run_task.sh --task khmer-embedding-training --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task khmer-embedding-training --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -142,7 +162,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/khmer-embedding-training/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -155,3 +175,9 @@ credentials or hardware. Results are written under `jobs/khmer-embedding-trainin | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/khmer-embedding-training/environment/Dockerfile b/tasks/khmer-embedding-training/environment/Dockerfile index a517a8b..52ebdd9 100644 --- a/tasks/khmer-embedding-training/environment/Dockerfile +++ b/tasks/khmer-embedding-training/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/khmer-embedding-training/task.toml b/tasks/khmer-embedding-training/task.toml index 9df03c2..ba193f1 100644 --- a/tasks/khmer-embedding-training/task.toml +++ b/tasks/khmer-embedding-training/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/khmer-embedding-training" -version = "0.8.0" +version = "1.0.0" description = "Train a Khmer-to-English embedding model and build a reusable document retrieval system." authors = [{ name = "Haoran Jin" }] keywords = ["information-retrieval", "cross-lingual", "khmer", "embedding", "training", "gpu"] diff --git a/tasks/khmer-embedding-training/tests/Dockerfile b/tasks/khmer-embedding-training/tests/Dockerfile index ee06918..214595f 100644 --- a/tasks/khmer-embedding-training/tests/Dockerfile +++ b/tasks/khmer-embedding-training/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -34,5 +34,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/khmer-embedding-training/tests/rewardkit_deepseek.py b/tasks/khmer-embedding-training/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/khmer-embedding-training/tests/rewardkit_deepseek.py +++ b/tasks/khmer-embedding-training/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/large-scale-formula-retrieval/README.md b/tasks/large-scale-formula-retrieval/README.md index 673dd77..0b94534 100644 --- a/tasks/large-scale-formula-retrieval/README.md +++ b/tasks/large-scale-formula-retrieval/README.md @@ -1,4 +1,17 @@ -# large-scale-formula-retrieval +# Large Scale Formula Retrieval + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `large-scale-formula-retrieval` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | nDCG_prime | +| Compute | 2 CPUs; 4 GiB RAM; 64 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 19800 s | +| Public queries | 5 | +| Hidden queries | 76 | Retrieve relevant formulas from the complete ARQMath v3 corpus with 34 MiB prepared artifacts in a 40 MiB serving filesystem and 50 ms CPU queries. @@ -9,7 +22,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The complete formula corpus must be searchable with only 34 MiB of prepared artifacts, a 40 MiB serving filesystem and 50 ms CPU queries. Representation, indexing and ranking must balance mathematical relevance with these limits. + +## Verification Explanation + +The verifier maps returned formula instances to visual IDs, deduplicates them and computes nDCG prime using the official judgments. It averages across all 76 hidden questions, including failures, with compact-serving storage and runtime limits enforced. Final reward also requires the integrity audit to pass. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -23,7 +50,7 @@ restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document and [SOURCES.md](SOURCES.md). -## Run +### Run The task allocates 2 CPU cores, 4 GiB RAM and 64 GiB storage. Agent development has @@ -76,7 +103,7 @@ After every container/process for this run has exited, unmount its storage: python tasks/large-scale-formula-retrieval/tools/prepare_storage.py --run-id RUN_ID --unmount ``` -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/large-scale-formula-retrieval @@ -87,3 +114,9 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +## References + +- [Task reference](https://aclanthology.org/2026.findings-acl.1277/). + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/large-scale-formula-retrieval/environment/Dockerfile b/tasks/large-scale-formula-retrieval/environment/Dockerfile index 400550e..d68458d 100644 --- a/tasks/large-scale-formula-retrieval/environment/Dockerfile +++ b/tasks/large-scale-formula-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/large-scale-formula-retrieval/task.toml b/tasks/large-scale-formula-retrieval/task.toml index cca91ed..91aa567 100644 --- a/tasks/large-scale-formula-retrieval/task.toml +++ b/tasks/large-scale-formula-retrieval/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/large-scale-formula-retrieval" -version = "0.4.2" +version = "1.0.0" description = "Retrieve relevant formulas from the complete ARQMath v3 corpus with 34 MiB prepared artifacts in a 40 MiB serving filesystem and 50 ms CPU queries." authors = [{ name = "jianlyu" }] keywords = ["information-retrieval", "formula", "latex", "arqmath", "cpu"] diff --git a/tasks/large-scale-formula-retrieval/tests/Dockerfile b/tasks/large-scale-formula-retrieval/tests/Dockerfile index 447a310..5349f5c 100644 --- a/tasks/large-scale-formula-retrieval/tests/Dockerfile +++ b/tasks/large-scale-formula-retrieval/tests/Dockerfile @@ -12,7 +12,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/large-scale-formula-retrieval/tests/rewardkit_deepseek.py b/tasks/large-scale-formula-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/large-scale-formula-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/large-scale-formula-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/layout-page-search/README.md b/tasks/layout-page-search/README.md index c1e1d99..0bd3b46 100644 --- a/tasks/layout-page-search/README.md +++ b/tasks/layout-page-search/README.md @@ -1,10 +1,22 @@ # Visual-Document Evidence Page Retrieval -Find the complete set of supporting pages within a specified visual document. +## Task Metadata -**Task:** `layout-page-search` · **Metric:** EvidencePageSetF1 +| Field | Value | +| --- | --- | +| Task | `layout-page-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | EvidencePageSetF1 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 12000 s | +| Dataset | MMLongBench-Doc original chart/figure/layout evidence annotations | +| Public queries | 5 | +| Hidden queries | 515 | -## Overview +Find the complete set of supporting pages within a specified visual document. + +### Overview A question about a chart, figure or page layout may require more than one page of evidence. The system must return a compact set of pages that together @@ -14,15 +26,9 @@ The task evaluates page selection rather than answer generation. The document is supplied in the query; it is not a collection-wide document-identification task. -## What This Task Tests - -- Combining page text and visual evidence. -- Selecting a complete evidence set without adding unrelated pages. -- Managing page-level representations and query-dependent comparisons. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -46,7 +52,7 @@ identify visual evidence, but do not certify that text-only retrieval must fail. This task overlaps the document source used by `visual-document-qa`; the output and metric differ. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The corpus, specified document, evidence labels and output interface are fixed. Supplied encoder weights remain fixed. An optional scoring or adaptation @@ -57,13 +63,27 @@ token, region or learned pairwise comparisons. Representations, index structure and page selection are implementation choices. No precomputed vectors, starter or external API resource is supplied. -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has six hours, including the optional adaptation allowance. Build has 3,600 seconds; each of two query calls has 2,700 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Combining page text and visual evidence. +- Selecting a complete evidence set without adding unrelated pages. +- Managing page-level representations and query-dependent comparisons. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting files under `/app`. Return up to ten distinct page IDs from the specified document, @@ -75,9 +95,9 @@ Use the supplied index, output, cache and temporary paths for generated files. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality For each question, compute set F1 between the returned pages and the complete original evidence-page set. Quality is mean F1 over 515 held-out questions. @@ -87,7 +107,7 @@ The set metric penalizes both missing evidence and extra pages. Returning the complete annotated evidence set achieves full per-query credit. It does not measure answer accuracy or bounding-box localization. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.5734305608092015` and integer score anchor `a = 20` on the 0–100 scale: @@ -102,7 +122,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -111,7 +131,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -121,7 +141,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -136,7 +156,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task layout-page-search -bash scripts/run_task.sh --task layout-page-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task layout-page-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -144,7 +164,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/layout-page-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -157,3 +177,9 @@ credentials or hardware. Results are written under `jobs/layout-page-search/`. | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/layout-page-search/environment/Dockerfile b/tasks/layout-page-search/environment/Dockerfile index ba82144..30854ee 100644 --- a/tasks/layout-page-search/environment/Dockerfile +++ b/tasks/layout-page-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/layout-page-search/task.toml b/tasks/layout-page-search/task.toml index b3c0b29..950870a 100644 --- a/tasks/layout-page-search/task.toml +++ b/tasks/layout-page-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/layout-page-search" -version = "0.5.0" +version = "1.0.0" description = "Retrieve concise, complete supporting page sets for questions about long visual documents." authors = [{ name = "Haoran Jin" }] keywords = ["information-retrieval", "visual-document", "layout", "multivector", "gpu"] diff --git a/tasks/layout-page-search/tests/Dockerfile b/tasks/layout-page-search/tests/Dockerfile index d984100..6d9fdcb 100644 --- a/tasks/layout-page-search/tests/Dockerfile +++ b/tasks/layout-page-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -33,8 +33,8 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple "colpali-engine==0.3.18" "peft==0.19.1" RUN apt-get update && apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng tesseract-ocr-fra tesseract-ocr-deu && rm -rf /var/lib/apt/lists/* diff --git a/tasks/layout-page-search/tests/rewardkit_deepseek.py b/tasks/layout-page-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/layout-page-search/tests/rewardkit_deepseek.py +++ b/tasks/layout-page-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/legal-stance-retrieval/README.md b/tasks/legal-stance-retrieval/README.md index bcd8323..73f4718 100644 --- a/tasks/legal-stance-retrieval/README.md +++ b/tasks/legal-stance-retrieval/README.md @@ -1,6 +1,16 @@ # Legal stance retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `legal-stance-retrieval` | +| Version | 1.0.0 | +| Authors | Cooki-e | +| Primary metric | Baseline-normalized mean NDCG@10 | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 5100 s | +| Hidden queries | 311 | Given a legal claim and `supported`, `refuted` or `all`, retrieve ten ranked court-summary document IDs. `all` is the union of both evidence directions; @@ -8,7 +18,7 @@ either direction can be absent. This is retrieval over recorded majority holdings in a fixed snapshot, not a current-law validity check or verdict classification. The supplied Qwen3-Embedding-0.6B and public examples are available to the search system. Designated preparation/retrieval APIs remain permitted. -## Frozen data and metric +### Frozen data and metric | Item | Count | | --- | ---: | @@ -59,7 +69,19 @@ of every corpus pair. Added positives have explicit author-agent decisions. Machine-neutral and out-of-pool pairs are not exhaustively adjudicated. Original training labels remain noisy. The task retains these documented limitations. -## Execution and resources +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +A precedent can be topically relevant while expressing the wrong stance. The system must interpret stance conditions and improve legal retrieval over the fixed baseline within the query budget. + +## Verification Explanation + +### Execution and resources The agent receives 16 CPUs, 32 GiB RAM, 100 GiB storage, one NVIDIA GPU and a 6-hour development phase. Both images use the standard GPU base. @@ -83,7 +105,7 @@ threshold is imposed by the audit; an honest baseline is a valid zero-improvemen submission. Optional task API credentials are shared `OPENROUTER_API_KEY` and `JINA_API_KEY`; judge keys are never passed to submission commands. -## Provenance and local assets +### Provenance and local assets The source is [IDIRLab/CaseFacts](https://github.com/idirlab/CaseFacts), downloaded at repository snapshot `16a4072067c39daa463495e10e3bb029c29beb5b`. @@ -103,7 +125,7 @@ verifier image. Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. -## Structural precedents and validation +### Structural precedents and validation The reviewed `tasks/query-encoder-alignment` and `tasks/reasoning-query-rewriting` packages at checkout @@ -123,12 +145,29 @@ this overrides the skill's generic prose-only convention. All source obligations were checked against the request and grader; see [instruction-audit.md](instruction-audit.md). -## Instruction revision 0.1.1 +### Instruction revision 0.1.1 The user requested a shorter agent-facing prompt. It now states the provided inputs and model, executable interfaces, whole-batch deadline and mean NDCG@10 objective. Detailed annotation guidance, training-label quality commentary, implementation suggestions and the reward anchor/formula remain author-side. The grader, frozen qrels and reward normalization did not change. General resource and integrity restrictions remain in the linked resource document. The redundant `model-metadata` directory and its two asset entries were removed. Its files were a model-card copy and a ModelScope `configuration.json`, neither used by the supplied Transformers loader. The actual `config.json`, model weights, tokenizer and SentenceTransformers configuration remain supplied and pinned. The packaging script excludes the removed files so they will not be recreated. -## Public examples — v0.1.2 +### Public examples — v0.1.2 Public examples are now under `/task/data/examples`, with `queries.jsonl` and `ground_truth.jsonl`. The host directory, Compose mount, asset manifest and packaging output use this name. Public query IDs replace the source `train_` prefix with `example_` in both files; the claim text, stance and relevance maps are unchanged. The agent-facing instruction, resource docs and task description provide the examples and model without suggesting a training method. Original source-split names remain in author provenance and audit records. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task legal-stance-retrieval +bash scripts/run_task.sh --task legal-stance-retrieval --gpu-device 0 --model YOUR_AGENT_MODEL +``` + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/legal-stance-retrieval/environment/Dockerfile b/tasks/legal-stance-retrieval/environment/Dockerfile index 090e39c..487638b 100644 --- a/tasks/legal-stance-retrieval/environment/Dockerfile +++ b/tasks/legal-stance-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/legal-stance-retrieval/task.toml b/tasks/legal-stance-retrieval/task.toml index 322354a..c914b15 100644 --- a/tasks/legal-stance-retrieval/task.toml +++ b/tasks/legal-stance-retrieval/task.toml @@ -3,7 +3,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/legal-stance-retrieval" -version = "0.1.2" +version = "1.0.0" description = "Build stance-conditioned legal precedent retrieval over a supplied corpus, with examples and a local model." authors = [{ name = "Cooki-e" }] keywords = ["information-retrieval", "legal", "stance", "embedding", "gpu"] diff --git a/tasks/legal-stance-retrieval/tests/Dockerfile b/tasks/legal-stance-retrieval/tests/Dockerfile index d28ab66..61bbb89 100644 --- a/tasks/legal-stance-retrieval/tests/Dockerfile +++ b/tasks/legal-stance-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/legal-stance-retrieval/tests/rewardkit_deepseek.py b/tasks/legal-stance-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/legal-stance-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/legal-stance-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/lightweight-query-retrieval/README.md b/tasks/lightweight-query-retrieval/README.md index fa25f5b..13db2d5 100644 --- a/tasks/lightweight-query-retrieval/README.md +++ b/tasks/lightweight-query-retrieval/README.md @@ -1,15 +1,26 @@ # Lightweight query retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata -Author: Bingyu. Task version 0.4.0. Built with Llama. +| Field | Value | +| --- | --- | +| Task | `lightweight-query-retrieval` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Retrieval runtime after a macro nDCG@10 quality gate | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 3900 s | +| Dataset | BEIR SciFact, NFCorpus and FiQA; author-released LightRetriever examples | +| Hidden queries | 1271 | + +Author: Bingyu. Built with Llama. Build a fast batch retrieval service, subject to a required search-quality threshold. The agent receives the original Llama-3.2-1B base model and 80,000 original labeled examples. The task does not supply a trained retrieval model, reference implementation, prebuilt index, or test answers. -## Data and provenance +### Data and provenance | Original BEIR collection | Corpus documents | Hidden test queries | | --- | ---: | ---: | @@ -50,7 +61,19 @@ Source checksums, row-level sampling provenance, audit records and reproducible preparation scripts are retained in `../../../task_development/lightretriever`, outside this task package. -## Runtime and scoring +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The objective covers complete query processing, including representation and search, across three collections. Faster execution earns credit only after meeting the macro retrieval-quality gate. + +## Verification Explanation + +### Runtime and scoring One NVIDIA A100 40GB, 16 CPU cores, 32 GiB RAM, 100 GiB storage, six hours of development, and no task network/API access. The shared GPU image supplies @@ -114,17 +137,18 @@ configuration. The overall verifier budget is 3,900 seconds to include the audit; the existing 2,300-second retrieval supervisor and all candidate limits are unchanged. -## Validation and release state +### Validation and release state Before the 0.4.0 audit integration, local assets, images, GPU forward/backward, scoring and isolation checks were executed. A legal implementation trained from the supplied original base and only supplied examples passed the complete separate verifier with minimum quality **0.454410**, median **0.860873 seconds** and reward **0.000000**. See `VALIDATION.md` for the evidence and limits. -Version 0.4.0 adds local reward-gate, subprocess-isolation and shell-orchestration -checks. No new GPU retrieval, coding-agent or live judge run has been performed; -the earlier witness does not establish the new judge's acceptance or rejection -behavior. See `VALIDATION.md` for the separate validation layers. +The later v0.4.0 verifier run completed its live integrity audit. Macro nDCG@10 +was 0.4303102260205993, below the 0.45 gate, and median query-batch time was +0.41057249531149864 seconds. See [RUN_RESULTS.md](RUN_RESULTS.md) for that +measurement and its remaining validation limits, and [VALIDATION.md](VALIDATION.md) +for the separate local checks. The two derived data assets are published in [`search-swe/Search-SWE`](https://huggingface.co/datasets/search-swe/Search-SWE/tree/ff16485accd6d258610c158fce45e36b50a47c3f/tasks/lightweight-query-retrieval) @@ -134,3 +158,20 @@ Their official manifest sizes and SHA-256 hashes match the local inputs. access to Meta's gated model must also be configured. Authoring scripts, baselines and trained witness checkpoints are not included in the agent mounts. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task lightweight-query-retrieval +bash scripts/run_task.sh --task lightweight-query-retrieval --gpu-device 0 --model YOUR_AGENT_MODEL +``` + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +None. diff --git a/tasks/lightweight-query-retrieval/environment/Dockerfile b/tasks/lightweight-query-retrieval/environment/Dockerfile index a85ce23..7b36e7a 100644 --- a/tasks/lightweight-query-retrieval/environment/Dockerfile +++ b/tasks/lightweight-query-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/lightweight-query-retrieval/task.toml b/tasks/lightweight-query-retrieval/task.toml index 708e2f9..80e82bb 100644 --- a/tasks/lightweight-query-retrieval/task.toml +++ b/tasks/lightweight-query-retrieval/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/lightweight-query-retrieval" -version = "0.4.0" +version = "1.0.0" description = "Reduce complete query processing time while maintaining a required retrieval quality across three document collections." authors = [{ name = "Bingyu" }] keywords = ["information-retrieval", "efficiency", "query-encoding", "gpu"] diff --git a/tasks/lightweight-query-retrieval/tests/Dockerfile b/tasks/lightweight-query-retrieval/tests/Dockerfile index 03eb642..5a159dc 100644 --- a/tasks/lightweight-query-retrieval/tests/Dockerfile +++ b/tasks/lightweight-query-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/lightweight-query-retrieval/tests/rewardkit_deepseek.py b/tasks/lightweight-query-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/lightweight-query-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/lightweight-query-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/long-document-reranking/README.md b/tasks/long-document-reranking/README.md index b04568a..b47bdb8 100644 --- a/tasks/long-document-reranking/README.md +++ b/tasks/long-document-reranking/README.md @@ -1,10 +1,22 @@ # Long-document Reranking -Improve long-document ranking without changing the candidate pool or the relevance model. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `long-document-reranking` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | Accuracy@5 | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 4800 s | +| Dataset | MTEB NarrativeQARetrieval | +| Public queries | 100 | +| Hidden queries | 100 | -**Task:** `long-document-reranking` · **Metric:** Accuracy@5 +Improve long-document ranking without changing the candidate pool or the relevance model. -## Overview +### Overview A relevant passage in a long document may lie far beyond the first 512 model tokens. Simply truncating every candidate can therefore discard the evidence @@ -16,16 +28,9 @@ BGE reranker. The engineering opportunity is in how documents are prepared, which windows are scored, and how local scores inform a document-level decision. Finding more evidence must remain practical within the runtime budget. -## What This Task Tests +### Task Setup -- Handling long documents with a short-input cross-encoder. -- Selecting useful document windows and aggregating their evidence. -- Improving preprocessing and inference without altering learned model behavior. -- Preserving candidate-set identity across a multi-stage ranking pipeline. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -38,7 +43,7 @@ Downloaded data and models are mounted read-only under `/task/data` and `/task/models`. The hidden evaluation contains 100 disjoint queries. [Starter notes](environment/starter/README.md) describe the initial implementation. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The candidate documents and reranker model, tokenizer, and configuration are fixed. The agent may optimize preprocessing, window selection, score aggregation, @@ -49,7 +54,7 @@ The Docker mounts place the model at `/task/models/bge-reranker-large`. The introductory model path in `instruction.md` currently contains a typo; the mounted path and runtime configuration are the operational reference. -### Environment and Resource Limits +#### Environment and Resource Limits The CPU Python 3.12 environment provides 16 CPUs, 64 GiB memory, 100 GiB storage, and no GPU. The agent has two hours and the Harbor verifier 80 minutes. @@ -57,7 +62,22 @@ The grader's build and run caps are 1,800 and 3,600 seconds respectively, but these stages and the integrity check must still fit inside the overall verifier phase; their individual maxima are not additive allowances. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Handling long documents with a short-input cross-encoder. +- Selecting useful document windows and aggregating their evidence. +- Improving preprocessing and inference without altering learned model behavior. +- Preserving candidate-set identity across a multi-stage ranking pipeline. + +## Verification Explanation + +### Submission Contract The deliverable is `/app/build.sh` and `/app/run.sh` with the implementation needed to rebuild preprocessing artifacts and serve reranking requests. @@ -68,21 +88,21 @@ The verifier uses a separate environment and unprivileged submission execution. The index directory is supplied by the caller. Consult [instruction.md](instruction.md) for the complete CLI and result format. -## Evaluation +### Evaluation -### Search Quality +#### Search Quality A hidden query is a hit if a relevant document is among the five returned candidates. Accuracy@5 is the fraction of hits over all hidden queries. Valid runs receive this fraction as a continuous quality score. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks executable behavior, complete outputs, valid ranking fields, candidate membership, and compliance with the fixed-model rules. Retrieval outside the pool is not a valid way to improve this task's score. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory judge checks task compliance. Final reward equals Accuracy@5 only when evaluation is valid and the judge passes; otherwise zero. @@ -90,7 +110,7 @@ The model being optimized around is the fixed reranker, not the trajectory judge The trajectory judge itself uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0 and receives its credentials only in the verifier. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -113,7 +133,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -126,9 +146,13 @@ assets, credentials, or hardware. | [Starter notes](environment/starter/README.md) | Initial pipeline and local validation guidance | | [Verifier notes](tests/README.md) | Additional verification details | -## Instruction source +### Instruction source `raw-instruction.md` preserves the original structured instruction. The request in `instruction.md` corrects its `/tast/model/bge-reranker-large` path typo to the actual read-only mount `/task/models/bge-reranker-large`; the fixed model and all candidate/model restrictions are unchanged. + +## References + +None. diff --git a/tasks/long-document-reranking/environment/Dockerfile b/tasks/long-document-reranking/environment/Dockerfile index 4b5f9ff..80763d1 100644 --- a/tasks/long-document-reranking/environment/Dockerfile +++ b/tasks/long-document-reranking/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/long-document-reranking/task.toml b/tasks/long-document-reranking/task.toml index f6e106e..b216e21 100644 --- a/tasks/long-document-reranking/task.toml +++ b/tasks/long-document-reranking/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/long-document-reranking" -version = "0.1" +version = "1.0.0" description = "Optimize long-document reranking over a frozen BM25 Top-100 candidate pool with a fixed local BGE reranker." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "search", "reranking", "cross-encoder", "long-document", "cpu"] diff --git a/tasks/long-document-reranking/tests/Dockerfile b/tasks/long-document-reranking/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/long-document-reranking/tests/Dockerfile +++ b/tasks/long-document-reranking/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/long-document-reranking/tests/rewardkit_deepseek.py b/tasks/long-document-reranking/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/long-document-reranking/tests/rewardkit_deepseek.py +++ b/tasks/long-document-reranking/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/long-pdf-evidence-vectorless/README.md b/tasks/long-pdf-evidence-vectorless/README.md index 344222f..f5f63a4 100644 --- a/tasks/long-pdf-evidence-vectorless/README.md +++ b/tasks/long-pdf-evidence-vectorless/README.md @@ -1,18 +1,43 @@ # Long-PDF Sentence Evidence — Vectorless -Build a reusable, embedding-free index over the supplied *War and Peace* PDF and its fixed sentence inventory. Return up to five complete original sentence units supporting each interpretive question. Version 0.3.1 adopts the revised `long-pdf-evidence` v0.4.1 data and **SentenceEvidenceAccuracy**. Earlier page Recall@5 checks do not validate this version. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `long-pdf-evidence-vectorless` | +| Version | 1.0.0 | +| Authors | Jianlyu | +| Primary metric | SentenceEvidenceAccuracy | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 10800 s | +| Dataset | single long PDF | +| Hidden queries | 12 | + +Build a reusable, embedding-free index over the supplied *War and Peace* PDF and its fixed sentence inventory. Return up to five complete original sentence units supporting each interpretive question. The task uses **SentenceEvidenceAccuracy** and the sentence-evidence inputs attributed in [SOURCES.md](SOURCES.md). The unchanged PDF has 2,043 physical pages; the question-independent inventory has 28,798 units. Ten public questions include support requirements and sufficient evidence. Twelve private questions use disjoint reference units. Page numbers are provenance only; alternative correct sentence evidence is accepted. See [instruction.md](instruction.md) for the agent request and [raw-instruction.md](raw-instruction.md) for the structured contract. -## Runtime and isolation +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The system must construct a reusable embedding-free index and retrieve original sentence evidence from a long novel. Evidence must jointly support the question within the output and runtime budgets. + +## Verification Explanation + +### Runtime and isolation The CPU image provides 16 CPUs, 64 GiB RAM and 100 GiB storage, without a GPU. Development is limited to 7,200 seconds. `/app/build.sh` creates a persistent index in 2,400 seconds. All build workers stop before private queries become visible, and the staged PDF **and sentence inventory** become root-owned (files 0400, directories 0700). Query processing must use only the index, supplied queries and submission code. Optional query-time services must start from that index alone. Embeddings are prohibited during both build and search; lexical indexing and permitted generation/non-embedding reranking remain available. The fresh separate verifier transfers `/app`, rebuilds the index, and runs submission commands as UID 10001. Up to five query processes share the index, each with 900 seconds and a shared 1,800-second deadline; cleanup has another 60 seconds. Failed or invalid queries remain in the twelve-query denominator with zero. The root-side grader reads a private corpus mount to verify fixed hashes and original sentence membership. Public validation and all judge credentials are excluded from submission processes. -The answer judge uses OpenRouter `deepseek/deepseek-v4.1-flash`, effort `high`, temperature 0, two workers and at most two attempts/request, within 2,700 seconds. Complete evidence support earns 1; topical or incomplete evidence earns 0. API/parse failure invalidates the measurement and fails closed. The separate trajectory audit uses RewardKit 0.2.0, LiteLLM 1.97.0 and Codex 0.147.0 with `deepseek/deepseek-v4.1-flash`, bounded at 3,600 seconds. Only a valid binary audit pass preserves accuracy as the final 0–1 reward. The entire verifier has 10,800 seconds. +The answer judge uses OpenRouter `deepseek/deepseek-v4.1-flash`, effort `high`, temperature 0, two workers and at most two attempts/request, within 2,700 seconds. Complete evidence support earns 1; topical or incomplete evidence earns 0. API/parse failure invalidates the measurement and fails closed. The separate trajectory audit uses RewardKit 0.2.0, LiteLLM 1.97.0 and Codex 0.164.0 with `deepseek/deepseek-v4.1-flash`, bounded at 3,600 seconds. Only a valid binary audit pass preserves accuracy as the final 0–1 reward. The entire verifier has 10,800 seconds. -## Credentials and evaluation +### Credentials and evaluation Configure the ignored repository `.env`: `AGENT_OPENROUTER_API_KEY` for the coding agent; `ANSWER_JUDGE_API_KEY` for the semantic judge; and `VERIFIER_OPENAI_API_KEY` / `VERIFIER_OPENAI_BASE_URL` for the trajectory audit. Both judge endpoints are `https://openrouter.ai/api/v1`. Set `ANSWER_JUDGE_BASE_URL` to that endpoint and `ANSWER_JUDGE_MODEL_NAME` to `deepseek/deepseek-v4.1-flash` for launcher validation; the task also pins both values. Optional solver calls use the separate `OPENROUTER_API_KEY` and only the models/endpoints in [available_resources.md](environment/docs/available_resources.md). The judge model is not a solver resource. Never include credential values in this package or artifacts. @@ -28,7 +53,7 @@ bash scripts/run_task.sh --task long-pdf-evidence-vectorless --agent codex --ope --output jobs/long-pdf-evidence-vectorless-codex-openrouter-luna-max-RUN_ID ``` -## Sources and validation status +### Sources and validation status All four public inputs use the reviewed original task's published immutable pins, sizes and SHA-256 hashes. Private queries/references are reused unchanged from its v0.4.1 verifier. See [SOURCES.md](SOURCES.md) for provenance and publication limits. Private files are excluded from agent mounts but remain visible to repository readers, so this is not a secret evaluation set. @@ -41,3 +66,7 @@ The October 16 Codex + OpenRouter Luna max trial completed verification with **S The four public inputs intentionally reuse the original task's published paths and immutable pins. Search-SWE `ac7dcc1` supports cross-task reuse while retaining source-path and integrity checks. On October 17, both checkouts and remote main contained that fix; this package passed the contribution checkout's merge-ready checker without asset changes. All 4 asset hashes, 20 task tests and the 174-test shared suite passed. The proposed duplicate upload was cancelled, and no data was published. See `contributing/dev/contribution-readiness/2026-10-17/REVIEW.md` for the current review. A public cart-purpose question omits the requester, allowing competing original scenes. Its supplied public reference requirements identify the intended furniture request; this observation does not invalidate the separate hidden-set measurement. On October 17 the contributor requested retaining wording unless modification was necessary, so the original public data are retained and this optional clarification is not a submission blocker. The diagnostic and proposed wording remain recorded in `public-query-ambiguity.md` under the October 16 author evidence directory. No question, grader or resource budget was changed in this review. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/long-pdf-evidence-vectorless/environment/Dockerfile b/tasks/long-pdf-evidence-vectorless/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/long-pdf-evidence-vectorless/environment/Dockerfile +++ b/tasks/long-pdf-evidence-vectorless/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/long-pdf-evidence-vectorless/task.toml b/tasks/long-pdf-evidence-vectorless/task.toml index 178c520..3fff0ae 100644 --- a/tasks/long-pdf-evidence-vectorless/task.toml +++ b/tasks/long-pdf-evidence-vectorless/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/long-pdf-evidence-vectorless" -version = "0.3.1" +version = "1.0.0" description = "Build a reusable, embedding-free index for retrieving complete sentence evidence from a long PDF." authors = [{ name = "Jianlyu" }] keywords = ["information-retrieval", "search", "evidence locator"] diff --git a/tasks/long-pdf-evidence-vectorless/tests/Dockerfile b/tasks/long-pdf-evidence-vectorless/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/long-pdf-evidence-vectorless/tests/Dockerfile +++ b/tasks/long-pdf-evidence-vectorless/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/long-pdf-evidence-vectorless/tests/rewardkit_deepseek.py b/tasks/long-pdf-evidence-vectorless/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/long-pdf-evidence-vectorless/tests/rewardkit_deepseek.py +++ b/tasks/long-pdf-evidence-vectorless/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/long-pdf-evidence/README.md b/tasks/long-pdf-evidence/README.md index c73a337..f13ef3f 100644 --- a/tasks/long-pdf-evidence/README.md +++ b/tasks/long-pdf-evidence/README.md @@ -1,8 +1,19 @@ # Long-PDF Sentence Evidence Retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata -Retrieve up to five original sentence units that fully support a short interpretive question about *War and Peace*. Version 0.4.3 uses **SentenceEvidenceAccuracy** with the DeepSeek evidence judge and a 30-second limit per query. The old 0.95 page-retrieval result is not a measurement of this revised task. +| Field | Value | +| --- | --- | +| Task | `long-pdf-evidence` | +| Version | 1.0.0 | +| Authors | Search-SWE, Bingyu | +| Primary metric | SentenceEvidenceAccuracy | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 10800 s | +| Dataset | single long PDF | +| Hidden queries | 12 | + +Retrieve up to five original sentence units that fully support a short interpretive question about *War and Peace*. The task uses **SentenceEvidenceAccuracy** with the DeepSeek evidence judge and a 30-second limit per query. The unchanged PDF has 2,043 physical pages. Its question-independent inventory contains 28,798 units, preserving source-page provenance and normalizing PDF typography and whitespace. Unit boundaries are fixed for consistent evaluation, including dialogue and sentences crossing pages. The ten public and twelve hidden questions use disjoint reference units and opaque shuffled IDs. Questions ask about motives, tensions, reversals, or relationships without restating the relevant passage. @@ -14,6 +25,20 @@ The unchanged PDF has 2,043 physical pages. Its question-independent inventory c | `data/validation/golden_answers.jsonl` | Public support requirements and sufficient original evidence | | `tests/data/` | Twelve private queries and references, excluded from agent mounts | +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Interpretive questions about a long novel may refer to motives, relationships or reversals without repeating the relevant wording. The system must select at most five original sentence units that jointly provide complete support. + +## Verification Explanation + +### Contract, scoring and runtime + Deliver executable `/app/build.sh` and `/app/run.sh` following [instruction.md](instruction.md). Return `{query_id, results: [{sentence_id, sentence}]}`, with zero to five distinct units and no generated answer. Evidence order is irrelevant. The verifier checks corpus checksums, source membership, and output validity before judging sufficiency. Alternative correct evidence is accepted; reference IDs are not a scoring gate. The answer judge is pinned to OpenRouter `deepseek/deepseek-v4.1-flash`, reasoning effort `high`, temperature 0. It awards 1 for complete evidence support and 0 otherwise. Mean support over all twelve questions is the metric; a failed or timed-out query remains in the denominator with zero. API/parse failures invalidate the measurement and fail closed, separately from ordinary unsupported evidence. A separate `deepseek/deepseek-v4.1-flash` trajectory audit through RewardKit 0.2.0 gates the final reward. Judge health and individual decisions are retained in verifier reports. @@ -28,8 +53,23 @@ Public inputs are published and pinned to immutable revisions in `assets.json`, Author development and calibration records live outside the package in `task_development/long-pdf-evidence` in the surrounding build workspace. [raw-instruction.md](raw-instruction.md) retains the structured specification and evaluator details; [instruction.md](instruction.md) presents the task, available inputs, executable interface, output requirements, limits and scoring criteria without data-construction explanations or judge implementation details. The prior task supplies the corpus, executable lifecycle and isolation pattern; `scientific-paper-qa` was consulted for separate semantic-judge credentials. The requested `agentic-search` reference currently uses deterministic GoldRecall@5, so its grader is not a semantic-judge implementation. -## Evidence-judge revision +### Evidence-judge revision -Version 0.4.3 pins DeepSeek V4.1 Flash in the grader, task metadata, verifier environment and structured authoring source. The agent-facing instruction describes the evidence requirements and 30-second per-query limit without naming the judge or explaining its implementation. The twelve questions, reference evidence, sufficiency rubric, averaging rule, reasoning effort and temperature are unchanged. The completed v0.4.3 Codex / `openai/gpt-6-luna` / `max` evaluation scored 1/12; both evidence evaluation and trajectory audit succeeded. See [RUN_RESULTS.md](RUN_RESULTS.md). +The task pins DeepSeek V4.1 Flash in the grader, task metadata, verifier environment and structured authoring source. The agent-facing instruction describes the evidence requirements and 30-second per-query limit without naming the judge or explaining its implementation. The twelve questions, reference evidence, sufficiency rubric, averaging rule, reasoning effort and temperature are unchanged. The completed v0.4.3 Codex / `openai/gpt-6-luna` / `max` evaluation scored 1/12; both evidence evaluation and trajectory audit succeeded. See [RUN_RESULTS.md](RUN_RESULTS.md). The earlier DeepSeek rejudgment of a Luna-generated submission returned 8/20 on the old twenty-question version. It is historical reference evidence, not a score for this twelve-question revision. The current trial's result and judge-health records are summarized in RUN_RESULTS.md. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task long-pdf-evidence +bash scripts/run_task.sh --task long-pdf-evidence --model YOUR_AGENT_MODEL +``` + +## References + +None. diff --git a/tasks/long-pdf-evidence/environment/Dockerfile b/tasks/long-pdf-evidence/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/long-pdf-evidence/environment/Dockerfile +++ b/tasks/long-pdf-evidence/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/long-pdf-evidence/task.toml b/tasks/long-pdf-evidence/task.toml index 38af0f3..2677efb 100644 --- a/tasks/long-pdf-evidence/task.toml +++ b/tasks/long-pdf-evidence/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/long-pdf-evidence" -version = "0.4.3" +version = "1.0.0" description = "Retrieve original sentences that completely support concise questions about a long novel." authors = [{ name = "Search-SWE" }, { name = "Bingyu" }] keywords = ["information-retrieval", "search", "evidence locator"] diff --git a/tasks/long-pdf-evidence/tests/Dockerfile b/tasks/long-pdf-evidence/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/long-pdf-evidence/tests/Dockerfile +++ b/tasks/long-pdf-evidence/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/long-pdf-evidence/tests/rewardkit_deepseek.py b/tasks/long-pdf-evidence/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/long-pdf-evidence/tests/rewardkit_deepseek.py +++ b/tasks/long-pdf-evidence/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/memory-constrained-dense-retrieval/README.md b/tasks/memory-constrained-dense-retrieval/README.md index 6d3322e..9c25976 100644 --- a/tasks/memory-constrained-dense-retrieval/README.md +++ b/tasks/memory-constrained-dense-retrieval/README.md @@ -1,10 +1,21 @@ # Memory-constrained Dense Retrieval -Search a fixed vector collection larger than available RAM while meeting strict query deadlines. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `memory-constrained-dense-retrieval` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | Accuracy@3 | +| Compute | 16 CPUs; 0.5 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 7200 s | +| Dataset | NaturalQuestions | +| Hidden queries | 20 | -**Task:** `memory-constrained-dense-retrieval` · **Version:** 0.7.1 · **Metric:** Accuracy@3 × P95 latency factor, with a 95% quality floor +Search a fixed vector collection larger than available RAM while meeting strict query deadlines. -## Overview +### Overview This task contains 1.5 million precomputed NaturalQuestions document vectors, each with 1,024 float32 dimensions. The raw vectors occupy about 6.14 GB, @@ -16,16 +27,9 @@ the system must become ready quickly and serve independent query invocations within a tight end-to-end deadline. The architecture is open, so memory management and service lifecycle are as important as the search algorithm. -## What This Task Tests +### Task Setup -- Designing dense-vector search under a memory budget smaller than the corpus. -- Balancing index preparation, retrieval quality, and query latency. -- Keeping a reusable service available across short-lived query clients. -- Accounting for process startup and serialization in end-to-end performance. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -52,7 +56,7 @@ Top-3 hits, favoring small relevant-document margins over rank four and balancing those margins between splits. This margin is a difficulty proxy, not evidence of failure by a measured ANN baseline. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes Document and query vectors are supplied inputs. Index organization, compression, storage access, and the retrieval implementation may change. External models, @@ -60,7 +64,7 @@ retrieval services, and answer mappings are not part of the task. An internal representation must still produce valid results for the supplied vectors and preserve their document identities. -### Environment and Resource Limits +#### Environment and Resource Limits Both phase containers provide 16 CPUs for development and offline construction, 512 MiB aggregate memory, no swap, 100 GiB storage allowance and no GPU. Final online queries have a separate one-logical-CPU budget: the client, restarted service and every descendant share the same single allowed CPU. Both phases use a per-process open-file soft and hard limit of 1,048,576, including non-root verifier build and query processes. The memory cgroup includes the build, resident service, clients, all descendants, charged file-cache pages, and runtime/verifier overhead; it is not a per-process allowance. Both agent development and final verification have a 120-minute total limit. Failed development experiments do not by themselves set the final reward; final verification starts in a fresh container. The final build has a 150-second limit and each query has a 0.5-second hard deadline. The 100 GiB storage allowance is an audited policy, not a kernel filesystem quota; index sizes are reported. @@ -72,7 +76,22 @@ The submitted build must finish within **150 seconds**, including service startup. Every query invocation has a **0.5-second wall-clock limit**, measured from process start through exit and protected output capture, including the full query path. The grader records index size. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Designing dense-vector search under a memory budget smaller than the corpus. +- Balancing index preparation, retrieval quality, and query latency. +- Keeping a reusable service available across short-lived query clients. +- Accounting for process startup and serialization in end-to-end performance. + +## Verification Explanation + +### Submission Contract The deliverable contains `/app/build.sh` and `/app/run.sh`. The first builds the index and starts a reusable service; the second accepts query-vector files and @@ -83,9 +102,9 @@ separately for each question. Submission execution is unprivileged, with persistent and runtime artifacts written to the supplied index directory. The complete interface is in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Each query earns 1 only when its invocation exits successfully, its valid output is captured within 0.5 seconds, and its top three results contain a relevant @@ -93,7 +112,7 @@ document. Otherwise that query earns 0. The primary metric is the mean of these 20 binary scores; 17 passing queries give Accuracy@3 = 0.85 and fail the minimum 0.95 quality requirement. The report also records retrieval accuracy without the latency gate for diagnosis. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks output coverage, document IDs, finite scores, ordering, and the execution deadline for each invocation. Incorrect answers, malformed @@ -103,7 +122,7 @@ process group while preserving the separately started query service. Build failure, an invalid index, or invalid evaluator inputs still invalidate the evaluation. The 512 MiB container memory limit applies throughout execution. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A trajectory audit independently checks task compliance. Let `A` be the mean of the 20 latency-gated Accuracy@3 values and `L95` the P95 duration in seconds. The service must achieve `A >= 0.95`: at least 19 of the 20 queries must pass the accuracy, output and 0.5-second execution gates. Below this quality floor the reward is zero. At or above it, the normalized reward before the integrity gate is `A * min(1, 0.030 / L95)`; the displayed score is 100 times this reward. Accuracy and the latency factor are still reported when the quality floor is missed. This is an acceptance threshold on this fixed workload, not a statistical confidence guarantee. At `A=1`, P95 values of 30, 60 and 150 ms earn 100, 50 and 20 points. At `A=0.95`, 30 and 150 ms earn 95 and 19 points; `A=0.90` earns zero regardless of speed. A failed build, invalid or oversized index, invalid evaluator inputs, out-of-memory kill observed in the final verifier container, or failed retrieval-integrity check makes the overall reward zero. @@ -112,7 +131,7 @@ The verifier starts from an empty index workspace and builds once. After the suc It uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0 and receives its DeepSeek endpoint and key only in the verifier. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -135,7 +154,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -146,8 +165,12 @@ assets, credentials, or hardware. | [Environment configuration](environment/docker-compose.yaml) | Read-only mounts and hardware requests | | [Verifier](tests/) | Execution, output validation, and scoring implementation | -## Local regression checks +### Local regression checks Run `python -m unittest discover -s tasks/memory-constrained-dense-retrieval/tests -p "test_*.py"`. Runtime grading requires Linux blocking process waits and a cgroup-v2 container with memory.max=536870912 and memory.swap.max=0. The grader records final-verifier container memory peak/events; any OOM kill in that container fails the evaluation. The two Compose overlays set identical 16-CPU, 512-MiB/no-swap container limits. The native query launcher enforces one logical CPU for all online submission processes. The verifier snapshots each output immediately after observing client exit, in a root-only directory. Query timing ends after capture completes, so every scored byte was received within the measured interval. Validation uses only that protected snapshot; writes through retained descriptors after capture cannot change it. The same capture-inclusive duration controls the 0.5-second deadline and P95. + +## References + +None. diff --git a/tasks/memory-constrained-dense-retrieval/environment/Dockerfile b/tasks/memory-constrained-dense-retrieval/environment/Dockerfile index e5260ad..9befce2 100644 --- a/tasks/memory-constrained-dense-retrieval/environment/Dockerfile +++ b/tasks/memory-constrained-dense-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/memory-constrained-dense-retrieval/task.toml b/tasks/memory-constrained-dense-retrieval/task.toml index 5c727ab..431c209 100644 --- a/tasks/memory-constrained-dense-retrieval/task.toml +++ b/tasks/memory-constrained-dense-retrieval/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/memory-constrained-dense-retrieval" -version = "0.7.1" +version = "1.0.0" description = "Build a memory-constrained dense-vector retrieval system over 1.5 million precomputed NQ corpus vectors." authors = [{ name = "Search-SWE" }] keywords = ["search", "retrieval", "dense-retrieval", "vector-search", "nq", "memory-constrained", "cpu", "indexing"] diff --git a/tasks/memory-constrained-dense-retrieval/tests/Dockerfile b/tasks/memory-constrained-dense-retrieval/tests/Dockerfile index 205a51c..23eb3da 100644 --- a/tasks/memory-constrained-dense-retrieval/tests/Dockerfile +++ b/tasks/memory-constrained-dense-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/memory-constrained-dense-retrieval/tests/rewardkit_deepseek.py b/tasks/memory-constrained-dense-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/memory-constrained-dense-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/memory-constrained-dense-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/methodology-inspiration-retrieval/README.md b/tasks/methodology-inspiration-retrieval/README.md index f870dd6..8d620cb 100644 --- a/tasks/methodology-inspiration-retrieval/README.md +++ b/tasks/methodology-inspiration-retrieval/README.md @@ -1,6 +1,18 @@ # Methodology Inspiration Retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `methodology-inspiration-retrieval` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Mean Recall@3 improvement | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 3000 s | +| Dataset | MIR, screened fixed-corpus derivative | +| Public queries | 0 | +| Hidden queries | 122 | Build a working search engine that retrieves papers offering methodological inspiration for a research problem and motivation. The solver receives a fixed @@ -8,7 +20,7 @@ abstract collection, citation examples, Stella 400M, one A100 40GB and the documented OpenRouter/Jina APIs. No implementation or training procedure is required. Author: **Bingyu**. -## Package contract +### Package contract | Item | Setting | | --- | --- | @@ -34,7 +46,7 @@ shared environment. `/opt/stella/bin/python` supplies an isolated Transformers the base PyTorch installation. Model loading and readiness belong to the build budget; initialization deferred to `run.sh` counts against the 12-second limit. -## Data and provenance +### Data and provenance Source: [MIR: Methodology Inspiration Retrieval for Scientific Research Problems](https://aclanthology.org/2025.acl-long.1390/), Garikaparthi et al., ACL 2025; @@ -76,7 +88,19 @@ per-paper redistribution license inventory. No blanket unrestricted license is claimed. Stella's upstream model card declares MIT; its revision is `ffeb2b7ee715c226d4ffe5e4619f7dbb48624c20`. -## Evaluation +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Relevant papers may share a useful methodology with a research proposal without sharing its topic vocabulary. The system must retrieve those connections from abstracts within the short batch retrieval deadline. + +## Verification Explanation + +### Evaluation For each query, `recall_q = |top3 ∩ positives_q| / |positives_q|`. `Q` is the arithmetic mean over all 122 queries. Positive-set sizes are @@ -99,7 +123,7 @@ receives zero. A successful passing trajectory audit is required for final reward; unavailable or failed judges never silently pass. The numerical normalization is author/verifier information, not solver implementation advice. -## Authoring and release status +### Authoring and release status Construction scripts, source maps, review decisions, baseline implementations and runtime evidence live in the sibling author workspace @@ -118,3 +142,20 @@ metric, normalization and deadlines. Fenced command examples follow the user's preferred instruction style. No starter or reference solution is shipped. Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task methodology-inspiration-retrieval +bash scripts/run_task.sh --task methodology-inspiration-retrieval --gpu-device 0 --model YOUR_AGENT_MODEL +``` + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Task reference](https://aclanthology.org/2025.acl-long.1390/). diff --git a/tasks/methodology-inspiration-retrieval/environment/Dockerfile b/tasks/methodology-inspiration-retrieval/environment/Dockerfile index f004382..4b96a49 100644 --- a/tasks/methodology-inspiration-retrieval/environment/Dockerfile +++ b/tasks/methodology-inspiration-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/methodology-inspiration-retrieval/task.toml b/tasks/methodology-inspiration-retrieval/task.toml index 5d376c4..f58edff 100644 --- a/tasks/methodology-inspiration-retrieval/task.toml +++ b/tasks/methodology-inspiration-retrieval/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/methodology-inspiration-retrieval" -version = "0.1.1" +version = "1.0.0" description = "Retrieve papers that provide methodological inspiration for scientific research problems." authors = [{ name = "Bingyu" }] keywords = ["information-retrieval", "scientific-literature", "methodology", "gpu"] diff --git a/tasks/methodology-inspiration-retrieval/tests/Dockerfile b/tasks/methodology-inspiration-retrieval/tests/Dockerfile index dae2363..31fbb25 100644 --- a/tasks/methodology-inspiration-retrieval/tests/Dockerfile +++ b/tasks/methodology-inspiration-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/methodology-inspiration-retrieval/tests/rewardkit_deepseek.py b/tasks/methodology-inspiration-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/methodology-inspiration-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/methodology-inspiration-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/multilingual-vector-postprocessing/README.md b/tasks/multilingual-vector-postprocessing/README.md index 28013b2..9f59f3a 100644 --- a/tasks/multilingual-vector-postprocessing/README.md +++ b/tasks/multilingual-vector-postprocessing/README.md @@ -1,4 +1,16 @@ -# multilingual-vector-postprocessing +# Multilingual Vector Postprocessing + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `multilingual-vector-postprocessing` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | clip((Q-B)/(U-B),0,1), with B and U frozen before agent evaluation | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 10800 s | +| Dataset | Belebele and XQuAD full multilingual pools, original-content-grouped development/test partition | Improve multilingual retrieval by postprocessing frozen 1024-dimensional vectors within one A100 and six hours. @@ -9,7 +21,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Only transformations of frozen vectors are available. Improvements must transfer across datasets and query languages while preserving performance under fixed preparation and query budgets. + +## Verification Explanation + +The verifier normalizes transformed vectors, ranks by FP32 cosine and computes binary nDCG@20 using the full positive sets. It averages equally across query languages and datasets, then normalizes improvement over frozen baseline/reference anchors. Query failures retain their denominator; preparation, resource and integrity failures invalidate the submission. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -22,7 +48,7 @@ python scripts/download_assets.py --task multilingual-vector-postprocessing --ve restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 64 GiB RAM and 100 GiB storage. Agent development has @@ -30,10 +56,8 @@ The task allocates 16 CPU cores, 64 GiB RAM and specified in the task contract; the verifier service timeout also includes its trusted setup and checks. -Both phases require one A100 40GB GPU. Reserve an idle device and configure -both Docker phases to expose that same device explicitly. Verify the actual -container mapping before running; a host CUDA_VISIBLE_DEVICES value alone is -insufficient. Release the reservation after all run containers/processes exit. +Both phases require one A100 40GB GPU. Use `--gpu-device` to select the same +host GPU for both phases. Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or authorized process environment, following the repository README. @@ -44,7 +68,7 @@ are not submission inputs. Invoke the standard entrypoint directly: ```bash -bash scripts/run_task.sh --task multilingual-vector-postprocessing --agent codex --openrouter \ +bash scripts/run_task.sh --task multilingual-vector-postprocessing --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max \ --output "jobs/multilingual-vector-postprocessing-codex-openrouter-luna-max-RUN_ID" ``` @@ -53,7 +77,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/multilingual-vector-postprocessing @@ -64,3 +88,9 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +None. diff --git a/tasks/multilingual-vector-postprocessing/environment/Dockerfile b/tasks/multilingual-vector-postprocessing/environment/Dockerfile index 866eefb..7673bce 100644 --- a/tasks/multilingual-vector-postprocessing/environment/Dockerfile +++ b/tasks/multilingual-vector-postprocessing/environment/Dockerfile @@ -1,7 +1,7 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts --registry=https://registry.npmmirror.com "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" "@earendil-works/pi-coding-agent@${SEARCH_SWE_PI_VERSION}" && npm install --global --registry=https://registry.npmmirror.com "@anthropic-ai/claude-code@${SEARCH_SWE_CLAUDE_CODE_VERSION}" && codex --version | grep -Fx "codex-cli ${SEARCH_SWE_CODEX_VERSION}" RUN test "$(pi --version)" = "${SEARCH_SWE_PI_VERSION}" && test "$(claude --version)" = "${SEARCH_SWE_CLAUDE_CODE_VERSION} (Claude Code)" RUN claude --help | grep -F "(low, medium, high, xhigh, max)" >/dev/null diff --git a/tasks/multilingual-vector-postprocessing/task.toml b/tasks/multilingual-vector-postprocessing/task.toml index ea1eb91..23ad386 100644 --- a/tasks/multilingual-vector-postprocessing/task.toml +++ b/tasks/multilingual-vector-postprocessing/task.toml @@ -3,7 +3,7 @@ artifacts = ["/logs/agent/collected-submission", "/logs/agent/submission-freeze- [task] name = "search-swe/multilingual-vector-postprocessing" -version = "0.1.2" +version = "1.0.0" description = "Improve multilingual retrieval by postprocessing frozen 1024-dimensional vectors within one A100 and six hours." authors = [{ name = "jianlyu" }] keywords = ["information-retrieval", "multilingual", "vector-postprocessing", "representation-learning", "gpu"] diff --git a/tasks/multilingual-vector-postprocessing/tests/Dockerfile b/tasks/multilingual-vector-postprocessing/tests/Dockerfile index 151f12c..19a373d 100644 --- a/tasks/multilingual-vector-postprocessing/tests/Dockerfile +++ b/tasks/multilingual-vector-postprocessing/tests/Dockerfile @@ -1,7 +1,7 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 ENV LITELLM_LOCAL_MODEL_COST_MAP=true RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple "harbor-rewardkit==0.2.0" "litellm==1.97.0" "nvidia-ml-py==13.580.82" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 RUN npm install --global --registry=https://registry.npmmirror.com "@openai/codex@${REWARDKIT_CODEX_VERSION}" && mkdir -p "/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}/bin" && ln -sf "$(command -v codex)" "/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}/bin/codex" && codex --version | grep -Fx "codex-cli ${REWARDKIT_CODEX_VERSION}" RUN useradd --create-home --uid 10001 --user-group submission && mkdir -p /app /private /task/runtime /logs/verifier /audit COPY . /tests/ diff --git a/tasks/multilingual-vector-postprocessing/tests/rewardkit_deepseek.py b/tasks/multilingual-vector-postprocessing/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/multilingual-vector-postprocessing/tests/rewardkit_deepseek.py +++ b/tasks/multilingual-vector-postprocessing/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/narrative-clue-search/README.md b/tasks/narrative-clue-search/README.md index 9cd6e92..1c945a8 100644 --- a/tasks/narrative-clue-search/README.md +++ b/tasks/narrative-clue-search/README.md @@ -1,10 +1,22 @@ # Narrative-Clue Search -Find a work from a user's long, incomplete and uncertain recollection. +## Task Metadata -**Task:** `narrative-clue-search` · **Metric:** QueryMeanNDCG@10 +| Field | Value | +| --- | --- | +| Task | `narrative-clue-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | QueryMeanNDCG@10 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 12000 s | +| Dataset | TREC ToT 2023 | +| Public queries | 5 | +| Hidden queries | 445 | -## Overview +Find a work from a user's long, incomplete and uncertain recollection. + +### Overview A user may remember a film through scattered plot details, comparisons and uncertain impressions rather than its title. The system must search article @@ -13,15 +25,9 @@ text and return the matching work near the start of its ranking. This task uses human recollections and their released relevance judgments. It leaves query interpretation and retrieval design to the implementation. -## What This Task Tests - -- Extracting useful retrieval evidence from a long recollection. -- Handling uncertain or incorrect remembered details. -- Ranking works from a full article collection. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -44,7 +50,7 @@ verified unaffected. Questions retain their original recollections but omit target links and answer metadata. Public sentence annotations add no hidden-query supervision. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The article collection, questions, relevance labels and ranked-output interface are fixed. Index design, query formulations, candidate selection and ranking @@ -52,13 +58,27 @@ are implementation choices. There is no required BM25 pipeline, fixed rewrite count or ban on semantic reranking. Optional OpenRouter and Jina services follow the [resource policy](environment/docs/available_resources.md). No starter is supplied. -### Environment and Resource Limits +#### Environment and Resource Limits The CPU environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and no GPU. Development has two hours. Index construction has 3,600 seconds; each of two query calls has 2,700 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Extracting useful retrieval evidence from a long recollection. +- Handling uncertain or incorrect remembered details. +- Ranking works from a full article collection. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting files under `/app`. Each request contains the query ID and recollection text. @@ -70,9 +90,9 @@ Use the supplied index, output, cache and temporary paths for generated files. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Compute nDCG@10 using complete original qrels, then average equally over 445 held-out questions. @@ -81,7 +101,7 @@ Four category means based on original human uncertainty annotations and a fixed median narrative length are diagnostics. They do not change query weights or introduce model-generated categories. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.40178618363049623` and integer score anchor `a = 20` on the 0–100 scale: @@ -96,7 +116,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -105,7 +125,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -115,7 +135,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -139,7 +159,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/narrative-clue-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -152,3 +172,7 @@ credentials or hardware. Results are written under `jobs/narrative-clue-search/` | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/narrative-clue-search/environment/Dockerfile b/tasks/narrative-clue-search/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/narrative-clue-search/environment/Dockerfile +++ b/tasks/narrative-clue-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/narrative-clue-search/task.toml b/tasks/narrative-clue-search/task.toml index d827ca4..bf0f2fd 100644 --- a/tasks/narrative-clue-search/task.toml +++ b/tasks/narrative-clue-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/narrative-clue-search" -version = "0.5.0" +version = "1.0.0" description = "Build a document search system for long, uncertain recollections." authors = [{ name = "Haoran Jin" }] keywords = ["information-retrieval", "semantic-retrieval", "query-rewriting", "narrative"] diff --git a/tasks/narrative-clue-search/tests/Dockerfile b/tasks/narrative-clue-search/tests/Dockerfile index f9bd398..cdf7647 100644 --- a/tasks/narrative-clue-search/tests/Dockerfile +++ b/tasks/narrative-clue-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,5 +30,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/narrative-clue-search/tests/rewardkit_deepseek.py b/tasks/narrative-clue-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/narrative-clue-search/tests/rewardkit_deepseek.py +++ b/tasks/narrative-clue-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/negative-constraint-retrieval/README.md b/tasks/negative-constraint-retrieval/README.md index 94872d9..e57dcd4 100644 --- a/tasks/negative-constraint-retrieval/README.md +++ b/tasks/negative-constraint-retrieval/README.md @@ -1,6 +1,18 @@ # Negative-constraint retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `negative-constraint-retrieval` | +| Version | 1.0.0 | +| Authors | Cooki-e | +| Primary metric | Accuracy@1 | +| Compute | 4 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 5400 s | +| Dataset | NegConstraint original passages plus broad local Qwen3.5-9B augmentation; authored natural-language queries | +| Public queries | 10 | +| Hidden queries | 10 | Build a Top-1 search engine over 8,011 fixed passages. The 20 authored queries require a named subject and semantic exclusions such as roles, events, named @@ -15,13 +27,13 @@ execution/output still gives zero for the submission. A required RewardKit traje resource and integrity policy; an unavailable or failed audit gives zero final reward and is an infrastructure failure, not evidence of poor retrieval. -## Fixed inputs and provenance +### Fixed inputs and provenance The original text comes from `xgl-git/NS-IR-main` at historical commit `ebdfa4f62b3c2af01d43cf30560910fba3003c73`, archive `NegConstraint.zip` with SHA-256 `a10dc25f13b816b4cd7c17c221b2916a97c981b0b8f086dbf3826a1c6e47cbfe`. Its 3,946 raw document IDs become 3,907 documents after exact-text deduplication. -All original passage texts and IDs remain unchanged. Version 0.3.0 adds 4,104 +All original passage texts and IDs remain unchanged. The corpus includes 4,104 source-conditioned passages generated locally with Qwen3.5-9B across 365 topic cohorts: 3,889 background passages and 215 evaluated-subject negatives. The same generation, fact-preservation and automatic filtering policy covers both groups. @@ -51,9 +63,17 @@ or exhaustive certification of every generated factual detail. The upstream dataset redistribution-license question remains unresolved. Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. -## Difficulty evidence +## Change Log -Version 0.3.0 expands the corpus while retaining the 20 conversational queries, +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +### Difficulty evidence + +The expanded corpus uses the 20 conversational queries, labels, unique-positive texts, 8-second deadline and 4-CPU verifier. Evaluated subjects now have 6–20 matching passages, and 346 additional background cohorts also receive generated passages. No query, label, split, source map or generation @@ -117,16 +137,18 @@ scored raw 1/10 on hidden queries, with no query timeouts and 3.318–5.032 seco per query. Public self-tests scored 10/10. Final Harbor reward was zero because RewardKit retried an already completed audit after a recovered stream event and exhausted its audit deadline. v0.2.3 fixes that transport handling, with unchanged -queries, retrieval scoring and budgets. A verifier-only regrade is prepared but -awaits explicit approval after an automatic approval rejection; it has not run. -See `VALIDATION.md` for evidence and the limits of this single trial. +queries, retrieval scoring and budgets. These are historical trial records. +The reconciled v0.3.0 verifier-only run scored 1/10 with a passing audit; +see [RUN_RESULTS.md](RUN_RESULTS.md) for that saved-submission measurement. The Agent implements the search engine under `/app` from the documented interfaces. Author construction evidence lives outside this task package at `task_development/negative-constraint-retrieval/construction/final-20/` in the surrounding author workspace, including `review.md` and `evidence.jsonl`. -## Interface, runtime and isolation +## Verification Explanation + +### Interface, runtime and isolation `instruction.md` is the complete Agent request; `raw-instruction.md` is its structured authoring source. Corpus records use `doc_id,text`, queries use @@ -154,7 +176,7 @@ only index/cache/temp and output locations are writable. `/tests`, labels, trajectory and reward logs are root-private. Judge credentials are not passed to submitted programs. Both phases use the pinned repository CPU base image. -## Validation and local configuration +### Validation and local configuration From the Search-SWE root: @@ -166,12 +188,9 @@ docker build -t search-swe-local:nc-agent tasks/negative-constraint-retrieval/en docker build -t search-swe-local:nc-verifier tasks/negative-constraint-retrieval/tests ``` -The strict task/release check reports the three unpublished revisions until -publication; it must not be made green using fabricated SHAs. Author staging -checks use `check_release(..., packages=[task], allow_unpublished=True)` with the -local task-only staging manifest. See `VALIDATION.md` for actual executed checks -and remaining prerequisites. Static checks and deterministic fixtures are not -an end-to-end coding-model trial. +Public inputs use the immutable revisions, exact sizes and hashes in +`assets.json`. [VALIDATION.md](VALIDATION.md) retains the original development +checks; [RUN_RESULTS.md](RUN_RESULTS.md) records the reconciled verifier result. For a future configured trial, preserve any existing ignored checkout `.env` and set the chosen coding model/provider and its credential, for example @@ -179,11 +198,11 @@ and set the chosen coding model/provider and its credential, for example `VERIFIER_OPENAI_BASE_URL=https://openrouter.ai/api/v1` and `VERIFIER_OPENAI_API_KEY`. Optional submission resources need the shared `OPENROUTER_API_KEY` and/or `JINA_API_KEY`, as in `agentic-search`. -Do not share secret values in chat or write them into artifacts. The archived v0.1.1 luna trial achieved 8/10 with about 59 seconds for the whole batch; it does not validate the revised setting. The v0.2.2 trial and pending v0.2.3 audit regrade are recorded above. For a separately authorized fresh trial, launch with +Do not share secret values in chat or write them into artifacts. The archived v0.1.1 luna trial achieved 8/10 with about 59 seconds for the whole batch; it does not validate the revised setting. The historical trial and completed saved-submission regrade are recorded above. For a separately authorized fresh trial, launch with `bash scripts/run_task.sh --task negative-constraint-retrieval` and the chosen model/provider flags, previewing with `--dry-run` first. -## Structural precedent +### Structural precedent Reviewed base: `46197a43f8cbb364e72b6d63d5db873ae7d2cac8`. `tasks/reasoning-query-rewriting` supplied the full-corpus search CLI, @@ -196,3 +215,9 @@ remote PR collision checking was not performed. The two existing repository test inventories and the network-policy table were extended with this task's network profile and RewardKit use. Existing unrelated network-policy and join-aware task changes were preserved. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +## References + +None. diff --git a/tasks/negative-constraint-retrieval/environment/Dockerfile b/tasks/negative-constraint-retrieval/environment/Dockerfile index a4d8eac..2e0f6c3 100644 --- a/tasks/negative-constraint-retrieval/environment/Dockerfile +++ b/tasks/negative-constraint-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/negative-constraint-retrieval/task.toml b/tasks/negative-constraint-retrieval/task.toml index 97bd45b..aad929b 100644 --- a/tasks/negative-constraint-retrieval/task.toml +++ b/tasks/negative-constraint-retrieval/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/negative-constraint-retrieval" -version = "0.3.0" +version = "1.0.0" description = "Build a Top-1 search engine that satisfies positive and negative semantic constraints over a fixed passage corpus." authors = [{ name = "Cooki-e" }] keywords = ["information-retrieval", "negation", "constraints", "semantic-retrieval"] diff --git a/tasks/negative-constraint-retrieval/tests/Dockerfile b/tasks/negative-constraint-retrieval/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/negative-constraint-retrieval/tests/Dockerfile +++ b/tasks/negative-constraint-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/negative-constraint-retrieval/tests/rewardkit_deepseek.py b/tasks/negative-constraint-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/negative-constraint-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/negative-constraint-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/paper-metadata-retrieval/README.md b/tasks/paper-metadata-retrieval/README.md index e7879f7..07f07ed 100644 --- a/tasks/paper-metadata-retrieval/README.md +++ b/tasks/paper-metadata-retrieval/README.md @@ -1,13 +1,39 @@ # Paper Metadata Retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `paper-metadata-retrieval` | +| Version | 1.0.0 | +| Authors | Cooki-e | +| Primary metric | Accuracy@1 | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 5400 s | +| Dataset | Fixed full-text paper collection with authored requests | +| Public queries | 3 | +| Hidden queries | 8 | Author: Cooki-e. Implement reusable Top-1 paper retrieval for natural-language research needs with source conditions. The CPU package provides 70,000 complete papers, 3 public validation requests and 8 verifier-only test requests. No training or fine-tuning. The public database is `data/corpus.sqlite`, with a single raw `documents` table containing document IDs, bibliographic IDs, titles, abstracts, complete text, metadata JSON and body checksums. It contains no search index, answer annotations, extracted eligibility masks or authoring traces. Source-collection and construction provenance is retained separately in the authoring workspace. Each paper keeps its actual per-record bibliographic and licensing metadata; no blanket redistribution license is asserted for the collection. +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + The fixed hidden split consists of the eight requests missed at Top-1 by all 15 original BM25/BGE configurations and all 8 subsequently measured Qwen3 8B configurations. Public validation uses three of the other twelve requests, selected by fewest Qwen Top-1 hits and then lowest mean reciprocal rank. Public and hidden questions and target papers are disjoint; the remaining nine requests are excluded. These are deliberately selected challenge questions, not an unbiased evaluation sample. Source/condition and retrieved-alternative review supports the labels; exhaustive corpus-wide uniqueness and independent human blind review have not been established. +The system must combine research relevance with source conditions over 70,000 full papers. The challenge split contains requests missed by the recorded lexical and neural baselines, with only one returned paper and a per-query deadline. + +## Verification Explanation + +### Contract and validation + `instruction.md` is the agent request, and `raw-instruction.md` is its structured contract. The public instruction names no source collection and prescribes no retrieval method. Generic CPU and API documents live under `environment/docs/`. The agent receives only the database, public validation and generic docs. Private questions and labels are copied into the separate verifier image under `/tests/data`; they are never mounted into the agent. At verification, submitted code runs as UID 10001 with `/app` read-only and filtered credentials. Judge keys, verifier files and trajectories remain root-only. Only `/app` and the agent trajectory are transferred. A build has 1,800 seconds; eight sequential single-query invocations each have 30 seconds, including all API time. Query timeouts lose only that query. Other contract failures invalidate the submission. Accuracy@1 is averaged across all eight queries, gated by a separate trajectory audit. @@ -23,3 +49,20 @@ python scripts/check_tasks.py tasks/paper-metadata-retrieval ``` Completed checks and unrun validation layers are recorded in `VALIDATION.md`. Oracle mechanics checks are not coding-agent solutions or live trajectory-judge evidence. + +The executable contract is in [instruction.md](instruction.md); [raw-instruction.md](raw-instruction.md) records the structured specification. See [tests/](tests/) for the separate verifier and [task.toml](task.toml) for phase budgets. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task paper-metadata-retrieval +bash scripts/run_task.sh --task paper-metadata-retrieval --model YOUR_AGENT_MODEL +``` + +## References + +None. diff --git a/tasks/paper-metadata-retrieval/environment/Dockerfile b/tasks/paper-metadata-retrieval/environment/Dockerfile index a4d8eac..2e0f6c3 100644 --- a/tasks/paper-metadata-retrieval/environment/Dockerfile +++ b/tasks/paper-metadata-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/paper-metadata-retrieval/task.toml b/tasks/paper-metadata-retrieval/task.toml index 3bacd7c..e1290d1 100644 --- a/tasks/paper-metadata-retrieval/task.toml +++ b/tasks/paper-metadata-retrieval/task.toml @@ -4,7 +4,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/paper-metadata-retrieval" -version = "0.1.1" +version = "1.0.0" description = "Build a search system that identifies a paper satisfying a research need and its stated source conditions." authors = [{ name = "Cooki-e" }] keywords = ["information-retrieval", "scientific-literature", "metadata", "constraints"] diff --git a/tasks/paper-metadata-retrieval/tests/Dockerfile b/tasks/paper-metadata-retrieval/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/paper-metadata-retrieval/tests/Dockerfile +++ b/tasks/paper-metadata-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/paper-metadata-retrieval/tests/rewardkit_deepseek.py b/tasks/paper-metadata-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/paper-metadata-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/paper-metadata-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/protein-text-bidirectional-retrieval/README.md b/tasks/protein-text-bidirectional-retrieval/README.md index b75df5c..58520ae 100644 --- a/tasks/protein-text-bidirectional-retrieval/README.md +++ b/tasks/protein-text-bidirectional-retrieval/README.md @@ -1,4 +1,15 @@ -# protein-text-bidirectional-retrieval +# Protein Text Bidirectional Retrieval + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `protein-text-bidirectional-retrieval` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | Equal-weight mean of per-dataset strict reciprocal-top1 paired success fractions | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 3600 s / 130000 s | Learn compact 64-dimensional protein/text embeddings and maximize strict reciprocal-top1 pair recovery under online latency and submission limits. @@ -9,7 +20,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Compact protein and text representations must recover the same pair as the top result in both directions. The 64-dimensional space, deployment size and online latency limits constrain both representation learning and search. + +## Verification Explanation + +A pair succeeds only when its independently executed protein-to-text and text-to-protein queries both rank the counterpart first. The verifier averages those success fractions equally across the two datasets. All official pairs remain in the denominator, and deployment, latency, resource and integrity gates apply. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -23,7 +48,7 @@ restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document and [SOURCES.md](SOURCES.md). -## Run +### Run The task allocates 16 CPU cores, 64 GiB RAM and 100 GiB storage. Agent development has @@ -31,10 +56,8 @@ The task allocates 16 CPU cores, 64 GiB RAM and specified in the task contract; the verifier service timeout also includes its trusted setup and checks. -Both phases require one A100 40GB GPU. Reserve an idle device and configure -both Docker phases to expose that same device explicitly. Verify the actual -container mapping before running; a host CUDA_VISIBLE_DEVICES value alone is -insufficient. Release the reservation after all run containers/processes exit. +Both phases require one A100 40GB GPU. Use `--gpu-device` to select the same +host GPU for both phases. Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or authorized process environment, following the repository README. @@ -45,7 +68,7 @@ are not submission inputs. Invoke the standard entrypoint directly: ```bash -bash scripts/run_task.sh --task protein-text-bidirectional-retrieval --agent codex --openrouter \ +bash scripts/run_task.sh --task protein-text-bidirectional-retrieval --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max \ --output "jobs/protein-text-bidirectional-retrieval-codex-openrouter-luna-max-RUN_ID" ``` @@ -54,7 +77,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/protein-text-bidirectional-retrieval @@ -65,3 +88,11 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Task reference](https://aclanthology.org/2024.acl-long.324/). + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/protein-text-bidirectional-retrieval/environment/Dockerfile b/tasks/protein-text-bidirectional-retrieval/environment/Dockerfile index 426abeb..9a4cac0 100644 --- a/tasks/protein-text-bidirectional-retrieval/environment/Dockerfile +++ b/tasks/protein-text-bidirectional-retrieval/environment/Dockerfile @@ -3,9 +3,9 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 RUN /opt/conda/bin/python -m pip install --no-cache-dir --index-url https://pypi.tuna.tsinghua.edu.cn/simple "nvidia-ml-py==13.580.82" "peft==0.20.0" RUN useradd --create-home --uid 10001 --user-group submission \ && mkdir -p /app /task/corpus /task/models /task/runtime /task/public /task/docs -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/protein-text-bidirectional-retrieval/task.toml b/tasks/protein-text-bidirectional-retrieval/task.toml index 27dbc45..a12c4f3 100644 --- a/tasks/protein-text-bidirectional-retrieval/task.toml +++ b/tasks/protein-text-bidirectional-retrieval/task.toml @@ -4,7 +4,7 @@ artifacts = ["/var/tmp/search-swe-submission.tar", "/logs/agent/trajectory.json" [task] name = "search-swe/protein-text-bidirectional-retrieval" -version = "0.6.1" +version = "1.0.0" description = "Learn compact 64-dimensional protein/text embeddings and maximize strict reciprocal-top1 pair recovery under online latency and submission limits." authors = [{name = "jianlyu", email = "2512674094@qq.com"}] keywords = ["protein-retrieval", "cross-modal", "contrastive-learning", "online-retrieval", "gpu"] diff --git a/tasks/protein-text-bidirectional-retrieval/tests/Dockerfile b/tasks/protein-text-bidirectional-retrieval/tests/Dockerfile index 44c3c83..9eed37f 100644 --- a/tasks/protein-text-bidirectional-retrieval/tests/Dockerfile +++ b/tasks/protein-text-bidirectional-retrieval/tests/Dockerfile @@ -3,7 +3,7 @@ ENV LITELLM_LOCAL_MODEL_COST_MAP=true RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --index-url https://pypi.tuna.tsinghua.edu.cn/simple \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" "nvidia-ml-py==13.580.82" "peft==0.20.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/protein-text-bidirectional-retrieval/tests/rewardkit_deepseek.py b/tasks/protein-text-bidirectional-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/protein-text-bidirectional-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/protein-text-bidirectional-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/quantity-aware-retrieval/README.md b/tasks/quantity-aware-retrieval/README.md index 6049df1..9d707c3 100644 --- a/tasks/quantity-aware-retrieval/README.md +++ b/tasks/quantity-aware-retrieval/README.md @@ -1,14 +1,26 @@ # Quantity-aware Retrieval -Current run and release status (2026-10-23): [RUN_RESULTS.md](RUN_RESULTS.md). Earlier validation-stage statements are superseded by that dated record. - -Author: **Bingyu**. This is a local, unpublished Search-SWE task package over +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `quantity-aware-retrieval` | +| Version | 1.0.0 | +| Authors | Bingyu | +| Primary metric | Normalized binary nDCG@10 | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 4800 s | +| Dataset | MedQuant, original passages and qrels | +| Public queries | 0 | +| Hidden queries | 210 | + +Author: **Bingyu**. This Search-SWE task uses MedQuant. The required deliverable is an executable retrieval system. Resources include labeled examples, Qwen3-Embedding-0.6B and documented OpenRouter/Jina APIs. No baseline implementation, authoring helper, or precomputed document vectors are provided to the agent. -## Data and split +### Data and split | Input | Size | | --- | ---: | @@ -36,7 +48,19 @@ fixtures are in `../../../task_development/quantity-aware-retrieval/`, outside this task. No `authoring/`, `bundled/`, `environment/tools/` or public starter is part of the package. -## Evaluation +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +Clinical relevance depends on numerical conditions as well as topic similarity. The system must interpret quantities and units from the supplied text and generalize from synthetic examples to the original hidden questions. + +## Verification Explanation + +### Evaluation The engine implements `/app/build.sh` and `/app/run.sh`, as specified in `instruction.md`. A separate verifier runs the submission as UID 10001, with @@ -72,7 +96,30 @@ a 21,600-second (6-hour) agent phase shared by implementation, optional training validation. Both phases use the repository shared CUDA 13.0/Python 3.12 image. The GPU environment document is copied from `docker/gpu/environment.md`. -## Provenance and limitations +### Local operation and validation + +Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. + +```bash +python scripts/download_assets.py --task quantity-aware-retrieval --verify-only +python -m unittest discover -s tasks/quantity-aware-retrieval/tests -p 'test_*.py' +python scripts/run_task.py --task quantity-aware-retrieval --agent codex --openrouter --dry-run +``` + +The coding model and verifier credentials are configured in the repository's +ignored `.env`; do not place credentials in this task. Task APIs are optional. +The verifier judge requires `VERIFIER_OPENAI_BASE_URL` and +`VERIFIER_OPENAI_API_KEY`. A real coding-model trial and live judge health check +are separate from package/unit/container smoke tests. See `VALIDATION.md` for +commands actually run, observed results, and unperformed release/evaluation steps. + +See [RUN_RESULTS.md](RUN_RESULTS.md) for recorded run outcomes and judge health, and [VALIDATION.md](VALIDATION.md) for validation evidence. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +### Provenance and limitations MedQuant and the underlying training material are from [QuantityAwareRankers](https://github.com/satya77/QuantityAwareRankers), associated @@ -99,20 +146,3 @@ files were independently checked against their manifest sizes and SHA-256. The model card declares Apache-2.0. The enclosing repository's code license does not establish redistribution rights for the MedQuant/training sources; those terms remain unresolved and are recorded in the published source notices. - -## Local operation and validation - -Public inputs are published and pinned to immutable revisions in `assets.json`, with exact sizes and SHA-256 hashes. Unchanged model pins are retained. See [RUN_RESULTS.md](RUN_RESULTS.md) for the current run evidence and publication status. Existing source terms and attribution remain applicable. - -```bash -python scripts/download_assets.py --task quantity-aware-retrieval --verify-only -python -m unittest discover -s tasks/quantity-aware-retrieval/tests -p 'test_*.py' -python scripts/run_task.py --task quantity-aware-retrieval --agent codex --openrouter --dry-run -``` - -The coding model and verifier credentials are configured in the repository's -ignored `.env`; do not place credentials in this task. Task APIs are optional. -The verifier judge requires `VERIFIER_OPENAI_BASE_URL` and -`VERIFIER_OPENAI_API_KEY`. A real coding-model trial and live judge health check -are separate from package/unit/container smoke tests. See `VALIDATION.md` for -commands actually run, observed results, and unperformed release/evaluation steps. diff --git a/tasks/quantity-aware-retrieval/environment/Dockerfile b/tasks/quantity-aware-retrieval/environment/Dockerfile index 3e8e43d..969f3a3 100644 --- a/tasks/quantity-aware-retrieval/environment/Dockerfile +++ b/tasks/quantity-aware-retrieval/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/quantity-aware-retrieval/task.toml b/tasks/quantity-aware-retrieval/task.toml index 27f698d..8874de7 100644 --- a/tasks/quantity-aware-retrieval/task.toml +++ b/tasks/quantity-aware-retrieval/task.toml @@ -4,7 +4,7 @@ artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/quantity-aware-retrieval" -version = "0.1.1" +version = "1.0.0" description = "Build a retrieval system that ranks clinical passages by relevance to natural-language numerical conditions." authors = [{ name = "Bingyu" }] keywords = ["information-retrieval", "numerical-reasoning", "units", "gpu"] diff --git a/tasks/quantity-aware-retrieval/tests/Dockerfile b/tasks/quantity-aware-retrieval/tests/Dockerfile index d28ab66..61bbb89 100644 --- a/tasks/quantity-aware-retrieval/tests/Dockerfile +++ b/tasks/quantity-aware-retrieval/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/quantity-aware-retrieval/tests/rewardkit_deepseek.py b/tasks/quantity-aware-retrieval/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/quantity-aware-retrieval/tests/rewardkit_deepseek.py +++ b/tasks/quantity-aware-retrieval/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/query-encoder-alignment/README.md b/tasks/query-encoder-alignment/README.md index 5a8e5c5..5d1f75f 100644 --- a/tasks/query-encoder-alignment/README.md +++ b/tasks/query-encoder-alignment/README.md @@ -1,10 +1,22 @@ # Query-side Encoder Alignment -Improve a compact query encoder against a document collection whose embeddings cannot change. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `query-encoder-alignment` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | Accuracy@1 error reduction above frozen 0.6B ridge, after runtime gate | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 43200 s / 5700 s | +| Dataset | local frozen retrieval corpus | +| Public queries | 100 | +| Hidden queries | 100 | -**Task:** `query-encoder-alignment` v0.2.0 · **Metric:** Accuracy@1 error reduction above frozen ridge, with a runtime gate +Improve a compact query encoder against a document collection whose embeddings cannot change. -## Overview +### Overview Replacing or tuning a query encoder does not automatically preserve its compatibility with an existing document index. This task fixes the document @@ -16,16 +28,9 @@ reference. It is therefore a joint representation and execution problem: a query vector must land in the right document space, and the complete query pipeline must stay within the relative runtime budget. -## What This Task Tests +### Task Setup -- Aligning a compact query representation with a frozen document-vector space. -- Preserving document identities and vector-row correspondence. -- Balancing query encoding quality with end-to-end execution time. -- Packaging deterministic inference and retrieval for offline verification. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -42,7 +47,7 @@ signals on public inputs. Its private permissions keep it inaccessible to the unprivileged final process; it is not an allowed final inference backbone. There are 100 hidden evaluation queries, isolated from the agent environment. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The query-side system may be adapted within the supplied 0.6B backbone requirement. Document vectors must not be modified, re-encoded, replaced, @@ -51,24 +56,35 @@ or reordered. Task network access during development is restricted to evaluation labels, or external retrieval results. The final submission must be self-contained and offline. -### Environment and Resource Limits +#### Environment and Resource Limits The Python 3.12 runtime uses CUDA-enabled PyTorch, 8 CPUs, 32 GiB memory, and a 100 GiB storage budget. The local Docker backend does not enforce that budget with a filesystem quota; the completed calibration stayed well below it. Both Compose configurations and `task.toml` request **one -NVIDIA GPU**. Set `QUERY_ALIGNMENT_GPU_DEVICE` to the reserved host GPU UUID or -index before rendering/building a trial. Both phases explicitly map this device. -For local Docker, the shared launcher works around Harbor 0.22.0's GPU -capability preflight with `--override-gpus 0`; the Compose reservations still -provide the GPU to both containers. +NVIDIA GPU**. Use `--gpu-device ` to select the same host GPU for both phases. The agent has 12 hours. Task-specific verification has 70 minutes inside the 95-minute Harbor verifier phase, with the integrity check bounded separately. The candidate must finish within **60% of the measured reference runtime**. The evaluation runner also caps each reference/candidate run at 1,800 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Aligning a compact query representation with a frozen document-vector space. +- Preserving document identities and vector-row correspondence. +- Balancing query encoding quality with end-to-end execution time. +- Packaging deterministic inference and retrieval for offline verification. + +## Verification Explanation + +### Submission Contract The complete deliverable lives under `/app/submission` and includes an executable `run.sh`. It accepts document vectors and query files, and returns one ranked @@ -78,9 +94,9 @@ Outputs must be deterministic and preserve query order, and execution must leave no background processes. [instruction.md](instruction.md) defines the complete interface; no separate submitted build step is required. -## Evaluation +### Evaluation -### Search Quality +#### Search Quality The public quality reference freezes Qwen3-Embedding-0.6B and fits centered ridge once on all public corpus/text-vector pairs. Its fixed settings and @@ -105,7 +121,7 @@ Both references exclude same-ID documents and all canonical-text copies: NFKC, collapse whitespace, casefold. They rank all unchanged vectors, filter, then refill; ties use corpus row order. The grader rejects excluded results. -### Correctness and Resource Gates +#### Correctness and Resource Gates Candidate wall time must be at most `0.60 × starter wall time`, measured on the same hidden workload. Missing the gate makes reward zero even if accuracy is @@ -116,7 +132,7 @@ sets time and a second fresh process checks byte-identical results. Timeout cleanup time is logged separately. Vector checksums and shape, query coverage/order, result validity, and clean process termination are also checked. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward The normalized quality reward is accepted only after runtime and validity gates and the independent trajectory audit pass. Failure at any gate produces zero. @@ -127,7 +143,7 @@ network denial. The audit uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0 with verifier-only DeepSeek endpoint and key settings. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -142,8 +158,7 @@ so the reference model's private file permissions retain their intended effect. ```bash python scripts/download_assets.py --task query-encoder-alignment python scripts/download_assets.py --task query-encoder-alignment --verify-only -export QUERY_ALIGNMENT_GPU_DEVICE=YOUR_RESERVED_GPU_UUID -bash scripts/run_task.sh --task query-encoder-alignment --agent codex --openrouter \ +bash scripts/run_task.sh --task query-encoder-alignment --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max --output jobs/query-encoder-alignment-UNIQUE_RUN_ID ``` @@ -155,7 +170,7 @@ Configure `AGENT_OPENROUTER_API_KEY`, `VERIFIER_OPENAI_API_KEY` and command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -168,7 +183,7 @@ assets, credentials, or hardware. | [Starter notes](environment/starter/README.md) | Teacher-side baseline and public validation | | [Resource policy](environment/docs/available_resources.md) | Development access and offline submission rules | -## Data provenance and local checks +### Data provenance and local checks The packaged ArguAna corpus matches mteb/arguana commit `6c1bcf74b13dfd823aff056b79d4d93e702f19c7`, with task-specific IDs; its source @@ -212,3 +227,9 @@ This one accepted Luna/max solve supports the approximate 20-point difficulty target; the historical Opus artifact replay scores below 50. A legal author solution achieving 100% hidden accuracy has not been demonstrated; synthetic grader tests establish the full-score numeric boundary only. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Task reference](https://arxiv.org/abs/2506.05176v3). diff --git a/tasks/query-encoder-alignment/environment/Dockerfile b/tasks/query-encoder-alignment/environment/Dockerfile index 14d060b..b6d40b0 100644 --- a/tasks/query-encoder-alignment/environment/Dockerfile +++ b/tasks/query-encoder-alignment/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/query-encoder-alignment/environment/docker-compose.yaml b/tasks/query-encoder-alignment/environment/docker-compose.yaml index 096164f..46747b5 100644 --- a/tasks/query-encoder-alignment/environment/docker-compose.yaml +++ b/tasks/query-encoder-alignment/environment/docker-compose.yaml @@ -6,7 +6,7 @@ services: reservations: devices: - driver: nvidia - device_ids: ["${QUERY_ALIGNMENT_GPU_DEVICE:?Set the reserved GPU UUID or index}"] + count: 1 capabilities: [gpu] volumes: - type: bind diff --git a/tasks/query-encoder-alignment/task.toml b/tasks/query-encoder-alignment/task.toml index e11c7d2..aac1e5c 100644 --- a/tasks/query-encoder-alignment/task.toml +++ b/tasks/query-encoder-alignment/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/query-encoder-alignment" -version = "0.2.0" +version = "1.0.0" description = "Optimize a query encoder based on a supplied small backbone against fixed document vectors." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "query-encoder", "distillation", "embedding", "gpu"] diff --git a/tasks/query-encoder-alignment/tests/Dockerfile b/tasks/query-encoder-alignment/tests/Dockerfile index 6dd03ac..62f0613 100644 --- a/tasks/query-encoder-alignment/tests/Dockerfile +++ b/tasks/query-encoder-alignment/tests/Dockerfile @@ -9,7 +9,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --timeout 120 --retries 10 --resume-retries 10 \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/query-encoder-alignment/tests/docker-compose.yaml b/tasks/query-encoder-alignment/tests/docker-compose.yaml index 5092e8c..1686544 100644 --- a/tasks/query-encoder-alignment/tests/docker-compose.yaml +++ b/tasks/query-encoder-alignment/tests/docker-compose.yaml @@ -6,7 +6,7 @@ services: reservations: devices: - driver: nvidia - device_ids: ["${QUERY_ALIGNMENT_GPU_DEVICE:?Set the reserved GPU UUID or index}"] + count: 1 capabilities: [gpu] volumes: - type: bind diff --git a/tasks/query-encoder-alignment/tests/rewardkit_deepseek.py b/tasks/query-encoder-alignment/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/query-encoder-alignment/tests/rewardkit_deepseek.py +++ b/tasks/query-encoder-alignment/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/reasoning-query-rewriting/README.md b/tasks/reasoning-query-rewriting/README.md index 2b8e3f2..bca14a9 100644 --- a/tasks/reasoning-query-rewriting/README.md +++ b/tasks/reasoning-query-rewriting/README.md @@ -1,10 +1,22 @@ # Reasoning-intensive Query Rewriting -Build a retrieval system that connects difficult biology questions to supporting documents. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `reasoning-query-rewriting` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | Accuracy@3 | +| Compute | 16 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 5400 s | +| Dataset | Bright biology short-document | +| Public queries | 10 | +| Hidden queries | 10 | -**Task:** `reasoning-query-rewriting` · **Metric:** Accuracy@3 with per-query partial credit +Build a retrieval system that connects difficult biology questions to supporting documents. -## Overview +### Overview A question can describe a biological mechanism without using the terminology of the document that explains it. This task tests whether a coding agent can @@ -16,16 +28,9 @@ retrieval and ranking. A useful rewrite must improve access to evidence without losing the original information need. The benchmark evaluates the executable system on held-out questions, rather than judging the wording of its rewrites. -## What This Task Tests +### Task Setup -- Translating reasoning-intensive queries into effective retrieval representations. -- Combining retrieval stages while keeping their evidence tied to the supplied corpus. -- Using a very small public validation set without specializing to known answers. -- Packaging indexing and query execution for a fresh verifier environment. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -47,7 +52,7 @@ content-only BM25, BM25 Top-100 + Qwen3-Reranker-8B, and Qwen3-Embedding-8B baselines. After shuffling, BM25 was recomputed and the neural scores were rechecked with the new tie-break order; all selected cases still missed. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The corpus and submission interface are fixed. Query rewriting, indexing, candidate selection, and ranking are implementation choices. Selected @@ -56,7 +61,7 @@ OpenRouter and Jina resources are permitted under the not authorize arbitrary external search, answer services, or evaluation data. The resource document lists exact providers and model IDs. -### Environment and Resource Limits +#### Environment and Resource Limits The CPU Python 3.12 environment provides 16 CPUs, 32 GiB memory, and 100 GiB storage. The agent has 120 minutes; the Harbor verifier phase has 90 minutes. @@ -64,7 +69,22 @@ Within verification, the current grader allows 1,800 seconds for the submitted build and 900 seconds for the query run. These submission deadlines are distinct from Docker image construction. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Translating reasoning-intensive queries into effective retrieval representations. +- Combining retrieval stages while keeping their evidence tied to the supplied corpus. +- Using a very small public validation set without specializing to known answers. +- Packaging indexing and query execution for a fresh verifier environment. + +## Verification Explanation + +### Submission Contract The deliverable is an executable system under `/app`, with `build.sh` preparing an index and `run.sh` returning three ranked corpus documents per question. @@ -75,9 +95,9 @@ See [instruction.md](instruction.md) for arguments, result schema, and tie-break Transferred files must recreate the required runtime: a development-time background process is not itself a submission. -## Evaluation +### Evaluation -### Search Quality +#### Search Quality A question is a hit when at least one relevant document appears in its top three. The verifier reports Accuracy@3 as the number of hits divided by the ten hidden @@ -85,12 +105,12 @@ questions. For example, seven hits produce `0.7`; each missed query contributes zero for that query. The instruction expresses the same accuracy on a 0–100 scale; the stored metric and reward use the 0–1 scale. -### Correctness and Resource Gates +#### Correctness and Resource Gates Build/run success, complete query coverage, valid corpus IDs, ranking format, and the execution budgets must all pass. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory judge checks task and resource compliance. Final reward equals Accuracy@3 when the execution, output, and integrity checks pass; @@ -99,7 +119,7 @@ The judge assesses the agent's trajectory, not answer quality. It uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0; configure its verifier-only DeepSeek endpoint and key as described in the evaluation guide. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -122,7 +142,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -133,3 +153,7 @@ assets, credentials, or hardware. | [Environment configuration](environment/docker-compose.yaml) | Read-only mounts and hardware requests | | [Verifier](tests/) | Execution, output validation, and scoring implementation | | [Resource policy](environment/docs/available_resources.md) | Permitted models, endpoints, and task-resource credentials | + +## References + +None. diff --git a/tasks/reasoning-query-rewriting/environment/Dockerfile b/tasks/reasoning-query-rewriting/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/reasoning-query-rewriting/environment/Dockerfile +++ b/tasks/reasoning-query-rewriting/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/reasoning-query-rewriting/task.toml b/tasks/reasoning-query-rewriting/task.toml index 93a1ae9..eceabf1 100644 --- a/tasks/reasoning-query-rewriting/task.toml +++ b/tasks/reasoning-query-rewriting/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/reasoning-query-rewriting" -version = "0.3.1" +version = "1.0.0" description = "Build a reasoning-assisted semantic retrieval system over Bright biology short documents." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "search", "biology", "BM25", "query-rewriting", "semantic-retrieval"] diff --git a/tasks/reasoning-query-rewriting/tests/Dockerfile b/tasks/reasoning-query-rewriting/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/reasoning-query-rewriting/tests/Dockerfile +++ b/tasks/reasoning-query-rewriting/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/reasoning-query-rewriting/tests/rewardkit_deepseek.py b/tasks/reasoning-query-rewriting/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/reasoning-query-rewriting/tests/rewardkit_deepseek.py +++ b/tasks/reasoning-query-rewriting/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/resource-bounded-multivector-search/README.md b/tasks/resource-bounded-multivector-search/README.md index 873b4f3..5bf290a 100644 --- a/tasks/resource-bounded-multivector-search/README.md +++ b/tasks/resource-bounded-multivector-search/README.md @@ -1,4 +1,17 @@ -# resource-bounded-multivector-search +# Resource Bounded Multivector Search + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `resource-bounded-multivector-search` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | Normalized relative query-time improvement at Recall@100 >= 0.99 | +| Compute | 8 CPUs; 16 GiB RAM; 32 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 3300 s | +| Public queries | 5000 | +| Hidden queries | 10000 | Reduce complete-workload query latency while retaining at least 99% exact MaxSim Recall@100. @@ -9,7 +22,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +MaxSim scoring compares multiple vectors per document and query. The implementation must reduce total query latency while keeping at least 99% exact Recall@100 within the memory and storage limits. + +## Verification Explanation + +The verifier measures Recall@100 against exhaustive MaxSim neighbors from the supplied vectors and requires at least 0.99 mean recall over all 10,000 queries. Reward then measures normalized complete-workload query-time improvement. Invalid outputs, resource limits and the required integrity audit are checked separately. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -22,7 +49,7 @@ python scripts/download_assets.py --task resource-bounded-multivector-search --v restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 8 CPU cores, 16 GiB RAM and 32 GiB storage. Agent development has @@ -75,7 +102,7 @@ After every container/process for this run has exited, unmount its storage: python tasks/resource-bounded-multivector-search/tools/prepare_storage.py --run-id RUN_ID --unmount ``` -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/resource-bounded-multivector-search @@ -86,3 +113,7 @@ Build both `environment/Dockerfile` and `tests/Dockerfile` with their respective directories as build contexts. Run the public helper and verifier with the task's resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. + +## References + +None. diff --git a/tasks/resource-bounded-multivector-search/environment/Dockerfile b/tasks/resource-bounded-multivector-search/environment/Dockerfile index d357952..98407b6 100644 --- a/tasks/resource-bounded-multivector-search/environment/Dockerfile +++ b/tasks/resource-bounded-multivector-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/resource-bounded-multivector-search/task.toml b/tasks/resource-bounded-multivector-search/task.toml index 3e54c30..d805630 100644 --- a/tasks/resource-bounded-multivector-search/task.toml +++ b/tasks/resource-bounded-multivector-search/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/resource-bounded-multivector-search" -version = "0.3.2" +version = "1.0.0" description = "Reduce complete-workload query latency while retaining at least 99% exact MaxSim Recall@100." authors = [{ name = "jianlyu" }] keywords = ["information-retrieval", "multi-vector", "colbert", "maxsim", "cpu", "quora"] diff --git a/tasks/resource-bounded-multivector-search/tests/Dockerfile b/tasks/resource-bounded-multivector-search/tests/Dockerfile index 029d65c..b67e211 100644 --- a/tasks/resource-bounded-multivector-search/tests/Dockerfile +++ b/tasks/resource-bounded-multivector-search/tests/Dockerfile @@ -15,7 +15,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/resource-bounded-multivector-search/tests/rewardkit_deepseek.py b/tasks/resource-bounded-multivector-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/resource-bounded-multivector-search/tests/rewardkit_deepseek.py +++ b/tasks/resource-bounded-multivector-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/rust-bm25-search/README.md b/tasks/rust-bm25-search/README.md index b1d68c6..7b9e9d6 100644 --- a/tasks/rust-bm25-search/README.md +++ b/tasks/rust-bm25-search/README.md @@ -1,12 +1,37 @@ # Rust BM25 search +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `rust-bm25-search` | +| Version | 1.0.0 | +| Authors | MXueguang | +| Primary metric | quality-gated BM25 quality/time/disk tradeoff | +| Compute | 4 CPUs; 16 GiB RAM; 64 GiB storage; 0 GPUs | +| Agent / verifier time limits | 10800 s / 18000 s | + This CPU task asks an agent to implement a complete Rust inverted index and BM25 searcher from the 8,841,823 raw MS MARCO Passage v1 passages. Generic crates are allowed; existing search engines and prebuilt indexes are not. A separate verifier recompiles the source, rebuilds an empty index, and evaluates all 6,980 standard dev queries. -**Contribution status: submitted for review; official asset pins pending.** The exact Pyserini +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The system must build an inverted index from millions of raw passages and match a fixed retrieval-quality floor. Query latency and index size also affect the score, requiring coordinated choices in parsing, compression and ranking. + +## Verification Explanation + +### Scoring and recorded validation + +The exact Pyserini quality floor has been measured, and an independently built author Rust implementation passed it on all dev queries (MRR@10 0.1872948219). A Harbor Oracle trial also passed the actual source-artifact transfer and independent @@ -17,7 +42,7 @@ review fixes; full outputs match their previous runs. Current timings and regression evidence are in [review-fixes-observation.json](authoring/review-fixes-observation.json). The coding trial used a context-free hosted agent with an isolated task container; Harbor's Codex API route remains untested because the API preflight found exhausted credits. -Official asset publication and terms review remain necessary before merge. See +See [validation.md](authoring/validation.md) for the evidence and limitations. Author: MXueguang (`x93ma@edu.uwaterloo.ca`). @@ -55,7 +80,7 @@ access, including through io_uring. Index files, directories and the root must have no extended attributes. These checks prevent IPC and xattr payloads from bypassing the measured index-file bytes. -## Restore and check +### Restore and check From the repository root, with the host dependencies documented by the repo: @@ -72,15 +97,13 @@ docker run --rm --network none --cpus 4 --memory 16g --memory-swap 16g --pids-li search-swe-local:rust-bm25-verifier python /check.py ``` -The data manifest currently pins existing Tevatron repositories. No upload -has been made. Hidden queries/qrels are small files in `tests/data`, with +The data manifest pins existing Tevatron repositories. Hidden queries/qrels are small files in `tests/data`, with immutable source URLs and fingerprints. Restore them if necessary using `python tasks/rust-bm25-search/authoring/prepare_queries.py`. Input terms and -toolchain provenance are in [SOURCES.md](authoring/SOURCES.md). Official -publication must preserve applicable MS MARCO/TREC terms and use the reviewed -`search-swe/Search-SWE` task path before merge. +toolchain provenance are in [SOURCES.md](authoring/SOURCES.md); the original +MS MARCO/TREC terms apply to the data. -## Reproduce the reference +### Reproduce the reference The author wrappers invoke the same trusted converter and reference code used by the verifier. Create fresh output paths; these commands refuse to overwrite @@ -116,7 +139,7 @@ sentinel solely to satisfy that constructor. BM25 makes no API request and runs with networking disabled. Startup measurements include Python/JVM/module startup as well as index opening, as required by the process-cold definition. -## Reviewed precedent and instruction audit +### Reviewed precedent and instruction audit The structural precedent is `tasks/sparse-retrieval-pruning`, reviewed on base commit `46197a43f8cbb364e72b6d63d5db873ae7d2cac8`. Reused patterns: CPU runtime, @@ -132,3 +155,16 @@ agent-facing [instruction.md](instruction.md) is a prose rewrite; the mapping and review boundaries are in [instruction-equivalence.md](authoring/instruction-equivalence.md). Author candidates, full indexes and trial logs live in ignored local paths; they are not included in the agent image or a public solution directory. + +### Running the task + +From the repository root, configure the agent and any task-specific judge credentials using the [evaluation guide](../../docs/evaluation.md), then run: + +```bash +python scripts/download_assets.py --task rust-bm25-search +bash scripts/run_task.sh --task rust-bm25-search --model YOUR_AGENT_MODEL +``` + +## References + +- [Data, reference implementation and toolchain sources](authoring/SOURCES.md). diff --git a/tasks/rust-bm25-search/environment/Dockerfile b/tasks/rust-bm25-search/environment/Dockerfile index 2a5f16b..51c87be 100644 --- a/tasks/rust-bm25-search/environment/Dockerfile +++ b/tasks/rust-bm25-search/environment/Dockerfile @@ -12,9 +12,9 @@ WORKDIR /app/submission ENV CARGO_TARGET_DIR=/app/build COPY smoke.py /task/smoke.py -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/rust-bm25-search/task.toml b/tasks/rust-bm25-search/task.toml index b471358..c438a97 100644 --- a/tasks/rust-bm25-search/task.toml +++ b/tasks/rust-bm25-search/task.toml @@ -6,7 +6,7 @@ artifacts = [ [task] name = "search-swe/rust-bm25-search" -version = "0.1.0" +version = "1.0.0" description = "Build a Rust BM25 index from MS MARCO passages and improve quality, search time and index size above a pinned Pyserini quality floor." authors = [{ name = "MXueguang", email = "x93ma@edu.uwaterloo.ca" }] keywords = ["information-retrieval", "bm25", "rust", "cpu", "efficiency"] diff --git a/tasks/same-place-image-search/README.md b/tasks/same-place-image-search/README.md index f3dbdb5..0b7cec0 100644 --- a/tasks/same-place-image-search/README.md +++ b/tasks/same-place-image-search/README.md @@ -1,10 +1,22 @@ # Same-Place Image Search -Retrieve photographs of the same physical building across viewpoints, scale and occlusion. +## Task Metadata -**Task:** `same-place-image-search` · **Metric:** HardPlaceMacroNDCG@5 +| Field | Value | +| --- | --- | +| Task | `same-place-image-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | HardPlaceMacroNDCG@5 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 7200 s / 12000 s | +| Dataset | Revisited Oxford and Paris, released expert ground truth | +| Public queries | 5 | +| Hidden queries | 135 | -## Overview +Retrieve photographs of the same physical building across viewpoints, scale and occlusion. + +### Overview A cropped photograph may show only part of a building. The system must find other views of that physical place rather than buildings with similar style. @@ -13,15 +25,9 @@ Each request includes gallery image IDs that are excluded from its ranking. The scope is retrieval within the supplied landmark galleries. The task does not ask for web geolocation or external webpage evidence. -## What This Task Tests - -- Matching physical identity across different views and image crops. -- Combining gallery indexing and bounded local visual verification. -- Respecting per-query exclusions and the source relevance protocol. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -45,7 +51,7 @@ query-source images are excluded from the gallery. No GPS-derived or generated same-place labels are added. The limited landmark families do not establish worldwide geographic generalization. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The gallery, query crops, Hard labels and output interface are fixed. CLIP weights remain fixed. Index structure, global and local representations @@ -53,13 +59,27 @@ and ranking are implementation choices; no precomputed vectors or starter are supplied. Additional pixel-based geometric verification may examine at most 20 candidates per request. External APIs are unavailable. -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has two hours. Index construction has 3,600 seconds; each of two query calls has 2,700 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Matching physical identity across different views and image crops. +- Combining gallery indexing and bounded local visual verification. +- Respecting per-query exclusions and the source relevance protocol. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with their supporting files under `/app`. For each cropped image and exclusion list, return up to @@ -72,9 +92,9 @@ use the supplied writable output and temporary paths. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Compute binary nDCG@5 against the original Hard positives after removing excluded images. Average within Oxford and Paris, then give the two sources @@ -83,7 +103,7 @@ equal weight. This task uses the source Hard relevance protocol with a task-specific top-five metric. Invalid or excluded returned IDs invalidate that query. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.5364309790092764` and integer score anchor `a = 15` on the 0–100 scale: @@ -98,7 +118,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -107,7 +127,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -117,7 +137,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -132,7 +152,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task same-place-image-search -bash scripts/run_task.sh --task same-place-image-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task same-place-image-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -140,7 +160,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/same-place-image-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -153,3 +173,9 @@ credentials or hardware. Results are written under `jobs/same-place-image-search | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/same-place-image-search/environment/Dockerfile b/tasks/same-place-image-search/environment/Dockerfile index a517a8b..52ebdd9 100644 --- a/tasks/same-place-image-search/environment/Dockerfile +++ b/tasks/same-place-image-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/same-place-image-search/task.toml b/tasks/same-place-image-search/task.toml index 4dc33ed..3bb00fd 100644 --- a/tasks/same-place-image-search/task.toml +++ b/tasks/same-place-image-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/same-place-image-search" -version = "0.5.0" +version = "1.0.0" description = "Build image search for photographs of the same physical place across viewpoints." authors = [{ name = "Haoran Jin" }] keywords = ["image-retrieval", "landmark", "geometry", "gpu"] diff --git a/tasks/same-place-image-search/tests/Dockerfile b/tasks/same-place-image-search/tests/Dockerfile index 1438804..0f20b22 100644 --- a/tasks/same-place-image-search/tests/Dockerfile +++ b/tasks/same-place-image-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -33,5 +33,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/same-place-image-search/tests/rewardkit_deepseek.py b/tasks/same-place-image-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/same-place-image-search/tests/rewardkit_deepseek.py +++ b/tasks/same-place-image-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/scientific-paper-qa/README.md b/tasks/scientific-paper-qa/README.md index 086375b..5cb4d50 100644 --- a/tasks/scientific-paper-qa/README.md +++ b/tasks/scientific-paper-qa/README.md @@ -1,10 +1,21 @@ # Scientific-Paper Question Answering -Build a RAG system that answers scientific questions and identifies the supporting paper. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `scientific-paper-qa` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | LLMJudgeAccuracy | +| Compute | 16 CPUs; 64 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 14400 s | +| Dataset | research-paper PDFs | +| Hidden queries | 10 | -**Task:** `scientific-paper-qa` · **Metric:** LLMJudgeAccuracy +Build a RAG system that answers scientific questions and identifies the supporting paper. -## Overview +### Overview Finding a related paper is only one part of scientific question answering. The system must recover relevant evidence from PDFs, interpret it, and @@ -15,16 +26,9 @@ It evaluates the complete pipeline: an otherwise plausible answer receives no credit when the submitted evidence document is wrong. It also tests whether the system can share its index and services reliably across concurrent questions. -## What This Task Tests +### Task Setup -- Turning scientific PDFs into searchable representations with document provenance. -- Connecting retrieved evidence to concise answers. -- Maintaining attribution across parsing, retrieval, and generation. -- Sharing a retrieval service safely across concurrent query processes. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -37,7 +41,7 @@ the same PDF corpus, but their questions and labels are separated from the agent environment. [Long-PDF evidence](../long-pdf-evidence/README.md) instead evaluates page localization within a single long PDF. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The corpus and output interface are fixed. PDF processing, index design, retrieval, evidence selection, and answer synthesis are implementation choices. @@ -45,7 +49,7 @@ Optional OpenRouter and Jina services are subject to the [resource policy](environment/docs/available_resources.md); they do not permit replacing corpus-grounded work with external answers. -### Environment and Resource Limits +#### Environment and Resource Limits The CPU Python 3.12 environment provides 16 CPUs, 64 GiB memory, 100 GiB storage, and no GPU. The agent has two hours and the Harbor verifier four hours. @@ -56,7 +60,22 @@ is excluded and workers share the same resources. The verifier also bounds query execution, answer scoring, and trajectory review as stages inside its overall budget. Stage limits do not extend that budget. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Turning scientific PDFs into searchable representations with document provenance. +- Connecting retrieved evidence to concise answers. +- Maintaining attribution across parsing, retrieval, and generation. +- Sharing a retrieval service safely across concurrent query processes. + +## Verification Explanation + +### Submission Contract The deliverable is `/app/build.sh` plus `/app/run.sh` and the implementation they need. Each query produces a concise `answer` and one evidence document ID @@ -66,9 +85,9 @@ concurrent invocations. The full schema and invocation details live in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality A question scores one only when **both** conditions hold: the evidence document matches the reference, and an answer judge accepts the answer as semantically @@ -76,13 +95,13 @@ equivalent to the reference. LLMJudgeAccuracy is the mean of these binary outcomes over 10 hidden questions. The report's 0–100 score is 100 times the normalized reward. -### Correctness and Resource Gates +#### Correctness and Resource Gates Execution, complete query coverage, non-empty answers, valid document IDs, and runtime limits must pass. A failed query process or structurally invalid output invalidates the submission, rather than merely counting as an incorrect answer. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward The answer judge and trajectory audit are separate. The first evaluates answer equivalence; the second checks task compliance and can set the whole reward @@ -97,7 +116,7 @@ and experimental details. The splits have disjoint query IDs, questions, and reference papers; the hidden split uses ten distinct reference papers. Current task identity and execution settings are recorded in `task.toml`. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -121,7 +140,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -132,3 +151,7 @@ assets, credentials, or hardware. | [Environment configuration](environment/docker-compose.yaml) | Read-only mounts and hardware requests | | [Verifier](tests/) | Execution, output validation, and scoring implementation | | [Resource policy](environment/docs/available_resources.md) | Allowed submission APIs and model restrictions | + +## References + +None. diff --git a/tasks/scientific-paper-qa/environment/Dockerfile b/tasks/scientific-paper-qa/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/scientific-paper-qa/environment/Dockerfile +++ b/tasks/scientific-paper-qa/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/scientific-paper-qa/task.toml b/tasks/scientific-paper-qa/task.toml index 279b586..dc2a5bc 100644 --- a/tasks/scientific-paper-qa/task.toml +++ b/tasks/scientific-paper-qa/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/scientific-paper-qa" -version = "0.3.0" +version = "1.0.0" description = "Build an evidence-grounded RAG system over a local collection of research-paper PDFs." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "search", "RAG"] diff --git a/tasks/scientific-paper-qa/tests/Dockerfile b/tasks/scientific-paper-qa/tests/Dockerfile index ec30eb2..47373f0 100644 --- a/tasks/scientific-paper-qa/tests/Dockerfile +++ b/tasks/scientific-paper-qa/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/scientific-paper-qa/tests/rewardkit_deepseek.py b/tasks/scientific-paper-qa/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/scientific-paper-qa/tests/rewardkit_deepseek.py +++ b/tasks/scientific-paper-qa/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/sketch-text-search/README.md b/tasks/sketch-text-search/README.md index 3f3d748..9becefa 100644 --- a/tasks/sketch-text-search/README.md +++ b/tasks/sketch-text-search/README.md @@ -1,10 +1,22 @@ # Sketch-and-Text Image Search -Retrieve an artwork from a partial sketch and its written description. +## Task Metadata -**Task:** `sketch-text-search` · **Metric:** LevelMacroRecallMRR@3 +| Field | Value | +| --- | --- | +| Task | `sketch-text-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | LevelMacroRecallMRR@3 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 12000 s | +| Dataset | CIRThan with a deterministic partial-sketch input condition | +| Public queries | 15 | +| Hidden queries | 1242 | -## Overview +Retrieve an artwork from a partial sketch and its written description. + +### Overview A rough sketch may omit parts of the desired scene. The system must combine the visible drawing and description to locate the target artwork. @@ -13,15 +25,9 @@ The collection retains three description levels. Each sketch-description pair is processed independently; descriptions for other requests are not extra query context. -## What This Task Tests - -- Combining partial visual structure with text descriptions. -- Adapting scoring or fusion from supplied labeled examples. -- Maintaining quality across broad and detailed descriptions. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -46,21 +52,35 @@ the remaining 40 percent is masked. Selection uses only the opaque sketch ID. Complete original sketches are not supplied. This is a partial-sketch task, not a claim to reproduce the original paper's input condition. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The gallery, partial sketches, description levels and output interface are fixed. Supplied encoders remain fixed; optional scoring or fusion components may use up to 30 minutes of adaptation. Indexing and ranking are implementation choices. No precomputed vectors, starter or external APIs are supplied. -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has six hours, including the adaptation allowance. Build has 3,600 seconds. Each of three query calls, one per description level, has 1,800 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Combining partial visual structure with text descriptions. +- Adapting scoring or fusion from supplied labeled examples. +- Maintaining quality across broad and detailed descriptions. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting code and learned files under `/app`. Each request contains a sketch path and description; @@ -74,9 +94,9 @@ temporary paths. A development-only permissions helper is available at The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Per-query quality is `0.5 × Recall@3 + 0.5 × MRR@3`. Average within each of the three description levels, then give the levels @@ -86,7 +106,7 @@ Up to ten results are accepted by the interface, but only the first three affect the metric. The composite and partial-sketch input are task adaptations; they do not reproduce the source paper's reported scores. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.32306763285024154` and integer score anchor `a = 15` on the 0–100 scale: @@ -101,7 +121,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -110,7 +130,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -120,7 +140,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -135,7 +155,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task sketch-text-search -bash scripts/run_task.sh --task sketch-text-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task sketch-text-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -143,7 +163,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/sketch-text-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -157,3 +177,9 @@ credentials or hardware. Results are written under `jobs/sketch-text-search/`. | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [environment/docs/check_runtime.py](environment/docs/check_runtime.py) | Development-only check of build and query filesystem permissions | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/sketch-text-search/environment/Dockerfile b/tasks/sketch-text-search/environment/Dockerfile index a517a8b..52ebdd9 100644 --- a/tasks/sketch-text-search/environment/Dockerfile +++ b/tasks/sketch-text-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/sketch-text-search/task.toml b/tasks/sketch-text-search/task.toml index 3fb9673..914a11f 100644 --- a/tasks/sketch-text-search/task.toml +++ b/tasks/sketch-text-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/sketch-text-search" -version = "0.6.0" +version = "1.0.0" description = "Build a sketch-and-text image search system over original gallery images." authors = [{ name = "Haoran Jin" }] keywords = ["information-retrieval", "sketch", "image", "composition", "gpu"] diff --git a/tasks/sketch-text-search/tests/Dockerfile b/tasks/sketch-text-search/tests/Dockerfile index 1438804..0f20b22 100644 --- a/tasks/sketch-text-search/tests/Dockerfile +++ b/tasks/sketch-text-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -33,5 +33,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/sketch-text-search/tests/rewardkit_deepseek.py b/tasks/sketch-text-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/sketch-text-search/tests/rewardkit_deepseek.py +++ b/tasks/sketch-text-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/sparse-retrieval-pruning/README.md b/tasks/sparse-retrieval-pruning/README.md index 879f075..eade360 100644 --- a/tasks/sparse-retrieval-pruning/README.md +++ b/tasks/sparse-retrieval-pruning/README.md @@ -1,10 +1,21 @@ # Sparse Retrieval Pruning -Optimize a sparse retrieval system while preserving strict quality requirements. +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `sparse-retrieval-pruning` | +| Version | 1.0.0 | +| Authors | Search-SWE | +| Primary metric | quality_gated_linear_starter_speedup | +| Compute | 1 CPUs; 32 GiB RAM; 80 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 16200 s | +| Dataset | MS MARCO Passage v1 with frozen SPLADE sparse vectors | +| Hidden queries | 1000 | -**Task:** `sparse-retrieval-pruning` · **Metric:** Quality-gated linear speedup reward +Optimize a sparse retrieval system while preserving strict quality requirements. -## Overview +### Overview Sparse retrieval efficiency depends on how terms, document identities, and posting lists fit together. A compact index is not useful if remapped terms @@ -19,16 +30,9 @@ wall time relative to that reference. Quality thresholds are hard gates; passing submissions receive `clip((speedup - 4) / 12, 0, 1)`: 4× starts scoring, 10× earns 50 points and 16× earns 100 points. -## What This Task Tests +### Task Setup -- Building a bounded-memory inverted index over a large sparse corpus. -- Keeping dictionaries, document mappings, and posting metadata consistent. -- Reducing retrieval work without sacrificing required ranking quality. -- Achieving speedups through Python data structures and algorithms rather than parallelism. - -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -45,7 +49,7 @@ The public assets are pinned to an immutable dataset revision in [assets.json](assets.json). Downloading verifies file sizes and SHA-256 checksums. Their Hugging Face source paths are under `tasks/sparse-retrieval-pruning/`. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes The corpus, executable interface, and quality/runtime requirements are fixed. The agent may replace the starter's index format or retrieval algorithm. @@ -58,7 +62,7 @@ code, compiled submission executables, and parallel query execution are outside the task. The input is set-valued; readers should not assume that weighted encoder outputs or a retrainable transformer are supplied. -### Environment and Resource Limits +#### Environment and Resource Limits The CPU Python 3.12 environment has one CPU, 32 GiB memory, 80 GiB storage, and no GPU. The agent has two hours; the Harbor verifier has 270 minutes (worst-case stage budget). @@ -69,7 +73,22 @@ within those bounds, and all verification stages share the overall phase budget. The formal build/search path requires no external service, credentials, or downloads. The private integrity judge's API access is a separate concern. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Building a bounded-memory inverted index over a large sparse corpus. +- Keeping dictionaries, document mappings, and posting metadata consistent. +- Reducing retrieval work without sacrificing required ranking quality. +- Achieving speedups through Python data structures and algorithms rather than parallelism. + +## Verification Explanation + +### Submission Contract The transferred system includes `/app/build.sh` and `/app/run.sh`. The build creates all persistent state under the caller's index directory; the query @@ -81,14 +100,14 @@ The exact fields and CLI are in [instruction.md](instruction.md). The [index format guide](environment/docs/index_format.md) explains the starter, but that particular storage format is not a required output artifact. -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Quality is measured using mean NDCG@10 and qrels-based Recall@100. The current minimums are **0.89** and **0.99** respectively. -### Correctness and Latency Reward +#### Correctness and Latency Reward The fixed corpus contains 1,000,000 documents. Each formal search invocation processes the same 1,000 hidden queries; the public development set contains @@ -120,7 +139,7 @@ Let `s = median(starter wall time) / median(candidate wall time)`. If every round meets both quality gates and all execution/output/integrity checks pass, reward is `max(0, min(1, (s - 4) / 12))`, and score is 100 times reward. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward If either quality threshold, output and execution checks, or the trajectory audit fails, final reward is `0`. Otherwise the final reward is the speedup @@ -128,7 +147,7 @@ reward described above. The verifier measures reference wall time afresh for eac The audit uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0 with verifier-only DeepSeek endpoint and key settings. -## Running This Task +### Running This Task From the repository root, follow the [quick start](../../docs/quickstart.md) to install the pinned Harbor dependencies and prepare Docker. Use the @@ -152,7 +171,7 @@ Replace `YOUR_AGENT_MODEL` with your configured model. Add `--dry-run` to inspec command construction without starting an evaluation; this does not validate assets, credentials, or hardware. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -165,7 +184,7 @@ assets, credentials, or hardware. | [Index format guide](environment/docs/index_format.md) | Starter layout and consistency relationships | | [Resource policy](environment/docs/available_resources.md) | Python dependency and execution restrictions | -## Revision 0.5.2 validation +### Revision 0.5.2 validation This revision implements the approved 4×–16× speedup calibration and five-round alternating protocol. Linux x86_64 Landlock and seccomp are required by the @@ -196,3 +215,7 @@ Revision 0.5.2 removes a per-UID `RLIMIT_NPROC` that incorrectly counted unrelated containers on the host and could block the starter's serial shell launch. The container PID limit, kernel thread prohibition, worker monitoring, quality gates and scoring are unchanged. + +## References + +None. diff --git a/tasks/sparse-retrieval-pruning/environment/Dockerfile b/tasks/sparse-retrieval-pruning/environment/Dockerfile index f4c3314..e8b52a1 100644 --- a/tasks/sparse-retrieval-pruning/environment/Dockerfile +++ b/tasks/sparse-retrieval-pruning/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/sparse-retrieval-pruning/task.toml b/tasks/sparse-retrieval-pruning/task.toml index 46ad37d..0bafbb1 100644 --- a/tasks/sparse-retrieval-pruning/task.toml +++ b/tasks/sparse-retrieval-pruning/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/sparse-retrieval-pruning" -version = "0.5.2" +version = "1.0.0" description = "Optimize Python learned-sparse retrieval under quality gates and a continuous 4x-to-16x starter-relative speedup reward." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "search", "ms-marco", "splade", "sparse-retrieval", "postings"] diff --git a/tasks/sparse-retrieval-pruning/tests/Dockerfile b/tasks/sparse-retrieval-pruning/tests/Dockerfile index 8831900..4539d88 100644 --- a/tasks/sparse-retrieval-pruning/tests/Dockerfile +++ b/tasks/sparse-retrieval-pruning/tests/Dockerfile @@ -7,7 +7,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --timeout 120 --retries 10 --resume-retries 10 \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/sparse-retrieval-pruning/tests/rewardkit_deepseek.py b/tasks/sparse-retrieval-pruning/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/sparse-retrieval-pruning/tests/rewardkit_deepseek.py +++ b/tasks/sparse-retrieval-pruning/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/spoken-query-search/README.md b/tasks/spoken-query-search/README.md index f150dde..bf28658 100644 --- a/tasks/spoken-query-search/README.md +++ b/tasks/spoken-query-search/README.md @@ -1,10 +1,22 @@ # Spoken-Query Document Search -Retrieve documents from fixed, uncertain speech transcriptions and their alternatives. +## Task Metadata -**Task:** `spoken-query-search` · **Metric:** SpokenMacroNDCG@10 +| Field | Value | +| --- | --- | +| Task | `spoken-query-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | SpokenMacroNDCG@10 | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 7200 s / 19200 s | +| Dataset | SQuTR, six released corpora and four acoustic conditions | +| Public queries | 10 | +| Hidden queries | 4550 | -## Overview +Retrieve documents from fixed, uncertain speech transcriptions and their alternatives. + +### Overview Speech recognition can mishear names and important query terms. This task provides a best transcription, alternative hypotheses and confidence information @@ -14,15 +26,9 @@ relevant documents. The recognizer is fixed. Document indexing, uncertainty handling and ranked retrieval belong to the submission. -## What This Task Tests - -- Using transcription alternatives and confidence information. -- Retrieving across six complete multilingual document collections. -- Maintaining retrieval quality under different acoustic conditions. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -50,7 +56,7 @@ SNR0/SNR10 variants of the public families are unused. No question family crosse the split. Here clean names an acoustic condition; clean reference text and audio are not runtime inputs. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes Transcriptions, confidence fields, document collections and source relevance labels are fixed. At most five retrieval formulations and 50 further @@ -61,13 +67,27 @@ Indexing and ranking are implementation choices. No document or query vectors or starter are supplied. Optional OpenRouter and Jina access follows the [resource policy](environment/docs/available_resources.md). -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has two hours. Build has 10,800 seconds; each of four query calls has 1,350 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Using transcription alternatives and confidence information. +- Retrieving across six complete multilingual document collections. +- Maintaining retrieval quality under different acoustic conditions. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting files under `/app`. Requests name a collection domain and provide the best @@ -81,9 +101,9 @@ calls with fresh scratch; requests must be processed independently. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Compute original graded nDCG@10, average within each collection and acoustic condition, then give the six collections and four conditions equal weight. @@ -93,7 +113,7 @@ The split separates question families, but it is not an independently verified speaker-disjoint benchmark. Per-condition and per-collection scores are reported to show the effect of recognition noise. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.4530094614433184` and integer score anchor `a = 15` on the 0–100 scale: @@ -108,7 +128,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query coverage, IDs, duplicate handling, finite scores and stage limits. @@ -117,7 +137,7 @@ invalid rows score zero. A non-timeout query-command failure invalidates that call. See [instruction.md](instruction.md) and the verifier for the exact task-specific validity checks. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -127,7 +147,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -143,7 +163,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task spoken-query-search -bash scripts/run_task.sh --task spoken-query-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task spoken-query-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -151,7 +171,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/spoken-query-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -164,3 +184,9 @@ credentials or hardware. Results are written under `jobs/spoken-query-search/`. | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/spoken-query-search/environment/Dockerfile b/tasks/spoken-query-search/environment/Dockerfile index 090e39c..487638b 100644 --- a/tasks/spoken-query-search/environment/Dockerfile +++ b/tasks/spoken-query-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/spoken-query-search/task.toml b/tasks/spoken-query-search/task.toml index b4d94d8..a3e6f25 100644 --- a/tasks/spoken-query-search/task.toml +++ b/tasks/spoken-query-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/spoken-query-search" -version = "0.5.0" +version = "1.0.0" description = "Build document search that handles uncertainty in noisy speech transcriptions." authors = [{ name = "Haoran Jin" }] keywords = ["speech", "uncertainty", "document-retrieval", "multilingual"] diff --git a/tasks/spoken-query-search/tests/Dockerfile b/tasks/spoken-query-search/tests/Dockerfile index ff89a9a..e1bd391 100644 --- a/tasks/spoken-query-search/tests/Dockerfile +++ b/tasks/spoken-query-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,5 +30,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/spoken-query-search/tests/rewardkit_deepseek.py b/tasks/spoken-query-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/spoken-query-search/tests/rewardkit_deepseek.py +++ b/tasks/spoken-query-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/training-error-tracing/README.md b/tasks/training-error-tracing/README.md index 906e3c6..29f583c 100644 --- a/tasks/training-error-tracing/README.md +++ b/tasks/training-error-tracing/README.md @@ -1,4 +1,15 @@ -# training-error-tracing +# Training Error Tracing + +## Task Metadata + +| Field | Value | +| --- | --- | +| Task | `training-error-tracing` | +| Version | 1.0.0 | +| Authors | jianlyu | +| Primary metric | quality-gated paired query-latency and GPU-memory cost | +| Compute | 16 CPUs; 64 GiB RAM; 64 GiB storage; 1 GPU | +| Agent / verifier time limits | 14400 s / 8400 s | Improve a working training-record retriever beyond training-text routing and layer-streamed fixed-base fallback. @@ -9,7 +20,21 @@ The coding agent receives [instruction.md](instruction.md), Those documents specify the submission interface, allowed inputs, scoring, failure behavior and public self-check commands. -## Restore inputs +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +The retriever must identify influential training records beyond the provided routing and fixed-base fallbacks. Quality gates must hold while reducing paired query latency and GPU-memory cost. + +## Verification Explanation + +The verifier measures the proportion of modified training records among the returned Top-10, requiring mean quality of at least 0.98 and at least 0.80 for every group on every repetition. Passing quality earns a 20-point milestone; the remaining credit measures paired latency and sampled GPU-memory improvement over a fresh run of the immutable public control. Harbor stores the displayed score divided by 100. See the [verification contract](raw-instruction.md#verification) for exact formulas, thresholds and failure handling. + +### Restore inputs Run from the repository root using the repository's existing Python environment: @@ -22,7 +47,7 @@ python scripts/download_assets.py --task training-error-tracing --verify-only restoration paths. Keep verifier inputs outside agent mounts. Model cards, licenses and data-use restrictions are described in the resource document. -## Run +### Run The task allocates 16 CPU cores, 64 GiB RAM and 64 GiB storage. Agent development has @@ -30,10 +55,8 @@ The task allocates 16 CPU cores, 64 GiB RAM and specified in the task contract; the verifier service timeout also includes its trusted setup and checks. -Both phases require one A100 40GB GPU. Reserve an idle device and configure -both Docker phases to expose that same device explicitly. Verify the actual -container mapping before running; a host CUDA_VISIBLE_DEVICES value alone is -insufficient. Release the reservation after all run containers/processes exit. +Both phases require one A100 40GB GPU. Use `--gpu-device` to select the same +host GPU for both phases. Configure `AGENT_OPENROUTER_API_KEY` through the existing ignored `.env` or authorized process environment, following the repository README. @@ -44,7 +67,7 @@ are not submission inputs. Invoke the standard entrypoint directly: ```bash -bash scripts/run_task.sh --task training-error-tracing --agent codex --openrouter \ +bash scripts/run_task.sh --task training-error-tracing --gpu-device 0 --agent codex --openrouter \ --model openai/gpt-6-luna --reasoning-effort max \ --output "jobs/training-error-tracing-codex-openrouter-luna-max-RUN_ID" ``` @@ -53,7 +76,7 @@ Inspect the resulting agent trajectory, submission and verifier report under the job directory. The score alone does not distinguish an invalid submission from an infrastructure error. -## Validate the package +### Validate the package ```bash python scripts/check_tasks.py tasks/training-error-tracing @@ -66,3 +89,9 @@ resource controls and mounts. Tests that exercise process isolation or GPU enforcement belong in disposable task containers, not the shared host. The verifier Compose service grants `SYS_PTRACE` and `CHECKPOINT_RESTORE` so the trusted supervisor can account for open-unlinked and memory-mapped task files. The host kernel/runtime must support these capabilities. Candidate commands drop all capability sets. Storage inspection errors fail closed; unit and real descriptor/mapping regressions are in `tests/test_storage_accounting.py` and can run in a disposable verifier without a GPU. + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +None. diff --git a/tasks/training-error-tracing/environment/Dockerfile b/tasks/training-error-tracing/environment/Dockerfile index 729d268..282cd5d 100644 --- a/tasks/training-error-tracing/environment/Dockerfile +++ b/tasks/training-error-tracing/environment/Dockerfile @@ -9,9 +9,9 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir --no-deps \ --index-url https://pypi.tuna.tsinghua.edu.cn/simple \ "gemlite==0.6.0.post2" "hqq==0.2.8.post1" "termcolor==3.3.0" -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/training-error-tracing/task.toml b/tasks/training-error-tracing/task.toml index 0c07a0e..8579afb 100644 --- a/tasks/training-error-tracing/task.toml +++ b/tasks/training-error-tracing/task.toml @@ -4,7 +4,7 @@ artifacts = ["/app/submission", "/logs/agent/trajectory.json"] [task] name = "search-swe/training-error-tracing" -version = "0.6.2" +version = "1.0.0" description = "Improve a working training-record retriever beyond training-text routing and layer-streamed fixed-base fallback." authors = [{ name = "jianlyu" }] keywords = ["information-retrieval", "training-data-attribution", "gradient-retrieval", "gpu"] diff --git a/tasks/training-error-tracing/tests/Dockerfile b/tasks/training-error-tracing/tests/Dockerfile index 949aed0..7a9b5cb 100644 --- a/tasks/training-error-tracing/tests/Dockerfile +++ b/tasks/training-error-tracing/tests/Dockerfile @@ -15,7 +15,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ --timeout 120 --retries 10 --resume-retries 10 \ "harbor-rewardkit==0.2.0" "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ diff --git a/tasks/training-error-tracing/tests/rewardkit_deepseek.py b/tasks/training-error-tracing/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/training-error-tracing/tests/rewardkit_deepseek.py +++ b/tasks/training-error-tracing/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/video-moment-search/README.md b/tasks/video-moment-search/README.md index bcb8602..deb658d 100644 --- a/tasks/video-moment-search/README.md +++ b/tasks/video-moment-search/README.md @@ -1,10 +1,22 @@ # Video-Moment Search -Locate the time interval matching a natural-language description within a specified video. +## Task Metadata -**Task:** `video-moment-search` · **Metric:** SourceMacroGridUtility +| Field | Value | +| --- | --- | +| Task | `video-moment-search` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | SourceMacroGridUtility | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 1 GPU | +| Agent / verifier time limits | 21600 s / 12000 s | +| Dataset | Charades-STA and ActivityNet Captions, pinned Lighthouse feature release | +| Public queries | 5 | +| Hidden queries | 450 | -## Overview +Locate the time interval matching a natural-language description within a specified video. + +### Overview Searching within a video requires finding both the right event and its time boundaries. Each request names a video and an original description. The system @@ -13,15 +25,9 @@ returns one interval on the video's supplied time grid. The task uses released frame features and local encoder weights, rather than raw videos or precomputed query representations. -## What This Task Tests - -- Matching descriptions to temporal evidence in fixed video features. -- Choosing accurate start and end boundaries. -- Using labeled intervals within a bounded adaptation budget. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -50,7 +56,7 @@ This supervised adaptation differs from the weak-supervision setting of the seed paper. Raw videos, generated query vectors and tokenized descriptions are not supplied. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes Video features, time grids and local encoders are fixed. An optional temporal/scoring component may have at most five million trainable parameters @@ -61,13 +67,27 @@ and boundary proposals remain allowed. Indexing and interval selection are implementation choices. No starter or external API resource is supplied. -### Environment and Resource Limits +#### Environment and Resource Limits The environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and one 40 GiB GPU. Development has six hours, including the optional training allowance. Build has 3,600 seconds; each of two query calls has 2,700 seconds. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Matching descriptions to temporal evidence in fixed video features. +- Choosing accurate start and end boundaries. +- Using labeled intervals within a bounded adaptation budget. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting files under `/app`. Each request produces one nonempty start/end interval whose @@ -79,9 +99,9 @@ sees the completed index read-only and uses fresh output and temporary paths. The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality Let x be the best IoU between the predicted interval and the original acceptable intervals. Its utility is: @@ -99,7 +119,7 @@ nonfinite or non-grid intervals score zero. The grid correction accounts for boundary precision limits; a long interval still loses IoU when it includes unrelated time. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.14696558206208937` and integer score anchor `a = 15` on the 0–100 scale: @@ -114,7 +134,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks successful index construction, executable behavior, query IDs, finite start/end values, interval order, grid endpoints and stage @@ -122,7 +142,7 @@ limits. Completed valid JSONL rows remain usable after a query timeout; missing or invalid query results score zero. A non-timeout query-command failure invalidates that call. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -132,7 +152,7 @@ RewardKit 0.2.0 with verifier-only OpenRouter settings. Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -147,7 +167,7 @@ The commands below apply in a checkout with this package installed at ```bash python scripts/download_assets.py --task video-moment-search -bash scripts/run_task.sh --task video-moment-search --model "YOUR_AGENT_MODEL" +bash scripts/run_task.sh --task video-moment-search --gpu-device 0 --model "YOUR_AGENT_MODEL" ``` The shared launcher defaults to Codex and also supports Pi and Claude Code. @@ -155,7 +175,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/video-moment-search/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -168,3 +188,9 @@ credentials or hardware. Results are written under `jobs/video-moment-search/`. | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +`--gpu-device 0` selects host GPU 0; a full GPU UUID is also accepted. Omit the option to use the default single-GPU allocation. Choose an available device with the task's required capacity. The [GPU guide](../../docs/evaluation.md#gpu-device-selection) explains selection and per-run records. + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/video-moment-search/environment/Dockerfile b/tasks/video-moment-search/environment/Dockerfile index a517a8b..52ebdd9 100644 --- a/tasks/video-moment-search/environment/Dockerfile +++ b/tasks/video-moment-search/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:gpu-cu13.0-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/video-moment-search/task.toml b/tasks/video-moment-search/task.toml index 00a5b43..485b5f4 100644 --- a/tasks/video-moment-search/task.toml +++ b/tasks/video-moment-search/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/video-moment-search" -version = "0.6.0" +version = "1.0.0" description = "Build temporal search over fixed video representations." authors = [{ name = "Haoran Jin" }] keywords = ["video", "moment-retrieval", "temporal", "gpu"] diff --git a/tasks/video-moment-search/tests/Dockerfile b/tasks/video-moment-search/tests/Dockerfile index 1438804..0f20b22 100644 --- a/tasks/video-moment-search/tests/Dockerfile +++ b/tasks/video-moment-search/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -33,5 +33,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/video-moment-search/tests/rewardkit_deepseek.py b/tasks/video-moment-search/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/video-moment-search/tests/rewardkit_deepseek.py +++ b/tasks/video-moment-search/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)", diff --git a/tasks/visual-document-qa/README.md b/tasks/visual-document-qa/README.md index 60a6763..761aacd 100644 --- a/tasks/visual-document-qa/README.md +++ b/tasks/visual-document-qa/README.md @@ -1,10 +1,22 @@ # Visual-Document Question Answering -Answer questions from text, tables and figures in a specified long document. +## Task Metadata -**Task:** `visual-document-qa` · **Metric:** SourceMacroAnswerAccuracy +| Field | Value | +| --- | --- | +| Task | `visual-document-qa` | +| Version | 1.0.0 | +| Authors | Haoran Jin | +| Primary metric | SourceMacroAnswerAccuracy | +| Compute | 8 CPUs; 32 GiB RAM; 100 GiB storage; 0 GPUs | +| Agent / verifier time limits | 7200 s / 45000 s | +| Dataset | DocLensDatasets: MMLongBench-Doc and FinRAGBench-V | +| Public queries | 5 | +| Hidden queries | 2471 | -## Overview +Answer questions from text, tables and figures in a specified long document. + +### Overview Relevant information in a long PDF may span prose, tables, charts and several pages. The system must locate and read that evidence, then give a direct @@ -13,15 +25,9 @@ answer or identify that the document does not contain the requested information. Each request identifies a document. Evidence-page citations are reported, while final answer quality is the primary metric. -## What This Task Tests - -- Indexing long PDFs while retaining page provenance. -- Combining textual and visual evidence for answer generation. -- Handling numeric, list and unanswerable questions. +### Task Setup -## Task Setup - -### Provided Assets +#### Provided Assets | Asset | Purpose | | --- | --- | @@ -43,7 +49,7 @@ can also appear in private questions. Four upstream answer-type metadata fields are corrected deterministically; the questions, canonical answers and comparison rules are unchanged. -### Fixed Components and Allowed Changes +#### Fixed Components and Allowed Changes PDFs, original question/answer labels and output interfaces are fixed. Parsing, rendering, indexing, retrieval and answer synthesis are implementation @@ -51,14 +57,28 @@ choices. No starter, local GPU or precomputed vectors are supplied. Optional OpenRouter and Jina resources follow the [resource policy](environment/docs/available_resources.md). -### Environment and Resource Limits +#### Environment and Resource Limits The CPU environment provides eight CPU cores, 32 GiB RAM, 100 GiB storage and no GPU. Development has two hours. Build has 14,400 seconds; each of two query calls has an independent 10,800-second allowance. These limits remain inside the 45,000-second overall verifier budget. -## Submission Contract +## Change Log + +| Date | Version | Description | +| --- | --- | --- | +| 2026-10-23 | 1.0.0 | Added initial task | + +## Difficulty Explanation + +- Indexing long PDFs while retaining page provenance. +- Combining textual and visual evidence for answer generation. +- Handling numeric, list and unanswerable questions. + +## Verification Explanation + +### Submission Contract Submit executable `/app/build.sh` and `/app/run.sh` with supporting files under `/app`. Each input names a document and question. Return the query ID, @@ -71,9 +91,9 @@ writable. Generated files must use the supplied index, output and temporary path The full schema and invocation details are in [instruction.md](instruction.md). -## Evaluation +### Evaluation -### Search Quality +#### Search Quality MMLongBench-Doc uses the source answer-extraction prompt and unchanged rule comparators, including ANLS, numeric, list and unanswerable handling. @@ -86,7 +106,7 @@ Evidence-page precision, recall and F1 are diagnostics, not part of the primary answer score. Out-of-range source page references are excluded only from citation diagnostics and do not remove the corresponding answer question. -### Score Normalization +#### Score Normalization The final reward uses the raw quality described above, with fixed reference quality `b = 0.4350989780543772` and integer score anchor `a = 20` on the 0–100 scale: @@ -101,7 +121,7 @@ auxiliary diagnostics are reported separately. Harbor retains the 0–1 reward precision; a displayed 0–100 score is rounded to one decimal place. Correctness and trajectory-audit gates take precedence over this conversion. -### Correctness and Resource Gates +#### Correctness and Resource Gates The verifier checks execution, output structure, document/page validity and stage limits. Completed valid rows survive query-call timeout; missing or invalid @@ -109,7 +129,7 @@ rows receive zero. Other query-command failures invalidate that call. Trusted answer-extraction or judge transport failures invalidate the measurement rather than establishing a low baseline or model score. -### Integrity Checks and Final Reward +#### Integrity Checks and Final Reward A separate trajectory audit checks task compliance and gates the quality reward. A confirmed violation sets reward to zero; unavailable or incomplete @@ -120,7 +140,7 @@ Private questions, reference labels and audit credentials are excluded from the agent environment and protected from submitted programs. Answer judges, where used, are distinct from trajectory auditing. -## Running This Task +### Running This Task Follow the repository [quick start](../../docs/quickstart.md) to prepare Harbor and Docker, the [evaluation guide](../../docs/evaluation.md) for agent and @@ -148,7 +168,7 @@ Replace `YOUR_AGENT_MODEL` with the configured model. Add `--dry-run` to inspect launch construction without starting an evaluation; it does not validate assets, credentials or hardware. Results are written under `jobs/visual-document-qa/`. -## Task Files +### Task Files | File or directory | What to read it for | | --- | --- | @@ -161,3 +181,7 @@ credentials or hardware. Results are written under `jobs/visual-document-qa/`. | [tests/](tests/) | Execution, output validation, isolation and scoring | | [environment/docs/available_resources.md](environment/docs/available_resources.md) | Permitted APIs, models and credential handling | | [SOURCES.md](SOURCES.md) | Dataset provenance, preparation and runtime asset sources | + +## References + +- [Data, model and method provenance](SOURCES.md). diff --git a/tasks/visual-document-qa/environment/Dockerfile b/tasks/visual-document-qa/environment/Dockerfile index 3358c9f..e5882ec 100644 --- a/tasks/visual-document-qa/environment/Dockerfile +++ b/tasks/visual-document-qa/environment/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 -ARG SEARCH_SWE_CODEX_VERSION=0.157.1 -ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.283 -ARG SEARCH_SWE_PI_VERSION=0.87.1 +ARG SEARCH_SWE_CODEX_VERSION=0.164.0 +ARG SEARCH_SWE_CLAUDE_CODE_VERSION=2.1.309 +ARG SEARCH_SWE_PI_VERSION=1.4.0 RUN npm install --global --ignore-scripts \ --registry=https://registry.npmmirror.com \ "@openai/codex@${SEARCH_SWE_CODEX_VERSION}" \ diff --git a/tasks/visual-document-qa/task.toml b/tasks/visual-document-qa/task.toml index 2945887..8eed748 100644 --- a/tasks/visual-document-qa/task.toml +++ b/tasks/visual-document-qa/task.toml @@ -2,7 +2,7 @@ schema_version = "1.4" artifacts = ["/app", "/logs/agent/trajectory.json"] [task] name = "search-swe/visual-document-qa" -version = "0.4.0" +version = "1.0.0" description = "Build a page search and question answering system over long visual documents." authors = [{ name = "Haoran Jin" }] keywords = ["visual-documents", "page-search", "question-answering"] diff --git a/tasks/visual-document-qa/tests/Dockerfile b/tasks/visual-document-qa/tests/Dockerfile index f9bd398..cdf7647 100644 --- a/tasks/visual-document-qa/tests/Dockerfile +++ b/tasks/visual-document-qa/tests/Dockerfile @@ -8,7 +8,7 @@ RUN /opt/conda/bin/python -m pip install --no-cache-dir \ "harbor-rewardkit==0.2.0" \ "litellm==1.97.0" -ARG REWARDKIT_CODEX_VERSION=0.147.0 +ARG REWARDKIT_CODEX_VERSION=0.164.0 ARG REWARDKIT_CODEX_NPM_REGISTRY=https://registry.npmmirror.com RUN set -eux; \ root="/root/.cache/harbor-rewardkit/codex/${REWARDKIT_CODEX_VERSION}"; \ @@ -30,5 +30,5 @@ RUN install -d -m 0755 /opt/audit \ && install -m 0755 /tests/rewardkit_deepseek.py /opt/audit/runner.py \ && install -m 0755 /tests/readonly_audit.py /opt/audit/readonly_audit.py \ && install -m 0644 /tests/landlock_audit.py /opt/audit/landlock_audit.py \ - && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin \ - && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.147.0/bin/codex + && install -d -o audit -g audit -m 0700 /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin \ + && ln -s /usr/local/bin/codex /home/audit/.cache/harbor-rewardkit/codex/0.164.0/bin/codex diff --git a/tasks/visual-document-qa/tests/rewardkit_deepseek.py b/tasks/visual-document-qa/tests/rewardkit_deepseek.py index b8b025d..dac2f3b 100644 --- a/tasks/visual-document-qa/tests/rewardkit_deepseek.py +++ b/tasks/visual-document-qa/tests/rewardkit_deepseek.py @@ -4,13 +4,23 @@ from __future__ import annotations import json +from importlib.metadata import version import os from pathlib import Path from urllib.parse import urlsplit +from rewardkit import agents as rewardkit_agents from rewardkit.agents import CodexBackend, register_agent +# RewardKit 0.2.0 selects its CLI/cache through this module-level pin. Keep its +# installer, isolated authentication and JSON parser while using our tested CLI. +CODEX_VERSION = "0.164.0" +if version("harbor-rewardkit") != "0.2.0": + raise RuntimeError("Revalidate the Codex runtime pin before upgrading RewardKit") +rewardkit_agents._CODEX_VERSION = CODEX_VERSION + + MODEL = { "slug": "deepseek/deepseek-v4.1-flash", "display_name": "DeepSeek V4.1 Flash (OpenRouter)",