diff --git a/tasks/long-document-reranking/README.md b/tasks/long-document-reranking/README.md index d7a3263..fbe3870 100644 --- a/tasks/long-document-reranking/README.md +++ b/tasks/long-document-reranking/README.md @@ -5,11 +5,11 @@ | Field | Value | | --- | --- | | Task | `long-document-reranking` | -| Version | 1.0.0 | +| Version | 1.0.1 | | Authors | Search-SWE | | Primary metric | Accuracy@5 | | Compute | 16 CPUs; 16 GiB RAM; 100 GiB storage; 0 GPUs | -| Agent / verifier time limits | 7200 s / 4800 s | +| Agent / verifier time limits | 7200 s / 13200 s | | Dataset | MTEB NarrativeQARetrieval | | Public queries | 100 | | Hidden queries | 100 | @@ -57,15 +57,18 @@ the mounted path and runtime configuration are the operational reference. #### Environment and Resource Limits The CPU Python 3.12 environment provides 16 CPUs, 16 GiB memory, 100 GiB storage, -and no GPU. The agent has two hours and the Harbor verifier 80 minutes. -The grader's build and run caps are 1,800 and 3,600 seconds respectively, -but these stages and the integrity check must still fit inside the overall -verifier phase; their individual maxima are not additive allowances. +and no GPU. The agent has two hours and the Harbor verifier 13,200 seconds +(3 hours 40 minutes). The verifier window covers a 1,800-second build, two +query runs of up to 3,600 seconds each, a 3,600-second trajectory judge, +and 600 seconds for setup and cleanup: +`1800 + 2 * 3600 + 3600 + 600 = 13200`. +Each submission stage remains subject to its own time limit. ## Change Log | Date | Version | Description | | --- | --- | --- | +| 2026-10-26 | 1.0.1 | Allow the verifier window to cover all existing stage budgets plus setup and cleanup. | | 2026-10-23 | 1.0.0 | Added initial task | ## Difficulty Explanation diff --git a/tasks/long-document-reranking/task.toml b/tasks/long-document-reranking/task.toml index 672e4e6..af13162 100644 --- a/tasks/long-document-reranking/task.toml +++ b/tasks/long-document-reranking/task.toml @@ -7,7 +7,7 @@ artifacts = [ [task] name = "search-swe/long-document-reranking" -version = "1.0.0" +version = "1.0.1" description = "Optimize long-document reranking over a frozen BM25 Top-100 candidate pool with a fixed local BGE reranker." authors = [{ name = "Search-SWE" }] keywords = ["information-retrieval", "search", "reranking", "cross-encoder", "long-document", "cpu"] @@ -28,7 +28,9 @@ user = "root" network_mode = "no-network" [verifier] -timeout_sec = 4800.0 +# Build + two query runs + integrity judge + setup/cleanup allowance. +# 1800 + 2 * 3600 + 3600 + 600 = 13200 seconds. +timeout_sec = 13200.0 user = "root" environment_mode = "separate" network_mode = "allowlist" diff --git a/tasks/long-document-reranking/tests/README.md b/tasks/long-document-reranking/tests/README.md index 99d2550..12d4bb3 100644 --- a/tasks/long-document-reranking/tests/README.md +++ b/tasks/long-document-reranking/tests/README.md @@ -6,6 +6,12 @@ The hidden query and ground truth files under `data/` are verifier-only. top-5 candidate validity and deterministic scores, and writes the evaluation report. +The outer verifier window is 13,200 seconds: 1,800 seconds for the build, +two query runs of 3,600 seconds each, 3,600 seconds for the trajectory judge, +and a 600-second setup/cleanup allowance. The build and query limits are +enforced independently by `grader.py`; the judge timeout is configured in +`jailbreak_judge/codex.toml`. + After the task-specific grader finishes, `test.sh` invokes the verifier-side AgentJudge configured in `jailbreak_judge/codex.toml`. `finalize_reward.py` applies that result as a hard gate: a valid `Accuracy@5` score is retained only