From 5ba6f38042613b733952a0f1aecc4add040e0fff Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:12:26 +0800 Subject: [PATCH 01/19] fix(webfetch): unify source validation and bound browser capture --- .../references/evaluation-design.md | 2 +- harness/web_fetch/README.md | 29 ++++++-- harness/web_fetch/browser_observations.py | 74 ++++++++++++++++++- harness/web_fetch/reference_lifecycle.py | 9 +-- harness/web_fetch/render_evidence.py | 23 +++++- tests/package_smoke.py | 6 +- tests/test_browser_observation_contracts.py | 31 +++++++- tests/test_browser_reference_scope.py | 6 +- tests/test_dynamic_entry_matching.py | 32 ++++---- tests/test_dynamic_reference.py | 42 +++++++++-- tests/test_freeze_headless_references.py | 10 +-- tests/test_reference_lifecycle.py | 39 ++++++++-- tests/test_render_evidence.py | 54 ++++++++++++++ 13 files changed, 302 insertions(+), 55 deletions(-) diff --git a/.agents/skills/benchmark-report/references/evaluation-design.md b/.agents/skills/benchmark-report/references/evaluation-design.md index ea412c7ea..73e1c5a85 100644 --- a/.agents/skills/benchmark-report/references/evaluation-design.md +++ b/.agents/skills/benchmark-report/references/evaluation-design.md @@ -38,7 +38,7 @@ When comparing builds of one implementation, do not infer a code-induced latency Prefer invariant contracts for changing pages: stable labels, required fields, record shape, types, count bounds, uniqueness, and independently observed explanatory text. Do not freeze ranks, prices, member counts, or current titles unless the benchmark is explicitly a time-bound snapshot. -Ground truth must be independent of the provider under test. Retain source location, bytes or a durable observation, hash, capture method, and scope. Partial annotations support only the requirements they cover. They are not a full-body reference. +Ground truth must be independent of the provider under test. Retain source location, bytes or a durable observation, hash, capture method, and scope. Partial annotations support only the requirements they cover. They are not a full-body reference. When authoring or expanding a task, inspect its actual reference text against the reader's main content needs. A dashboard's labels do not establish its values; a feed's navigation does not establish its entries. Select applicable fact, body-region, record, pattern, resource and file checks through the shared scorer. Give the source-complete answer and a page shell or incomplete answer opposite outcomes before freezing the new version. Preserve earlier input versions. ## Machine-readable current profile diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 915dd6507..f6e95f0c2 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -114,6 +114,19 @@ a changed experiment. Checkpoints and final outputs are created exclusively and cannot replace an existing observation. Store them under the configured data home, outside Git. +The batch isolates each browser worker in a child process. Its wall-clock budget +covers all declared attempts plus two seconds for startup and cleanup, including +browser transport calls that do not honor Playwright's page timeout. A stuck or +crashed worker produces a failed observation and leaves other cases intact. + +Explicit `source_verification.reference_mode: live_exact` applies to every track +and output kind, including stable contracts and mixed text/resource deliveries. +A required live reference must exist before execution. Without that declaration, +compiled references remain frozen. Rendered-reference checks apply the declared +body regions, record contracts, composite fields and patterns as well as facts; +a fact group is optional when another +content requirement establishes the source scope. + ## WCXB scoring `wcxb_scoring.py` owns certified frozen-HTML scoring. The maintained command is @@ -519,9 +532,10 @@ For each case, independently capture and retain: Use scoped visible text/DOM evidence, not script-containing `textContent`. Save the rendered DOM or screenshot alongside text when useful for review. 3. The complete reference for that declared scope, derived independently of - provider outputs. Freeze required fact groups before scoring; every group - needs at least one nonempty alternative and must occur in both reference and - rendered text. Include tail/item-count checks to catch partial lists. + provider outputs. Freeze the applicable body, record, field, pattern or fact + requirements before scoring and check them against both reference and rendered + text. Each declared fact group needs a nonempty alternative. Include tail and + item-count checks to catch partial lists. Use a condition such as “12 product cards visible and loading indicator gone, within 10 seconds,” not merely `networkidle` or a fixed sleep. A delayed request @@ -556,7 +570,9 @@ The result includes exact SHA-256 hashes of initial HTML, rendered text and `ground_truth_main_content`, plus group-by-group coverage for initial static DOM text, initial payload candidates, rendered text and reference. Matching uses the scoring normalizer and numeric boundaries; all groups are mandatory -regardless of `fact_coverage_threshold`. Empty facts are invalid, not 100%. +regardless of `fact_coverage_threshold`. Without fact groups, another declared +content requirement must supply acceptance. Numeric fields require values; +labels or loading placeholders do not establish a completed dashboard. Classifications describe the required facts, not the entire page: @@ -566,7 +582,8 @@ Classifications describe the required facts, not the entire page: on these facts alone. It includes hidden/template/metadata content. - `partial_initial_content`: only some groups found in the initial response. - `absent_from_initial_evidence`: no groups found by this mechanical audit. -- `unclassified`: no required facts were supplied; validation fails. +- `unclassified`: no fact groups were supplied. Other declared content checks + still determine validity. The static visibility proxy excludes scripts, head, templates, noscript and explicitly hidden elements; it cannot compute stylesheets, layout or browser @@ -574,7 +591,7 @@ HTML repair. Payload detection recognizes source literals and JSON string escapes, not arbitrary expressions, compression or external scripts. Therefore absence is not proof of JS necessity: a follow-up HTTP/API request may suffice. `js_required_proven` is always false. Likewise, `valid` checks declared -completion and fact/length gates; it does not independently verify the declared +completion, length and declared content gates; it does not independently verify the declared capture outcome or prove full-page completeness. Retain the actual evidence. Run the synthetic checks without credentials or network: diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index dc346704b..e9e510a8f 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -7,6 +7,9 @@ import hashlib import ipaddress import json +import os +import signal +import subprocess import socket import sys import tempfile @@ -53,6 +56,8 @@ ? node.textContent : ''; } if (node.nodeType !== Node.ELEMENT_NODE) return ''; + // Unrendered elements can return textContent from innerText. + if (['NOSCRIPT', 'TEMPLATE'].includes(node.tagName)) return ''; const style = getComputedStyle(node); if (style.display === 'none' || Number(style.opacity) === 0) return ''; if (!affected.has(node)) return node.innerText || ''; @@ -564,6 +569,11 @@ def capture_with_retries( "message": str(error)[:1000], } ) + return _failed_observation(case, observer, errors, attempts, timeout_ms, last_row) + + +def _failed_observation(case, observer, errors, attempts, timeout_ms, last_row=None): + last_row = last_row or {} return { **last_row, "case_id": case["id"], @@ -599,6 +609,68 @@ def capture_source_identity() -> dict: "lexbench_webfetch/scoring": SCORING_SOURCE_SHA256} + +_CAPTURE_PROGRAM = ( + "import json,sys; from harness.web_fetch.browser_observations import capture_with_retries; " + "print(json.dumps(capture_with_retries(**json.load(sys.stdin))))" +) + + +def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolved_ip=None, + *, observation_scope="reference", dns_mode="system"): + """Bound browser transport and cleanup as well as page operations.""" + payload = dict(case=case, timeout_ms=timeout_ms, attempts=attempts, + browser_binary=browser_binary, resolved_ip=resolved_ip, + observation_scope=observation_scope, dns_mode=dns_mode) + budget = timeout_ms * attempts / 1000 + 2 + began = time.monotonic() + diagnostics = "" + with subprocess.Popen( + [sys.executable, "-c", _CAPTURE_PROGRAM], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + cwd=Path(__file__).resolve().parents[2], start_new_session=os.name == "posix", + env={**os.environ, "DEBUG": "pw:api"}, + ) as process: + try: + stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) + if process.returncode: + raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") + result = json.loads(stdout) + if not isinstance(result, dict) or result.get('case_id') != case['id']: + raise ValueError("Capture worker returned another case") + return result + except subprocess.TimeoutExpired: + # Each POSIX worker owns a new process group, including its driver. + # Terminate only that group so a stuck transport cannot hold pipes open. + signals = (signal.SIGTERM, signal.SIGKILL) if os.name == "posix" else (None, None) + for sig in signals: + try: + if os.name == "posix": + os.killpg(process.pid, sig) + else: + process.kill() + except ProcessLookupError: + pass + try: + _stdout, diagnostics = process.communicate(timeout=1) + break + except subprocess.TimeoutExpired: + continue + message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" + error_type = "TimeoutError" + except (RuntimeError, ValueError) as error: + message, error_type = str(error), type(error).__name__ + row = _failed_observation( + case, {"browser": "Chromium", "binary": "explicit" if browser_binary else "Playwright default", + "dns_mode": dns_mode}, + [{"attempt": None, "type": error_type, "message": message}], None, timeout_ms) + row['completion']['elapsed_ms'] = round((time.monotonic() - began) * 1000, 2) + row['worker_wall_budget_ms'] = budget * 1000 + if diagnostics: + row['worker_diagnostics'] = diagnostics[-4000:] + return row + + def capture_batch( cases, *, @@ -683,7 +755,7 @@ def observe(case): ) if dns_mode != "system": options["dns_mode"] = dns_mode - row = capture_with_retries( + row = _capture_in_process( case, timeout_ms, attempts, browser_binary, resolved_ip, **options ) guard() diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index cca40a390..1049670c7 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -143,16 +143,11 @@ def requires_fresh_reference(case) -> bool: if contract: from lexbench_webfetch.stable_tasks import validate_acceptance_contract validate_acceptance_contract(contract) - return False verification = _get(case, "source_verification", {}) or {} mode = verification.get("reference_mode", "frozen") if mode not in {"frozen", "live_exact"}: raise ReferenceLifecycleError("Unknown reference mode; use frozen or live_exact") - return (mode == "live_exact" - and _get(case, "track", "web") in {"web", "dynamic", "challenge"} - and _get(case, "task_kind", "body") == "body" - and bool((_get(case, "ground_truth_main_content", "") or "").strip()) - and bool(verification)) + return mode == "live_exact" def expected_providers(dataset, case): @@ -593,6 +588,8 @@ def _binding(case, dataset_path: Path, now: datetime) -> dict[str, Any]: identifier = safe_identifier(_get(case, "id")) verification = _get(case, "source_verification") reference = _get(case, "ground_truth_main_content") + if not isinstance(reference, str) or not reference.strip(): + raise ReferenceLifecycleError(f"{identifier}: required live reference is missing") if not isinstance(verification, dict) or verification.get("state") != "valid": raise ReferenceLifecycleError(f"{identifier}: independent source verification is not valid") if not isinstance(verification.get("method"), str) or not verification["method"].strip(): diff --git a/harness/web_fetch/render_evidence.py b/harness/web_fetch/render_evidence.py index 36ca672f8..58ee0a543 100644 --- a/harness/web_fetch/render_evidence.py +++ b/harness/web_fetch/render_evidence.py @@ -117,7 +117,12 @@ def validate_render_reference( errors.append(f"duplicate_fact_id:{identifier}") seen.add(identifier) facts.append({"id": identifier, "alternatives": alternatives}) - if not facts: + body_requirements = get("body_requirements", {}) or {} + required_patterns = get("required_patterns", []) or [] + acceptance_contract = get("acceptance_contract", {}) or {} + composite_acceptance = get("composite_acceptance", {}) or {} + if not facts and not (body_requirements or required_patterns or acceptance_contract + or composite_acceptance): errors.append("required_facts_empty") texts = {"initial_visible": _initial_visible(html), "initial_payload": _payload_candidates(html), @@ -147,6 +152,22 @@ def validate_render_reference( for group in coverage[key]["groups"]: if not group["pass"]: errors.append(f"{key}_missing_fact:{group['id']}") + if body_requirements or required_patterns or acceptance_contract or composite_acceptance: + from lexbench_webfetch.metrics import evaluate_content + for key in ("reference", "rendered"): + checks = evaluate_content(texts[key], [], body_requirements=body_requirements, + required_patterns=required_patterns, + acceptance_contract=acceptance_contract, + composite_acceptance=composite_acceptance, + base_url=get("url", "")) + if not checks["body_coverage_pass"]: + errors.append(f"{key}_body_requirements_not_met") + if not all(row['pass'] for row in checks['pattern_results']): + errors.append(f"{key}_pattern_requirements_not_met") + for field, label in (("stable_task_pass", "record"), + ("composite_acceptance_pass", "composite")): + if checks.get(field) is False: + errors.append(f"{key}_{label}_requirements_not_met") evidence = (get("source_benchmark", {}) or {}).get("render_evidence", {}) completion = evidence.get("completion", {}) condition = completion.get("condition") diff --git a/tests/package_smoke.py b/tests/package_smoke.py index 51ad940e7..aa81ed8c9 100644 --- a/tests/package_smoke.py +++ b/tests/package_smoke.py @@ -1040,13 +1040,13 @@ def verify_browser_observation_capture(root, source_root): binary = root / 'observation-browser' binary.write_bytes(b'synthetic installed browser identity') calls = [] - original = observations.capture_with_retries + original = observations._capture_in_process def observe(case, *args, **kwargs): calls.append(case['id']) return {'case_id': case['id'], 'status': 'success', 'content_text': 'Installed capture'} - observations.capture_with_retries = observe + observations._capture_in_process = observe try: options = dict(timeout_ms=1000, attempts=1, browser_binary=str(binary), checkpoints=root / 'observation-checkpoints') @@ -1054,7 +1054,7 @@ def observe(case, *args, **kwargs): assert observations.capture_batch([task], **options) == first assert calls == ['installed-observation'] finally: - observations.capture_with_retries = original + observations._capture_in_process = original def verify_reference_promotion(root, source_root): diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 95914ed98..39c56bf15 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -36,7 +36,7 @@ def test_invalid_policy_rejected_before_browser_calls(rule, monkeypatch): def unexpected(*args, **kwargs): pytest.fail("Invalid policy reached browser capture") - monkeypatch.setattr(capture, "capture_with_retries", unexpected) + monkeypatch.setattr(capture, "_capture_in_process", unexpected) with pytest.raises(ValueError, match="policy"): capture.capture_batch([case_with(rule)], timeout_ms=1000, attempts=1) @@ -93,7 +93,7 @@ def observe(case, *args, **kwargs): calls.append(case["id"]) return {"case_id": case["id"], "status": "success", "content_text": "Article"} - monkeypatch.setattr(capture, "capture_with_retries", observe) + monkeypatch.setattr(capture, "_capture_in_process", observe) options = dict( timeout_ms=1000, attempts=1, @@ -133,7 +133,7 @@ def observe(case, *args, **kwargs): binary.write_bytes(b"changed during observation") return {"case_id": case["id"]} - monkeypatch.setattr(capture, "capture_with_retries", observe) + monkeypatch.setattr(capture, "_capture_in_process", observe) with pytest.raises(ValueError, match="implementation changed"): capture.capture_batch([case_with()], **options) assert not (options["checkpoints"] / "new-task.json").exists() @@ -207,6 +207,8 @@ def do_GET(self): body = b"""
Hidden panel
Transparent menu
+ + Visible duplicate
Visible duplicate

Translucent text

Inline formatting preserved

@@ -256,6 +258,8 @@ def test_live_multielement_completion_preserves_declared_content_scope(rendered_ if mode == "visible": assert "Inline formatting preserved" in row["content_text"] assert row["content_text"].count("Visible duplicate") == 1 + assert "Fallback code" not in row["content_text"] + assert "Template payload" not in row["content_text"] assert ("Hidden panel" in row["content_text"]) == (mode != "visible") assert ("Excluded panel" in row["content_text"]) == (mode == "dom") @@ -268,3 +272,24 @@ def test_live_ambiguous_content_scope_is_rejected_instead_of_truncated(rendered_ task["url"] = rendered_page with pytest.raises(ValueError, match="exactly one element"): capture.capture(task, 10000, os.environ["BROWSER_EVAL_TEST_BROWSER_BINARY"]) + + +def test_stuck_capture_worker_is_bounded_and_remains_an_unqualified_observation(monkeypatch): + import time + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', 'import time; time.sleep(30)') + began = time.monotonic() + row = capture._capture_in_process(case_with(), 20, 1) + assert time.monotonic() - began < 8 + assert row['status'] == 'failure' + assert row['completion']['satisfied'] is False + assert row['attempt_count'] is None + assert row['attempt_errors'][0]['type'] == 'TimeoutError' + assert 'wall-clock budget' in row['attempt_errors'][0]['message'] + + +def test_crashed_capture_worker_does_not_erase_other_case_results(monkeypatch): + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', "raise RuntimeError('controlled worker failure')") + row = capture._capture_in_process(case_with(), 1000, 1) + assert row['case_id'] == 'new-task' + assert row['status'] == 'failure' + assert 'controlled worker failure' in row['attempt_errors'][0]['message'] diff --git a/tests/test_browser_reference_scope.py b/tests/test_browser_reference_scope.py index c7d9f7dbc..7ccf545ae 100644 --- a/tests/test_browser_reference_scope.py +++ b/tests/test_browser_reference_scope.py @@ -85,7 +85,7 @@ def observe(case, *args, **kwargs): calls.append(case['id']) return {'case_id':case['id'],'status':'failure' if case['id']=='blocked' else 'success', 'content_text':'Verify you are human' if case['id']=='blocked' else 'Article body'} - monkeypatch.setattr(browser_capture,'capture_with_retries',observe) + monkeypatch.setattr(browser_capture,'_capture_in_process',observe) cases=[{'id':k,'url':'https://example.com/'+k} for k in ('article','blocked')] binary = tmp_path / 'browser' binary.write_bytes(b'synthetic browser identity') @@ -109,7 +109,7 @@ def observe(case, *args, **kwargs): def test_checkpoint_cannot_cross_dns_modes(tmp_path, monkeypatch): binary = tmp_path / 'browser' binary.write_bytes(b'synthetic browser identity') - monkeypatch.setattr(browser_capture, 'capture_with_retries', lambda case, *args, **kwargs: { + monkeypatch.setattr(browser_capture, '_capture_in_process', lambda case, *args, **kwargs: { 'case_id': case['id'], 'status': 'success', 'completion': {'satisfied': True}, }) options = dict(timeout_ms=1000, attempts=1, checkpoints=tmp_path / 'checkpoints', @@ -403,7 +403,7 @@ def fake_capture(case, timeout_ms, attempts, browser_binary, resolved_ip): 'completion': {'satisfied': succeeded}, } - monkeypatch.setattr(browser_capture, 'capture_with_retries', fake_capture) + monkeypatch.setattr(browser_capture, '_capture_in_process', fake_capture) monkeypatch.setattr('sys.argv', [ 'capture_headless_browser_observations.py', '--dataset', str(dataset), '--output', str(output), diff --git a/tests/test_dynamic_entry_matching.py b/tests/test_dynamic_entry_matching.py index acc2ffca0..064968064 100644 --- a/tests/test_dynamic_entry_matching.py +++ b/tests/test_dynamic_entry_matching.py @@ -4,7 +4,7 @@ from pathlib import Path from lexbench_webfetch.dynamic_entry_matching import matching_entry_facts -from lexbench_webfetch.dynamic_reference import entry_facts +from tests.test_dynamic_reference import _entry_facts from lexbench_webfetch.metrics import fact_in_text, normalize_for_match @@ -25,7 +25,7 @@ def assert_complete(self, text, facts): def test_corrupt_one_price_and_shipping_with_repeated_values(self): entries = ['First Shirt\n$10.90\nor 9 x$1.21\nFree shipping', 'Second Shirt\n$10.90\nor 9 x$1.21\nFree shipping'] - facts = entry_facts('dynamic-react-cart', entries) + facts = _entry_facts('dynamic-react-cart', entries) reference = '\n\n'.join(entries) for corrupted in [reference.replace('$10.90', '$999.99', 1), reference.replace('Free shipping', '', 1), @@ -36,7 +36,7 @@ def test_corrupt_one_price_and_shipping_with_repeated_values(self): self.assertTrue(hits['entry-2-name']) def test_prefix_shipping_cannot_be_borrowed_from_next_card(self): - facts = entry_facts('dynamic-react-cart', [ + facts = _entry_facts('dynamic-react-cart', [ 'First\n$10\nFree shipping', 'Second\n$10\nFree shipping']) self.assert_complete('Free shipping First $10 Add to cart Free shipping Second $10 Add to cart', facts) hits = self.hits('First $10 Add to cart Free shipping Second $10 Add to cart', facts) @@ -44,7 +44,7 @@ def test_prefix_shipping_cannot_be_borrowed_from_next_card(self): self.assertTrue(hits['entry-2-shipping']) def test_swagger_method_and_path_belong_to_summary(self): - facts = entry_facts('dynamic-swagger', [ + facts = _entry_facts('dynamic-swagger', [ 'POST\n/pet/{petId}/uploadImage\nuploads an image', 'GET\n/pet\nList pets', 'POST\n/user\nCreate user']) text = 'GET /pet/{petId}/uploadImage uploads an image GET /pet List pets POST /user Create user' @@ -54,7 +54,7 @@ def test_swagger_method_and_path_belong_to_summary(self): self.assertFalse(self.hits(text.replace('/user', '/user/login'), facts)['entry-3-path']) def test_native_markdown_link_can_adjoin_method_path_and_summary(self): - facts = entry_facts('dynamic-swagger', [ + facts = _entry_facts('dynamic-swagger', [ 'POST\n/pet\nAdd a new pet to the store', 'GET\n/pet/{petId}\nFind pet by ID']) self.assert_complete( @@ -62,7 +62,7 @@ def test_native_markdown_link_can_adjoin_method_path_and_summary(self): 'GET[/pet/{petId}](#/pet/getPetById)Find pet by ID', facts) def test_overlapping_titles_do_not_invent_missing_record(self): - facts = entry_facts('dynamic-react-cart', [ + facts = _entry_facts('dynamic-react-cart', [ 'Blue T-Shirt\n$10', 'Marine Blue T-shirt\n$10']) hits = self.hits('Marine Blue T-shirt $10', facts) self.assertFalse(hits['entry-1-name']) @@ -73,30 +73,30 @@ def test_overlapping_titles_do_not_invent_missing_record(self): def test_shuffled_and_identical_titles_with_distinct_descriptions(self): entries = ['$299\nSame title\nDark grey laptop\n9 reviews', '$299\nSame title\nGold laptop\n9 reviews'] - facts = entry_facts('dynamic-catalog', entries) + facts = _entry_facts('dynamic-catalog', entries) self.assert_complete('\n\n'.join(reversed(entries)), facts) hits = self.hits(entries[1], facts) self.assertFalse(hits['entry-1-description']) self.assertTrue(hits['entry-2-description']) def test_duplicate_identical_records_need_disjoint_tuples(self): - facts = entry_facts('dynamic-product-reviews', ['2022-07-22\nGood', '2022-07-22\nGood']) + facts = _entry_facts('dynamic-product-reviews', ['2022-07-22\nGood', '2022-07-22\nGood']) hits = self.hits('2022-07-22 Good', facts) self.assertEqual(sum(bool(v) for k, v in hits.items() if k.endswith('-text')), 1) self.assert_complete('2022-07-22 Good 2022-07-22 Good', facts) def test_labels_separators_and_explicit_layouts(self): - facts = entry_facts('dynamic-swagger', ['POST\n/pet\nCreate pet']) + facts = _entry_facts('dynamic-swagger', ['POST\n/pet\nCreate pet']) self.assert_complete('Method: POST | Path: /pet | Summary: Create pet', facts) - facts = entry_facts('dynamic-catalog', ['$10\nTitle\nDescription\n1 reviews']) + facts = _entry_facts('dynamic-catalog', ['$10\nTitle\nDescription\n1 reviews']) self.assert_complete('Title | Price: $10 | Description: Description | 1reviews', facts) - facts = entry_facts('dynamic-quotes-api', ['“Hello.”\nby Someone\nTags: one two']) + facts = _entry_facts('dynamic-quotes-api', ['“Hello.”\nby Someone\nTags: one two']) self.assert_complete('“Hello.”by Someone Tags: one two', facts) - facts = entry_facts('dynamic-products-api', ['Hoodie\n$52']) + facts = _entry_facts('dynamic-products-api', ['Hoodie\n$52']) self.assert_complete('Hoodie\\\\ \\ $52', facts) def test_arbitrary_windows_order_and_numeric_fragments_rejected(self): - facts = entry_facts('dynamic-products-api', ['Shirt\n$10']) + facts = _entry_facts('dynamic-products-api', ['Shirt\n$10']) for text in ['Shirt unrelated product $10', 'Shirt $100', 'Shirt $10.90', 'Shirt -$10', 'Shirt $99 Other $10', '$10 Shirt']: self.assertFalse(any(self.hits(text, facts).values()), text) @@ -107,7 +107,7 @@ def test_fails_closed_without_mutating_frozen_facts(self): before = copy.deepcopy(facts) self.assertEqual(self.hits('Hi', facts), {'entry-1-unknown': []}) self.assertEqual(facts, before) - facts = entry_facts('dynamic-products-api', ['Shirt\n$10']) + facts = _entry_facts('dynamic-products-api', ['Shirt\n$10']) self.assertFalse(any(self.hits('Shirt $10', facts + [facts[0]]).values())) def test_all_frozen_captures_and_missing_records(self): @@ -115,7 +115,7 @@ def test_all_frozen_captures_and_missing_records(self): for filename in ('dynamic-browser-captures.json', 'dynamic-expansion-captures.json'): captures += json.loads((ROOT/'research'/filename).read_text()) for capture in captures: - facts = entry_facts(capture['id'], capture['entries']) + facts = _entry_facts(capture['id'], capture['entries']) with self.subTest(case=capture['id']): self.assert_complete(capture['rendered_text'], facts) self.assert_complete('\n\n'.join(reversed(capture['entries'])), facts) @@ -125,7 +125,7 @@ def test_all_frozen_captures_and_missing_records(self): def test_every_new_capture_field_corruption_is_rejected(self): captures = json.loads((ROOT/'research/dynamic-expansion-captures.json').read_text()) for capture in captures: - facts = entry_facts(capture['id'], capture['entries']) + facts = _entry_facts(capture['id'], capture['entries']) for fact in facts: index = int(fact['id'].split('-')[1]) - 1 entries = capture['entries'][:] diff --git a/tests/test_dynamic_reference.py b/tests/test_dynamic_reference.py index 2eb026ec7..fc0eb526c 100644 --- a/tests/test_dynamic_reference.py +++ b/tests/test_dynamic_reference.py @@ -1,18 +1,50 @@ import copy import json +import re import unittest from pathlib import Path -from lexbench_webfetch.dynamic_reference import entry_facts from lexbench_webfetch.metrics import evaluate_content ROOT=Path(__file__).resolve().parents[1] + +def _entry_facts(case_id, entries): + fields = (['quote', 'author', 'tags'] if case_id in {'dynamic-quotes-api','dynamic-delayed'} + else ['name','price'] if case_id=='dynamic-products-api' + else ['price','title','description','reviews'] if case_id=='dynamic-catalog' + else ['date','text'] if case_id in {'dynamic-reviews-graphql','dynamic-product-reviews'} + else ['name','description'] if case_id=='dynamic-angular-phones' + else ['method','path','summary'] if case_id=='dynamic-swagger' + else ['family'] if case_id=='dynamic-iframe' + else ['name','price','financing','shipping'] if case_id=='dynamic-react-cart' else None) + if not fields:raise ValueError('Unknown dynamic case') + facts=[] + for i,entry in enumerate(entries,1): + values=[line.strip() for line in entry.splitlines() if line.strip()] + entry_fields = fields + if case_id == 'dynamic-react-cart': + # Shipping and financing are optional visible fields on these cards. + entry_fields = ['name', 'price'] + if len(values)>2 and values[2].startswith('or '):entry_fields.append('financing') + if values[-1]=='Free shipping':entry_fields.append('shipping') + if len(values)!=len(entry_fields):raise ValueError(f'Incomplete or unexpected fields: {case_id} entry {i}') + for field,value in zip(entry_fields,values): + if field=='author':value=value.removeprefix('by ') + if field=='tags':value=value.removeprefix('Tags: ') + alternatives=[value] + # Numeric count + its label can be adjacent in Markdown extraction. + # Keep the whole number/unit, never remove arbitrary word boundaries. + if field=='reviews' and re.fullmatch(r'\d+\s+reviews?',value): + alternatives.append(re.sub(r'\s+','',value)) + facts.append({'id':f'entry-{i}-{field}','alternatives':alternatives}) + return facts + class DynamicReferenceTests(unittest.TestCase): def test_all_observed_fields_are_required(self): captures=json.loads((ROOT/'research/dynamic-browser-captures.json').read_text()) captures+=json.loads((ROOT/'research/dynamic-expansion-captures.json').read_text()) for c in captures: - facts=entry_facts(c['id'],c['entries']) + facts=_entry_facts(c['id'],c['entries']) ref=c['rendered_text'] score=lambda text:evaluate_content(text,[],required_facts=facts,min_content_chars=1) self.assertTrue(score(ref)['content_pass'],c['id']) @@ -23,10 +55,10 @@ def test_all_observed_fields_are_required(self): def test_placeholder_or_incomplete_card_is_rejected(self): for entries in [[''],['Only a product name']]: - with self.assertRaises(ValueError):entry_facts('dynamic-products-api',entries) + with self.assertRaises(ValueError):_entry_facts('dynamic-products-api',entries) def test_count_label_spacing_preserves_numeric_identity(self): - facts=entry_facts('dynamic-catalog',['$10\nTitle\nDescription\n1 reviews']) + facts=_entry_facts('dynamic-catalog',['$10\nTitle\nDescription\n1 reviews']) score=lambda text:evaluate_content(text,[],required_facts=facts,min_content_chars=1)['content_pass'] self.assertTrue(score('$10 Title Description 1reviews')) self.assertFalse(score('$10 Title Description 11reviews')) @@ -39,7 +71,7 @@ def test_new_observed_values_replace_old_fact_values(self): updated = copy.deepcopy(case['entries']) old = updated[0] updated[0] = 'Updated family name' - facts = entry_facts(case['id'], updated) + facts = _entry_facts(case['id'], updated) fresh = '\n'.join(updated) stale = '\n'.join(case['entries']) score = lambda text: evaluate_content(text, [], required_facts=facts, min_content_chars=1) diff --git a/tests/test_freeze_headless_references.py b/tests/test_freeze_headless_references.py index b5780f3d5..921357061 100644 --- a/tests/test_freeze_headless_references.py +++ b/tests/test_freeze_headless_references.py @@ -165,7 +165,7 @@ def test_unavailable_body_reference_recovers_from_complete_clean_capture(self): self.assertEqual(frozen['cases'][0]['source_verification']['state'],'valid') self.assertEqual(changes[0]['case_id'],case['id']) - def test_unavailable_body_reference_stays_unavailable_when_capture_is_blocked(self): + def test_required_live_reference_cannot_freeze_a_blocked_empty_capture(self): with tempfile.TemporaryDirectory() as directory: captures=Path(directory) case={'id':'blocked-reference','name':'Blocked reference', @@ -179,10 +179,10 @@ def test_unavailable_body_reference_stays_unavailable_when_capture_is_blocked(se 'sha256':'b'*64,'quality_issues':['http_status_403']} (captures/'audit.json').write_text(json.dumps([audit])) - frozen,changes,_=compute_reference_update({'cases':[case]},captures) - - self.assertEqual(frozen['cases'][0],case) - self.assertEqual(changes,[]) + with self.assertRaisesRegex(ValueError, 'requires independent browser observation'): + compute_reference_update({'cases':[case]},captures) + self.assertEqual(case['ground_truth_main_content'], '') + self.assertEqual(case['source_verification']['state'], 'unavailable') def test_freeze_writes_nothing_until_changed_scope_is_promoted(self): with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index cb0ac631e..be8250d5e 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -1,4 +1,5 @@ """Synthetic clocks and captures only: no network, browsers or real providers.""" +import pytest import json import tempfile import unittest @@ -147,10 +148,11 @@ def test_equivalent_capture_allows_selector_wording_but_not_scoring_scope(self): with self.assertRaises(ReferenceLifecycleError): check_case_equivalent_reference_binding(replace(equivalent, task_prompt="Different task"), binding) - def test_only_body_with_independent_metadata_is_required(self): - for case in (replace(self.case, source_verification={}), - replace(self.case, task_kind="structure"), replace(self.case, ground_truth_main_content="")): - self.assertIsNone(self.checks(case)) + def test_explicit_reference_requirements_are_independent_of_output_kind(self): + self.assertIsNone(self.checks(replace(self.case, source_verification={}))) + self.assertIsNotNone(self.checks(replace(self.case, task_kind="structure"))) + with self.assertRaisesRegex(ReferenceLifecycleError, "required live reference is missing"): + self.checks(replace(self.case, ground_truth_main_content="")) with self.assertRaisesRegex(ReferenceLifecycleError, "resource evidence binding"): self.checks(replace(self.case, track="image", task_kind="resource", expected_resources=[{"id": "image", "urls": ["https://example.test/a.png"]}])) @@ -622,6 +624,7 @@ def test_public_guard_allows_complete_current_batch_without_live_references(self stable = replace( self.case, id="stable", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}, acceptance_contract={ "version": 1, "schema": "product", @@ -629,7 +632,8 @@ def test_public_guard_allows_complete_current_batch_without_live_references(self "stable_text_anchors": [{"id": "section", "alternatives": ["Catalog"]}], }, ) - resource = replace(self.case, id="resource", track="image", task_kind="resource") + resource = replace(self.case, id="resource", track="image", task_kind="resource", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}) records = [ {"attempt_id": "p__stable__r001", "case_id": "stable", "provider": "p", "started_at": "2026-01-01T00:00:01+00:00", "elapsed_ms": 10, @@ -655,6 +659,7 @@ def test_public_guard_rejects_incomplete_current_batch_without_live_references(s stable = replace( self.case, id="stable", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}, acceptance_contract={ "version": 1, "schema": "product", @@ -687,6 +692,7 @@ def test_current_batch_is_strict_inside_mixed_historical_manifests(self): stable = replace( self.case, id="stable", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}, acceptance_contract={ "version": 1, "schema": "product", @@ -799,3 +805,26 @@ def test_legacy_rescore_still_allowed_without_claiming_protocol(self): if __name__ == "__main__": unittest.main() + + +@pytest.mark.parametrize('track', ['web', 'dynamic', 'challenge', 'risk_control', 'document', 'image', 'video']) +@pytest.mark.parametrize('kind', ['body', 'structure', 'resource', 'file']) +def test_explicit_live_reference_is_selected_for_every_mixed_contract(track, kind): + from harness.web_fetch.reference_lifecycle import requires_fresh_reference + case = {'id': 'live', 'url': 'https://example.test/', 'track': track, 'task_kind': kind, + 'ground_truth_main_content': 'Substantive independently observed article', + 'acceptance_contract': {'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'title', 'alternatives': ['Article']}]}, + 'source_verification': {'reference_mode': 'live_exact'}} + assert requires_fresh_reference(case) + case['source_verification'] = {} + assert not requires_fresh_reference(case) + + +def test_unknown_reference_mode_is_not_hidden_by_a_stable_contract(): + from harness.web_fetch.reference_lifecycle import requires_fresh_reference + with pytest.raises(ReferenceLifecycleError, match='reference mode'): + requires_fresh_reference({'acceptance_contract': { + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'title', 'alternatives': ['Article']}]}, + 'source_verification': {'reference_mode': 'live_excat'}}) diff --git a/tests/test_render_evidence.py b/tests/test_render_evidence.py index 6dcd1c640..49550ee6a 100644 --- a/tests/test_render_evidence.py +++ b/tests/test_render_evidence.py @@ -163,3 +163,57 @@ def test_elapsed_zero_exact_deadline_and_omission_are_allowed(self): if __name__ == "__main__": unittest.main() + + +class DeclaredContentReferenceTests(unittest.TestCase): + def test_body_scope_is_sufficient_without_an_artificial_fact_group(self): + from lexbench_webfetch.body_coverage import body_requirements + text = 'An article explains how batteries store energy and how charging changes their capacity.' + case = {'id': 'article', 'url': 'https://example.test/', 'required_facts': [], + 'ground_truth_main_content': text, 'body_requirements': body_requirements(text), + 'source_benchmark': {'render_evidence': { + 'scope': 'Complete article body', 'rendered_truncated': False, + 'completion': {'condition': 'Article loaded', 'satisfied': True, 'timeout_ms': 10000}}}} + self.assertTrue(validate_render_reference(case, '
Loading
', text)['valid']) + missing = validate_render_reference(case, '
Loading
', 'Home Terms Privacy') + self.assertFalse(missing['valid']) + self.assertIn('rendered_body_requirements_not_met', missing['errors']) + + def test_pattern_scope_is_checked_on_both_reference_and_observation(self): + case = {'id': 'code', 'url': 'https://example.test/', 'required_facts': [], + 'ground_truth_main_content': 'revenue = 123', + 'required_patterns': [{'id': 'formula', 'pattern': r'revenue\s*=\s*\d+', 'minimum': 1}], + 'source_benchmark': {'render_evidence': { + 'scope': 'Visible worked example', 'rendered_truncated': False, + 'completion': {'condition': 'Example loaded', 'satisfied': True, 'timeout_ms': 10000}}}} + self.assertTrue(validate_render_reference(case, '
Loading
', 'revenue = 123')['valid']) + self.assertFalse(validate_render_reference(case, '
Loading
', 'Home Terms')['valid']) + case['ground_truth_main_content'] = 'Home Terms' + self.assertFalse(validate_render_reference(case, '
Loading
', 'revenue = 123')['valid']) + + + def test_declared_dashboard_fields_require_values_in_both_source_surfaces(self): + case = {'id': 'dashboard', 'url': 'https://example.test/', + 'ground_truth_main_content': 'Submitted: 123\nApproved: 98', + 'acceptance_contract': {'version': 1, 'schema': 'body_text', + 'minimum_records': 0, 'stable_text_anchors': [ + {'id': 'submitted', 'alternatives': ['Submitted']}]}, + 'composite_acceptance': {'version': 1, 'groups': [ + {'id': 'counts', 'kind': 'field_report', 'required_fields': [ + {'id': 'submitted', 'kind': 'integer', 'aliases': ['Submitted']}, + {'id': 'approved', 'kind': 'integer', 'aliases': ['Approved']}]}]}, + 'source_benchmark': {'render_evidence': { + 'scope': 'Submitted and approved counts', 'rendered_truncated': False, + 'completion': {'condition': 'Counts loaded', 'satisfied': True, 'timeout_ms': 10000}}}} + full = case['ground_truth_main_content'] + self.assertTrue(validate_render_reference(case, '
Loading
', full)['valid']) + for invalid in ('Submitted: -\nApproved: -', 'Submitted: 123', 'Privacy Terms'): + with self.subTest(invalid=invalid): + self.assertFalse(validate_render_reference(case, '
Loading
', invalid)['valid']) + case['ground_truth_main_content'] = invalid + self.assertFalse(validate_render_reference(case, '
Loading
', full)['valid']) + case['ground_truth_main_content'] = full + case['acceptance_contract']['stable_text_anchors'][0]['alternatives'] = ['Quarterly totals'] + self.assertFalse(validate_render_reference(case, '
Loading
', full)['valid']) + del case['acceptance_contract'] + self.assertTrue(validate_render_reference(case, '
Loading
', full)['valid']) From 8ec939a618d20274615bc6e754f853aa0b2492b6 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:05:16 +0800 Subject: [PATCH 02/19] fix(webfetch): bind live rules and clean up capture descendants --- harness/web_fetch/browser_observations.py | 26 ++++++-- harness/web_fetch/reference_lifecycle.py | 6 +- tests/test_browser_observation_contracts.py | 66 +++++++++++++++++++++ tests/test_reference_lifecycle.py | 30 ++++++++++ 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index e9e510a8f..dd341c7e0 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -619,6 +619,8 @@ def capture_source_identity() -> dict: def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolved_ip=None, *, observation_scope="reference", dns_mode="system"): """Bound browser transport and cleanup as well as page operations.""" + if browser_binary: + browser_binary = str(Path(browser_binary).resolve()) payload = dict(case=case, timeout_ms=timeout_ms, attempts=attempts, browser_binary=browser_binary, resolved_ip=resolved_ip, observation_scope=observation_scope, dns_mode=dns_mode) @@ -640,20 +642,34 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve raise ValueError("Capture worker returned another case") return result except subprocess.TimeoutExpired: - # Each POSIX worker owns a new process group, including its driver. - # Terminate only that group so a stuck transport cannot hold pipes open. + # Chrome starts a detached group. Collect descendants before killing + # the driver, while their parent relationships still identify ownership. + owned = {process.pid} + if os.name == "posix": + table = subprocess.check_output( + ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) + parents = {int(pid): int(parent) for pid, parent in + (line.split() for line in table.splitlines())} + while True: + children = {pid for pid, parent in parents.items() if parent in owned} + if children <= owned: + break + owned.update(children) signals = (signal.SIGTERM, signal.SIGKILL) if os.name == "posix" else (None, None) for sig in signals: try: if os.name == "posix": - os.killpg(process.pid, sig) + for pid in sorted(owned, reverse=True): + try: + os.kill(pid, sig) + except ProcessLookupError: + pass else: process.kill() except ProcessLookupError: pass try: _stdout, diagnostics = process.communicate(timeout=1) - break except subprocess.TimeoutExpired: continue message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" @@ -699,6 +715,8 @@ def capture_batch( public_ip(resolved_ip) if checkpoints and not browser_binary: raise ValueError("Checkpoint reuse requires an explicit browser binary") + if browser_binary: + browser_binary = str(Path(browser_binary).resolve()) sources = capture_source_identity() binary_sha = ( hashlib.sha256(Path(browser_binary).read_bytes()).hexdigest() if browser_binary else None diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 1049670c7..54459c236 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -220,7 +220,11 @@ def scope_identity(case) -> dict[str, Any]: ("expected_any", []), ("required_facts", []), ("required_patterns", []), ("forbidden_any", []), ("min_content_chars", 1), ("fact_coverage_threshold", 1.0), ("content_score_applicable", True), ("expected_resources", []), - ("resource_output_contract", {}), ("reference_answer", ""))}, + ("resource_output_contract", {}), ("reference_answer", ""), + ("acceptance_contract", {}), ("body_requirements", {}), + ("composite_acceptance", {})) + if key not in {"acceptance_contract", "body_requirements", "composite_acceptance"} + or _get(case, key, default)}, }) diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 39c56bf15..a12e76248 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -293,3 +293,69 @@ def test_crashed_capture_worker_does_not_erase_other_case_results(monkeypatch): assert row['case_id'] == 'new-task' assert row['status'] == 'failure' assert 'controlled worker failure' in row['attempt_errors'][0]['message'] + + +@pytest.mark.skipif(not os.getenv('BROWSER_EVAL_TEST_BROWSER_BINARY'), reason='explicit Chromium required') +def test_batch_preserves_browser_path_relative_to_external_caller(tmp_path, monkeypatch, rendered_page): + from pathlib import Path + + (tmp_path / 'browser').mkdir() + (tmp_path / 'browser/chrome').symlink_to(Path(os.environ['BROWSER_EVAL_TEST_BROWSER_BINARY']).resolve()) + monkeypatch.chdir(tmp_path) + # Only the loopback fixture bypasses public DNS qualification. + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', + 'import harness.web_fetch.browser_observations as c; ' + 'c.resolve_public_host=lambda url, ip: ("127.0.0.1", None, "local-fixture"); ' + + capture._CAPTURE_PROGRAM) + task = case_with({'selector': 'main', 'text_mode': 'visible'}) + task['url'] = rendered_page + result = capture.capture_batch([task], timeout_ms=10000, attempts=1, + browser_binary='browser/chrome', checkpoints=tmp_path / 'checkpoints') + assert result['new-task']['completion']['satisfied'], json.dumps(result, indent=2) + assert 'Inline formatting preserved' in result['new-task']['content_text'] + + +@pytest.mark.skipif(os.name != 'posix' or not os.getenv('BROWSER_EVAL_TEST_BROWSER_BINARY'), + reason='POSIX and explicit Chromium required') +def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser(tmp_path, monkeypatch): + import subprocess + from playwright.sync_api import sync_playwright + + pid_file = tmp_path / 'owned-browser.json' + binary = os.environ['BROWSER_EVAL_TEST_BROWSER_BINARY'] + program = f''' +import json, os, signal, subprocess, time +from pathlib import Path +from playwright.sync_api import sync_playwright +with sync_playwright() as p: + browser = p.chromium.launch(executable_path={binary!r}, headless=True, args=['--no-sandbox']) + driver = p._impl_obj._connection._transport._proc.pid + table = subprocess.check_output(['ps', '-A', '-o', 'pid=,ppid='], text=True) + children = [int(pid) for pid, parent in (line.split() for line in table.splitlines()) + if int(parent) == driver] + Path({str(pid_file)!r}).write_text(json.dumps(children)) + os.kill(driver, signal.SIGSTOP) + time.sleep(30) +''' + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) + with sync_playwright() as p: + unrelated = p.chromium.launch(executable_path=binary, headless=True, args=['--no-sandbox']) + try: + result = capture._capture_in_process(case_with(), 4000, 1, binary) + assert result['status'] == 'failure' + owned = json.loads(pid_file.read_text()) + assert owned + for pid in owned: + state = subprocess.run(['ps', '-o', 'stat=', '-p', str(pid)], + capture_output=True, text=True).stdout.strip() + assert not state or state.startswith('Z'), (pid, state) + assert unrelated.new_page().evaluate('1 + 1') == 2 + finally: + unrelated.close() + if pid_file.exists(): + import signal + for pid in json.loads(pid_file.read_text()): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index be8250d5e..818557eb7 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -562,6 +562,36 @@ def test_rescore_preserves_binding_without_claiming_current_freshness(self): self.assertEqual(records[0]["metrics"]["reference_sha256"], sha256(self.case.ground_truth_main_content)) self.assertEqual(json.loads((self.run / "manifest.json").read_text())["reference_checks"], original["reference_checks"]) + def test_live_contract_changes_require_new_source_certification(self): + self.case = replace(self.case, acceptance_contract={ + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'price', 'alternatives': ['Price']}]}) + original = self.prepare_run() + record = json.loads((self.run / 'results.jsonl').read_text()) + record.update(provider_type='fixture', case_name='Body', category='static') + (self.run / 'results.jsonl').write_text(json.dumps(record) + '\n') + self.finalize([self.observation(original)]) + manifest = json.loads((self.run / 'manifest.json').read_text()) + require_verified_reference_checks([manifest], cases=[self.case], source=self.run) + rescore_evaluation(run_dir=self.run, dataset_meta=self.dataset(), + dataset_path=self.dataset_path, cases=[self.case]) + before = {p: p.read_bytes() for p in self.run.rglob('*') if p.is_file()} + changes = { + 'acceptance_contract': {**self.case.acceptance_contract, + 'stable_text_anchors': [{'id': 'price', 'alternatives': ['Provider raw output']}]}, + 'body_requirements': {'required_regions': [{'id': 'new', 'alternatives': ['Different']}]}, + 'composite_acceptance': {'required_fields': ['Different']}, + } + for field, value in changes.items(): + with self.subTest(field=field): + changed = replace(self.case, **{field: value}) + with self.assertRaises(ReferenceLifecycleError): + require_verified_reference_checks([manifest], cases=[changed], source=self.run) + with self.assertRaises(ReferenceLifecycleError): + rescore_evaluation(run_dir=self.run, dataset_meta=self.dataset(changed), + dataset_path=self.dataset_path, cases=[changed]) + self.assertEqual(before, {p: p.read_bytes() for p in self.run.rglob('*') if p.is_file()}) + def verified_run(self): original = self.prepare_run() self.finalize([self.observation(original)]) From 4ff6fdbe53acdb8e6b732dbfb3ad084f3199c8d5 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:13:35 +0800 Subject: [PATCH 03/19] fix(webfetch): retain timeout bounds when process discovery fails --- harness/web_fetch/browser_observations.py | 15 +++++++++++---- tests/test_browser_observation_contracts.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index dd341c7e0..b829ec6ed 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -646,10 +646,13 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve # the driver, while their parent relationships still identify ownership. owned = {process.pid} if os.name == "posix": - table = subprocess.check_output( - ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) - parents = {int(pid): int(parent) for pid, parent in - (line.split() for line in table.splitlines())} + try: + table = subprocess.check_output( + ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) + parents = {int(pid): int(parent) for pid, parent in + (line.split() for line in table.splitlines())} + except (OSError, subprocess.SubprocessError): + parents = {} while True: children = {pid for pid, parent in parents.items() if parent in owned} if children <= owned: @@ -659,6 +662,10 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve for sig in signals: try: if os.name == "posix": + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + pass for pid in sorted(owned, reverse=True): try: os.kill(pid, sig) diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index a12e76248..ff25edda5 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -359,3 +359,24 @@ def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser( os.killpg(pid, signal.SIGKILL) except ProcessLookupError: pass + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX process enumeration') +@pytest.mark.parametrize('failure', ['missing', 'exit', 'timeout']) +def test_capture_timeout_remains_bounded_when_process_enumeration_fails(monkeypatch, failure): + import subprocess + import time + + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', 'import time; time.sleep(4)') + def unavailable(*args, **kwargs): + if failure == 'missing': + raise FileNotFoundError('ps is unavailable') + if failure == 'exit': + raise subprocess.CalledProcessError(1, 'ps') + raise subprocess.TimeoutExpired('ps', 1) + monkeypatch.setattr(capture.subprocess, 'check_output', unavailable) + began = time.monotonic() + result = capture._capture_in_process(case_with(), 20, 1) + assert time.monotonic() - began < 3.8 + assert result['status'] == 'failure' + assert result['attempt_errors'][0]['type'] == 'TimeoutError' From 8a065e5eb8703ce0e47d5d52c0bb44cac70f3ec6 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:31:48 +0800 Subject: [PATCH 04/19] fix(webfetch): preserve capture isolation when process tools fail --- harness/web_fetch/browser_observations.py | 122 +++++++++++--------- tests/test_browser_observation_contracts.py | 61 ++++++++-- 2 files changed, 122 insertions(+), 61 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index b829ec6ed..6c625cd74 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -8,6 +8,7 @@ import ipaddress import json import os +import re import signal import subprocess import socket @@ -627,62 +628,77 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve budget = timeout_ms * attempts / 1000 + 2 began = time.monotonic() diagnostics = "" - with subprocess.Popen( - [sys.executable, "-c", _CAPTURE_PROGRAM], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - cwd=Path(__file__).resolve().parents[2], start_new_session=os.name == "posix", - env={**os.environ, "DEBUG": "pw:api"}, - ) as process: - try: - stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) - if process.returncode: - raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") - result = json.loads(stdout) - if not isinstance(result, dict) or result.get('case_id') != case['id']: - raise ValueError("Capture worker returned another case") - return result - except subprocess.TimeoutExpired: - # Chrome starts a detached group. Collect descendants before killing - # the driver, while their parent relationships still identify ownership. - owned = {process.pid} - if os.name == "posix": - try: - table = subprocess.check_output( - ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) - parents = {int(pid): int(parent) for pid, parent in - (line.split() for line in table.splitlines())} - except (OSError, subprocess.SubprocessError): - parents = {} - while True: - children = {pid for pid, parent in parents.items() if parent in owned} - if children <= owned: - break - owned.update(children) - signals = (signal.SIGTERM, signal.SIGKILL) if os.name == "posix" else (None, None) - for sig in signals: - try: - if os.name == "posix": - try: - os.killpg(process.pid, sig) - except ProcessLookupError: - pass - for pid in sorted(owned, reverse=True): + try: + process = subprocess.Popen( + [sys.executable, "-c", _CAPTURE_PROGRAM], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + cwd=Path(__file__).resolve().parents[2], start_new_session=os.name == "posix", + env={**os.environ, "DEBUG": "pw:api,pw:browser"}, + ) + except OSError as error: + message, error_type = str(error), type(error).__name__ + else: + with process: + try: + stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) + if process.returncode: + raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") + result = json.loads(stdout) + if not isinstance(result, dict) or result.get('case_id') != case['id']: + raise ValueError("Capture worker returned another case") + return result + except subprocess.TimeoutExpired as error: + # Chrome starts a detached group. Collect descendants before killing + # the driver, while their parent relationships still identify ownership. + # The launcher receipt survives an unavailable process-list command. + partial = error.stderr or b"" + diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial + browser_pids = {int(pid) for pid in re.findall( + r"(?m)^\S+\s+pw:browser pid=(\d+)\s*$", diagnostics)} + owned = {process.pid} + if os.name == "posix": + try: + table = subprocess.check_output( + ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) + parents = {int(pid): int(parent) for pid, parent in + (line.split() for line in table.splitlines())} + except (OSError, subprocess.SubprocessError): + parents = {} + while True: + children = {pid for pid, parent in parents.items() if parent in owned} + if children <= owned: + break + owned.update(children) + signals = (signal.SIGTERM, signal.SIGKILL) if os.name == "posix" else (None, None) + for sig in signals: + try: + if os.name == "posix": + for browser_pid in browser_pids: + try: + os.kill(browser_pid, sig) + except ProcessLookupError: + pass try: - os.kill(pid, sig) + os.killpg(process.pid, sig) except ProcessLookupError: pass - else: - process.kill() - except ProcessLookupError: - pass - try: - _stdout, diagnostics = process.communicate(timeout=1) - except subprocess.TimeoutExpired: - continue - message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" - error_type = "TimeoutError" - except (RuntimeError, ValueError) as error: - message, error_type = str(error), type(error).__name__ + for pid in sorted(owned, reverse=True): + try: + os.kill(pid, sig) + except ProcessLookupError: + pass + else: + process.kill() + except ProcessLookupError: + pass + try: + _stdout, diagnostics = process.communicate(timeout=1) + except subprocess.TimeoutExpired: + continue + message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" + error_type = "TimeoutError" + except (RuntimeError, ValueError) as error: + message, error_type = str(error), type(error).__name__ row = _failed_observation( case, {"browser": "Chromium", "binary": "explicit" if browser_binary else "Playwright default", "dns_mode": dns_mode}, diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index ff25edda5..11bb22752 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -317,7 +317,8 @@ def test_batch_preserves_browser_path_relative_to_external_caller(tmp_path, monk @pytest.mark.skipif(os.name != 'posix' or not os.getenv('BROWSER_EVAL_TEST_BROWSER_BINARY'), reason='POSIX and explicit Chromium required') -def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser(tmp_path, monkeypatch): +@pytest.mark.parametrize('discovery_failure', [None, 'missing', 'exit', 'timeout']) +def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser(tmp_path, monkeypatch, discovery_failure): import subprocess from playwright.sync_api import sync_playwright @@ -329,15 +330,30 @@ def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser( from playwright.sync_api import sync_playwright with sync_playwright() as p: browser = p.chromium.launch(executable_path={binary!r}, headless=True, args=['--no-sandbox']) + browser.new_page() driver = p._impl_obj._connection._transport._proc.pid table = subprocess.check_output(['ps', '-A', '-o', 'pid=,ppid='], text=True) - children = [int(pid) for pid, parent in (line.split() for line in table.splitlines()) - if int(parent) == driver] + parents = {{int(pid): int(parent) for pid, parent in (line.split() for line in table.splitlines())}} + children = {{pid for pid, parent in parents.items() if parent == driver}} + while True: + descendants = {{pid for pid, parent in parents.items() if parent in children}} + if descendants <= children: + break + children.update(descendants) + children = sorted(children) Path({str(pid_file)!r}).write_text(json.dumps(children)) os.kill(driver, signal.SIGSTOP) time.sleep(30) ''' monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) + if discovery_failure: + def unavailable(*args, **kwargs): + if discovery_failure == 'missing': + raise FileNotFoundError('ps missing') + if discovery_failure == 'exit': + raise subprocess.CalledProcessError(1, 'ps') + raise subprocess.TimeoutExpired('ps', 1) + monkeypatch.setattr(capture.subprocess, 'check_output', unavailable) with sync_playwright() as p: unrelated = p.chromium.launch(executable_path=binary, headless=True, args=['--no-sandbox']) try: @@ -345,10 +361,16 @@ def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser( assert result['status'] == 'failure' owned = json.loads(pid_file.read_text()) assert owned - for pid in owned: - state = subprocess.run(['ps', '-o', 'stat=', '-p', str(pid)], - capture_output=True, text=True).stdout.strip() - assert not state or state.startswith('Z'), (pid, state) + import time + deadline = time.monotonic() + 2 + while True: + alive = [(pid, subprocess.run(['ps', '-o', 'stat=', '-p', str(pid)], + capture_output=True, text=True).stdout.strip()) for pid in owned] + alive = [(pid, state) for pid, state in alive if state and not state.startswith('Z')] + if not alive or time.monotonic() >= deadline: + break + time.sleep(0.05) + assert not alive, alive assert unrelated.new_page().evaluate('1 + 1') == 2 finally: unrelated.close() @@ -356,7 +378,7 @@ def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser( import signal for pid in json.loads(pid_file.read_text()): try: - os.killpg(pid, signal.SIGKILL) + os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass @@ -380,3 +402,26 @@ def unavailable(*args, **kwargs): assert time.monotonic() - began < 3.8 assert result['status'] == 'failure' assert result['attempt_errors'][0]['type'] == 'TimeoutError' + + +def test_worker_launch_failure_is_checkpointed_without_aborting_the_batch(tmp_path, monkeypatch): + original = capture.subprocess.Popen + calls = [] + def launch(*args, **kwargs): + calls.append(1) + if len(calls) == 1: + raise OSError('controlled process launch failure') + return original(*args, **kwargs) + monkeypatch.setattr(capture.subprocess, 'Popen', launch) + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', + 'import json,sys; p=json.load(sys.stdin); print(json.dumps({' + '"case_id":p["case"]["id"],"status":"success","completion":{"satisfied":True}}))') + binary = tmp_path / 'browser' + binary.write_bytes(b'fixture binary identity') + cases = [case_with(), {**case_with(), 'id': 'next-task'}] + result = capture.capture_batch(cases, timeout_ms=1000, attempts=1, + browser_binary=str(binary), checkpoints=tmp_path / 'checkpoints') + assert result['new-task']['status'] == 'failure' + assert result['new-task']['attempt_errors'][0]['type'] == 'OSError' + assert result['next-task']['status'] == 'success' + assert len(list((tmp_path / 'checkpoints').glob('*.json'))) == 2 From 1ecd137d6f67a9f63d360b0ddf89178e98f95d16 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:46:54 +0800 Subject: [PATCH 05/19] fix(webfetch): verify process identity before timeout escalation --- harness/web_fetch/README.md | 2 +- harness/web_fetch/browser_observations.py | 73 +++++++++++---------- tests/test_browser_observation_contracts.py | 43 ++++++++++++ 3 files changed, 82 insertions(+), 36 deletions(-) diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index f6e95f0c2..4f3a9e48d 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -80,7 +80,7 @@ report-group registry used by configuration, execution and presentation. `browser_observations.py` owns browser reference capture; the corresponding `scripts/capture_headless_browser_observations.py` command only parses the shared entry point. Install the `headless-eval` extra and a Playwright Chromium -browser, or pass `--browser-binary` with an installed Chromium executable. +browser, or pass `--browser-binary` with an installed Chromium executable. Batch capture requires POSIX process isolation. New tasks declare their completion and content scope in the task definition: ```json diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index 6c625cd74..c95ce1136 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -617,9 +617,21 @@ def capture_source_identity() -> dict: ) +def _capture_process_snapshot(): + try: + table = subprocess.check_output( + ["ps", "-A", "-o", "pid=,ppid=,lstart="], text=True, timeout=1) + return {int(pid): (int(parent), started) for pid, parent, started in + (line.split(maxsplit=2) for line in table.splitlines())} + except (OSError, ValueError, subprocess.SubprocessError): + return {} + + def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolved_ip=None, *, observation_scope="reference", dns_mode="system"): """Bound browser transport and cleanup as well as page operations.""" + if os.name != "posix": + raise ValueError("Browser reference batch capture requires POSIX process isolation") if browser_binary: browser_binary = str(Path(browser_binary).resolve()) payload = dict(case=case, timeout_ms=timeout_ms, attempts=attempts, @@ -656,41 +668,30 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve browser_pids = {int(pid) for pid in re.findall( r"(?m)^\S+\s+pw:browser pid=(\d+)\s*$", diagnostics)} owned = {process.pid} - if os.name == "posix": - try: - table = subprocess.check_output( - ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) - parents = {int(pid): int(parent) for pid, parent in - (line.split() for line in table.splitlines())} - except (OSError, subprocess.SubprocessError): - parents = {} - while True: - children = {pid for pid, parent in parents.items() if parent in owned} - if children <= owned: - break - owned.update(children) - signals = (signal.SIGTERM, signal.SIGKILL) if os.name == "posix" else (None, None) - for sig in signals: - try: - if os.name == "posix": - for browser_pid in browser_pids: - try: - os.kill(browser_pid, sig) - except ProcessLookupError: - pass - try: - os.killpg(process.pid, sig) - except ProcessLookupError: - pass - for pid in sorted(owned, reverse=True): - try: - os.kill(pid, sig) - except ProcessLookupError: - pass - else: - process.kill() - except ProcessLookupError: - pass + initial = _capture_process_snapshot() + while True: + children = {pid for pid, (parent, _) in initial.items() if parent in owned} + if children <= owned: + break + owned.update(children) + for sig in (signal.SIGTERM, signal.SIGKILL): + targets = (owned | browser_pids) - {process.pid} + if sig == signal.SIGKILL: + current = _capture_process_snapshot() + targets = {pid for pid in targets if pid in initial and pid in current + and initial[pid][1] == current[pid][1]} + for pid in targets: + try: + os.kill(pid, sig) + except ProcessLookupError: + pass + # A timed-out communicate has not reaped the worker, so this + # process-group identity remains owned until communication finishes. + if process.returncode is None: + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + pass try: _stdout, diagnostics = process.communicate(timeout=1) except subprocess.TimeoutExpired: @@ -723,6 +724,8 @@ def capture_batch( workers=1, ): """Checkpoint independent observations against capture code, policy and binary.""" + if os.name != "posix": + raise ValueError("Browser reference batch capture requires POSIX process isolation") if any(type(value) is not int or value < 1 for value in (timeout_ms, attempts, workers)): raise ValueError("Browser observation budgets and workers must be positive integers") dns_mode = validate_dns_mode(dns_mode) diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 11bb22752..d1c687b0e 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -425,3 +425,46 @@ def launch(*args, **kwargs): assert result['new-task']['attempt_errors'][0]['type'] == 'OSError' assert result['next-task']['status'] == 'success' assert len(list((tmp_path / 'checkpoints').glob('*.json'))) == 2 + + +def test_unsupported_batch_platform_is_rejected_before_launch(monkeypatch): + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='nt')) + monkeypatch.setattr(capture.subprocess, 'Popen', lambda *a, **kw: pytest.fail('unsupported launch')) + with pytest.raises(ValueError, match='POSIX'): + capture.capture_batch([case_with()], timeout_ms=1000, attempts=1) + with pytest.raises(ValueError, match='POSIX'): + capture._capture_in_process(case_with(), 1000, 1) + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX process escalation') +@pytest.mark.parametrize('reused', [False, True]) +def test_timeout_escalation_preserves_reused_pids_and_kills_surviving_owned_children(monkeypatch, reused): + import signal + + launch = capture.subprocess.Popen + started = [] + signals = [] + groups = [] + snapshots = [] + child = 99999999 + def worker(*args, **kwargs): + process = launch(*args, **kwargs) + started.append(process.pid) + return process + def process_table(*args, **kwargs): + snapshots.append(1) + stamp = 'new-start' if reused and len(snapshots) > 1 else 'old-start' + return f'{started[0]} 0 worker-start\n{child} {started[0]} {stamp}\n' + def terminate_group(pid, sig): + groups.append((pid, sig)) + os.killpg(pid, sig) + monkeypatch.setattr(capture.subprocess, 'Popen', worker) + monkeypatch.setattr(capture.subprocess, 'check_output', process_table) + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, + kill=lambda pid, sig: signals.append((pid, sig)), killpg=terminate_group)) + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', 'import time; time.sleep(30)') + result = capture._capture_in_process(case_with(), 20, 1) + assert result['status'] == 'failure' + assert (child, signal.SIGTERM) in signals + assert ((child, signal.SIGKILL) in signals) == (not reused) + assert groups == [(started[0], signal.SIGTERM)] From be558d441c2174a698a53140f0a5fb80c259743f Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:54:41 +0800 Subject: [PATCH 06/19] fix(webfetch): ignore stale browser launch identities during cleanup --- harness/web_fetch/browser_observations.py | 4 +++ tests/test_browser_observation_contracts.py | 29 +++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index c95ce1136..a9588b5f4 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -667,6 +667,8 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial browser_pids = {int(pid) for pid in re.findall( r"(?m)^\S+\s+pw:browser pid=(\d+)\s*$", diagnostics)} + browser_pids.difference_update(int(pid) for pid in re.findall( + r"(?m)^\S+\s+pw:browser \[pid=(\d+)\] pid={unrelated}\n' + if mode == 'exited': + log += f'2026-10-26T00:00:01Z pw:browser [pid={unrelated}] \n' + monkeypatch.setattr(capture.subprocess, 'Popen', worker) + monkeypatch.setattr(capture.subprocess, 'check_output', processes) + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, + kill=lambda pid, sig: signalled.append((pid, sig)), killpg=os.killpg)) + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', + f'import sys,time; sys.stderr.write({log!r}); sys.stderr.flush(); time.sleep(30)') + result = capture._capture_in_process(case_with(), 20, 1) + assert result['status'] == 'failure' + assert signalled == [] From 931acdf147b745bd3e1a5b3e0f5aeeeb7a53af10 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:01:32 +0800 Subject: [PATCH 07/19] fix(webfetch): enforce capture hard deadlines in one termination pass --- harness/web_fetch/browser_observations.py | 42 ++++++++++----------- tests/test_browser_observation_contracts.py | 18 ++++----- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index a9588b5f4..e764f8274 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -620,9 +620,9 @@ def capture_source_identity() -> dict: def _capture_process_snapshot(): try: table = subprocess.check_output( - ["ps", "-A", "-o", "pid=,ppid=,lstart="], text=True, timeout=1) - return {int(pid): (int(parent), started) for pid, parent, started in - (line.split(maxsplit=2) for line in table.splitlines())} + ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) + return {int(pid): int(parent) for pid, parent in + (line.split() for line in table.splitlines())} except (OSError, ValueError, subprocess.SubprocessError): return {} @@ -672,34 +672,32 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve owned = {process.pid} initial = _capture_process_snapshot() while True: - children = {pid for pid, (parent, _) in initial.items() if parent in owned} + children = {pid for pid, parent in initial.items() if parent in owned} if children <= owned: break owned.update(children) if initial: browser_pids.intersection_update(owned) - for sig in (signal.SIGTERM, signal.SIGKILL): - targets = (owned | browser_pids) - {process.pid} - if sig == signal.SIGKILL: - current = _capture_process_snapshot() - targets = {pid for pid in targets if pid in initial and pid in current - and initial[pid][1] == current[pid][1]} - for pid in targets: - try: - os.kill(pid, sig) - except ProcessLookupError: - pass - # A timed-out communicate has not reaped the worker, so this - # process-group identity remains owned until communication finishes. - if process.returncode is None: + # The acquisition has already failed its hard deadline. Kill once, + # before reaping any process, rather than signalling saved PIDs again. + for pid in browser_pids: + try: + os.killpg(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): try: - os.killpg(process.pid, sig) + os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass + for pid in owned - browser_pids - {process.pid}: try: - _stdout, diagnostics = process.communicate(timeout=1) - except subprocess.TimeoutExpired: - continue + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + _stdout, diagnostics = process.communicate(timeout=1) message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" error_type = "TimeoutError" except (RuntimeError, ValueError) as error: diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 5340397ce..8ab66bd9c 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -318,7 +318,8 @@ def test_batch_preserves_browser_path_relative_to_external_caller(tmp_path, monk @pytest.mark.skipif(os.name != 'posix' or not os.getenv('BROWSER_EVAL_TEST_BROWSER_BINARY'), reason='POSIX and explicit Chromium required') @pytest.mark.parametrize('discovery_failure', [None, 'missing', 'exit', 'timeout']) -def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser(tmp_path, monkeypatch, discovery_failure): +@pytest.mark.parametrize('stalled_process', ['driver', 'browser']) +def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser(tmp_path, monkeypatch, discovery_failure, stalled_process): import subprocess from playwright.sync_api import sync_playwright @@ -342,7 +343,7 @@ def test_hung_driver_cleanup_removes_owned_chrome_and_preserves_another_browser( children.update(descendants) children = sorted(children) Path({str(pid_file)!r}).write_text(json.dumps(children)) - os.kill(driver, signal.SIGSTOP) + os.kill(driver if {stalled_process!r} == 'driver' else children[0], signal.SIGSTOP) time.sleep(30) ''' monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) @@ -438,7 +439,7 @@ def test_unsupported_batch_platform_is_rejected_before_launch(monkeypatch): @pytest.mark.skipif(os.name != 'posix', reason='POSIX process escalation') @pytest.mark.parametrize('reused', [False, True]) -def test_timeout_escalation_preserves_reused_pids_and_kills_surviving_owned_children(monkeypatch, reused): +def test_hard_timeout_signals_only_currently_owned_children_once(monkeypatch, reused): import signal launch = capture.subprocess.Popen @@ -453,8 +454,8 @@ def worker(*args, **kwargs): return process def process_table(*args, **kwargs): snapshots.append(1) - stamp = 'new-start' if reused and len(snapshots) > 1 else 'old-start' - return f'{started[0]} 0 worker-start\n{child} {started[0]} {stamp}\n' + parent = 1 if reused else started[0] + return f'{started[0]} 0\n{child} {parent}\n' def terminate_group(pid, sig): groups.append((pid, sig)) os.killpg(pid, sig) @@ -465,9 +466,8 @@ def terminate_group(pid, sig): monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', 'import time; time.sleep(30)') result = capture._capture_in_process(case_with(), 20, 1) assert result['status'] == 'failure' - assert (child, signal.SIGTERM) in signals - assert ((child, signal.SIGKILL) in signals) == (not reused) - assert groups == [(started[0], signal.SIGTERM)] + assert signals == ([] if reused else [(child, signal.SIGKILL)]) + assert groups == [(started[0], signal.SIGKILL)] @pytest.mark.skipif(os.name != 'posix', reason='POSIX browser ownership') @@ -484,7 +484,7 @@ def worker(*args, **kwargs): def processes(*args, **kwargs): if mode == 'exited': raise FileNotFoundError('no process listing') - return f'{worker_pid[0]} 0 worker-start\n{unrelated} 1 unrelated-start\n' + return f'{worker_pid[0]} 0\n{unrelated} 1\n' log = f'2026-10-26T00:00:00Z pw:browser pid={unrelated}\n' if mode == 'exited': log += f'2026-10-26T00:00:01Z pw:browser [pid={unrelated}] \n' From 026e0e01db507ff85869650889577875e8267291 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:22:40 +0800 Subject: [PATCH 08/19] fix(webfetch): retain browser ownership after driver exit --- harness/web_fetch/browser_observations.py | 105 ++++++++++++-------- tests/test_browser_observation_contracts.py | 48 ++++++++- 2 files changed, 111 insertions(+), 42 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index e764f8274..8f705741b 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -620,13 +620,62 @@ def capture_source_identity() -> dict: def _capture_process_snapshot(): try: table = subprocess.check_output( - ["ps", "-A", "-o", "pid=,ppid="], text=True, timeout=1) - return {int(pid): int(parent) for pid, parent in - (line.split() for line in table.splitlines())} + ["ps", "-ww", "-A", "-o", "pid=,ppid=,args="], text=True, timeout=1) + return {int(pid): (int(parent), command) for pid, parent, command in + (line.split(maxsplit=2) for line in table.splitlines())} except (OSError, ValueError, subprocess.SubprocessError): return {} +def _terminate_capture_processes(process, diagnostics): + """Stop one worker and browsers authenticated by its launch receipts.""" + browser_pids = {int(pid) for pid in re.findall( + r"(?m)^\S+\s+pw:browser pid=(\d+)\s*$", diagnostics)} + browser_pids.difference_update(int(pid) for pid in re.findall( + r"(?m)^\S+\s+pw:browser \[pid=(\d+)\] ", line): + profile = re.search(r"--user-data-dir=(.+?) --remote-debugging-pipe(?: |$)", line) + launching = profile[1] if profile else None + launched = re.match(r"^\S+\s+pw:browser pid=(\d+)\s*$", line) + if launched and launching: + profiles[int(launched[1])] = launching + if process.returncode is not None and not browser_pids: + return + owned = {process.pid} if process.returncode is None else set() + initial = _capture_process_snapshot() + while True: + children = {pid for pid, (parent, _) in initial.items() if parent in owned} + if children <= owned: + break + owned.update(children) + if initial: + browser_pids = {pid for pid in browser_pids if pid in owned or ( + pid in initial and pid in profiles + and ('--user-data-dir=' + profiles[pid] + ' ') in (initial[pid][1] + ' '))} + # Signal each owned target once, before further waiting or reaping. + for pid in browser_pids: + try: + os.killpg(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + for pid in owned - browser_pids - {process.pid}: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if process.returncode is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolved_ip=None, *, observation_scope="reference", dns_mode="system"): """Bound browser transport and cleanup as well as page operations.""" @@ -645,14 +694,18 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve [sys.executable, "-c", _CAPTURE_PROGRAM], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=Path(__file__).resolve().parents[2], start_new_session=os.name == "posix", - env={**os.environ, "DEBUG": "pw:api,pw:browser"}, + env={**os.environ, "DEBUG": "pw:api,pw:browser", "DEBUG_COLORS": "0"}, ) except OSError as error: message, error_type = str(error), type(error).__name__ else: - with process: + with ExitStack() as cleanup: + for pipe in (process.stdin, process.stdout, process.stderr): + if pipe is not None: + cleanup.callback(pipe.close) try: stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) + _terminate_capture_processes(process, stderr) if process.returncode: raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") result = json.loads(stdout) @@ -660,44 +713,18 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve raise ValueError("Capture worker returned another case") return result except subprocess.TimeoutExpired as error: - # Chrome starts a detached group. Collect descendants before killing - # the driver, while their parent relationships still identify ownership. - # The launcher receipt survives an unavailable process-list command. partial = error.stderr or b"" diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial - browser_pids = {int(pid) for pid in re.findall( - r"(?m)^\S+\s+pw:browser pid=(\d+)\s*$", diagnostics)} - browser_pids.difference_update(int(pid) for pid in re.findall( - r"(?m)^\S+\s+pw:browser \[pid=(\d+)\] pid={unrelated}\n' if mode == 'exited': log += f'2026-10-26T00:00:01Z pw:browser [pid={unrelated}] \n' @@ -497,3 +500,42 @@ def processes(*args, **kwargs): result = capture._capture_in_process(case_with(), 20, 1) assert result['status'] == 'failure' assert signalled == [] + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX inherited pipe writer') +def test_surviving_inherited_pipe_writer_does_not_abort_batch_or_checkpointing(tmp_path, monkeypatch): + import signal + import time + + pid_file = tmp_path / 'writer.pid' + child = ("import os,sys,time; from pathlib import Path; pid=os.fork();\n" + "if pid: sys.exit(0)\n" + f"Path({str(pid_file)!r}).write_text(str(os.getpid())); " + "sys.stderr.write('controlled inherited pipe writer\\n'); sys.stderr.flush(); time.sleep(30)") + program = f'''import json,subprocess,sys,time +params=json.load(sys.stdin) +if params['case']['id']=='new-task': + subprocess.run([sys.executable,'-c',{child!r}],check=True) + time.sleep(30) +else: + print(json.dumps(dict(case_id=params['case']['id'],status='success',completion=dict(satisfied=True)))) +''' + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) + binary = tmp_path / 'browser' + binary.write_bytes(b'fixture browser identity') + cases = [case_with(), {**case_with(), 'id': 'next-task'}] + began = time.monotonic() + try: + result = capture.capture_batch(cases, timeout_ms=20, attempts=1, + browser_binary=str(binary), checkpoints=tmp_path / 'checkpoints') + assert time.monotonic() - began < 6 + assert result['new-task']['status'] == 'failure' + assert 'controlled inherited pipe writer' in result['new-task']['worker_diagnostics'] + assert result['next-task']['status'] == 'success' + assert len(list((tmp_path / 'checkpoints').glob('*.json'))) == 2 + finally: + if pid_file.exists(): + try: + os.kill(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass From 412a358a401480a716a023e54aaf4057edad3b89 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:51:48 +0800 Subject: [PATCH 09/19] fix(webfetch): clean up browsers launched during timeout handling --- harness/web_fetch/browser_observations.py | 18 ++++--- tests/test_browser_observation_contracts.py | 57 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index 8f705741b..fc8cb0c05 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -627,8 +627,9 @@ def _capture_process_snapshot(): return {} -def _terminate_capture_processes(process, diagnostics): +def _terminate_capture_processes(process, diagnostics, signalled=None): """Stop one worker and browsers authenticated by its launch receipts.""" + signalled = set() if signalled is None else signalled browser_pids = {int(pid) for pid in re.findall( r"(?m)^\S+\s+pw:browser pid=(\d+)\s*$", diagnostics)} browser_pids.difference_update(int(pid) for pid in re.findall( @@ -642,9 +643,10 @@ def _terminate_capture_processes(process, diagnostics): launched = re.match(r"^\S+\s+pw:browser pid=(\d+)\s*$", line) if launched and launching: profiles[int(launched[1])] = launching - if process.returncode is not None and not browser_pids: - return - owned = {process.pid} if process.returncode is None else set() + browser_pids.difference_update(signalled) + if (process.returncode is not None or process.pid in signalled) and not browser_pids: + return signalled + owned = {process.pid} if process.returncode is None and process.pid not in signalled else set() initial = _capture_process_snapshot() while True: children = {pid for pid, (parent, _) in initial.items() if parent in owned} @@ -655,6 +657,8 @@ def _terminate_capture_processes(process, diagnostics): browser_pids = {pid for pid in browser_pids if pid in owned or ( pid in initial and pid in profiles and ('--user-data-dir=' + profiles[pid] + ' ') in (initial[pid][1] + ' '))} + owned.difference_update(signalled) + signalled.update(owned | browser_pids) # Signal each owned target once, before further waiting or reaping. for pid in browser_pids: try: @@ -669,11 +673,12 @@ def _terminate_capture_processes(process, diagnostics): os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass - if process.returncode is None: + if process.pid in owned: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass + return signalled def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolved_ip=None, @@ -715,7 +720,7 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve except subprocess.TimeoutExpired as error: partial = error.stderr or b"" diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial - _terminate_capture_processes(process, diagnostics) + signalled = _terminate_capture_processes(process, diagnostics) try: _stdout, diagnostics = process.communicate(timeout=1) except subprocess.TimeoutExpired as drain_error: @@ -725,6 +730,7 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve process.wait(timeout=1) except subprocess.TimeoutExpired: pass + _terminate_capture_processes(process, diagnostics, signalled) message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" error_type = "TimeoutError" except (RuntimeError, ValueError) as error: diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 63b2f36e3..d9cc96040 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -539,3 +539,60 @@ def test_surviving_inherited_pipe_writer_does_not_abort_batch_or_checkpointing(t os.kill(int(pid_file.read_text()), signal.SIGKILL) except ProcessLookupError: pass + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX late browser launch') +@pytest.mark.parametrize('inherit_pipes', [False, True]) +def test_browser_launched_during_failed_discovery_is_stopped(tmp_path, monkeypatch, inherit_pipes): + import signal + import time + + process_status = capture.subprocess.run + trigger = tmp_path / 'launch' + pid_file = tmp_path / 'browser.pid' + program = f'''import json,subprocess,sys,time +from pathlib import Path +json.load(sys.stdin) +while not Path({str(trigger)!r}).exists(): time.sleep(.01) +child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)'], + start_new_session=True, stdout={"None" if inherit_pipes else "subprocess.DEVNULL"}, + stderr={"None" if inherit_pipes else "subprocess.DEVNULL"}) +Path({str(pid_file)!r}).write_text(str(child.pid)) +print(f'2026-10-26T00:00:00Z pw:browser pid={{child.pid}}',file=sys.stderr,flush=True) +time.sleep(30) +''' + def unavailable(*args, **kwargs): + trigger.touch() + deadline = time.monotonic() + 1 + while not pid_file.exists() and time.monotonic() < deadline: + time.sleep(.01) + time.sleep(.05) + raise FileNotFoundError('process discovery unavailable') + signals = [] + def kill_group(pid, sig): + signals.append(pid) + os.killpg(pid, sig) + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) + monkeypatch.setattr(capture.subprocess, 'check_output', unavailable) + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, + kill=os.kill, killpg=kill_group)) + try: + result = capture._capture_in_process(case_with(), 20, 1) + assert result['status'] == 'failure' + browser_pid = int(pid_file.read_text()) + assert browser_pid in signals + assert len(signals) == len(set(signals)) + deadline = time.monotonic() + 2 + while True: + state = process_status(['ps', '-o', 'stat=', '-p', str(browser_pid)], + capture_output=True, text=True, timeout=1).stdout.strip() + if not state or state.startswith('Z'): + break + assert time.monotonic() < deadline, 'late browser survived capture timeout' + time.sleep(.02) + finally: + if pid_file.exists(): + try: + os.kill(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass From 211486ca5a7107a43f1e2fd06ad8ea97147c3a10 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:58:06 +0800 Subject: [PATCH 10/19] fix(webfetch): checkpoint worker communication failures --- harness/web_fetch/browser_observations.py | 67 ++++++++++++--------- tests/test_browser_observation_contracts.py | 40 ++++++++++++ 2 files changed, 80 insertions(+), 27 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index fc8cb0c05..18612bf12 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -704,37 +704,50 @@ def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolve except OSError as error: message, error_type = str(error), type(error).__name__ else: - with ExitStack() as cleanup: - for pipe in (process.stdin, process.stdout, process.stderr): - if pipe is not None: - cleanup.callback(pipe.close) - try: - stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) - _terminate_capture_processes(process, stderr) - if process.returncode: - raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") - result = json.loads(stdout) - if not isinstance(result, dict) or result.get('case_id') != case['id']: - raise ValueError("Capture worker returned another case") - return result - except subprocess.TimeoutExpired as error: - partial = error.stderr or b"" - diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial - signalled = _terminate_capture_processes(process, diagnostics) + signalled = set() + try: + with ExitStack() as cleanup: + for pipe in (process.stdin, process.stdout, process.stderr): + if pipe is not None: + cleanup.callback(pipe.close) try: - _stdout, diagnostics = process.communicate(timeout=1) - except subprocess.TimeoutExpired as drain_error: - partial = drain_error.stderr or diagnostics + stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) + _terminate_capture_processes(process, stderr, signalled) + if process.returncode: + raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") + result = json.loads(stdout) + if not isinstance(result, dict) or result.get('case_id') != case['id']: + raise ValueError("Capture worker returned another case") + return result + except subprocess.TimeoutExpired as error: + partial = error.stderr or b"" diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial + _terminate_capture_processes(process, diagnostics, signalled) try: - process.wait(timeout=1) - except subprocess.TimeoutExpired: - pass + _stdout, diagnostics = process.communicate(timeout=1) + except subprocess.TimeoutExpired as drain_error: + partial = drain_error.stderr or diagnostics + diagnostics = partial.decode(errors="replace") if isinstance(partial, bytes) else partial + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + pass + _terminate_capture_processes(process, diagnostics, signalled) + message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" + error_type = "TimeoutError" + except (RuntimeError, ValueError) as error: + message, error_type = str(error), type(error).__name__ + except OSError as error: + message, error_type = str(error), type(error).__name__ + try: _terminate_capture_processes(process, diagnostics, signalled) - message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" - error_type = "TimeoutError" - except (RuntimeError, ValueError) as error: - message, error_type = str(error), type(error).__name__ + except OSError: + pass + try: + process.kill() + process.wait(timeout=1) + except (OSError, subprocess.TimeoutExpired): + pass row = _failed_observation( case, {"browser": "Chromium", "binary": "explicit" if browser_binary else "Playwright default", "dns_mode": dns_mode}, diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index d9cc96040..f61f695be 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -596,3 +596,43 @@ def kill_group(pid, sig): os.kill(int(pid_file.read_text()), signal.SIGKILL) except ProcessLookupError: pass + + +@pytest.mark.parametrize('failure_call', [1, 2]) +def test_worker_pipe_error_is_checkpointed_and_next_case_runs(tmp_path, monkeypatch, failure_call): + launch = capture.subprocess.Popen + workers = [] + def start(*args, **kwargs): + process = launch(*args, **kwargs) + if not workers: + communicate = process.communicate + calls = [] + def fail_pipe(*args, **kwargs): + calls.append(1) + if len(calls) == failure_call: + raise OSError('controlled worker pipe failure') + return communicate(*args, **kwargs) + process.communicate = fail_pipe + workers.append(process) + return process + monkeypatch.setattr(capture.subprocess, 'Popen', start) + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', + 'import json,sys,time; p=json.load(sys.stdin); ' + 'time.sleep(30 if p["case"]["id"]=="new-task" else 0); ' + 'print(json.dumps(dict(case_id=p["case"]["id"],status="success",completion=dict(satisfied=True))))') + binary = tmp_path / 'browser' + binary.write_bytes(b'fixture binary') + try: + result = capture.capture_batch([case_with(), {**case_with(), 'id': 'next-task'}], + timeout_ms=20, attempts=1, browser_binary=str(binary), checkpoints=tmp_path / 'checkpoints') + assert result['new-task']['status'] == 'failure' + assert result['new-task']['attempt_errors'][0]['type'] == 'OSError' + assert 'controlled worker pipe failure' in result['new-task']['attempt_errors'][0]['message'] + assert result['next-task']['status'] == 'success' + assert len(list((tmp_path / 'checkpoints').glob('*.json'))) == 2 + assert workers[0].poll() is not None + finally: + for process in workers: + if process.poll() is None: + process.kill() + process.wait(timeout=2) From 33c8732eccdfff0d03d8eaf66a064aa798f95fed Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:05:05 +0800 Subject: [PATCH 11/19] fix(webfetch): verify browser identity when process discovery fails --- harness/web_fetch/browser_observations.py | 15 +++++++++ tests/test_browser_observation_contracts.py | 35 ++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index 18612bf12..74a2efbcb 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -657,6 +657,21 @@ def _terminate_capture_processes(process, diagnostics, signalled=None): browser_pids = {pid for pid in browser_pids if pid in owned or ( pid in initial and pid in profiles and ('--user-data-dir=' + profiles[pid] + ' ') in (initial[pid][1] + ' '))} + else: + verified = set() + for pid in browser_pids: + if pid not in profiles: + continue + try: + matches = subprocess.run( + ["pgrep", "-f", "--", re.escape("--user-data-dir=" + profiles[pid]) + "( |$)"], + capture_output=True, text=True, timeout=1, check=False, + ) + if matches.returncode == 0 and str(pid) in matches.stdout.split(): + verified.add(pid) + except (OSError, subprocess.SubprocessError): + pass + browser_pids = verified owned.difference_update(signalled) signalled.update(owned | browser_pids) # Signal each owned target once, before further waiting or reaping. diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index f61f695be..705b30e21 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -554,10 +554,13 @@ def test_browser_launched_during_failed_discovery_is_stopped(tmp_path, monkeypat from pathlib import Path json.load(sys.stdin) while not Path({str(trigger)!r}).exists(): time.sleep(.01) -child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)'], +child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)', + '--user-data-dir='+{str(tmp_path / 'profile')!r},'--remote-debugging-pipe'], start_new_session=True, stdout={"None" if inherit_pipes else "subprocess.DEVNULL"}, stderr={"None" if inherit_pipes else "subprocess.DEVNULL"}) Path({str(pid_file)!r}).write_text(str(child.pid)) +print('2026-10-26T00:00:00Z pw:browser browser --user-data-dir=' + +{str(tmp_path / 'profile')!r}+' --remote-debugging-pipe',file=sys.stderr,flush=True) print(f'2026-10-26T00:00:00Z pw:browser pid={{child.pid}}',file=sys.stderr,flush=True) time.sleep(30) ''' @@ -636,3 +639,33 @@ def fail_pipe(*args, **kwargs): if process.poll() is None: process.kill() process.wait(timeout=2) + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX browser ownership') +@pytest.mark.parametrize('fallback_available', [False, True]) +def test_unrelated_live_pid_in_stale_receipt_survives_failed_discovery(monkeypatch, fallback_available): + import subprocess + import sys + + run = subprocess.run + with subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], + start_new_session=True) as unrelated: + log = ('2026-10-26T00:00:00Z pw:browser chrome ' + '--user-data-dir=/tmp/browser-eval-expired-profile --remote-debugging-pipe\n' + f'2026-10-26T00:00:00Z pw:browser pid={unrelated.pid}\n') + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', + f'import sys,time; sys.stderr.write({log!r}); sys.stderr.flush(); time.sleep(30)') + def unavailable(*args, **kwargs): + raise FileNotFoundError('process listing unavailable') + def fallback(args, **kwargs): + if args[0] == 'pgrep' and not fallback_available: + raise FileNotFoundError('fallback unavailable') + return run(args, **kwargs) + monkeypatch.setattr(capture.subprocess, 'check_output', unavailable) + monkeypatch.setattr(capture.subprocess, 'run', fallback) + try: + result = capture._capture_in_process(case_with(), 20, 1) + assert result['status'] == 'failure' + assert unrelated.poll() is None, 'unrelated process killed by stale launch receipt' + finally: + unrelated.kill() From d37ade5fd6ffa23e7acf503ccab73b66669b7aab Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:11:23 +0800 Subject: [PATCH 12/19] fix(webfetch): terminate owned browser process groups --- harness/web_fetch/browser_observations.py | 5 +- tests/test_browser_observation_contracts.py | 61 ++++++++++++++++++++- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index 74a2efbcb..bb25bff50 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -685,7 +685,10 @@ def _terminate_capture_processes(process, diagnostics, signalled=None): pass for pid in owned - browser_pids - {process.pid}: try: - os.kill(pid, signal.SIGKILL) + if os.getpgid(pid) == pid: + os.killpg(pid, signal.SIGKILL) + else: + os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass if process.pid in owned: diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 705b30e21..a55b9990e 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -464,7 +464,7 @@ def terminate_group(pid, sig): os.killpg(pid, sig) monkeypatch.setattr(capture.subprocess, 'Popen', worker) monkeypatch.setattr(capture.subprocess, 'check_output', process_table) - monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, getpgid=lambda pid: started[0], kill=lambda pid, sig: signals.append((pid, sig)), killpg=terminate_group)) monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', 'import time; time.sleep(30)') result = capture._capture_in_process(case_with(), 20, 1) @@ -493,7 +493,7 @@ def processes(*args, **kwargs): log += f'2026-10-26T00:00:01Z pw:browser [pid={unrelated}] \n' monkeypatch.setattr(capture.subprocess, 'Popen', worker) monkeypatch.setattr(capture.subprocess, 'check_output', processes) - monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, getpgid=os.getpgid, kill=lambda pid, sig: signalled.append((pid, sig)), killpg=os.killpg)) monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', f'import sys,time; sys.stderr.write({log!r}); sys.stderr.flush(); time.sleep(30)') @@ -577,7 +577,7 @@ def kill_group(pid, sig): os.killpg(pid, sig) monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) monkeypatch.setattr(capture.subprocess, 'check_output', unavailable) - monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, + monkeypatch.setattr(capture, 'os', SimpleNamespace(name='posix', environ=os.environ, getpgid=os.getpgid, kill=os.kill, killpg=kill_group)) try: result = capture._capture_in_process(case_with(), 20, 1) @@ -669,3 +669,58 @@ def fallback(args, **kwargs): assert unrelated.poll() is None, 'unrelated process killed by stale launch receipt' finally: unrelated.kill() + + +@pytest.mark.skipif(os.name != 'posix', reason='POSIX late browser descendants') +def test_delayed_launch_receipt_cleans_children_created_after_snapshot(tmp_path, monkeypatch): + import signal + import subprocess + import time + + trigger = tmp_path / 'spawn-child' + browser_pid = tmp_path / 'browser.pid' + child_pid = tmp_path / 'child.pid' + browser_program = f'''import os,subprocess,sys,time +from pathlib import Path +Path({str(browser_pid)!r}).write_text(str(os.getpid())) +while not Path({str(trigger)!r}).exists(): time.sleep(.01) +child=subprocess.Popen([sys.executable,'-c','import os,signal,time; os.kill(os.getpid(),signal.SIGSTOP); time.sleep(30)']) +Path({str(child_pid)!r}).write_text(str(child.pid)) +print(f'2026-10-26T00:00:00Z pw:browser pid={{os.getpid()}}',file=sys.stderr,flush=True) +time.sleep(30) +''' + program = f'''import subprocess,sys,time +subprocess.Popen([sys.executable,'-c',{browser_program!r}], start_new_session=True) +time.sleep(30) +''' + snapshot = subprocess.check_output + def delayed_snapshot(*args, **kwargs): + table = snapshot(*args, **kwargs) + if browser_pid.exists() and not trigger.exists(): + trigger.touch() + deadline = time.monotonic() + 1 + while not child_pid.exists() and time.monotonic() < deadline: + time.sleep(.01) + time.sleep(.05) + return table + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', program) + monkeypatch.setattr(capture.subprocess, 'check_output', delayed_snapshot) + try: + result = capture._capture_in_process(case_with(), 20, 1) + assert result['status'] == 'failure' + pid = int(child_pid.read_text()) + deadline = time.monotonic() + 2 + while True: + state = subprocess.run(['ps', '-o', 'stat=', '-p', str(pid)], + capture_output=True, text=True, timeout=1).stdout.strip() + if not state or state.startswith('Z'): + break + assert time.monotonic() < deadline, 'late browser child survived cleanup' + time.sleep(.02) + finally: + for path in (child_pid, browser_pid): + if path.exists(): + try: + os.kill(int(path.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass From e9f89e284242a6172ed0d3cd0011478d1f34cf7b Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:31:58 +0800 Subject: [PATCH 13/19] fix(webfetch): preserve legacy evidence and complete mixed captures --- harness/web_fetch/artifacts.py | 2 + harness/web_fetch/browser_observations.py | 20 +++++- harness/web_fetch/reference_freeze.py | 2 +- harness/web_fetch/reference_lifecycle.py | 72 ++++++++++++++++++--- harness/web_fetch/runner.py | 10 ++- tests/fixtures/legacy-live-reference.json | 50 ++++++++++++++ tests/test_browser_observation_contracts.py | 35 ++++++++++ tests/test_browser_reference_scope.py | 51 +++++++++++++++ tests/test_reference_lifecycle.py | 37 +++++++++++ 9 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/legacy-live-reference.json diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py index 700ff45d2..fe4fc2316 100644 --- a/harness/web_fetch/artifacts.py +++ b/harness/web_fetch/artifacts.py @@ -406,9 +406,11 @@ def materialize_bound_artifacts(output: Path, source_root: Path, records: list[d def materialize_reference_checks(output: Path, source: Path, batches: list[dict]) -> None: """Retain hash-bound source observations and native execution contracts.""" + from .reference_lifecycle import preserve_legacy_reference_dataset from .native_routes import materialize_native_route_contracts materialize_native_route_contracts(output, source, batches) for batch in batches: + preserve_legacy_reference_dataset(batch, source, output) post = batch.get('reference_checks', {}).get('postcheck', {}) if not post: continue diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index bb25bff50..a5e75556a 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -33,6 +33,7 @@ _resource_target_matches, resource_source_scope_identity, scope_identity, + requires_fresh_reference, ) from lexbench_webfetch.paths import safe_identifier from harness.web_fetch.source_capture import extract_html @@ -309,6 +310,8 @@ def capture( if resolved_ip is not None: resolved_ip = public_ip(resolved_ip) frames = [] + captured = None + capture_text = case.get("task_kind") != "resource" or requires_fresh_reference(case) dns_metadata = {"dns_mode": dns_mode} browser_version = None primary_response_server_addr = None @@ -353,7 +356,7 @@ def capture( if case.get("task_kind") == "resource": captured = capture_resource_observation(page, case, timeout_ms, begin) final_url, title = page.url, page.title() - else: + if capture_text: page.wait_for_function( "([selector, minimum]) => " "document.querySelectorAll(selector).length >= minimum", @@ -440,7 +443,7 @@ def capture( "primary_response_server_addr": primary_response_server_addr, **dns_metadata, } - if case.get("task_kind") == "resource": + if not capture_text: return { "case_id": case["id"], "status": "success", @@ -471,6 +474,9 @@ def capture( "content_text": content, "selector": policy.description, "capture_policy": asdict(policy), + **({key: value for key, value in captured.items() + if key not in {"content_text", "selector", "observation_method", "observed_records", "completion"}} + if captured is not None else {}), "observation_method": observation_method, **( {"outer_html": outer_html, "settle_budget_ms": 2000, "visible_frames": frames} @@ -482,6 +488,7 @@ def capture( "observer": observer, "completion": { "satisfied": bool(normalized) + and (captured is None or captured["completion"]["satisfied"]) and elapsed <= timeout_ms and all(frame["status"] == "success" for frame in frames) and ( @@ -809,6 +816,15 @@ def capture_batch( raise ValueError("Checkpoint reuse requires an explicit browser binary") if browser_binary: browser_binary = str(Path(browser_binary).resolve()) + if not _capture_process_snapshot(): + try: + discovery = subprocess.run(["pgrep", "-f", "--", "--user-data-dir="], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=1) + usable = discovery.returncode in (0, 1) + except (OSError, subprocess.SubprocessError): + usable = False + if not usable: + raise ValueError("Browser capture requires working ps or pgrep process discovery") sources = capture_source_identity() binary_sha = ( hashlib.sha256(Path(browser_binary).read_bytes()).hexdigest() if browser_binary else None diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index 26152ba29..e20ce9c99 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -142,7 +142,7 @@ def load_browser_observations(dataset, path=None, directory=None): for cid,row in observations.items(): if not isinstance(row,dict) or row.get('case_id')!=cid: raise ValueError('Browser observation case ID mismatch: '+cid) - if row.get('scope_identity')!=scope_identity(cases[cid]): + if row.get('scope_identity')!=scope_identity(cases[cid], version=row.get('scope_identity', {}).get('schema_version', 1)): raise ValueError('Browser observation scope identity mismatch: '+cid) binary=(row.get('observer') or {}).get('binary') if (not isinstance(binary,str) or Path(binary).is_absolute() diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 54459c236..c5c86d65e 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -204,11 +204,12 @@ def completed_case_records(dataset, case, records): return relevant if set(methods) == expected else None -def scope_identity(case) -> dict[str, Any]: +def scope_identity(case, *, version=2) -> dict[str, Any]: verification = _get(case, "source_verification", {}) or {} benchmark = _get(case, "source_benchmark", {}) or {} render = benchmark.get("render_evidence", {}) return deepcopy({ + **({"schema_version": 2} if version == 2 else {}), "case_id": _get(case, "id"), "url": _get(case, "url"), "track": _get(case, "track", "web"), "task_kind": _get(case, "task_kind", "body"), "task_prompt": _get(case, "task_prompt", ""), @@ -224,7 +225,7 @@ def scope_identity(case) -> dict[str, Any]: ("acceptance_contract", {}), ("body_requirements", {}), ("composite_acceptance", {})) if key not in {"acceptance_contract", "body_requirements", "composite_acceptance"} - or _get(case, key, default)}, + or (version == 2 and _get(case, key, default))}, }) @@ -445,6 +446,7 @@ def evaluation_scope_identity(case) -> dict[str, Any]: exact selector in each batch's strict binding and postcheck. """ identity = scope_identity(case) + identity.pop("schema_version", None) identity.pop("selector", None) _normalize_draft_mutable_body_metadata(identity) return identity @@ -452,6 +454,7 @@ def evaluation_scope_identity(case) -> dict[str, Any]: def _binding_evaluation_scope_identity(binding: dict) -> dict[str, Any]: identity = deepcopy(binding.get("scope_identity", {})) + identity.pop("schema_version", None) identity.pop("selector", None) _normalize_draft_mutable_body_metadata(identity) return identity @@ -683,20 +686,67 @@ def check_case_reference_fresh(case, dataset_path: Path, binding: dict | None, n raise ReferenceLifecycleError("Frozen reference binding changed during the run", "reference_changed") -def check_case_reference_binding(case, binding: dict) -> None: +def legacy_reference_dataset(manifest, source=None, dataset_path=None): + """Recover omitted legacy rule fields from the original hash-bound input.""" + bindings = manifest.get('reference_checks', {}).get('cases', {}) + if not any(row.get('scope_identity', {}).get('schema_version', 1) == 1 + for row in bindings.values()): + return None + digests = {digest for digest in ( + manifest.get('dataset_sha256'), + manifest.get('reference_checks', {}).get('postcheck', {}).get('dataset_sha256')) + if isinstance(digest, str) and re.fullmatch(r'[0-9a-f]{64}', digest)} + candidates = [] + if source is not None: + candidates.extend(Path(source) / f'reference-checks/dataset-{digest}.json' for digest in digests) + candidates.append(Path(source) / 'dataset.json') + if dataset_path is not None: + candidates.append(Path(dataset_path)) + for path in candidates: + if path.is_file() and not path.is_symlink(): + raw = path.read_bytes() + if sha256(raw) in digests: + return raw + return None + + +def legacy_reference_cases(raw): + return {row['id']: row for row in json.loads(raw)['cases']} if raw else {} + + +def preserve_legacy_reference_dataset(manifest, source, output): + raw = legacy_reference_dataset(manifest, source) + if raw is not None: + target = contained_path(output, f'reference-checks/dataset-{sha256(raw)}.json', reject_symlinks=True) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(raw) + + +def check_case_reference_binding(case, binding: dict, *, original_case=None) -> None: """Offline rescores must use the original capture; no freshness claim is made.""" + version = binding.get('scope_identity', {}).get('schema_version', 1) + if version not in (1, 2) or (version == 1 and any( + _get(case, key, {}) != _get(original_case or {}, key, {}) + for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance'))): + raise ReferenceLifecycleError('Legacy rules differ from the hash-bound original dataset', 'reference_changed') reference = _get(case, "ground_truth_main_content", "") if (sha256(reference) != binding.get("reference_sha256") or sha256(normalize_for_match(reference)) != binding.get("expected_normalized_sha256") - or scope_identity(case) != binding.get("scope_identity") + or scope_identity(case, version=version) != binding.get("scope_identity") or _get(case, "source_verification", {}) != binding.get("source_verification") or ("render_evidence" in binding and (_get(case, "source_benchmark", {}) or {}).get("render_evidence") != binding["render_evidence"])): raise ReferenceLifecycleError("Rescore requires the original frozen reference and capture binding", "reference_changed") -def check_case_equivalent_reference_binding(case, binding: dict) -> None: +def check_case_equivalent_reference_binding(case, binding: dict, *, original_case=None) -> None: """Allow distinct independent captures only when reference bytes and scope are identical.""" + if binding.get('scope_identity', {}).get('schema_version', 1) == 1 and original_case is not None: + binding = deepcopy(binding) + binding['scope_identity']['evaluation_contract'].update({ + key: _get(original_case, key, {}) + for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance') + if _get(original_case, key, {})}) reference = _get(case, "ground_truth_main_content", "") verification = _get(case, "source_verification", {}) if (sha256(reference) != binding.get("reference_sha256") @@ -802,8 +852,8 @@ def _selected_records(batch, records, original_elapsed=None): return selected -def _postcheck_case(binding, case, observation, records, now): - check_case_reference_binding(case, binding) +def _postcheck_case(binding, case, observation, records, now, original_case=None): + check_case_reference_binding(case, binding, original_case=original_case) if observation.get("independent") is not True: raise ReferenceLifecycleError("Observation must attest independent capture, not provider output") if observation.get("scope_identity") != binding["scope_identity"]: @@ -896,6 +946,7 @@ def finalize_reference_checks(run_dir: Path, observations_path: Path, now=None) outcomes = [] for batch in guarded: checks = batch["reference_checks"] + original_cases = legacy_reference_cases(legacy_reference_dataset(batch, run_dir)) results = {} try: if input_error: @@ -906,7 +957,7 @@ def finalize_reference_checks(run_dir: Path, observations_path: Path, now=None) if identifier not in cases: raise ReferenceLifecycleError("Reference case absent from frozen dataset") results[identifier] = _postcheck_case(binding, cases[identifier], observations[identifier], - [r for r in selected if r["case_id"] == identifier], checked) + [r for r in selected if r["case_id"] == identifier], checked, original_cases.get(identifier)) except (ReferenceLifecycleError, KeyError, TypeError, ValueError, OverflowError) as error: results[identifier] = {"status": getattr(error, "status", "reference_unverified"), "error": str(error)} except (ReferenceLifecycleError, KeyError, TypeError, ValueError, OverflowError) as error: @@ -1583,6 +1634,7 @@ def valid_hash(value): checks = batch["reference_checks"] if not isinstance(checks, dict) or checks.get("status") != "verified": raise ReferenceLifecycleError("Live reference checks are not verified") + original_cases = legacy_reference_cases(legacy_reference_dataset(batch, source)) bindings, post = checks.get("cases"), checks.get("postcheck") if (checks.get("schema_version") != 1 or checks.get("max_age_seconds") != MAX_REFERENCE_AGE_SECONDS @@ -1660,9 +1712,9 @@ def valid_hash(value): elif allow_scoring_rule_revision: check_case_scoring_revision_binding(case_map[identifier], binding) elif equivalent_capture_bindings: - check_case_equivalent_reference_binding(case_map[identifier], binding) + check_case_equivalent_reference_binding(case_map[identifier], binding, original_case=original_cases.get(identifier)) else: - check_case_reference_binding(case_map[identifier], binding) + check_case_reference_binding(case_map[identifier], binding, original_case=original_cases.get(identifier)) relevant = [r for r in selected if r["case_id"] == identifier] if selected is not None else None if relevant is not None: if not relevant: diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index 99a6e8490..db4623611 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -41,6 +41,8 @@ from .reference_lifecycle import ( ReferenceLifecycleError, check_case_reference_binding, + legacy_reference_dataset, + legacy_reference_cases, check_case_reference_fresh, is_diagnostic_evidence, preflight_reference_checks, @@ -1007,6 +1009,8 @@ def rescore_evaluation( cases = [case_by_id[identifier] for identifier in selected_ids] # Validate the entire batch before touching metrics, datasets or results. # Historical records without a binding retain the legacy rescore behavior. + legacy_input = legacy_reference_dataset(manifest, run_dir, dataset_path) + original_cases = legacy_reference_cases(legacy_input) bindings = manifest.get("reference_checks", {}).get("cases", {}) for record in records: if "reference_sha256" in record.get("metrics", {}) or record["case_id"] in bindings: @@ -1015,7 +1019,7 @@ def rescore_evaluation( if (binding is None or case is None or record.get("metrics", {}).get("reference_sha256") != binding["reference_sha256"]): raise ReferenceLifecycleError("Rescore has no matching original reference binding") - check_case_reference_binding(case, binding) + check_case_reference_binding(case, binding, original_case=original_cases.get(case.id)) for record in records: case = case_by_id.get(record["case_id"]) if case is None: @@ -1034,6 +1038,10 @@ def rescore_evaluation( apply_request_status(record) record["metrics"]["deadline_overrun"] = previous_deadline + if legacy_input is not None: + legacy_path = contained_path(run_dir, f'reference-checks/dataset-{hashlib.sha256(legacy_input).hexdigest()}.json', reject_symlinks=True) + legacy_path.parent.mkdir(parents=True, exist_ok=True) + legacy_path.write_bytes(legacy_input) tmp = run_dir / "results.jsonl.tmp" tmp.write_text("".join(json.dumps(record, ensure_ascii=True) + "\n" for record in records), encoding="utf-8") tmp.replace(run_dir / "results.jsonl") diff --git a/tests/fixtures/legacy-live-reference.json b/tests/fixtures/legacy-live-reference.json new file mode 100644 index 000000000..f7ac10bd8 --- /dev/null +++ b/tests/fixtures/legacy-live-reference.json @@ -0,0 +1,50 @@ +{ + "source_revision": "e9cdf6b53a3733a2fa7fd449f6adcc8c345e75c5", + "source_sha256": "ab70a859cab9a76e7382fe96f523a6298fc99de67df161f34471827663b279ce", + "observation": { + "case_id": "body", + "independent": true, + "content_text": "Price 10 dollars", + "checked_at": "2026-09-03T12:00:03+00:00", + "scope_identity": { + "case_id": "body", + "url": "https://example.test/body", + "track": "web", + "task_kind": "body", + "task_prompt": "", + "reference_scope": "Complete article", + "selector": null, + "source_benchmark": {}, + "render_scope": null, + "evaluation_contract": { + "expected_any": [ + "Price" + ], + "required_facts": [], + "required_patterns": [], + "forbidden_any": [], + "min_content_chars": 1, + "fact_coverage_threshold": 1.0, + "content_score_applicable": true, + "expected_resources": [], + "resource_output_contract": {}, + "reference_answer": "" + } + }, + "completion": { + "satisfied": true, + "elapsed_ms": 100, + "timeout_ms": 1000 + }, + "observer": { + "binary": "explicit" + } + }, + "files": { + "results.jsonl": "{\"attempt_id\": \"p__body__r001\", \"case_id\": \"body\", \"provider\": \"p\", \"ok\": true, \"started_at\": \"2026-09-03T12:00:00+00:00\", \"elapsed_ms\": 1000, \"artifact_path\": \"artifacts/output.txt\", \"metrics\": {\"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\", \"content_sha256\": \"f03809d54d25826fd516a82676f1aa5727cb95b15e5aebe6d23b0a3439694e04\"}, \"provider_type\": \"fixture\", \"case_name\": \"Body\", \"category\": \"static\"}\n", + "manifest.json": "{\n \"schema_version\": 1,\n \"tool\": \"web_fetch\",\n \"started_at\": \"2026-09-03T12:00:00+00:00\",\n \"completed_at\": \"2026-09-03T12:00:02+00:00\",\n \"attempt_count\": 1,\n \"attempt_ids\": [\n \"p__body__r001\"\n ],\n \"providers\": [\n {\n \"name\": \"p\",\n \"type\": \"synthetic\"\n }\n ],\n \"dataset\": {\n \"id\": \"test\",\n \"case_count\": 1\n },\n \"parameters\": {\n \"runs\": 1\n },\n \"reference_checks\": {\n \"schema_version\": 1,\n \"status\": \"verified\",\n \"preflight_checked_at\": \"2026-09-03T12:00:00+00:00\",\n \"max_age_seconds\": 1800,\n \"cases\": {\n \"body\": {\n \"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\",\n \"expected_normalized_sha256\": \"5abcaeca8b85a7185181880e89ce1967736a71d99c7115e5664037cae4ab9a30\",\n \"checked_at\": \"2026-09-03T11:59:00+00:00\",\n \"scope_identity\": {\n \"case_id\": \"body\",\n \"url\": \"https://example.test/body\",\n \"track\": \"web\",\n \"task_kind\": \"body\",\n \"task_prompt\": \"\",\n \"reference_scope\": \"Complete article\",\n \"selector\": null,\n \"source_benchmark\": {},\n \"render_scope\": null,\n \"evaluation_contract\": {\n \"expected_any\": [\n \"Price\"\n ],\n \"required_facts\": [],\n \"required_patterns\": [],\n \"forbidden_any\": [],\n \"min_content_chars\": 1,\n \"fact_coverage_threshold\": 1.0,\n \"content_score_applicable\": true,\n \"expected_resources\": [],\n \"resource_output_contract\": {},\n \"reference_answer\": \"\"\n }\n },\n \"source_verification\": {\n \"state\": \"valid\",\n \"reference_mode\": \"live_exact\",\n \"method\": \"independent source capture\",\n \"checked_at\": \"2026-09-03T11:59:00+00:00\",\n \"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\",\n \"reference_scope\": \"Complete article\"\n }\n }\n },\n \"postcheck\": {\n \"checked_at\": \"2026-09-03T12:00:04+00:00\",\n \"status\": \"verified\",\n \"evidence_path\": \"reference-checks/observations-e73e1ba0b9e034178aa0befff88c8d6bbd652c3122865cb0828a238bd87f0278.json\",\n \"evidence_sha256\": \"e73e1ba0b9e034178aa0befff88c8d6bbd652c3122865cb0828a238bd87f0278\",\n \"results_sha256\": \"0ed79bc42eb7de483a6af2e2aaa0e1170506a4aaacc2ff0320facb06bbded49c\",\n \"dataset_sha256\": \"d0bc15c2d0aee7910ff073721507b3e35c83bc4b3a86ad0a70ed64da500f6478\",\n \"cases\": {\n \"body\": {\n \"status\": \"verified\",\n \"checked_at\": \"2026-09-03T12:00:03+00:00\",\n \"latest_call_finished_at\": \"2026-09-03T12:00:01+00:00\",\n \"content_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\",\n \"normalized_sha256\": \"5abcaeca8b85a7185181880e89ce1967736a71d99c7115e5664037cae4ab9a30\"\n }\n }\n }\n },\n \"dataset_sha256\": \"d0bc15c2d0aee7910ff073721507b3e35c83bc4b3a86ad0a70ed64da500f6478\"\n}\n", + "dataset.json": "{\"tool\": \"web_fetch\", \"id\": \"test\", \"cases\": [{\"id\": \"body\", \"name\": \"Body\", \"url\": \"https://example.test/body\", \"category\": \"static\", \"expected_any\": [\"Price\"], \"description\": \"\", \"tags\": [], \"required_facts\": [], \"forbidden_any\": [], \"diagnostic_any\": [], \"min_content_chars\": 1, \"fact_coverage_threshold\": 1.0, \"ground_truth_evidence\": [], \"task_prompt\": \"\", \"reference_answer\": \"\", \"source_benchmark\": {}, \"ground_truth_main_content\": \"Price 10 dollars\", \"expected_resources\": [], \"resource_output_contract\": {}, \"content_score_applicable\": true, \"track\": \"web\", \"required_patterns\": [], \"source_verification\": {\"state\": \"valid\", \"reference_mode\": \"live_exact\", \"method\": \"independent source capture\", \"checked_at\": \"2026-09-03T11:59:00+00:00\", \"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\", \"reference_scope\": \"Complete article\"}, \"task_kind\": \"body\", \"acceptance_contract\": {}, \"composite_acceptance\": {}, \"mutable_body_policy\": {}, \"body_requirements\": {\"version\": 3, \"unit\": \"region_compact_hexagram_recall\", \"minimum_coverage\": 0.8, \"minimum_floor\": 0.6, \"minimum_region_passes\": 1, \"segmentation_method\": \"balanced_tokens_fallback\", \"regions\": [{\"id\": \"body-region-1\", \"text\": \"price 10 dollars\", \"source_text\": \"price 10 dollars\", \"segmentation_method\": \"balanced_tokens_fallback\"}]}, \"retrieval_hints\": {}, \"source_review_url\": \"\", \"evaluation_url\": \"\", \"page_identity_required\": false, \"expected_final_url\": \"\", \"expected_document_language\": \"\", \"document_language_match\": \"primary\", \"expected_document_language_present\": true, \"target_language\": \"\", \"target_region\": \"\", \"rule_review\": {}}]}", + "artifacts/output.txt": "Provider raw output stays unchanged", + "reference-checks/observations-e73e1ba0b9e034178aa0befff88c8d6bbd652c3122865cb0828a238bd87f0278.json": "[{\"case_id\": \"body\", \"independent\": true, \"content_text\": \"Price 10 dollars\", \"checked_at\": \"2026-09-03T12:00:03+00:00\", \"scope_identity\": {\"case_id\": \"body\", \"url\": \"https://example.test/body\", \"track\": \"web\", \"task_kind\": \"body\", \"task_prompt\": \"\", \"reference_scope\": \"Complete article\", \"selector\": null, \"source_benchmark\": {}, \"render_scope\": null, \"evaluation_contract\": {\"expected_any\": [\"Price\"], \"required_facts\": [], \"required_patterns\": [], \"forbidden_any\": [], \"min_content_chars\": 1, \"fact_coverage_threshold\": 1.0, \"content_score_applicable\": true, \"expected_resources\": [], \"resource_output_contract\": {}, \"reference_answer\": \"\"}}, \"completion\": {\"satisfied\": true, \"elapsed_ms\": 100, \"timeout_ms\": 1000}, \"observer\": {\"binary\": \"explicit\"}}]" + } +} diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index a55b9990e..9d1a30584 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -412,6 +412,8 @@ def test_worker_launch_failure_is_checkpointed_without_aborting_the_batch(tmp_pa original = capture.subprocess.Popen calls = [] def launch(*args, **kwargs): + if capture._CAPTURE_PROGRAM not in args[0]: + return original(*args, **kwargs) calls.append(1) if len(calls) == 1: raise OSError('controlled process launch failure') @@ -606,6 +608,8 @@ def test_worker_pipe_error_is_checkpointed_and_next_case_runs(tmp_path, monkeypa launch = capture.subprocess.Popen workers = [] def start(*args, **kwargs): + if capture._CAPTURE_PROGRAM not in args[0]: + return launch(*args, **kwargs) process = launch(*args, **kwargs) if not workers: communicate = process.communicate @@ -724,3 +728,34 @@ def delayed_snapshot(*args, **kwargs): os.kill(int(path.read_text()), signal.SIGKILL) except ProcessLookupError: pass + + +@pytest.mark.parametrize('pgrep_status', [0, 1, 2]) +def test_batch_requires_working_process_discovery_before_launch(monkeypatch, pgrep_status): + from types import SimpleNamespace + launches = [] + monkeypatch.setattr(capture, '_capture_process_snapshot', lambda: {}) + monkeypatch.setattr(capture.subprocess, 'run', lambda *a, **kw: SimpleNamespace(returncode=pgrep_status)) + monkeypatch.setattr(capture, '_capture_in_process', lambda case, *a, **kw: launches.append(case['id']) or {'status': 'success'}) + def batch(): + return capture.capture_batch([case_with()], timeout_ms=1000, attempts=1) + if pgrep_status in (0, 1): + batch(); assert launches + else: + with pytest.raises(ValueError, match='process discovery'): + batch() + assert not launches + + +@pytest.mark.skipif(not os.getenv('BROWSER_EVAL_TEST_BROWSER_BINARY'), reason='explicit Chromium required') +def test_missing_process_tools_reject_real_browser_before_acquisition(monkeypatch): + launches = [] + def unavailable(*args, **kwargs): + raise FileNotFoundError('process query unavailable') + monkeypatch.setattr(capture.subprocess, 'check_output', unavailable) + monkeypatch.setattr(capture.subprocess, 'run', unavailable) + monkeypatch.setattr(capture.subprocess, 'Popen', lambda *a, **kw: launches.append(a)) + with pytest.raises(ValueError, match='process discovery'): + capture.capture_batch([case_with()], timeout_ms=20, attempts=1, + browser_binary=os.environ['BROWSER_EVAL_TEST_BROWSER_BINARY']) + assert launches == [] diff --git a/tests/test_browser_reference_scope.py b/tests/test_browser_reference_scope.py index 7ccf545ae..28d05f4c1 100644 --- a/tests/test_browser_reference_scope.py +++ b/tests/test_browser_reference_scope.py @@ -418,3 +418,54 @@ def fake_capture(case, timeout_ms, attempts, browser_binary, resolved_ip): assert set(rows) == {'unreachable', 'reachable'} assert rows['unreachable']['status'] == 'failure' assert rows['reachable']['status'] == 'success' + + +@pytest.mark.skipif(not os.getenv('BROWSER_EVAL_TEST_BROWSER_BINARY'), reason='explicit Chromium required') +@pytest.mark.parametrize('missing', [None, 'text', 'image']) +def test_mixed_live_resource_capture_freezes_text_and_preserves_resource_checks(tmp_path, missing): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from threading import Thread + from datetime import datetime, timezone + import hashlib + from harness.web_fetch.reference_freeze import compute_reference_update + from harness.web_fetch.reference_lifecycle import preflight_reference_checks + + class Page(BaseHTTPRequestHandler): + def do_GET(self): + body = ('' if missing == 'text' else '
Required article anchor
') + body += '' if missing == 'image' else '' + self.send_response(200); self.send_header('Content-Type', 'text/html'); self.end_headers() + self.wfile.write(('' + body + '').encode()) + def log_message(self, *args): pass + server = ThreadingHTTPServer(('127.0.0.1', 0), Page) + thread = Thread(target=server.serve_forever, daemon=True); thread.start() + url = f'http://127.0.0.1:{server.server_port}/' + evidence = {'case_id': 'mixed', 'url': url, 'source': 'Independent HTTP; no provider output', + 'state': 'captured', 'status': 200, 'checked_at': datetime.now(timezone.utc).isoformat(), + 'resources': [{'tag': 'img', 'url': url + 'photo.png'}]} + folder = tmp_path / 'sources/mixed'; folder.mkdir(parents=True) + raw = json.dumps(evidence).encode(); (folder / 'evidence.json').write_bytes(raw) + (tmp_path / 'audit.json').write_text('[]') + case = {'id': 'mixed', 'url': url, 'track': 'image', 'task_kind': 'resource', + 'ground_truth_main_content': 'Required article anchor', + 'required_facts': [{'id': 'body', 'alternatives': ['Required article anchor']}], + 'expected_resources': [{'id': 'photo', 'urls': [url + 'photo.png']}], + 'source_verification': {'reference_mode': 'live_exact', 'observation_route': 'browser', + 'state': 'valid', 'method': 'Independent browser', 'checked_at': evidence['checked_at'], + 'evidence_path': 'sources/mixed/evidence.json', 'evidence_sha256': hashlib.sha256(raw).hexdigest()}} + def freeze(): + row = browser_capture.capture(case, 3000, os.environ['BROWSER_EVAL_TEST_BROWSER_BINARY']) + frozen, _, _ = compute_reference_update({'cases': [case]}, tmp_path, {'mixed': row}) + path = tmp_path / 'dataset.json'; path.write_text(json.dumps(frozen)) + checks = preflight_reference_checks(frozen['cases'], path) + assert checks['cases']['mixed']['reference_sha256'] + assert frozen['cases'][0]['ground_truth_main_content'] == 'Required article anchor' + assert row['target_matches'][0]['satisfied'] + try: + if missing: + with pytest.raises((ValueError, TimeoutError)): + freeze() + else: + freeze() + finally: + server.shutdown(); server.server_close(); thread.join() diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index 818557eb7..09d01284b 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -858,3 +858,40 @@ def test_unknown_reference_mode_is_not_hidden_by_a_stable_contract(): 'version': 1, 'schema': 'body_text', 'minimum_records': 0, 'stable_text_anchors': [{'id': 'title', 'alternatives': ['Article']}]}, 'source_verification': {'reference_mode': 'live_excat'}}) + + +def test_original_scope_certificate_survives_unchanged_inputs_but_rejects_rule_changes(tmp_path): + from harness.web_fetch.reference_freeze import load_browser_observations + fixture = json.loads((Path(__file__).parent / 'fixtures/legacy-live-reference.json').read_text()) + for relative, text in fixture['files'].items(): + target = tmp_path / relative; target.parent.mkdir(parents=True, exist_ok=True); target.write_text(text) + dataset = json.loads((tmp_path / 'dataset.json').read_text()) + case = EvalCase(**dataset['cases'][0]) + manifest = json.loads((tmp_path / 'manifest.json').read_text()) + # The runner may serialize its saved dataset differently from the input file. + manifest['dataset_sha256'] = sha256(json.dumps(dataset, indent=4)) + (tmp_path / 'manifest.json').write_text(json.dumps(manifest)) + observation = tmp_path / 'browser-observation.json' + observation.write_text(json.dumps({case.id: fixture['observation']})) + assert load_browser_observations(dataset, path=observation)[case.id] == fixture['observation'] + require_verified_reference_checks([manifest], cases=[case], source=tmp_path) + require_verified_reference_checks([manifest], cases=[case], source=tmp_path, equivalent_capture_bindings=True) + for field in ('acceptance_contract', 'body_requirements', 'composite_acceptance'): + changed = replace(case, **{field: {'required_fields': ['changed']}}) + with pytest.raises(ReferenceLifecycleError): + require_verified_reference_checks([manifest], cases=[changed], source=tmp_path) + with pytest.raises(ReferenceLifecycleError): + rescore_evaluation(run_dir=tmp_path, dataset_meta={**dataset, 'cases': [changed.__dict__]}, + dataset_path=tmp_path / 'dataset.json', cases=[changed]) + rescore_evaluation(run_dir=tmp_path, dataset_meta=dataset, dataset_path=tmp_path / 'dataset.json', cases=[case]) + require_verified_reference_checks([manifest], cases=[case], source=tmp_path) + + from harness.web_fetch.artifacts import materialize_reference_checks + packaged = tmp_path / 'package'; packaged.mkdir() + (packaged / 'dataset.json').write_text(json.dumps({**dataset, 'presentation': 'new'})) + materialize_reference_checks(packaged, tmp_path, [manifest]) + require_verified_reference_checks([manifest], cases=[case], source=packaged) + original_input = next(packaged.glob('reference-checks/dataset-*.json')) + original_input.write_text(original_input.read_text() + ' ') + with pytest.raises(ReferenceLifecycleError): + require_verified_reference_checks([manifest], cases=[case], source=packaged) From c90e29f9a321078c2b7f8ac6d99a6027c0596635 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:32:53 +0800 Subject: [PATCH 14/19] fix(webfetch): require original rules for legacy certificates --- harness/web_fetch/reference_lifecycle.py | 10 ++++++---- tests/test_reference_lifecycle.py | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index c5c86d65e..fdfcdc5b0 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -725,9 +725,9 @@ def preserve_legacy_reference_dataset(manifest, source, output): def check_case_reference_binding(case, binding: dict, *, original_case=None) -> None: """Offline rescores must use the original capture; no freshness claim is made.""" version = binding.get('scope_identity', {}).get('schema_version', 1) - if version not in (1, 2) or (version == 1 and any( - _get(case, key, {}) != _get(original_case or {}, key, {}) - for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance'))): + if version not in (1, 2) or (version == 1 and (original_case is None or any( + _get(case, key, {}) != _get(original_case, key, {}) + for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance')))): raise ReferenceLifecycleError('Legacy rules differ from the hash-bound original dataset', 'reference_changed') reference = _get(case, "ground_truth_main_content", "") if (sha256(reference) != binding.get("reference_sha256") @@ -741,7 +741,9 @@ def check_case_reference_binding(case, binding: dict, *, original_case=None) -> def check_case_equivalent_reference_binding(case, binding: dict, *, original_case=None) -> None: """Allow distinct independent captures only when reference bytes and scope are identical.""" - if binding.get('scope_identity', {}).get('schema_version', 1) == 1 and original_case is not None: + if binding.get('scope_identity', {}).get('schema_version', 1) == 1: + if original_case is None: + raise ReferenceLifecycleError('Legacy rules require the hash-bound original dataset', 'reference_changed') binding = deepcopy(binding) binding['scope_identity']['evaluation_contract'].update({ key: _get(original_case, key, {}) diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index 09d01284b..97c39a16d 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -895,3 +895,9 @@ def test_original_scope_certificate_survives_unchanged_inputs_but_rejects_rule_c original_input.write_text(original_input.read_text() + ' ') with pytest.raises(ReferenceLifecycleError): require_verified_reference_checks([manifest], cases=[case], source=packaged) + + stripped = replace(case, body_requirements={}, acceptance_contract={}, composite_acceptance={}) + for equivalent in (False, True): + with pytest.raises(ReferenceLifecycleError): + require_verified_reference_checks([manifest], cases=[stripped], source=packaged, + equivalent_capture_bindings=equivalent) From f66336adb971538e19b8038dbe3bb8a6c0f045c7 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:37:53 +0800 Subject: [PATCH 15/19] fix(webfetch): share text qualification across live task kinds --- harness/web_fetch/reference_freeze.py | 4 +++ harness/web_fetch/reference_lifecycle.py | 5 ++- harness/web_fetch/render_evidence.py | 39 ++++++++++++++---------- tests/test_browser_reference_scope.py | 5 +-- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index e20ce9c99..604c75a9e 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -13,6 +13,7 @@ from harness.record_io import canonical_json_sha256, read_completed_records from harness.web_fetch.artifacts import materialize_task_references +from harness.web_fetch.render_evidence import content_requirement_errors from lexbench_webfetch.body_coverage import body_requirements from lexbench_webfetch.metrics import normalize_for_match from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes, require_mutable_inputs @@ -240,6 +241,9 @@ def compute_reference_update(dataset, captures, observations=None, *, refresh_ca case,row,browser_observation=cid in observations)}) case['ground_truth_main_content']=text if old_requirements is not None:case['body_requirements']=new_requirements + errors=content_requirement_errors(case,text) + if errors: + raise ValueError(f'Reference requirements not met: {cid}: {errors}') case['source_verification'].update(state='valid',checked_at=timestamp, selector=selector,method=method, reference_sha256=hashlib.sha256(text.encode()).hexdigest()) diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index fdfcdc5b0..f0d0bb042 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -18,7 +18,7 @@ from lexbench_webfetch.metrics import normalize_for_match, normalize_resource_url from .reference_inputs import input_tree, read_dataset_bytes -from .render_evidence import validate_render_reference +from .render_evidence import content_requirement_errors, validate_render_reference HOMEPAGE_READER_SCOPE_PROMPTS = frozenset({ @@ -602,6 +602,9 @@ def _binding(case, dataset_path: Path, now: datetime) -> dict[str, Any]: if not isinstance(verification.get("method"), str) or not verification["method"].strip(): raise ReferenceLifecycleError(f"{identifier}: independent verification method is missing") _fresh(verification.get("checked_at"), now) + errors = content_requirement_errors(case, reference) + if errors: + raise ReferenceLifecycleError(f"{identifier}: reference requirements not met: {errors}") actual_hash = sha256(reference) if verification.get("reference_sha256") != actual_hash: raise ReferenceLifecycleError(f"{identifier}: reference SHA-256 mismatch", "reference_changed") diff --git a/harness/web_fetch/render_evidence.py b/harness/web_fetch/render_evidence.py index 58ee0a543..1be7bfec3 100644 --- a/harness/web_fetch/render_evidence.py +++ b/harness/web_fetch/render_evidence.py @@ -87,6 +87,27 @@ def _coverage(text: str, facts: list[dict[str, Any]]) -> dict[str, Any]: "all_present": bool(groups) and hits == len(groups), "groups": groups} +def content_requirement_errors(case: EvalCase | Mapping[str, Any], text: str) -> list[str]: + """Check declared text requirements for any independently captured task.""" + get = case.get if isinstance(case, Mapping) else lambda key, default=None: getattr(case, key, default) + requirements = {key: get(key, default) or default for key, default in ( + ('body_requirements', {}), ('required_patterns', []), + ('acceptance_contract', {}), ('composite_acceptance', {}))} + if not any(requirements.values()): + return [] + from lexbench_webfetch.metrics import evaluate_content + checks = evaluate_content(text, [], **requirements, base_url=get('url', '')) + errors = [] + if not checks['body_coverage_pass']: + errors.append('body_requirements_not_met') + if not all(row['pass'] for row in checks['pattern_results']): + errors.append('pattern_requirements_not_met') + for field, label in (('stable_task_pass', 'record'), ('composite_acceptance_pass', 'composite')): + if checks.get(field) is False: + errors.append(f'{label}_requirements_not_met') + return errors + + def validate_render_reference( case: EvalCase | Mapping[str, Any], initial_html: str | bytes, rendered_text: str, ) -> dict[str, Any]: @@ -152,22 +173,8 @@ def validate_render_reference( for group in coverage[key]["groups"]: if not group["pass"]: errors.append(f"{key}_missing_fact:{group['id']}") - if body_requirements or required_patterns or acceptance_contract or composite_acceptance: - from lexbench_webfetch.metrics import evaluate_content - for key in ("reference", "rendered"): - checks = evaluate_content(texts[key], [], body_requirements=body_requirements, - required_patterns=required_patterns, - acceptance_contract=acceptance_contract, - composite_acceptance=composite_acceptance, - base_url=get("url", "")) - if not checks["body_coverage_pass"]: - errors.append(f"{key}_body_requirements_not_met") - if not all(row['pass'] for row in checks['pattern_results']): - errors.append(f"{key}_pattern_requirements_not_met") - for field, label in (("stable_task_pass", "record"), - ("composite_acceptance_pass", "composite")): - if checks.get(field) is False: - errors.append(f"{key}_{label}_requirements_not_met") + for key in ("reference", "rendered"): + errors.extend(f"{key}_{error}" for error in content_requirement_errors(case, texts[key])) evidence = (get("source_benchmark", {}) or {}).get("render_evidence", {}) completion = evidence.get("completion", {}) condition = completion.get("condition") diff --git a/tests/test_browser_reference_scope.py b/tests/test_browser_reference_scope.py index 28d05f4c1..7cfaff97f 100644 --- a/tests/test_browser_reference_scope.py +++ b/tests/test_browser_reference_scope.py @@ -432,7 +432,7 @@ def test_mixed_live_resource_capture_freezes_text_and_preserves_resource_checks( class Page(BaseHTTPRequestHandler): def do_GET(self): - body = ('' if missing == 'text' else '
Required article anchor
') + body = ('
Unrelated article text
' if missing == 'text' else '
Required article anchor
') body += '' if missing == 'image' else '' self.send_response(200); self.send_header('Content-Type', 'text/html'); self.end_headers() self.wfile.write(('' + body + '').encode()) @@ -448,7 +448,8 @@ def log_message(self, *args): pass (tmp_path / 'audit.json').write_text('[]') case = {'id': 'mixed', 'url': url, 'track': 'image', 'task_kind': 'resource', 'ground_truth_main_content': 'Required article anchor', - 'required_facts': [{'id': 'body', 'alternatives': ['Required article anchor']}], + 'acceptance_contract': {'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'body', 'alternatives': ['Required article anchor']}]}, 'expected_resources': [{'id': 'photo', 'urls': [url + 'photo.png']}], 'source_verification': {'reference_mode': 'live_exact', 'observation_route': 'browser', 'state': 'valid', 'method': 'Independent browser', 'checked_at': evidence['checked_at'], From 96dd15d36ba4097db2fa98c2a60e0bca4fb7ebfa Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:47:27 +0800 Subject: [PATCH 16/19] fix(webfetch): retain recorded fields in unversioned scopes --- harness/web_fetch/reference_freeze.py | 3 +- harness/web_fetch/reference_lifecycle.py | 23 ++++-- tests/fixtures/legacy-contract-reference.json | 79 +++++++++++++++++++ tests/test_reference_lifecycle.py | 5 +- 4 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/legacy-contract-reference.json diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index 604c75a9e..a22406347 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -24,6 +24,7 @@ requires_browser_observation, requires_fresh_reference, scope_identity, + matches_scope_identity, timestamp, validate_postcheck_observation, ) @@ -143,7 +144,7 @@ def load_browser_observations(dataset, path=None, directory=None): for cid,row in observations.items(): if not isinstance(row,dict) or row.get('case_id')!=cid: raise ValueError('Browser observation case ID mismatch: '+cid) - if row.get('scope_identity')!=scope_identity(cases[cid], version=row.get('scope_identity', {}).get('schema_version', 1)): + if not matches_scope_identity(cases[cid], row.get('scope_identity')): raise ValueError('Browser observation scope identity mismatch: '+cid) binary=(row.get('observer') or {}).get('binary') if (not isinstance(binary,str) or Path(binary).is_absolute() diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index f0d0bb042..f88b5896a 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -229,6 +229,20 @@ def scope_identity(case, *, version=2) -> dict[str, Any]: }) +def matches_scope_identity(case, stored_scope) -> bool: + if not isinstance(stored_scope, dict): + return False + version = stored_scope.get('schema_version', 1) + if version not in (1, 2): + return False + expected = scope_identity(case, version=version) + if version == 1: + for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance'): + if key in stored_scope.get('evaluation_contract', {}): + expected['evaluation_contract'][key] = _get(case, key, {}) + return stored_scope == expected + + def resource_source_scope_identity(case) -> dict[str, Any]: """Stable source-observation scope for resource ground truth.""" return deepcopy({ @@ -735,7 +749,7 @@ def check_case_reference_binding(case, binding: dict, *, original_case=None) -> reference = _get(case, "ground_truth_main_content", "") if (sha256(reference) != binding.get("reference_sha256") or sha256(normalize_for_match(reference)) != binding.get("expected_normalized_sha256") - or scope_identity(case, version=version) != binding.get("scope_identity") + or not matches_scope_identity(case, binding.get("scope_identity")) or _get(case, "source_verification", {}) != binding.get("source_verification") or ("render_evidence" in binding and (_get(case, "source_benchmark", {}) or {}).get("render_evidence") != binding["render_evidence"])): @@ -748,10 +762,9 @@ def check_case_equivalent_reference_binding(case, binding: dict, *, original_cas if original_case is None: raise ReferenceLifecycleError('Legacy rules require the hash-bound original dataset', 'reference_changed') binding = deepcopy(binding) - binding['scope_identity']['evaluation_contract'].update({ - key: _get(original_case, key, {}) - for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance') - if _get(original_case, key, {})}) + for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance'): + if _get(original_case, key, {}): + binding['scope_identity']['evaluation_contract'].setdefault(key, _get(original_case, key)) reference = _get(case, "ground_truth_main_content", "") verification = _get(case, "source_verification", {}) if (sha256(reference) != binding.get("reference_sha256") diff --git a/tests/fixtures/legacy-contract-reference.json b/tests/fixtures/legacy-contract-reference.json new file mode 100644 index 000000000..ce84cc070 --- /dev/null +++ b/tests/fixtures/legacy-contract-reference.json @@ -0,0 +1,79 @@ +{ + "source_revision": "5d7c04e87d9ca7639293b6f55f58538b2969ce2c", + "source_sha256": "7934b0be67d2f71a1178cbfcbf74dabdf4590176fa8c8d29a1b575227ffd849e", + "observation": { + "case_id": "body", + "independent": true, + "content_text": "Price 10 dollars", + "checked_at": "2026-09-03T12:00:03+00:00", + "scope_identity": { + "case_id": "body", + "url": "https://example.test/body", + "track": "web", + "task_kind": "body", + "task_prompt": "", + "reference_scope": "Complete article", + "selector": null, + "source_benchmark": {}, + "render_scope": null, + "evaluation_contract": { + "expected_any": [ + "Price" + ], + "required_facts": [], + "required_patterns": [], + "forbidden_any": [], + "min_content_chars": 1, + "fact_coverage_threshold": 1.0, + "content_score_applicable": true, + "expected_resources": [], + "resource_output_contract": {}, + "reference_answer": "", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "price", + "alternatives": [ + "Price" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "price 10 dollars", + "source_text": "price 10 dollars", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + } + }, + "completion": { + "satisfied": true, + "elapsed_ms": 100, + "timeout_ms": 1000 + }, + "observer": { + "binary": "explicit" + } + }, + "files": { + "results.jsonl": "{\"attempt_id\": \"p__body__r001\", \"case_id\": \"body\", \"provider\": \"p\", \"ok\": true, \"started_at\": \"2026-09-03T12:00:00+00:00\", \"elapsed_ms\": 1000, \"artifact_path\": \"artifacts/output.txt\", \"metrics\": {\"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\", \"content_sha256\": \"f03809d54d25826fd516a82676f1aa5727cb95b15e5aebe6d23b0a3439694e04\"}, \"provider_type\": \"fixture\", \"case_name\": \"Body\", \"category\": \"static\"}\n", + "manifest.json": "{\n \"schema_version\": 1,\n \"tool\": \"web_fetch\",\n \"started_at\": \"2026-09-03T12:00:00+00:00\",\n \"completed_at\": \"2026-09-03T12:00:02+00:00\",\n \"attempt_count\": 1,\n \"attempt_ids\": [\n \"p__body__r001\"\n ],\n \"providers\": [\n {\n \"name\": \"p\",\n \"type\": \"synthetic\"\n }\n ],\n \"dataset\": {\n \"id\": \"test\",\n \"case_count\": 1\n },\n \"parameters\": {\n \"runs\": 1\n },\n \"reference_checks\": {\n \"schema_version\": 1,\n \"status\": \"verified\",\n \"preflight_checked_at\": \"2026-09-03T12:00:00+00:00\",\n \"max_age_seconds\": 1800,\n \"cases\": {\n \"body\": {\n \"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\",\n \"expected_normalized_sha256\": \"5abcaeca8b85a7185181880e89ce1967736a71d99c7115e5664037cae4ab9a30\",\n \"checked_at\": \"2026-09-03T11:59:00+00:00\",\n \"scope_identity\": {\n \"case_id\": \"body\",\n \"url\": \"https://example.test/body\",\n \"track\": \"web\",\n \"task_kind\": \"body\",\n \"task_prompt\": \"\",\n \"reference_scope\": \"Complete article\",\n \"selector\": null,\n \"source_benchmark\": {},\n \"render_scope\": null,\n \"evaluation_contract\": {\n \"expected_any\": [\n \"Price\"\n ],\n \"required_facts\": [],\n \"required_patterns\": [],\n \"forbidden_any\": [],\n \"min_content_chars\": 1,\n \"fact_coverage_threshold\": 1.0,\n \"content_score_applicable\": true,\n \"expected_resources\": [],\n \"resource_output_contract\": {},\n \"reference_answer\": \"\",\n \"acceptance_contract\": {\n \"version\": 1,\n \"schema\": \"body_text\",\n \"minimum_records\": 0,\n \"stable_text_anchors\": [\n {\n \"id\": \"price\",\n \"alternatives\": [\n \"Price\"\n ]\n }\n ]\n },\n \"body_requirements\": {\n \"version\": 3,\n \"unit\": \"region_compact_hexagram_recall\",\n \"minimum_coverage\": 0.8,\n \"minimum_floor\": 0.6,\n \"minimum_region_passes\": 1,\n \"segmentation_method\": \"balanced_tokens_fallback\",\n \"regions\": [\n {\n \"id\": \"body-region-1\",\n \"text\": \"price 10 dollars\",\n \"source_text\": \"price 10 dollars\",\n \"segmentation_method\": \"balanced_tokens_fallback\"\n }\n ]\n }\n }\n },\n \"source_verification\": {\n \"state\": \"valid\",\n \"reference_mode\": \"live_exact\",\n \"method\": \"independent source capture\",\n \"checked_at\": \"2026-09-03T11:59:00+00:00\",\n \"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\",\n \"reference_scope\": \"Complete article\"\n }\n }\n },\n \"postcheck\": {\n \"checked_at\": \"2026-09-03T12:00:04+00:00\",\n \"status\": \"verified\",\n \"evidence_path\": \"reference-checks/observations-4be14ef59e82865f9b9055a832f364fa0bfa8fc32bfc019e2f4d2da7d17051ba.json\",\n \"evidence_sha256\": \"4be14ef59e82865f9b9055a832f364fa0bfa8fc32bfc019e2f4d2da7d17051ba\",\n \"results_sha256\": \"0ed79bc42eb7de483a6af2e2aaa0e1170506a4aaacc2ff0320facb06bbded49c\",\n \"dataset_sha256\": \"ef0272d6ca23773180e8130ccd307e72f25e44b65e8ec0fab788a407d59eec18\",\n \"cases\": {\n \"body\": {\n \"status\": \"verified\",\n \"checked_at\": \"2026-09-03T12:00:03+00:00\",\n \"latest_call_finished_at\": \"2026-09-03T12:00:01+00:00\",\n \"content_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\",\n \"normalized_sha256\": \"5abcaeca8b85a7185181880e89ce1967736a71d99c7115e5664037cae4ab9a30\"\n }\n }\n }\n },\n \"dataset_sha256\": \"ef0272d6ca23773180e8130ccd307e72f25e44b65e8ec0fab788a407d59eec18\"\n}\n", + "dataset.json": "{\"tool\": \"web_fetch\", \"id\": \"test\", \"cases\": [{\"id\": \"body\", \"name\": \"Body\", \"url\": \"https://example.test/body\", \"category\": \"static\", \"expected_any\": [\"Price\"], \"description\": \"\", \"tags\": [], \"required_facts\": [], \"forbidden_any\": [], \"diagnostic_any\": [], \"min_content_chars\": 1, \"fact_coverage_threshold\": 1.0, \"ground_truth_evidence\": [], \"task_prompt\": \"\", \"reference_answer\": \"\", \"source_benchmark\": {}, \"ground_truth_main_content\": \"Price 10 dollars\", \"expected_resources\": [], \"resource_output_contract\": {}, \"content_score_applicable\": true, \"track\": \"web\", \"required_patterns\": [], \"source_verification\": {\"state\": \"valid\", \"reference_mode\": \"live_exact\", \"method\": \"independent source capture\", \"checked_at\": \"2026-09-03T11:59:00+00:00\", \"reference_sha256\": \"b8981cae1071848c08e8e784911a07169ae111c3fad7599cba42d485f3602423\", \"reference_scope\": \"Complete article\"}, \"task_kind\": \"body\", \"acceptance_contract\": {\"version\": 1, \"schema\": \"body_text\", \"minimum_records\": 0, \"stable_text_anchors\": [{\"id\": \"price\", \"alternatives\": [\"Price\"]}]}, \"composite_acceptance\": {}, \"mutable_body_policy\": {}, \"body_requirements\": {\"version\": 3, \"unit\": \"region_compact_hexagram_recall\", \"minimum_coverage\": 0.8, \"minimum_floor\": 0.6, \"minimum_region_passes\": 1, \"segmentation_method\": \"balanced_tokens_fallback\", \"regions\": [{\"id\": \"body-region-1\", \"text\": \"price 10 dollars\", \"source_text\": \"price 10 dollars\", \"segmentation_method\": \"balanced_tokens_fallback\"}]}, \"retrieval_hints\": {}, \"source_review_url\": \"\", \"evaluation_url\": \"\", \"page_identity_required\": false, \"expected_final_url\": \"\", \"expected_document_language\": \"\", \"document_language_match\": \"primary\", \"expected_document_language_present\": true, \"target_language\": \"\", \"target_region\": \"\", \"rule_review\": {}}]}", + "artifacts/output.txt": "Provider raw output stays unchanged", + "reference-checks/observations-4be14ef59e82865f9b9055a832f364fa0bfa8fc32bfc019e2f4d2da7d17051ba.json": "[{\"case_id\": \"body\", \"independent\": true, \"content_text\": \"Price 10 dollars\", \"checked_at\": \"2026-09-03T12:00:03+00:00\", \"scope_identity\": {\"case_id\": \"body\", \"url\": \"https://example.test/body\", \"track\": \"web\", \"task_kind\": \"body\", \"task_prompt\": \"\", \"reference_scope\": \"Complete article\", \"selector\": null, \"source_benchmark\": {}, \"render_scope\": null, \"evaluation_contract\": {\"expected_any\": [\"Price\"], \"required_facts\": [], \"required_patterns\": [], \"forbidden_any\": [], \"min_content_chars\": 1, \"fact_coverage_threshold\": 1.0, \"content_score_applicable\": true, \"expected_resources\": [], \"resource_output_contract\": {}, \"reference_answer\": \"\", \"acceptance_contract\": {\"version\": 1, \"schema\": \"body_text\", \"minimum_records\": 0, \"stable_text_anchors\": [{\"id\": \"price\", \"alternatives\": [\"Price\"]}]}, \"body_requirements\": {\"version\": 3, \"unit\": \"region_compact_hexagram_recall\", \"minimum_coverage\": 0.8, \"minimum_floor\": 0.6, \"minimum_region_passes\": 1, \"segmentation_method\": \"balanced_tokens_fallback\", \"regions\": [{\"id\": \"body-region-1\", \"text\": \"price 10 dollars\", \"source_text\": \"price 10 dollars\", \"segmentation_method\": \"balanced_tokens_fallback\"}]}}}, \"completion\": {\"satisfied\": true, \"elapsed_ms\": 100, \"timeout_ms\": 1000}, \"observer\": {\"binary\": \"explicit\"}}]" + } +} diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index 97c39a16d..fa54024b7 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -860,9 +860,10 @@ def test_unknown_reference_mode_is_not_hidden_by_a_stable_contract(): 'source_verification': {'reference_mode': 'live_excat'}}) -def test_original_scope_certificate_survives_unchanged_inputs_but_rejects_rule_changes(tmp_path): +@pytest.mark.parametrize('fixture_name', ['legacy-live-reference.json', 'legacy-contract-reference.json']) +def test_original_scope_certificate_survives_unchanged_inputs_but_rejects_rule_changes(tmp_path, fixture_name): from harness.web_fetch.reference_freeze import load_browser_observations - fixture = json.loads((Path(__file__).parent / 'fixtures/legacy-live-reference.json').read_text()) + fixture = json.loads((Path(__file__).parent / 'fixtures' / fixture_name).read_text()) for relative, text in fixture['files'].items(): target = tmp_path / relative; target.parent.mkdir(parents=True, exist_ok=True); target.write_text(text) dataset = json.loads((tmp_path / 'dataset.json').read_text()) From abaf20029cc98842818e7dae481c5881861ced72 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:56:13 +0800 Subject: [PATCH 17/19] fix(webfetch): verify legacy observation rule authority --- harness/web_fetch/reference_freeze.py | 18 ++++++++++++++++++ tests/test_reference_lifecycle.py | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index a22406347..775da6c10 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -25,6 +25,9 @@ requires_fresh_reference, scope_identity, matches_scope_identity, + check_case_reference_binding, + legacy_reference_dataset, + legacy_reference_cases, timestamp, validate_postcheck_observation, ) @@ -141,11 +144,26 @@ def load_browser_observations(dataset, path=None, directory=None): unknown=set(observations)-set(cases) if unknown: raise ValueError('Unknown browser observation case: '+sorted(unknown)[0]) + original_manifest={};original_cases={} + if any(isinstance(row,dict) and isinstance(row.get('scope_identity',{}),dict) + and row.get('scope_identity',{}).get('schema_version',1)==1 + for row in observations.values()): + source=path.parent if path else directory + manifest_path=contained_path(source,'manifest.json',reject_symlinks=True) + if not manifest_path.is_file(): + raise ValueError('Legacy browser observations require their original run manifest; capture again with scope version 2') + original_manifest=json.loads(manifest_path.read_text()) + original_cases=legacy_reference_cases(legacy_reference_dataset(original_manifest,source)) for cid,row in observations.items(): if not isinstance(row,dict) or row.get('case_id')!=cid: raise ValueError('Browser observation case ID mismatch: '+cid) if not matches_scope_identity(cases[cid], row.get('scope_identity')): raise ValueError('Browser observation scope identity mismatch: '+cid) + if row.get('scope_identity',{}).get('schema_version',1)==1: + binding=original_manifest.get('reference_checks',{}).get('cases',{}).get(cid) + if not isinstance(binding,dict) or binding.get('scope_identity')!=row.get('scope_identity'): + raise ValueError('Legacy browser observation differs from its original capture scope: '+cid) + check_case_reference_binding(cases[cid],binding,original_case=original_cases.get(cid)) binary=(row.get('observer') or {}).get('binary') if (not isinstance(binary,str) or Path(binary).is_absolute() or PureWindowsPath(binary).drive or '/' in binary or '\\' in binary): diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index fa54024b7..5fc083fcd 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -879,11 +879,22 @@ def test_original_scope_certificate_survives_unchanged_inputs_but_rejects_rule_c require_verified_reference_checks([manifest], cases=[case], source=tmp_path, equivalent_capture_bindings=True) for field in ('acceptance_contract', 'body_requirements', 'composite_acceptance'): changed = replace(case, **{field: {'required_fields': ['changed']}}) + with pytest.raises(ValueError): + load_browser_observations({**dataset, 'cases': [changed.__dict__]}, path=observation) + if getattr(case, field): + removed = replace(case, **{field: {}}) + with pytest.raises(ValueError): + load_browser_observations({**dataset, 'cases': [removed.__dict__]}, path=observation) with pytest.raises(ReferenceLifecycleError): require_verified_reference_checks([manifest], cases=[changed], source=tmp_path) with pytest.raises(ReferenceLifecycleError): rescore_evaluation(run_dir=tmp_path, dataset_meta={**dataset, 'cases': [changed.__dict__]}, dataset_path=tmp_path / 'dataset.json', cases=[changed]) + unbound = tmp_path / 'unbound'; unbound.mkdir() + unbound_observation = unbound / 'browser-observation.json' + unbound_observation.write_bytes(observation.read_bytes()) + with pytest.raises(ValueError): + load_browser_observations(dataset, path=unbound_observation) rescore_evaluation(run_dir=tmp_path, dataset_meta=dataset, dataset_path=tmp_path / 'dataset.json', cases=[case]) require_verified_reference_checks([manifest], cases=[case], source=tmp_path) From bcd5b18e4d63afb0edf512659319b67a2a21fe0b Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:41:26 +0800 Subject: [PATCH 18/19] refactor: standardize imports and preserve runtime loading --- CONTRIBUTING.md | 4 + Makefile | 2 +- harness/browser_use/__init__.py | 2 +- harness/browser_use/failure_cause_review.py | 6 +- harness/browser_use/report.py | 48 +++-- harness/browser_use/report_check.py | 4 +- harness/browser_use/runner.py | 66 +------ harness/browser_use/selection.py | 6 +- harness/environments/webarena/cli.py | 6 +- harness/environments/webarena/config.py | 4 +- harness/environments/webarena/manager.py | 4 +- harness/environments/webarena/map_setup.py | 3 +- harness/environments/webarena/rainbond.py | 5 +- harness/lexbench/report.py | 16 +- harness/lexbench/resource_evidence.py | 4 +- harness/lexbench/resource_profiles.py | 3 +- harness/lexbench/resource_summary.py | 6 +- harness/lexbench/verification.py | 3 +- harness/record_io.py | 1 - harness/report_languages.py | 4 +- harness/reporting/assets.py | 7 +- harness/reporting/packs.py | 19 +- harness/reporting/publication.py | 3 +- harness/reporting/versions.py | 3 +- harness/reporting/visual.py | 1 + harness/trajectory_replay/__init__.py | 2 +- .../trajectory_replay/browser_use_seeds.py | 3 +- .../trajectory_replay/catalog_action_audit.py | 8 +- .../trajectory_replay/catalog_comparison.py | 8 +- harness/trajectory_replay/cli.py | 2 +- harness/trajectory_replay/diagnostics.py | 10 +- .../trajectory_replay/evaluator_transition.py | 7 +- harness/trajectory_replay/executor.py | 2 +- .../public_webarena/__init__.py | 2 +- .../trajectory_replay/public_webarena/cli.py | 11 +- .../public_webarena/diagnostics.py | 51 +++-- .../public_webarena/evaluators.py | 4 +- .../public_webarena/executor.py | 4 +- .../public_webarena/manifest_repair.py | 9 +- .../public_webarena/publication.py | 20 +- .../public_webarena/qualification.py | 21 +- .../public_webarena/release.py | 7 +- .../public_webarena/report.py | 8 +- .../public_webarena/response_builder.py | 2 +- .../public_webarena/runner.py | 44 ++--- .../public_webarena/suite.py | 9 +- .../public_webarena/webarena.py | 10 +- .../public_webarena/webarena_environment.py | 3 +- harness/trajectory_replay/remote_runtime.py | 6 +- .../trajectory_replay/report_compaction.py | 3 +- harness/trajectory_replay/report_languages.py | 4 +- .../trajectory_replay/retrieval_acceptance.py | 2 +- harness/trajectory_replay/runner.py | 23 +-- .../trajectory_replay/webarena_corrections.py | 28 ++- harness/trajectory_replay/webarena_dataset.py | 5 +- harness/trajectory_replay/webarena_plan.py | 5 +- harness/trajectory_replay/webarena_replay.py | 13 +- harness/trajectory_replay/webarena_report.py | 37 ++-- .../webarena_report_check.py | 5 +- harness/trajectory_replay/webarena_results.py | 15 +- harness/trajectory_replay/webarena_review.py | 6 +- harness/web_fetch/__main__.py | 2 +- harness/web_fetch/accounts.py | 3 +- harness/web_fetch/adjudication_evidence.py | 26 +-- harness/web_fetch/anomaly_review.py | 6 +- harness/web_fetch/artifact_auth.py | 16 +- harness/web_fetch/artifacts.py | 25 +-- harness/web_fetch/assembly.py | 24 ++- harness/web_fetch/benchmark_runner.py | 25 +-- harness/web_fetch/browser_observations.py | 12 +- harness/web_fetch/call_boundary.py | 8 +- harness/web_fetch/certification.py | 45 ++--- harness/web_fetch/cli.py | 26 ++- harness/web_fetch/comparison_audit.py | 21 +- harness/web_fetch/config.py | 5 +- harness/web_fetch/dataset_assembly.py | 11 +- harness/web_fetch/dynamic_cohort.py | 11 +- harness/web_fetch/evidence.py | 28 +-- harness/web_fetch/failure_diagnosis.py | 7 +- harness/web_fetch/headless_runner.py | 25 ++- harness/web_fetch/local_audit.py | 17 +- harness/web_fetch/local_execution_contract.py | 30 +-- harness/web_fetch/local_execution_legacy.py | 8 +- harness/web_fetch/local_revision.py | 75 +++---- harness/web_fetch/methods.py | 14 +- harness/web_fetch/models.py | 1 - harness/web_fetch/native_projection.py | 10 +- harness/web_fetch/native_routes.py | 51 ++--- harness/web_fetch/network_dns.py | 31 ++- harness/web_fetch/network_preflight.py | 8 +- harness/web_fetch/providers/__init__.py | 3 +- harness/web_fetch/providers/base.py | 37 ++-- harness/web_fetch/providers/cloudflare.py | 25 ++- harness/web_fetch/providers/cowagent.py | 8 +- harness/web_fetch/providers/exa.py | 10 +- harness/web_fetch/providers/firecrawl.py | 11 +- harness/web_fetch/providers/fixture.py | 5 +- harness/web_fetch/providers/headless.py | 36 ++-- .../web_fetch/providers/headless_worker.py | 31 +-- harness/web_fetch/providers/http.py | 5 +- harness/web_fetch/providers/http_client.py | 3 +- harness/web_fetch/providers/lexmount.py | 8 +- harness/web_fetch/providers/moli.py | 36 ++-- .../web_fetch/providers/subprocess_jsonl.py | 19 +- harness/web_fetch/providers/tavily.py | 10 +- harness/web_fetch/publication.py | 10 +- harness/web_fetch/publication_check.py | 20 +- harness/web_fetch/publication_privacy.py | 16 +- harness/web_fetch/published_snapshot.py | 28 +-- harness/web_fetch/reference_freeze.py | 34 ++-- harness/web_fetch/reference_inputs.py | 20 +- harness/web_fetch/reference_lifecycle.py | 22 +-- harness/web_fetch/reference_revision.py | 19 +- harness/web_fetch/refresh_assembly.py | 15 +- harness/web_fetch/render_evidence.py | 3 +- harness/web_fetch/report.py | 185 +++++++++--------- harness/web_fetch/report_build.py | 175 ++++++++--------- harness/web_fetch/report_capabilities.py | 15 +- harness/web_fetch/report_changes.py | 8 +- harness/web_fetch/report_check.py | 65 +++--- harness/web_fetch/report_components.py | 24 +-- harness/web_fetch/report_diagnostic.py | 24 +-- harness/web_fetch/report_explainer.py | 51 +++-- harness/web_fetch/report_i18n.py | 37 +--- harness/web_fetch/report_interfaces.py | 34 ++-- harness/web_fetch/report_methods.py | 30 ++- harness/web_fetch/report_package.py | 53 ++--- harness/web_fetch/report_pricing.py | 3 +- harness/web_fetch/report_publish.py | 19 +- harness/web_fetch/report_quality.py | 4 +- harness/web_fetch/report_reading.py | 8 +- harness/web_fetch/report_resources.py | 10 +- harness/web_fetch/report_version.py | 5 +- harness/web_fetch/rescoring.py | 33 ++-- harness/web_fetch/resource_cohort.py | 16 +- harness/web_fetch/result_ledger.py | 12 +- harness/web_fetch/result_review.py | 38 ++-- harness/web_fetch/retrieval_hints.py | 10 +- harness/web_fetch/review_execution.py | 19 +- harness/web_fetch/risk_control.py | 9 +- harness/web_fetch/runner.py | 60 +++--- harness/web_fetch/safety.py | 6 +- harness/web_fetch/scoring_revision.py | 12 +- harness/web_fetch/source_capture.py | 21 +- harness/web_fetch/speed_cohort.py | 6 +- harness/web_fetch/storage.py | 10 +- harness/web_fetch/task_outputs.py | 15 +- harness/web_fetch/title_quality.py | 3 +- harness/web_fetch/wcxb_latency.py | 31 ++- harness/web_fetch/wcxb_publication.py | 21 +- harness/web_fetch/wcxb_report_check.py | 28 ++- harness/web_fetch/wcxb_scoring.py | 3 +- harness/web_fetch/webmainbench_assembly.py | 5 +- .../web_fetch/webmainbench_case_correction.py | 18 +- .../web_fetch/webmainbench_certification.py | 7 +- harness/web_fetch/webmainbench_correction.py | 13 +- harness/web_fetch/webmainbench_diagnostics.py | 44 +++-- harness/web_fetch/webmainbench_evidence.py | 6 +- .../webmainbench_latency_correction.py | 15 +- .../webmainbench_latency_diagnostic.py | 37 ++-- .../web_fetch/webmainbench_report_check.py | 11 +- harness/web_fetch/webmainbench_runner.py | 12 +- harness/web_fetch/webmainbench_scoring.py | 6 +- harness/web_fetch/workers/cowagent_worker.py | 5 +- harness/web_fetch/workers/crawl4ai_worker.py | 7 +- .../workers/langchain_webbase_worker.py | 1 + harness/web_search/adapters.py | 6 +- harness/web_search/comparison.py | 14 +- harness/web_search/core.py | 14 +- harness/web_search/delivery.py | 7 +- harness/web_search/diagnostics.py | 3 +- harness/web_search/example.py | 5 +- harness/web_search/messages.py | 1 + harness/web_search/pages.py | 3 +- harness/web_search/profiles.py | 14 +- harness/web_search/publication.py | 38 ++-- harness/web_search/relevance.py | 50 ++--- harness/web_search/report.py | 109 ++++++----- harness/web_search/review.py | 36 ++-- harness/web_search/revisions.py | 14 +- harness/web_search/runner.py | 29 +-- harness/web_search/visual.py | 20 +- pyproject.toml | 5 +- scripts/apply_webmainbench_case_correction.py | 3 +- scripts/audit_initial_references.py | 7 +- scripts/build_native_dns.py | 4 +- scripts/build_report.py | 21 +- scripts/check_diagnostic_report.py | 4 +- scripts/check_track_naming.py | 3 +- scripts/finalize_reference_checks.py | 2 +- scripts/run_certified_webmainbench_full.py | 3 +- scripts/run_webmainbench_benchmark.py | 3 +- scripts/score_webmainbench_results.py | 3 +- scripts/select_wcxb_sample.py | 3 +- tests/fixtures/native-route-baseline.json | 72 +++---- 195 files changed, 1729 insertions(+), 1691 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 237700ba7..1a3a996d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,10 @@ Keep original-language task text, source quotations and multilingual test fixtur of shared renderers: read them from the selected profile and bound evidence. Published historical reports retain their own content and provenance. +Use absolute Python imports and keep ordinary imports at the top of the module. +Keep an import local when loading an optional dependency, preparing a runtime +module, avoiding a circular import or preserving test isolation. + Test both accepted and rejected scenarios. Fix scoring rules rather than editing saved verdicts. Refactoring must preserve historical measurements and certificates. Write documentation in English. Keep the [Chinese README](docs/README.zh-CN.md) aligned when setup or usage changes; other docs do not need duplicate translations. Preserve original task text, quotations and frozen acceptance inputs. Reports follow their track's language settings. diff --git a/Makefile b/Makefile index 5d877bd24..f2456e2ce 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ naming-check: runtime-check .PHONY: lint-check lint-check: runtime-check - $(TEST_PY) -m ruff check --config 'include=["*.py"]' --select F . + $(TEST_PY) -m ruff check --config 'include=["*.py"]' --select F,TID252 . $(TEST_PY) -m ruff check test-python: runtime-check diff --git a/harness/browser_use/__init__.py b/harness/browser_use/__init__.py index e1ff81886..404250d98 100644 --- a/harness/browser_use/__init__.py +++ b/harness/browser_use/__init__.py @@ -1,5 +1,5 @@ """Autonomous browser-use evaluation built on WebArena-Verified.""" -from .selection import PILOT_SCHEMA, PilotTask, load_pilot +from harness.browser_use.selection import PILOT_SCHEMA, PilotTask, load_pilot __all__ = ["PILOT_SCHEMA", "PilotTask", "load_pilot"] diff --git a/harness/browser_use/failure_cause_review.py b/harness/browser_use/failure_cause_review.py index 03e26cfc4..a4c9e3d1a 100644 --- a/harness/browser_use/failure_cause_review.py +++ b/harness/browser_use/failure_cause_review.py @@ -7,6 +7,10 @@ from pathlib import Path from typing import Any +from lexbench_webfetch.paths import contained_path + +from harness.web_fetch.safety import unlinked_path + LAYERED_REVIEW_SCHEMA = "lex_browser_use_layered_failure_cause_review/2" DEFAULT_REVIEW_PATH = ( Path(__file__).parents[2] / "research/browser-use-failure-cause-review.json" @@ -19,9 +23,7 @@ def sha256(path: Path) -> str | None: def review_path(profile: dict[str, Any], *, authority_root: Path | None = None) -> Path: """Resolve an explicitly bound external review or a repository-relative review.""" - from lexbench_webfetch.paths import contained_path - from harness.web_fetch.safety import unlinked_path value = (profile.get("failure_cause_review") or {}).get("source_path") if value is None: value = "research/browser-use-failure-cause-review.json" diff --git a/harness/browser_use/report.py b/harness/browser_use/report.py index 50659d48f..2f6bc3833 100644 --- a/harness/browser_use/report.py +++ b/harness/browser_use/report.py @@ -18,26 +18,20 @@ from pathlib import Path, PurePosixPath from typing import Any -from harness.record_io import load_jsonl -from harness.report_languages import write_language_contract -from harness.reporting.assets import bound_json_asset -from harness.reporting.templates import template_identity -from harness.web_fetch.report_style import STYLE as SHARED_STYLE - -from .failure_cause_review import ( +from harness.browser_use.failure_cause_review import ( LAYERED_REVIEW_SCHEMA, extract_failure_cause_review_cohort, load_failure_cause_review, localized_cause, validate_failure_cause_review, ) -from .failure_cause_review import ( +from harness.browser_use.failure_cause_review import ( review_path as failure_review_path, ) -from .failure_cause_review import ( +from harness.browser_use.failure_cause_review import ( sha256 as file_sha256, ) -from .publication import ( +from harness.browser_use.publication import ( evaluator_summary, evidence_metadata, read_public_json, @@ -47,7 +41,23 @@ trajectory, trajectory_label, ) -from .selection import PilotSelection, load_pilot, report_selection_path, validate_against_official +from harness.browser_use.selection import ( + PilotSelection, + load_pilot, + report_selection_path, + validate_against_official, +) +from harness.record_io import load_jsonl +from harness.report_languages import ( + add_language_argument, + includes_english, + primary_report_language, + select_report_language, + write_language_contract, +) +from harness.reporting.assets import bound_json_asset +from harness.reporting.templates import template_identity +from harness.web_fetch.report_style import STYLE as SHARED_STYLE ENGINE_LABELS = {"chromium": "Chromium", "moli": "Moli"} MODEL_LABELS = {"gpt-5.6-sol": "GPT-5.6 Sol"} @@ -62,7 +72,6 @@ RESEARCH_ROOT = Path(__file__).parents[2] / "research" - def _finding_text(diagnosis: dict[str, Any], field: str) -> str: value = diagnosis.get(field, {}) return str(value.get("zh") or value.get("en") or "") if isinstance(value, dict) else str(value) @@ -2256,7 +2265,9 @@ def _trajectory_replay_control( """Explain the selected replay comparison without inheriting a past article.""" if bridge is None: return "" - from .report_i18n import bilingual_text + + # Report assets are loaded only when rendering, not for CLI help. + from harness.browser_use.report_i18n import bilingual_text def text(zh, en): return Safe(bilingual_text(zh, en)) @@ -2338,7 +2349,7 @@ def _article_body( automatic_failure_counts: Counter[str], trajectory_replay_bridge: dict[str, Any] | None, ) -> str: - from .report_i18n import bilingual_text + from harness.browser_use.report_i18n import bilingual_text text = bilingual_text task_count = len(tasks) @@ -2750,7 +2761,6 @@ def _load_fix_log(profile: dict[str, Any]) -> dict[str, Any]: return load_fix_log(profile, sanitize_value=sanitize) - def _write_fix_log(stage: Path, model: dict[str, Any], *, english: bool = False) -> None: """Render both views from one manifest; screenshots never imply pixel change.""" lang = "en" if english else "zh" @@ -2776,7 +2786,7 @@ def link(identifier: str, label: str) -> str: def save(name: str, title: str, body: str) -> None: page = _page(title=title, active="fixes", body=body, depth=1) if english: - from .report_i18n import localize_html + from harness.browser_use.report_i18n import localize_html page = localize_html(page, PurePosixPath(f"fixes/{name}")) (folder / name).write_text(page, encoding="utf-8") @@ -2847,9 +2857,8 @@ def save(name: str, title: str, body: str) -> None: def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_path: Path, include_english: bool = False, language: str | None = None, profile_path: Path | None = None) -> Path: - from harness.report_languages import includes_english, select_report_language + from harness.browser_use.report_i18n import bilingual_text - from .report_i18n import bilingual_text include_english = includes_english(language, include_english) if profile_path is None: raise ValueError("An explicit report profile is required; pass --profile") @@ -3984,7 +3993,7 @@ def translated(zh, en): ) _write_fix_log(stage, fix_log) if include_english: - from .report_i18n import write_english_report + from harness.browser_use.report_i18n import write_english_report write_english_report(stage) _write_fix_log(stage, fix_log, english=True) @@ -4031,7 +4040,6 @@ def main() -> None: type=Path, default=None, ) - from harness.report_languages import add_language_argument, primary_report_language add_language_argument(parser) parser.add_argument("--include-english", action="store_true", help="Generate both editions; --language selects the homepage") args = parser.parse_args() diff --git a/harness/browser_use/report_check.py b/harness/browser_use/report_check.py index 118bbf66f..35d860307 100644 --- a/harness/browser_use/report_check.py +++ b/harness/browser_use/report_check.py @@ -9,6 +9,7 @@ from collections import Counter from pathlib import Path +from bs4 import BeautifulSoup from lexbench_webfetch.paths import contained_path from harness.browser_use.failure_cause_review import ( @@ -39,6 +40,7 @@ from harness.reporting.assets import bound_json_asset from harness.reporting.presentation import visible_text as _visible_text from harness.reporting.templates import template_identity +from harness.reporting.versions import check_build_version from harness.web_fetch.safety import reject_evidence_symlinks FORBIDDEN_READER_TERMS = ( @@ -310,7 +312,6 @@ def _check_public_privacy(report_dir: Path) -> None: def check_methodology_evidence(content: str, tasks: list[dict], environment: dict) -> None: """Require displayed deployment and evaluator facts to match the selected evidence.""" - from bs4 import BeautifulSoup soup = BeautifulSoup(content, "html.parser") expected = { @@ -478,7 +479,6 @@ def check( raise ValueError("report is missing one or more required report pages") selection = load_pilot(selection_path) audit = json.loads(audit_path.read_text(encoding="utf-8")) - from harness.reporting.versions import check_build_version check_build_version(report_dir, audit) if audit.get("profile_sha256") and audit.get("template") != template_identity("browser-use"): raise ValueError("report bypasses the maintained Browser Use template") diff --git a/harness/browser_use/runner.py b/harness/browser_use/runner.py index 42523ca2e..fb489c975 100644 --- a/harness/browser_use/runner.py +++ b/harness/browser_use/runner.py @@ -21,6 +21,8 @@ from webarena_verified import WebArenaVerified from webarena_verified.environments.container.manager import ContainerManager +from harness.browser_use.model import OpenAIResponsesModel +from harness.browser_use.selection import PilotTask, load_pilot, validate_against_official from harness.container_isolation import require_linux_cgroup_v2 from harness.environments.browser import ( ENGINE_PORTS, @@ -52,14 +54,12 @@ site_runtime_compatibility, verify_session_viewport, ) +from harness.record_io import load_jsonl from harness.resource_accounting import ( DockerStatsResources, ResourceSampler, ) -from .model import OpenAIResponsesModel -from .selection import PilotTask, load_pilot, validate_against_official - ENGINE_IMAGES = configured_browser_images() SCHEMA = "lex_browser_use_attempt/1" @@ -70,14 +70,6 @@ # it only keeps environment preparation from aborting a formal cohort. - - - - - - - - def _load_repo_env(repo_root: Path) -> dict[str, str]: """Load repository defaults, with explicit process overrides for automation.""" env_file = repo_root / ".env" @@ -92,16 +84,6 @@ def _load_repo_env(repo_root: Path) -> dict[str, str]: return values - - - - - - - - - - def _limit_cowagent_transport_retries(executor_class: Any) -> None: """Limit CowAgent API errors to the initial request plus one retry.""" if getattr(executor_class, "_lex_browser_eval_retry_limit", False): @@ -127,45 +109,6 @@ def limited( executor_class._lex_browser_eval_retry_limit = True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def _system_prompt() -> str: return """You are the autonomous web agent under evaluation. Use only the browser tool. Complete the objective from the supplied start URLs. Do not ask for help, download files, or @@ -234,8 +177,6 @@ def _parse_agent_response(text: str) -> dict[str, Any] | str: return repaired if isinstance(repaired, dict) else text - - def _run_attempt( # noqa: PLR0913 *, lock_path: Path, @@ -484,7 +425,6 @@ def _load_completed_attempts( if not path.exists(): return {} completed: dict[tuple[str, str, int, int], dict[str, Any]] = {} - from harness.record_io import load_jsonl for record_number, row in enumerate(load_jsonl(path, skip_blank=True), 1): model = str(row.get("model") or expected_model or "") diff --git a/harness/browser_use/selection.py b/harness/browser_use/selection.py index b98a225cf..bb2261c18 100644 --- a/harness/browser_use/selection.py +++ b/harness/browser_use/selection.py @@ -11,6 +11,10 @@ from pathlib import Path from typing import Any +from lexbench_webfetch.paths import contained_path + +from harness.web_fetch.safety import unlinked_path + PILOT_SCHEMA = "lex_webarena_verified_pilot/1" ALLOWED_SITES = frozenset({"shopping", "shopping_admin", "reddit", "gitlab"}) TIER_COUNTS = {"easy": 15, "medium": 30, "hard": 45} @@ -331,9 +335,7 @@ def report_selection_path( profile: dict, explicit: Path | None = None, *, authority_root: Path | None = None ) -> Path: """Use the same frozen selection in report construction and packaging.""" - from lexbench_webfetch.paths import contained_path - from harness.web_fetch.safety import unlinked_path binding = profile.get("task_selection") or {} value = explicit or binding.get("path") or "evaluation/browser-use/webarena-verified-pilot.json" path = Path(value) diff --git a/harness/environments/webarena/cli.py b/harness/environments/webarena/cli.py index 99c43cc60..cf6507e0c 100644 --- a/harness/environments/webarena/cli.py +++ b/harness/environments/webarena/cli.py @@ -10,9 +10,9 @@ from webarena_verified.environments.setup import setup_init from webarena_verified.types.task import WebArenaSite -from .config import SITES, RuntimeConfig -from .manager import EnvironmentManager -from .map_archives import verify_archives +from harness.environments.webarena.config import SITES, RuntimeConfig +from harness.environments.webarena.manager import EnvironmentManager +from harness.environments.webarena.map_archives import verify_archives def _write_receipt(path: Path | None, payload: dict[str, Any]) -> None: diff --git a/harness/environments/webarena/config.py b/harness/environments/webarena/config.py index a0a016cdb..d55e90b9c 100644 --- a/harness/environments/webarena/config.py +++ b/harness/environments/webarena/config.py @@ -12,6 +12,8 @@ from typing import Any from urllib.parse import urlparse +from harness.web_fetch.safety import unlinked_path, write_new_file + SITES = ("shopping_admin", "shopping", "reddit", "gitlab", "map", "wikipedia") SITE_TOKENS = {site: "__" + site.upper() + "__" for site in SITES} WEBARENA_VERIFIED_VERSION = "1.2.3" @@ -151,7 +153,6 @@ def main_image_lock(argv=None) -> None: parser = argparse.ArgumentParser() parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) - from harness.web_fetch.safety import unlinked_path args.output = unlinked_path(args.output) if args.output.exists(): @@ -165,7 +166,6 @@ def main_image_lock(argv=None) -> None: for site, reference in official_image_references().items() }, } - from harness.web_fetch.safety import write_new_file write_new_file( args.output, (json.dumps(payload, indent=2, ensure_ascii=False) + "\n").encode("utf-8") diff --git a/harness/environments/webarena/manager.py b/harness/environments/webarena/manager.py index 65de486ea..75ed8b797 100644 --- a/harness/environments/webarena/manager.py +++ b/harness/environments/webarena/manager.py @@ -14,8 +14,8 @@ from webarena_verified.environments.container.manager import ContainerManager from webarena_verified.types.task import WebArenaSite -from .backend import DockerBackend -from .config import RuntimeConfig, SiteConfig, normalized_architecture +from harness.environments.webarena.backend import DockerBackend +from harness.environments.webarena.config import RuntimeConfig, SiteConfig, normalized_architecture STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) diff --git a/harness/environments/webarena/map_setup.py b/harness/environments/webarena/map_setup.py index 8e19888b3..25ae943d5 100644 --- a/harness/environments/webarena/map_setup.py +++ b/harness/environments/webarena/map_setup.py @@ -26,6 +26,8 @@ from urllib.parse import urlencode from urllib.request import Request, urlopen +from harness.web_fetch.safety import unlinked_path, write_new_file + GIB = 1024**3 MAPPINGS = { "nominatim_volumes.tar": { @@ -556,7 +558,6 @@ def map_check_main(argv=None): parser.add_argument("--url", required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) - from harness.web_fetch.safety import unlinked_path, write_new_file output = unlinked_path(args.output) if output.exists(): diff --git a/harness/environments/webarena/rainbond.py b/harness/environments/webarena/rainbond.py index 0b514fde8..d2abf068e 100644 --- a/harness/environments/webarena/rainbond.py +++ b/harness/environments/webarena/rainbond.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse import fcntl import hashlib import json @@ -1079,10 +1080,6 @@ def release_slice(config_path: Path, output_dir: Path, lease_id: str) -> dict[st def pool_main(argv=None) -> None: - import argparse - import json - from pathlib import Path - parser = argparse.ArgumentParser() parser.add_argument("--config", type=Path, required=True) subparsers = parser.add_subparsers(dest="command", required=True) diff --git a/harness/lexbench/report.py b/harness/lexbench/report.py index c43198eaf..e855f8b4e 100644 --- a/harness/lexbench/report.py +++ b/harness/lexbench/report.py @@ -11,17 +11,19 @@ import shutil import statistics -ROOT = pathlib.Path(os.environ.get("BROWSER_EVAL_DATA_HOME", pathlib.Path.home() / ".browser-eval")) +from lexbench_webfetch.paths import contained_path, safe_identifier + +from harness.lexbench import charts, reproduction +from harness.lexbench.charts import combine_comparisons, render_chart +from harness.lexbench.resource_evidence import load_matrix +from harness.lexbench.resource_summary import summarize as summarize_linux_profiles +from harness.lexbench.resource_summary import validate_summary as validate_linux_summary from harness.record_io import load_jsonl from harness.report_languages import english_notice, generated_languages, write_language_contract from harness.resource_calibration import duration_calibration -from lexbench_webfetch.paths import contained_path, safe_identifier -from . import charts, reproduction -from .charts import combine_comparisons, render_chart -from .resource_evidence import load_matrix -from .resource_summary import summarize as summarize_linux_profiles -from .resource_summary import validate_summary as validate_linux_summary +ROOT = pathlib.Path(os.environ.get("BROWSER_EVAL_DATA_HOME", pathlib.Path.home() / ".browser-eval")) + REPOSITORY_ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/harness/lexbench/resource_evidence.py b/harness/lexbench/resource_evidence.py index ce1c09e65..e0273954a 100644 --- a/harness/lexbench/resource_evidence.py +++ b/harness/lexbench/resource_evidence.py @@ -9,10 +9,10 @@ import statistics from pathlib import Path -from .resource_matrix import verify_calls +from lexbench_webfetch.paths import contained_path +from harness.lexbench.resource_matrix import verify_calls from harness.record_io import load_jsonl -from lexbench_webfetch.paths import contained_path def digest(path: Path) -> str: diff --git a/harness/lexbench/resource_profiles.py b/harness/lexbench/resource_profiles.py index f65314647..962ab2c97 100644 --- a/harness/lexbench/resource_profiles.py +++ b/harness/lexbench/resource_profiles.py @@ -18,9 +18,10 @@ import subprocess from pathlib import Path +from lexbench_webfetch.paths import contained_path, safe_identifier + from harness.lexbench.resource_summary import validate_population from harness.record_io import load_jsonl -from lexbench_webfetch.paths import contained_path, safe_identifier def sha256(path: Path) -> str: diff --git a/harness/lexbench/resource_summary.py b/harness/lexbench/resource_summary.py index 47530515f..1db9d37ef 100644 --- a/harness/lexbench/resource_summary.py +++ b/harness/lexbench/resource_summary.py @@ -9,11 +9,11 @@ import statistics from pathlib import Path -from harness.record_io import load_jsonl -from harness.resource_calibration import duration_calibration from lexbench_webfetch.paths import contained_path -from .resource_evidence import digest as sha256 +from harness.lexbench.resource_evidence import digest as sha256 +from harness.record_io import load_jsonl +from harness.resource_calibration import duration_calibration def stats(values: list[float]) -> dict: diff --git a/harness/lexbench/verification.py b/harness/lexbench/verification.py index f0db83c0b..b68ac8e0c 100644 --- a/harness/lexbench/verification.py +++ b/harness/lexbench/verification.py @@ -12,11 +12,12 @@ import subprocess import tempfile +from lexbench_webfetch.paths import contained_path, safe_identifier + from harness.lexbench.charts import combine_comparisons, render_chart from harness.lexbench.resource_evidence import validate_matrix from harness.lexbench.resource_summary import validate_summary from harness.report_languages import check_language_contract, generated_languages -from lexbench_webfetch.paths import contained_path, safe_identifier from harness.web_fetch.safety import reject_evidence_symlinks diff --git a/harness/record_io.py b/harness/record_io.py index 7c078f757..bbde73baf 100644 --- a/harness/record_io.py +++ b/harness/record_io.py @@ -4,7 +4,6 @@ from pathlib import Path - def file_sha256(path): """Hash file bytes without loading the entire artifact into memory. diff --git a/harness/report_languages.py b/harness/report_languages.py index 9d7448287..13913c848 100644 --- a/harness/report_languages.py +++ b/harness/report_languages.py @@ -7,6 +7,9 @@ from pathlib import Path from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from dotenv import dotenv_values + + class ReportLanguageError(ValueError): """Invalid report-language configuration, before evaluation starts.""" @@ -21,7 +24,6 @@ def configured_report_language( *, supported: tuple[str, ...] = SUPPORTED_REPORT_LANGUAGES ) -> str: """Return the environment-selected language, or the public English default.""" - from dotenv import dotenv_values configured = os.environ.get(REPORT_LANGUAGE_ENV) if configured is None: diff --git a/harness/reporting/assets.py b/harness/reporting/assets.py index 6b35ae249..ee7f5381a 100644 --- a/harness/reporting/assets.py +++ b/harness/reporting/assets.py @@ -1,4 +1,6 @@ """Materialize bound report assets from a Git-external source tree.""" + +import argparse import hashlib import json import os @@ -9,6 +11,8 @@ from pathlib import Path from lexbench_webfetch.paths import contained_path + +from harness.reporting.packs import read_packed_file from harness.web_fetch.safety import unlinked_path @@ -22,7 +26,6 @@ def authored_main(raw: bytes) -> str: def restore_bound_content(export, binding, destination): """Recover declared authored fields from an immutable, verified export.""" - from harness.reporting.packs import read_packed_file source = binding['source'] if source['recovery']['kind'] != 'html-main-json-v1': @@ -74,8 +77,6 @@ def read_file(name): def restore_main(): - import argparse - parser = argparse.ArgumentParser(description='Restore authored content from a pinned export') parser.add_argument('--binding', type=Path, required=True) parser.add_argument('--export', type=Path, required=True) diff --git a/harness/reporting/packs.py b/harness/reporting/packs.py index 632620a98..9278b51e9 100644 --- a/harness/reporting/packs.py +++ b/harness/reporting/packs.py @@ -5,13 +5,21 @@ """ from __future__ import annotations +import base64 import gzip import hashlib import json +import mmap +import sys import tempfile +from array import array +from bisect import bisect_right +from collections import OrderedDict from collections.abc import Iterable from pathlib import Path +from fastcdc import fastcdc + class ByteBlockStore: """Deduplicate ordered blocks without retaining the report payload in RAM.""" @@ -102,16 +110,12 @@ def flush(): return catalog, index - def write_compact_packs(entries, destination, *, architecture=None, pack_limit=16*1024*1024): """Content-defined deduplication with a compact, hash-bound file catalog. File hashes remain authoritative. Catalog IDs address immutable byte ranges; they do not duplicate millions of individual block hashes in the browser. """ - from array import array - from fastcdc import fastcdc - import sys destination.mkdir(parents=True, exist_ok=True) ids, files, packs, starts = {}, {}, [], [] @@ -192,7 +196,6 @@ def __len__(self): class _CompactBlocks: def __init__(self, sizes, packs, starts): - from array import array self.sizes, self.packs, self.starts = sizes, packs, starts self.offsets = array('I') offset, pack = 0, 0 @@ -211,7 +214,6 @@ def __len__(self): return len(self.sizes) def __getitem__(self, key): - from bisect import bisect_right if type(key) is not int or not 0 <= key < len(self): raise KeyError(key) pack = bisect_right(self.starts, key)-1 @@ -228,8 +230,6 @@ def for_pack(self, name): def read_packed_manifest(read): """Normalize both maintained wire encodings without expanding the catalog.""" - from array import array - import sys manifest = json.loads(read('manifest.json')) if manifest.get('schema_version') == 2: return manifest @@ -276,7 +276,6 @@ def read_packed_manifest(read): def reconstruct_packed_file(manifest,name,pack): - import base64 entry = manifest['files'][name] if 'inline_base64' in entry: raw = base64.b64decode(entry['inline_base64'],validate=True) @@ -302,7 +301,6 @@ class DecompressedPackCache: the selected byte range immediately, rather than retain mapped views. """ def __init__(self, read, *, max_maps=16): - from collections import OrderedDict if max_maps < 1: raise ValueError('Mapped pack cache must have a positive capacity') self.read, self.max_maps = read, max_maps @@ -313,7 +311,6 @@ def __enter__(self): return self def __call__(self, name): - import mmap if name in self.maps: self.maps.move_to_end(name) return self.maps[name] diff --git a/harness/reporting/publication.py b/harness/reporting/publication.py index 96dbf4193..fbb47b308 100644 --- a/harness/reporting/publication.py +++ b/harness/reporting/publication.py @@ -10,6 +10,7 @@ import stat import tempfile import urllib.parse +import uuid import zipfile from collections.abc import Callable, Sequence from contextlib import contextmanager @@ -18,6 +19,7 @@ from urllib.error import HTTPError from lexbench_webfetch.paths import contained_path + from harness.web_fetch.safety import evidence_tree_identity, reject_evidence_symlinks @@ -382,7 +384,6 @@ def pending(self, identity: dict, *, allow_create: bool) -> tuple[dict, bool]: return {**pending, 'key': key}, True if not allow_create: raise ValueError('no bound site or matching pending creation operation') - import uuid pending = {**identity, 'key': self.prefix + uuid.uuid4().hex} try: self.writer(self.journal, json_bytes(pending)) diff --git a/harness/reporting/versions.py b/harness/reporting/versions.py index 10991f26c..0e1239c2a 100644 --- a/harness/reporting/versions.py +++ b/harness/reporting/versions.py @@ -10,6 +10,8 @@ from pathlib import Path from urllib.parse import urljoin, urlsplit +from bs4 import BeautifulSoup + from harness.web_fetch import __version__ FILENAME = "browser-eval-version.json" @@ -87,7 +89,6 @@ def bind_build_version(root): def check_build_version(root, manifest): """Keep legacy reports readable; verify every declared build and label.""" - from bs4 import BeautifulSoup binding = manifest.get("browser_eval") path = root / FILENAME diff --git a/harness/reporting/visual.py b/harness/reporting/visual.py index 374a2440a..cfe216dea 100644 --- a/harness/reporting/visual.py +++ b/harness/reporting/visual.py @@ -16,6 +16,7 @@ from urllib.parse import unquote, urlsplit from lexbench_webfetch.paths import contained_path + from harness.web_fetch.safety import reject_evidence_symlinks diff --git a/harness/trajectory_replay/__init__.py b/harness/trajectory_replay/__init__.py index 5d8e0824d..e2d4ca124 100644 --- a/harness/trajectory_replay/__init__.py +++ b/harness/trajectory_replay/__init__.py @@ -1,5 +1,5 @@ """Reproducible browser-engine evaluation with isolated resource accounting.""" -from .suite import BrowserTask, TaskSuite, load_suite +from harness.trajectory_replay.suite import BrowserTask, TaskSuite, load_suite __all__ = ["BrowserTask", "TaskSuite", "load_suite"] diff --git a/harness/trajectory_replay/browser_use_seeds.py b/harness/trajectory_replay/browser_use_seeds.py index ae3f52a79..4a9d9d9f4 100644 --- a/harness/trajectory_replay/browser_use_seeds.py +++ b/harness/trajectory_replay/browser_use_seeds.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import Any +from harness.record_io import load_jsonl + SCHEMA = "lex_browser_use_replay_seeds/2" _REF_ACTIONS = {"click", "fill", "select"} @@ -212,7 +214,6 @@ def _authoritative_rows(root: Path) -> tuple[dict[str, Any], list[dict[str, Any] if not model: raise ValueError(f"evidence manifest does not identify a model: {root}") grouped: dict[tuple[str, int], list[dict[str, Any]]] = defaultdict(list) - from harness.record_io import load_jsonl for row in load_jsonl(root / "attempts.jsonl", skip_blank=True): grouped[(str(row["engine"]), int(row["task_id"]))].append(row) diff --git a/harness/trajectory_replay/catalog_action_audit.py b/harness/trajectory_replay/catalog_action_audit.py index 47cecc52c..a3214e134 100644 --- a/harness/trajectory_replay/catalog_action_audit.py +++ b/harness/trajectory_replay/catalog_action_audit.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse import json import subprocess from pathlib import Path @@ -9,6 +10,7 @@ from harness.trajectory_replay.catalog_contracts import file_sha256, load_action_overrides from harness.trajectory_replay.public_webarena.suite import load_suite from harness.trajectory_replay.webarena_replay import _json_sha256 +from harness.web_fetch.safety import unlinked_path, write_new_file AUDIT_SCHEMA = "lex_webarena_replay_action_override_audit/1" @@ -166,12 +168,6 @@ def _finding_modes(audit: dict) -> dict[int, dict[int, str]]: def main_audit(argv=None) -> None: - import argparse - import json - from pathlib import Path - - from harness.web_fetch.safety import unlinked_path, write_new_file - parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--catalog", type=Path, required=True) parser.add_argument("--portable-suite", type=Path, required=True) diff --git a/harness/trajectory_replay/catalog_comparison.py b/harness/trajectory_replay/catalog_comparison.py index be50730eb..5c8548770 100644 --- a/harness/trajectory_replay/catalog_comparison.py +++ b/harness/trajectory_replay/catalog_comparison.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse import hashlib import json from pathlib import Path @@ -25,6 +26,7 @@ ) from harness.trajectory_replay.webarena_corrections import apply_correction, load_corrections from harness.trajectory_replay.webarena_replay import _json_sha256 +from harness.web_fetch.safety import unlinked_path, write_new_file RESULTS_SCHEMA = "lex_webarena_catalog_comparison_results/1" DATASET_SCHEMA = "lex_webarena_catalog_comparison_dataset/1" @@ -376,13 +378,7 @@ def build_catalog_comparison_results( def inputs_main(argv=None) -> None: - import argparse - import hashlib - import json - from pathlib import Path - from harness.trajectory_replay.catalog_contracts import file_sha256 - from harness.web_fetch.safety import unlinked_path, write_new_file parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", type=Path, required=True) diff --git a/harness/trajectory_replay/cli.py b/harness/trajectory_replay/cli.py index d19d2fc9f..ebf349aaf 100644 --- a/harness/trajectory_replay/cli.py +++ b/harness/trajectory_replay/cli.py @@ -5,7 +5,7 @@ import argparse from pathlib import Path -from .runner import run_evaluation +from harness.trajectory_replay.runner import run_evaluation def main() -> None: diff --git a/harness/trajectory_replay/diagnostics.py b/harness/trajectory_replay/diagnostics.py index f90e97b5d..4dd97c588 100644 --- a/harness/trajectory_replay/diagnostics.py +++ b/harness/trajectory_replay/diagnostics.py @@ -18,6 +18,11 @@ from typing import Any from harness.record_io import same_json_record +from harness.trajectory_replay.webarena_corrections import ( + apply_correction, + evaluate_corrected_contract, + load_corrections, +) from harness.web_fetch.safety import unlinked_path, write_new_file @@ -780,11 +785,6 @@ def audit_saved_correction( "after_catalog_sha256": _diagnostic_digest(after_catalog), } from harness.trajectory_replay.remote_runtime import load_remote_sites, remote_benchmark - from harness.trajectory_replay.webarena_corrections import ( - apply_correction, - evaluate_corrected_contract, - load_corrections, - ) benchmark = remote_benchmark(load_remote_sites(site_map)) response = json.loads((task_dir / "actual_response.json").read_text()) diff --git a/harness/trajectory_replay/evaluator_transition.py b/harness/trajectory_replay/evaluator_transition.py index 95056daaa..bab579d9b 100644 --- a/harness/trajectory_replay/evaluator_transition.py +++ b/harness/trajectory_replay/evaluator_transition.py @@ -15,11 +15,14 @@ from harness.trajectory_replay.webarena_corrections import ( apply_correction, evaluate_corrected_contract, + load_corrections, ) from harness.trajectory_replay.webarena_runtime_source import ( WEBARENA_VERIFIED_FORK_URL, WEBARENA_VERIFIED_REVISION, + verify_webarena_runtime_source, ) +from harness.web_fetch.safety import unlinked_path, write_new_file TRANSITION_SCHEMA = "lex_webarena_evaluator_transition/1" RECEIPT_SCHEMA = "lex_webarena_evaluator_transition_receipt/3" @@ -158,7 +161,6 @@ def reevaluate_pass( corrections: Any, ) -> tuple[dict[str, Any], dict[str, Any]]: """Return the current evaluation and a binding for the immutable old evidence.""" - from harness.web_fetch.safety import unlinked_path task_dir = unlinked_path(task_dir) @@ -308,9 +310,6 @@ def _old_evaluation_identity(value: str) -> tuple[int, str, str]: def main(argv=None) -> None: from harness.trajectory_replay.catalog_contracts import verify_physical_evidence from harness.trajectory_replay.remote_runtime import remote_benchmark - from harness.trajectory_replay.webarena_corrections import load_corrections - from harness.trajectory_replay.webarena_runtime_source import verify_webarena_runtime_source - from harness.web_fetch.safety import unlinked_path, write_new_file parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--transition", type=Path, required=True) diff --git a/harness/trajectory_replay/executor.py b/harness/trajectory_replay/executor.py index 55318d1eb..ea10eaf5d 100644 --- a/harness/trajectory_replay/executor.py +++ b/harness/trajectory_replay/executor.py @@ -7,7 +7,7 @@ from contextlib import nullcontext from typing import Any -from .suite import validate_selector +from harness.trajectory_replay.suite import validate_selector def _body_text(page: Any, timeout: int, limit: int) -> str: diff --git a/harness/trajectory_replay/public_webarena/__init__.py b/harness/trajectory_replay/public_webarena/__init__.py index 5d8e0824d..c7f6e2a46 100644 --- a/harness/trajectory_replay/public_webarena/__init__.py +++ b/harness/trajectory_replay/public_webarena/__init__.py @@ -1,5 +1,5 @@ """Reproducible browser-engine evaluation with isolated resource accounting.""" -from .suite import BrowserTask, TaskSuite, load_suite +from harness.trajectory_replay.public_webarena.suite import BrowserTask, TaskSuite, load_suite __all__ = ["BrowserTask", "TaskSuite", "load_suite"] diff --git a/harness/trajectory_replay/public_webarena/cli.py b/harness/trajectory_replay/public_webarena/cli.py index 04c7b0ae1..cbe80bb8c 100644 --- a/harness/trajectory_replay/public_webarena/cli.py +++ b/harness/trajectory_replay/public_webarena/cli.py @@ -10,9 +10,12 @@ from contextlib import contextmanager, nullcontext from pathlib import Path -from .config import configure_browser_docker_host, load_runtime_env -from .qualification import task_ids_from_scope -from .runner import run_evaluation +from harness.trajectory_replay.public_webarena.config import ( + configure_browser_docker_host, + load_runtime_env, +) +from harness.trajectory_replay.public_webarena.qualification import task_ids_from_scope +from harness.trajectory_replay.public_webarena.runner import run_evaluation @contextmanager @@ -38,8 +41,6 @@ def qualification_lock( handle.close() - - def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--suite", default="/work/tasks.v2.json") diff --git a/harness/trajectory_replay/public_webarena/diagnostics.py b/harness/trajectory_replay/public_webarena/diagnostics.py index 50dec1781..d95cdbf2d 100644 --- a/harness/trajectory_replay/public_webarena/diagnostics.py +++ b/harness/trajectory_replay/public_webarena/diagnostics.py @@ -24,11 +24,14 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from copy import deepcopy from pathlib import Path +from types import SimpleNamespace from typing import Any from urllib.parse import parse_qs, urlsplit from lexbench_webfetch.paths import contained_path, safe_identifier +from harness.environments import browser as browser_runtime +from harness.environments.webarena import har as browser_har from harness.environments.webarena.config import SITE_TOKENS from harness.record_io import parse_jsonl from harness.trajectory_replay.diagnostics import ( @@ -37,11 +40,13 @@ diagnostic_output_path, write_diagnostic, ) +from harness.trajectory_replay.public_webarena.qualification import ( + build_probe, + task_contract_sha256, +) +from harness.trajectory_replay.public_webarena.suite import SUPPORTED_ACTIONS, load_suite from harness.web_fetch.safety import unlinked_path -from .qualification import build_probe, task_contract_sha256 -from .suite import SUPPORTED_ACTIONS, load_suite - def classify_failure(error: str, trace: list[dict[str, Any]]) -> dict[str, Any]: """Extract a deterministic failure signature from the raw trace.""" @@ -1294,17 +1299,6 @@ def site_urls(config: dict[str, Any], sites: tuple[str, ...]) -> dict[str, str]: def runtime_smoke_main(argv=None) -> None: - from harness.environments.browser import ( - ENGINE_PORTS, - cleanup_browser_runtime, - configured_browser_images, - published_endpoint, - start_browser_container, - validate_resource_controls, - wait_cdp, - ) - from harness.web_fetch.safety import unlinked_path - parser = argparse.ArgumentParser() parser.add_argument("--config", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) @@ -1314,7 +1308,7 @@ def runtime_smoke_main(argv=None) -> None: action="store_true", help="authenticate with credentials_env before starting diagnostic HAR capture", ) - parser.add_argument("--engine", action="append", choices=tuple(ENGINE_PORTS)) + parser.add_argument("--engine", action="append", choices=tuple(browser_runtime.ENGINE_PORTS)) parser.add_argument("--cpus", type=float, default=None) args = parser.parse_args(argv) config_path = unlinked_path(args.config) @@ -1327,21 +1321,18 @@ def runtime_smoke_main(argv=None) -> None: urls = site_urls(config, sites) if args.cpus is not None and (not math.isfinite(args.cpus) or args.cpus <= 0): raise ValueError("--cpus must be finite and positive") - from types import SimpleNamespace import docker from playwright.sync_api import sync_playwright - from harness.environments.webarena.har import CdpHarRecorder - - from .webarena_environment import WebArenaEnvironment + from harness.trajectory_replay.public_webarena.webarena_environment import WebArenaEnvironment output.mkdir(parents=True) client = docker.from_env() if args.cpus is not None: if args.cpus <= 0: raise ValueError("--cpus must be positive") - validate_resource_controls(client.info()) + browser_runtime.validate_resource_controls(client.info()) receipt: dict[str, Any] = { "schema": "lex_webarena_runtime_smoke/1", "diagnostic_only": True, @@ -1353,8 +1344,8 @@ def runtime_smoke_main(argv=None) -> None: "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "engines": {}, } - images = configured_browser_images() - for engine in tuple(dict.fromkeys(args.engine or ENGINE_PORTS)): + images = browser_runtime.configured_browser_images() + for engine in tuple(dict.fromkeys(args.engine or browser_runtime.ENGINE_PORTS)): client.images.get(images[engine]) run_id = f"runtime-smoke-{uuid.uuid4().hex[:10]}" network = client.networks.create( @@ -1367,9 +1358,9 @@ def runtime_smoke_main(argv=None) -> None: playwright = None try: playwright = sync_playwright().start() - container = start_browser_container( + container = browser_runtime.start_browser_container( client, - image=images[engine], cdp_port=ENGINE_PORTS[engine], + image=images[engine], cdp_port=browser_runtime.ENGINE_PORTS[engine], engine=engine, run_id=run_id, cpus=args.cpus, @@ -1378,11 +1369,11 @@ def runtime_smoke_main(argv=None) -> None: ) containers[engine] = container container.reload() - endpoint = published_endpoint( - container, engine=engine, cdp_port=ENGINE_PORTS[engine] + endpoint = browser_runtime.published_endpoint( + container, engine=engine, cdp_port=browser_runtime.ENGINE_PORTS[engine] ) browser = playwright.chromium.connect_over_cdp( - wait_cdp(endpoint, timeout_seconds=30, container=container) + browser_runtime.wait_cdp(endpoint, timeout_seconds=30, container=container) ) browsers[engine] = browser page = browser.contexts[0].pages[0] @@ -1399,7 +1390,7 @@ def runtime_smoke_main(argv=None) -> None: ), page, ) - recorder = CdpHarRecorder(page) + recorder = browser_har.CdpHarRecorder(page) pages = [] for site, url in urls.items(): response = page.goto(url, wait_until="domcontentloaded", timeout=60_000) @@ -1423,7 +1414,9 @@ def runtime_smoke_main(argv=None) -> None: "har": {**har, "sha256": _sha256(har_path)}, } finally: - cleanup_browser_runtime(browsers, playwright, containers, {engine: network}) + browser_runtime.cleanup_browser_runtime( + browsers, playwright, containers, {engine: network} + ) if config_path.read_bytes() != frozen_config: raise ValueError("Runtime configuration changed during smoke") (output / "receipt.json").write_text( diff --git a/harness/trajectory_replay/public_webarena/evaluators.py b/harness/trajectory_replay/public_webarena/evaluators.py index ea53c00c3..c2260e56a 100644 --- a/harness/trajectory_replay/public_webarena/evaluators.py +++ b/harness/trajectory_replay/public_webarena/evaluators.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any, Protocol -from .executor import judge_outputs -from .suite import BrowserTask +from harness.trajectory_replay.public_webarena.executor import judge_outputs +from harness.trajectory_replay.public_webarena.suite import BrowserTask @dataclass(frozen=True) diff --git a/harness/trajectory_replay/public_webarena/executor.py b/harness/trajectory_replay/public_webarena/executor.py index c6dcec76b..6641d90a0 100644 --- a/harness/trajectory_replay/public_webarena/executor.py +++ b/harness/trajectory_replay/public_webarena/executor.py @@ -10,8 +10,8 @@ from typing import Any from urllib.parse import urlsplit -from .suite import validate_selector -from .targets import locator_for +from harness.trajectory_replay.public_webarena.suite import validate_selector +from harness.trajectory_replay.public_webarena.targets import locator_for TABLE_COLUMN_EXPRESSION = r"""(table, options) => { const normalize = value => (value || '').replace(/\s+/g, ' ').trim(); diff --git a/harness/trajectory_replay/public_webarena/manifest_repair.py b/harness/trajectory_replay/public_webarena/manifest_repair.py index 1578c49ad..e6acc4062 100644 --- a/harness/trajectory_replay/public_webarena/manifest_repair.py +++ b/harness/trajectory_replay/public_webarena/manifest_repair.py @@ -14,13 +14,12 @@ from lexbench_webfetch.paths import contained_path, safe_identifier from harness.record_io import load_jsonl, same_json_record +from harness.trajectory_replay.public_webarena.diagnostics import diagnose_summary +from harness.trajectory_replay.public_webarena.metrics import summarize_attempts +from harness.trajectory_replay.public_webarena.report import write_latest_report +from harness.trajectory_replay.public_webarena.result_audit import audit_result_set from harness.web_fetch.safety import unlinked_path -from .diagnostics import diagnose_summary -from .metrics import summarize_attempts -from .report import write_latest_report -from .result_audit import audit_result_set - def _read_rows(path: Path) -> list[dict[str, Any]]: return load_jsonl(path, skip_blank=True) diff --git a/harness/trajectory_replay/public_webarena/publication.py b/harness/trajectory_replay/public_webarena/publication.py index da0f44933..9c86d7990 100644 --- a/harness/trajectory_replay/public_webarena/publication.py +++ b/harness/trajectory_replay/public_webarena/publication.py @@ -17,6 +17,12 @@ from lexbench_webfetch.paths import contained_path +from harness.reporting.publication import archive_inventory +from harness.trajectory_replay.public_webarena.result_audit import audit_result_set +from harness.web_fetch.publication_target import publication_document, require_publication_version +from harness.web_fetch.report_publish import PublishConfig, publish +from harness.web_fetch.safety import unlinked_path + REPORT_STEM = "webarena-trajectory-replay-latest" PRIVATE_IPV4 = re.compile( @@ -102,7 +108,7 @@ def _read_json(path: Path) -> dict: def _report_files(formal_root: Path, summary: dict) -> list[Path]: - from .report import report_file_names + from harness.trajectory_replay.public_webarena.report import report_file_names files = [] for name in sorted(report_file_names(summary)): @@ -272,7 +278,7 @@ def build(formal_root: Path, archive: Path, attestation: Path, *, authority: Pat def check_report_bundle(formal_root: Path) -> dict[str, int]: - from .report import METRICS, PAGES + from harness.trajectory_replay.public_webarena.report import METRICS, PAGES reports = formal_root / "reports" pages_checked = 0 @@ -335,8 +341,7 @@ def check_report_bundle(formal_root: Path) -> dict[str, int]: def stage_report(formal_root: Path, destination: Path) -> None: - from .report import write_latest_report - from .result_audit import audit_result_set + from harness.trajectory_replay.public_webarena.report import write_latest_report if destination.exists() or destination.is_symlink(): raise ValueError("report staging destination already exists") @@ -425,13 +430,6 @@ def publish_current( transport=None, ) -> dict: """Submit only the profile-bound public replay package through the shared publisher.""" - from harness.reporting.publication import archive_inventory - from harness.web_fetch.publication_target import ( - publication_document, - require_publication_version, - ) - from harness.web_fetch.report_publish import PublishConfig, publish - from harness.web_fetch.safety import unlinked_path paths = [ unlinked_path(path) diff --git a/harness/trajectory_replay/public_webarena/qualification.py b/harness/trajectory_replay/public_webarena/qualification.py index abe6e21db..92e17e8c5 100644 --- a/harness/trajectory_replay/public_webarena/qualification.py +++ b/harness/trajectory_replay/public_webarena/qualification.py @@ -23,12 +23,16 @@ from lexbench_webfetch.paths import contained_path, safe_identifier from harness.record_io import load_jsonl, same_json_record +from harness.trajectory_replay.public_webarena.har import HAR_CREATOR_NAME, HAR_CREATOR_VERSION +from harness.trajectory_replay.public_webarena.suite import ( + TRAJECTORY_SCHEMA, + TaskSuite, + load_suite, + validate_selector, +) +from harness.trajectory_replay.public_webarena.targets import validate_target from harness.web_fetch.safety import unlinked_path, write_new_file -from .har import HAR_CREATOR_NAME, HAR_CREATOR_VERSION -from .suite import TRAJECTORY_SCHEMA, TaskSuite, load_suite, validate_selector -from .targets import validate_target - _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") QUALIFICATION_SCHEMA = "lex_browser_use_qualification/2" _PASS_STATUS_FIELDS = ( @@ -727,7 +731,6 @@ def task_ids_from_scope( return task_ids - def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -1201,7 +1204,6 @@ def run_batch( ) - ALLOWED_DISPOSITIONS = { "eligible", "needs_evidence_revalidation", @@ -1300,7 +1302,6 @@ def main_subset(argv=None) -> None: _write_qualified_suite(args.output, result) - def build_requalification_suite( suite: dict[str, Any], scope: dict[str, Any], *, predecessor_sha256: str ) -> dict[str, Any]: @@ -1435,7 +1436,6 @@ def build_probe(suite: dict[str, Any], proposal: dict[str, Any]) -> dict[str, An return result - def _proposal_digest(value: Any) -> str: encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() return hashlib.sha256(encoded).hexdigest() @@ -1519,7 +1519,6 @@ def _default_skip_reason(decision: dict[str, Any]) -> str: ) - def qualification_from_applied_suite(suite: dict[str, Any]) -> dict[str, Any]: """Rehydrate adjudication state without requiring old private evidence paths.""" decisions: list[dict[str, Any]] = [] @@ -1557,7 +1556,6 @@ def qualification_from_applied_suite(suite: dict[str, Any]) -> dict[str, Any]: } - def _collect_validated_run( canonical_tasks: dict[str, dict[str, Any]], decisions: dict[str, dict[str, Any]], @@ -1637,7 +1635,6 @@ def _collect_validated_run( candidate["passing_paths"].append(attempt_path) - def _overlay_two_pass_candidates( canonical_tasks: dict[str, dict[str, Any]], decisions: dict[str, dict[str, Any]], @@ -1692,7 +1689,6 @@ def _overlay_two_pass_candidates( return overlaid - def finalize_adjudication( suite: dict[str, Any], qualification: dict[str, Any], @@ -1756,7 +1752,6 @@ def finalize_adjudication( return result - def main_finalize_adjudication(argv=None) -> None: parser = argparse.ArgumentParser( description="Finalize complete two-pass qualification decisions", diff --git a/harness/trajectory_replay/public_webarena/release.py b/harness/trajectory_replay/public_webarena/release.py index 41f1995ac..c27b833c6 100644 --- a/harness/trajectory_replay/public_webarena/release.py +++ b/harness/trajectory_replay/public_webarena/release.py @@ -16,8 +16,11 @@ json_bytes, recover_binding, ) - -from .publication import build, check_report_bundle, stage_report +from harness.trajectory_replay.public_webarena.publication import ( + build, + check_report_bundle, + stage_report, +) def _run_json(command: list[str], project_root: Path) -> dict: diff --git a/harness/trajectory_replay/public_webarena/report.py b/harness/trajectory_replay/public_webarena/report.py index ef909b536..1e49dfe9e 100644 --- a/harness/trajectory_replay/public_webarena/report.py +++ b/harness/trajectory_replay/public_webarena/report.py @@ -12,9 +12,11 @@ from urllib.parse import urlparse from harness.distribution import installed_data_path - -from .publication import REPORT_STEM, public_projection -from .result_audit import ResultAuditError, audit_result_set +from harness.trajectory_replay.public_webarena.publication import REPORT_STEM, public_projection +from harness.trajectory_replay.public_webarena.result_audit import ( + ResultAuditError, + audit_result_set, +) ROOT = Path(__file__).resolve().parents[3] PAGES = tuple( diff --git a/harness/trajectory_replay/public_webarena/response_builder.py b/harness/trajectory_replay/public_webarena/response_builder.py index c9a423a0c..079ee54ba 100644 --- a/harness/trajectory_replay/public_webarena/response_builder.py +++ b/harness/trajectory_replay/public_webarena/response_builder.py @@ -6,7 +6,7 @@ import re from typing import Any -from .suite import BrowserTask +from harness.trajectory_replay.public_webarena.suite import BrowserTask def _resolve(value: Any, outputs: dict[str, str]) -> Any: diff --git a/harness/trajectory_replay/public_webarena/runner.py b/harness/trajectory_replay/public_webarena/runner.py index 871d1e710..01159a278 100644 --- a/harness/trajectory_replay/public_webarena/runner.py +++ b/harness/trajectory_replay/public_webarena/runner.py @@ -35,17 +35,27 @@ ResourceSampler, delta_counters, ) - -from .diagnostics import diagnose_summary -from .evaluators import EvaluatorRegistry, TaskEvidence, write_agent_response -from .executor import execute_steps -from .har import CdpHarRecorder -from .metrics import summarize_attempts -from .qualification import task_contract_sha256 -from .report import write_latest_report -from .response_builder import build_agent_response -from .suite import BrowserTask, load_suite, validate_ground_truth_evidence -from .webarena_environment import WebArenaEnvironment, wall_clock_timeout +from harness.trajectory_replay.public_webarena.diagnostics import diagnose_summary +from harness.trajectory_replay.public_webarena.evaluators import ( + EvaluatorRegistry, + TaskEvidence, + write_agent_response, +) +from harness.trajectory_replay.public_webarena.executor import execute_steps +from harness.trajectory_replay.public_webarena.har import CdpHarRecorder +from harness.trajectory_replay.public_webarena.metrics import summarize_attempts +from harness.trajectory_replay.public_webarena.qualification import task_contract_sha256 +from harness.trajectory_replay.public_webarena.report import write_latest_report +from harness.trajectory_replay.public_webarena.response_builder import build_agent_response +from harness.trajectory_replay.public_webarena.suite import ( + BrowserTask, + load_suite, + validate_ground_truth_evidence, +) +from harness.trajectory_replay.public_webarena.webarena_environment import ( + WebArenaEnvironment, + wall_clock_timeout, +) ENGINE_IMAGES = dict(DEFAULT_BROWSER_IMAGES) ENCODING = tiktoken.get_encoding("o200k_base") @@ -58,10 +68,6 @@ def _remaining_task_budget_ms(deadline: float, *, phase: str) -> int: return remaining_ms - - - - def _http_origin(url: str) -> str | None: parts = urllib.parse.urlsplit(url) if parts.scheme not in {"http", "https"} or not parts.netloc: @@ -256,8 +262,6 @@ def _capture_final_page_evidence(page: Page, artifact_dir: Path) -> dict[str, An } - - def _prepare_page( page: Page, visited_origins: set[str], @@ -523,8 +527,6 @@ def _environment_failure_row( } - - class _QuietFixtureHandler(SimpleHTTPRequestHandler): def log_message(self, format: str, *args: Any) -> None: return @@ -634,10 +636,6 @@ def _environment(client: Any, containers: dict[str, Any]) -> dict[str, Any]: } - - - - def _write_run_artifacts( output: Path, rows: list[dict[str, Any]], summary: dict[str, Any], manifest: dict[str, Any] ) -> None: diff --git a/harness/trajectory_replay/public_webarena/suite.py b/harness/trajectory_replay/public_webarena/suite.py index 533c267ff..f020b9035 100644 --- a/harness/trajectory_replay/public_webarena/suite.py +++ b/harness/trajectory_replay/public_webarena/suite.py @@ -11,7 +11,8 @@ from typing import Any from urllib.parse import urlparse -from .targets import validate_target +from harness.trajectory_replay.public_webarena.targets import validate_target +from harness.web_fetch.safety import unlinked_path, write_new_file SCHEMA = "lex_browser_use_tasks/4" LEGACY_SCHEMA = "lex_browser_use_tasks/3" @@ -392,7 +393,10 @@ def _tertile_bounds(counts: list[int]) -> tuple[int, int]: def build_suite(catalog: dict[str, Any], trace_dir: Path) -> dict[str, Any]: - from .webarena import import_playwright_trace, validate_trace_catalog + from harness.trajectory_replay.public_webarena.webarena import ( + import_playwright_trace, + validate_trace_catalog, + ) validate_trace_catalog(catalog) imported: list[tuple[dict[str, Any], dict[str, Any]]] = [] @@ -512,7 +516,6 @@ def main_build_suite(argv=None) -> None: parser.add_argument("--trace-dir", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) - from harness.web_fetch.safety import unlinked_path, write_new_file args.output = unlinked_path(args.output) if args.output.exists(): diff --git a/harness/trajectory_replay/public_webarena/webarena.py b/harness/trajectory_replay/public_webarena/webarena.py index 77ab1cbe9..382a01ae8 100644 --- a/harness/trajectory_replay/public_webarena/webarena.py +++ b/harness/trajectory_replay/public_webarena/webarena.py @@ -21,7 +21,8 @@ from typing import Any from urllib.parse import urlparse -from .suite import TRAJECTORY_SCHEMA +from harness.trajectory_replay.public_webarena.suite import TRAJECTORY_SCHEMA +from harness.web_fetch.safety import unlinked_path, write_new_file HUMAN_TRAJECTORY_FOLDER_ID = "1NrN_sawtYK2V_uHnmmS8ugmGIKUAsPgt" HUMAN_TRAJECTORY_FOLDER_URL = "https://drive.google.com/drive/folders/" + HUMAN_TRAJECTORY_FOLDER_ID @@ -633,7 +634,6 @@ def main_audit_traces(argv=None) -> None: parser.add_argument("--trace-dir", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) - from harness.web_fetch.safety import unlinked_path, write_new_file output = unlinked_path(args.output) if output.exists(): @@ -715,12 +715,6 @@ def main_audit_traces(argv=None) -> None: def main_catalog(argv=None) -> None: - import argparse - import json - from pathlib import Path - - from harness.web_fetch.safety import unlinked_path, write_new_file - parser = argparse.ArgumentParser() parser.add_argument("--folder-html", type=Path, required=True) parser.add_argument("--webarena-dataset", type=Path, required=True) diff --git a/harness/trajectory_replay/public_webarena/webarena_environment.py b/harness/trajectory_replay/public_webarena/webarena_environment.py index 4cdb0aff5..914d94c99 100644 --- a/harness/trajectory_replay/public_webarena/webarena_environment.py +++ b/harness/trajectory_replay/public_webarena/webarena_environment.py @@ -24,8 +24,7 @@ from harness.environments.webarena.backend import MAGENTO_INIT_TIMEOUT_SECONDS, DockerBackend from harness.environments.webarena.manager import EnvironmentManager from harness.environments.webarena.rainbond import RainbondSlicePool - -from .suite import BrowserTask +from harness.trajectory_replay.public_webarena.suite import BrowserTask STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) LEGACY_PORT_SITES = { diff --git a/harness/trajectory_replay/remote_runtime.py b/harness/trajectory_replay/remote_runtime.py index a4f2ee268..c64e2e2f6 100644 --- a/harness/trajectory_replay/remote_runtime.py +++ b/harness/trajectory_replay/remote_runtime.py @@ -12,6 +12,7 @@ import json import os import re +import time from pathlib import Path from typing import Any from urllib.parse import urlsplit, urlunsplit @@ -1014,11 +1015,6 @@ def main_revise(argv=None) -> None: def main_prepared_reset(argv=None) -> None: - import os - import time - - from harness.web_fetch.safety import unlinked_path, write_new_file - parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--method-id", required=True) parser.add_argument("--app-id", required=True, type=int) diff --git a/harness/trajectory_replay/report_compaction.py b/harness/trajectory_replay/report_compaction.py index 5a0af9276..8750b2549 100644 --- a/harness/trajectory_replay/report_compaction.py +++ b/harness/trajectory_replay/report_compaction.py @@ -12,6 +12,8 @@ from pathlib import Path from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from harness.reporting.visual import browser_executable, local_report_server + SHARD_COUNT = 16 @@ -222,7 +224,6 @@ def check_compact_navigation(report: Path) -> None: """Exercise the exported query routes and evidence, in every saved language.""" from playwright.sync_api import sync_playwright - from harness.reporting.visual import browser_executable, local_report_server manifest = json.loads((report / 'report-manifest.json').read_text()) rows = json.loads((report / 'data/results.json').read_text()) diff --git a/harness/trajectory_replay/report_languages.py b/harness/trajectory_replay/report_languages.py index 97a3c6d91..3f4923c41 100644 --- a/harness/trajectory_replay/report_languages.py +++ b/harness/trajectory_replay/report_languages.py @@ -2,10 +2,10 @@ from pathlib import Path +from harness.report_languages import check_primary_report_language, generated_languages -def report_languages(root: Path) -> tuple[str, ...]: - from harness.report_languages import check_primary_report_language, generated_languages +def report_languages(root: Path) -> tuple[str, ...]: check_primary_report_language(root) return generated_languages(root) diff --git a/harness/trajectory_replay/retrieval_acceptance.py b/harness/trajectory_replay/retrieval_acceptance.py index 73a9a2ecb..8adee738a 100644 --- a/harness/trajectory_replay/retrieval_acceptance.py +++ b/harness/trajectory_replay/retrieval_acceptance.py @@ -1069,7 +1069,7 @@ def verify_attempt( evaluation = loaded["evaluation"] if evaluation.get("task_id") != attempt["task_id"]: raise ValueError("official evaluator task differs from retrieval attempt") - from .webarena_dataset import _validate_evaluation_provenance + from harness.trajectory_replay.webarena_dataset import _validate_evaluation_provenance _validate_evaluation_provenance(evaluation, attempt, expected_data_checksum) accepted = ( diff --git a/harness/trajectory_replay/runner.py b/harness/trajectory_replay/runner.py index 8f71e5701..13928960b 100644 --- a/harness/trajectory_replay/runner.py +++ b/harness/trajectory_replay/runner.py @@ -29,21 +29,17 @@ wait_cdp, ) from harness.resource_accounting import DockerStatsResources, ResourceSampler, delta_counters - -from .diagnostics import diagnose_summary -from .executor import execute_steps, judge_outputs, validate_outputs -from .metrics import summarize_attempts -from .report import write_latest_report -from .suite import BrowserTask, load_suite, validate_ground_truth_evidence +from harness.trajectory_replay.diagnostics import diagnose_summary +from harness.trajectory_replay.executor import execute_steps, judge_outputs, validate_outputs +from harness.trajectory_replay.metrics import summarize_attempts +from harness.trajectory_replay.report import write_latest_report +from harness.trajectory_replay.suite import BrowserTask, load_suite, validate_ground_truth_evidence +from harness.web_fetch.safety import unlinked_path ENGINE_IMAGES = configured_browser_images() ENCODING = tiktoken.get_encoding("o200k_base") - - - - def _http_origin(url: str) -> str | None: parts = urllib.parse.urlsplit(url) if parts.scheme not in {"http", "https"} or not parts.netloc: @@ -120,8 +116,6 @@ def _resource_result( return result - - def _prepare_page(page: Page, visited_origins: set[str], *, settle_ms: int) -> Page: """Reset task state and replace only the renderer page, keeping the browser/context warm.""" context = page.context @@ -228,8 +222,6 @@ def record_origin(frame: Any) -> None: } - - class _QuietFixtureHandler(SimpleHTTPRequestHandler): def log_message(self, format: str, *args: Any) -> None: return @@ -299,8 +291,6 @@ def _environment(client: Any, containers: dict[str, Any]) -> dict[str, Any]: } - - def _write_run_artifacts( output: Path, rows: list[dict[str, Any]], summary: dict[str, Any], manifest: dict[str, Any] ) -> None: @@ -337,7 +327,6 @@ def run_evaluation( ) -> dict[str, Any]: if max_retries < 0: raise ValueError("max_retries must be non-negative") - from harness.web_fetch.safety import unlinked_path output = unlinked_path(Path(output_dir)) if output.exists(): raise ValueError("Output must be a new directory; existing evidence is immutable") diff --git a/harness/trajectory_replay/webarena_corrections.py b/harness/trajectory_replay/webarena_corrections.py index 74eb27272..99bbb2478 100644 --- a/harness/trajectory_replay/webarena_corrections.py +++ b/harness/trajectory_replay/webarena_corrections.py @@ -8,10 +8,13 @@ from __future__ import annotations +import argparse import copy import hashlib import json +import os import re +import tempfile from collections.abc import Mapping from dataclasses import dataclass from importlib.resources import files @@ -19,14 +22,19 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any -if TYPE_CHECKING: - from webarena_verified.types.eval import TaskEvalResult - from webarena_verified.types.task import WebArenaVerifiedTask +from lexbench_webfetch.paths import contained_path +from harness.trajectory_replay.retrieval_acceptance import validate_contract from harness.trajectory_replay.webarena_runtime_source import ( WEBARENA_VERIFIED_FORK_URL, verify_webarena_runtime_source, ) +from harness.web_fetch.safety import unlinked_path, write_new_file + +if TYPE_CHECKING: + from webarena_verified.types.eval import TaskEvalResult + from webarena_verified.types.task import WebArenaVerifiedTask + SCHEMA = "lex_webarena_replay_task_corrections/1" DEFAULT_CORRECTIONS_PATH = ( @@ -845,8 +853,6 @@ def check_replay_assertions( def _revision_file_sha256(path: Path) -> str: - from harness.web_fetch.safety import unlinked_path - path = unlinked_path(path) if path.is_symlink() or not path.is_file(): raise ValueError(f"missing or symlinked source: {path}") @@ -854,10 +860,6 @@ def _revision_file_sha256(path: Path) -> str: def _load_dataset_revision_bindings(path: Path | None = None) -> dict[int, dict[str, Any]]: - from lexbench_webfetch.paths import contained_path - - from harness.web_fetch.safety import unlinked_path - root = Path(__file__).resolve().parents[2] path = unlinked_path( path @@ -893,8 +895,6 @@ def revise_dataset( revision_bindings_path: Path | None = None, ) -> dict[str, Any]: from harness.environments.webarena.runtime import configure_benchmark - from harness.trajectory_replay.retrieval_acceptance import validate_contract - from harness.web_fetch.safety import unlinked_path base_path = unlinked_path(base_path) corrections_path = unlinked_path(corrections_path) @@ -1040,10 +1040,6 @@ def revise_dataset( def main_dataset_revision(argv=None) -> None: - import argparse - - from harness.web_fetch.safety import unlinked_path, write_new_file - parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", type=Path, required=True) parser.add_argument("--corrections", type=Path, required=True) @@ -1076,8 +1072,6 @@ def main_dataset_revision(argv=None) -> None: args.output.parent.mkdir(parents=True, exist_ok=True) payload = (json.dumps(result, ensure_ascii=False, indent=2) + "\n").encode("utf-8") if replacing: - import os - import tempfile with tempfile.NamedTemporaryFile(dir=args.output.parent, delete=False) as stream: staged = Path(stream.name) diff --git a/harness/trajectory_replay/webarena_dataset.py b/harness/trajectory_replay/webarena_dataset.py index edc733d6e..67959f9f2 100644 --- a/harness/trajectory_replay/webarena_dataset.py +++ b/harness/trajectory_replay/webarena_dataset.py @@ -18,6 +18,7 @@ require_current_retrieval, verify_attempt, ) +from harness.trajectory_replay.webarena_corrections import apply_correction, load_corrections from harness.trajectory_replay.webarena_plan import ( load_replay_plan, replay_calls_binding_sha256, @@ -74,10 +75,6 @@ def _effective_correction_bindings() -> dict[int, tuple[str, str, str, tuple[str from harness.environments.webarena.runtime import ( configure_benchmark, ) - from harness.trajectory_replay.webarena_corrections import ( # noqa: PLC0415 - apply_correction, - load_corrections, - ) benchmark = configure_benchmark() catalog = load_corrections() diff --git a/harness/trajectory_replay/webarena_plan.py b/harness/trajectory_replay/webarena_plan.py index 5d0ec4294..502fd393d 100644 --- a/harness/trajectory_replay/webarena_plan.py +++ b/harness/trajectory_replay/webarena_plan.py @@ -13,6 +13,7 @@ from harness.environments.webarena.runtime import ( SITE_PORTS, ) +from harness.trajectory_replay import webarena_corrections from harness.trajectory_replay.browser_use_seeds import ( _authoritative_rows, _contains_ref, @@ -329,13 +330,12 @@ def load_replay_plan(profile_path: str | Path, seed_manifest_path: str | Path) - ) from harness.webarena_state import load_profile_contracts, require_supersession - from .webarena_corrections import load_corrections state_contracts = load_profile_contracts( (profile.get("scoring") or {}).get("observed_state"), repo_root ) if state_contracts: - corrections = load_corrections() + corrections = webarena_corrections.load_corrections() for task_id, contract in state_contracts.items(): if task_id not in {int(task["task_id"]) for task in seed_tasks}: raise ValueError("state contract outside replay task selection") @@ -375,7 +375,6 @@ def load_replay_plan(profile_path: str | Path, seed_manifest_path: str | Path) - ) - def _render_observation_script(script: str, values: dict[str, Any]) -> str: """Insert JSON literals once without interpreting their contents as template tokens.""" tokens = re.compile(r"\b(?:" + "|".join(re.escape(key) for key in values) + r")\b") diff --git a/harness/trajectory_replay/webarena_replay.py b/harness/trajectory_replay/webarena_replay.py index 696c5e8af..c83fa6f6c 100644 --- a/harness/trajectory_replay/webarena_replay.py +++ b/harness/trajectory_replay/webarena_replay.py @@ -44,6 +44,13 @@ verify_session_viewport, ) from harness.resource_accounting import DockerStatsResources, ResourceSampler +from harness.trajectory_replay.retrieval_acceptance import ( + PROTOCOL, + build_actual_response, + digest, + require_current_retrieval, + verify_attempt, +) from harness.trajectory_replay.webarena_corrections import ( apply_correction, check_replay_assertions, @@ -296,8 +303,7 @@ def select_methods(plan: dict[str, Any], engines: list[str]) -> list[dict[str, A def validate_qualification_dataset(plan: dict[str, Any], path: str | Path) -> str: """Bind a formal comparison to the complete pre-qualified trajectory set.""" - from .retrieval_acceptance import digest, require_current_retrieval - from .webarena_dataset import OBSERVATION_PROTOCOL + from harness.trajectory_replay.webarena_dataset import OBSERVATION_PROTOCOL require_current_retrieval(plan) dataset_path = Path(path).expanduser().resolve() @@ -513,7 +519,6 @@ def _run_one( # noqa: PLR0913 resource_started = None recorder = CdpHarRecorder() records: list[dict[str, Any]] = [] - from .retrieval_acceptance import PROTOCOL, build_actual_response attempt_id = uuid.uuid4().hex actual_response = None @@ -1191,7 +1196,6 @@ def required_receipt_int(item: object, field: str, record: str) -> int: if item.get("process_status") == "pass" and item.get("status") != "unsupported": verify_state_attempt(item, plan["state_contracts"][task_id]) if task_id in plan.get("retrieval_task_ids", []): - from .retrieval_acceptance import verify_attempt if item.get("status") == "pass" and not ( item.get("process_status") == item.get("evaluator_status") == "pass" @@ -1360,7 +1364,6 @@ def run_replay_suite( # noqa: PLR0913 scope = "formal" if diagnostic_task_ids is None else "diagnostic" qualification_dataset_sha256 = None if scope != "diagnostic": - from .retrieval_acceptance import require_current_retrieval require_current_retrieval(plan) if scope in {"formal", "formal_correction_shard"}: diff --git a/harness/trajectory_replay/webarena_report.py b/harness/trajectory_replay/webarena_report.py index 59d0149ac..ef34a4e5c 100644 --- a/harness/trajectory_replay/webarena_report.py +++ b/harness/trajectory_replay/webarena_report.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse import hashlib import html import importlib.resources @@ -10,6 +11,7 @@ import re import shutil import statistics +import tempfile import zipfile from collections import Counter, defaultdict from pathlib import Path @@ -17,8 +19,16 @@ from urllib.parse import urlencode, urlsplit from harness.record_io import load_jsonl +from harness.report_languages import ( + add_language_argument, + includes_english, + primary_report_language, + select_report_language, +) +from harness.reporting.assets import copy_bound_assets from harness.reporting.privacy import LOCAL_PATH_RE, WEB_URL_RE from harness.reporting.templates import template_identity +from harness.reporting.versions import bind_build_version from harness.trajectory_replay.catalog_contracts import ( LEGACY_CATALOG_CONTRACTS_SCHEMA, LEGACY_CATALOG_REPORT_SCHEMA, @@ -36,12 +46,14 @@ resolve_participant_id, ) from harness.trajectory_replay.remote_runtime import load_remote_sites, remote_benchmark +from harness.trajectory_replay.report_inputs import bound_dataset_path, verified_dataset_path from harness.trajectory_replay.runtime_identity import request_bridge_identity from harness.trajectory_replay.webarena_corrections import load_corrections from harness.trajectory_replay.webarena_review import ( load_validated_failure_review, programmatic_failure, ) +from harness.web_fetch.storage import data_home # This notice describes one retained historical cohort, not trajectory replay in # general. Keying all three immutable inputs prevents a future requalified run @@ -835,8 +847,6 @@ def _public_evidence( return public - - def _detail_page( lang: str, attempt: dict[str, Any], @@ -1383,14 +1393,6 @@ def _assets(output: Path) -> None: (output / "assets/app.js").write_text(js, encoding="utf-8") - - - - - - - - def build_webarena_replay_report( *, results_path: str | Path, @@ -1404,10 +1406,6 @@ def build_webarena_replay_report( include_english: bool = False, language: str | None = None, ) -> dict[str, Any]: - from harness.report_languages import ( - includes_english, - select_report_language, - ) include_english = includes_english(language, include_english) languages = ("zh", "en") if include_english else ("zh",) profile_source = Path(profile_path) @@ -2183,7 +2181,6 @@ def build_webarena_replay_report( "measurement_notes": {lang: profile["measurement_notes"][lang] for lang in languages} if profile.get("measurement_notes") else {}, } - from harness.reporting.versions import bind_build_version manifest['browser_eval'] = bind_build_version(output) _write_json(output / "report-manifest.json", manifest) if language is not None: @@ -2655,15 +2652,6 @@ def validate_catalog_acceptance_report( def build_main(argv=None) -> None: - import argparse - import json - import tempfile - from pathlib import Path - - from harness.reporting.assets import copy_bound_assets - from harness.trajectory_replay.report_inputs import bound_dataset_path, verified_dataset_path - from harness.web_fetch.storage import data_home - parser = argparse.ArgumentParser() parser.add_argument("--results", required=True) parser.add_argument("--profile", type=Path, required=True) @@ -2687,7 +2675,6 @@ def build_main(argv=None) -> None: action="store_true", help="Explicitly generate the English report and its assets", ) - from harness.report_languages import add_language_argument, primary_report_language add_language_argument(parser) args = parser.parse_args(argv) profile = json.loads(args.profile.read_text(encoding="utf-8")) diff --git a/harness/trajectory_replay/webarena_report_check.py b/harness/trajectory_replay/webarena_report_check.py index 5919c6462..e3ef9d255 100644 --- a/harness/trajectory_replay/webarena_report_check.py +++ b/harness/trajectory_replay/webarena_report_check.py @@ -9,6 +9,7 @@ import json import math import re +from collections import Counter from pathlib import Path from urllib.parse import parse_qs, unquote, urlsplit @@ -17,6 +18,7 @@ from harness.reporting.presentation import visible_text as _visible_text from harness.reporting.privacy import LOCAL_PATH_RE, WEB_URL_RE from harness.reporting.templates import template_identity +from harness.reporting.versions import check_build_version from harness.trajectory_replay.report_languages import language_roots, report_languages from harness.web_fetch.safety import reject_evidence_symlinks @@ -455,8 +457,6 @@ def _check_result_detail_semantics(text: str, evidence: dict, status: str) -> No def _check_failure_matrix(page: str, results: list[dict], participants: list[str]) -> None: """Recompute cells independently; also verify each count's filter target.""" - from collections import Counter - from urllib.parse import parse_qs soup = BeautifulSoup(page, "html.parser") matrix = soup.select_one("[data-failure-matrix]") @@ -526,7 +526,6 @@ def check(report_dir: Path, *, profile_path: Path | None = None) -> dict[str, in reject_evidence_symlinks(report_dir) root = report_dir.resolve() manifest = json.loads((root / "report-manifest.json").read_text(encoding="utf-8")) - from harness.reporting.versions import check_build_version check_build_version(root, manifest) if (manifest.get("profile_sha256") and manifest.get("template") != template_identity("trajectory-replay")): diff --git a/harness/trajectory_replay/webarena_results.py b/harness/trajectory_replay/webarena_results.py index 4ac619db4..2e60bf8fc 100644 --- a/harness/trajectory_replay/webarena_results.py +++ b/harness/trajectory_replay/webarena_results.py @@ -8,15 +8,17 @@ from pathlib import Path from typing import Any -from .remote_runtime import ( +from harness.trajectory_replay.remote_runtime import ( COHORT_RESET_SCOPE, checked_file_sha256, input_digest, selected_task_ids, validate_remote_cohort_reset_receipt, ) -from .webarena_plan import load_replay_plan -from .webarena_replay import RECEIPT_SCHEMA, validate_qualification_dataset +from harness.trajectory_replay.retrieval_acceptance import verify_attempt +from harness.trajectory_replay.webarena_plan import load_replay_plan +from harness.trajectory_replay.webarena_replay import RECEIPT_SCHEMA, validate_qualification_dataset +from harness.web_fetch.safety import unlinked_path, write_new_file EFFECTIVE_SCHEMA = "lex_webarena_trajectory_replay_effective_suite/1" @@ -157,7 +159,6 @@ def build_effective_results( if stale: raise ValueError(f"effective replay matrix contains stale trajectories: {stale}") - from .retrieval_acceptance import verify_attempt seed_candidates = { int(task["task_id"]): next( @@ -624,12 +625,6 @@ def main_aggregate(argv: list[str] | None = None) -> None: def main_merge(argv=None) -> None: - import argparse - import json - from pathlib import Path - - from harness.web_fetch.safety import unlinked_path, write_new_file - parser = argparse.ArgumentParser() parser.add_argument("--profile", required=True) diff --git a/harness/trajectory_replay/webarena_review.py b/harness/trajectory_replay/webarena_review.py index e4e2aca00..3b62a6b03 100644 --- a/harness/trajectory_replay/webarena_review.py +++ b/harness/trajectory_replay/webarena_review.py @@ -8,7 +8,11 @@ from pathlib import Path from typing import Any -from .participants import bind_participant, participant_registry, resolve_participant_id +from harness.trajectory_replay.participants import ( + bind_participant, + participant_registry, + resolve_participant_id, +) REVIEW_SCHEMA = "lex_webarena_trajectory_replay_failure_review/1" diff --git a/harness/web_fetch/__main__.py b/harness/web_fetch/__main__.py index 2f05ddc22..e3f08735f 100644 --- a/harness/web_fetch/__main__.py +++ b/harness/web_fetch/__main__.py @@ -1,5 +1,5 @@ -from .cli import main +from harness.web_fetch.cli import main if __name__ == "__main__": main() diff --git a/harness/web_fetch/accounts.py b/harness/web_fetch/accounts.py index 2063307e9..044320161 100644 --- a/harness/web_fetch/accounts.py +++ b/harness/web_fetch/accounts.py @@ -7,8 +7,7 @@ from pathlib import Path from typing import Any - -from .methods import report_method_catalog +from harness.web_fetch.methods import report_method_catalog def account_checklist() -> list[dict[str, Any]]: diff --git a/harness/web_fetch/adjudication_evidence.py b/harness/web_fetch/adjudication_evidence.py index 14d2ad439..ef8ce8916 100644 --- a/harness/web_fetch/adjudication_evidence.py +++ b/harness/web_fetch/adjudication_evidence.py @@ -1,13 +1,18 @@ """Require inspectable task and answer evidence when a verdict overrides review.""" -import json + +import copy import hashlib +import json from dataclasses import dataclass + +from lexbench_webfetch.metrics import normalize_for_match, task_measured, task_passed from lexbench_webfetch.paths import contained_path -from lexbench_webfetch.metrics import task_passed -from .local_execution_contract import local_measurement_call_sha256 -from harness.record_io import file_sha256, canonical_json_sha256 +from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256, _same_page_url, score_attempt -from .publication_privacy import read_evidence_binding, evidence_excerpt_matches +from harness.record_io import canonical_json_sha256, file_sha256, load_jsonl +from harness.web_fetch.local_execution_contract import local_measurement_call_sha256 +from harness.web_fetch.publication_privacy import evidence_excerpt_matches, read_evidence_binding +from harness.web_fetch.result_review import review_input_identity def requires_adjudication(row): @@ -18,7 +23,6 @@ def requires_adjudication(row): def current_rule_outcomes(case, record): """Expose only declared, computed rules as structured adjudication facts.""" - from lexbench_webfetch.metrics import task_passed m = record['metrics'] outcomes = {'task_result': {'passed': task_passed(record['ok'], m)}, 'request_status': {'passed': bool(m.get('request_ok', record['ok']))}} @@ -179,7 +183,6 @@ def require_adjudication_evidence(source, dataset, records, review): def require_current_review_state(records, review): - from lexbench_webfetch.metrics import task_measured, task_passed indexed={record['attempt_id']:record for record in records} rows=review.get('attempts',[]) unreviewed=review.get('unreviewed_attempt_ids',[]) @@ -288,8 +291,6 @@ def require_retained_reviews(source, dataset, records, review): if retained: raise ValueError('Retained reviews require their original evidence') return set() - from .result_review import review_input_identity - from harness.record_io import load_jsonl receipt = json.loads(read_evidence_binding(source, binding).read_text()) if receipt.get('kind') == 'current_retained_review_dependencies': return _require_current_retained_reviews(source, dataset, records, review, receipt) @@ -350,13 +351,11 @@ def require_retained_reviews(source, dataset, records, review): def compact_retained_review_dependencies(source, dataset, records, review): """Keep actual reused opinions and decision identities, not old matrices.""" require_retained_reviews(source, dataset, records, review) - from .result_review import review_input_identity receipt = json.loads(read_evidence_binding(source, review['retained_reviews']).read_text()) if receipt.get('kind') == 'current_retained_review_dependencies': return receipt prior = json.loads(read_evidence_binding(source, receipt['prior_review']).read_text()) before = json.loads(read_evidence_binding(source, receipt['prior_dataset']).read_text()) - from harness.record_io import load_jsonl previous = {r['attempt_id']: r for r in load_jsonl(read_evidence_binding(source, receipt['prior_results']))} old_cases = {c['id']: c for c in before['cases']} cases = {c['id']: c for c in dataset['cases']} @@ -384,7 +383,6 @@ def compact_retained_review_dependencies(source, dataset, records, review): def _require_current_retained_reviews(source, dataset, records, review, receipt): - from .result_review import review_input_identity rows = receipt.get('attempts', []) dependencies = {r['attempt_id']: r for r in rows} retained = {r['attempt_id'] for r in review.get('attempts', []) @@ -444,8 +442,6 @@ def _require_current_retained_reviews(source, dataset, records, review, receipt) def score_reviewed_page_identity(source, case, record, annotation, captures, *, historical_scoring_source=None): """Apply a bound retrospective content correspondence, never a native URL.""" - from lexbench_webfetch.metrics import normalize_for_match - from lexbench_webfetch.scoring import score_attempt, _same_page_url, SCORING_SOURCE_SHA256 if (annotation.get("attempt_id") != record["attempt_id"] or annotation.get("case_sha256") != canonical_json_sha256(case) @@ -580,7 +576,6 @@ def _require_local_review(source, dataset, records, *, review_document=None, sco or len(indexed) != len(rows) or set(indexed) != set(expected)): raise ValueError("Local independent review scope differs") unresolved = 0 - from lexbench_webfetch.metrics import task_measured for identifier, row in indexed.items(): record = expected[identifier] verdict = ("unavailable" if not task_measured(record["metrics"]) else @@ -602,7 +597,6 @@ def _require_local_review(source, dataset, records, *, review_document=None, sco def _legacy_diagnostic_migrated_case(case): """Decode the exact task transition in retained legacy review receipts.""" - import copy result = copy.deepcopy(case) hints = result.pop('forbidden_any', []) if hints: diff --git a/harness/web_fetch/anomaly_review.py b/harness/web_fetch/anomaly_review.py index da96f4bfe..c1c41046d 100644 --- a/harness/web_fetch/anomaly_review.py +++ b/harness/web_fetch/anomaly_review.py @@ -3,16 +3,16 @@ This is deliberately separate from verdict review: triggers are computed only after the blind reviewer has judged the saved answer, so they cannot prime that verdict. """ -from collections import defaultdict + import hashlib import json -from pathlib import Path import re +from collections import defaultdict +from pathlib import Path from statistics import median from lexbench_webfetch.paths import contained_path - LAYERS = ('input', 'reference', 'request', 'raw', 'adapter', 'scorer', 'final_verdict') TRIGGER_VERSION = 2 _SHORT_OUTPUT = 512 diff --git a/harness/web_fetch/artifact_auth.py b/harness/web_fetch/artifact_auth.py index 08ad04bb0..1869d7528 100644 --- a/harness/web_fetch/artifact_auth.py +++ b/harness/web_fetch/artifact_auth.py @@ -1,11 +1,18 @@ """Artifact Site's user-scoped publishing credential, separate from eval secrets.""" + +import argparse +import json import os -from pathlib import Path import stat import tempfile +import time +import urllib.request import uuid +from pathlib import Path from urllib.parse import urlsplit +from harness.web_fetch.publication_target import validate_service_base + def token_path(base_url): root = Path.home() / '.config/artifact-site' @@ -94,12 +101,6 @@ def save_token(token, *, base_url, replace_existing=False): def authorize_main(argv=None): - import argparse - import json - import os - import time - import urllib.request - parser = argparse.ArgumentParser(description=__doc__) action = parser.add_mutually_exclusive_group(required=True) action.add_argument('--device-code') @@ -118,7 +119,6 @@ def authorize_main(argv=None): if backup:print('Previous credential backed up:',backup) print('Existing authorization saved to the user credential file; token not displayed.') return - from harness.web_fetch.publication_target import validate_service_base args.base_url = validate_service_base(args.base_url) if args.timeout <= 0: diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py index fe4fc2316..7b968cd10 100644 --- a/harness/web_fetch/artifacts.py +++ b/harness/web_fetch/artifacts.py @@ -7,13 +7,21 @@ from __future__ import annotations import hashlib -from harness.record_io import file_sha256 import json import shutil from pathlib import Path from typing import Any, Iterable +from lexbench_webfetch import file_acceptance from lexbench_webfetch.paths import contained_path, safe_identifier + +from harness.record_io import file_sha256 +from harness.web_fetch import native_routes +from harness.web_fetch.publication_privacy import original_evidence_size, read_evidence_binding +from harness.web_fetch.reference_lifecycle import ( + latency_correction_original_elapsed, + preserve_legacy_reference_dataset, +) from harness.web_fetch.safety import reject_evidence_symlinks PRIVATE_ARTIFACT_ROOT = "private_artifacts" @@ -22,7 +30,6 @@ def file_scoring_metadata(case, metadata: dict, source: Path) -> dict: """Recheck delivered file bytes against current rules without changing call evidence.""" - from lexbench_webfetch.file_acceptance import evaluate_files case = case if isinstance(case, dict) else vars(case) evaluation = metadata.get("file_evaluation") @@ -47,7 +54,7 @@ def file_scoring_metadata(case, metadata: dict, source: Path) -> dict: if hashlib.sha256(raw).hexdigest() != item.get("sha256"): raise ValueError("Delivered file integrity failure: " + item["artifact_path"]) payloads.append(raw) - current = evaluate_files(case, payloads) + current = file_acceptance.evaluate_files(case, payloads) if len(current["files"]) != len(delivered): raise ValueError("Delivered file evaluation changed the file count") current["files"] = [{**old, **new} for old, new in zip(delivered, current["files"], strict=True)] @@ -168,7 +175,6 @@ def verify_public_call_log(root: Path, binding: dict[str, Any], *, allow_public_ raise ValueError("Public call log output must stay under artifacts") claimed = payload["output"] if allow_public_views: - from .publication_privacy import read_evidence_binding, original_evidence_size answer = {'path': output_path.as_posix(), 'sha256': claimed['sha256']} read_evidence_binding(root, answer) size = original_evidence_size(root, answer) @@ -374,15 +380,13 @@ def verify_record_artifacts(root: Path, record: dict[str, Any], *, native_cases= if not source.is_file() or file_sha256(source) != digest: raise ValueError('Artifact differs from its result binding: ' + relative) verify_record_public_call_logs(root, record) - from harness.web_fetch.native_routes import verify_saved_native_answer - verify_saved_native_answer(root, record, cases_by_id=native_cases) + native_routes.verify_saved_native_answer(root, record, cases_by_id=native_cases) def materialize_bound_artifacts(output: Path, source_root: Path, records: list[dict]) -> None: """Verify acquisition-time authority before copying declared evidence.""" copies = {} - from .native_routes import native_cases_by_id - native_cases = native_cases_by_id(source_root) + native_cases = native_routes.native_cases_by_id(source_root) for record in records: verify_record_artifacts(source_root, record, native_cases=native_cases) for relative, digest in artifact_bindings(record): @@ -406,9 +410,7 @@ def materialize_bound_artifacts(output: Path, source_root: Path, records: list[d def materialize_reference_checks(output: Path, source: Path, batches: list[dict]) -> None: """Retain hash-bound source observations and native execution contracts.""" - from .reference_lifecycle import preserve_legacy_reference_dataset - from .native_routes import materialize_native_route_contracts - materialize_native_route_contracts(output, source, batches) + native_routes.materialize_native_route_contracts(output, source, batches) for batch in batches: preserve_legacy_reference_dataset(batch, source, output) post = batch.get('reference_checks', {}).get('postcheck', {}) @@ -489,7 +491,6 @@ def copy_cohort_evidence(source, target, dataset, records, batches, *, if include_private and (source / 'result-review.json').is_file(): allowed.add('result-review.json') if 'latency-corrections.json' in allowed: - from .reference_lifecycle import latency_correction_original_elapsed latency_correction_original_elapsed(source) native_cases = {case['id']: case for case in dataset['cases']} for record in records: diff --git a/harness/web_fetch/assembly.py b/harness/web_fetch/assembly.py index 336815728..96a08227d 100644 --- a/harness/web_fetch/assembly.py +++ b/harness/web_fetch/assembly.py @@ -1,4 +1,5 @@ """Assemble exact headless measurements, never selecting retries by score.""" + import argparse import hashlib import json @@ -8,29 +9,37 @@ from datetime import datetime, timedelta, timezone from pathlib import Path +from lexbench_webfetch.metrics import task_passed +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.scoring_regressions import ( + active_scoring_regressions, + build_correction_provenance, + valid_correction_provenance, +) + from harness.record_io import file_sha256, load_jsonl +from harness.reporting.publication import atomic_write from harness.web_fetch.artifacts import ( copy_cohort_evidence, materialize_bound_artifacts, materialize_reference_checks, materialize_task_references, ) -from lexbench_webfetch.metrics import task_passed from harness.web_fetch.publication_privacy import project_execution_manifest from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes from harness.web_fetch.reference_lifecycle import ( latency_correction_original_elapsed, reference_revision_cases, - require_verified_reference_checks, require_certifiable_evidence, + require_verified_reference_checks, requires_fresh_reference, scoring_revision_history_sha256, valid_scoring_revision, ) -from harness.web_fetch.result_review import require_result_review -from lexbench_webfetch.paths import contained_path +from harness.web_fetch.resource_cohort import validate_cohort +from harness.web_fetch.result_review import require_blind_result_review, require_result_review from harness.web_fetch.safety import reject_evidence_symlinks, unlinked_path, validate_report_ids -from lexbench_webfetch.scoring_regressions import build_correction_provenance, valid_correction_provenance +from harness.web_fetch.scoring_revision import revise from harness.web_fetch.speed_cohort import build_summary as build_speed_summary REPOSITORY = Path(__file__).resolve().parents[2] @@ -421,8 +430,6 @@ def bind_assembled_scoring_revision(output, *, source=None, historical_sources=( inherited=path.read_bytes() if inherited != (source/'scoring-revisions.json').read_bytes(): raise ValueError('Inherited scoring authority differs from declared source') - from lexbench_webfetch.scoring_regressions import active_scoring_regressions - from .scoring_revision import revise controls=active_scoring_regressions() history=json.loads(inherited) identifiers=list(dict.fromkeys(identifier for receipt in history @@ -456,7 +463,6 @@ def bind_assembled_scoring_revision(output, *, source=None, historical_sources=( dataset=json.loads((output/'dataset.json').read_text()) records=read_records(output) if historical_sources: - from .result_review import require_blind_result_review require_blind_result_review(output, dataset, records, historical_sources=historical_sources) else: require_result_review(output, dataset, records) @@ -510,7 +516,6 @@ def bind_assembled_scoring_revision(output, *, source=None, historical_sources=( (check/'scoring-revisions.json').write_text(candidate) if not valid_scoring_revision(check): raise ValueError('Assembled scoring revision would leave an invalid history') - from harness.reporting.publication import atomic_write atomic_write(path, candidate.encode()) return True @@ -1245,7 +1250,6 @@ def _assemble(run, certificates, replacements, output, additions=None, provider_ _copy_speed_bundle(speed_cohort,output/'speed-cohort',speed_dataset, speed_rows,speed_batches,excluded_providers=exclude_providers) if resource_cohort: - from .resource_cohort import validate_cohort resource_dataset, resource_rows, resource_batches = validate_cohort(resource_cohort) if {row['provider'] for row in resource_rows} & set(exclude_providers): raise ValueError('Excluded providers remain in the resource cohort') diff --git a/harness/web_fetch/benchmark_runner.py b/harness/web_fetch/benchmark_runner.py index 5126242d6..cfff58e4c 100644 --- a/harness/web_fetch/benchmark_runner.py +++ b/harness/web_fetch/benchmark_runner.py @@ -3,20 +3,25 @@ import argparse import json +import math from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from pathlib import Path -from .artifacts import materialize_bound_artifacts, materialize_task_references -from .config import ( - DEFAULT_CALL_TIMEOUT_SECONDS, load_dataset, load_provider_config, - load_runtime_env, add_failure_retry_argument, +from harness.web_fetch.artifacts import materialize_bound_artifacts, materialize_task_references +from harness.web_fetch.config import ( + DEFAULT_CALL_TIMEOUT_SECONDS, + add_failure_retry_argument, + load_dataset, + load_provider_config, + load_runtime_env, ) -from .providers import build_provider -from .reference_inputs import input_tree -from .report_build import build_report -from .runner import preflight_declared_capacity, run_evaluation -from .storage import data_home +from harness.web_fetch.network_preflight import preflight_guarded_providers +from harness.web_fetch.providers import build_provider +from harness.web_fetch.reference_inputs import input_tree +from harness.web_fetch.report_build import build_report +from harness.web_fetch.runner import preflight_declared_capacity, run_evaluation +from harness.web_fetch.storage import data_home ROOT = Path(__file__).resolve().parents[2] @@ -41,7 +46,6 @@ def main(argv=None): help='Positive per-call deadline in seconds, retained in execution evidence') add_failure_retry_argument(parser) args=parser.parse_args(argv) - import math if not math.isfinite(args.timeout) or args.timeout <= 0: parser.error('--timeout must be positive and finite') if args.provider_config and not args.provider_config.is_file(): @@ -96,7 +100,6 @@ def main(argv=None): missing=provider.validate() if missing:errors.append(f'{scope}/{name}: missing {", ".join(missing)}') if errors:parser.error('\n'.join(errors)) - from .network_preflight import preflight_guarded_providers try: network_audit = preflight_guarded_providers(selected_providers, cases) except ValueError as error: diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index a5e75556a..94060a499 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -10,8 +10,8 @@ import os import re import signal -import subprocess import socket +import subprocess import sys import tempfile import time @@ -22,8 +22,11 @@ from pathlib import Path from urllib.parse import urlsplit -from harness.reporting.publication import atomic_create, json_bytes from lexbench_webfetch.metrics import normalize_for_match +from lexbench_webfetch.paths import safe_identifier +from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 + +from harness.reporting.publication import atomic_create, json_bytes from harness.web_fetch.network_dns import ( configure_chromium_dns_profile, validate_dns_mode, @@ -31,11 +34,10 @@ from harness.web_fetch.reference_inputs import read_dataset_bytes from harness.web_fetch.reference_lifecycle import ( _resource_target_matches, + requires_fresh_reference, resource_source_scope_identity, scope_identity, - requires_fresh_reference, ) -from lexbench_webfetch.paths import safe_identifier from harness.web_fetch.source_capture import extract_html # innerText includes opacity-zero content. Descend only through ancestors of @@ -611,13 +613,11 @@ def capture_source_identity() -> dict: "data/browser-observation-contracts.json", ) root = Path(__file__).parent - from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 return {**{name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in names}, "lexbench_webfetch/scoring": SCORING_SOURCE_SHA256} - _CAPTURE_PROGRAM = ( "import json,sys; from harness.web_fetch.browser_observations import capture_with_retries; " "print(json.dumps(capture_with_retries(**json.load(sys.stdin))))" diff --git a/harness/web_fetch/call_boundary.py b/harness/web_fetch/call_boundary.py index c7bbad3c0..6bf53d44e 100644 --- a/harness/web_fetch/call_boundary.py +++ b/harness/web_fetch/call_boundary.py @@ -1,12 +1,14 @@ """One end-to-end answer deadline for every adapter, including local processing.""" + import queue import threading import time -from .models import ProviderResult -from .diagnostics import diagnostic_context, emit -from lexbench_webfetch.response_input import answer_input from lexbench_webfetch.file_acceptance import evaluate_files +from lexbench_webfetch.response_input import answer_input + +from harness.web_fetch.diagnostics import diagnostic_context, emit +from harness.web_fetch.models import ProviderResult def prepare_answer(result, case): diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py index e73d14ec1..5170d1d36 100644 --- a/harness/web_fetch/certification.py +++ b/harness/web_fetch/certification.py @@ -6,11 +6,16 @@ import argparse import hashlib import json +import math import shutil import sys from collections.abc import Callable from pathlib import Path +from lexbench_webfetch.metrics import normalize_for_match +from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 +from lexbench_webfetch.versions import verify_release + from harness.record_io import load_jsonl from harness.web_fetch import ( benchmark_runner, @@ -18,10 +23,21 @@ evidence, reference_freeze, source_capture, + webmainbench_certification, webmainbench_runner, ) -from harness.web_fetch.reference_inputs import read_dataset_bytes from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS, add_failure_retry_argument +from harness.web_fetch.reference_inputs import read_dataset_bytes +from harness.web_fetch.reference_lifecycle import ( + requires_browser_observation, + requires_fresh_reference, +) +from harness.web_fetch.retrieval_hints import native_fetch_dataset, parse_retrieval_hints +from harness.web_fetch.risk_control import main as risk_control_main +from harness.web_fetch.wcxb_dataset import validate_manifest +from harness.web_fetch.webmainbench_correction import certify_case_correction as correction_main +from harness.web_fetch.webmainbench_runner import WCXB_PROVIDERS, load_wcxb_source +from harness.web_search.runner import main as search_main ROOT = Path(__file__).resolve().parents[2] @@ -36,11 +52,6 @@ def browser_observation_cases( cannot supply the facts/reference file that the freezer consumes. Malformed or missing audit rows fail closed instead of being guessed at. """ - from lexbench_webfetch.metrics import normalize_for_match - from harness.web_fetch.reference_lifecycle import ( - requires_browser_observation, - requires_fresh_reference, - ) try: rows = json.loads((captures / "audit.json").read_text(encoding="utf-8")) @@ -109,37 +120,31 @@ def run(stage: Callable[[list[str]], None], arguments: list[str]) -> None: def main(argv: list[str] | None = None) -> None: argv = sys.argv[1:] if argv is None else argv if argv[:1] == ["web-search"]: - from harness.web_search.runner import main as search_main search_main(argv[1:], certified=True) return if argv[:1] == ["webmainbench-latency"]: - from .webmainbench_latency_correction import certify + from harness.web_fetch.webmainbench_latency_correction import certify certify(argv[1:]) return if argv[:1] == ["risk-control"]: - from harness.web_fetch.risk_control import main as risk_control_main risk_control_main(argv[1:], certified=True) return if argv[:1] == ["webmainbench-correction"]: - from harness.web_fetch.webmainbench_correction import ( - certify_case_correction as correction_main, - ) correction_main(argv[1:]) return if argv[:1] == ["webmainbench-full"]: - from harness.web_fetch.webmainbench_certification import main as webmainbench_full_main - webmainbench_full_main(argv[1:]) + webmainbench_certification.main(argv[1:]) return if argv[:1] == ["wcxb"]: run_wcxb(argv[1:]) return if argv[:1] == ["wcxb-latency"]: - from .wcxb_latency import certify + from harness.web_fetch.wcxb_latency import certify options = argv[1:] certify(options) return @@ -181,7 +186,6 @@ def main(argv: list[str] | None = None) -> None: ), ) args = parser.parse_args(argv) - import math if not math.isfinite(args.timeout) or args.timeout <= 0: parser.error("--timeout must be positive and finite") if args.provider_config and not args.provider_config.is_file(): @@ -210,7 +214,6 @@ def main(argv: list[str] | None = None) -> None: # evidence does not turn a version rerun into live source qualification. unqualified = [] - from harness.web_fetch.retrieval_hints import native_fetch_dataset, parse_retrieval_hints if args.retrieval_mode == "native": original_sha = hashlib.sha256(read_dataset_bytes(args.dataset)).hexdigest() @@ -225,18 +228,12 @@ def main(argv: list[str] | None = None) -> None: release_raw = None release_path = args.benchmark_release or args.dataset.parent / 'benchmark-release.json' if args.benchmark_release or release_path.exists(): - from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 - from lexbench_webfetch.versions import verify_release if release_path.is_symlink(): parser.error('Benchmark release rejects symlinks') release_raw = release_path.read_bytes() verify_release(json.loads(release_raw), dataset, exact_dataset=False, scoring_source_sha256=SCORING_SOURCE_SHA256) # Reject malformed execution contracts before doing any external captures. - from harness.web_fetch.reference_lifecycle import ( - requires_browser_observation, - requires_fresh_reference, - ) for case in dataset["cases"]: if case["id"] in case_ids and requires_fresh_reference(case): @@ -428,12 +425,10 @@ def run_wcxb(argv: list[str]) -> None: help="Comma-separated methods to rerun from the prior certified cohort", ) args = parser.parse_args(argv) - from harness.web_fetch.webmainbench_runner import WCXB_PROVIDERS, load_wcxb_source if args.output_root.exists(): parser.error("WCXB output root must be new and immutable") source_records, before = load_wcxb_source(args.dataset) - from harness.web_fetch.wcxb_dataset import validate_manifest manifest = json.loads(args.sample_manifest.read_text()) profile = json.loads(args.profile.read_text()) diff --git a/harness/web_fetch/cli.py b/harness/web_fetch/cli.py index 9235f72e9..a2466e8c0 100644 --- a/harness/web_fetch/cli.py +++ b/harness/web_fetch/cli.py @@ -4,15 +4,27 @@ from datetime import datetime, timezone from pathlib import Path -from .accounts import print_account_checklist -from .config import load_dataset, load_provider_config, load_runtime_env, select_diagnostic_scope, add_failure_retry_argument, DEFAULT_CALL_TIMEOUT_SECONDS from lexbench_webfetch.metrics import task_passed from lexbench_webfetch.models import TASK_KINDS -from .providers import build_provider -from .report_build import build_report -from .runner import rescore_evaluation, run_evaluation -from .storage import data_home -from harness.report_languages import ReportLanguageError, add_language_argument, resolve_report_language + +from harness.report_languages import ( + ReportLanguageError, + add_language_argument, + resolve_report_language, +) +from harness.web_fetch.accounts import print_account_checklist +from harness.web_fetch.config import ( + DEFAULT_CALL_TIMEOUT_SECONDS, + add_failure_retry_argument, + load_dataset, + load_provider_config, + load_runtime_env, + select_diagnostic_scope, +) +from harness.web_fetch.providers import build_provider +from harness.web_fetch.report_build import build_report +from harness.web_fetch.runner import rescore_evaluation, run_evaluation +from harness.web_fetch.storage import data_home EXAMPLES = Path(__file__).parent / "data" diff --git a/harness/web_fetch/comparison_audit.py b/harness/web_fetch/comparison_audit.py index 2a6d51e3e..b068f62fe 100644 --- a/harness/web_fetch/comparison_audit.py +++ b/harness/web_fetch/comparison_audit.py @@ -9,17 +9,22 @@ from pathlib import Path from typing import Any -from harness.record_io import parse_jsonl - -from .artifacts import diagnostic_log_binding, verify_public_call_log, verify_record_artifacts from lexbench_webfetch.metrics import task_passed -from .reference_lifecycle import require_verified_reference_checks -from .reference_lifecycle import is_diagnostic_evidence from lexbench_webfetch.paths import contained_path, safe_identifier -from harness.web_fetch.safety import reject_evidence_symlinks -from .task_outputs import require_native_task_routes -from .retrieval_hints import parse_retrieval_hints +from harness.record_io import parse_jsonl +from harness.web_fetch.artifacts import ( + diagnostic_log_binding, + verify_public_call_log, + verify_record_artifacts, +) +from harness.web_fetch.reference_lifecycle import ( + is_diagnostic_evidence, + require_verified_reference_checks, +) +from harness.web_fetch.retrieval_hints import parse_retrieval_hints +from harness.web_fetch.safety import reject_evidence_symlinks +from harness.web_fetch.task_outputs import require_native_task_routes _IDENTITY_FIELDS = { "label", diff --git a/harness/web_fetch/config.py b/harness/web_fetch/config.py index 8f17ff201..c4b08cdda 100644 --- a/harness/web_fetch/config.py +++ b/harness/web_fetch/config.py @@ -9,8 +9,9 @@ from urllib.parse import urlsplit from lexbench_webfetch.models import EvalCase, validate_case -from .reference_inputs import read_dataset_bytes -from .retrieval_hints import ( + +from harness.web_fetch.reference_inputs import read_dataset_bytes +from harness.web_fetch.retrieval_hints import ( native_default_case, parse_retrieval_hints, task_definition_sha256, diff --git a/harness/web_fetch/dataset_assembly.py b/harness/web_fetch/dataset_assembly.py index 00bdd26b8..e95815a4e 100644 --- a/harness/web_fetch/dataset_assembly.py +++ b/harness/web_fetch/dataset_assembly.py @@ -6,11 +6,15 @@ import json import re from pathlib import Path -from harness.record_io import canonical_json_sha256 +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.scoring_regressions import require_scoring_regressions + +from harness.record_io import canonical_json_sha256 from harness.web_fetch.reference_inputs import read_dataset_bytes +from harness.web_fetch.reference_lifecycle import homepage_prompt_anchor_correction from harness.web_fetch.retrieval_hints import historical_native_fetch_dataset, native_fetch_dataset -from lexbench_webfetch.paths import contained_path +from harness.web_fetch.safety import reject_evidence_symlinks ROOT = Path(__file__).resolve().parents[2] @@ -54,8 +58,6 @@ def _apply_case_corrections( definition: dict, selected: dict[str, dict], repository: Path ) -> list[str]: """Apply compact, hash-locked reference and scoring corrections to imported cases.""" - from lexbench_webfetch.scoring_regressions import require_scoring_regressions - from harness.web_fetch.reference_lifecycle import homepage_prompt_anchor_correction declaration = definition.get("case_corrections") if declaration is None: @@ -161,7 +163,6 @@ def assemble_profile_dataset( profile_path = contained_path(repository, declared_profile, reject_symlinks=True) reproduction_profile = str(declared_profile) else: - from .safety import reject_evidence_symlinks profile_path = (profile_path_override if profile_path_override.is_absolute() else repository / profile_path_override) reject_evidence_symlinks(profile_path) diff --git a/harness/web_fetch/dynamic_cohort.py b/harness/web_fetch/dynamic_cohort.py index 42e0b5f5f..33883f3d1 100644 --- a/harness/web_fetch/dynamic_cohort.py +++ b/harness/web_fetch/dynamic_cohort.py @@ -10,18 +10,23 @@ from datetime import datetime from pathlib import Path +from lexbench_webfetch.models import EvalCase +from lexbench_webfetch.paths import contained_path + from harness.record_io import parse_jsonl from harness.web_fetch.artifacts import ( materialize_bound_artifacts, materialize_reference_checks, verify_record_artifacts, ) -from lexbench_webfetch.models import EvalCase from harness.web_fetch.publication_privacy import project_execution_manifest from harness.web_fetch.reference_lifecycle import require_verified_reference_checks from harness.web_fetch.render_evidence import validate_render_reference -from harness.web_fetch.safety import commit_staged_evidence, reject_evidence_symlinks, validate_report_ids -from lexbench_webfetch.paths import contained_path +from harness.web_fetch.safety import ( + commit_staged_evidence, + reject_evidence_symlinks, + validate_report_ids, +) from harness.web_fetch.storage import current_evidence_dir, data_home diff --git a/harness/web_fetch/evidence.py b/harness/web_fetch/evidence.py index 07598ee3c..24a3dcce4 100644 --- a/harness/web_fetch/evidence.py +++ b/harness/web_fetch/evidence.py @@ -11,24 +11,33 @@ from datetime import datetime, timedelta, timezone from pathlib import Path -from harness.record_io import file_sha256, load_jsonl, parse_jsonl, read_completed_records +from lexbench_webfetch.paths import contained_path, safe_identifier -from .artifacts import ( +from harness.record_io import file_sha256, load_jsonl, parse_jsonl, read_completed_records +from harness.web_fetch.artifacts import ( materialize_bound_artifacts, materialize_reference_checks, materialize_task_references, + public_call_log_binding, + verify_public_call_log, verify_record_artifacts, ) -from .reference_inputs import input_tree -from .reference_lifecycle import ( +from harness.web_fetch.reference_inputs import input_tree +from harness.web_fetch.reference_lifecycle import ( completed_case_records, expected_providers, finalize_reference_checks, reference_cases, require_verified_reference_checks, + requires_fresh_reference, ) -from harness.web_fetch.safety import commit_staged_evidence, reject_evidence_symlinks, validate_report_ids -from lexbench_webfetch.paths import contained_path, safe_identifier +from harness.web_fetch.safety import ( + commit_staged_evidence, + reject_evidence_symlinks, + validate_report_ids, +) +from harness.web_fetch.storage import current_evidence_dir + def recover_completed_batch(source: Path, manifest: dict, records: list[dict]) -> dict: """Bind all flushed calls from an interrupted fixed-answer physical batch. @@ -37,9 +46,6 @@ def recover_completed_batch(source: Path, manifest: dict, records: list[dict]) - saved completed calls, and explicitly retains the original unfinished state. It never supplies missing live-reference certification or chooses scores. """ - from datetime import datetime - from .artifacts import public_call_log_binding, verify_public_call_log - from .reference_lifecycle import requires_fresh_reference if (manifest.get("schema_version") != 1 or manifest.get("tool") != "web_fetch" or manifest.get("completed_at") or manifest.get("attempt_ids") is not None @@ -427,8 +433,6 @@ def _project_provider_run(source: Path, output: Path, provider: str, reference_r ) - - def project_provider_run(source: Path, output: Path, provider: str, reference_root: Path | None = None, case_ids: set[str] | None = None, *, retain_completed: bool = False) -> None: @@ -472,8 +476,6 @@ def projection_main(argv=None) -> None: def addition_main(argv=None) -> None: - from .storage import current_evidence_dir - parser = argparse.ArgumentParser(description='Validate and add a complete provider run.') parser.add_argument('run_dir', type=Path) parser.add_argument('--provider', required=True) diff --git a/harness/web_fetch/failure_diagnosis.py b/harness/web_fetch/failure_diagnosis.py index b58431f69..2dd394325 100644 --- a/harness/web_fetch/failure_diagnosis.py +++ b/harness/web_fetch/failure_diagnosis.py @@ -1,5 +1,9 @@ """Evidence-based failure stage, preserving uncertainty about remote internals.""" -import json,re + +import json +import re + +from lexbench_webfetch.metrics import task_measured def _platform_capability_restricted(message): @@ -53,7 +57,6 @@ def _stable_failure(metrics): def diagnose(r,c,raw): - from lexbench_webfetch.metrics import task_measured if not task_measured(r['metrics'], c.get('task_kind')): return None try:payload=json.loads(raw) diff --git a/harness/web_fetch/headless_runner.py b/harness/web_fetch/headless_runner.py index c43404c6a..7befc8ded 100644 --- a/harness/web_fetch/headless_runner.py +++ b/harness/web_fetch/headless_runner.py @@ -3,15 +3,21 @@ import argparse import hashlib import json +from collections import Counter +from copy import deepcopy from pathlib import Path +from lexbench_webfetch.paths import contained_path, safe_identifier +from lexbench_webfetch.scoring import score_attempt + +from harness.record_io import load_jsonl from harness.web_fetch.artifacts import materialize_task_references from harness.web_fetch.config import ( + add_failure_retry_argument, derive_comparison, load_dataset, load_provider_settings, load_runtime_env, - add_failure_retry_argument, ) from harness.web_fetch.providers import build_provider from harness.web_fetch.providers.headless import HEADLESS_CONVERSION @@ -20,8 +26,10 @@ version_number, ) from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes -from harness.web_fetch.runner import run_evaluation -from lexbench_webfetch.paths import safe_identifier +from harness.web_fetch.reference_lifecycle import require_verified_reference_checks +from harness.web_fetch.runner import load_list_reference, run_evaluation +from harness.web_fetch.safety import unlinked_path, validate_report_ids +from harness.web_fetch.storage import require_current_report_profile ROOT = Path(__file__).resolve().parents[2] @@ -301,17 +309,6 @@ def check_evidence( provenance_path=None, report_profile=None, ): - from collections import Counter - from copy import deepcopy - - from harness.record_io import load_jsonl - from harness.web_fetch.reference_lifecycle import require_verified_reference_checks - from harness.web_fetch.runner import load_list_reference - from lexbench_webfetch.paths import contained_path - from harness.web_fetch.safety import unlinked_path, validate_report_ids - from lexbench_webfetch.scoring import score_attempt - from harness.web_fetch.storage import require_current_report_profile - root = unlinked_path(root) canonical_dataset = canonical_dataset or ROOT / "evaluation/dataset.json" provider_config = provider_config or ROOT / "configs/providers.headless.json" diff --git a/harness/web_fetch/local_audit.py b/harness/web_fetch/local_audit.py index 7b0439970..47058b476 100644 --- a/harness/web_fetch/local_audit.py +++ b/harness/web_fetch/local_audit.py @@ -1,12 +1,18 @@ """Fail-closed validation for explicitly non-publishable local audit bundles.""" + import json from pathlib import Path -from harness.record_io import file_sha256, canonical_json_sha256 from lexbench_webfetch.metrics import task_passed -from .reference_inputs import input_tree from lexbench_webfetch.paths import contained_path -from . import adjudication_evidence, local_execution_contract +from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 +from lexbench_webfetch.scoring_regressions import build_correction_provenance + +from harness.record_io import canonical_json_sha256, file_sha256 +from harness.web_fetch import adjudication_evidence, local_execution_contract +from harness.web_fetch.publication_privacy import read_evidence_binding +from harness.web_fetch.reference_inputs import benchmark_input_path, input_tree +from harness.web_fetch.reliability import assess_reliability ROOT = Path(__file__).resolve().parents[2] MANIFEST_NAME = "local-audit-manifest.json" @@ -15,7 +21,6 @@ def _repository_file(relative: str) -> Path: if not isinstance(relative, str) or not relative: raise ValueError("Repository evidence path is missing") - from .reference_inputs import benchmark_input_path path = benchmark_input_path(contained_path(ROOT, relative)) cursor = ROOT has_symlink = False @@ -280,7 +285,6 @@ def _require_review(source: Path, dataset: dict, records: list[dict], def _read_bound_receipt(source: Path, receipt: dict, relative: str) -> dict: - from .publication_privacy import read_evidence_binding expected = receipt.get("review", {}).get("bound_receipts", {}).get(relative) if not expected: raise ValueError("Local audit review receipt changed: " + relative) @@ -357,7 +361,6 @@ def _require_lineage(source: Path, receipt: dict) -> None: def _require_local_scoring_revision(source, contract, records): - from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 if any(r.get("metrics", {}).get("scoring_source_sha256") != SCORING_SOURCE_SHA256 for r in records): raise ValueError("Local measurement has a stale scoring implementation") @@ -372,7 +375,6 @@ def _require_local_scoring_revision(source, contract, records): or receipt.get("call_identities") != { r["attempt_id"]: local_execution_contract.local_measurement_call_sha256(r) for r in records}): raise ValueError("Invalid local scoring correction binding") - from lexbench_webfetch.scoring_regressions import build_correction_provenance provenance = receipt.get("provenance") or {} expected_provenance = build_correction_provenance( provenance.get("category"), provenance.get("root_cause"), @@ -394,7 +396,6 @@ def _require_local_scoring_revision(source, contract, records): def require_complete_local_inventory(dataset: dict, records: list[dict]) -> dict: """Allow explicit unknown verdicts locally, never missing physical results.""" - from .reliability import assess_reliability comparison = dataset.get("comparison", {}) if (comparison.get("kind") != "uniform_native_local_measurement" diff --git a/harness/web_fetch/local_execution_contract.py b/harness/web_fetch/local_execution_contract.py index 875274636..160b5fa0f 100644 --- a/harness/web_fetch/local_execution_contract.py +++ b/harness/web_fetch/local_execution_contract.py @@ -1,12 +1,25 @@ """Verify frozen local execution inputs and physical calls, independently of review.""" + import hashlib import json +import math +from datetime import datetime from pathlib import Path from types import MappingProxyType +from urllib.parse import urlsplit -from harness.record_io import canonical_json_sha256 from lexbench_webfetch.paths import contained_path -from .reference_inputs import read_dataset_bytes + +from harness.record_io import canonical_json_sha256 +from harness.web_fetch.artifacts import ( + record_public_call_log_bindings, + verify_record_public_call_logs, +) +from harness.web_fetch.native_projection import native_field_value +from harness.web_fetch.network_dns import validate_dns_mode +from harness.web_fetch.network_preflight import GUARDED_NATIVE_TYPES, require_usable_public_dns +from harness.web_fetch.publication_privacy import read_evidence_binding +from harness.web_fetch.reference_inputs import read_dataset_bytes ROOT = Path(__file__).resolve().parents[2] # Frozen schema-1 semantics, not defaults for a future multi-round protocol. @@ -64,8 +77,6 @@ def _require_local_record_correction(source, original, corrected, receipt=None): if changed - {"final_url"}: raise ValueError("Local correction changed measured call facts") if "final_url" in changed: - from .native_projection import native_field_value - from .publication_privacy import read_evidence_binding proof = receipt.get("field_projections", {}).get(identifier, {}).get("final_url", {}) if not proof.get('path') or not proof.get('sha256'): raise ValueError('Local URL correction lost its native response') @@ -140,7 +151,6 @@ def local_measurement_cases(contract: dict, source=None) -> list[dict]: def _local_timeout_seconds(contract): - import math value = contract.get('timeout_seconds') if (isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0): @@ -150,8 +160,6 @@ def _local_timeout_seconds(contract): def require_local_execution_contract(source, dataset, records, contract, execution): """Shared physical-call boundary for complete cohorts and task replacements.""" - from .artifacts import verify_record_public_call_logs, record_public_call_log_bindings - from datetime import datetime timeout = _local_timeout_seconds(contract) # This mode binds one authoritative physical call per method/task. Multiple # rounds and retries require the certified cohort's separate lineage model. @@ -178,7 +186,7 @@ def validate_current_batches(): _require_local_method_bindings(contract, execution["batches"], field) for batch in execution["batches"]: _require_local_batch_segments(batch) - from .local_execution_legacy import require_execution_lineage + from harness.web_fetch.local_execution_legacy import require_execution_lineage require_execution_lineage( source, contract, execution["batches"], records, cases=canonical["cases"], timeout_seconds=timeout, @@ -230,8 +238,6 @@ def validate_current_batches(): def _require_local_runner_inputs(source, contract, execution, records): """Verify retained runner inputs and source observations, without certifying them.""" - from datetime import datetime - from .publication_privacy import read_evidence_binding inputs = contract['runner_inputs'] def read(key): return json.loads(read_evidence_binding(source, inputs[key]).read_text()) @@ -246,9 +252,7 @@ def read(key): dns_mode = _require_bound_dns_mode(configuration, contract['providers'], network) if network.get('usable_for_guarded_local_comparison') is not True: raise ValueError('Local runner lacks usable public DNS evidence') - from .network_preflight import GUARDED_NATIVE_TYPES, require_usable_public_dns require_usable_public_dns(network) - from urllib.parse import urlsplit method_types = {name: configuration['providers'][name].get('method_type', configuration['providers'][name]['type']) for name in contract['providers']} if any(value in GUARDED_NATIVE_TYPES for value in method_types.values()): @@ -300,8 +304,6 @@ def read(key): def _require_bound_dns_mode(configuration, providers, network): """Match observed DNS to guarded adapters' frozen configuration.""" - from .network_dns import validate_dns_mode - from .network_preflight import GUARDED_NATIVE_TYPES observed = validate_dns_mode(network.get('dns_mode')) modes = { validate_dns_mode(config.get('dns_mode', 'system')) diff --git a/harness/web_fetch/local_execution_legacy.py b/harness/web_fetch/local_execution_legacy.py index 34b93a792..851ebda5a 100644 --- a/harness/web_fetch/local_execution_legacy.py +++ b/harness/web_fetch/local_execution_legacy.py @@ -9,11 +9,14 @@ and historical negative controls have been retired together. A new cohort's lack of such plans alone does not establish that historical evidence is unused. """ -from harness.record_io import file_sha256 + import json +from copy import deepcopy from lexbench_webfetch.paths import contained_path -from .local_execution_contract import LOCAL_SCHEMA1_PARAMETERS + +from harness.record_io import file_sha256 +from harness.web_fetch.local_execution_contract import LOCAL_SCHEMA1_PARAMETERS def require_execution_lineage( @@ -77,7 +80,6 @@ def read(binding, *, json_lines=False): for key, plan in plan_values.items(): if plans[key]["configuration"]["sha256"] != plan["provider_config_sha256"]: raise ValueError("Local execution configuration differs from its pre-call plan") - from copy import deepcopy normalized = deepcopy(configs["resumed"]) for provider in normalized.get("providers", {}).values(): for field in ("command", "required_paths"): diff --git a/harness/web_fetch/local_revision.py b/harness/web_fetch/local_revision.py index 07ba0ee24..4abcbf642 100644 --- a/harness/web_fetch/local_revision.py +++ b/harness/web_fetch/local_revision.py @@ -1,19 +1,52 @@ """Validate task replacements and offline revisions of a bound local snapshot.""" + +import copy +import fcntl import hashlib import json -import fcntl import shutil import tempfile +from collections import Counter from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path -from harness.record_io import file_sha256, canonical_json_sha256, load_jsonl - -from .cohort_exclusions import require_cohort_exclusions +from lexbench_webfetch.metrics import task_passed from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256, load_list_reference, score_attempt from lexbench_webfetch.scoring_regressions import require_scoring_regressions -from .scoring_revision import ALLOWED_FIELDS -from .publication_privacy import EvidenceBinding, read_evidence_binding, staged_evidence_reads + +from harness.record_io import canonical_json_sha256, file_sha256, load_jsonl +from harness.web_fetch.adjudication_evidence import ( + _require_local_review, + _require_page_identity_review, + compact_retained_review_dependencies, + prepare_reviewed_page_identities, +) +from harness.web_fetch.cohort_exclusions import apply_cohort_exclusions, require_cohort_exclusions +from harness.web_fetch.local_execution_contract import ( + local_measurement_cases, + require_local_execution_contract, +) +from harness.web_fetch.native_projection import ( + native_projection_regressions, + project_native_answer, + project_native_result_fields, +) +from harness.web_fetch.native_routes import verify_native_execution_manifests +from harness.web_fetch.providers.moli import replay_native_http_error_context +from harness.web_fetch.publication_privacy import ( + EvidenceBinding, + bound_public_view, + original_evidence_size, + project_file, + read_evidence_binding, + staged_evidence_reads, +) +from harness.web_fetch.report import runtime_log_text +from harness.web_fetch.result_review import review_input_identity +from harness.web_fetch.runner import _public_text +from harness.web_fetch.scoring_revision import ALLOWED_FIELDS @dataclass(frozen=True) @@ -70,7 +103,6 @@ def _require_revision_regressions(identifiers): """A revision cites controls in the layer whose boundary it changes.""" if not identifiers: raise ValueError('Local revision requires executable regression provenance') - from .native_projection import native_projection_regressions native = native_projection_regressions() scoring = [identifier for identifier in identifiers if identifier not in native] if scoring: @@ -207,7 +239,6 @@ def require_local_task_replacement(source, binding, dataset, records): raise ValueError('Task replacement altered an inherited or newly measured result') if any(r['metrics'].get('evaluation_excluded') for r in calls): raise ValueError('Replacement task cannot be excluded after seeing its results') - from .local_execution_contract import require_local_execution_contract if not receipt.get('execution_contract') or not receipt.get('cohort_execution'): raise ValueError('Task replacement requires its complete execution contract') contract = json.loads(read_evidence_binding(source, receipt['execution_contract']).read_text()) @@ -258,7 +289,6 @@ def require_local_revision(source, binding, dataset, records): def _current_row_identity(case, record): - from .result_review import review_input_identity return {'attempt_id': record['attempt_id'], 'case_sha256': canonical_json_sha256(case), 'record_sha256': canonical_json_sha256(record), 'scoring_source_sha256': record['metrics']['scoring_source_sha256'], @@ -266,7 +296,6 @@ def _current_row_identity(case, record): def _require_current_row_evidence(source, record, diagnostic_projection=None): - from .result_review import review_input_identity identity = review_input_identity(record) read_evidence_binding(source, {'path': record['artifact_path'], 'sha256': record['metrics']['content_sha256']}) @@ -282,7 +311,6 @@ def _require_current_row_evidence(source, record, diagnostic_projection=None): def _require_diagnostic_projection(source, record, proof): - from .result_review import review_input_identity diagnostic = review_input_identity(record)['diagnostic_log'] call = record.get('provider_metadata', {}).get('call_log') if (proof.get('attempt_id') != record['attempt_id'] @@ -309,10 +337,6 @@ def _require_diagnostic_projection(source, record, proof): def _compact_diagnostic_projections(source, records, private_manifest, *, public_log_updates=None): """Verify private originals once; emit honest public-view receipts only.""" - from .result_review import review_input_identity - from .report import runtime_log_text - from .runner import _public_text - from .publication_privacy import project_file private = None bound_public_logs = set() def collect_paths(value): @@ -508,7 +532,6 @@ def require_current_task_replacement(source, receipt, dataset, records): read_evidence_binding(source, capture) contract = json.loads(read_evidence_binding(source, scope['execution_contract']).read_text()) execution = json.loads(read_evidence_binding(source, scope['execution']).read_text()) - from .local_execution_contract import local_measurement_cases, require_local_execution_contract frozen = local_measurement_cases(contract, source) current = {case['id']: case for case in dataset['cases']} if {case['id'] for case in frozen} != {scope['added_case_id']}: @@ -580,15 +603,12 @@ def require_material(value): execution_binding = receipt.get('current_execution') if not execution_binding or read_evidence_binding(source, execution_binding) != execution: raise ValueError('Current execution inventory lacks its actual binding') - from collections import Counter physical = Counter(identifier for batch in json.loads(execution.read_text())['batches'] for identifier in batch['attempt_ids']) if physical != Counter(row['attempt_id'] for row in records): raise ValueError('Current execution physical/logical inventory differs') - from .native_routes import verify_native_execution_manifests verify_native_execution_manifests(source, json.loads(execution.read_text())['batches'], dataset, records) - from .adjudication_evidence import _require_local_review review = receipt.get('current_review') declared = dataset.get('comparison', {}).get('independent_review') if declared is not None and declared != review: @@ -612,7 +632,6 @@ def _install_compact_local_snapshot(source, binding, *, reference_authorities=() opinions remain exact. The caller may delete obsolete states after this succeeds and the ordinary presentation boundary accepts the returned binding. """ - from .adjudication_evidence import compact_retained_review_dependencies source = Path(source) dataset = json.loads((source/'dataset.json').read_text()) records = load_jsonl(source/'results.jsonl') @@ -715,7 +734,6 @@ def compact_local_snapshot(source, binding, **options): execution_path = source/'execution.json' original_execution = execution_path.read_bytes() if execution_path.is_file() else None if original_execution is not None: - from collections import Counter execution = json.loads(original_execution) current = {row['attempt_id'] for row in records} physical = Counter(identifier for batch in execution['batches'] @@ -744,8 +762,6 @@ def compact_local_snapshot(source, binding, **options): aggregate['batch_count'] = len(execution['batches']) execution['inventory_scope'] = 'current_snapshot_attempts' execution_path.write_text(json.dumps(execution, ensure_ascii=False, indent=2)+'\n') - import tempfile - import shutil temporary = tempfile.TemporaryDirectory(prefix='current-diagnostic-projection-') backups = {} try: @@ -778,8 +794,6 @@ def compact_local_snapshot(source, binding, **options): def _project_native_correction(source, original, request, *, historical_scoring_source=None): """One projection path for preparing and verifying an offline correction.""" - from .native_projection import project_native_answer, project_native_result_fields - from .publication_privacy import original_evidence_size metadata = original.get('provider_metadata', {}) native = read_evidence_binding(source, request['native_response']) if (request['native_response']['sha256'] != metadata.get('raw_response_sha256') @@ -827,7 +841,6 @@ def _require_native_answer_correction(source, original, corrected, proof, *, his # A disclosed native public view projects public answer bytes. Compare # its explicitly paired answer view even while private scoring bytes # are temporarily restored; neither view reconstructs the original. - from .publication_privacy import bound_public_view native_view = bound_public_view(source, proof['native_response']) answer_view = bound_public_view(source, after) if (native_view is None or answer_view is None @@ -850,8 +863,6 @@ def _require_native_answer_correction(source, original, corrected, proof, *, his def _native_context_proof(source, original, request): """Replay context from one bound physical call, preserving its answer bytes.""" - from .providers.moli import replay_native_http_error_context - from .publication_privacy import original_evidence_size if (set(request) != {'attempt_id', 'stdout', 'diagnostic', 'call_log', 'projection'} or request['attempt_id'] != original['attempt_id'] or original.get('provider_type') != 'moli_webfetch'): @@ -999,7 +1010,6 @@ def _require_revision_step(source, receipt, dataset, records, outputs): if any(new_rows[key]['metrics'].get('scoring_source_sha256') != value for key, value in expected_scoring.items()): raise ValueError('Local revision scoring source binding changed') - from lexbench_webfetch.metrics import task_passed if receipt.get('input_verdicts'): verdicts = _read_input_verdicts(source, receipt['input_verdicts'], receipt['input_results']['sha256'], old_rows) @@ -1011,7 +1021,6 @@ def _require_revision_step(source, receipt, dataset, records, outputs): and verdicts[row['attempt_id']] != task_passed(row['ok'], row['metrics'])] if transitions != receipt['task_verdict_changes']: raise ValueError('Local revision verdict transitions differ from input proof') - from .adjudication_evidence import _require_page_identity_review _require_page_identity_review(source, dataset, records, scoring_sources=expected_scoring) require_cohort_exclusions(source, dataset, records) return expected_scoring, before_dataset, previous @@ -1037,11 +1046,6 @@ def revise_local_snapshot(source, configuration): def _revise_local_snapshot(source, configuration, transaction): """Apply declared score-only changes in a separate snapshot, retaining predecessors.""" - import copy - from datetime import datetime, timezone - from lexbench_webfetch.scoring import score_attempt, load_list_reference, SCORING_SOURCE_SHA256 - from .cohort_exclusions import apply_cohort_exclusions - from lexbench_webfetch.metrics import task_passed config = json.loads(configuration.read_text()) regressions = config.get('regression_ids', []) @@ -1142,7 +1146,6 @@ def _revise_local_snapshot(source, configuration, transaction): 'rule_revision': bool(config.get('rule_revision')), 'input_results_sha256': input_results_sha256} selected.append(row['attempt_id']) - from .adjudication_evidence import prepare_reviewed_page_identities identity_updates = prepare_reviewed_page_identities(source, original_dataset, dataset, records) identity_binding = identity_updates.binding if identity_updates else None if identity_updates: diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py index fe6a00e05..90aa92c28 100644 --- a/harness/web_fetch/methods.py +++ b/harness/web_fetch/methods.py @@ -1,17 +1,19 @@ from __future__ import annotations -import json import hashlib -import inspect -from copy import deepcopy import importlib.metadata +import inspect +import json import re from collections import Counter -from functools import lru_cache, wraps from contextvars import ContextVar +from copy import deepcopy +from functools import lru_cache, wraps from pathlib import Path from typing import Any +from harness.distribution import installed_data_path + def _resolve_catalog_templates(data): """Expand named defaults once; methods override dictionaries and replace lists.""" @@ -203,7 +205,6 @@ def _execution_browser_method_registry(): """Use the execution registry for release and source-build participants.""" path = Path(__file__).resolve().parents[2] / "configs/providers.headless.json" if not path.is_file(): - from harness.distribution import installed_data_path try: path = installed_data_path("share/browser-harness-eval/providers.headless.json") except importlib.metadata.PackageNotFoundError: @@ -278,7 +279,6 @@ def method_version(record, provenance=None, *, language="zh"): return str(browser_version) if browser_version else None - def method_versions(records, provenance=None): """Keep distinct measured binaries visible when their reported versions coincide.""" versions={} @@ -293,7 +293,6 @@ def method_versions(records, provenance=None): if multiple and hashes else '') for version,hashes in sorted(versions.items())] - def with_measured_method_labels(render): """Scope all participant labels to recorded versions and binary identities.""" signature = inspect.signature(render) @@ -386,7 +385,6 @@ def method_info(provider_type: str, *, language="zh") -> dict[str, Any]: return registry.get(provider_type, report_method_catalog(language)["unregistered_adapter"]) - ROUTE_LABELS = report_method_catalog()["route_labels"] diff --git a/harness/web_fetch/models.py b/harness/web_fetch/models.py index 9eca80551..518797a42 100644 --- a/harness/web_fetch/models.py +++ b/harness/web_fetch/models.py @@ -4,7 +4,6 @@ from dataclasses import asdict, dataclass, field from typing import Any - # The frozen dataset retains image/video as source tracks. Public comparisons # group those cases by the deliverable a caller must receive. REPORT_FAMILIES = { diff --git a/harness/web_fetch/native_projection.py b/harness/web_fetch/native_projection.py index e24745f04..0560f0a70 100644 --- a/harness/web_fetch/native_projection.py +++ b/harness/web_fetch/native_projection.py @@ -4,16 +4,19 @@ implementation in one owner lets acquisition and saved-evidence checks replay the same policy rather than accepting any later declaration of allowed fields. """ + import base64 import binascii import hashlib import json import re from functools import lru_cache +from pathlib import Path +from types import SimpleNamespace from urllib.parse import parse_qsl, quote, urljoin, urlsplit from lexbench_webfetch.paths import validate_http_url -from pathlib import Path +from lexbench_webfetch.resource_urls import srcset_urls def native_projection_source(): @@ -138,8 +141,7 @@ def native_answer_projection(provider_type, case, payload, status_code=None, *, if key in _object(payload.get('metadata'))}, **({'files': {'pointer': ['files'], 'format': 'worker_files'}} if payload.get('files') else {})} if provider_type == 'moli_webfetch': - from .providers.moli import _route_for_case - from types import SimpleNamespace + from harness.web_fetch.providers.moli import _route_for_case projected_case = SimpleNamespace(**case) if isinstance(case, dict) else case raw = _route_for_case(projected_case, native_default=True) in {'raw_file', 'native_document_bytes'} return {'schema_version': 2, 'source_format': 'bytes', @@ -192,7 +194,6 @@ def _srcset_urls(value, parser, historical_scoring_source): else: parser = historical if parser == SRCSET_PARSER: - from lexbench_webfetch.resource_urls import srcset_urls return srcset_urls(value) if parser == 'legacy-comma-split': return [part.strip().split()[0] for part in value.split(',') if part.strip()] @@ -476,7 +477,6 @@ def identity(value: str) -> tuple | None: return candidate - def project_native_result_fields(payload, declaration, *, request_url=None, diagnostic=None, transport_receipt=None): """Replay scored title/address from native fields or same-call evidence.""" diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py index abccf1281..51b3d226e 100644 --- a/harness/web_fetch/native_routes.py +++ b/harness/web_fetch/native_routes.py @@ -1,12 +1,29 @@ """Declared native input and route contracts, independent of URLs and answers.""" + +import hashlib import json +import re +import shutil +import threading +from copy import deepcopy from dataclasses import dataclass -from harness.record_io import canonical_json_sha256, file_sha256 -from .config import DEFAULT_CALL_TIMEOUT_SECONDS +from datetime import datetime, timezone + +from lexbench_webfetch.file_acceptance import evaluate_files +from lexbench_webfetch.paths import contained_path -from .native_projection import ( - SCORE_ANSWER_FIELDS, require_native_projection_schema, require_native_answer_projection, +from harness.record_io import canonical_json_sha256, file_sha256 +from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS +from harness.web_fetch.native_projection import ( + SCORE_ANSWER_FIELDS, + native_answer_projection, + native_projection_policy, + native_projection_source, + require_native_answer_projection, + require_native_projection_schema, ) +from harness.web_fetch.publication_privacy import read_evidence_binding +from harness.web_fetch.retrieval_hints import native_default_case INPUT_MEDIA_KINDS = frozenset({ 'html_page', 'direct_pdf', 'direct_docx', 'direct_text', @@ -45,7 +62,6 @@ class NativeRequestSignature: @classmethod def parse(cls, item, *, field): - import re def reject(name): raise ValueError('Native route ' + field + name + ' is invalid') if not isinstance(item, dict): @@ -113,7 +129,6 @@ def native_route_cell(provider, case, *, timeout_seconds=DEFAULT_CALL_TIMEOUT_SE endpoint = None if transport == 'http': endpoint, _ = provider.native_http_arguments(case, timeout_seconds) - from .native_projection import native_projection_policy cell = {'method': key[0], 'provider_type': method, 'track': track, 'task_kind': kind, 'input_media_kind': media, 'transport': transport, 'interface': interface, 'answer_fields': fields, 'configured_parameters': provider.native_configured_parameters(), @@ -146,7 +161,6 @@ def _request_parameters(provider, case, cell, timeout_seconds): def normalized_http_parameters(case, arguments): - from copy import deepcopy body = deepcopy(arguments.get('json_body')) if not isinstance(body, dict): raise ValueError('Native HTTP request lacks its declared input field') @@ -162,7 +176,6 @@ def normalized_http_parameters(case, arguments): 'timeout_seconds': arguments.get('timeout'), 'max_bytes': arguments.get('max_bytes')} - class NativeRouteLedger: """Freeze class-level intent before calls; reject observed per-case drift. @@ -172,8 +185,6 @@ class NativeRouteLedger: response, URL pattern, or expected answer selects a route. """ def __init__(self, cases, providers, *, timeout_seconds=DEFAULT_CALL_TIMEOUT_SECONDS): - from datetime import datetime, timezone - import threading self.timeout_seconds = timeout_seconds self.cells, self.observed, self.calls, self.errors = {}, {}, {}, [] self.physical_attempts = {} @@ -189,7 +200,6 @@ def __init__(self, cases, providers, *, timeout_seconds=DEFAULT_CALL_TIMEOUT_SEC for provider in self.providers.values(): key, cell = native_route_cell(provider, case, timeout_seconds=self.timeout_seconds) if case.track == 'dynamic': - from .retrieval_hints import native_default_case if not native_default_case(case): raise ValueError('Native dynamic inputs require a stable acceptance contract') cell['request_parameters'] = _request_parameters(provider, case, cell, timeout_seconds) @@ -204,15 +214,11 @@ def __init__(self, cases, providers, *, timeout_seconds=DEFAULT_CALL_TIMEOUT_SEC self.binding = None def write(self, source, *, dataset_sha256, runtime_source_sha256): - import hashlib - import json self.document.update(dataset_sha256=dataset_sha256, runtime_source_sha256=runtime_source_sha256) raw = (json.dumps(self.document, ensure_ascii=False, indent=2)+'\n').encode() path = source/'native-route-contract.json' if path.exists(): raise ValueError('Native route contract already exists; do not overwrite a run') - from .native_projection import native_projection_source - from lexbench_webfetch.paths import contained_path for cell in self.cells.values(): policy = cell['answer_selection_policy'] policy_source = native_projection_source() @@ -273,7 +279,6 @@ def observe(self, provider, case, *, transport, parameters, endpoint=None, inter def receipt(self, provider, case, *, ok=None): """Transport started means entry to the outbound call, not server receipt.""" - from copy import deepcopy key, _ = native_route_cell(provider, case, timeout_seconds=self.timeout_seconds) with self.lock: ordinal = self.physical_attempts.get((provider.name, case.id), 1) @@ -289,13 +294,8 @@ def receipt(self, provider, case, *, ok=None): 'protocol_errors': [dict(e) for e in self.errors if e['method']==provider.name and e['case_id']==case.id]} - def verify_native_route_contract(source, manifest, dataset, records): """Shared execution/release check for newly declared native cohorts only.""" - import hashlib - import json - from datetime import datetime - from lexbench_webfetch.paths import contained_path if manifest.get('native_route_errors'): raise ValueError('Native route protocol was violated') binding = manifest.get('native_route_contract') @@ -421,9 +421,6 @@ def verify_native_execution_manifests(source, manifests, dataset, records): def materialize_native_route_contracts(output, source, manifests): """Copy hash-bound frozen contracts, deduplicating multiple cohort roots.""" - import re - import shutil - from lexbench_webfetch.paths import contained_path for manifest in manifests: binding = manifest.get('native_route_contract') if not binding: @@ -473,7 +470,6 @@ def materialize_native_route_contracts(output, source, manifests): def native_cases_by_id(root): """Read one operation's case authority; never cache mutable evidence globally.""" - from lexbench_webfetch.paths import contained_path path = contained_path(root, 'dataset.json', reject_symlinks=True) if not path.is_file(): return None @@ -485,7 +481,6 @@ def native_cases_by_id(root): def _expected_native_declaration(root, record, raw, cases_by_id): - from .native_projection import native_answer_projection cases_by_id = native_cases_by_id(root) if cases_by_id is None else cases_by_id case = (cases_by_id or {}).get(record.get('case_id')) if case is None or case.get('url') != record.get('url'): @@ -506,13 +501,10 @@ def verify_saved_native_answer(root, record, *, cases_by_id=None): Older calls with no declaration need a separately recorded historical audit; their missing acquisition declarations must never be invented in place. """ - from .publication_privacy import read_evidence_binding - from lexbench_webfetch.paths import contained_path metadata = record.get('provider_metadata', {}) declaration = metadata.get('native_answer_projection') policy = metadata.get('native_route_receipt', {}).get('answer_selection_policy') if policy is not None: - from .native_projection import native_projection_policy if policy != native_projection_policy(record.get('provider_type')): raise ValueError('Native answer selection policy implementation differs from its pre-call freeze') read_evidence_binding(root, {'path': policy['source_path'], @@ -570,7 +562,6 @@ def verify_saved_native_answer(root, record, *, cases_by_id=None): diagnostic=diagnostic, transport_receipt=metadata.get('source_transport_receipt'), historical_scoring_source=record.get('metrics', {}).get('scoring_source_sha256')) if files or 'file_evaluation' in metadata: - from lexbench_webfetch.file_acceptance import evaluate_files source_verification = {} cases_by_id = native_cases_by_id(root) if cases_by_id is None else cases_by_id if cases_by_id is not None: diff --git a/harness/web_fetch/network_dns.py b/harness/web_fetch/network_dns.py index 2d60e8eef..711f65834 100644 --- a/harness/web_fetch/network_dns.py +++ b/harness/web_fetch/network_dns.py @@ -1,12 +1,23 @@ """Explicit process-local DNS environment; never change product safety policy.""" -from contextlib import contextmanager, asynccontextmanager + +import asyncio +import hashlib import ipaddress import json +import math +import os +import pathlib +import platform +import secrets import socket +import socketserver +import subprocess +import tempfile import threading import time import urllib.parse import urllib.request +from contextlib import asynccontextmanager, contextmanager from pathlib import Path DNS_ENDPOINT = "https://cloudflare-dns.com/dns-query" @@ -157,8 +168,6 @@ class NativeDnsBroker: resolver is used only by this parent's HTTPS transport to the fixed DoH host. """ def __init__(self, *, resolver=None, diagnostics=None): - import secrets - import socketserver self.resolver = resolver or lookup_public_dns self.diagnostics = diagnostics self.token = secrets.token_hex(32) @@ -221,7 +230,6 @@ class Server(socketserver.ThreadingTCPServer): self.thread.start() def begin_call(self, timeout): - import math if not math.isfinite(timeout) or timeout <= 0: raise ValueError("DNS timeout must be positive and finite") with self.trace_lock: @@ -230,7 +238,6 @@ def begin_call(self, timeout): self.traces = [] def environment(self, library, base=None): - import os env = dict(os.environ if base is None else base) env.update(DYLD_INSERT_LIBRARIES=str(library), BROWSER_EVAL_DNS_PORT=str(self.server.server_address[1]), @@ -260,11 +267,8 @@ async def crawl4ai_public_dns_browser(config): Native launch arguments and extraction remain owned by Crawl4AI. The profile, browser and driver are all closed when the worker exits. """ - import asyncio - import pathlib - import tempfile - from playwright.async_api import async_playwright from crawl4ai.browser_manager import BrowserManager + from playwright.async_api import async_playwright if config.chrome_channel != "chrome": raise ValueError("public_doh requires the frozen Chrome channel") @@ -293,15 +297,11 @@ async def crawl4ai_public_dns_browser(config): def _native_dns_sources(): - import pathlib return [pathlib.Path(__file__).resolve().parent / "workers/native_dns_darwin.c"] def validate_native_dns_library(library): """Bind native resolver bytes to current source, architecture and platform.""" - import hashlib - import pathlib - import platform if platform.system() != "Darwin": raise ValueError("native public DNS bridge currently requires Darwin") if not library: @@ -323,11 +323,6 @@ def validate_native_dns_library(library): def build_native_dns_library(destination): """Explicit preparation step; never compile or swap runtimes during a call.""" - import hashlib - import pathlib - import platform - import subprocess - import tempfile if platform.system() != "Darwin": raise ValueError("native public DNS bridge currently requires Darwin") path = pathlib.Path(destination).expanduser().resolve() diff --git a/harness/web_fetch/network_preflight.py b/harness/web_fetch/network_preflight.py index b2de69826..ad20dbd35 100644 --- a/harness/web_fetch/network_preflight.py +++ b/harness/web_fetch/network_preflight.py @@ -1,9 +1,12 @@ """Reject non-public DNS addresses before a public local/remote comparison spends calls.""" + import ipaddress import socket from concurrent.futures import ThreadPoolExecutor from urllib.parse import urlsplit +from harness.web_fetch import network_dns + GUARDED_NATIVE_TYPES = {"openclaw_builtin", "deepseek_harness_builtin", "cowagent_builtin", "langchain_webbase", "moli_webfetch", "crawl4ai"} _FAKE_IP_RANGE = ipaddress.ip_network("198.18.0.0/15") @@ -67,7 +70,6 @@ def require_usable_public_dns(audit, *, allow_partial_resolution_failures=False) def preflight_guarded_providers(providers, cases): """Require one declared DNS environment for every local comparison method.""" - from .network_dns import lookup_public_dns, validate_dns_mode native = {"openclaw_builtin", "deepseek_harness_builtin", "cowagent_builtin", "langchain_webbase", "moli_webfetch", "crawl4ai"} selected = [ provider for provider in providers @@ -91,7 +93,7 @@ def preflight_guarded_providers(providers, cases): selected = [provider for provider in selected if provider not in isolated] if not selected: return None - modes = {validate_dns_mode(provider.config.get("dns_mode", "system")) for provider in selected} + modes = {network_dns.validate_dns_mode(provider.config.get("dns_mode", "system")) for provider in selected} if len(modes) != 1: raise ValueError("Selected local methods must declare the same dns_mode") mode = modes.pop() @@ -101,7 +103,7 @@ def preflight_guarded_providers(providers, cases): def resolve(host, port, **kwargs): return [(socket.AF_INET if family == 4 else socket.AF_INET6, socket.SOCK_STREAM, 6, '', (address, port)) - for address, family in lookup_public_dns(host, timeout=45)] + for address, family in network_dns.lookup_public_dns(host, timeout=45)] audit = inspect_system_dns([case.url for case in cases], resolver=resolve) audit["dns_mode"] = mode audit["unresolved_hosts"] = [ diff --git a/harness/web_fetch/providers/__init__.py b/harness/web_fetch/providers/__init__.py index 4f4ae9db0..807ff8aa5 100644 --- a/harness/web_fetch/providers/__init__.py +++ b/harness/web_fetch/providers/__init__.py @@ -1,3 +1,4 @@ -from .base import Provider, build_provider + +from harness.web_fetch.providers.base import Provider, build_provider __all__ = ["Provider", "build_provider"] diff --git a/harness/web_fetch/providers/base.py b/harness/web_fetch/providers/base.py index e5090a931..935c7c6ee 100644 --- a/harness/web_fetch/providers/base.py +++ b/harness/web_fetch/providers/base.py @@ -1,15 +1,20 @@ from __future__ import annotations -from abc import ABC, abstractmethod import os import signal import subprocess import time +from abc import ABC, abstractmethod +from copy import deepcopy +from functools import partial from typing import Any from lexbench_webfetch.models import EvalCase + from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import NATIVE_RETRIEVAL_HINTS +from harness.web_fetch.native_routes import normalized_http_parameters +from harness.web_fetch.providers.http_client import inner_timeout_budget +from harness.web_fetch.retrieval_hints import NATIVE_RETRIEVAL_HINTS def terminate_owned_process_tree(process): @@ -89,7 +94,6 @@ class Provider(ABC): def native_configured_parameters(self): """Return the adapter's explicitly public configuration diagnostics.""" - from copy import deepcopy return {key: deepcopy(self.config[key]) for key in self.native_configuration_fields if key in self.config} @@ -101,8 +105,6 @@ def __init__(self, name: str, config: dict[str, Any]): self.name = name self.config = config self.type = str(config.get("type", name)) - from functools import partial - from .http_client import inner_timeout_budget self._native_mode = bool(config.get('uniform_native_parameters')) self.http_timeout_budget = partial(inner_timeout_budget, native_default=self._native_mode) self.request_arguments = (self._native_request_arguments if self._native_mode @@ -148,7 +150,6 @@ def native_http_request(self, case, callback, endpoint, *, native_route_ledger=N raise ValueError('Native adapter arguments differ from its canonical request builder') endpoint, kwargs = native_endpoint, native_arguments if ledger is not None: - from ..native_routes import normalized_http_parameters self.capture_native_request(case, transport='http', endpoint=endpoint, native_route_ledger=ledger, parameters=normalized_http_parameters(case, kwargs)) return callback(endpoint, **kwargs) @@ -198,50 +199,50 @@ def begin_call(self) -> None: def build_provider(name: str, config: dict[str, Any]) -> Provider: provider_type = config.get("type") if provider_type in {"headless_html", "headless_markdown"}: - from .headless import HeadlessProvider + from harness.web_fetch.providers.headless import HeadlessProvider return HeadlessProvider(name, config) if provider_type == "moli_webfetch": - from .moli import MoliWebFetchProvider + from harness.web_fetch.providers.moli import MoliWebFetchProvider return MoliWebFetchProvider(name, config) if provider_type == "http": - from .http import HttpProvider + from harness.web_fetch.providers.http import HttpProvider return HttpProvider(name, config) if provider_type == "cowagent_builtin": - from .cowagent import CowAgentBuiltinProvider + from harness.web_fetch.providers.cowagent import CowAgentBuiltinProvider return CowAgentBuiltinProvider(name, config) if provider_type in {"lexmount", "lexmount_cn"}: - from .lexmount import LexmountProvider + from harness.web_fetch.providers.lexmount import LexmountProvider return LexmountProvider(name, config) if provider_type == "firecrawl": - from .firecrawl import FirecrawlProvider + from harness.web_fetch.providers.firecrawl import FirecrawlProvider return FirecrawlProvider(name, config) if provider_type == "tavily_extract": - from .tavily import TavilyExtractProvider + from harness.web_fetch.providers.tavily import TavilyExtractProvider return TavilyExtractProvider(name, config) if provider_type == "exa_contents": - from .exa import ExaContentsProvider + from harness.web_fetch.providers.exa import ExaContentsProvider return ExaContentsProvider(name, config) if provider_type == "cloudflare_markdown": - from .cloudflare import CloudflareMarkdownProvider + from harness.web_fetch.providers.cloudflare import CloudflareMarkdownProvider return CloudflareMarkdownProvider(name, config) if provider_type == "cloudflare_content": - from .cloudflare import CloudflareContentProvider + from harness.web_fetch.providers.cloudflare import CloudflareContentProvider return CloudflareContentProvider(name, config) if provider_type == "subprocess_jsonl": - from .subprocess_jsonl import SubprocessJsonlProvider + from harness.web_fetch.providers.subprocess_jsonl import SubprocessJsonlProvider return SubprocessJsonlProvider(name, config) if provider_type == "fixture": - from .fixture import FixtureProvider + from harness.web_fetch.providers.fixture import FixtureProvider return FixtureProvider(name, config) raise ValueError(f"unknown provider type: {provider_type!r}") diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py index d516a2310..39d747daa 100644 --- a/harness/web_fetch/providers/cloudflare.py +++ b/harness/web_fetch/providers/cloudflare.py @@ -6,18 +6,26 @@ import time import urllib.error import urllib.request -from pathlib import Path -from pathlib import PurePosixPath +from pathlib import Path, PurePosixPath from urllib.parse import urlencode, urlsplit from lexbench_webfetch.models import EvalCase -from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import native_contract_for_case, native_default_case, plan_for_case from lexbench_webfetch.paths import validate_http_url -from ..native_projection import native_answer_projection -from .base import Provider -from ..native_routes import NativeRoute -from .http_client import HttpOnlyRedirectHandler, HttpResponse, request + +from harness.web_fetch.models import ProviderResult +from harness.web_fetch.native_projection import ( + SRCSET_PARSER, + native_answer_projection, + project_native_answer, +) +from harness.web_fetch.native_routes import NativeRoute +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import HttpOnlyRedirectHandler, HttpResponse, request +from harness.web_fetch.retrieval_hints import ( + native_contract_for_case, + native_default_case, + plan_for_case, +) _DIRECT_DOCUMENT_EXTENSIONS = ( ".pdf", @@ -499,7 +507,6 @@ def _fetch_media_resources( if not isinstance(payload, dict): payload = {} envelope_success = payload.get("success") is True - from ..native_projection import project_native_answer, SRCSET_PARSER answer_key = 'images' if case.track == 'image' else 'videos' meta = payload.get('meta') if isinstance(payload.get('meta'), dict) else {} effective_url = meta.get('finalUrl') or case.url diff --git a/harness/web_fetch/providers/cowagent.py b/harness/web_fetch/providers/cowagent.py index 4dc22d5f5..6acd76216 100644 --- a/harness/web_fetch/providers/cowagent.py +++ b/harness/web_fetch/providers/cowagent.py @@ -8,15 +8,15 @@ import sys from pathlib import Path -from .subprocess_jsonl import SubprocessJsonlProvider +from harness.web_fetch.providers.subprocess_jsonl import SubprocessJsonlProvider +from harness.web_fetch.safety import unlinked_path, write_new_file +from harness.web_fetch.storage import data_home class CowAgentBuiltinProvider(SubprocessJsonlProvider): """Invoke CowAgent's repository-local WebFetch implementation unchanged.""" def __init__(self, name, config): - from ..storage import data_home - worker = Path(__file__).parents[1] / 'workers/cowagent_worker.py' workspace = Path(str(config.get( "workspace", data_home() / "runs/cowagent-workspace" @@ -32,7 +32,6 @@ def __init__(self, name, config): super().__init__(name, derived) - EXTRACTOR_FIXTURE = '''Fixture

Example article

BODY_FACT

DETAIL_LABEL @@ -66,7 +65,6 @@ def probe_extractor(source): def probe_main(argv=None): """Execute an extractor from trusted local source, without importing its module.""" - from ..safety import unlinked_path, write_new_file parser = argparse.ArgumentParser(description="Offline synthetic CowAgent extractor probe") parser.add_argument("source", type=Path) diff --git a/harness/web_fetch/providers/exa.py b/harness/web_fetch/providers/exa.py index 6a6b057df..011fc817b 100644 --- a/harness/web_fetch/providers/exa.py +++ b/harness/web_fetch/providers/exa.py @@ -4,12 +4,12 @@ import hashlib from lexbench_webfetch.models import EvalCase -from harness.web_fetch.models import ProviderResult -from ..native_projection import native_answer_projection -from .base import Provider -from ..native_routes import NativeRoute -from .http_client import request +from harness.web_fetch.models import ProviderResult +from harness.web_fetch.native_projection import native_answer_projection +from harness.web_fetch.native_routes import NativeRoute +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import request # Exa's per-URL failure envelope contains free-form JSON. Publish only the # stable diagnostic enum values observed in the API contract; the complete diff --git a/harness/web_fetch/providers/firecrawl.py b/harness/web_fetch/providers/firecrawl.py index 541e77455..504b4d8fb 100644 --- a/harness/web_fetch/providers/firecrawl.py +++ b/harness/web_fetch/providers/firecrawl.py @@ -7,12 +7,13 @@ from urllib.parse import urlsplit from lexbench_webfetch.models import EvalCase + from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import plan_for_case, native_default_case -from ..native_projection import native_answer_projection -from .base import Provider -from ..native_routes import NativeRoute -from .http_client import request +from harness.web_fetch.native_projection import native_answer_projection +from harness.web_fetch.native_routes import NativeRoute +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import request +from harness.web_fetch.retrieval_hints import native_default_case, plan_for_case class FirecrawlProvider(Provider): diff --git a/harness/web_fetch/providers/fixture.py b/harness/web_fetch/providers/fixture.py index 02849cfea..cad0aa100 100644 --- a/harness/web_fetch/providers/fixture.py +++ b/harness/web_fetch/providers/fixture.py @@ -5,9 +5,10 @@ import json from lexbench_webfetch.models import EvalCase + from harness.web_fetch.models import ProviderResult -from .base import Provider -from ..native_routes import NativeRoute +from harness.web_fetch.native_routes import NativeRoute +from harness.web_fetch.providers.base import Provider class FixtureProvider(Provider): diff --git a/harness/web_fetch/providers/headless.py b/harness/web_fetch/providers/headless.py index 82fb1d702..5b112c611 100644 --- a/harness/web_fetch/providers/headless.py +++ b/harness/web_fetch/providers/headless.py @@ -2,29 +2,32 @@ from __future__ import annotations import base64 -from contextlib import contextmanager -import socket import hashlib import importlib.util import json import math import os import shutil +import socket import subprocess import sys import tempfile import threading import time +from contextlib import contextmanager from pathlib import Path from urllib.parse import urlsplit -from ..markdown import converter_package, require_converter from lexbench_webfetch.models import EvalCase -from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import native_contract_for_case, plan_for_case from lexbench_webfetch.paths import validate_http_url -from .base import Provider, terminate_owned_process_tree -from .runtime_identity import require_expected_provider_versions + +from harness.web_fetch.markdown import converter_package, require_converter +from harness.web_fetch.models import ProviderResult +from harness.web_fetch.network_dns import validate_dns_mode, validate_native_dns_library +from harness.web_fetch.providers.base import Provider, terminate_owned_process_tree +from harness.web_fetch.providers.cloudflare import CloudflareMarkdownProvider +from harness.web_fetch.providers.runtime_identity import require_expected_provider_versions +from harness.web_fetch.retrieval_hints import native_contract_for_case, plan_for_case ENGINES = ('kitesurf', 'lightpanda', 'obscura', 'moli', 'chromium') LOOPBACK_SANDBOX_PROFILE = ( @@ -234,7 +237,11 @@ def cli_command( raise ValueError('unsupported native metadata route') ms = str(max(1, math.ceil(timeout * 1000))) if output_mode in {'resource', 'media_metadata', 'document_page'}: - from ..task_outputs import MEDIA_METADATA_EXPRESSION, DOCUMENT_PAGE_EXPRESSION, resource_expression + from harness.web_fetch.task_outputs import ( + DOCUMENT_PAGE_EXPRESSION, + MEDIA_METADATA_EXPRESSION, + resource_expression, + ) expression = resource_expression(track, detail_links) if output_mode == 'resource' else {'media_metadata': MEDIA_METADATA_EXPRESSION, 'document_page': DOCUMENT_PAGE_EXPRESSION}[output_mode] if engine == 'obscura': command = [binary, 'fetch', '--eval', expression, '--timeout', str(math.ceil(timeout)), '--quiet'] @@ -379,7 +386,6 @@ def validate(self): except (KeyError, OSError, ValueError) as error: errors.append('native_metadata:' + str(error)) resource_interval = self.config.get('resource_sample_interval_ms') - from ..network_dns import validate_dns_mode, validate_native_dns_library try: dns_mode = validate_dns_mode(self.config.get('dns_mode', 'system')) if engine == 'kitesurf' and dns_mode != 'system': @@ -438,7 +444,7 @@ def _expected_version_errors(self, errors): return errors def _resource_runner(self): - from ..container_resources import ContainerResourceRunner + from harness.web_fetch.container_resources import ContainerResourceRunner return ContainerResourceRunner( self.config['resource_container_image'], @@ -575,7 +581,12 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: else 'public cloud CDP' if engine == 'kitesurf' else 'local process')} if output_mode in {'resource', 'media_metadata', 'document_page'}: - from ..task_outputs import MEDIA_METADATA_EXPRESSION, DOCUMENT_PAGE_EXPRESSION, resource_expression, resource_detail_links + from harness.web_fetch.task_outputs import ( + DOCUMENT_PAGE_EXPRESSION, + MEDIA_METADATA_EXPRESSION, + resource_detail_links, + resource_expression, + ) projection = resource_expression(case.track, resource_detail_links(case)) if output_mode == 'resource' else {'media_metadata': MEDIA_METADATA_EXPRESSION, 'document_page': DOCUMENT_PAGE_EXPRESSION}[output_mode] metadata['output_projection_sha256'] = hashlib.sha256(projection.encode('utf-8')).hexdigest() normalized_plan = completion_plan.worker_dict() if completion_plan else None @@ -615,7 +626,6 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: 'harness_content_assembly': False, } if engine == 'kitesurf' and output_mode == 'markdown' and resource_interval is None: - from .cloudflare import CloudflareMarkdownProvider cloudflare = CloudflareMarkdownProvider(self.name, { 'account_id': self.config['account_id'], 'api_token': self.config['api_token'], 'browser': 'kitesurf', 'wait_until': self.config.get('wait_until', 'networkidle2'), @@ -827,7 +837,7 @@ def native_cdp_browser(playwright, request, deadline, metadata=None): browser = None with tempfile.TemporaryFile() as log: environment = {**os.environ, 'SSL_CERT_FILE': str(request['ca_cert'])} if engine == 'obscura' and request.get('ca_cert') else None - from ..network_dns import native_process_dns_environment + from harness.web_fetch.network_dns import native_process_dns_environment with native_process_dns_environment( request.get('dns_mode', 'system'), request.get('native_dns_library'), diff --git a/harness/web_fetch/providers/headless_worker.py b/harness/web_fetch/providers/headless_worker.py index 344d0088a..481a187c7 100644 --- a/harness/web_fetch/providers/headless_worker.py +++ b/harness/web_fetch/providers/headless_worker.py @@ -7,8 +7,8 @@ import shutil import subprocess import sys -import time import tempfile +import time from importlib.metadata import version from pathlib import Path @@ -19,11 +19,25 @@ if sys.path and Path(sys.path[0]).resolve() == _WORKER_DIRECTORY: sys.path.pop(0) sys.path.insert(0, str(Path(__file__).resolve().parents[3])) -from harness.web_fetch.markdown import CONVERTER, converter_command, require_converter -from harness.web_fetch.document_text import extract_document_text -from harness.web_fetch.providers.headless import cli_command, completion_metadata, native_cdp_browser + from lexbench_webfetch.paths import validate_http_url +from harness.web_fetch import network_dns +from harness.web_fetch.document_text import extract_document_text +from harness.web_fetch.markdown import CONVERTER, converter_command, require_converter +from harness.web_fetch.providers.headless import ( + cli_command, + completion_metadata, + native_cdp_browser, +) +from harness.web_fetch.retrieval_hints import native_contract_for_case, parse_retrieval_hints +from harness.web_fetch.task_outputs import ( + DOCUMENT_PAGE_EXPRESSION, + DOCUMENT_PAGE_READY, + MEDIA_METADATA_EXPRESSION, + resource_expression, +) + def browser_error_type(error): """Normalize known main-navigation failures without retrying security errors.""" @@ -47,7 +61,6 @@ def browser_error_type(error): def require_normalized_retrieval_plan(request): """Keep the declared family while validating explicit native completion.""" - from harness.web_fetch.retrieval_hints import native_contract_for_case, parse_retrieval_hints plan = request.get('retrieval_plan') track = request.get('track') hints = request.get('retrieval_hints') @@ -97,10 +110,9 @@ def _native_browser_file(page, request, route_timeout, start, meta): remaining_ms()/1000, output_mode='file', ca_cert=request.get('ca_cert'), http_proxy=request.get('http_proxy')) if request.get('block_private_networks'): command.append('--block-private-networks') - from harness.web_fetch.network_dns import native_process_dns_environment attachment_dns = {} meta['native_attachment_dns'] = attachment_dns - with native_process_dns_environment( + with network_dns.native_process_dns_environment( request.get('dns_mode', 'system'), request.get('native_dns_library'), start + route_timeout, attachment_dns, ) as process_environment: @@ -144,7 +156,6 @@ def _native_browser_file(page, request, route_timeout, start, meta): def _projection_expression(output_mode, track, detail_links=False): - from harness.web_fetch.task_outputs import MEDIA_METADATA_EXPRESSION, DOCUMENT_PAGE_EXPRESSION, resource_expression return resource_expression(track, detail_links) if output_mode == 'resource' else {'media_metadata': MEDIA_METADATA_EXPRESSION, 'document_page': DOCUMENT_PAGE_EXPRESSION}[output_mode] @@ -202,7 +213,6 @@ def remaining_ms(): def _page_projection(page, request, output_mode, meta, deadline): if output_mode == 'document_page': - from harness.web_fetch.task_outputs import DOCUMENT_PAGE_READY page.wait_for_function(DOCUMENT_PAGE_READY, timeout=max(1, (deadline-time.monotonic())*1000)) return _projection_text(_evaluate_document(page, _projection_expression(output_mode, request.get('track'), request.get('resource_detail_links', False)), deadline, meta), output_mode, meta, deadline) @@ -288,8 +298,7 @@ def fetch(request): environment = None if engine == 'obscura' and request.get('ca_cert'): environment = {**os.environ, 'SSL_CERT_FILE': str(request['ca_cert'])} - from harness.web_fetch.network_dns import native_process_dns_environment - with native_process_dns_environment( + with network_dns.native_process_dns_environment( request.get('dns_mode', 'system'), request.get('native_dns_library'), start + route_timeout, diff --git a/harness/web_fetch/providers/http.py b/harness/web_fetch/providers/http.py index 25944f086..f8652e662 100644 --- a/harness/web_fetch/providers/http.py +++ b/harness/web_fetch/providers/http.py @@ -5,9 +5,10 @@ from html.parser import HTMLParser from lexbench_webfetch.models import EvalCase + from harness.web_fetch.models import ProviderResult -from .base import Provider -from .http_client import inner_timeout_budget, request +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import inner_timeout_budget, request class TextExtractor(HTMLParser): diff --git a/harness/web_fetch/providers/http_client.py b/harness/web_fetch/providers/http_client.py index 8dc717ea4..7fe1132b1 100644 --- a/harness/web_fetch/providers/http_client.py +++ b/harness/web_fetch/providers/http_client.py @@ -9,7 +9,8 @@ from typing import Any from lexbench_webfetch.paths import validate_http_url -from ..diagnostics import emit + +from harness.web_fetch.diagnostics import emit @dataclass diff --git a/harness/web_fetch/providers/lexmount.py b/harness/web_fetch/providers/lexmount.py index 8c44a3331..a0c1daa11 100644 --- a/harness/web_fetch/providers/lexmount.py +++ b/harness/web_fetch/providers/lexmount.py @@ -1,11 +1,11 @@ from __future__ import annotations from lexbench_webfetch.models import EvalCase -from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import plan_for_case -from .base import Provider -from .http_client import inner_timeout_budget, request +from harness.web_fetch.models import ProviderResult +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import inner_timeout_budget, request +from harness.web_fetch.retrieval_hints import plan_for_case _ENGINES = { "auto", diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py index fdda5dbb9..5d1d5fdf1 100644 --- a/harness/web_fetch/providers/moli.py +++ b/harness/web_fetch/providers/moli.py @@ -3,7 +3,6 @@ from __future__ import annotations import base64 -from contextlib import ExitStack import hashlib import math import os @@ -13,19 +12,28 @@ import tempfile import threading import time +from contextlib import ExitStack from pathlib import Path from urllib.parse import urlsplit -from ..native_projection import main_document_url_from_fetch_log as _diagnostic_main_document_url +from lexbench_webfetch.models import TRACKS, EvalCase +from lexbench_webfetch.paths import validate_http_url -from lexbench_webfetch.models import EvalCase, TRACKS from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import NATIVE_RETRIEVAL_HINTS, native_contract_for_case, plan_for_case -from lexbench_webfetch.paths import validate_http_url -from ..native_projection import native_answer_projection -from .base import Provider, terminate_owned_process_tree -from ..native_routes import NativeRoute -from .http_client import inner_timeout_budget +from harness.web_fetch.native_projection import ( + main_document_url_from_fetch_log as _diagnostic_main_document_url, +) +from harness.web_fetch.native_projection import native_answer_projection, native_fetch_diagnostics +from harness.web_fetch.native_routes import NativeRoute, declared_input_media_kind +from harness.web_fetch.network_dns import validate_dns_mode, validate_native_dns_library +from harness.web_fetch.providers.base import Provider, terminate_owned_process_tree +from harness.web_fetch.providers.headless import HeadlessProvider +from harness.web_fetch.providers.http_client import inner_timeout_budget +from harness.web_fetch.retrieval_hints import ( + NATIVE_RETRIEVAL_HINTS, + native_contract_for_case, + plan_for_case, +) _WAIT_UNTIL_VALUES = { "done", @@ -112,7 +120,6 @@ def _moli_native_error_type(stderr: str | bytes) -> str: def _native_fetch_diagnostics(stderr: bytes, requested_url: str) -> tuple[int | None, bool]: """Use the same capture-bounded native parser as saved-call replay.""" - from ..native_projection import native_fetch_diagnostics return native_fetch_diagnostics(stderr, requested_url) @@ -131,8 +138,6 @@ def replay_native_http_error_context(stdout: bytes, diagnostic: bytes, requested 'error_message': f'Moli returned HTTP {status}'} - - def _route_for_case(case: EvalCase, *, native_default: bool = False) -> str: """Select one output contract before applying page-family extraction rules.""" if case.task_kind == "file": @@ -140,7 +145,6 @@ def _route_for_case(case: EvalCase, *, native_default: bool = False) -> str: if native_default: if case.task_kind == "resource": return "native_resource_html" - from ..native_routes import declared_input_media_kind media = declared_input_media_kind(case, required=case.track == "document") if case.track == "document" and media in {"direct_pdf", "direct_docx"}: return "native_document_bytes" @@ -185,7 +189,6 @@ def _resolve_binary(self) -> str | None: def _configuration_errors(self, binary: str | None) -> list[str]: errors = [] - from ..network_dns import validate_dns_mode, validate_native_dns_library try: if validate_dns_mode(self.config.get("dns_mode", "system")) == "public_doh": validate_native_dns_library(self.config.get("native_dns_library")) @@ -305,7 +308,7 @@ def _validate_resolved_binary(self, binary: str | None) -> list[str]: "binary_sha256": binary_sha256, } if self.config.get("dns_mode", "system") == "public_doh": - from ..network_dns import validate_native_dns_library + from harness.web_fetch.network_dns import validate_native_dns_library _, binding = validate_native_dns_library(self.config.get("native_dns_library")) self._implementation_identity["native_dns"] = binding return errors @@ -332,7 +335,6 @@ def _native_file_delivery(case, binary, call_deadline): def _diagnostic_file_delivery(self, case, binary, call_deadline): if case.task_kind != 'file': return None - from .headless import HeadlessProvider native = HeadlessProvider(self.name, {**self.config, 'type':'headless_markdown', 'engine':'moli', 'binary':binary, 'max_original_file_bytes':int(self.config.get('max_bytes',20_000_000))}) @@ -627,7 +629,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> diagnostic = spool_stack.enter_context( Path(str(sink.path) + ".moli.stderr").open("x+b") if sink else tempfile.TemporaryFile()) if sink: sink.emit("caller", "native_cli_start", command=command) - from ..network_dns import NativeDnsBroker, validate_native_dns_library + from harness.web_fetch.network_dns import NativeDnsBroker, validate_native_dns_library dns_broker = None process_environment = None metadata["dns_mode"] = self.config.get("dns_mode", "system") diff --git a/harness/web_fetch/providers/subprocess_jsonl.py b/harness/web_fetch/providers/subprocess_jsonl.py index 52679d353..316559b67 100644 --- a/harness/web_fetch/providers/subprocess_jsonl.py +++ b/harness/web_fetch/providers/subprocess_jsonl.py @@ -1,8 +1,8 @@ from __future__ import annotations import base64 -import json import hashlib +import json import os import queue import subprocess @@ -12,12 +12,15 @@ from typing import Any from lexbench_webfetch.models import EvalCase + +from harness.record_io import canonical_json_sha256 from harness.web_fetch.models import ProviderResult -from ..retrieval_hints import plan_for_case, native_default_case -from ..native_projection import native_answer_projection -from .base import Provider -from ..native_routes import NativeRoute -from .runtime_identity import validate_runtime_identity +from harness.web_fetch.native_projection import native_answer_projection +from harness.web_fetch.native_routes import NativeRoute, declared_input_media_kind +from harness.web_fetch.network_dns import validate_dns_mode +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.runtime_identity import validate_runtime_identity +from harness.web_fetch.retrieval_hints import native_default_case, plan_for_case class SubprocessJsonlProvider(Provider): @@ -89,7 +92,6 @@ def _resolve_token(self, value: Any) -> str: def validate(self) -> list[str]: if hasattr(self, "_implementation_identity"): del self._implementation_identity - from ..network_dns import validate_dns_mode try: mode = validate_dns_mode(self.config.get("dns_mode", "system")) except ValueError as error: @@ -203,7 +205,6 @@ def _request_stderr(self, cursor: tuple[int, int]) -> dict[str, Any]: } def native_worker_request(self, case, timeout, diagnostic_path=None): - from ..native_routes import declared_input_media_kind media = declared_input_media_kind(case, required=False) return {'url': case.url, 'log_level': self.config.get('log_level'), 'diagnostic_path': diagnostic_path, 'timeout_seconds': timeout, @@ -235,7 +236,6 @@ def _diagnostic_worker_request(self, case, timeout, diagnostic_path, input_media def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> ProviderResult: self.check_native_call(case) - from ..native_routes import declared_input_media_kind input_media = declared_input_media_kind(case, required=bool( self._native_mode and case.track == "document")) answer_begin = time.perf_counter() @@ -262,7 +262,6 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> str(self._active_diagnostics.path) if self._active_diagnostics else None, input_media, retrieval_plan) if native_route_ledger is not None: - from harness.record_io import canonical_json_sha256 projected = {**request, 'url': '', 'diagnostic_path': '' if request['diagnostic_path'] else None} self.capture_native_request(case, native_route_ledger=native_route_ledger, transport='jsonl', interface='native_worker_command', diff --git a/harness/web_fetch/providers/tavily.py b/harness/web_fetch/providers/tavily.py index 8e1429028..dbfd4c021 100644 --- a/harness/web_fetch/providers/tavily.py +++ b/harness/web_fetch/providers/tavily.py @@ -4,12 +4,12 @@ import hashlib from lexbench_webfetch.models import EvalCase -from harness.web_fetch.models import ProviderResult -from ..native_projection import native_answer_projection -from .base import Provider -from ..native_routes import NativeRoute -from .http_client import request +from harness.web_fetch.models import ProviderResult +from harness.web_fetch.native_projection import native_answer_projection +from harness.web_fetch.native_routes import NativeRoute +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import request # Tavily's failed_results entries are free-form. Keep the useful public API # diagnostics, but never project arbitrary response text into public evidence. diff --git a/harness/web_fetch/publication.py b/harness/web_fetch/publication.py index 38f7e76f2..7c776d0ac 100644 --- a/harness/web_fetch/publication.py +++ b/harness/web_fetch/publication.py @@ -1,21 +1,22 @@ """Artifact Hub large-bundle transport; credentials remain with the caller.""" + +import hashlib +import io import json import math import socket import time import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed from email.utils import parsedate_to_datetime from http.client import RemoteDisconnected -from pathlib import PurePosixPath +from pathlib import Path, PurePosixPath from urllib.error import HTTPError, URLError from urllib.parse import quote def verify_published_tree(source, archive, *, excluded=('.generated-report',)): """Compare every deployed file with the submitted tree without extracting ZIPs.""" - import hashlib - import io - from pathlib import Path source = Path(source) paths = list(source.rglob('*')) if any(p.is_symlink() for p in paths): @@ -124,7 +125,6 @@ def check_baseline(): put_file(request, endpoint + '/files/index.html', bundle.read(entry)) remaining = [info for info in files if info.filename != 'index.html'] if max_workers > 1: - from concurrent.futures import ThreadPoolExecutor, as_completed with ThreadPoolExecutor(max_workers=max_workers) as pool: pending = { pool.submit(put_file, request, diff --git a/harness/web_fetch/publication_check.py b/harness/web_fetch/publication_check.py index f1aa38f5b..1f7833def 100644 --- a/harness/web_fetch/publication_check.py +++ b/harness/web_fetch/publication_check.py @@ -6,23 +6,28 @@ import json import zipfile from contextlib import ExitStack -from urllib.parse import urlsplit, parse_qs, quote from pathlib import Path +from urllib.parse import parse_qs, quote, urlsplit from bs4 import BeautifulSoup +from lexbench_webfetch.paths import contained_path from harness.record_io import parse_jsonl from harness.reporting.links import validate_virtual_links -from harness.reporting.packs import DecompressedPackCache, read_packed_manifest, reconstruct_packed_file +from harness.reporting.packs import ( + DecompressedPackCache, + read_packed_manifest, + reconstruct_packed_file, +) from harness.reporting.publication import archive_inventory from harness.web_fetch.publication_privacy import ( + bound_public_view, project_file, project_redundant_result_streams, - publication_risk, publication_file_risk, - bound_public_view, + publication_risk, ) -from lexbench_webfetch.paths import contained_path +from harness.web_fetch.report_package import DATA_ATTACHMENT_HTML, validate_release from harness.web_fetch.safety import evidence_tree_identity from harness.web_fetch.webmainbench_diagnostics import ( require_valid_teds_for_release, @@ -39,7 +44,6 @@ def scored_output_hash(row: dict) -> str: def check(archive, source, overlay=None, *, presentation_binding=None, external_archives=None, allow_pending_external=False, snapshot_audit=False): - from .report_package import validate_release, DATA_ATTACHMENT_HTML validate_release(source, presentation_binding=presentation_binding, snapshot_audit=snapshot_audit) source_files = evidence_tree_identity(source) physical = archive_inventory(archive) @@ -100,7 +104,7 @@ def read(name): if overlay and (manifest["blocks"] != baseline["blocks"] or set(manifest["files"]) != set(baseline["files"])): raise ValueError("Incremental overlay changed the frozen pack or file inventory") if overlay: - from .report_package import explanatory_route + from harness.web_fetch.report_package import explanatory_route for name, entry in manifest["files"].items(): if entry != baseline["files"][name] and not (explanatory_route(name) or name == "publication-privacy.json"): raise ValueError("Incremental overlay changed measured evidence: " + name) @@ -252,7 +256,7 @@ def data(name): binding={'path':row['artifact_path'],'sha256':expected_hash} public = bound_public_view(source,binding) if public is None and snapshot_audit: - from .published_snapshot import read_snapshot_public_file + from harness.web_fetch.published_snapshot import read_snapshot_public_file retained=read_snapshot_public_file(source,binding) public=(retained,{'role':'scored_answer','published_sha256':hashlib.sha256(original).hexdigest()}) if (public is None or public[1]['role'] != 'scored_answer' diff --git a/harness/web_fetch/publication_privacy.py b/harness/web_fetch/publication_privacy.py index 2fb4a2fca..73facb46f 100644 --- a/harness/web_fetch/publication_privacy.py +++ b/harness/web_fetch/publication_privacy.py @@ -1,7 +1,8 @@ """Project public evidence and execution metadata without changing measurements.""" + import hashlib -from harness.record_io import file_sha256 import html +import io import json import re from collections import OrderedDict @@ -12,12 +13,15 @@ from functools import lru_cache from pathlib import Path, PurePosixPath, PureWindowsPath from threading import Lock -from urllib.parse import unquote_plus +from urllib.parse import unquote_plus, urlsplit + +from lexbench_webfetch.paths import contained_path + +from harness.record_io import file_sha256 def require_public_method_options(configuration): """Reject private values even when their option name is displayable.""" - from urllib.parse import urlsplit def contains_url_credentials(value): if isinstance(value,dict):return any(contains_url_credentials(v) for v in value.values()) if isinstance(value,list):return any(contains_url_credentials(v) for v in value) @@ -489,7 +493,6 @@ def _scan_plain_publication_risk(data): return None - def _secret_placeholder(value): return bool((value.startswith('<') and value.endswith('>')) or (value.startswith('[') and value.endswith(']')) @@ -911,8 +914,6 @@ def bound_public_view(source, binding): A returned view verifies public bytes plus the recorded original identity; it does not claim that omitted original bytes have been reconstructed. """ - from pathlib import Path - from lexbench_webfetch.paths import contained_path source = Path(source) profile_path = contained_path(source, 'presentation-binding.json', reject_symlinks=True) if not profile_path.is_file(): @@ -946,7 +947,6 @@ def read_evidence_binding(source, binding, *, allow_public_view=True): accept their separately verified public projection. Both use the same path, staged-write and hash checks. """ - from lexbench_webfetch.paths import contained_path identity = EvidenceBinding.from_mapping(binding) path = contained_path(source, identity.path, reject_symlinks=True) staged = _STAGED_EVIDENCE.get() @@ -995,7 +995,6 @@ def evidence_excerpt_matches(source, binding, sample, text): def require_public_evidence_view_inventory(source, binding, records): """Require complete current-answer/native-response view coverage at release.""" - from lexbench_webfetch.paths import contained_path if (source/'private_artifacts').exists(): raise ValueError('Public evidence snapshot must not contain private artifact trees') index = json.loads(read_evidence_binding(source, binding).read_text()) @@ -1118,7 +1117,6 @@ def publication_file_risk(name, data): """ if not name.endswith('.jsonl'): return publication_risk(data) - import io for line in io.BytesIO(data): if not line.strip(): continue diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py index 13310e47d..b3c185186 100644 --- a/harness/web_fetch/published_snapshot.py +++ b/harness/web_fetch/published_snapshot.py @@ -6,11 +6,22 @@ import tempfile from pathlib import Path -from harness.record_io import canonical_json_sha256, file_sha256, load_jsonl +from bs4 import BeautifulSoup from lexbench_webfetch.metrics import task_passed from lexbench_webfetch.paths import contained_path from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 -from .scoring_revision import ALLOWED_FIELDS, correction_provenance + +from harness.record_io import canonical_json_sha256, file_sha256, load_jsonl +from harness.report_languages import generated_languages +from harness.reporting.presentation import check_presentation +from harness.reporting.templates import resolve_template +from harness.reporting.visual import browser_executable, local_report_server, settle_theme +from harness.web_fetch.report import generate +from harness.web_fetch.report_build import _install_report_tree, write_frozen_result_summary +from harness.web_fetch.report_check import check_saved_result_summary +from harness.web_fetch.report_explainer import check_current_browser_article +from harness.web_fetch.report_version import check_release, load_release +from harness.web_fetch.scoring_revision import ALLOWED_FIELDS, correction_provenance def _bound(root, binding): @@ -66,7 +77,6 @@ def validate_snapshot_audit(source, profile_path): audit = json.loads(inputs['review'].read_text()) scoring_source = SCORING_SOURCE_SHA256 if config.get('benchmark_release'): - from .report_version import load_release _, release = load_release(source, dataset, records, profile=profile_path) scoring_source = release['scoring_source_sha256'] old_cases = {row['id']: row for row in baseline['cases']} @@ -156,10 +166,6 @@ def validate_snapshot_audit(source, profile_path): def build_snapshot_audit(source, output, *, profile): """Keep the original report routes and metric populations, adding score review.""" - from .report import generate - from .report_build import _install_report_tree, write_frozen_result_summary - from harness.reporting.templates import resolve_template - from harness.report_languages import generated_languages source, output = Path(source).resolve(), Path(output).resolve() if source == output or source in output.parents or output in source.parents: raise ValueError('Snapshot audit needs a separate output tree') @@ -217,7 +223,7 @@ def build_snapshot_audit(source, output, *, profile): (staging / 'dataset.json').write_text(json.dumps(dataset, ensure_ascii=False, indent=2)+'\n') shutil.copy2(inputs['results'], staging / 'results.jsonl') batches = json.loads((staging / 'execution.json').read_text())['batches'] - from .report_version import bind_release + from harness.web_fetch.report_version import bind_release release_binding = bind_release(source, staging, dataset, records, profile=profile) generate(staging, dataset, records, batches, authored_interpretation=True, frozen_scores=True) write_frozen_result_summary(staging, dataset, records) @@ -240,8 +246,6 @@ def build_snapshot_audit(source, output, *, profile): def check_snapshot_audit(root): """Deterministic checks for matrix values, disclosure and reviewed evidence.""" - from bs4 import BeautifulSoup - from .report_check import check_saved_result_summary root = Path(root) manifest=json.loads((root/'report-manifest.json').read_text()) if manifest.get('evidence_state')!='published_snapshot_score_audit' or manifest.get('certified') is not False: @@ -252,7 +256,6 @@ def check_snapshot_audit(root): raise ValueError('Snapshot audit input changed: '+name) dataset = json.loads((root/'dataset.json').read_text()) rows = load_jsonl(root/'results.jsonl') - from .report_version import check_release check_release(root, dataset, rows) if dataset.get('definition_revision'): revision_path=_bound(root,dataset['definition_revision']) @@ -274,8 +277,6 @@ def check_snapshot_audit(root): if scope is None or any(str(value) not in scope.get_text() for value in expected_scope): raise ValueError('Snapshot report lost its scoring input scope') audit_html = BeautifulSoup((root/'audit/index.html').read_text(), 'html.parser') - from .report_explainer import check_current_browser_article - from harness.reporting.presentation import check_presentation check_current_browser_article(root,{case['id']:case for case in dataset['cases']},rows,dataset['comparison']['report_interpretation']) for route in ['index.html','guide/index.html','analysis/index.html','audit/index.html']: check_presentation((root/route).read_text()) @@ -324,7 +325,6 @@ def read_snapshot_public_file(root, binding): def check_snapshot_visual(root, receipt): """Exercise native route classes, themes, narrow layouts and evidence dialogs.""" from playwright.sync_api import sync_playwright - from harness.reporting.visual import browser_executable, local_report_server, settle_theme root, receipt = Path(root).resolve(), Path(receipt).resolve() if receipt == root or root in receipt.parents: raise ValueError('Visual receipts belong outside the public report') diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index 775da6c10..fa079cce6 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -1,4 +1,5 @@ """Freeze independent references; require explicit promotion for scope changes.""" + import argparse import hashlib import json @@ -11,29 +12,39 @@ from pathlib import Path, PureWindowsPath from urllib.parse import parse_qs, urlsplit -from harness.record_io import canonical_json_sha256, read_completed_records -from harness.web_fetch.artifacts import materialize_task_references -from harness.web_fetch.render_evidence import content_requirement_errors from lexbench_webfetch.body_coverage import body_requirements from lexbench_webfetch.metrics import normalize_for_match -from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes, require_mutable_inputs +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.reference_quality import clean_reference_text, reference_anchor_in_text +from lexbench_webfetch.scoring_regressions import ( + ROOT_CAUSE_CATEGORIES, + build_correction_provenance, + valid_correction_provenance, +) + +from harness.record_io import canonical_json_sha256, read_completed_records +from harness.web_fetch.artifacts import materialize_task_references +from harness.web_fetch.reference_inputs import ( + input_tree, + read_dataset_bytes, + require_mutable_inputs, +) from harness.web_fetch.reference_lifecycle import ( + _completion, + check_case_reference_binding, completed_case_records, + legacy_reference_cases, + legacy_reference_dataset, load_capture_observation, + matches_scope_identity, reference_cases, requires_browser_observation, requires_fresh_reference, scope_identity, - matches_scope_identity, - check_case_reference_binding, - legacy_reference_dataset, - legacy_reference_cases, timestamp, validate_postcheck_observation, ) -from lexbench_webfetch.reference_quality import clean_reference_text, reference_anchor_in_text -from lexbench_webfetch.scoring_regressions import ROOT_CAUSE_CATEGORIES, build_correction_provenance, valid_correction_provenance -from lexbench_webfetch.paths import contained_path +from harness.web_fetch.render_evidence import content_requirement_errors from harness.web_fetch.source_capture import capture ROOT = Path(__file__).resolve().parents[2] @@ -173,7 +184,6 @@ def load_browser_observations(dataset, path=None, directory=None): def compute_reference_update(dataset, captures, observations=None, *, refresh_case_ids=()): """Compute the one authoritative frozen dataset/change set from captures.""" - from harness.web_fetch.reference_lifecycle import _completion frozen=deepcopy(dataset);observations=observations or {} audit_rows=json.loads((captures/'audit.json').read_text()) diff --git a/harness/web_fetch/reference_inputs.py b/harness/web_fetch/reference_inputs.py index c5044c914..afa5ad061 100644 --- a/harness/web_fetch/reference_inputs.py +++ b/harness/web_fetch/reference_inputs.py @@ -12,17 +12,25 @@ from urllib.parse import quote, urlsplit, urlunsplit from urllib.request import urlopen +from lexbench_webfetch.inputs import ( + definition_path, + digest, + verify_catalog, + verify_tree, +) +from lexbench_webfetch.inputs import ( + load_manifest as _load_manifest, +) +from lexbench_webfetch.inputs import ( + require_mutable_inputs as validate_mutable_candidate, +) +from lexbench_webfetch.paths import contained_path, safe_identifier + from harness.reporting.assets import copy_bound_assets from harness.reporting.packs import read_packed_file -from lexbench_webfetch.paths import contained_path, safe_identifier from harness.web_fetch.safety import reject_evidence_symlinks from harness.web_fetch.storage import data_home -from lexbench_webfetch.inputs import ( - digest, load_manifest as _load_manifest, verify_catalog, verify_tree, definition_path, - require_mutable_inputs as validate_mutable_candidate, -) - MANIFEST_NAME = "source-inputs.json" ROOT = Path(__file__).resolve().parents[2] diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index f88b5896a..462667f0b 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +import argparse import hashlib import json import math @@ -14,12 +15,15 @@ from pathlib import Path from typing import Any -from harness.record_io import file_sha256, load_jsonl, parse_jsonl - from lexbench_webfetch.metrics import normalize_for_match, normalize_resource_url -from .reference_inputs import input_tree, read_dataset_bytes -from .render_evidence import content_requirement_errors, validate_render_reference +from lexbench_webfetch.paths import contained_path, safe_identifier +from lexbench_webfetch.scoring_regressions import valid_correction_provenance +from lexbench_webfetch.stable_tasks import validate_acceptance_contract +from harness.record_io import file_sha256, load_jsonl, parse_jsonl +from harness.web_fetch.native_routes import verify_native_execution_manifests +from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes +from harness.web_fetch.render_evidence import content_requirement_errors, validate_render_reference HOMEPAGE_READER_SCOPE_PROMPTS = frozenset({ "直接获取这个热门网站首次加载后可见的稳定正文，保留产品、功能、栏目和支持入口文字。" @@ -34,7 +38,6 @@ "不翻页、不滚动触发更多内容、不登录，也不执行其他 Browser Use 操作。图片文字、按钮控件文字、隐藏名称、" "网页标题和脚本文字不能代替正文。", }) -from lexbench_webfetch.paths import contained_path, safe_identifier def _same_homepage_reader_constraints(old: dict, new: dict) -> bool: @@ -141,7 +144,6 @@ def requires_fresh_reference(case) -> bool: """Existing answers stay frozen unless the task explicitly selects live checks.""" contract = _get(case, "acceptance_contract", {}) if contract: - from lexbench_webfetch.stable_tasks import validate_acceptance_contract validate_acceptance_contract(contract) verification = _get(case, "source_verification", {}) or {} mode = verification.get("reference_mode", "frozen") @@ -588,7 +590,6 @@ def build_observations(run_dir: Path, captures_root: Path, def observations_main(argv=None) -> None: """Command boundary for assembling independently captured postchecks.""" - import argparse parser = argparse.ArgumentParser(description=build_observations.__doc__) parser.add_argument("--run-dir", type=Path, required=True) @@ -1050,7 +1051,7 @@ def _valid_scoring_revision(source, *, validate_reviews, review_sha256=None) -> dataset = None records = None if dataset is not None and records is not None: - from .result_review import require_blind_result_review + from harness.web_fetch.result_review import require_blind_result_review for candidate in review_candidates: try: raw = candidate.read_bytes() @@ -1070,7 +1071,7 @@ def _valid_scoring_revision(source, *, validate_reviews, review_sha256=None) -> return False if review_path.is_file(): try: - from .result_review import require_result_review + from harness.web_fetch.result_review import require_result_review if dataset is None or records is None: dataset = json.loads((root/'dataset.json').read_text()) records = load_jsonl(root/'results.jsonl') @@ -1079,7 +1080,6 @@ def _valid_scoring_revision(source, *, validate_reviews, review_sha256=None) -> except (OSError, ValueError, TypeError, KeyError, StopIteration): assembled_review_valid = False - from lexbench_webfetch.scoring_regressions import valid_correction_provenance def revision_edge(value): if not isinstance(value, dict): @@ -1352,7 +1352,6 @@ def reference_revision_cases(source) -> dict[str, set[str]]: return {} current_dataset = sha256((root / "dataset.json").read_bytes()) current_results = file_sha256(root / "results.jsonl") - from lexbench_webfetch.scoring_regressions import valid_correction_provenance if not isinstance(receipts, list): return {} @@ -1611,7 +1610,6 @@ def current_protocol(batch): def valid_hash(value): return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None - from .native_routes import verify_native_execution_manifests verify_native_execution_manifests(source, manifests, {'cases': [dict(c) if isinstance(c, dict) else vars(c) for c in case_map.values()]} if case_map is not None else None, records) diff --git a/harness/web_fetch/reference_revision.py b/harness/web_fetch/reference_revision.py index af9eecac5..1a404beac 100644 --- a/harness/web_fetch/reference_revision.py +++ b/harness/web_fetch/reference_revision.py @@ -13,7 +13,12 @@ from datetime import datetime, timezone from pathlib import Path -from harness.record_io import parse_jsonl +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.scoring_regressions import ROOT_CAUSE_CATEGORIES, build_correction_provenance + +from harness.record_io import load_jsonl, parse_jsonl +from harness.web_fetch.artifacts import verify_record_artifacts +from harness.web_fetch.native_routes import native_cases_by_id from harness.web_fetch.reference_inputs import ( input_tree, read_dataset_bytes, @@ -26,9 +31,12 @@ ) from harness.web_fetch.rescoring import rescore from harness.web_fetch.result_review import archive_superseded_result_review -from harness.web_fetch.safety import commit_staged_evidence, evidence_tree_identity, reject_evidence_symlinks, validate_report_ids -from lexbench_webfetch.paths import contained_path -from lexbench_webfetch.scoring_regressions import ROOT_CAUSE_CATEGORIES, build_correction_provenance +from harness.web_fetch.safety import ( + commit_staged_evidence, + evidence_tree_identity, + reject_evidence_symlinks, + validate_report_ids, +) ROOT = Path(__file__).resolve().parents[2] @@ -299,8 +307,6 @@ def revise( fields: set[str] | None = None, ) -> dict: """Publish a complete reference correction, or preserve the original bundle.""" - from harness.record_io import load_jsonl - from harness.web_fetch.artifacts import verify_record_artifacts reject_evidence_symlinks(source) source = source.resolve() @@ -319,7 +325,6 @@ def revise( or (rows and not validator(source)) ): raise ValueError("Invalid or unbound revision history: " + name) - from .native_routes import native_cases_by_id native_cases = native_cases_by_id(source) for record in load_jsonl(source / "results.jsonl"): verify_record_artifacts(source, record, native_cases=native_cases) diff --git a/harness/web_fetch/refresh_assembly.py b/harness/web_fetch/refresh_assembly.py index c18a644cf..4b415ca66 100644 --- a/harness/web_fetch/refresh_assembly.py +++ b/harness/web_fetch/refresh_assembly.py @@ -13,6 +13,15 @@ from datetime import UTC, datetime from pathlib import Path +from lexbench_webfetch.metrics import task_passed +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.scoring import load_list_reference, score_attempt +from lexbench_webfetch.scoring_regressions import ( + build_correction_provenance, + load_scoring_regressions, + valid_correction_provenance, +) + from harness.record_io import load_jsonl from harness.web_fetch.artifacts import ( materialize_bound_artifacts, @@ -26,17 +35,14 @@ _copy_speed_bundle, _prune_shadowed_batch_attempts, ) -from lexbench_webfetch.metrics import task_passed +from harness.web_fetch.native_routes import native_cases_by_id from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes from harness.web_fetch.reference_lifecycle import ( reference_revision_cases, require_verified_reference_checks, valid_scoring_revision, ) -from lexbench_webfetch.paths import contained_path from harness.web_fetch.safety import reject_evidence_symlinks, validate_report_ids -from lexbench_webfetch.scoring import load_list_reference, score_attempt -from lexbench_webfetch.scoring_regressions import build_correction_provenance, load_scoring_regressions, valid_correction_provenance from harness.web_fetch.speed_cohort import ( build_summary as build_speed_summary, ) @@ -729,7 +735,6 @@ def _validate_layer_execution( def _verify_artifacts(root: Path, rows: list[dict]) -> None: - from .native_routes import native_cases_by_id native_cases = native_cases_by_id(root) for row in rows: verify_record_artifacts(root, row, native_cases=native_cases) diff --git a/harness/web_fetch/render_evidence.py b/harness/web_fetch/render_evidence.py index 1be7bfec3..f1cfd237c 100644 --- a/harness/web_fetch/render_evidence.py +++ b/harness/web_fetch/render_evidence.py @@ -10,7 +10,7 @@ from html.parser import HTMLParser from typing import Any -from lexbench_webfetch.metrics import fact_in_text, normalize_for_match +from lexbench_webfetch.metrics import evaluate_content, fact_in_text, normalize_for_match from lexbench_webfetch.models import EvalCase @@ -95,7 +95,6 @@ def content_requirement_errors(case: EvalCase | Mapping[str, Any], text: str) -> ('acceptance_contract', {}), ('composite_acceptance', {}))} if not any(requirements.values()): return [] - from lexbench_webfetch.metrics import evaluate_content checks = evaluate_content(text, [], **requirements, base_url=get('url', '')) errors = [] if not checks['body_coverage_pass']: diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index b31590f7b..beb1e76f0 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -8,47 +8,118 @@ import shutil import statistics from collections import Counter, defaultdict +from collections import Counter as WordCounter from contextvars import ContextVar from dataclasses import dataclass from functools import wraps +from math import ceil from pathlib import Path from urllib.parse import urlencode, urlsplit -from harness.record_io import load_jsonl +from bs4 import BeautifulSoup +from lexbench_webfetch.body_coverage import ( + _compact_with_positions, + _global_region_allocation, + compact_ngrams, + evaluate_body_requirements, + native_markup_body_text, + tokens, +) +from lexbench_webfetch.dynamic_lists import visible_html_text +from lexbench_webfetch.metrics import normalize_for_match, percentile, task_measured, task_passed +from lexbench_webfetch.models import TRACKS +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.reference_quality import is_html_document_or_fragment +from lexbench_webfetch.response_input import answer_input +from lexbench_webfetch.stable_tasks import SCHEMA_FIELDS, _parse_records, _visible -from .failure_diagnosis import diagnose from harness.numbers import nonnegative_number -from .methods import ROUTE_LABELS, method_page_model, is_browser_comparison, browser_method_label, browser_method_registry, native_browser_engine - -from lexbench_webfetch.metrics import percentile, task_measured, task_passed +from harness.record_io import load_jsonl +from harness.report_languages import ( + check_language_contract, + english_notice, + generated_languages, + primary_report_language, + select_report_language, + write_language_contract, +) +from harness.web_fetch.adjudication_evidence import adjudication_reason +from harness.web_fetch.failure_diagnosis import diagnose +from harness.web_fetch.methods import ( + ROUTE_LABELS, + browser_method_label, + browser_method_registry, + is_browser_comparison, + method_access, + method_page_model, + method_version, + method_versions, + native_browser_engine, + report_method, + report_method_catalog, + versioned_method_label, + with_measured_method_labels, +) from harness.web_fetch.models import REPORT_FAMILIES, execution_id, report_family -from lexbench_webfetch.models import TRACKS -from .report_components import ( +from harness.web_fetch.publication_privacy import ( + project_evidence_view, + public_render_views, + read_evidence_binding, + require_public_method_options, +) +from harness.web_fetch.reliability import assess_reliability +from harness.web_fetch.report_changes import ( + ROOT, + localized_release_history, + render_changes, + render_localized_release_history, + render_webmainbench_history, + require_review_rounds, +) +from harness.web_fetch.report_components import ( basis_label, + comparable_body_reference, examples, + list_rule_description, + measurement_presentation, metric_guide, metric_legend, - measurement_presentation, reference_label, reference_status, reference_text, + render_acceptance_scope, sources_panel, stable_contract_details, tabs, + task_contract_guide, task_instruction, + task_pass_guide, ) -from .report_interfaces import method_provenance -from .methods import method_access, method_version, method_versions - -from .report_methods import PROFILES, adoption_counts -from .methods import report_method, report_method_catalog -from .report_resources import render_resource_comparison -from .report_style import STYLE -from lexbench_webfetch.paths import contained_path +from harness.web_fetch.report_diagnostic import ( + FAILURE_TYPES, + diagnostic_call_summary, + diagnostic_cohort_summary, + diagnostic_failure_counts, + diagnostic_findings, + diagnostic_specimen_excerpt, +) +from harness.web_fetch.report_explainer import ( + _article_speed_population, + load_browser_failure_review, + render_explainer, +) +from harness.web_fetch.report_i18n import write_english_report +from harness.web_fetch.report_interfaces import _observed_route, method_provenance, method_spec +from harness.web_fetch.report_methods import PROFILES, adoption_counts, render_methods +from harness.web_fetch.report_quality import render_quality_observations +from harness.web_fetch.report_reading import render_archived_reading +from harness.web_fetch.report_resources import render_resource_comparison +from harness.web_fetch.report_style import STYLE +from harness.web_fetch.result_ledger import validate_ledger from harness.web_fetch.safety import safe_href, validate_report_ids -from .webmainbench_diagnostics import ERROR_PAGE - - +from harness.web_fetch.wcxb_latency import load_full_paired_latency, summarize_full_paired_latency +from harness.web_fetch.wcxb_scoring import paired_latency_methods +from harness.web_fetch.webmainbench_diagnostics import ERROR_PAGE def format_wcxb_conditional_latency(value: float | None) -> str: @@ -65,7 +136,6 @@ def generate_wcxb(out: Path, rows: list[dict], summary: dict, ground_truth: dict or any(not isinstance(report_content['guide'].get(lang),str) or not report_content['guide'][lang].strip() for lang in languages)): raise ValueError('WCXB authored interpretation lacks requested languages') - from collections import Counter as WordCounter providers=list(summary['providers']) comparison=profile.get('paired_comparison') or {} baseline=comparison.get('baseline') @@ -88,8 +158,6 @@ def generate_wcxb(out: Path, rows: list[dict], summary: dict, ground_truth: dict for row in rows:by_case[row['case_id']][row['provider']]=row if len(by_case)!=case_count or any(set(group)!=set(providers) for group in by_case.values()): raise ValueError('WCXB same-case matrix is incomplete') - from .wcxb_latency import load_full_paired_latency, summarize_full_paired_latency - from .wcxb_scoring import paired_latency_methods paired_methods={} pair={} paired_latency=None @@ -161,7 +229,6 @@ def type_label(lang,value):return type_labels.get(value,value) if lang=='zh' els def browser_label(provider): if profile.get('participants',{}).get(provider,{}).get('label'): return profile['participants'][provider]['label'] - from .methods import versioned_method_label record={'provider':provider, 'provider_metadata':identities[provider]} registered=report_method(provider,record) base=registered.get('label',{}).get('en',identities[provider].get('engine') or provider) @@ -268,8 +335,6 @@ def score_digits(values, direction='max'): flow_labels=(['冻结 HTML',f'{len(providers)} 种配置转换','最终响应与重试','官方逐页评分','有效参考汇总'] if lang=='zh' else ['Frozen HTML',f'{len(providers)} conversions','Final response and retries','Official page score','Qualified aggregate']) flow_source='flowchart LR\n A["'+flow_labels[0]+'"] --> B["'+flow_labels[1]+'"]\n B --> C["'+flow_labels[2]+'"]\n C --> D["'+flow_labels[3]+'"]\n D --> E["'+flow_labels[4]+'"]\n C -- '+('失败重试' if lang=='zh' else 'failure retry')+' --> B' flow='

'+('一页怎样变成一行结果' if lang=='zh' else 'From one page to one result')+'

'+' → '.join(f'{esc(label)}' for label in flow_labels)+'
' - from .report_interfaces import method_spec, _observed_route - from .methods import native_browser_engine measured_records={p:{'provider':p,'provider_metadata':identities[p]} for p in providers} route_descriptions={p:_observed_route(native_browser_engine(p,measured_records[p]),identities[p],language=lang) for p in providers} interface_specs={p:method_spec(p,measured_records[p],language=lang) for p in providers} @@ -353,7 +418,6 @@ def score_digits(values, direction='max'): preview_section=f'

{"冻结 HTML 阅读视图" if lang=="zh" else "Frozen HTML reading view"}

{preview_note} {"下载完整输入与证据 JSON" if lang=="zh" else "Download complete input and evidence JSON"}

{preview}
' body=f'

{esc(case)} · {esc(type_label(lang,gt["page_type"]))}

{esc(group[providers[0]]["original_url"])}

{qualification}

{esc(x["summary"])}

'+''.join(f'' for k in ('provider','precision','recall','f1'))+f''+same+f'
{esc(x[k])}{"应包含命中" if lang=="zh" else "Required hits"}{"应排除误命中" if lang=="zh" else "Forbidden hits"}{esc(x["diagnosis"])}
{preview_section}

{esc(x["reference"])}

{"完整证据 JSON" if lang=="zh" else "Complete evidence JSON"} · {esc(x["source"])} HTML.gz

{"冻结 HTML 输入" if lang=="zh" else "Frozen HTML input"}
{esc(readable_output(reference))}
'+''.join(panels)
    page(lang,f'cases/{case}.html',case,body)
- from harness.report_languages import write_language_contract
  write_language_contract(out, include_english=include_english)
 
 def provider_label(name):
@@ -675,7 +739,7 @@ def run_log_section(out,record,output):
   if not path.is_file():continue
   raw=path.read_bytes();observed=hashlib.sha256(raw).hexdigest()
   if _RENDER_CONTEXT.get().score_audit:
-   from .published_snapshot import read_snapshot_public_file
+   from harness.web_fetch.published_snapshot import read_snapshot_public_file
    read_snapshot_public_file(out,binding)
   else:
    if binding.get('sha256') and binding['sha256']!=observed:raise ValueError(f'Call log hash mismatch: {record["attempt_id"]}')
@@ -916,14 +980,6 @@ def write(out,path,title,body,*,root_links=False):
 @_scoped_render
 def generate_cohort_diagnostic(out, model):
  """Render live cohort diagnostics through the maintained WebFetch components."""
- from .report_diagnostic import (
-  FAILURE_TYPES,
-  diagnostic_call_summary,
-  diagnostic_cohort_summary,
-  diagnostic_failure_counts,
-  diagnostic_findings,
-  diagnostic_specimen_excerpt,
- )
  token=_RENDER_CONTEXT.set(RenderContext('
实时网页诊断
'+esc(model['scope_note'])+'
'))
  out.mkdir(parents=True,exist_ok=True);(out/'assets').mkdir(exist_ok=True)
  diagnostic_style='''
@@ -1094,7 +1150,6 @@ def generate_public_benchmark_research(out:Path,research:dict,*,include_english:
  out.mkdir(parents=True,exist_ok=True);(out/'assets').mkdir(exist_ok=True)
  (out/'assets/style.css').write_text(STYLE+EXTRA,encoding='utf-8')
  (out/'assets/report.js').write_text(JS,encoding='utf-8')
- from .report_changes import ROOT, localized_release_history, render_localized_release_history
  release_history=localized_release_history(
   json.loads((ROOT/'research/report-changes.json').read_text(encoding='utf-8')),
   'public_benchmark_research')
@@ -1208,7 +1263,6 @@ def research_table(headers,rows,**kwargs):return table(headers,rows,index_header
   accounts+=section(h('实测阶段','Measurement phase'),f'
{h("届时按方法单独列出官方安装入口、版本和所需环境变量名，不公开值。","The measured edition will list official installation sources, versions and required environment-variable names per method, never their values.")}
')
   _research_write(out,'accounts/index.html',h('账号与安装','Accounts and setup'),accounts,english)
  (out/'research.json').write_text(json.dumps(research,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
- from harness.report_languages import write_language_contract
  write_language_contract(out,include_english=include_english)
  return out/'index.html'
 
@@ -1294,7 +1348,6 @@ def _evidence_chunks(text, width=620, overlap=100):
  return chunks
 
 def _best_evidence_excerpt(expected, output):
- from lexbench_webfetch.body_coverage import tokens
  wanted=Counter(tokens(expected))
  if not wanted:return ''
  best='';best_score=0
@@ -1304,12 +1357,10 @@ def _best_evidence_excerpt(expected, output):
  return best if best_score>=min(.25,3/max(1,sum(wanted.values()))) else ''
 
 def _normalized_evidence_lines(output):
- from lexbench_webfetch.metrics import normalize_for_match
  return [[line,normalize_for_match(line),None] for raw in output.splitlines() if (line:=raw.strip())]
 
 def _exact_evidence_lines(expected, output, *, normalized_lines=None):
  """Locate one bounded, verbatim fragment; never repeat a whole long HTML line."""
- from lexbench_webfetch.metrics import normalize_for_match
  normalized_expected=normalize_for_match(expected)
  lines=_normalized_evidence_lines(output) if normalized_lines is None else normalized_lines
  for entry in lines:
@@ -1356,9 +1407,7 @@ def _call_log_excerpt(item):
 
 def _body_region_evidence(content,contract):
  """Map canonical scorer allocations to complete lines without changing scoring output."""
- from math import ceil
 
- from lexbench_webfetch.body_coverage import _global_region_allocation, compact_ngrams, tokens
  version=contract['version'];size=6 if version==3 else 3
  units=tokens(content) if version==1 else compact_ngrams(content,size)
  predicted=Counter(units);expected={}
@@ -1414,21 +1463,16 @@ def _position_label(index,total):
 
 def _body_scoring_text(record, output):
  """Use the scorer's body input; titles belong only to answer diagnostics."""
- from lexbench_webfetch.response_input import answer_input
  content=answer_input(output.encode('utf-8'),record.get('provider_metadata'),
                       ok=record.get('ok',False),status_code=record.get('status_code'))[0]
- from lexbench_webfetch.reference_quality import is_html_document_or_fragment
  if is_html_document_or_fragment(content):
-  from lexbench_webfetch.dynamic_lists import visible_html_text
   content=visible_html_text(content)
  else:
-  from lexbench_webfetch.body_coverage import native_markup_body_text
   fallback=html.unescape(re.sub(r"<(script|style|template)\b[^>]*>.*?","",content,flags=re.I|re.S))
   content=native_markup_body_text(content,fallback)
  return content
 
 def body_region_rows(case, metrics, output, *, frozen_scores=False):
- from lexbench_webfetch.body_coverage import evaluate_body_requirements
  contract=case.get('body_requirements') or {}
  if not contract:return [],''
  saved_regions=metrics.get('body_region_results',[])
@@ -1437,7 +1481,6 @@ def body_region_rows(case, metrics, output, *, frozen_scores=False):
   # The frozen-build boundary validates the saved metric and answer hashes.
   # A newer normalizer or a redacted public answer cannot reproduce the old
   # matching-text identity. Render the saved allocation without rescoring it.
-  from .publication_privacy import project_evidence_view
   evidence={}
   for item in saved_regions:
    passages=[]
@@ -1449,7 +1492,6 @@ def body_region_rows(case, metrics, output, *, frozen_scores=False):
   evidence_counts={item['id']:item.get('matched_units',item.get('matched_tokens',0)) for item in saved_regions}
   recalculated={item['id']:item for item in saved_regions}
  elif saved_regions and all('matched_output_passages' in item for item in saved_regions):
-  from lexbench_webfetch.body_coverage import _compact_with_positions
   matching_text=_compact_with_positions(output)[0]
   digest=hashlib.sha256(matching_text.encode()).hexdigest()
   if any(item.get('matching_text_sha256')!=digest for item in saved_regions):
@@ -1561,8 +1603,6 @@ def _stable_field_comparison(metrics,item):
  """Find the closest parsed record for a structured anchor, for display only."""
  schema=metrics.get('stable_task_schema')
  if schema!='quote':return None
- from lexbench_webfetch.metrics import normalize_for_match
- from lexbench_webfetch.stable_tasks import SCHEMA_FIELDS, _parse_records, _visible
  candidates=[]
  for alternative in item.get('alternatives',[]):
   expected=_parse_records(_visible(alternative,schema),schema)
@@ -1586,7 +1626,6 @@ def _quote_field_issue(metrics):
   if comparison and comparison[2].get('quote') and comparison[2].get('author'):
    comparisons.append(comparison)
  if not comparisons:return ''
- from lexbench_webfetch.metrics import normalize_for_match
  compact=0
  for expected,returned,_ in comparisons:
   left=normalize_for_match(expected.get('tags','')).replace(' ','')
@@ -1615,7 +1654,6 @@ def stable_anchor_evidence(metrics,output):
   if comparison:
    expected_fields,returned_fields,exact=comparison
    labels={'quote':'引文','author':'作者','tags':'标签'};rows=[]
-   from lexbench_webfetch.metrics import normalize_for_match
    for field in ('quote','author','tags'):
     wanted=expected_fields.get(field,'');got=returned_fields.get(field,'')
     compact=(normalize_for_match(wanted).replace(' ','')==normalize_for_match(got).replace(' ',''))
@@ -1727,7 +1765,6 @@ def reason(r,c):
   return '正文没有抓全；低于目标的位置为：'+'、'.join(failed)+'。下方会把完整标准答案与实际返回并排展示。'
  if r.get('diagnosis'):return r['diagnosis']['evidence']
  if c.get('task_kind')=='structure':
-  from .report_components import list_rule_description
   return f"检出 {r['metrics'].get('named_entry_count',0)} 个具名详情入口。"+list_rule_description(c)
  if r['metrics'].get('page_identity_required') and r['metrics'].get('page_identity_status') != 'verified':
   return '未通过：最终网页身份缺少证据，保留在计划任务分母中。'
@@ -1764,7 +1801,6 @@ def stable_checks(m):
 def ranking(records,cases,track,base='',id=None,compact=False,audit=None,*,diagnostic=False):
  excluded={r['case_id'] for r in records if r['metrics'].get('evaluation_excluded')}
  records=[r for r in records if not r['metrics'].get('evaluation_excluded')]
- from .report_components import comparable_body_reference
  title_scope=track=='challenge' and any('title_quality' in r for r in records if report_family(cases[r['case_id']])==track)
  selected={k:c for k,c in cases.items() if report_family(c)==track and k not in excluded}
  text_track=track in {'web','dynamic','document','challenge','risk_control'} and any(c.get('task_kind','body') in {'body','structure'} for c in selected.values())
@@ -1817,7 +1853,6 @@ def avg(key):
  if text_track:
   covered=sum(comparable_body_reference(c) for c in selected.values())
   note=f'完整参考覆盖 {covered}/{len(selected)}；缺少参考只影响全文指标，不影响任务结果。有参考时，请求失败计 0；请求成功但任务未通过仍按实际正文评分。'
-  from .report_components import list_rule_description
   identity_rules=list(dict.fromkeys(list_rule_description(c) for c in selected.values()
    if c.get('task_kind')=='structure' and not c.get('acceptance_contract')))
   if identity_rules:note+=' '+esc(' '.join(identity_rules))
@@ -1845,7 +1880,6 @@ def avg(key):
   result='
指标说明 · 通过率、质量与耗时'+metric_legend(heads)+'
'+result
  if title_scope:result+=title_diagnostics(groups,selected,base,id or 'ranking-'+track)
  result+=matched_latency(groups,cases,base)
- from .report_components import task_pass_guide
  return task_pass_guide(track,base,selected)+'
'+note+'
'+result
 
 def matched_latency(groups,cases,base):
@@ -1883,7 +1917,6 @@ def overview_all_call_latency(records,cases):
 def result_ledger_section(out, records, prefix=''):
  path=out/'local-evidence/result-ledger.json'
  if not path.is_file():return ''
- from .result_ledger import validate_ledger
  ledger=validate_ledger(json.loads(path.read_text()),records,source=out)
  rows=[('',cell(esc(row['id']))+cell(str(row['before_passes']))+cell(str(row['added_passes']))+
            cell(str(row['removed_passes']))+cell(str(row['totals']['pass']))+
@@ -1891,7 +1924,6 @@ def result_ledger_section(out, records, prefix=''):
  return section('评分修正账目','
每轮增加、减少与最终通过数均从逐条状态记录计算；排除结果保留原始调用，不计入比较分母。
'+table(['阶段','修正前通过','新增通过','减少通过','修正后通过','排除比较'],rows,metrics=[])+link(prefix+'local-evidence/result-ledger.json','下载逐条增减账目'),'result-ledger')
 
 def overview_matched_latency(records,cases):
- from .report_explainer import _article_speed_population
  names=sorted({r['provider'] for r in records})
  keys={(r['provider'],r['case_id']) for r in records}
  if not names or len(keys)!=len(records) or any((name,cid) not in keys for name in names for cid in cases):
@@ -2133,12 +2165,10 @@ def local_audit_banner(dataset):
   return ''
  return '
'+model_text('本地审核版','Local audit report')+'
'+model_text('本报告未正式发布。工具版本、调用时间与核验范围以保存记录为准。','This report is not a formal release. Consult the saved records for tool versions, call times and verification scope.')+'
'
 
-from .methods import with_measured_method_labels
 
 @with_measured_method_labels
 @_scoped_render
 def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english:bool=False,authored_interpretation:bool=True,presentation_only:bool=False,show_local_status_banners:bool=True,detail_attempt_ids:set[str]|None=None,dataset_case_ids:set[str]|None=None,frozen_scores:bool=False,english_routes=None,diagnostic_view_paths=()):
- from .publication_privacy import public_render_views
  _public_views=public_render_views(out,records)
  if english_routes is not None and not presentation_only:
   raise ValueError('Scoped English refresh requires presentation_only=True')
@@ -2160,7 +2190,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
  diagnostic=bool(score_audit) or (local_snapshot and not local_pass_rate_ranking(dataset))
  if local_snapshot and not show_local_status_banners:
   raise ValueError('Uncertified local diagnostic status cannot be hidden')
- from .reliability import assess_reliability
  reliability=assess_reliability(dataset,records)
  if reliability['complete']:
   reliability_banner=''
@@ -2202,7 +2231,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
  failure_reviews={}
  content_binding=dataset.get('comparison',{}).get('report_interpretation')
  if content_binding:
-  from .report_explainer import load_browser_failure_review
   article_content=json.loads(contained_path(out,content_binding['path']).read_text())
   failure_reviews=load_browser_failure_review(out,cases,records,article_content)
  bycase=defaultdict(list);byprovider=defaultdict(list)
@@ -2220,7 +2248,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
   release=json.loads(release_path.read_text())
   description=''+description+' · '+esc('bench v'+release['version'])+''
  home='
'+hero('Web Fetch 评测报告',description)
- from .report_explainer import render_explainer
  TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else '本报告的任务范围与验收规则'
  if authored_interpretation and dataset.get('comparison',{}).get('report_interpretation'):
   binding=dataset['comparison']['report_interpretation']
@@ -2235,7 +2262,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
   guide=section('如何读取本轮结果','
比例只对应本报告列出的案例与配置。先看总榜和分榜，再打开具体案例，对照验收要求与实际返回。请求成功只表示方法正常返回；任务通过表示返回内容满足该案例的全部要求。
评分规则与计时范围见“评测方法”，检查和修正记录见“核验记录”，方法版本、固定配置与日志范围见“环境与复现”。
')
  else:
   guide=render_explainer(out,cases,records,table=table,cell=cell,link=link,authored=authored_interpretation)
- from .report_reading import render_archived_reading
  guide=render_archived_reading(out) or guide
  write(out,'guide/index.html','报告解读',guide)
  panels=[]
@@ -2282,7 +2308,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
   if c.get('resource_output_contract',{}).get('unexpected')=='reject':
    status_note+='
资源输出契约：只允许在答案图片字段中返回登记的目标、别名或匹配规则；额外图片会使任务未通过。
'
   if c.get('task_kind')=='structure':
-   from .report_components import list_rule_description
    status_note='
'+esc(list_rule_description(c))+'
任务结果与耗时进入主榜；全文指标无完整参考时显示 —。
'
    if (out/'list-references'/f'{c["id"]}.json').exists():status_note+='
'+link('../list-references/'+c['id']+'.json','名称与链接核验依据')+'
'
    rules='
登记任务规则
'+esc(list_rule_description(c))+'
稳定内容与变化字段
稳定标题、栏目、说明按登记内容核对；频繁变化的名单、条目标题、排序、价格或 ID 不固定为旧快照取值，按声明的字段结构、类型、数量及唯一性核验。登录页、重复入口、空链接不计入。
'
@@ -2334,7 +2359,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
   if not c.get('ground_truth_main_content'):
    rows[6]=('',cell('全文完整度 / 纯净度 / F1')+cell('— / — / —（全文指标无完整参考，未评测）'))
   else:
-   from .report_components import comparable_body_reference
+   from harness.web_fetch.report_components import comparable_body_reference
    if not comparable_body_reference(c):
     rows[6]=('',cell('快照文字诊断 · 完整度 / 纯净度 / F1')+cell(' / '.join(pct(m.get(k)) for k in ['word_recall','word_precision','word_f1'])+'；参考含不固定取值的字段，仅为历史快照诊断，不进入主表全文均值。'))
   if r.get('status_code') is None:
@@ -2398,7 +2423,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
   caveat=''
   if c.get('track')=='dynamic':caveat='
按本例实际登记规则判定。“条目未完整匹配”表示未找到完整字段组合，不表示每个字段都不存在。稳定标题、栏目、说明按登记内容核对；频繁变化的名单和价格按字段结构、类型、数量及唯一性核验，不固定旧快照取值。全文指标仅在有完整、可比的参考时另算，不证明变化值的事实准确性或实时性。
'
   if c.get('task_kind')=='structure':
-   from .report_components import list_rule_description
    caveat='
'+esc(list_rule_description(c))+link('../datasets/'+c['id']+'.html','规则与独立参考')+'
'
   if 'stable_task_pass' in m:
    caveat='
本次任务按稳定内容与条目结构共同判定。全文指标须有完整、可比的参考，不能由条目数量推算。
'
@@ -2442,7 +2466,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
    if s:facts.extend([('独立源页复查',f"{s.get('checked_at','')} · HTTP {s.get('status','未取得')} · {s.get('title') or s.get('error','')}"),('复查边界',d['source_check_boundary'])])
    summary+='
查看失败分类与归因边界'+table(['项目','依据'],[('',cell(esc(k))+cell(esc(v))) for k,v in facts],metrics=[])+"
"
   body=hero(esc(c['name']),hero_meta)+section('结论',summary,'overview')
-  from .report_quality import render_quality_observations
   body+=section('目标覆盖与附加内容',render_quality_observations(r,c,output,esc=esc),'quality-diagnostics')
   reviewed=audited_attempts.get(r['attempt_id'])
   if reviewed:
@@ -2458,7 +2481,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
     proof=reviewed['adjudication_evidence']
     adjudication_en=proof.get('reason_en')
     if not adjudication_en and proof.get('rule_assertions'):
-     from .adjudication_evidence import adjudication_reason
      adjudication_en=adjudication_reason(proof['rule_assertions'],language='en')
     review_rows.append(('分歧裁定',proof.get('summary_zh',reviewed['adjudication'])))
    body+=section('独立结果核验',table(
@@ -2602,7 +2624,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
  write_support(out,records,manifests,authored_interpretation=authored_interpretation)
  if score_audit:
   render_score_audit(out,score_audit,score_binding)
- from .report_changes import render_changes
  if records and all(row.get('provider_type') == 'fixture' for row in records):
   write(out,'changes/index.html','示例说明',hero('合成示例','本报告由本地测试适配器生成。没有正式评测发布记录。'))
  elif not authored_interpretation:
@@ -2613,7 +2634,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
  else:
   render_changes(out,cases,records,dataset)
  for relative in diagnostic_view_paths:
-  from bs4 import BeautifulSoup
   with (out/relative).open(encoding='utf-8') as source_header:
    previous=BeautifulSoup(source_header.read(8192),'html.parser')
   heading=previous.find('h1');source_heading=previous.find('h2',string='记录来源')
@@ -2621,18 +2641,13 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
   note=source_heading.find_next('p')
   diagnostic_view_page(out,str(Path(relative).with_suffix('.txt')),heading.get_text(),note.get_text() if note else '')
  if include_english:
-  from .report_i18n import write_english_report
   write_english_report(out,source_routes=english_routes)
- from harness.report_languages import write_language_contract
  if presentation_only and (out/'report-languages.json').is_file():
-  from harness.report_languages import check_language_contract, generated_languages
   if ('en' in generated_languages(out)) != include_english:
    raise ValueError('Presentation refresh cannot change generated languages')
   if not include_english:
-   from harness.report_languages import english_notice
    (out/'en').mkdir(exist_ok=True)
    (out/'en/index.html').write_text(english_notice(), encoding='utf-8')
-  from harness.report_languages import primary_report_language, select_report_language
   if primary_report_language(out) == 'en':
    select_report_language(out, 'en')
   check_language_contract(out)
@@ -2643,7 +2658,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
 
 def current_local_review_findings(out,dataset,records,audit):
  """Describe the current snapshot from bound calls, never cached release prose."""
- from .publication_privacy import read_evidence_binding
  receipt=json.loads(read_evidence_binding(out,dataset['comparison']['task_replacement']).read_text())
  added=receipt['provider_calls_added']
  cases={c['id']:c for c in dataset['cases']}
@@ -2698,7 +2712,6 @@ def write_review_page(out,records):
  if not metadata_path.is_file():
   raise ValueError('A reviewed report requires research/report-changes.json')
  changes=json.loads(metadata_path.read_text())
- from .report_changes import require_review_rounds
  rounds=(audit['review_rounds'] if audit.get('kind')=='local_independent_review' else require_review_rounds(changes))
  rounds_html=''.join(
   f'
第 {i} 轮
{esc(row.get("date",""))}
{model_text(row.get("title",""),row.get("title_en"))}
范围：{model_text(row.get("scope",""),row.get("scope_en"))}
发现：{model_text(row.get("finding",""),row.get("finding_en"))}
纠正：{model_text(row.get("fix",""),row.get("fix_en"))}
未完成：{model_text(row.get("pending",""),row.get("pending_en"))}
'
@@ -2748,7 +2761,6 @@ def coverage(label,item):
 
 def load_frozen_method_options(out,records):
  """Bind displayed controls to their actual configuration and call scope."""
- from .publication_privacy import read_evidence_binding, require_public_method_options
  document=json.loads((out/'frozen-method-options.json').read_text())
  if document.get('schema_version')!=2 or not document.get('source_config'):
   raise ValueError('Frozen option display requires a bound configuration and call scope')
@@ -2850,7 +2862,6 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
  dataset=json.loads((out/'dataset.json').read_text()) if (out/'dataset.json').is_file() else {}
  local_matrix=measurement_presentation(dataset).local_matrix
  headless=is_headless_cohort(records)
- from .report_components import task_contract_guide
  methodology=hero('评测方法','按任务定义验收规则。同组已测方法使用相同的冻结任务与参考，覆盖数量在各榜单列明。')
  if headless:
   methodology+=section('从参考到发布','
1 · 冻结参考
锁定既定题目、答案和验收规则；只有明确声明需要实时参考的任务才重新采集源页。
2 · 执行调用
同一 URL、任务和截止时间；各方法使用其最快的正式任务原生产物路线，保留版本、输出与哈希。
3 · 测后核验
核对冻结题目、答案、规则和调用记录；需要实时参考的任务另做前后采集，确认参考未变。
4 · 结果独立审查
独立审查按明确指定的范围开展，检查输出、参考答案和验收规则；本轮审查覆盖见下方，不把程序评分当成独立审查。
'+review_summary(out,'../'),'评测流程')
@@ -2910,7 +2921,6 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
   methodology+='
'+model_text(
    '正文验收使用原生接口交付的内容，不以额外解析器补齐回答。原始文件任务按其字节验收规则核对；请求、输出与错误保留在逐次调用证据中。',
    'Text acceptance uses content delivered by the native interface, without an additional parser completing the answer. Original-file tasks use their byte-acceptance rules; requests, outputs and errors remain in per-call evidence.')+'
'
- from .report_components import render_acceptance_scope
  if dataset.get('cases'):
   methodology+=render_acceptance_scope({case['id']:case for case in dataset['cases']},records)
  score_binding=dataset.get('comparison',{}).get('score_audit')
@@ -3024,7 +3034,6 @@ def option_text(value):
  write(out,'reproduce/index.html','环境与复现',support)
  write(out,'accounts/index.html','账号与安装',render_account_requirements(records))
 
- from .report_methods import render_methods
  write(out,'adoption/index.html','方法介绍',render_methods(out,records,hero=hero,section=section,table=table,cell=cell,link=link,esc=esc,authored=authored_interpretation))
 
 
@@ -3038,7 +3047,6 @@ def option_text(value):
 
 
 def _wmb_html_preview(source,english=False):
- from bs4 import BeautifulSoup
  soup=BeautifulSoup(source,'html.parser')
  # This is an inert reading view, never a replay of untrusted website scripts.
  for node in soup.select('script,iframe,object,embed,base,meta,link,svg,math,video,audio,source'):
@@ -3099,8 +3107,6 @@ def _wmb_denominator_example(records, english, case_count, provider=None):
  return '
'+h('公式计分范围：','Formula-scoring population: ')+esc(provider)+' · '+str(scored)+' / '+str(case_count)+'
'+fallback+'
'+matrix+'
'+h(f'公式分只平均右上、左下、右下共 {scored} 页；其中 {counts[(False,True)]+counts[(True,False)]} 页计 0 分。分母由两份 Markdown 中识别出的公式决定，不是“有正文的页数”，也不是“答对的页数”。',f'The formula mean uses the upper-right, lower-left and lower-right cells: {scored} pages, including {counts[(False,True)]+counts[(True,False)]} zero scores. Recognition in the two Markdown files determines the denominator; it is neither the body-text count nor a count of correct pages.')+'
'
 
 
-
-
 def _wmb_latency_summary(records,names):
  """Final successful fetch time; retain all rows and historical execution cost."""
  by_method={name:[r for r in records if r['provider']==name] for name in names}
@@ -3240,7 +3246,6 @@ def specimen(metric):
    stats.append(''+esc(labels.get(row['provider'],row['provider']))+': '+_wmb_pct(details.get('score'))+' · '+length_text+'')
   evidence=''
   if out is not None:
-   from pathlib import Path
    paths=[(h('参考内容','Reference'),Path('references')/(case_id+'.md'))]
    if selected:paths.append((labels.get(selected[0]['provider'],selected[0]['provider']),Path(selected[0]['artifact_path'])))
    for title,relative in paths:
@@ -3279,7 +3284,6 @@ def specimen(metric):
  return '
'+h('结果解读','Reading the results')+'
'+overall+scope+content+tables+version+delivery+decision+'
'
 
 
-
 def _wmb_write(out,path,title,body,english=False):
  if english:
   body=body.replace('
Web Fetch · 评测报告
','
Web Fetch · Measured report
')
@@ -3471,7 +3475,6 @@ def dialog(target,title,body):
 @with_measured_method_labels
 def generate_webmainbench(out:Path,dataset:dict,records:list,summaries:dict,manifests:list,review:dict|None=None,research:dict|None=None,corrections:dict|None=None,adoption_data:dict|None=None,fixture_correction:dict|None=None,teds_revision:tuple|None=None,anomaly_review:dict|None=None,latency_diagnostic:dict|None=None,*,include_english:bool=False,report_content:dict|None=None):
  """Render the current WebMainBench report with the established report shell."""
- from .report_changes import render_webmainbench_history
  out.mkdir(parents=True,exist_ok=True);(out/'assets').mkdir(exist_ok=True)
  (out/'assets/style.css').write_text(STYLE+EXTRA,encoding='utf-8')
  (out/'assets/report.js').write_text(JS,encoding='utf-8')
@@ -3722,8 +3725,6 @@ def evidence_links(urls):
   adoption=hero(h('方法介绍','Methods'),h('比较本轮登记的 {method_count} 种执行配置；版本与接口见下表。','Compare the {method_count} registered configurations; versions and interfaces are listed below.'))
   adoption+=section(h('本轮实现','Measured implementations'),table([h('方法','Method'),h('版本','Version'),h('二进制 SHA-256','Binary SHA-256'),h('部署','Deployment')],provider_rows,metrics=[]))
   interface_rows=[]
-  from .report_interfaces import method_spec, _observed_route
-  from .methods import report_method_catalog
   language='en' if english else 'zh'
   for name in names:
    example=next((r for r in records if r['provider']==name and r['ok']),next(r for r in records if r['provider']==name))
@@ -3821,7 +3822,6 @@ def evidence_links(urls):
    reference_text=contained_path(out,'references/'+case_id+'.md').read_text(encoding='utf-8')
    attempt_body=_wmb_result_detail(r,c,output_text,reference_text,labels,metric_labels,failure_notes,corrected,english)
    _wmb_write(out,'attempts/'+attempt_id+'.html',labels[r['provider']]+' · '+case_id,attempt_body,english)
- from harness.report_languages import write_language_contract
  write_language_contract(out,include_english=include_english)
 
 
@@ -4491,7 +4491,6 @@ def verdict(value):return '通过' if value else '失败'
  controls=''
  markup=table(['方法','题目','旧判定','修复前','修复后','原因分类','证据'],inventory,metrics=[])
  # Keep paragraphs in the evidence sections, outside the fixed-height inventory.
- from bs4 import BeautifulSoup
  document=BeautifulSoup(markup,'html.parser')
  for element,item in zip(document.select('tbody tr'),audit['changes']):element['data-audit-provider']=item['provider']
  body+=section('逐项判定',controls+'
'+str(document)+'
')
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 34b11ecaf..0b0125729 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -1,16 +1,28 @@
 """Build the single current report from an explicit evidence directory."""
+
 import hashlib
 import json
 import shutil
 import tempfile
+from copy import deepcopy
 from pathlib import Path
 
-from harness.reporting.versions import record_build_version
+from lexbench_webfetch.composite_tasks import validate_composite_acceptance
+from lexbench_webfetch.paths import contained_path, safe_identifier
+from lexbench_webfetch.reference_quality import bind_annotation_evidence
+from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
+from lexbench_webfetch.stable_tasks import validate_acceptance_contract
 
 from harness.record_io import file_sha256, load_jsonl
-from harness.reporting.assets import copy_bound_assets
-
-from .artifacts import (
+from harness.report_languages import generated_languages, includes_english, select_report_language
+from harness.reporting.assets import copy_bound_assets, load_bound_content
+from harness.reporting.templates import resolve_template
+from harness.reporting.versions import bind_build_version, record_build_version
+from harness.web_fetch.adjudication_evidence import (
+    require_adjudication_evidence,
+    require_current_review_state,
+)
+from harness.web_fetch.artifacts import (
     copy_cohort_evidence,
     diagnostic_log_binding,
     materialize_reference_checks,
@@ -20,13 +32,31 @@
     verify_record_private_artifacts,
     verify_record_public_call_logs,
 )
-from .failure_diagnosis import diagnose
-from .reference_lifecycle import require_verified_reference_checks
-from .reliability import assess_reliability
-from .report import (
+from harness.web_fetch.cohort_exclusions import require_cohort_exclusions
+from harness.web_fetch.failure_diagnosis import diagnose
+from harness.web_fetch.local_audit import require_local_audit
+from harness.web_fetch.local_revision import (
+    require_current_task_replacement,
+    require_local_task_replacement,
+    validate_revision_chain,
+)
+from harness.web_fetch.native_routes import verify_native_execution_manifests
+from harness.web_fetch.publication_privacy import (
+    read_evidence_binding,
+    require_host_metadata_projection,
+    require_public_evidence_view_inventory,
+)
+from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes
+from harness.web_fetch.reference_lifecycle import (
+    require_certifiable_evidence,
+    require_verified_reference_checks,
+)
+from harness.web_fetch.reliability import assess_reliability
+from harness.web_fetch.report import (
     TRACKS,
     failure_category,
     generate,
+    generate_cohort_diagnostic,
     generate_public_benchmark_research,
     generate_wcxb,
     generate_webmainbench,
@@ -35,25 +65,47 @@
     succeeded,
     webmainbench_full_machine_issue,
 )
-from .report_changes import webmainbench_history
-from lexbench_webfetch.paths import contained_path, safe_identifier
-from harness.web_fetch.safety import reject_evidence_symlinks, validate_report_ids
-from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
-from .storage import data_home, require_selection_history
-from .wcxb_latency import load_full_paired_latency
-from .webmainbench_diagnostics import validate_teds_revision_if_present
-from .webmainbench_latency_correction import prior_results_path
-from .webmainbench_latency_diagnostic import load_diagnostic as load_latency_diagnostic
+from harness.web_fetch.report_changes import webmainbench_history
+from harness.web_fetch.report_diagnostic import (
+    check_diagnostic_report,
+    load_diagnostic,
+    prepare_diagnostic_evidence,
+)
+from harness.web_fetch.report_version import check_release
+from harness.web_fetch.resource_cohort import validate_cohort
+from harness.web_fetch.result_review import require_result_review
+from harness.web_fetch.risk_control import validate_risk_control_certification
+from harness.web_fetch.safety import (
+    copy_evidence_file,
+    evidence_tree_identity,
+    reject_evidence_symlinks,
+    unlinked_path,
+    validate_report_ids,
+)
+from harness.web_fetch.storage import data_home, require_selection_history
+from harness.web_fetch.wcxb_dataset import validate_manifest
+from harness.web_fetch.wcxb_latency import load_full_paired_latency
+from harness.web_fetch.wcxb_scoring import paired_latency_methods
+from harness.web_fetch.webmainbench_correction import validate as validate_case_correction
+from harness.web_fetch.webmainbench_diagnostics import (
+    validate_reference_revision,
+    validate_teds_revision,
+    validate_teds_revision_if_present,
+    validate_webmainbench_scoring_revision,
+)
+from harness.web_fetch.webmainbench_evidence import read_method_binding
+from harness.web_fetch.webmainbench_latency_correction import prior_results_path
+from harness.web_fetch.webmainbench_latency_diagnostic import (
+    load_diagnostic as load_latency_diagnostic,
+)
+from harness.web_fetch.webmainbench_runner import load_wcxb_source
 
 REPOSITORY = Path(__file__).resolve().parents[2]
 
 
 def _validate_stable_annotation_evidence(case, source, dataset):
     """Validate anchors against their primary or registered qualification evidence."""
-    from copy import deepcopy
 
-    from .reference_inputs import input_tree, read_dataset_bytes
-    from lexbench_webfetch.reference_quality import bind_annotation_evidence
 
     checked = deepcopy(case)
     try:
@@ -94,20 +146,12 @@ def _validate_stable_annotation_evidence(case, source, dataset):
 
 def build_diagnostic_cohorts(source: Path | None, output: Path, profile: Path | None = None) -> Path:
     """Build an explicitly requested diagnostic edition using the WebFetch owner."""
-    from harness.reporting.templates import resolve_template
 
-    from .report import generate_cohort_diagnostic
-    from .report_diagnostic import (
-        check_diagnostic_report,
-        load_diagnostic,
-        prepare_diagnostic_evidence,
-    )
 
     if profile is not None:
         configuration = json.loads(profile.read_text())
         binding = configuration.get('diagnostic_input')
         if binding:
-            from .storage import data_home
             selected = source or contained_path(data_home(), binding['path'])
             if (not selected.is_file() or selected.is_symlink()
                     or file_sha256(selected) != binding['sha256']):
@@ -154,7 +198,6 @@ def build_diagnostic_cohorts(source: Path | None, output: Path, profile: Path |
 
 def require_risk_control_certification(source: Path, dataset: dict, records: list[dict]) -> None:
     """Validate the complete profile-bound risk-control cohort before reporting."""
-    from .risk_control import validate_risk_control_certification
 
     comparison = dataset.get("comparison", {})
     if comparison.get("kind") != "risk_control_100_moli":
@@ -189,11 +232,9 @@ def copy_methodology_assets(output, assets=None):
             shutil.copy2(source, target)
 
 
-
 def build_public_benchmark_research(source: Path, output: Path, *, include_english: bool = False,
                                 language: str | None = None) -> Path:
     """Build the research-only companion without measured evidence."""
-    from harness.report_languages import includes_english, select_report_language
     include_english = includes_english(language, include_english)
     source, output = source.resolve(), output.resolve()
     marker = '.generated-report'
@@ -257,8 +298,6 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
                                include_english, language, authored_interpretation,
                                show_local_status_banners, profile_path=None):
     """Render a bound historical report using saved scores and public evidence."""
-    from harness.report_languages import select_report_language
-    from .safety import evidence_tree_identity, copy_evidence_file
     if source in output.parents:
         raise ValueError("Frozen presentation output must be outside its source tree")
     before = evidence_tree_identity(source)
@@ -296,7 +335,7 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
                 if relative in before:
                     metadata["public_" + kind + "_path"] = relative
                     metadata["public_" + kind + "_sha256"] = before[relative]
-        from .report_version import bind_release
+        from harness.web_fetch.report_version import bind_release
         release_binding = bind_release(source, staging, dataset, records, profile=profile_path)
         if release_binding:
             manifest['benchmark_release'] = release_binding
@@ -319,14 +358,11 @@ def _build_frozen_presentation(source, output, presentation_binding, *,
 
 def write_frozen_presentation_manifest(output, manifest, *, show_local_status_banners=True, authored_interpretation=False):
     """Refresh renderer metadata while retaining the frozen measurement bindings."""
-    from harness.report_languages import generated_languages
-    from harness.reporting.templates import resolve_template
     current = dict(manifest)
     binding_path = output / 'presentation-binding.json'
     if binding_path.is_file():
         binding = json.loads(binding_path.read_text())
         if binding.get('offline_revision'):
-            from .publication_privacy import read_evidence_binding
             authority = json.loads(read_evidence_binding(output, binding['offline_revision']).read_text())
             if authority.get('kind') == 'current_local_snapshot_authority':
                 if authority.get('schema_version') != 1:
@@ -339,7 +375,6 @@ def write_frozen_presentation_manifest(output, manifest, *, show_local_status_ba
     current.update(template=resolve_template('webfetch'), languages=languages, generated_languages=languages,
                    presentation={"show_local_status_banners": show_local_status_banners,
                                  "frozen_scores": True, "authored_interpretation": authored_interpretation})
-    from harness.reporting.versions import bind_build_version
     current['browser_eval'] = bind_build_version(output)
     (output / "report-manifest.json").write_text(json.dumps(current, ensure_ascii=False, indent=2) + "\n")
 
@@ -377,7 +412,6 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                  diagnostic_preview: bool = False, language: str | None = None,
                  show_local_status_banners: bool = True,
                  presentation_binding: Path | None = None, profile_path: Path | None = None) -> Path:
-    from harness.report_languages import includes_english, select_report_language
     include_english = includes_english(language, include_english)
     source, output = source.resolve(), output.resolve()
     if source == output or output in source.parents:
@@ -396,7 +430,6 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
     diagnostic_preview = (diagnostic_preview and
         dataset_probe.get('comparison', {}).get('kind') == 'web_fetch_diagnostic')
     if not local_audit and not diagnostic_preview:
-        from .reference_lifecycle import require_certifiable_evidence
         require_certifiable_evidence(dataset_probe)
     if dataset_probe.get('comparison', {}).get('kind') == 'webmainbench_frozen_html_to_markdown':
         return build_webmainbench_report(source, output, include_english=include_english,
@@ -410,9 +443,7 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
     validate_report_ids(dataset,records)
     require_risk_control_certification(source, dataset, records)
     cases = {c['id']:c for c in dataset['cases']}
-    from .result_review import require_result_review
     if local_audit:
-        from .local_audit import require_local_audit
         result_review = require_local_audit(source, dataset, records)
     else:
         result_review = require_result_review(source, dataset, records)
@@ -441,7 +472,6 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                 raise ValueError('Speed cohort output integrity failure: '+row['attempt_id'])
     resource_root=source/'resource-cohort'
     if resource_root.exists():
-        from .resource_cohort import validate_cohort
         resource_dataset, resource_records, resource_batches = validate_cohort(resource_root)
     for c in cases.values():
         annotation=c.get('source_benchmark',{}).get('stable_annotation',{})
@@ -462,14 +492,12 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                 f'Stale scoring implementation; rescore before reporting: {r["attempt_id"]}')
         contract = cases[r['case_id']].get('acceptance_contract')
         if contract:
-            from lexbench_webfetch.stable_tasks import validate_acceptance_contract
             validate_acceptance_contract(contract)
             expected = hashlib.sha256(json.dumps(contract, sort_keys=True, ensure_ascii=False).encode('utf-8')).hexdigest()
             if r['metrics'].get('acceptance_contract_sha256') != expected:
                 raise ValueError(f'Acceptance contract hash missing or mismatched: {r["attempt_id"]}')
         composite = cases[r['case_id']].get('composite_acceptance')
         if composite:
-            from lexbench_webfetch.composite_tasks import validate_composite_acceptance
             validate_composite_acceptance(composite)
             expected = hashlib.sha256(json.dumps(
                 composite, sort_keys=True, ensure_ascii=False).encode('utf-8')).hexdigest()
@@ -495,7 +523,6 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
         raise ValueError('A report requires tasks and measured records')
     execution = source/'execution.json'
     manifests = json.loads(execution.read_text())['batches'] if execution.exists() else [json.loads((source/'manifest.json').read_text())]
-    from .native_routes import verify_native_execution_manifests
     verify_native_execution_manifests(source, manifests, dataset, records)
     if not local_audit and not diagnostic_preview:
         require_verified_reference_checks(manifests, cases=dataset, records=records, source=source,
@@ -543,9 +570,9 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
             if local_audit:
                 meta["public_log_note"] = dataset.get("comparison", {}).get("call_chain_audit", {}).get("call_log_notes", {}).get(r["attempt_id"], "")
             if local_audit and meta.get("diagnostic_log_path"):
-                from .runner import _public_text
+                from harness.web_fetch.runner import _public_text
                 diagnostic_source = contained_path(source, meta["diagnostic_log_path"])
-                from .report import runtime_log_text
+                from harness.web_fetch.report import runtime_log_text
                 raw_log = runtime_log_text(diagnostic_source.read_text(encoding="utf-8", errors="replace"))
                 safe_log = _public_text(raw_log, limit=len(raw_log)) or ""
                 public_log = "logs/" + r["attempt_id"] + ".debug.txt"
@@ -555,7 +582,7 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                 meta["public_debug_log_path"] = public_log
                 meta["public_debug_log_sha256"] = file_sha256(destination)
             if local_audit and meta.get("raw_response_path"):
-                from .runner import _public_text
+                from harness.web_fetch.runner import _public_text
                 response_source = contained_path(source, meta["raw_response_path"])
                 response_text = response_source.read_text(encoding="utf-8", errors="replace")
                 public_response = "logs/" + r["attempt_id"] + ".response.txt"
@@ -581,7 +608,7 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                 shutil.copy2(log_source, log_target)
                 verify_public_call_log(staging, binding)
             text=raw.read_text(encoding='utf-8',errors='replace')
-            from .title_quality import add_title_quality
+            from harness.web_fetch.title_quality import add_title_quality
             add_title_quality(r, cases[r['case_id']], source)
             saved_diagnosis = r.get('diagnosis')
             if local_audit and (saved_diagnosis or {}).get('classification_revision') == 'failure-classification-v2':
@@ -631,7 +658,7 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
         if (source / 'certification.json').is_file():
             shutil.copy2(source / 'certification.json', staging / 'certification.json')
             if dataset.get('comparison', {}).get('kind') == 'risk_control_100_moli':
-                from .risk_control import risk_control_verification_binding
+                from harness.web_fetch.risk_control import risk_control_verification_binding
                 receipt = json.loads((source / 'certification.json').read_text())
                 verification_name, _ = risk_control_verification_binding(receipt)
                 for name in ('dataset.json', 'manifest.json', 'execution.json', verification_name):
@@ -741,8 +768,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
         if dataset.get("comparison", {}).get("kind") == "uniform_native_local_measurement":
             shutil.copy2(source / "local-run-contract.json", staging / "local-run-contract.json")
             if (source/"providers.json").is_file():
-                from .report import NATIVE_OPTION_LABELS
-                from .publication_privacy import require_public_method_options
+                from harness.web_fetch.publication_privacy import require_public_method_options
+                from harness.web_fetch.report import NATIVE_OPTION_LABELS
                 config_bytes = contained_path(source, 'providers.json', reject_symlinks=True).read_bytes()
                 config_hash = hashlib.sha256(config_bytes).hexdigest()
                 contract = json.loads((source / 'local-run-contract.json').read_text())
@@ -771,7 +798,6 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                     'scope_description_en': 'Only explicitly configured public options for this report are shown; omitted parameters do not mean disabled. The original configuration remains private; this public copy contains only allowed fields.'}
                 (staging / 'frozen-method-options.json').write_text(
                     json.dumps(options, ensure_ascii=False, indent=2) + '\n')
-            from harness.reporting.templates import resolve_template
             (staging / "report-manifest.json").write_text(json.dumps({
                 "schema_version": 1, "template": resolve_template("webfetch"),
                 "evidence_state": "local_measurement", "certified": False,
@@ -780,16 +806,15 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
                 "results_sha256": file_sha256(source / "results.jsonl"),
                 "presentation": {"show_local_status_banners": show_local_status_banners},
             }, ensure_ascii=False, indent=2))
-        from .report_reading import copy_archived_reading
+        from harness.web_fetch.report_reading import copy_archived_reading
         copy_archived_reading(source,staging)
-        from .report_version import bind_release
+        from harness.web_fetch.report_version import bind_release
         release_binding = bind_release(source, staging, dataset, records, profile=profile_path)
         generate(staging,dataset,records,manifests,include_english=include_english,
                  authored_interpretation=authored_interpretation,
                  show_local_status_banners=show_local_status_banners)
         if language is not None:
             select_report_language(staging, language)
-        from harness.reporting.templates import resolve_template
         from harness.report_languages import generated_languages
         template_manifest = {'schema_version': 1, 'template': None if diagnostic_preview else resolve_template('webfetch'),
             'dataset_sha256': file_sha256(staging/'dataset.json'),
@@ -887,9 +912,7 @@ def validate_webmainbench_anomaly_review(receipt: dict, source: Path,
 def build_webmainbench_report(source: Path, output: Path, *, include_english: bool = False,
                                 language: str | None = None, profile_path: Path | None = None) -> Path:
     """Build the current public-benchmark report without legacy task predicates."""
-    from harness.report_languages import includes_english, select_report_language
     include_english = includes_english(language, include_english)
-    from harness.reporting.assets import copy_bound_assets, load_bound_content
 
     authority = Path(__file__).resolve().parents[2]
     if profile_path is None:
@@ -1026,11 +1049,6 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root
     """Build the separately bound complete 545-case report from certified evidence."""
     from harness.web_fetch.webmainbench_evidence import digest, read_jsonl
 
-    from .webmainbench_diagnostics import (
-        validate_reference_revision,
-        validate_teds_revision,
-        validate_webmainbench_scoring_revision,
-    )
     if source.is_symlink() or any(path.is_symlink() for path in source.rglob('*')):
         raise ValueError('Symlinked report evidence is forbidden')
     if authority_root is None:
@@ -1054,7 +1072,6 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root
     repo = authority_root
     registry_path = repo / 'evaluation/webmainbench-full/moli-variants.json'
     registry = json.loads(registry_path.read_text(encoding='utf-8'))
-    from .webmainbench_evidence import read_method_binding
     binding_name, method_binding = read_method_binding(source)
     teds_path = source / 'teds-correction.json'
     teds_receipt = json.loads(teds_path.read_text(encoding='utf-8')) if teds_path.is_file() else None
@@ -1115,7 +1132,6 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root
     content_binding_path = authority_root / 'evaluation/webmainbench-full/content-binding.json'
     content_binding = json.loads(content_binding_path.read_text(encoding='utf-8'))
     content_path = data_home() / 'report-assets/webmainbench-full' / content_binding['relative_path']
-    from harness.reporting.assets import load_bound_content
     load_bound_content(data_home() / 'report-assets/webmainbench-full', content_binding)
     failure_binding_path = authority_root / 'evaluation/webmainbench-full/failure-analysis-binding.json'
     failure_binding = json.loads(failure_binding_path.read_text(encoding='utf-8'))
@@ -1127,7 +1143,6 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root
     correction_root = None
     corrected_keys = set()
     if correction_binding_path.exists():
-        from .webmainbench_correction import validate as validate_case_correction
         correction_binding = json.loads(correction_binding_path.read_text())
         correction_root = data_home() / correction_binding['relative_path']
         if digest(correction_root / 'certificate.json') != correction_binding['sha256']:
@@ -1191,7 +1206,7 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root
         shutil.copy2(Path(__file__).resolve().parents[2] / 'research/provider-adoption.json', staging / 'provider-adoption.json')
         summaries = json.loads((source / 'score-summaries.json').read_text(encoding='utf-8'))
         if correction_root:
-            from .webmainbench_correction import apply as apply_case_correction
+            from harness.web_fetch.webmainbench_correction import apply as apply_case_correction
             raw_records = read_jsonl(source / 'original-results.jsonl')
             records, summaries = apply_case_correction(
                 correction_root, staging, records, summaries,
@@ -1249,10 +1264,7 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
                       *, content_binding: dict, authority_root: Path,
                       include_english: bool = False, language: str | None = None) -> Path:
     """Build the one current report from a fully certified WCXB matrix."""
-    from harness.report_languages import includes_english, select_report_language
     include_english = includes_english(language, include_english)
-    from harness.web_fetch.wcxb_dataset import validate_manifest
-    from harness.web_fetch.webmainbench_runner import load_wcxb_source
     source, output = source.resolve(), output.resolve()
     providers = profile['providers']
     if not providers or len(set(providers)) != len(providers):
@@ -1264,7 +1276,6 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
     if comparison and (baseline == candidate_id or {baseline,candidate_id} - set(providers)):
         raise ValueError('WCXB paired comparison must name two registered participants')
 
-    from harness.reporting.assets import load_bound_content
     report_content = load_bound_content(
         source / "report-assets", content_binding,
         evidence_root=source, authority_root=authority_root)
@@ -1293,7 +1304,6 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
         raise ValueError('WCXB official evaluator differs from the certified profile')
     if [x['provider'] for x in certificate['providers']] != providers:
         raise ValueError('WCXB participant registry differs from the certificate')
-    from harness.web_fetch.wcxb_scoring import paired_latency_methods
     paired_methods = paired_latency_methods(profile, certificate) if profile.get('latency_correction_receipt_sha256') else {}
     rows = load_jsonl(source / 'wcxb-scores.jsonl', skip_blank=True)
     case_ids = {r['track_id'] for r in records}
@@ -1469,7 +1479,7 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
         (staging / '.generated-report').write_text('generated\n')
         if language is not None:
             select_report_language(staging, language)
-        from .wcxb_publication import write_inventory
+        from harness.web_fetch.wcxb_publication import write_inventory
         record_build_version(staging)
         write_inventory(staging, profile)
         if output.exists():
@@ -1494,8 +1504,6 @@ def validate_local_presentation(output: Path, binding_path: Path):
     This is a presentation update of an existing hash-bound diagnostic snapshot,
     not a fresh build or certification. All measured files remain untouched.
     """
-    from lexbench_webfetch.paths import contained_path
-    from harness.report_languages import generated_languages
     output = output.resolve()
     binding = json.loads(binding_path.read_text())
     manifest_path = contained_path(output, 'report-manifest.json')
@@ -1517,28 +1525,21 @@ def validate_local_presentation(output: Path, binding_path: Path):
         if file_sha256(path) != item['sha256']:
             raise ValueError('Presentation evidence binding changed: ' + item['path'])
     if binding.get('publication_projection'):
-        from .publication_privacy import require_host_metadata_projection
         require_host_metadata_projection(output, binding['publication_projection'])
     dataset = json.loads((output / 'dataset.json').read_text())
     records = load_jsonl(contained_path(output, 'results.jsonl'))
-    from .report_version import check_release
     check_release(output, dataset, records)
     if binding.get('publication_evidence_views'):
-        from .publication_privacy import require_public_evidence_view_inventory
         require_public_evidence_view_inventory(output, binding['publication_evidence_views'], records)
-    from .native_routes import verify_native_execution_manifests
     execution = output/'execution.json'
     if execution.is_file():
         verify_native_execution_manifests(output, json.loads(execution.read_text())['batches'], dataset, records)
     scoring_sources = {row['attempt_id']: binding.get('measured_scoring_source_sha256') for row in records}
     if binding.get('offline_revision'):
-        from .local_revision import validate_revision_chain
-        from .publication_privacy import read_evidence_binding
         scoring_sources, input_dataset, input_results = validate_revision_chain(
             output, binding['offline_revision'], dataset, records)
         revision_authority = json.loads(read_evidence_binding(output, binding['offline_revision']).read_text())
         if revision_authority.get('kind') == 'current_local_snapshot_authority':
-            from .local_revision import require_current_task_replacement
             if (revision_authority.get('schema_version') != 1
                     or input_dataset is not None or input_results is not None
                     or binding.get('task_replacement')):
@@ -1552,12 +1553,10 @@ def validate_local_presentation(output: Path, binding_path: Path):
             if input_dataset is None or input_results is None:
                 raise ValueError('Historical presentation revision has no input snapshot')
             if binding.get('task_replacement'):
-                from .local_revision import require_local_task_replacement
                 previous_dataset = json.loads(read_evidence_binding(output, input_dataset).read_text())
                 previous_records = load_jsonl(read_evidence_binding(output, input_results))
                 require_local_task_replacement(output, binding['task_replacement'], previous_dataset, previous_records)
     elif binding.get('task_replacement'):
-        from .local_revision import require_local_task_replacement
         scoring_sources = require_local_task_replacement(output, binding['task_replacement'], dataset, records)
     if (len(records) != binding.get('result_count')
             or len(dataset['cases']) != binding.get('case_count')
@@ -1568,14 +1567,12 @@ def validate_local_presentation(output: Path, binding_path: Path):
         raise ValueError("Presentation snapshot must not contain symlinks")
     validate_report_ids(dataset, records)
     reliability = assess_reliability(dataset, records)
-    from .cohort_exclusions import require_cohort_exclusions
     excluded = require_cohort_exclusions(output, dataset, records)
     if (not reliability['complete'] and
             (any(reliability[key] for key in ('missing', 'extra', 'duplicates'))
              or any(item['case_id'] not in excluded for item in reliability['unjudged']))):
         raise ValueError('Presentation refresh requires the complete declared matrix')
     for row in records:
-        from .publication_privacy import read_evidence_binding
         try:
             read_evidence_binding(output, {'path': row['artifact_path'],
                                            'sha256': row['metrics']['content_sha256']})
@@ -1593,7 +1590,6 @@ def _review_refresh_changes(output, binding, dataset, records):
     baseline = binding.get('review_refresh_baseline')
     if baseline is None:
         return set(), set()
-    from .publication_privacy import read_evidence_binding
     before_dataset = json.loads(read_evidence_binding(output, baseline['dataset']).read_text())
     before_records = load_jsonl(read_evidence_binding(output, baseline['results']))
     old_cases = {case['id']: case for case in before_dataset['cases']}
@@ -1627,7 +1623,6 @@ def _current_template_refresh(output, binding, manifest):
     revision = binding.get('offline_revision')
     if not revision:
         return False
-    from .publication_privacy import read_evidence_binding
     authority = json.loads(read_evidence_binding(output, revision).read_text())
     return (authority.get('kind') == 'current_local_snapshot_authority'
             and authority.get('schema_version') == 1)
@@ -1635,7 +1630,6 @@ def _current_template_refresh(output, binding, manifest):
 
 def refresh_local_presentation(output: Path, binding_path: Path, *, review_only: bool = False) -> Path:
     """Install explanatory routes and, on request, bound fresh review details."""
-    from .safety import unlinked_path, evidence_tree_identity, copy_evidence_file
     output = unlinked_path(output)
     before_tree = evidence_tree_identity(output)
     binding, dataset, records, manifest = validate_local_presentation(output, binding_path)
@@ -1644,7 +1638,6 @@ def refresh_local_presentation(output: Path, binding_path: Path, *, review_only:
     revised_attempt_ids = set()
     refresh_all_current_pages = False
     if review_only:
-        from .adjudication_evidence import require_adjudication_evidence, require_current_review_state
         review_path = contained_path(output, 'result-review.json', reject_symlinks=True)
         if not any(item.get('path') == 'result-review.json'
                    and item.get('sha256') == file_sha256(review_path)
diff --git a/harness/web_fetch/report_capabilities.py b/harness/web_fetch/report_capabilities.py
index e28ea83ab..e15f0db8a 100644
--- a/harness/web_fetch/report_capabilities.py
+++ b/harness/web_fetch/report_capabilities.py
@@ -1,10 +1,18 @@
 """Official capability evidence, deliberately independent of evaluation scores."""
+
 import json
-from harness.numbers import positive_number
 from html import escape
-from .report_components import tabs, bilingual_text
 
-from .methods import report_method, report_method_catalog, browser_method_label, method_version, normalized_method_version
+from harness.numbers import positive_number
+from harness.web_fetch.methods import (
+    browser_method_label,
+    method_version,
+    normalized_method_version,
+    report_method,
+    report_method_catalog,
+)
+from harness.web_fetch.report_components import bilingual_text, tabs
+from harness.web_fetch.report_interfaces import browser_task_interface_matrix
 
 
 def c(status, detail):
@@ -19,7 +27,6 @@ def task_capabilities(out, names, records, *, sources=None, language="zh"):
     """Describe the configured native task routes, without upgrading product claims."""
     catalog = report_method_catalog() if language == "zh" else report_method_catalog(language)
     sources = {key:(item["label"],item["url"]) for key,item in catalog["sources"].items()} if sources is None else sources
-    from .report_interfaces import browser_task_interface_matrix
     measured=[r for r in records if r.get('provider_metadata',{}).get('output_contract')]
     if (not measured or not all(report_method(name).get('kind') == 'browser' for name in names)
             or not (out/'dataset.json').is_file()):return None
diff --git a/harness/web_fetch/report_changes.py b/harness/web_fetch/report_changes.py
index 149cee0ea..c255115e9 100644
--- a/harness/web_fetch/report_changes.py
+++ b/harness/web_fetch/report_changes.py
@@ -1,9 +1,12 @@
 """Compact public release history: conclusions and audit evidence, not work logs."""
+
 import json
 import shutil
 from collections import defaultdict
 from pathlib import Path
 
+from harness.web_fetch.storage import research_evidence_dir
+
 ROOT=Path(__file__).resolve().parents[2]
 HEADLESS_HISTORY_START='2026-09-04-five-browser-fastest-markdown'
 FILE_RERUN_EVIDENCE = (
@@ -124,7 +127,7 @@ def webmainbench_history(data):
 
 def render_localized_release_history(data, english, evidence_links=(), *, kind='webmainbench_frozen_html_to_markdown'):
     """Render reader-facing release history without embedding release prose in report.py."""
-    from .report import hero, section, table, cell, esc, link
+    from harness.web_fetch.report import cell, esc, hero, link, section, table
     releases = localized_release_history(data, kind)
     latest, *older = releases
     def localized(row, key):
@@ -162,8 +165,7 @@ def render_webmainbench_history(data, english, evidence_links=()):
 
 
 def render_changes(out, cases, records, dataset=None):
-    from .report import hero, section, table, cell, esc, link, write
-    from .storage import research_evidence_dir
+    from harness.web_fetch.report import cell, esc, hero, link, section, table, write
     evidence_root=research_evidence_dir()
     selection=out/'file-rerun-selection.json'
     rerun_sources=[]
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index 4e007e15b..a8f2c0d18 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -7,22 +7,47 @@
 from collections import Counter
 from html.parser import HTMLParser
 from pathlib import Path
-from urllib.parse import parse_qsl, urlencode, urlsplit
+from urllib.parse import parse_qsl, unquote, urlencode, urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+from lexbench_webfetch.metrics import task_measured, task_passed
+from lexbench_webfetch.paths import contained_path, safe_identifier
+from lexbench_webfetch.stable_tasks import validate_acceptance_contract
 
 from harness.record_io import file_sha256, load_jsonl
 from harness.report_languages import check_language_contract, generated_languages, visible_numbers
 from harness.reporting.presentation import check_presentation
+from harness.reporting.templates import resolve_template
+from harness.web_fetch.adjudication_evidence import (
+    require_adjudication_evidence,
+    require_current_review_state,
+)
+from harness.web_fetch.local_audit import (
+    require_complete_local_inventory,
+    require_local_measurement,
+)
+from harness.web_fetch.local_execution_contract import local_measurement_cases
+from harness.web_fetch.methods import native_browser_engine
+from harness.web_fetch.publication_privacy import public_render_views, read_evidence_binding
 from harness.web_fetch.reference_lifecycle import validate_resource_source_evidence
 from harness.web_fetch.reliability import require_reliable
-from lexbench_webfetch.paths import contained_path, safe_identifier
+from harness.web_fetch.report import local_pass_rate_ranking
+from harness.web_fetch.report_build import validate_local_presentation
+from harness.web_fetch.report_explainer import check_current_browser_article
+from harness.web_fetch.report_interfaces import SPECS
+from harness.web_fetch.report_methods import PROFILES
+from harness.web_fetch.report_reading import render_archived_reading
+from harness.web_fetch.report_resources import check_resource_presentation
+from harness.web_fetch.report_version import check_release
+from harness.web_fetch.result_ledger import validate_ledger
 from harness.web_fetch.safety import reject_evidence_symlinks
 from harness.web_fetch.storage import require_canonical_inputs, require_current_report_profile
+from harness.web_fetch.wcxb_report_check import check as check_wcxb
+from harness.web_fetch.webmainbench_report_check import check_sample_report
 
 
 def require_public_evidence_page(content, relative, views):
     """Require inspectable public downloads and honest original/public hashes."""
-    from bs4 import BeautifulSoup
-    from urllib.parse import unquote, urljoin
     content = re.sub(r'<(pre|script|style)\b[^>]*>.*?', '', content, flags=re.I | re.S)
     soup = BeautifulSoup(content, 'html.parser')
     visible = soup.get_text(' ', strip=True)
@@ -49,7 +74,6 @@ def require_public_evidence_page(content, relative, views):
 
 
 def check_public_evidence_pages(root, records):
-    from .publication_privacy import public_render_views
     views = public_render_views(root, records)
     languages = generated_languages(root)
     for row in records:
@@ -213,7 +237,6 @@ def check_localized_image_sources(zh, en, canonical):
 
 
 def check_saved_result_summary(root, dataset, records):
-    from lexbench_webfetch.metrics import task_measured, task_passed
     cases = {case['id']: case for case in dataset['cases']}
     summary = json.loads((root / 'summary.json').read_text())
     groups = {}
@@ -232,7 +255,6 @@ def check_saved_result_summary(root, dataset, records):
 
 def require_replacement_execution(root, binding, physical, logical):
     """Keep superseded task calls as history without counting them as current rows."""
-    from .publication_privacy import read_evidence_binding
     receipt = json.loads(read_evidence_binding(root, binding).read_text())
     previous = load_jsonl(read_evidence_binding(root, receipt['baseline_results']))
     calls = load_jsonl(read_evidence_binding(root, receipt['cohort_results']))
@@ -301,16 +323,14 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
             (repository / profile).resolve() if not profile.is_absolute() else profile.resolve()
         )
     if (root / 'dataset.json').is_file() and (root / 'results.jsonl').is_file():
-        from .report_version import check_release
         check_release(root, json.loads((root / 'dataset.json').read_text()),
                       load_jsonl(root / 'results.jsonl'), profile=profile_path)
     if (root / 'report-manifest.json').is_file() and json.loads((root / 'report-manifest.json').read_text()).get('evidence_state') == 'published_snapshot_score_audit':
-        from .published_snapshot import check_snapshot_audit
+        from harness.web_fetch.published_snapshot import check_snapshot_audit
         if profile_path and file_sha256(profile_path) != json.loads((root / 'report-manifest.json').read_text()).get('profile_sha256'):
             raise ValueError('Snapshot audit profile changed')
         return check_snapshot_audit(root)
     if (root / "wcxb-score-summary.json").is_file():
-        from harness.web_fetch.wcxb_report_check import check as check_wcxb
 
         check_wcxb(root, authority_root=repository, profile_path=profile_path)
         return None
@@ -326,7 +346,6 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
     d = json.loads((root / "dataset.json").read_text())
     manifest_path = root / 'report-manifest.json'
     if manifest_path.is_file():
-        from harness.reporting.templates import resolve_template
         manifest = json.loads(manifest_path.read_text())
         expected_template = resolve_template('webfetch', root=repository)
         if manifest.get('template') != expected_template:
@@ -352,7 +371,6 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
     check_language_contract(root)
     has_english = "en" in generated_languages(root)
     if d.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown":
-        from harness.web_fetch.webmainbench_report_check import check_sample_report
 
         check_sample_report(root, d, rs, authority_root=repository, profile_path=profile_path)
         return (
@@ -365,30 +383,23 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
     local_measurement = d.get("comparison", {}).get("kind") == "uniform_native_local_measurement"
     if presentation_binding and not local_measurement:
         raise ValueError("Presentation binding requires a frozen uniform-native diagnostic snapshot")
-    from .report import local_pass_rate_ranking
     sample_ranking = local_measurement and local_pass_rate_ranking(d)
     if local_measurement:
         check_local_diagnostic_presentation(root, ranked=sample_ranking)
         if (root / 'result-review.json').is_file():
-            from .adjudication_evidence import require_adjudication_evidence, require_current_review_state
             review=json.loads((root / 'result-review.json').read_text())
             require_current_review_state(rs,review)
             require_adjudication_evidence(root,d,rs,review)
         if (root / 'local-evidence/result-ledger.json').is_file():
-            from .result_ledger import validate_ledger
             ledger = validate_ledger(json.loads((root / 'local-evidence/result-ledger.json').read_text()), rs, source=root)
             if ledger['states'][-1]['source_results_sha256'] != hashlib.sha256((root / 'results.jsonl').read_bytes()).hexdigest():
                 raise ValueError('Result ledger source hash differs from current results')
-        from .local_audit import require_local_measurement
-        from .local_execution_contract import local_measurement_cases
         if presentation_binding:
             if profile:
                 raise ValueError("Presentation binding cannot certify a release profile")
-            from .report_build import validate_local_presentation
             checked_binding = validate_local_presentation(root, presentation_binding)[0]
             public_payload_views = bool(checked_binding.get("publication_evidence_views"))
             if checked_binding.get('offline_revision'):
-                from .publication_privacy import read_evidence_binding
                 current_receipt = json.loads(read_evidence_binding(root, checked_binding['offline_revision']).read_text())
                 current_snapshot_authority = (current_receipt.get('kind') == 'current_local_snapshot_authority'
                                               and current_receipt.get('schema_version') == 1)
@@ -473,7 +484,6 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
         safe_identifier(case["id"])
         validate_resource_source_evidence(case, canonical_dataset)
     if local_measurement:
-        from .local_audit import require_complete_local_inventory
         reliability = require_complete_local_inventory(d, rs)
     else:
         reliability = require_reliable(d, rs)
@@ -487,7 +497,6 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
         safe_identifier(r["provider"])
         safe_identifier(r["case_id"])
         if public_payload_views:
-            from .publication_privacy import read_evidence_binding
             read_evidence_binding(root, {'path': r['artifact_path'],
                                          'sha256': r['metrics']['content_sha256']})
         else:
@@ -510,7 +519,6 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
                 or c.get("acceptance_contract")
             )
             if local_measurement:
-                from lexbench_webfetch.stable_tasks import validate_acceptance_contract
                 if c.get("acceptance_contract"):
                     validate_acceptance_contract(c["acceptance_contract"])
                 elif not (stable_checks or c.get("task_kind") == "file"):
@@ -622,7 +630,6 @@ def canonical_link(page, value):
     }
     archived_manifest = root / "archived-reading.json"
     if archived_manifest.is_file():
-        from .report_reading import render_archived_reading
         # Validate the archived source before allowing its original case names.
         render_archived_reading(root, "en" if has_english else "zh")
         allowed_english_source_text.update(json.loads(archived_manifest.read_text()).get("preserved_source_names", []))
@@ -733,7 +740,6 @@ def canonical_link(page, value):
     track_count = len({case.get("track", "web") for case in d["cases"]})
     require_home_comparison_layout(home, track_count, diagnostic=local_measurement and not sample_ranking, sample_ranking=sample_ranking)
     if (root / "resource-cohort/resource-comparison.json").is_file():
-        from .report_resources import check_resource_presentation
         resource_summary = json.loads((root / "resource-cohort/resource-comparison.json").read_text())
         guide = (root / "guide/index.html").read_text()
         for page_name, text in [("home", home), ("guide", guide)]:
@@ -778,7 +784,7 @@ def main(argv=None, *, authority_root: Path | None = None):
     parser.add_argument("--visual-receipt",type=Path,help="Full snapshot-audit browser checks; save receipt outside the report")
     args = parser.parse_args(argv)
     if args.visual_receipt:
-        from .published_snapshot import check_snapshot_visual
+        from harness.web_fetch.published_snapshot import check_snapshot_visual
         print(check_snapshot_visual(args.root,args.visual_receipt))
     result = check(args.root, authority_root=args.authority_root, profile=args.profile, presentation_binding=args.presentation_binding, source_dataset=args.source_dataset)
     if result is not None:
@@ -790,15 +796,12 @@ def check_revision(out: Path, *, authority_root: Path, profile_path: Path | None
     out = out.resolve()
     repository = authority_root.resolve()
     if (out / "wcxb-score-summary.json").is_file():
-        from harness.web_fetch.wcxb_report_check import check as check_wcxb
 
         check_wcxb(out, authority_root=repository, profile_path=profile_path)
         return "PASS: WCXB release scope and report acceptance checks verified"
     dataset = json.loads((out / "dataset.json").read_text())
-    from bs4 import BeautifulSoup
 
     if dataset.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown":
-        from harness.web_fetch.webmainbench_report_check import check_sample_report
 
         check_sample_report(out, authority_root=repository, profile_path=profile_path)
         return (
@@ -806,12 +809,10 @@ def check_revision(out: Path, *, authority_root: Path, profile_path: Path | None
         )
     binding = dataset.get('comparison', {}).get('report_interpretation')
     if binding:
-        from .report_explainer import check_current_browser_article
         check_current_browser_article(out, {case['id']:case for case in dataset['cases']},
                                       load_jsonl(out/'results.jsonl'), binding)
     provenance_path = out / "implementation-provenance.json"
     if provenance_path.is_file():
-        from .methods import native_browser_engine
         provenance = json.loads(provenance_path.read_text())
         records = load_jsonl(out / "results.jsonl")
         home = BeautifulSoup((out / "index.html").read_text(), "html.parser")
@@ -927,7 +928,6 @@ def check_revision(out: Path, *, authority_root: Path, profile_path: Path | None
             if expected_failure not in [h.get_text() for h in soup.select("#filtered-table th")]:
                 raise ValueError(path)
     records = load_jsonl(out / 'results.jsonl')
-    from .methods import native_browser_engine
     synthetic = dataset.get('comparison', {}).get('kind') == 'synthetic_browser_report'
     if records and all(row.get('provider_type') == 'fixture' for row in records) and (
         synthetic or all(not native_browser_engine(row['provider']) for row in records)
@@ -939,7 +939,6 @@ def check_revision(out: Path, *, authority_root: Path, profile_path: Path | None
         return 'PASS: synthetic report presentation, declared adapters and bound interpretation verified'
     counts = json.loads((out / "adoption/counts.json").read_text())["methods"]
     names = {row["provider"] for row in load_jsonl(out / "results.jsonl")}
-    from harness.web_fetch.report_methods import PROFILES
 
     if not {item["provider"] for item in counts} == set(PROFILES) | names:
         raise ValueError("Report presentation check failed")
@@ -948,8 +947,6 @@ def check_revision(out: Path, *, authority_root: Path, profile_path: Path | None
     interfaces = BeautifulSoup((out / "adoption/index.html").read_text(), "html.parser").select(
         '.interface-panel[id^="interface-"]'
     )
-    from harness.web_fetch.report_interfaces import SPECS
-    from harness.web_fetch.methods import native_browser_engine
     interface_names = set(SPECS) | {name for name in names if native_browser_engine(name)}
 
     if not (
diff --git a/harness/web_fetch/report_components.py b/harness/web_fetch/report_components.py
index 068c0e989..6d5070d74 100644
--- a/harness/web_fetch/report_components.py
+++ b/harness/web_fetch/report_components.py
@@ -1,8 +1,17 @@
 """Small, shared presentation components; never change measured results."""
-from html import escape
-from pathlib import Path
+
 import json
+from collections import Counter, defaultdict
 from dataclasses import dataclass
+from html import escape
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from lexbench_webfetch.metrics import task_passed
+from lexbench_webfetch.stable_tasks import SCHEMA_FIELDS, UNIQUE_FIELDS
+
+from harness.web_fetch.models import report_family
+from harness.web_fetch.report_i18n import TEXT
 
 
 @dataclass(frozen=True)
@@ -118,7 +127,6 @@ def metric_legend(headers):
 
 def media_population_text(track, cases):
     """Describe the declared population, without assuming a historical cohort."""
-    from .models import report_family
     selected = [case for case in (cases or {}).values() if report_family(case) == track]
     if track == 'media_file':
         return (f'输入直接指向媒体文件，共 {len(selected)} 题：必须取得原始字节，并通过各题登记的格式、长度与文件指纹要求。',
@@ -147,7 +155,6 @@ def task_pass_guide(track, base='', cases=None):
     }
     rules.update({key: value['pass_rule'] for key, value in TRACK_CONTEXT.items() if 'pass_rule' in value})
     text = rules.get(track, '按案例中登记的验收规则逐项检查。')
-    from .report_i18n import TEXT
     missing_translations = {
         'dynamic': 'Validate the initial-load scope: stable text and required record fields, counts, uniqueness and types jointly determine acceptance. Volatile members, titles and prices are checked structurally rather than frozen to old values. An HTTP shell is insufficient; rendering, embedded data and public APIs are valid routes. Full-text quality is separate and requires a complete comparable reference.',
         'document': 'Document text must satisfy request status, minimum length and every registered content check; a download link alone is insufficient. Verification and error hints are diagnostic. F1 is reported separately and is not a pass threshold.',
@@ -273,7 +280,6 @@ def list_rule_description(c):
     return f'本例按登记的稳定内容锚点及字段／数量规则核验，至少 {minimum} 个不同的具名详情入口；稳定标题、栏目、说明按登记内容核对，频繁变化的条目标题和价格只核对声明的结构与类型。通过不证明变化值的事实准确性或实时性。'
 
 
-
 SCHEMA_LABELS = {
     'table_row':'表格行',
     'earthquake':'地震条目',
@@ -312,7 +318,6 @@ def task_instruction(c):
         if c.get('track') == 'document' and 0 < len(reference) < 100:
             return '读取这个最小文档的全部可见正文。该样本只检查能否取得短文档文字，不代表多页、表格或附录能力。'
         return c.get('task_prompt', '')
-    from lexbench_webfetch.stable_tasks import SCHEMA_FIELDS
     if contract['schema'] == 'page_identity':
         return '核对声明的页面、栏目或查询身份；不以变化中的条目名单作为固定答案。'
     if contract['schema'] == 'body_text':
@@ -349,7 +354,6 @@ def stable_rule_description(c):
 
 
 def stable_contract_details(c, *, evidence_root=None):
-    from lexbench_webfetch.stable_tasks import SCHEMA_FIELDS, UNIQUE_FIELDS
     contract = c['acceptance_contract']
     schema = contract['schema']
     fields = ''.join('
  • '+escape(FIELD_REQUIREMENTS[f][0])+'：'+escape(FIELD_REQUIREMENTS[f][1])+'
    • ' for f in SCHEMA_FIELDS[schema])
@@ -456,9 +460,6 @@ def sources_panel():
 
 def acceptance_scope(cases, records):
     """Describe frozen requirements and saved outcomes without changing either."""
-    from collections import Counter, defaultdict
-    from urllib.parse import urlsplit
-    from lexbench_webfetch.metrics import task_passed
 
     families = defaultdict(Counter)
     affiliated = []
@@ -504,7 +505,8 @@ def acceptance_scope(cases, records):
 
 
 def render_acceptance_scope(cases, records):
-    from .report import model_text as h, section, table, cell, esc, pct, provider_label, link
+    from harness.web_fetch.report import cell, esc, link, pct, provider_label, section, table
+    from harness.web_fetch.report import model_text as h
     scope = acceptance_scope(cases, records)
     fields = ('tasks', 'identity', 'literal', 'facts', 'body_regions', 'records', 'resources', 'full_reference', 'short_length_floor')
     rows = [('', cell(esc(track)) + ''.join(cell(str(counts[key])) for key in fields))
diff --git a/harness/web_fetch/report_diagnostic.py b/harness/web_fetch/report_diagnostic.py
index 53b5bceb6..13d5967b7 100644
--- a/harness/web_fetch/report_diagnostic.py
+++ b/harness/web_fetch/report_diagnostic.py
@@ -1,4 +1,5 @@
 """Validate a diagnostic report model without upgrading it to certification."""
+
 import hashlib
 import json
 import math
@@ -8,10 +9,20 @@
 from pathlib import Path
 from urllib.parse import unquote, urljoin, urlsplit
 
-from harness.record_io import load_jsonl
-
+from bs4 import BeautifulSoup
+from lexbench_webfetch.metrics import task_passed
 from lexbench_webfetch.paths import contained_path, safe_identifier, validate_http_url
 
+from harness.record_io import load_jsonl
+from harness.reporting.presentation import check_presentation
+from harness.reporting.versions import check_build_version
+from harness.web_fetch.artifacts import (
+    verify_record_private_artifacts,
+    verify_record_public_call_logs,
+)
+from harness.web_fetch.publication_privacy import project_file, publication_risk
+from harness.web_fetch.reference_lifecycle import require_verified_reference_checks
+
 
 def require_diagnostic_scope(model, profile):
     """Compare the report with an independent planned population, not itself."""
@@ -63,10 +74,6 @@ def _saved_implementation_identity(record, batches):
 
 def prepare_diagnostic_evidence(spec_path, output):
     """Adapt complete saved cohorts and bound reviews, never make provider calls."""
-    from .artifacts import verify_record_private_artifacts, verify_record_public_call_logs
-    from lexbench_webfetch.metrics import task_passed
-    from .publication_privacy import project_file
-    from .reference_lifecycle import require_verified_reference_checks
 
     spec_path, output = Path(spec_path).resolve(), Path(output)
     spec = json.loads(spec_path.read_text())
@@ -431,18 +438,13 @@ def load_diagnostic(source):
 
 def check_diagnostic_report(output, profile=None):
     """Enforce the shared reader structure and evidence dialogs at release."""
-    from bs4 import BeautifulSoup
-
-    from harness.reporting.presentation import check_presentation
 
-    from .publication_privacy import publication_risk
 
     output = Path(output)
     model, files = load_diagnostic(output)
     if profile is not None:
         require_diagnostic_scope(model, json.loads(Path(profile).read_text()))
     manifest = json.loads((output / "report-manifest.json").read_text())
-    from harness.reporting.versions import check_build_version
     check_build_version(output, manifest)
     if profile is not None:
         binding = json.loads(Path(profile).read_text()).get('diagnostic_input')
diff --git a/harness/web_fetch/report_explainer.py b/harness/web_fetch/report_explainer.py
index 4598f10e6..59d93dc9b 100644
--- a/harness/web_fetch/report_explainer.py
+++ b/harness/web_fetch/report_explainer.py
@@ -1,4 +1,5 @@
 """Bound report interpretations and a population-driven default summary."""
+
 import base64
 import hashlib
 import json
@@ -6,20 +7,32 @@
 import statistics
 from collections import Counter
 from html import escape
+from html.parser import HTMLParser
 from pathlib import Path
 from urllib.parse import urlsplit
 
-from harness.record_io import canonical_json_sha256
-
-from .report_resources import render_resource_comparison
-from .report_components import bilingual_text
-from lexbench_webfetch.metrics import task_passed, percentile
-from .methods import with_measured_method_labels
-from .report_style import STYLE
+from bs4 import BeautifulSoup
+from lexbench_webfetch.metrics import percentile, task_passed
 from lexbench_webfetch.paths import contained_path
 
-
-from .models import REPORT_FAMILIES as BROWSER_FAMILY_NAMES, report_family
+from harness.record_io import canonical_json_sha256
+from harness.web_fetch.artifacts import record_public_call_log_bindings, verify_public_call_log
+from harness.web_fetch.methods import (
+    browser_method_label,
+    method_versions,
+    native_browser_engine,
+    with_measured_method_labels,
+)
+from harness.web_fetch.models import REPORT_FAMILIES as BROWSER_FAMILY_NAMES
+from harness.web_fetch.models import report_family
+from harness.web_fetch.report_components import (
+    bilingual_text,
+    comparable_body_reference,
+    task_instruction,
+)
+from harness.web_fetch.report_interfaces import browser_task_interface_matrix, method_provenance
+from harness.web_fetch.report_resources import render_resource_comparison
+from harness.web_fetch.report_style import STYLE
 
 
 def _mean(values):
@@ -177,7 +190,6 @@ def load_browser_failure_review(out, cases, records, content):
 
 def browser_example_evidence(out, case, row, *, focus=None, reference_focus=None):
     """Project saved scoring evidence for a compact example; never rescore outputs."""
-    from .report_components import task_instruction
     m = row['metrics']
     def text(value):
         if isinstance(value, str): return value
@@ -321,7 +333,6 @@ def distribution(rows):
                 'p95': percentile(times, .95) if times else None}
     tracks = [track for track in BROWSER_FAMILY_NAMES if any(report_family(c) == track for c in cases.values())]
     failure_review = load_browser_failure_review(out, cases, records, content)
-    from .report_components import comparable_body_reference
     quality_cases=[cid for cid,case in cases.items() if report_family(case)=='web' and comparable_body_reference(case)]
     figures = content.get('figures', [])
     if not isinstance(figures, list):
@@ -341,8 +352,6 @@ def distribution(rows):
         examples.append({'row': row, 'explanation': item['explanation'],
                          'evidence': browser_example_evidence(out,cases[row['case_id']],row,focus=item.get('focus'),reference_focus=item.get('reference_focus'))})
     stats = {}
-    from .report_interfaces import method_provenance
-    from .methods import method_versions
 
     provenance = method_provenance(out)
     for name in providers:
@@ -386,8 +395,6 @@ def distribution(rows):
 
 def _offline_browser_article(body, out, cases, model):
     """Embed specimen evidence; an independent download needs no report host."""
-    from bs4 import BeautifulSoup
-    from .artifacts import record_public_call_log_bindings, verify_public_call_log
     document = BeautifulSoup(body, 'html.parser')
     for script in document.select('script'):
         script.decompose()
@@ -411,7 +418,7 @@ def evidence(path, anchor, title, payload):
         for binding in record_public_call_log_bindings(row):
             snapshot = json.loads((out/'dataset.json').read_text()).get('comparison',{}).get('score_audit') if (out/'dataset.json').is_file() else None
             if snapshot:
-                from .published_snapshot import read_snapshot_public_file
+                from harness.web_fetch.published_snapshot import read_snapshot_public_file
                 log = read_snapshot_public_file(out,binding).read_text()
             else:
                 log = verify_public_call_log(out,binding).read_text()
@@ -455,7 +462,6 @@ def evidence(path, anchor, title, payload):
 
 def _browser_network_rows(records, providers):
     """Summarize recorded DNS routes without filling missing metadata."""
-    from .methods import browser_method_label, native_browser_engine
     rows=[]
     for provider in providers:
         label=browser_method_label(provider)
@@ -529,7 +535,6 @@ def _browser_reviewed_error(out, model, row):
 def _render_current_browser_article(out, cases, records, binding):
     model = current_browser_article_model(out, cases, records, binding)
     content, names, stats = model['content'], model['providers'], model['stats']
-    from .methods import browser_method_label, native_browser_engine
     engine_count = len({native_browser_engine(name) or name for name in names})
     def label(name):
         return bilingual_text(browser_method_label(name) if browser_method_label(name) != name else name.title(),
@@ -565,7 +570,6 @@ def family_cell(name, track):
         rate = group['passed']/group['n']
         values = [stats[n]['tracks'][track]['passed']/stats[n]['tracks'][track]['n'] for n in names]
         return f"{group['passed']}/{group['n']}
    " + metric(name,'family_'+track,rate,values,True)
-    from .report_interfaces import browser_task_interface_matrix
     interfaces = browser_task_interface_matrix(cases, names, records, compact=True)
     english_interfaces = browser_task_interface_matrix(cases, names, records, compact=True, language='en')
     english_routes = {(name,row['track']):row['cells'][name] for row in english_interfaces['rows'] for name in names}
@@ -684,7 +688,6 @@ def reviewed_distribution(items, provider):
                     ('
    独立核验范围与修订见核验记录。逐题原因与原始证据可从结果分析打开。
    ' if review else
                      '
    独立结果审查：0/'+str(model['results'])+'。未经独立复判的程序通过结果：'+str(sum(stats[n]['passed'] for n in names))+'。本轮报告使用统一评分器的原始验收结果。
    '), 'conclusions')
     # Reuse the reference article's reading components with current measured data.
-    from bs4 import BeautifulSoup
     article = BeautifulSoup(body, 'html.parser')
     parts = {node['id']:node for node in article.select('article > section[id]')}
     reading = BeautifulSoup('
    ', 'html.parser')
@@ -840,7 +843,7 @@ def append_html(parent, markup):
                 'stats':{n:{k:v for k,v in stats[n].items() if k not in {'specimen','failure_reviews'}} for n in names}}
     guide = contained_path(out,'guide');guide.mkdir(parents=True,exist_ok=True)
     (guide/'interpretation-evidence.json').write_text(json.dumps(evidence,ensure_ascii=False,indent=2)+'\n')
-    from .report import EXTRA
+    from harness.web_fetch.report import EXTRA
     standalone = _offline_browser_article(body,out,cases,model)
     (guide/'article.html').write_text(''+escape(content['title'])+''+standalone+'')
     return body+''
@@ -849,10 +852,7 @@ def append_html(parent, markup):
 @with_measured_method_labels
 def check_current_browser_article(out, cases, records, binding):
     """Verify numerical populations, unique highlights and complete drilldowns."""
-    from html.parser import HTMLParser
-    from urllib.parse import urlsplit
     model = current_browser_article_model(out, cases, records, binding)
-    from .report_interfaces import browser_task_interface_matrix
     interfaces = browser_task_interface_matrix(cases, model['providers'], records, compact=True)
     expected_interfaces = {(name,row['track']):[(part['mode'],part['count'],str(part['count'])+' 题：'+part['description']) for part in row['cells'][name]]
                            for row in interfaces['rows'] for name in model['providers']}
@@ -905,7 +905,6 @@ def handle_endtag(self,tag):
     for relative in ('guide/index.html','guide/article.html'):
         page=contained_path(out,relative)
         inspect=Inspect();inspect.feed(page.read_text())
-        from bs4 import BeautifulSoup
         document = BeautifulSoup(page.read_text(), 'html.parser')
         chapter_ids = [node.get('id') for node in document.select('.reading-chapter')]
         if chapter_ids != ['reading-'+key for key in ('scope','results','method','configuration','failures','conclusion')]:
@@ -1088,7 +1087,6 @@ def handle_endtag(self,tag):
             if link!=expected_link or (not link.startswith('#') and not (page.parent/urlsplit(link).path).is_file()):
                 raise ValueError('Browser result evidence target is broken')
         if relative.endswith('article.html'):
-            from bs4 import BeautifulSoup
             offline = BeautifulSoup(page.read_text(),'html.parser')
             if offline.select('script,link[rel="stylesheet"]'):
                 raise ValueError('Independent article requires external runtime')
@@ -1109,7 +1107,6 @@ def render_explainer(out, cases, records, *, table=None, cell=None, link=None, a
         content_binding = json.loads(dataset_path.read_text()).get('comparison', {}).get('report_interpretation')
     if authored and content_binding is not None:
         return _render_current_browser_article(out, cases, records, content_binding)
-    from .report_components import bilingual_text
     passes = sum(task_passed(row['ok'], row['metrics'],
                  cases[row['case_id']].get('task_kind', 'body')) for row in records)
     return ('
    '
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index 84d310bb7..8626bb4b0 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -15,7 +15,9 @@
 from pathlib import Path, PurePosixPath
 from urllib.parse import urlsplit, urlunsplit
 
+from bs4 import BeautifulSoup, Comment
 
+from harness.web_fetch.report_reading import archived_translations, render_archived_reading
 
 NAV = [
     ("datasets/index.html", "Dataset"),
@@ -51,7 +53,6 @@
 # measured task names and prompts remain unchanged.
 
 
-
 # Some generated blocks concatenate several fixed sentences into one text node,
 # or split a sentence around an inline link/code element.  These replacements
 # are deliberately narrow report phrases; they never include task names,
@@ -250,25 +251,6 @@ def _body_region_summary(match):
 
 # Current standalone article: authored prose only.
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 
 def release_translations(history: dict) -> dict[str, str]:
     """Localize only the structured release records included in this report."""
@@ -306,22 +288,12 @@ def add(source, target):
 ))
 
 
-
-
-
-
-
-
-
-
 REGEX.insert(0, (
     re.compile(r"^相对 Chromium，Moli 的平均耗时减少 ([\d.]+)%，P50 减少 ([\d.]+)%，P95 减少 ([\d.]+)%。这三项在五款中均最低；跨批次条件下，尚不能确定差距有多少来自浏览器实现。$"),
     r"Compared with Chromium, Moli reduces mean latency by \1%, P50 by \2%, and P95 by \3%. All three are the lowest of the five; with collection across batches, the share of the difference attributable to browser implementation remains uncertain.",
 ))
 
 
-
-
 REGEX.insert(0, (
     re.compile(r"^(\d+) 份真实日志文件已保存并通过 SHA-256 校验。$"),
     r"\1 real log files were retained and verified by SHA-256.",
@@ -398,7 +370,6 @@ def add(source, target):
 ))
 
 
-
 REGEX.insert(0, (
     re.compile(r"^资源表覆盖预先确定的全部 (\d+) 个普通网页，包含失败调用。这组任务的通过数：(.*)。各项平均值、P50、P95 使用同一完整范围。$"),
     lambda match: (
@@ -743,7 +714,6 @@ def preserve(match):
     return _rewrite_links(localized, source, english)
 
 
-
 REGEX.insert(0, (re.compile(r"^源码提交：(.+)$"), r"Source commit: \1"))
 
 
@@ -771,7 +741,6 @@ def write_english_report(out: Path, *, source_routes=None) -> int:
             sources.append(out / str(route))
     history_path = out / "report-changes.json"
     translations = release_translations(json.loads(history_path.read_text())) if history_path.is_file() else {}
-    from .report_reading import archived_translations
     article_translations = archived_translations(out)
     english_root = out / "en"
     if source_routes is None and english_root.exists():
@@ -792,8 +761,6 @@ def write_english_report(out: Path, *, source_routes=None) -> int:
         target.parent.mkdir(parents=True, exist_ok=True)
         raw=source_path.read_text(encoding="utf-8")
         if str(source)=='guide/index.html' and (out/'archived-reading.json').is_file():
-            from .report_reading import render_archived_reading
-            from bs4 import BeautifulSoup, Comment
             parsed = BeautifulSoup(raw, 'html.parser')
             article = parsed.select_one('[data-archived-reading="true"]')
             if article is None:
diff --git a/harness/web_fetch/report_interfaces.py b/harness/web_fetch/report_interfaces.py
index f8c843be3..e702b85a8 100644
--- a/harness/web_fetch/report_interfaces.py
+++ b/harness/web_fetch/report_interfaces.py
@@ -1,15 +1,28 @@
 """Interfaces actually used by the benchmark, not a claim to cover entire SDKs."""
+
 import json
 import re
-from types import SimpleNamespace
+from collections import Counter
 from html import escape
-from harness.numbers import positive_number
+from types import SimpleNamespace
+
 from lexbench_webfetch.paths import contained_path
-from .methods import native_browser_engine, browser_method_label
-from .report_components import measurement_presentation, bilingual_text
-from .providers.headless import output_mode_for_case
 
-from .methods import report_method, report_method_catalog, method_access, metadata_matches, method_page_model, browser_method_registry
+from harness.numbers import positive_number
+from harness.web_fetch import artifacts as evidence_artifacts
+from harness.web_fetch.methods import (
+ browser_method_label,
+ browser_method_registry,
+ metadata_matches,
+ method_access,
+ method_page_model,
+ native_browser_engine,
+ report_method,
+ report_method_catalog,
+)
+from harness.web_fetch.models import REPORT_FAMILIES, report_family
+from harness.web_fetch.providers.headless import output_mode_for_case
+from harness.web_fetch.report_components import bilingual_text, measurement_presentation
 
 # Views of the shared catalog; execution aliases inherit their engine's metadata.
 SPECS = {name: item['interface'] for name, item in report_method_catalog()['methods'].items()
@@ -61,7 +74,6 @@ def method_spec(name, record, *, timeout_seconds=None, language="zh"):
  return base
 
 
-
 def _observed_wait(meta, *, language="zh"):
  navigation=meta.get('navigation_wait_until')
  completion=meta.get('wait_until')
@@ -73,7 +85,6 @@ def _observed_wait(meta, *, language="zh"):
  return messages['wait_unrecorded']
 
 
-
 def _route_template(rule, metadata):
  """Render a registered route only when all its evidence fields are present."""
  if not metadata_matches(metadata,rule.get('metadata_equals',{})):return None
@@ -120,7 +131,6 @@ def _observed_route(engine, meta, *, language="zh", compact=False):
 
 def browser_task_interface_matrix(cases, providers, records=None, *, compact=False, language="zh"):
  """Describe task routes, preferring persisted call metadata when available."""
- from collections import Counter
  catalog=report_method_catalog(language)
  messages=catalog['messages']
  methods=catalog['methods']
@@ -130,7 +140,6 @@ def browser_task_interface_matrix(cases, providers, records=None, *, compact=Fal
  routes={name:item.get('routes',{}) for name,item in methods.items()}
  capabilities={name:item.get('capabilities',messages['not_registered']) for name,item in routes.items()}
  tracks=[]
- from .models import REPORT_FAMILIES, report_family
  for case in cases.values():
   track=report_family(case)
   if track not in tracks:tracks.append(track)
@@ -203,7 +212,6 @@ def browser_task_interface_matrix(cases, providers, records=None, *, compact=Fal
 def render_interfaces(out,records):
  messages=report_method_catalog()['messages']
  english_messages=report_method_catalog('en')['messages']
- from .artifacts import record_public_call_log_bindings, verify_public_call_log
  dataset_path=out/"dataset.json"
  dataset=json.loads(dataset_path.read_text()) if dataset_path.is_file() else {}
  comparison=dataset.get("comparison",{})
@@ -223,11 +231,11 @@ def fields(rows,english):
   r=min((r for r in records if r['provider']==name), key=lambda row: case_order.get(row['case_id'], len(case_order)), default=None)
   s=method_spec(name,r,timeout_seconds=comparison.get("timeout_seconds"))
   english=method_spec(name,r,timeout_seconds=comparison.get("timeout_seconds"),language="en")
-  bindings=record_public_call_log_bindings(r) if r else []
+  bindings=evidence_artifacts.record_public_call_log_bindings(r) if r else []
   if r and native_default and not bindings:
    raise ValueError("Measured native interface requires a saved call log: "+name)
   if bindings:
-   request=json.loads(verify_public_call_log(out,bindings[-1]).read_text())["request"]
+   request=json.loads(evidence_artifacts.verify_public_call_log(out,bindings[-1]).read_text())["request"]
    model=method_page_model(name,name,[x for x in records if x["provider"]==name])
    s={**s,"entry":model["endpoint"],"params":[],"request":request,"notes":messages['interface_request_note']}
    english_model=method_page_model(name,name,[x for x in records if x["provider"]==name],language="en")
diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py
index a745cd608..c45c77c0f 100644
--- a/harness/web_fetch/report_methods.py
+++ b/harness/web_fetch/report_methods.py
@@ -1,13 +1,26 @@
 """Provider-centred introduction and reproducible, sample-scoped adoption counts."""
+
 import json
 from pathlib import Path
-from .methods import method_info, method_page_model, browser_method_label, method_version
-from .report_components import tabs,cards,measurement_presentation
-from .report_interfaces import render_interfaces
-from .report_capabilities import render_capabilities
-from .report_pricing import render_pricing
 
-from .methods import report_method, report_method_catalog
+from harness.web_fetch.methods import (
+    browser_method_label,
+    method_info,
+    method_page_model,
+    method_version,
+    report_method,
+    report_method_catalog,
+)
+from harness.web_fetch.reliability import implementation_inventory
+from harness.web_fetch.report_capabilities import render_capabilities
+from harness.web_fetch.report_components import (
+    bilingual_text,
+    cards,
+    measurement_presentation,
+    tabs,
+)
+from harness.web_fetch.report_interfaces import render_interfaces
+from harness.web_fetch.report_pricing import render_pricing
 
 PROFILES = {name: (item['profile']['kind'], item['profile']['description'], item['source_url'])
             for name,item in report_method_catalog()['methods'].items() if item.get('profile')}
@@ -16,7 +29,6 @@ def display_name(name):
     return browser_method_label(name)
 
 
-
 def adoption_counts(data,names):
     result=[]
     for name in names:
@@ -37,8 +49,7 @@ def adoption_counts(data,names):
     return sorted(result,key=lambda x:(x['repository_count'] is None,-(x['repository_count'] or 0),x['provider']))
 
 def _implementation_scope(records, *, section, table, cell, esc):
-    from .reliability import implementation_inventory
-    from .report import model_text
+    from harness.web_fetch.report import model_text
     inventories = implementation_inventory(records)
     if not inventories:
         return ""
@@ -114,7 +125,6 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
             english_desc=method_page_model(name,name,[row for row in records if row['provider']==name],language='en')['contract']
             english_info={**english_info,'output':'Native product output; actual responses and logs retained','freshness':'Cache and wait settings are in the frozen configuration'}
             info={**info,'output':'产品原生返回；保留实际输出与日志','freshness':'缓存与等待选项见本轮固定配置'}
-        from .report_components import bilingual_text
         intro.append(('',cell(link('../providers/'+name+'.html',display_name(name),browser_method_label(name,language='en')))+cell(bilingual_text(kind,english_kind))+cell(bilingual_text(desc,english_desc))+cell(bilingual_text(info['output'],english_info['output']))+cell(bilingual_text(info['freshness'],english_info['freshness']))+cell(link(url,'文档 / 源码 ↗'))))
     for item in counts:
         name=item['provider'];n=item['repository_count'];known=n is not None
diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py
index 08af4ab63..4574bcfc8 100644
--- a/harness/web_fetch/report_package.py
+++ b/harness/web_fetch/report_package.py
@@ -2,7 +2,6 @@
 
 import argparse
 import base64
-from functools import lru_cache
 import gzip
 import hashlib
 import html
@@ -10,17 +9,40 @@
 import mimetypes
 import os
 import re
-import sys
 import shutil
+import sys
 import tempfile
 import zipfile
+from functools import lru_cache
 from pathlib import Path
+from urllib.parse import quote, urlencode
+
+from lexbench_webfetch.paths import contained_path
 
 from harness.record_io import load_jsonl
-from harness.reporting.packs import ByteBlockStore, DecompressedPackCache, read_packed_manifest, reconstruct_packed_file, write_compact_packs, write_packs
 from harness.reporting.links import validate_virtual_links
-from harness.web_fetch.publication_privacy import project_file, project_redundant_result_streams, publication_file_risk
-from harness.web_fetch.safety import evidence_tree_identity
+from harness.reporting.packs import (
+    ByteBlockStore,
+    DecompressedPackCache,
+    read_packed_manifest,
+    reconstruct_packed_file,
+    write_compact_packs,
+    write_packs,
+)
+from harness.reporting.publication import archive_inventory
+from harness.web_fetch.publication_privacy import (
+    project_file,
+    project_redundant_result_streams,
+    publication_file_risk,
+)
+from harness.web_fetch.publication_target import validate_service_base
+from harness.web_fetch.published_snapshot import check_snapshot_audit
+from harness.web_fetch.reference_lifecycle import require_certifiable_evidence
+from harness.web_fetch.report_build import validate_local_presentation
+from harness.web_fetch.report_explainer import current_browser_article_model
+from harness.web_fetch.report_version import check_release
+from harness.web_fetch.result_review import require_result_review
+from harness.web_fetch.safety import evidence_tree_identity, unlinked_path
 
 
 def _require(condition, detail):
@@ -29,7 +51,6 @@ def _require(condition, detail):
 
 
 def validate_release(source: Path, *, presentation_binding: Path | None = None, snapshot_audit=False) -> bool:
-    from .report_version import check_release
     dataset_path = source / "dataset.json"
     dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {}
     records = load_jsonl(source / "results.jsonl")
@@ -39,16 +60,13 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None,
     if snapshot_audit:
         if not audit_state or presentation_binding is not None:
             raise ValueError('Snapshot packaging requires its explicit diagnostic report')
-        from .published_snapshot import check_snapshot_audit
         check_snapshot_audit(source)
         return False
     if audit_state:
         raise ValueError('Published snapshot audits require --snapshot-audit; they are not certified releases')
     if presentation_binding is not None:
-        from .report_build import validate_local_presentation
         validate_local_presentation(source, presentation_binding)
         return False
-    from .reference_lifecycle import require_certifiable_evidence
     manifests = []
     for name in ('manifest.json', 'execution.json'):
         path = source / name
@@ -58,7 +76,6 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None,
     require_certifiable_evidence(dataset, manifests)
     content_binding = dataset.get('comparison', {}).get('report_interpretation')
     if content_binding:
-        from .report_explainer import current_browser_article_model
         current_browser_article_model(source, {case['id']:case for case in dataset['cases']},
                                       records, content_binding)
     sampled = dataset.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown"
@@ -67,7 +84,6 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None,
         if reliability.get("complete") is not True:
             raise ValueError("Refusing to package an incomplete or unjudged benchmark report")
         if dataset.get("comparison", {}).get("require_independent_result_review"):
-            from .result_review import require_result_review
             require_result_review(source, dataset, records)
         return False
     review = json.loads((source / "result-review.json").read_text())
@@ -96,7 +112,6 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None,
     return True
 
 
-
 def _retain_inline_contract(text, original, available):
     """A lazy acceptance contract needs a real file; otherwise retain its bytes."""
     pattern = re.compile(r'
    ]+)">
    ')
@@ -429,7 +444,6 @@ def add(path, data, *, source_sha256=None):
     return manifest
 
 
-
 def _compact_cached_content(cached: Path, destination: Path, *, source: Path, refresh_privacy=False):
     """Reuse public files while keeping derived result projections source-authoritative."""
     manifest = read_packed_manifest(lambda name: (cached/name).read_bytes())
@@ -550,7 +564,6 @@ def _archive_identity(path: Path):
 
 def build(source: Path, archive: Path, *, presentation_binding: Path | None = None, cached_content: Path | None = None, parts_directory: Path | None = None, refresh_privacy=False, snapshot_audit=False):
     if parts_directory is not None:
-        from .safety import unlinked_path
         parts_directory = unlinked_path(parts_directory)
         if (parts_directory.exists() or parts_directory.is_relative_to(source.resolve())
             or source.resolve().is_relative_to(parts_directory)
@@ -577,7 +590,7 @@ def build(source: Path, archive: Path, *, presentation_binding: Path | None = No
             manifest = _compact_cached_content(cached_content, staging/'content', source=source, refresh_privacy=refresh_privacy)
             if parts_directory is None:
                 _write_archive(staging/'content', staged_archive)
-                from .publication_check import check
+                from harness.web_fetch.publication_check import check
                 check(staged_archive, source, presentation_binding=presentation_binding, snapshot_audit=snapshot_audit)
             if evidence_tree_identity(cached_content) != cached_identity:
                 raise ValueError('Cached public content changed during packaging')
@@ -585,7 +598,7 @@ def build(source: Path, archive: Path, *, presentation_binding: Path | None = No
             staged_parts = staging/'parts'
             partition_content(staging/'content',staged_parts)
             _write_archive(staging/'content',staged_archive)
-            from .publication_check import check
+            from harness.web_fetch.publication_check import check
             check(staged_archive,source,presentation_binding=presentation_binding,snapshot_audit=snapshot_audit,
                   external_archives=part_archives(staged_parts/'parts.json'),allow_pending_external=True)
             manifest = _wire_manifest(staging/'content')
@@ -684,7 +697,6 @@ def replace(name, data, *, source_sha256=None):
                 manifest["files"][name]["source_sha256"] = source_sha256
             changes[name] = manifest["files"][name]["sha256"]
 
-        from lexbench_webfetch.paths import contained_path
 
         for name in routes:
             allowed = explanatory_route(name)
@@ -732,7 +744,6 @@ def replace(name, data, *, source_sha256=None):
 
 def build_incremental(source: Path, baseline: Path, destination: Path, *, presentation_binding: Path | None = None):
     """Stage a presentation update against immutable source and baseline bytes."""
-    from .safety import unlinked_path
 
     source, baseline, destination = map(unlinked_path, (source, baseline, destination))
     if (
@@ -748,7 +759,7 @@ def build_incremental(source: Path, baseline: Path, destination: Path, *, presen
     with tempfile.TemporaryDirectory(prefix=".webfetch-update-", dir=destination.parent) as work:
         staged = Path(work) / "update"
         receipt = _build_incremental(source, baseline, staged)
-        from .publication_check import check
+        from harness.web_fetch.publication_check import check
 
         check(baseline, source, staged, presentation_binding=presentation_binding)
         if evidence_tree_identity(source) != before or _archive_identity(baseline) != baseline_hash:
@@ -856,7 +867,6 @@ def partition_content(content, output, *, max_bytes=300*1024*1024, max_files=200
 
 def part_archives(plan_path):
     """Resolve only immutable archives named by the private preparation plan."""
-    from lexbench_webfetch.paths import contained_path
     plan = json.loads(plan_path.read_text())
     result = {}
     for label,entry in plan['parts'].items():
@@ -868,9 +878,6 @@ def part_archives(plan_path):
 
 def bind_data_parts(archive, plan_path, publications):
     """Bind a prepared report to the exact committed versions of its data files."""
-    from urllib.parse import quote, urlencode
-    from harness.reporting.publication import archive_inventory
-    from .publication_target import validate_service_base
     part_archives(plan_path)
     plan = json.loads(plan_path.read_text())
     _require(set(publications) == set(plan['parts']),'Published data part registry differs')
diff --git a/harness/web_fetch/report_pricing.py b/harness/web_fetch/report_pricing.py
index 1262057b7..fc63e7d48 100644
--- a/harness/web_fetch/report_pricing.py
+++ b/harness/web_fetch/report_pricing.py
@@ -1,6 +1,8 @@
 """Source-backed commercial prices, separate from measured benchmark outcomes."""
+
 import json
 import math
+from html import escape
 from pathlib import Path
 
 SOURCE = Path(__file__).resolve().parents[2] / 'research/provider-pricing.json'
@@ -38,7 +40,6 @@ def esc(value):
         english = translations.get(value) if isinstance(value, str) else None
         if english is None:
             return original
-        from html import escape
         return '' + original + ''
 
     dest = out / 'adoption'
diff --git a/harness/web_fetch/report_publish.py b/harness/web_fetch/report_publish.py
index d80afa2e3..cc5121df1 100644
--- a/harness/web_fetch/report_publish.py
+++ b/harness/web_fetch/report_publish.py
@@ -6,9 +6,11 @@
 import hashlib
 import json
 import os
+import re
 import urllib.parse
 import urllib.request
 from dataclasses import dataclass
+from functools import partial
 from pathlib import Path
 
 from harness.reporting.publication import (
@@ -26,8 +28,10 @@
     publication_document,
     require_publication_site,
     require_publication_version,
+    validate_service_base,
 )
-from harness.web_fetch.safety import reject_evidence_symlinks
+from harness.web_fetch.report_package import part_archives, validate_release
+from harness.web_fetch.safety import reject_evidence_symlinks, unlinked_path
 
 STAGED_UPLOAD_THRESHOLD = 23 * 1024 * 1024
 
@@ -107,7 +111,6 @@ def publish(config: PublishConfig, *, transport=None, package_check=None, valida
     if config.mode not in {"create", "update", "reuse"}:
         raise ValueError("Unknown publication mode")
     if config.operation_namespace == 'webfetch':
-        from .report_package import validate_release
         validate_release(config.source, presentation_binding=config.presentation_binding,snapshot_audit=config.snapshot_audit)
     if config.visibility not in {None, "private", "unlisted", "public"}:
         raise ValueError("Unknown publication visibility")
@@ -115,12 +118,10 @@ def publish(config: PublishConfig, *, transport=None, package_check=None, valida
         validation_inputs = (*validation_inputs, config.presentation_binding)
     attachments, parts_plan = {}, None
     if config.parts_manifest is not None:
-        from .report_package import part_archives
         attachments = part_archives(config.parts_manifest)
         parts_plan = json.loads(config.parts_manifest.read_text())
         validation_inputs = (*validation_inputs,config.parts_manifest,*attachments.values())
     if package_check is None and (config.presentation_binding is not None or config.parts_manifest is not None or config.snapshot_audit):
-        from functools import partial
         package_check = partial(check,presentation_binding=config.presentation_binding,external_archives=attachments,snapshot_audit=config.snapshot_audit)
     protected = {config.profile.resolve(), config.archive.resolve(),
                  *(Path(path).resolve() for path in validation_inputs)}
@@ -373,16 +374,6 @@ def main(*, root: Path | None = None):
     print(json.dumps(result, ensure_ascii=False), flush=True)
 
 def rename_main(argv=None, *, default_root=None) -> None:
-    import argparse
-    import json
-    import os
-    import re
-    import urllib.request
-    from pathlib import Path
-
-    from harness.web_fetch.publication_target import validate_service_base
-    from harness.web_fetch.safety import unlinked_path
-
     root = default_root or Path.cwd()
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
diff --git a/harness/web_fetch/report_quality.py b/harness/web_fetch/report_quality.py
index 2d1bc7123..6f0212303 100644
--- a/harness/web_fetch/report_quality.py
+++ b/harness/web_fetch/report_quality.py
@@ -1,8 +1,10 @@
 """Present coverage and extra-content diagnostics as separate observations."""
+
 import re
 
 from lexbench_webfetch.metrics import task_passed
-from .report_components import bilingual_text
+
+from harness.web_fetch.report_components import bilingual_text
 
 
 def quality_observations(record, case, output=''):
diff --git a/harness/web_fetch/report_reading.py b/harness/web_fetch/report_reading.py
index 9eadd5b6a..ddbe67747 100644
--- a/harness/web_fetch/report_reading.py
+++ b/harness/web_fetch/report_reading.py
@@ -1,19 +1,21 @@
 """An explicitly supplied historical article, separate from current measurements."""
+
+import base64
 import hashlib
-from harness.record_io import file_sha256
 import json
 import re
 import shutil
-import base64
 from pathlib import Path
 from urllib.parse import urlsplit
 
+from bs4 import BeautifulSoup
 from lexbench_webfetch.paths import contained_path
 
+from harness.record_io import file_sha256
+
 
 def sanitize_article_html(fragment):
     """Keep static article content; never import executable HTML or global CSS."""
-    from bs4 import BeautifulSoup
     allowed = {'article', 'section', 'div', 'span', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
                'table', 'thead', 'tbody', 'tfoot', 'tr', 'th', 'td', 'ul', 'ol', 'li', 'a', 'img',
                'figure', 'figcaption', 'pre', 'code', 'blockquote', 'strong', 'em', 'b', 'i',
diff --git a/harness/web_fetch/report_resources.py b/harness/web_fetch/report_resources.py
index eec1cc171..97052358f 100644
--- a/harness/web_fetch/report_resources.py
+++ b/harness/web_fetch/report_resources.py
@@ -3,10 +3,12 @@
 
 from html import escape
 
-from .methods import report_method
-from .report_components import bilingual_text
-from harness.numbers import positive_number
+from bs4 import BeautifulSoup
 
+from harness.numbers import positive_number
+from harness.reporting.presentation import check_presentation
+from harness.web_fetch.methods import report_method
+from harness.web_fetch.report_components import bilingual_text
 
 GROUPS = (
     ('CPU 与计算', (
@@ -183,7 +185,6 @@ def lowest(key, statistic):
 
 def check_resource_presentation(html, summary):
     """Verify complete-population values and unique extrema at release time."""
-    from harness.reporting.presentation import check_presentation
     soup = check_presentation(html)
     if summary.get('primary_population') != 'all_declared_calls':
         raise ValueError('Resource ranking must retain every declared call')
@@ -193,7 +194,6 @@ def check_resource_presentation(html, summary):
         raise ValueError('Resource comparison has the wrong population or denominator')
     headline = soup.select('[data-resource-winner]')
     expected_html, _ = render_resource_comparison(summary, summary['providers'])
-    from bs4 import BeautifulSoup
     expected_leaders = BeautifulSoup(expected_html, 'html.parser').select_one(
         '[data-resource-winner]')['data-resource-winner']
     if len(headline) != 1 or headline[0]['data-resource-winner'] != expected_leaders:
diff --git a/harness/web_fetch/report_version.py b/harness/web_fetch/report_version.py
index e3f5b377e..2e6066abd 100644
--- a/harness/web_fetch/report_version.py
+++ b/harness/web_fetch/report_version.py
@@ -4,9 +4,12 @@
 import json
 from pathlib import Path
 
+from bs4 import BeautifulSoup
 from lexbench_webfetch.paths import contained_path
 from lexbench_webfetch.versions import verify_release
 
+from harness.reporting.versions import check_build_version
+
 FILENAME = "benchmark-release.json"
 
 
@@ -100,11 +103,9 @@ def bind_release(source, output, dataset, records, *, profile=None):
 
 def check_release(root, dataset, records, *, profile=None):
     """Check the edition, rendered label and scoring identity without rescoring."""
-    from bs4 import BeautifulSoup
 
     manifest_path = root / "report-manifest.json"
     manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {}
-    from harness.reporting.versions import check_build_version
     check_build_version(root, manifest)
     binding = manifest.get("benchmark_release")
     expected = _profile_binding(profile)
diff --git a/harness/web_fetch/rescoring.py b/harness/web_fetch/rescoring.py
index 237ddba4f..f25a3709d 100644
--- a/harness/web_fetch/rescoring.py
+++ b/harness/web_fetch/rescoring.py
@@ -1,16 +1,22 @@
 """Recompute shared task metrics from immutable evidence; never call providers."""
 
+import argparse
 import hashlib
 import json
+from collections import deque
+from concurrent.futures import ProcessPoolExecutor
 from copy import deepcopy
 from datetime import datetime, timezone
 from pathlib import Path
 
-ROOT = Path(__file__).resolve().parents[2]
-from harness.record_io import parse_jsonl
-from harness.web_fetch.artifacts import verify_record_artifacts, file_scoring_metadata
+from lexbench_webfetch.metrics import summarize, task_passed
+from lexbench_webfetch.paths import contained_path
+from lexbench_webfetch.response_input import apply_request_status
+from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256, load_list_reference, score_attempt
+
+from harness.record_io import canonical_json_sha256, file_sha256, parse_jsonl
+from harness.web_fetch.artifacts import file_scoring_metadata, verify_record_artifacts
 from harness.web_fetch.failure_diagnosis import diagnose
-from lexbench_webfetch.metrics import summarize
 from harness.web_fetch.models import execution_id
 from harness.web_fetch.reference_lifecycle import (
     ReferenceLifecycleError,
@@ -21,10 +27,10 @@
     requires_fresh_reference,
 )
 from harness.web_fetch.report import failure_category, succeeded
-from lexbench_webfetch.response_input import apply_request_status
-from lexbench_webfetch.paths import contained_path
 from harness.web_fetch.safety import validate_report_ids
-from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256, load_list_reference, score_attempt
+from harness.web_fetch.storage import current_evidence_dir
+
+ROOT = Path(__file__).resolve().parents[2]
 
 
 def _rescore_one(arguments):
@@ -159,8 +165,6 @@ def accept(index,computed):
         for index in selected:
             accept(index,_rescore_one(arguments(index)))
     else:
-        from collections import deque
-        from concurrent.futures import ProcessPoolExecutor
         # Bound queued records as well as workers: saved answers and diagnostics
         # can be large, so eager map would retain many duplicated payloads.
         with ProcessPoolExecutor(max_workers=jobs) as pool:
@@ -190,10 +194,6 @@ def accept(index,computed):
 
 
 def main(argv=None):
-    import argparse
-
-    from harness.web_fetch.storage import current_evidence_dir
-
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("source", type=Path, nargs="?", default=None)
     args = parser.parse_args(argv)
@@ -231,10 +231,6 @@ def score_published_snapshot(source, dataset, *, jobs=1, recovered=None, progres
     evidence. Every changed artifact must match the saved publication receipt;
     an exact recovered original takes precedence over its public view.
     """
-    from concurrent.futures import ProcessPoolExecutor
-    from collections import deque
-    from harness.record_io import file_sha256
-    from lexbench_webfetch.metrics import task_passed
     if type(jobs) is not int or not 1 <= jobs <= 8:
         raise ValueError('Offline scoring jobs must be between 1 and 8')
     source = Path(source).resolve()
@@ -243,7 +239,7 @@ def score_published_snapshot(source, dataset, *, jobs=1, recovered=None, progres
     validate_report_ids(original, records)
     cases = {case['id']: case for case in dataset['cases']}
     old_cases = {case['id']: case for case in original['cases']}
-    from .scoring_revision import ALLOWED_FIELDS
+    from harness.web_fetch.scoring_revision import ALLOWED_FIELDS
     if cases.keys() != old_cases.keys() or dataset.get('coverage_plan') != original.get('coverage_plan'):
         raise ValueError('Published rescoring must retain the complete matrix')
     for key, prior in old_cases.items():
@@ -301,7 +297,6 @@ def accept(result):
                 following = next(remaining, None)
                 if following is not None:
                     pending.append(pool.submit(_score_published_row, following))
-    from harness.record_io import canonical_json_sha256
     receipt = {'kind': 'published_snapshot_rescore', 'provider_calls_added': 0,
         'corrected_dataset_sha256': canonical_json_sha256(dataset),
         'input_dataset_sha256': file_sha256(source / 'dataset.json'),
diff --git a/harness/web_fetch/resource_cohort.py b/harness/web_fetch/resource_cohort.py
index 5569ee14f..887c86dc0 100644
--- a/harness/web_fetch/resource_cohort.py
+++ b/harness/web_fetch/resource_cohort.py
@@ -4,19 +4,19 @@
 import hashlib
 import json
 import math
-from harness.numbers import positive_number
 import statistics
 from pathlib import Path
 
-from harness.record_io import load_jsonl
-from harness.web_fetch.publication_privacy import project_execution_manifest
-
-from .artifacts import verify_record_artifacts
 from lexbench_webfetch.metrics import percentile, task_passed
-from .reference_lifecycle import require_verified_reference_checks
-from .result_review import require_result_review
 from lexbench_webfetch.paths import contained_path
-from .speed_cohort import provider_cohort
+
+from harness.numbers import positive_number
+from harness.record_io import load_jsonl
+from harness.web_fetch.artifacts import verify_record_artifacts
+from harness.web_fetch.publication_privacy import project_execution_manifest
+from harness.web_fetch.reference_lifecycle import require_verified_reference_checks
+from harness.web_fetch.result_review import require_result_review
+from harness.web_fetch.speed_cohort import provider_cohort
 
 METRICS = (
     "measurement_duration_ms",
diff --git a/harness/web_fetch/result_ledger.py b/harness/web_fetch/result_ledger.py
index be50503df..950f7d9e8 100644
--- a/harness/web_fetch/result_ledger.py
+++ b/harness/web_fetch/result_ledger.py
@@ -1,6 +1,12 @@
 """Conserve task judgments across explicit offline corrections and exclusions."""
+
+import json
 from collections import Counter
 
+from lexbench_webfetch.metrics import task_measured, task_passed
+
+from harness.record_io import load_jsonl
+from harness.web_fetch.publication_privacy import read_evidence_binding
 
 VERDICTS = {'pass', 'fail', 'excluded', 'unavailable'}
 
@@ -70,10 +76,7 @@ def validate_ledger(ledger, records=None, *, source=None):
     if ledger.get('predecessor'):
         if source is None:
             raise ValueError('Result ledger needs its bound predecessor evidence')
-        import json
-        from .local_revision import require_local_task_replacement
-        from .publication_privacy import read_evidence_binding
-        from harness.record_io import load_jsonl
+        from harness.web_fetch.local_revision import require_local_task_replacement
         prior = json.loads(read_evidence_binding(source, ledger['predecessor']).read_text())
         validate_ledger(prior, source=source)
         count = len(prior['states'])
@@ -107,7 +110,6 @@ def validate_ledger(ledger, records=None, *, source=None):
                     or set(event['added_attempt_ids']) != added):
                 raise ValueError('Task replacement inventory differs from its physical calls')
     if records is not None:
-        from lexbench_webfetch.metrics import task_measured, task_passed
         actual = {record['attempt_id']: (
             'excluded' if record['metrics'].get('evaluation_excluded') else
             'unavailable' if not task_measured(record['metrics']) else
diff --git a/harness/web_fetch/result_review.py b/harness/web_fetch/result_review.py
index fd0552616..3f784db8a 100644
--- a/harness/web_fetch/result_review.py
+++ b/harness/web_fetch/result_review.py
@@ -1,16 +1,27 @@
 """Validate an independently produced, attempt-complete result review."""
+
 import argparse
 import hashlib
 import json
+import shutil
 import tempfile
+from datetime import datetime, timezone
 from pathlib import Path
 
-from harness.record_io import file_sha256, load_jsonl, parse_jsonl
-
-from .anomaly_review import require_anomaly_review
 from lexbench_webfetch.metrics import task_passed
 from lexbench_webfetch.paths import contained_path
 
+from harness.record_io import file_sha256, load_jsonl, parse_jsonl
+from harness.web_fetch import artifacts as evidence_artifacts
+from harness.web_fetch.anomaly_review import detect_anomalies, require_anomaly_review
+from harness.web_fetch.reference_inputs import require_mutable_inputs
+from harness.web_fetch.reference_lifecycle import valid_reference_revision, valid_scoring_revision
+from harness.web_fetch.safety import (
+    commit_staged_evidence,
+    evidence_tree_identity,
+    stage_evidence_revision,
+)
+
 ANSWER_METADATA_FIELDS = ('links', 'images', 'videos', 'media', 'extras')
 
 
@@ -99,7 +110,7 @@ def _machine_verdict(source, record, case, reviewed=None, *, revision_checks=Non
             raise
         if historical_authority:
             return (reviewed or {}).get('machine_verdict')
-        from .reference_lifecycle import valid_scoring_revision_chain
+        from harness.web_fetch.reference_lifecycle import valid_scoring_revision_chain
         verdict = (reviewed or {}).get('machine_verdict')
         if verdict not in {'pass', 'fail'}:
             raise
@@ -142,7 +153,6 @@ def same_review(left, right):
         raise
 
 
-
 def _require_result_review(source, dataset, records, review_document=None, *, anomaly_gate, historical_sources=(), historical_authority=False):
     required = dataset.get('comparison', {}).get('require_independent_result_review', False)
     path = source / 'result-review.json'
@@ -173,7 +183,6 @@ def _require_result_review(source, dataset, records, review_document=None, *, an
         raise ValueError('The declared full review must include successful and failed attempts')
     anomaly_attempts = set()
     if anomaly_gate and dataset.get('comparison', {}).get('require_anomaly_review', False):
-        from .anomaly_review import detect_anomalies
         reviewer = {row.get('attempt_id'): row.get('reviewer_anomalies', [])
                     for row in review.get('attempts', [])}
         anomaly_attempts = set(detect_anomalies(source, records, dataset, reviewer))
@@ -408,7 +417,7 @@ def unchanged():
     installed = target.read_bytes()
     try:
         if historical_sources:
-            from .assembly import bind_assembled_scoring_revision
+            from harness.web_fetch.assembly import bind_assembled_scoring_revision
             bind_assembled_scoring_revision(evidence, historical_sources=historical_sources)
     except BaseException:
         if target.read_bytes() == installed:
@@ -1078,7 +1087,6 @@ def merge_shards_main(argv=None) -> None:
     print(f"reviewed={len(review['attempts'])} disagreements={review['disagreements']} unresolved=0")
 
 
-
 def refresh_review_attempt_ids(evidence: Path, prior_evidence: Path) -> set[str]:
     """Return attempts whose task or reviewer-visible evidence changed."""
     current_dataset, current_records, _ = _read_review_evidence(evidence, skip_blank=True)
@@ -1104,8 +1112,6 @@ def refresh_review_attempt_ids(evidence: Path, prior_evidence: Path) -> set[str]
 def prepare_review_subset(evidence: Path, output: Path, *, case_ids=None,
                           prior_evidence: Path | None = None):
     """Stage a complete hash-verified supplemental scope before exposing its directory."""
-    from .artifacts import materialize_bound_artifacts
-    from .safety import evidence_tree_identity, stage_evidence_revision
 
     if bool(case_ids) == bool(prior_evidence):
         raise ValueError("choose exactly one of case_ids or prior_evidence")
@@ -1153,7 +1159,7 @@ def prepare(staging):
         (staging / "dataset.json").write_text(json.dumps(subset, ensure_ascii=False, indent=2) + "\n")
         (staging / "results.jsonl").write_text(
             "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records))
-        materialize_bound_artifacts(staging, evidence, records)
+        evidence_artifacts.materialize_bound_artifacts(staging, evidence, records)
         if prior_evidence and evidence_tree_identity(prior_evidence) != prior_identity:
             raise ValueError("Prior review evidence changed during subset preparation")
         return {"cases": len(cases), "attempts": len(records)}
@@ -1176,14 +1182,8 @@ def prepare_subset_main(argv=None):
 
 def rebind_metadata_review(evidence: Path, prior_dataset: Path, *, rebound_at=None):
     """Rebind neutral metadata in a validated bundle, preserving immutable history."""
-    import shutil
-    from datetime import datetime, timezone
 
-    from .artifacts import verify_record_artifacts
-    from .assembly import bind_assembled_scoring_revision
-    from .reference_inputs import require_mutable_inputs
-    from .reference_lifecycle import valid_reference_revision, valid_scoring_revision
-    from .safety import commit_staged_evidence, evidence_tree_identity
+    from harness.web_fetch.assembly import bind_assembled_scoring_revision
 
     before = evidence_tree_identity(evidence)
     evidence = evidence.resolve()
@@ -1216,7 +1216,7 @@ def rebind_metadata_review(evidence: Path, prior_dataset: Path, *, rebound_at=No
             raise ValueError("Reference revision history is invalid or unbound")
         native_cases = {c['id']: c for c in prior_document['cases']}
         for record in records:
-            verify_record_artifacts(staged, record, native_cases=native_cases)
+            evidence_artifacts.verify_record_artifacts(staged, record, native_cases=native_cases)
         (staged / "dataset.json").write_bytes(current_bytes)
         results_bytes = (staged / "results.jsonl").read_bytes()
         if not rebind_unchanged_result_review(
diff --git a/harness/web_fetch/retrieval_hints.py b/harness/web_fetch/retrieval_hints.py
index 1c0453a7d..b52a64e21 100644
--- a/harness/web_fetch/retrieval_hints.py
+++ b/harness/web_fetch/retrieval_hints.py
@@ -1,7 +1,6 @@
 """One parser for the public, answer-free dynamic retrieval contract."""
 
 from __future__ import annotations
-from lexbench_webfetch.inputs import definition_path
 
 import hashlib
 import json
@@ -11,8 +10,13 @@
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
-from typing import Any
 from types import MappingProxyType
+from typing import Any
+
+from lexbench_webfetch.inputs import definition_path
+from lexbench_webfetch.stable_tasks import validate_acceptance_contract
+
+from harness.web_fetch.reference_inputs import input_tree
 
 # Compare against this immutable declaration; store a fresh dict in each case.
 NATIVE_RETRIEVAL_HINTS = MappingProxyType({'version': 3, 'mode': 'native'})
@@ -76,7 +80,6 @@ def task_definition_sha256(case: Mapping[str, Any]) -> str:
 
 @lru_cache(maxsize=1)
 def _verified_source_tree() -> Path:
-    from .reference_inputs import input_tree
     return input_tree(
         definition_path("evaluation/source-inputs.json").parent / "dataset.json", "sources"
     )
@@ -555,7 +558,6 @@ def native_default_case(case: Any) -> bool:
     contract = get("acceptance_contract")
     if not contract:
         return False
-    from lexbench_webfetch.stable_tasks import validate_acceptance_contract
     validate_acceptance_contract(contract)
     return True
 
diff --git a/harness/web_fetch/review_execution.py b/harness/web_fetch/review_execution.py
index 420f29161..285d27dc6 100644
--- a/harness/web_fetch/review_execution.py
+++ b/harness/web_fetch/review_execution.py
@@ -11,15 +11,20 @@
 from pathlib import Path
 
 from bs4 import BeautifulSoup
+from lexbench_webfetch import metrics
+from lexbench_webfetch.metrics import task_passed
+from lexbench_webfetch.paths import contained_path, safe_identifier
 
 from harness.record_io import load_jsonl, parse_jsonl
-
 from harness.web_fetch import anomaly_review, result_review
-from lexbench_webfetch import metrics
-from .anomaly_review import LAYERS, TRIGGER_VERSION, detect_anomalies
-from .artifacts import artifact_bindings, record_private_bindings, verify_record_artifacts
-from lexbench_webfetch.metrics import task_passed
-from .result_review import (
+from harness.web_fetch.anomaly_review import LAYERS, TRIGGER_VERSION, detect_anomalies
+from harness.web_fetch.artifacts import (
+    artifact_bindings,
+    record_private_bindings,
+    verify_record_artifacts,
+)
+from harness.web_fetch.native_routes import native_cases_by_id
+from harness.web_fetch.result_review import (
     install_result_review,
     require_blind_result_review,
     require_result_review,
@@ -27,7 +32,6 @@
     review_raw_artifacts,
     write_review_document,
 )
-from lexbench_webfetch.paths import contained_path, safe_identifier
 from harness.web_fetch.safety import reject_evidence_symlinks
 
 SCHEMA = {
@@ -1252,7 +1256,6 @@ def bind_review_workspace(evidence, work, records, bindings, options, *, prior=N
     if current != bindings:
         raise ValueError("Review evidence changed during execution")
     artifacts = {}
-    from .native_routes import native_cases_by_id
     native_cases = native_cases_by_id(evidence)
     for record in records:
         verify_record_artifacts(evidence, record, native_cases=native_cases)
diff --git a/harness/web_fetch/risk_control.py b/harness/web_fetch/risk_control.py
index 7ed645d26..975230d5b 100644
--- a/harness/web_fetch/risk_control.py
+++ b/harness/web_fetch/risk_control.py
@@ -10,14 +10,14 @@
 from pathlib import Path
 from typing import Any
 
+from lexbench_webfetch.paths import contained_path
+
 from harness.web_fetch.artifacts import diagnostic_log_binding
 from harness.web_fetch.config import load_dataset, load_provider_config, load_runtime_env
-from harness.web_fetch.reference_inputs import read_dataset_bytes, benchmark_input_path
+from harness.web_fetch.reference_inputs import benchmark_input_path, read_dataset_bytes
 from harness.web_fetch.retrieval_hints import plan_for_case
 from harness.web_fetch.storage import data_home
 
-from lexbench_webfetch.paths import contained_path
-
 
 def _digest(path: Path) -> str:
     return hashlib.sha256(benchmark_input_path(path).read_bytes()).hexdigest()
@@ -348,6 +348,8 @@ def validate(profile_path: Path) -> tuple[dict, dict, list, object, Path]:
     provider_name = profile["provider"]["name"]
     if list(config) != [provider_name]:
         raise ValueError("Risk-control provider configuration must contain one Moli method")
+
+    # Certificate validation does not load the provider execution stack.
     from harness.web_fetch.providers import build_provider
 
     provider = build_provider(provider_name, config[provider_name])
@@ -419,6 +421,7 @@ def main(argv: list[str] | None = None, *, certified: bool = False) -> None:
     execution = profile["execution"]
     proof_before = frozen_risk_control_input_snapshot(profile_path)
     pre_identity = implementation_identity(provider)
+
     from harness.web_fetch.runner import run_evaluation
 
     records, _ = run_evaluation(
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index db4623611..c815b5177 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -18,39 +18,54 @@
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
-from harness.record_io import load_jsonl
+from lexbench_webfetch.metrics import summarize, task_passed
+from lexbench_webfetch.models import EvalCase, validate_case
+from lexbench_webfetch.paths import contained_path, safe_identifier
+from lexbench_webfetch.response_input import apply_request_status
+from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256, load_list_reference, score_attempt
 
-from .artifacts import (
+from harness.record_io import load_jsonl
+from harness.reporting.versions import software_identity
+from harness.web_fetch.artifacts import (
     PRIVATE_ARTIFACT_ROOT,
     PUBLIC_CALL_LOG_ROOT,
     diagnostic_log_binding,
+    file_scoring_metadata,
     raw_response_binding,
     verify_record_artifacts,
     verify_record_private_artifacts,
     verify_record_public_call_logs,
 )
-from .call_boundary import fetch_with_deadline
-from .config import DEFAULT_FAILURE_RETRIES
-from lexbench_webfetch.metrics import summarize, task_passed
+from harness.web_fetch.call_boundary import fetch_with_deadline
+from harness.web_fetch.config import DEFAULT_FAILURE_RETRIES
+from harness.web_fetch.diagnostics import CallDiagnostics
 from harness.web_fetch.models import AttemptRecord, ProviderResult
-from lexbench_webfetch.models import EvalCase, validate_case
-from .providers import Provider
-from .reference_inputs import read_dataset_bytes, benchmark_input_path
-from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
-from .reference_lifecycle import (
+from harness.web_fetch.native_projection import (
+    require_native_answer_projection,
+    require_native_projection_schema,
+)
+from harness.web_fetch.native_routes import (
+    NativeRouteLedger,
+    declared_input_media_kind,
+    native_cases_by_id,
+    verify_native_route_contract,
+)
+from harness.web_fetch.network_preflight import preflight_guarded_providers
+from harness.web_fetch.providers import Provider
+from harness.web_fetch.publication_privacy import redact_url_queries
+from harness.web_fetch.reference_inputs import benchmark_input_path, read_dataset_bytes
+from harness.web_fetch.reference_lifecycle import (
     ReferenceLifecycleError,
     check_case_reference_binding,
-    legacy_reference_dataset,
-    legacy_reference_cases,
     check_case_reference_fresh,
     is_diagnostic_evidence,
+    legacy_reference_cases,
+    legacy_reference_dataset,
     preflight_reference_checks,
+    validate_resource_source_evidence,
 )
-from .diagnostics import CallDiagnostics
-from lexbench_webfetch.response_input import apply_request_status
-from lexbench_webfetch.paths import contained_path, safe_identifier
-from lexbench_webfetch.scoring import load_list_reference, score_attempt
 
 
 class IncompleteEvaluationError(RuntimeError):
@@ -170,7 +185,6 @@ def _public_text(value: Any, *, limit: int = 4096) -> str | None:
     text = _AUTH_SECRET_VALUE.sub(lambda match: match.group(1) + "[REDACTED]", text)
     text = _JSON_SECRET_VALUE.sub(lambda match: match.group(1) + "[REDACTED]" + match.group(3), text)
     text = _LINE_SECRET_VALUE.sub(lambda match: match.group(1) + match.group(2) + " [REDACTED]", text)
-    from .publication_privacy import redact_url_queries
     text, _ = redact_url_queries(text)
     text = _POSIX_LOCAL_PATH.sub("[local-path]", text)
     text = _WINDOWS_LOCAL_PATH.sub("[local-path]", text)
@@ -181,7 +195,6 @@ def _public_url(value: Any) -> str | None:
     """Keep public HTTP(S) targets while redacting credentials and secret query values."""
     if value is None:
         return None
-    from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
     try:
         parsed = urlsplit(str(value))
     except ValueError:
@@ -391,7 +404,6 @@ def persist_result_artifacts(
     """Persist one physical call without changing its scored answer bytes."""
     discarded_late_answer = (not result.ok and result.error_type == 'timeout'
                              and result.content == '')
-    from .native_projection import require_native_answer_projection, require_native_projection_schema
     if result.metadata.get('native_answer_projection'):
         require_native_projection_schema(result.metadata['native_answer_projection'])
     raw = result.metadata.get('_raw_response_b64')
@@ -525,7 +537,6 @@ def run_evaluation(
         if provider.config.get("uniform_native_parameters"):
             if runs != 1:
                 raise ValueError("fixed native calls require one run")
-            from .native_routes import declared_input_media_kind
             for case in cases:
                 provider.check_native_call(case)
                 declared_input_media_kind(case)
@@ -536,7 +547,6 @@ def run_evaluation(
         raise ValueError('Provider names must be unique within an execution')
     if len({case.id for case in cases}) != len(cases):
         raise ValueError('Task IDs must be unique within an execution')
-    from .native_routes import NativeRouteLedger
     native_routes = (NativeRouteLedger(cases, providers, timeout_seconds=timeout)
                      if any(p.config.get('uniform_native_parameters') for p in providers) else None)
     for filename in ['artifacts', PUBLIC_CALL_LOG_ROOT, 'manifest.json', 'dataset.json',
@@ -546,7 +556,6 @@ def run_evaluation(
         raise ValueError("unknown reference policy")
     if reference_policy == "frozen_existing":
         # Explicit local measurement of existing rules; no live-source claim.
-        from .reference_lifecycle import validate_resource_source_evidence
         for case in cases:
             validate_resource_source_evidence(case, dataset_path)
         frozen_dataset_sha256 = hashlib.sha256(read_dataset_bytes(dataset_path)).hexdigest()
@@ -559,7 +568,6 @@ def run_evaluation(
             raise ValueError(
                 f"provider {provider.name!r} missing configuration: {', '.join(missing)}"
             )
-    from .network_preflight import preflight_guarded_providers
     network_audit = preflight_guarded_providers(providers, cases)
     output_dir.mkdir(parents=True, exist_ok=True)
     if network_audit is not None:
@@ -570,7 +578,6 @@ def run_evaluation(
     started_at = utc_now()
     records: list[dict[str, Any]] = []
 
-    from harness.reporting.versions import software_identity
     manifest = {
         "schema_version": 1,
         "runtime_source_sha256": hashlib.sha256(SCORING_SOURCE_SHA256.encode()+b'\0'+b'\0'.join(
@@ -867,7 +874,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) ->
                         ).to_dict()
                         apply_request_status(record)
                         record["completed_at"] = finished
-                        from .models import execution_id
+                        from harness.web_fetch.models import execution_id
                         record['run_id'] = execution_id(record)
                         passed = task_passed(record['ok'],record['metrics'],case.task_kind)
                         state = "PASS" if passed else "FAIL"
@@ -951,7 +958,6 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) ->
         json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
     )
     if native_routes is not None:
-        from .native_routes import verify_native_route_contract
         verify_native_route_contract(output_dir, manifest, dataset_meta, records)
     expected_attempts = len(cases) * len(providers) * runs
     if len(records) != expected_attempts or manifest["unexecuted"]:
@@ -976,7 +982,6 @@ def rescore_evaluation(
         contained_path(run_dir,filename)
     case_by_id = {case.id: case for case in cases}
     records = load_records(run_dir)
-    from .native_routes import native_cases_by_id
     native_cases = native_cases_by_id(run_dir)
     for record in records:
         if record.get("metrics", {}).get("content_sha256"):
@@ -1026,7 +1031,6 @@ def rescore_evaluation(
             raise ValueError(f"run contains case absent from dataset: {record['case_id']}")
         artifact = contained_path(run_dir,record["artifact_path"])
         previous_deadline = bool(record.get("metrics", {}).get("deadline_overrun"))
-        from .artifacts import file_scoring_metadata
         record["metrics"] = score_attempt(
             case, artifact.read_bytes(), ok=bool(record.get("ok")),
             title=record.get('title'), final_url=record.get('final_url'),
diff --git a/harness/web_fetch/safety.py b/harness/web_fetch/safety.py
index 1139f0d93..ca7e4cc9e 100644
--- a/harness/web_fetch/safety.py
+++ b/harness/web_fetch/safety.py
@@ -1,10 +1,14 @@
 """Validation at filesystem and URL boundaries shared by runners and reports."""
+
+import ctypes
 import hashlib
 import os
+import shutil
 import sys
 import tempfile
 from pathlib import Path
 from urllib.parse import urlsplit
+
 from lexbench_webfetch.paths import safe_identifier, validate_http_url
 
 
@@ -121,10 +125,8 @@ def copy_evidence_file(source, destination):
     Never hard-link frozen inputs: writing a generated copy must not alter its
     source. Unsupported filesystems/platforms retain ordinary copy semantics.
     """
-    import shutil
     source, destination = unlinked_path(Path(source)), unlinked_path(Path(destination))
     if sys.platform == 'darwin':
-        import ctypes
         try:
             clone = ctypes.CDLL(None, use_errno=True).clonefile
         except (AttributeError, OSError):
diff --git a/harness/web_fetch/scoring_revision.py b/harness/web_fetch/scoring_revision.py
index adfe0d137..93a5f2894 100644
--- a/harness/web_fetch/scoring_revision.py
+++ b/harness/web_fetch/scoring_revision.py
@@ -8,7 +8,10 @@
 from datetime import datetime, timezone
 from pathlib import Path
 
-ROOT = Path(__file__).resolve().parents[2]
+from lexbench_webfetch.paths import contained_path
+from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
+from lexbench_webfetch.scoring_regressions import ROOT_CAUSE_CATEGORIES, build_correction_provenance
+
 from harness.record_io import file_sha256, parse_jsonl
 from harness.web_fetch.reference_inputs import read_dataset_bytes
 from harness.web_fetch.reference_lifecycle import (
@@ -22,10 +25,10 @@
     archive_superseded_result_review,
     rebind_unchanged_result_review,
 )
-from lexbench_webfetch.paths import contained_path
 from harness.web_fetch.safety import validate_report_ids
-from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
-from lexbench_webfetch.scoring_regressions import ROOT_CAUSE_CATEGORIES, build_correction_provenance
+
+ROOT = Path(__file__).resolve().parents[2]
+
 
 ALLOWED_FIELDS = {
     "page_identity_required",
@@ -274,7 +277,6 @@ def _revise_unlocked(
 
 def rebind_report_interpretation(source: Path, binding: dict, *, implementation_binding: dict | None = None):
     """Bind presentation content while preserving the scored output bytes."""
-    from .reference_lifecycle import valid_scoring_revision
     source = source.resolve()
     if not valid_scoring_revision(source):
         raise ValueError('Report binding requires a valid existing scoring chain')
diff --git a/harness/web_fetch/source_capture.py b/harness/web_fetch/source_capture.py
index eec314437..3646bdd10 100644
--- a/harness/web_fetch/source_capture.py
+++ b/harness/web_fetch/source_capture.py
@@ -14,6 +14,8 @@
 import re
 import socket
 import tempfile
+import urllib.error
+import urllib.request
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from datetime import datetime, timezone
@@ -22,12 +24,14 @@
 from urllib.parse import urljoin, urlsplit
 
 from bs4 import BeautifulSoup
+from lexbench_webfetch.metrics import normalize_for_match
+from lexbench_webfetch.paths import contained_path, safe_identifier, validate_http_url
+from lexbench_webfetch.reference_quality import audit_reference, clean_reference_text
 
 from harness.web_fetch.document_text import extract_document_text
-from lexbench_webfetch.metrics import normalize_for_match
+from harness.web_fetch.reference_inputs import read_dataset_bytes
 from harness.web_fetch.reference_lifecycle import scope_identity
-from lexbench_webfetch.reference_quality import audit_reference, clean_reference_text
-from lexbench_webfetch.paths import contained_path, safe_identifier, validate_http_url
+from harness.web_fetch.safety import unlinked_path, write_new_file
 
 
 def extract_html(raw: bytes, url: str):
@@ -327,7 +331,6 @@ def collect(
         raise ValueError("workers must be positive")
     if output.exists() or output.is_symlink():
         raise ValueError("Capture output must be a new directory")
-    from .reference_inputs import read_dataset_bytes
     cases = json.loads(read_dataset_bytes(dataset))["cases"]
     ids = [safe_identifier(case["id"]) for case in cases]
     if not ids or len(set(ids)) != len(ids):
@@ -404,16 +407,6 @@ def main(argv=None):
 
 
 def resource_links_main(argv=None) -> None:
-    import argparse
-    import json
-    import urllib.error
-    import urllib.request
-    from datetime import datetime, timezone
-    from pathlib import Path
-    from urllib.parse import urljoin
-    from harness.web_fetch.reference_inputs import read_dataset_bytes
-    from harness.web_fetch.safety import unlinked_path, write_new_file
-
     parser = argparse.ArgumentParser()
     parser.add_argument("--dataset", type=Path, default=Path("evaluation/dataset.json"))
     parser.add_argument("--output", type=Path, required=True)
diff --git a/harness/web_fetch/speed_cohort.py b/harness/web_fetch/speed_cohort.py
index 58d956231..7c0ca5093 100644
--- a/harness/web_fetch/speed_cohort.py
+++ b/harness/web_fetch/speed_cohort.py
@@ -1,4 +1,5 @@
 """Build a bound same-window latency comparison from a complete headless subcohort."""
+
 import argparse
 import hashlib
 import json
@@ -9,9 +10,11 @@
 from pathlib import Path
 from urllib.parse import urlsplit
 
+from lexbench_webfetch.metrics import percentile, task_passed
+
 from harness.record_io import load_jsonl
 from harness.web_fetch.artifacts import materialize_task_references
-from lexbench_webfetch.metrics import percentile, task_passed
+from harness.web_fetch.config import derive_comparison
 from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes
 from harness.web_fetch.reference_lifecycle import require_verified_reference_checks
 
@@ -203,7 +206,6 @@ def selection_main():
     except ValueError as error:
         parser.error(str(error))
     result=deepcopy(dataset);result['cases']=selected
-    from .config import derive_comparison
     result['comparison']=derive_comparison(dataset, {'kind':'headless_same_window_speed',
         'selection':'Provider-blind reference-length quantile sampling over ordinary web body tasks; at most one case per hostname',
         'planned_before_calls':True,'case_count':len(selected)})
diff --git a/harness/web_fetch/storage.py b/harness/web_fetch/storage.py
index 556f55822..1141b143b 100644
--- a/harness/web_fetch/storage.py
+++ b/harness/web_fetch/storage.py
@@ -7,8 +7,10 @@
 from collections import Counter
 from pathlib import Path
 
-from harness.record_io import load_jsonl
+from lexbench_webfetch.paths import contained_path
 
+from harness.record_io import load_jsonl
+from harness.web_fetch.methods import native_browser_engine
 
 
 def data_home() -> Path:
@@ -76,7 +78,7 @@ def frozen_input_bindings(evidence: Path) -> dict[str, str]:
 
 def require_canonical_inputs(evidence: Path, canonical: Path) -> None:
     """Reject a report build when its frozen inputs differ from Git."""
-    from .reference_inputs import input_tree, read_dataset_bytes
+    from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes
 
     for relative in (Path("dataset.json"), Path("sources"), Path("list-references")):
         expected = canonical / relative
@@ -141,7 +143,6 @@ def require_current_report_profile(evidence: Path, profile_path: Path) -> None:
     records = load_jsonl(evidence / "results.jsonl")
     evidence_binding = profile.get("evidence_binding")
     if evidence_binding is not None:
-        from lexbench_webfetch.paths import contained_path
 
         files = evidence_binding.get("files") if isinstance(evidence_binding, dict) else None
         required = {"dataset.json", "results.jsonl", "execution.json"}
@@ -268,13 +269,12 @@ def require_current_report_profile(evidence: Path, profile_path: Path) -> None:
     if any(not identifier for identifier in attempt_ids) or len(attempt_ids) != len(set(attempt_ids)):
         failures.append("attempt_ids")
     route_contract = profile.get("native_task_route_contract")
-    from .methods import native_browser_engine
     if profile.get("comparison_kind") == "headless_markdown" and any(native_browser_engine(name) for name in providers) and route_contract is None:
         raise ValueError("Native browser report requires native_task_route_contract")
     if route_contract is not None:
         if route_contract != 4:
             raise ValueError("Unsupported native task route contract")
-        from .task_outputs import require_native_task_routes
+        from harness.web_fetch.task_outputs import require_native_task_routes
         require_native_task_routes(cases, records)
     resource_profile = profile.get("resource_profile") or {}
     if resource_profile.get("required"):
diff --git a/harness/web_fetch/task_outputs.py b/harness/web_fetch/task_outputs.py
index f9798fe2a..71c9fbd96 100644
--- a/harness/web_fetch/task_outputs.py
+++ b/harness/web_fetch/task_outputs.py
@@ -5,6 +5,14 @@
 downloads document bytes for external conversion.
 """
 
+import hashlib
+import json
+from types import SimpleNamespace
+
+from harness.web_fetch.methods import native_browser_engine
+from harness.web_fetch.providers.headless import output_mode_for_case
+from harness.web_fetch.retrieval_hints import NATIVE_RETRIEVAL_HINTS, native_contract_for_case
+
 RESOURCE_PAGE_EXPRESSION = r"""
 (() => {
   const urls = new Set();
@@ -73,7 +81,6 @@ def resource_detail_links(case):
 
 def resource_expression(track, detail_links=False):
     """Select resource element types from the declared task, never its answers."""
-    import json
     selectors = {'image':'img,picture source',
                  'video':'video,video source,audio,audio source,iframe,a[href]'}
     if track not in selectors:
@@ -160,20 +167,14 @@ def resource_expression(track, detail_links=False):
 """
 
 
-
 def require_native_task_routes(cases, records):
     """Reject substitutions and stale task adapters before report publication."""
-    import hashlib
-    from types import SimpleNamespace
-    from .providers.headless import output_mode_for_case
-    from .retrieval_hints import NATIVE_RETRIEVAL_HINTS, native_contract_for_case
 
     inventory = {case['id']: SimpleNamespace(id=case['id'], url=case['url'], track=case.get('track','web'), task_kind=case.get('task_kind','body'), retrieval_hints=case.get('retrieval_hints',{}), source_benchmark=case.get('source_benchmark',{}), expected_resources=[{'kind':r.get('kind')} for r in case.get('expected_resources',[])]) for case in cases}
     expressions = {'resource':RESOURCE_PAGE_EXPRESSION, 'media_metadata':MEDIA_METADATA_EXPRESSION,
                    'document_page':DOCUMENT_PAGE_EXPRESSION}
     failures = []
     for row in records:
-        from .methods import native_browser_engine
         provider = (
             'moli'
             if row.get('provider_type') == 'moli_webfetch'
diff --git a/harness/web_fetch/title_quality.py b/harness/web_fetch/title_quality.py
index c6e40b578..24b41aba7 100644
--- a/harness/web_fetch/title_quality.py
+++ b/harness/web_fetch/title_quality.py
@@ -6,16 +6,17 @@
 conditional on returned, reference-covered titles; task acceptance still tests
 retrieval. Never write these values to word_* or use them to change task_pass.
 """
+
 import hashlib
 import json
 import re
 import statistics
 from pathlib import Path
-from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
 
 from lexbench_webfetch.list_verification import identity
 from lexbench_webfetch.metrics import word_overlap
 from lexbench_webfetch.paths import contained_path
+from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
 
 
 def reference_title(value):
diff --git a/harness/web_fetch/wcxb_latency.py b/harness/web_fetch/wcxb_latency.py
index 54995dcf8..bcd8a1396 100644
--- a/harness/web_fetch/wcxb_latency.py
+++ b/harness/web_fetch/wcxb_latency.py
@@ -1,21 +1,26 @@
 """Validate complete, certified WCXB version timing corrections."""
 from __future__ import annotations
 
+import argparse
 import hashlib
 import json
 import math
-import statistics
-import argparse
 import os
 import re
+import statistics
 import subprocess
+import sys
 import time
 from datetime import datetime, timezone
 from pathlib import Path
 
-from harness.record_io import parse_jsonl
 from lexbench_webfetch.paths import contained_path
-from harness.web_fetch.safety import reject_evidence_symlinks
+
+from harness.record_io import parse_jsonl
+from harness.web_fetch import webmainbench_runner
+from harness.web_fetch.providers import build_provider
+from harness.web_fetch.safety import evidence_tree_identity, reject_evidence_symlinks
+from harness.web_fetch.wcxb_dataset import validate_manifest
 
 
 def load_full_paired_latency(root: Path, profile: dict, valid_cases: set[str]) -> tuple[dict, dict]:
@@ -126,9 +131,7 @@ def collect(*, dataset: Path, manifest: Path, profile: Path, output: Path,
             providers: dict, binaries: dict, wait_quiet_seconds: int = 0,
             inputs: tuple[Path, ...] = ()) -> dict:
     """Collect the full declared two-build crossover; never select by outcome."""
-    from harness.web_fetch.wcxb_dataset import validate_manifest
     from harness.web_fetch.wcxb_scoring import participant_names
-    from harness.web_fetch.webmainbench_runner import fetch_record, fixture_server, load_wcxb_source
 
     if type(wait_quiet_seconds) is not int or wait_quiet_seconds < 0:
         raise ValueError('wait-quiet-seconds must be a nonnegative integer')
@@ -140,7 +143,6 @@ def collect(*, dataset: Path, manifest: Path, profile: Path, output: Path,
         raise ValueError('Timing output must stay outside the source dataset')
     watched = [profile, manifest, *binaries.values(), *inputs]
     watched += [dataset / 'test/html', dataset / 'test/ground-truth', dataset / 'evaluate.py']
-    from harness.web_fetch.safety import evidence_tree_identity
 
     def source_tree():
         reject_evidence_symlinks(contained_path(dataset, 'test', reject_symlinks=True))
@@ -162,7 +164,7 @@ def unchanged():
     participant_names(authority)
     if type(authority.get('concurrency', 1)) is not int or authority.get('concurrency', 1) != 1:
         raise ValueError('Paired timing requires serial execution')
-    records, source_hash = load_wcxb_source(dataset)
+    records, source_hash = webmainbench_runner.load_wcxb_source(dataset)
     manifest_bytes = manifest.read_bytes()
     selected, _ = validate_manifest(records, source_hash, json.loads(manifest_bytes))
     hashes = {method: before[path] for method, path in binaries.items()}
@@ -196,7 +198,7 @@ def unchanged():
     output.mkdir(parents=True)
     started = datetime.now(timezone.utc).isoformat()
     calls_path = output / 'calls.jsonl'
-    with fixture_server(selected) as (base, ca), calls_path.open('x', encoding='utf-8') as sink:
+    with webmainbench_runner.fixture_server(selected) as (base, ca), calls_path.open('x', encoding='utf-8') as sink:
         for provider in providers.values():
             provider.config['ca_cert'] = str(ca)
         for round_number in (0, 1):
@@ -206,7 +208,7 @@ def unchanged():
                 first = 'release' if (index + round_number) % 2 == 0 else 'pr'
                 for order, method in enumerate((first, 'pr' if first == 'release' else 'release'), 1):
                     start = time.monotonic()
-                    response = fetch_record(providers[method], None, record,
+                    response = webmainbench_runner.fetch_record(providers[method], None, record,
                                             f"{base}/case/{record['track_id']}",
                                             authority['timeout_seconds'])
                     row = {'round': round_number, 'case': record['track_id'],
@@ -243,12 +245,9 @@ def unchanged():
     return result
 
 
-
 def certify(argv=None) -> dict:
     """Bind the complete frozen timing receipt without changing any measured calls."""
-    from harness.web_fetch.wcxb_dataset import validate_manifest
     from harness.web_fetch.wcxb_scoring import participant_names
-    from harness.web_fetch.webmainbench_runner import load_wcxb_source
 
     parser = argparse.ArgumentParser(description='Certify a complete WCXB paired latency cohort')
     parser.add_argument('--dataset', type=Path, required=True)
@@ -268,7 +267,7 @@ def certify(argv=None) -> dict:
         raw[path] = path.read_bytes()
     authority = json.loads(raw[args.profile])
     participant_names(authority)
-    records, before = load_wcxb_source(args.dataset)
+    records, before = webmainbench_runner.load_wcxb_source(args.dataset)
     selected, _ = validate_manifest(records, before, json.loads(raw[args.sample_manifest]))
     if (type(authority.get('case_count')) is not int or len(selected) != authority.get('case_count')
             or hashlib.sha256(raw[args.dataset / 'evaluate.py']).hexdigest()
@@ -276,7 +275,7 @@ def certify(argv=None) -> dict:
         raise ValueError('Paired timing scope or evaluator differs from profile')
     receipt, rows = load_full_paired_latency(args.evidence_root, authority,
                                             {row['track_id'] for row in selected})
-    _, after = load_wcxb_source(args.dataset)
+    _, after = webmainbench_runner.load_wcxb_source(args.dataset)
     reject_evidence_symlinks(contained_path(args.dataset, 'test', reject_symlinks=True))
     if (before != after or before != receipt['source_sha256']
             or hashlib.sha256(raw[args.sample_manifest]).hexdigest() != receipt['manifest_sha256']
@@ -300,12 +299,10 @@ def certify(argv=None) -> dict:
 
 def main(argv=None, *, root: Path | None = None) -> None:
     """Caller-relative CLI; existing binary flags retain their two-build meaning."""
-    import sys
     argv = list(sys.argv[1:] if argv is None else argv)
     if argv[:1] == ['certify']:
         certify(argv[1:])
         return
-    from harness.web_fetch.providers import build_provider
 
     parser = argparse.ArgumentParser(description='Collect complete profile-bound WCXB paired timing')
     for name in ('dataset', 'manifest', 'release', 'candidate', 'output'):
diff --git a/harness/web_fetch/wcxb_publication.py b/harness/web_fetch/wcxb_publication.py
index dc8ec62bf..b3b69b923 100644
--- a/harness/web_fetch/wcxb_publication.py
+++ b/harness/web_fetch/wcxb_publication.py
@@ -9,15 +9,23 @@
 from pathlib import Path
 from urllib.parse import quote
 
+from lexbench_webfetch.paths import contained_path
+
+from harness.reporting import links as report_links
 from harness.reporting.publication import (
-    PublicationOperation, archive_inventory, atomic_write, json_bytes, package_archive,
-    public_share_context, snapshot_publication,
+    PublicationOperation,
+    archive_inventory,
+    atomic_write,
+    json_bytes,
+    package_archive,
+    public_share_context,
+    snapshot_publication,
 )
+from harness.web_fetch import wcxb_report_check
 from harness.web_fetch.artifact_auth import load_token
 from harness.web_fetch.publication import upload_large_archive
 from harness.web_fetch.publication_target import publication_base, require_publication_version
 from harness.web_fetch.report_publish import response_json
-from lexbench_webfetch.paths import contained_path
 from harness.web_fetch.safety import evidence_tree_identity, reject_evidence_symlinks
 from harness.web_fetch.storage import data_home
 
@@ -58,9 +66,6 @@ def inventory(report: Path, profile: dict) -> dict[str, str]:
 
 
 def package(source: Path, archive: Path, receipt: Path, *, authority_root: Path, profile_path: Path | None = None) -> dict:
-    from harness.reporting.links import check as check_links
-    from harness.web_fetch.wcxb_report_check import check as check_report
-
     profile_path = profile_path or authority_root / 'evaluation/wcxb/current-report.json'
     binding_path = profile_path.with_name('report-content-binding.json')
     reject_evidence_symlinks(profile_path)
@@ -71,8 +76,8 @@ def package(source: Path, archive: Path, receipt: Path, *, authority_root: Path,
 
     def validate():
         inventory(source, profile)
-        check_report(source, authority_root=authority_root, profile_path=profile_path)
-        check_links(source)
+        wcxb_report_check.check(source, authority_root=authority_root, profile_path=profile_path)
+        report_links.check(source)
 
     def attest(staged):
         files = inventory(source, profile)
diff --git a/harness/web_fetch/wcxb_report_check.py b/harness/web_fetch/wcxb_report_check.py
index 501e8b83a..c99cf4bde 100644
--- a/harness/web_fetch/wcxb_report_check.py
+++ b/harness/web_fetch/wcxb_report_check.py
@@ -13,13 +13,22 @@
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 from bs4 import BeautifulSoup
+from lexbench_webfetch.paths import contained_path
 
-from harness.web_fetch.safety import reject_evidence_symlinks
 from harness.record_io import load_jsonl
-from harness.report_languages import localized_report_routes, visible_numbers
+from harness.report_languages import (
+    check_language_contract,
+    generated_languages,
+    localized_report_routes,
+    visible_numbers,
+)
+from harness.reporting.assets import authored_main
 from harness.reporting.presentation import check_presentation
-from lexbench_webfetch.paths import contained_path
+from harness.reporting.visual import local_report_server
+from harness.web_fetch.safety import reject_evidence_symlinks
 from harness.web_fetch.storage import data_home
+from harness.web_fetch.wcxb_latency import load_full_paired_latency, summarize_full_paired_latency
+from harness.web_fetch.wcxb_scoring import paired_latency_methods
 
 ROOT = Path(__file__).resolve().parents[2]
 
@@ -238,7 +247,6 @@ def require_home_interpretation(html: str | BeautifulSoup, summary: dict,
 
 def require_guide_interpretation(html: str | BeautifulSoup, summary: dict,
                                  providers: list[str], lang: str, *, expected_html: str) -> BeautifulSoup:
-
     """A guide must work as a standalone explanation, with traceable examples."""
     soup = BeautifulSoup(html, 'html.parser') if isinstance(html, str) else html
     expected = BeautifulSoup(expected_html, 'html.parser')
@@ -294,9 +302,6 @@ def require_case_summary(html: str | BeautifulSoup, lang: str) -> BeautifulSoup:
 
 
 def check_bound_interpretation(root: Path, binding: dict) -> None:
-    from harness.report_languages import generated_languages
-    from harness.reporting.assets import authored_main
-
     for language in generated_languages(root):
         route = ('en/' if language == 'en' else '') + 'guide/index.html'
         page = contained_path(root, route, reject_symlinks=True)
@@ -318,7 +323,7 @@ def check(root: Path, *, authority_root: Path | None = None, profile_path: Path
     reject_evidence_symlinks(profile_path)
     profile = json.loads(profile_path.read_text())
     comparison = profile.get('paired_comparison') or {}
-    from .wcxb_publication import inventory
+    from harness.web_fetch.wcxb_publication import inventory
     inventory(root, profile)
     binding_path = profile_path.with_name('report-content-binding.json')
     reject_evidence_symlinks(binding_path)
@@ -352,7 +357,6 @@ def check(root: Path, *, authority_root: Path | None = None, profile_path: Path
     require(summary['benchmark'] == sample['dataset_id'] == certificate['benchmark'], 'WCXB certified dataset identity differs')
     require({(r['provider'], r['case_id']) for r in scores} == {(p, c) for p in providers for c in case_ids}, 'WCXB method-case matrix differs')
     require([r['provider'] for r in certificate['providers']] == providers, 'WCXB certificate participants differ')
-    from harness.web_fetch.wcxb_scoring import paired_latency_methods
     paired_methods = paired_latency_methods(profile, certificate) if profile.get('latency_correction_receipt_sha256') else {}
     if profile.get('optional_ui_receipt_sha256'):
         optional_paths = [name for name, sha in content_binding['evidence'].items()
@@ -378,10 +382,6 @@ def check(root: Path, *, authority_root: Path | None = None, profile_path: Path
                     for build in optional['builds'].values()),
                 'WCXB optional feature evidence lacks observed outcomes')
 
-    from harness.web_fetch.wcxb_latency import (
-        load_full_paired_latency,
-        summarize_full_paired_latency,
-    )
     paired_latency = None
     if profile.get('latency_correction_receipt_sha256'):
         _, selected_latency = load_full_paired_latency(root / 'research/wcxb-full-paired-latency', profile, case_ids)
@@ -601,8 +601,6 @@ def binary_from_env(path: Path) -> str:
 
 def check_visual(root: Path, browser_binary: str) -> None:
     from playwright.sync_api import sync_playwright
-    from harness.reporting.visual import local_report_server
-    from harness.report_languages import check_language_contract, generated_languages
 
     root = root.resolve()
     check_language_contract(root)
diff --git a/harness/web_fetch/wcxb_scoring.py b/harness/web_fetch/wcxb_scoring.py
index 8610f17c6..5525ac01f 100644
--- a/harness/web_fetch/wcxb_scoring.py
+++ b/harness/web_fetch/wcxb_scoring.py
@@ -14,9 +14,10 @@
 from numbers import Real
 from pathlib import Path
 
+from lexbench_webfetch.paths import contained_path, safe_identifier
+
 from harness.record_io import parse_jsonl
 from harness.reporting.publication import atomic_create, json_bytes
-from lexbench_webfetch.paths import contained_path, safe_identifier
 from harness.web_fetch.safety import reject_evidence_symlinks
 from harness.web_fetch.wcxb_dataset import validate_manifest
 from harness.web_fetch.wcxb_latency import load_full_paired_latency
diff --git a/harness/web_fetch/webmainbench_assembly.py b/harness/web_fetch/webmainbench_assembly.py
index 61effa874..789bbadf9 100644
--- a/harness/web_fetch/webmainbench_assembly.py
+++ b/harness/web_fetch/webmainbench_assembly.py
@@ -11,18 +11,19 @@
 from pathlib import Path
 
 from lexbench_webfetch.paths import contained_path, safe_identifier
+
 from harness.web_fetch.webmainbench_certification import verify_shard
 from harness.web_fetch.webmainbench_evidence import (
     alias_row,
     copy_alias_artifacts,
     copy_bound,
     digest,
+    load_sample_contract,
     read_jsonl,
     require_matrix,
     require_unique_attempt_ids,
-    load_sample_contract,
-    scored_shard_bindings,
     retry_chain_matches,
+    scored_shard_bindings,
 )
 from harness.web_fetch.webmainbench_runner import load_source
 from harness.web_fetch.webmainbench_scoring import aggregate, verify_score
diff --git a/harness/web_fetch/webmainbench_case_correction.py b/harness/web_fetch/webmainbench_case_correction.py
index e55532626..b7854c2eb 100644
--- a/harness/web_fetch/webmainbench_case_correction.py
+++ b/harness/web_fetch/webmainbench_case_correction.py
@@ -7,12 +7,22 @@
 from collections import defaultdict
 from pathlib import Path
 
-from harness.record_io import load_jsonl
-
 from lexbench_webfetch.paths import contained_path, safe_identifier
+
+from harness.record_io import load_jsonl
 from harness.web_fetch.safety import evidence_tree_identity, stage_evidence_revision
-from .webmainbench_evidence import copy_alias_artifacts, copy_bound, require_unique_attempt_ids, retry_chain_matches
-from .webmainbench_scoring import CORE_METRICS, DATASET_SHA256, SCORER_COMMIT, aggregate
+from harness.web_fetch.webmainbench_evidence import (
+    copy_alias_artifacts,
+    copy_bound,
+    require_unique_attempt_ids,
+    retry_chain_matches,
+)
+from harness.web_fetch.webmainbench_scoring import (
+    CORE_METRICS,
+    DATASET_SHA256,
+    SCORER_COMMIT,
+    aggregate,
+)
 
 AXES = ("language", "level", "table", "code", "equation")
 SHARD_FILES = ("scored-results.jsonl", "results.jsonl", "physical-attempts.jsonl",
diff --git a/harness/web_fetch/webmainbench_certification.py b/harness/web_fetch/webmainbench_certification.py
index fa138a823..364e795e1 100644
--- a/harness/web_fetch/webmainbench_certification.py
+++ b/harness/web_fetch/webmainbench_certification.py
@@ -10,10 +10,12 @@
 import sys
 from pathlib import Path
 
-from harness.web_fetch.config import load_provider_config, load_runtime_env
 from lexbench_webfetch.paths import contained_path, safe_identifier
+
+from harness.web_fetch.config import load_provider_config, load_runtime_env
 from harness.web_fetch.webmainbench_evidence import digest, read_jsonl, require_unique_attempt_ids
 from harness.web_fetch.webmainbench_runner import PROVIDERS, load_source
+from harness.web_fetch.webmainbench_scoring import verify_score
 
 
 def _read_json(root: Path, relative: str) -> dict:
@@ -233,10 +235,7 @@ def scorer_source_binding(scorer: Path) -> dict[str, str]:
         scorer, path.relative_to(scorer), reject_symlinks=True)) for path in files}
 
 
-
 def main_score(argv: list[str] | None = None) -> None:
-    from .webmainbench_scoring import verify_score
-
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--cohort", type=Path, required=True)
     parser.add_argument("--source", type=Path, required=True)
diff --git a/harness/web_fetch/webmainbench_correction.py b/harness/web_fetch/webmainbench_correction.py
index 5dfe271c7..63b1d062b 100644
--- a/harness/web_fetch/webmainbench_correction.py
+++ b/harness/web_fetch/webmainbench_correction.py
@@ -14,7 +14,12 @@
 import types
 from pathlib import Path
 
+from lexbench_webfetch.paths import contained_path, safe_identifier
+
+from harness.distribution import installed_data_path
 from harness.record_io import load_jsonl
+from harness.web_fetch.safety import stage_evidence_revision
+from harness.web_fetch.webmainbench_case_correction import dimensions
 from harness.web_fetch.webmainbench_certification import verify_shard
 from harness.web_fetch.webmainbench_diagnostics import corrected_distance
 from harness.web_fetch.webmainbench_scoring import (
@@ -25,10 +30,6 @@
     revised_metrics,
 )
 
-from lexbench_webfetch.paths import contained_path, safe_identifier
-from harness.web_fetch.safety import stage_evidence_revision
-from harness.distribution import installed_data_path
-
 
 def digest(path):
     return hashlib.sha256(path.read_bytes()).hexdigest()
@@ -107,9 +108,6 @@ def apply(
     validation_rows: list | None = None,
     expected_parent: str | None = None,
 ) -> tuple[list, dict]:
-    from harness.web_fetch.webmainbench_case_correction import dimensions
-    from harness.web_fetch.webmainbench_scoring import aggregate
-
     certificate, replacements, calls = validate(
         root,
         validation_rows if validation_rows is not None else original_rows,
@@ -702,7 +700,6 @@ def _revise_dataset_scoring(source: Path, corrected_dataset: Path, manifest_path
         summary = summaries[provider]
         summary["pre_scoring_correction_aggregate"] = copy.deepcopy(summary["official_aggregate"])
         summary["official_aggregate"] = aggregate(provider_rows)
-        from .webmainbench_case_correction import dimensions
         summary["dimensions"] = dimensions(provider_rows)
         summary["scorer"]["repository"] = "https://github.com/lexmount/WebMainBench"
         summary["scorer"]["commit"] = scorer_commit
diff --git a/harness/web_fetch/webmainbench_diagnostics.py b/harness/web_fetch/webmainbench_diagnostics.py
index 995f669e1..eb7499de5 100644
--- a/harness/web_fetch/webmainbench_diagnostics.py
+++ b/harness/web_fetch/webmainbench_diagnostics.py
@@ -12,26 +12,43 @@
 import tempfile
 import types
 from collections import defaultdict
+from datetime import datetime, timezone
 from pathlib import Path
 
-from harness.record_io import load_jsonl
 from lexbench_webfetch.metrics import task_passed
-from harness.web_fetch.result_review import write_review_document
+from lexbench_webfetch.paths import contained_path
 from lexbench_webfetch.scoring import score_attempt
-from lexbench_webfetch.scoring_regressions import build_correction_provenance, load_scoring_regressions
-from harness.web_fetch.webmainbench_evidence import require_matrix
+from lexbench_webfetch.scoring_regressions import (
+    build_correction_provenance,
+    load_scoring_regressions,
+    valid_correction_provenance,
+)
+
+from harness.record_io import load_jsonl
+from harness.web_fetch.result_review import write_review_document
+from harness.web_fetch.safety import (
+    evidence_tree_identity,
+    reject_evidence_symlinks,
+    stage_evidence_revision,
+    unlinked_path,
+    write_new_file,
+)
+from harness.web_fetch.webmainbench_evidence import (
+    load_sample_contract,
+    read_jsonl,
+    require_matrix,
+    scored_shard_bindings,
+)
+from harness.web_fetch.webmainbench_latency_correction import prior_results_path
 from harness.web_fetch.webmainbench_scoring import (
+    CORE_METRICS,
+    METRIC_SOURCE_MANIFEST_SHA256,
     SCORER_COMMIT,
     aggregate,
     metric_dict,
     official_calculator,
 )
 
-from lexbench_webfetch.paths import contained_path
-from harness.web_fetch.safety import evidence_tree_identity, reject_evidence_symlinks, stage_evidence_revision
-from lexbench_webfetch.scoring_regressions import valid_correction_provenance
-from .webmainbench_latency_correction import prior_results_path
-
 ERROR_PAGE = re.compile(
     "(?:^#\\s*Error response\\b|^Error code:\\s*404\\b|ERR[_\\\\]ACCESS[_\\\\]DENIED\\b|"
     "^#\\s*无法访问此网站\\b|^This site can.t be reached\\b)",
@@ -531,7 +548,8 @@ def category(row):
         (output / "score-summaries.json").write_text(
             json.dumps(summaries, ensure_ascii=False, indent=2) + "\n"
         )
-        from lexbench_webfetch import ordered_tree as ordered_tree_module, scoring as scoring_module
+        from lexbench_webfetch import ordered_tree as ordered_tree_module
+        from lexbench_webfetch import scoring as scoring_module
 
         def source_name(path):
             return path.relative_to(ROOT) if path.is_relative_to(ROOT) else Path("lexbench_webfetch") / path.name
@@ -804,11 +822,6 @@ def close(a: float, b: float) -> bool:
 
 
 def sample_review_main(argv=None, *, root=None) -> None:
-    from datetime import datetime, timezone
-
-    from .webmainbench_evidence import load_sample_contract, read_jsonl, scored_shard_bindings
-    from .webmainbench_scoring import CORE_METRICS, METRIC_SOURCE_MANIFEST_SHA256
-
     parser = argparse.ArgumentParser(description="Audit a profile-declared WebMainBench sample without the official scorer")
     parser.add_argument("--source-root", type=Path, required=True)
     parser.add_argument("--dataset", type=Path, required=True)
@@ -1168,7 +1181,6 @@ def suspicion_main(argv=None) -> None:
     parser.add_argument("--source-dataset", type=Path, required=True)
     parser.add_argument("--output", type=Path)
     args = parser.parse_args(argv)
-    from .safety import unlinked_path, write_new_file
 
     if args.output:
         args.output = unlinked_path(args.output)
diff --git a/harness/web_fetch/webmainbench_evidence.py b/harness/web_fetch/webmainbench_evidence.py
index c28c13115..462ad2c6a 100644
--- a/harness/web_fetch/webmainbench_evidence.py
+++ b/harness/web_fetch/webmainbench_evidence.py
@@ -8,10 +8,11 @@
 import shutil
 from pathlib import Path
 
-from harness.record_io import load_jsonl
-
 from lexbench_webfetch.paths import contained_path, safe_identifier
 
+from harness.record_io import load_jsonl
+from harness.web_fetch.webmainbench_runner import load_source
+
 
 def retry_chain_matches(row: dict, attempts: list[dict]) -> bool:
     correction = row.get("provider_metadata", {}).get("latency_correction", {})
@@ -162,7 +163,6 @@ def read_method_binding(root: Path) -> tuple[str, dict]:
 
 def load_sample_contract(source: Path, manifest_path: Path, profile_path: Path):
     """Read a predeclared sample and method matrix without assuming its size or providers."""
-    from .webmainbench_runner import load_source
 
     for path in (source, manifest_path, profile_path):
         contained_path(path.parent, path.name, reject_symlinks=True)
diff --git a/harness/web_fetch/webmainbench_latency_correction.py b/harness/web_fetch/webmainbench_latency_correction.py
index c16872dc9..d7a85b071 100644
--- a/harness/web_fetch/webmainbench_latency_correction.py
+++ b/harness/web_fetch/webmainbench_latency_correction.py
@@ -12,10 +12,11 @@
 import time
 from pathlib import Path
 
-from harness.record_io import parse_jsonl
-
 from lexbench_webfetch.paths import contained_path
 
+from harness.record_io import parse_jsonl
+from harness.web_fetch import webmainbench_runner
+
 
 def prior_results_path(root: Path) -> Path:
     if (root / "latency-corrections.json").is_file():
@@ -192,10 +193,8 @@ def match(pattern: str, value: str) -> float | None:
 
 
 def snapshot(dataset: Path, sample: Path, official: Path) -> dict:
-    from .webmainbench_runner import load_selected_ids, load_source
-
-    selected, binding = load_selected_ids(sample)
-    source = {row["track_id"]: row for row in load_source(dataset, selected)}
+    selected, binding = webmainbench_runner.load_selected_ids(sample)
+    source = {row["track_id"]: row for row in webmainbench_runner.load_source(dataset, selected)}
     recorded = json.loads((official / "dataset.json").read_text(encoding="utf-8"))
     cases = {row["id"]: row for row in recorded["cases"]}
     if (
@@ -236,7 +235,7 @@ def validate_host_sample(sample: dict) -> None:
 
 
 def certify(argv: list[str] | None = None) -> None:
-    from .webmainbench_latency_diagnostic import _repeat_calls
+    from harness.web_fetch.webmainbench_latency_diagnostic import _repeat_calls
 
     parser = argparse.ArgumentParser(description=__doc__)
     for name in (
@@ -376,7 +375,7 @@ def certify(argv: list[str] | None = None) -> None:
 
 
 def main(argv: list[str] | None = None) -> None:
-    from .webmainbench_latency_diagnostic import _repeat_calls
+    from harness.web_fetch.webmainbench_latency_diagnostic import _repeat_calls
 
     parser = argparse.ArgumentParser(description=__doc__)
     for name in ("base", "cohort", "dataset", "sample", "output"):
diff --git a/harness/web_fetch/webmainbench_latency_diagnostic.py b/harness/web_fetch/webmainbench_latency_diagnostic.py
index 663f82e8d..b78f13d9f 100644
--- a/harness/web_fetch/webmainbench_latency_diagnostic.py
+++ b/harness/web_fetch/webmainbench_latency_diagnostic.py
@@ -5,22 +5,25 @@
 import hashlib
 import json
 import math
-import re
 import random
+import re
+import statistics
 import subprocess
 import time
-import statistics
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
 
-from harness.record_io import load_jsonl, parse_jsonl
-
 from lexbench_webfetch.metrics import percentile
-from lexbench_webfetch.paths import contained_path
-from .webmainbench_diagnostics import ERROR_PAGE
-from .webmainbench_evidence import retry_chain_matches
-from .webmainbench_latency_correction import prior_results_path
+from lexbench_webfetch.paths import contained_path, safe_identifier
+
+from harness.record_io import load_jsonl, parse_jsonl
+from harness.web_fetch import webmainbench_runner
+from harness.web_fetch.providers.headless import LOOPBACK_SANDBOX_PROFILE
+from harness.web_fetch.safety import unlinked_path
+from harness.web_fetch.webmainbench_diagnostics import ERROR_PAGE
+from harness.web_fetch.webmainbench_evidence import retry_chain_matches
+from harness.web_fetch.webmainbench_latency_correction import prior_results_path
 
 RUNS = {'moli-115-control': 'moli', 'moli-117-repeat': 'moli-1.1.7'}
 
@@ -366,16 +369,6 @@ def prior_call_rows(path: Path) -> dict[str, dict]:
 
 
 def paired_tail_main(argv=None) -> None:
-    from .providers.headless import LOOPBACK_SANDBOX_PROFILE
-    from lexbench_webfetch.paths import safe_identifier
-    from harness.web_fetch.safety import unlinked_path
-    from .webmainbench_runner import (
-        fixture_server,
-        load_selected_ids,
-        load_source,
-        verify_loopback_sandbox,
-    )
-
     parser = argparse.ArgumentParser(
         description="Repeat declared paired Moli latency calls serially."
     )
@@ -425,14 +418,14 @@ def paired_tail_main(argv=None) -> None:
         )
         if result.stdout.strip() != f"moli {version}":
             raise ValueError("Moli version mismatch: " + version)
-    selected_ids, sample = load_selected_ids(args.sample)
+    selected_ids, sample = webmainbench_runner.load_selected_ids(args.sample)
     if args.formal_correction and sample["case_count"] != 200:
         raise ValueError("Expected frozen 200-page sample")
     source_options = {"expected_count": args.source_count}
     if args.source_sha256:
         source_options["expected_sha256"] = args.source_sha256
     source = {
-        row["track_id"]: row for row in load_source(args.dataset, selected_ids, **source_options)
+        row["track_id"]: row for row in webmainbench_runner.load_source(args.dataset, selected_ids, **source_options)
     }
     if not source:
         raise ValueError("Paired timing requires a nonempty sample")
@@ -494,8 +487,8 @@ def paired_tail_main(argv=None) -> None:
         }
     (args.output / "runner-source.py").write_bytes(frozen_source)
     (args.output / "selection.json").write_text(json.dumps(selection, indent=2) + "\n")
-    with fixture_server([source[case_id] for case_id in cases]) as (base, ca_cert):
-        selection["network_probe"] = verify_loopback_sandbox(base, ca_cert, source[cases[0]])
+    with webmainbench_runner.fixture_server([source[case_id] for case_id in cases]) as (base, ca_cert):
+        selection["network_probe"] = webmainbench_runner.verify_loopback_sandbox(base, ca_cert, source[cases[0]])
         (args.output / "selection.json").write_text(json.dumps(selection, indent=2) + "\n")
         with (args.output / "calls.jsonl").open("w") as stream:
             for round_index in range(args.rounds):
diff --git a/harness/web_fetch/webmainbench_report_check.py b/harness/web_fetch/webmainbench_report_check.py
index e75294726..e6cf90d4d 100644
--- a/harness/web_fetch/webmainbench_report_check.py
+++ b/harness/web_fetch/webmainbench_report_check.py
@@ -10,6 +10,8 @@
 from pathlib import Path
 
 from bs4 import BeautifulSoup
+from lexbench_webfetch.metrics import percentile
+from lexbench_webfetch.paths import contained_path
 
 from harness.record_io import load_jsonl
 from harness.report_languages import (
@@ -17,14 +19,13 @@
     generated_languages,
     localized_report_routes,
 )
+from harness.reporting.assets import load_bound_content
 from harness.reporting.presentation import check_presentation
-from lexbench_webfetch.metrics import percentile
 from harness.web_fetch.report import ERROR_PAGE, EXTRA, JS, STYLE
 from harness.web_fetch.report_changes import webmainbench_history
-from lexbench_webfetch.paths import contained_path
 from harness.web_fetch.safety import reject_evidence_symlinks
 from harness.web_fetch.webmainbench_diagnostics import validate_teds_revision_if_present
-from harness.web_fetch.webmainbench_evidence import retry_chain_matches
+from harness.web_fetch.webmainbench_evidence import read_method_binding, retry_chain_matches
 from harness.web_fetch.webmainbench_latency_correction import (
     prior_results_path,
     validate_latency_correction,
@@ -34,8 +35,6 @@
 
 
 def validate_sample_content(root: Path, *, authority_root: Path, binding_path: Path | None = None) -> None:
-    from harness.reporting.assets import load_bound_content
-
     selected = binding_path or contained_path(authority_root, 'evaluation/webmainbench-content-binding.json', reject_symlinks=True)
     reject_evidence_symlinks(selected)
     expected = selected.read_bytes()
@@ -199,7 +198,6 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
             or not str(target.get('share_url', '')).startswith(str(target.get('host', '')).rstrip('/') + '/v/')
             or not target.get('site_slug')):
         raise ValueError('WebMainBench publication target is not bound to the certified profile')
-    from .webmainbench_evidence import read_method_binding
     _, method_binding = read_method_binding(root)
     profile = json.loads(profile_path.read_text(encoding='utf-8'))
     expected_providers = profile["providers"] + variant_ids
@@ -886,7 +884,6 @@ def check_sample_report(root: Path, dataset: dict | None = None,
     validate_sample_content(root, authority_root=authority_root or repo, binding_path=profile_path.parent / "webmainbench-content-binding.json")
 
 
-
 def main():
     parser = argparse.ArgumentParser(description="Check a complete WebMainBench report")
     parser.add_argument("report", type=Path)
diff --git a/harness/web_fetch/webmainbench_runner.py b/harness/web_fetch/webmainbench_runner.py
index 3256b88f8..0c0fecb12 100644
--- a/harness/web_fetch/webmainbench_runner.py
+++ b/harness/web_fetch/webmainbench_runner.py
@@ -23,17 +23,23 @@
 from pathlib import Path
 from urllib.parse import urlsplit
 
-ROOT = Path(__file__).resolve().parents[2]
+from lexbench_webfetch.models import EvalCase
 
 from harness.record_io import parse_jsonl
-from harness.web_fetch.config import load_provider_config, load_runtime_env, add_failure_retry_argument
-from lexbench_webfetch.models import EvalCase
+from harness.web_fetch.config import (
+    add_failure_retry_argument,
+    load_provider_config,
+    load_runtime_env,
+)
 from harness.web_fetch.models import ProviderResult
 from harness.web_fetch.providers import build_provider
 from harness.web_fetch.providers.cloudflare import CloudflareMarkdownProvider
 from harness.web_fetch.providers.headless import LOOPBACK_SANDBOX_PROFILE
 from harness.web_fetch.runner import classify_exception
 
+ROOT = Path(__file__).resolve().parents[2]
+
+
 DATASET_SHA256 = "0efaa4b49a45e320a27fe6e5a0b6aad5b57259fc3321ac3448519cacc74c537e"
 SCORER_COMMIT = "9d991bdc00c57b57521499494d96be85c31317ba"
 PROVIDERS = ("kitesurf", "lightpanda", "obscura", "moli", "moli-1.1.7", "chromium")
diff --git a/harness/web_fetch/webmainbench_scoring.py b/harness/web_fetch/webmainbench_scoring.py
index 80f9179dc..38d7b0f96 100644
--- a/harness/web_fetch/webmainbench_scoring.py
+++ b/harness/web_fetch/webmainbench_scoring.py
@@ -13,7 +13,10 @@
 from pathlib import Path
 from threading import RLock
 
+from lexbench_webfetch.paths import contained_path
+
 from harness.record_io import load_jsonl
+from harness.web_fetch.webmainbench_evidence import digest, read_jsonl
 
 SCORER_COMMIT = "9d991bdc00c57b57521499494d96be85c31317ba"
 DATASET_SHA256 = "0efaa4b49a45e320a27fe6e5a0b6aad5b57259fc3321ac3448519cacc74c537e"
@@ -266,9 +269,6 @@ def main() -> None:
 
 
 def verify_score(shard: Path, provider: str, case_ids: set[str], scorer: Path, profile: dict) -> dict:
-    from lexbench_webfetch.paths import contained_path
-    from .webmainbench_evidence import digest, read_jsonl
-
     scored = read_jsonl(shard / "scored-results.jsonl")
     measured = read_jsonl(shard / "results.jsonl")
     summary = json.loads((shard / "score-summary.json").read_text(encoding="utf-8"))
diff --git a/harness/web_fetch/workers/cowagent_worker.py b/harness/web_fetch/workers/cowagent_worker.py
index 132212746..bc8638b91 100644
--- a/harness/web_fetch/workers/cowagent_worker.py
+++ b/harness/web_fetch/workers/cowagent_worker.py
@@ -7,6 +7,9 @@
 from contextlib import redirect_stdout
 from pathlib import Path
 
+from harness.web_fetch.diagnostics import diagnostic_context, redact
+from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode
+
 
 def main():
     root = Path.cwd()
@@ -16,8 +19,6 @@ def main():
     # JSONL response, including prints that happen during module initialization.
     sys.path.insert(0, str(root))
     sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
-    from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode
-    from harness.web_fetch.diagnostics import diagnostic_context, redact
 
     class NativeLogSink:
         def emit(self, source, event, **fields):
diff --git a/harness/web_fetch/workers/crawl4ai_worker.py b/harness/web_fetch/workers/crawl4ai_worker.py
index 043b46e3e..3021d5438 100755
--- a/harness/web_fetch/workers/crawl4ai_worker.py
+++ b/harness/web_fetch/workers/crawl4ai_worker.py
@@ -3,9 +3,9 @@
 
 import asyncio
 import base64
-import os
 import json
 import multiprocessing
+import os
 import sys
 import tempfile
 from contextlib import AsyncExitStack, redirect_stdout
@@ -14,13 +14,15 @@
 from urllib.parse import urlsplit
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
-from harness.web_fetch.network_dns import validate_dns_mode, crawl4ai_public_dns_browser
 
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
 from crawl4ai.async_configs import HTTPCrawlerConfig
 from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
 from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
 
+from harness.web_fetch.native_routes import declared_input_media_kind
+from harness.web_fetch.network_dns import crawl4ai_public_dns_browser, validate_dns_mode
+
 LIBRARY_VERSION = version("crawl4ai")
 PDF_IMPLEMENTATION = (
     "crawl4ai.processors.pdf.PDFCrawlerStrategy+crawl4ai.processors.pdf.PDFContentScrapingStrategy"
@@ -146,7 +148,6 @@ def is_direct_pdf_document(request):
     if request.get("task_kind", "body") != "body":
         return False
     if request.get("uniform_native_parameters"):
-        from harness.web_fetch.native_routes import declared_input_media_kind
         return declared_input_media_kind(request) == "direct_pdf"
     path = urlsplit(request["url"]).path.lower().rstrip("/")
     return path.endswith(".pdf") or "/pdf/" in path
diff --git a/harness/web_fetch/workers/langchain_webbase_worker.py b/harness/web_fetch/workers/langchain_webbase_worker.py
index 9cc5ade4e..25e49b948 100755
--- a/harness/web_fetch/workers/langchain_webbase_worker.py
+++ b/harness/web_fetch/workers/langchain_webbase_worker.py
@@ -7,6 +7,7 @@
 import sys
 from importlib.metadata import version
 from pathlib import Path
+
 sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
 from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode
 
diff --git a/harness/web_search/adapters.py b/harness/web_search/adapters.py
index ae9d226ac..26acad45a 100644
--- a/harness/web_search/adapters.py
+++ b/harness/web_search/adapters.py
@@ -1,9 +1,10 @@
 """Provider transport registry. Add a provider here without changing the collector."""
+
+import json
+import re
 from dataclasses import dataclass
 from datetime import date, timedelta
-import json
 from urllib.parse import urlencode
-import re
 
 BILLING_POLICIES = frozenset({'unknown', 'http-200', 'http-200-nonempty'})
 
@@ -43,7 +44,6 @@ def timing(self, payload):
         return {key: payload.get(key) for key in ("timing", "usage", "cache", "attempts", "stop_reason", "coverage")}
 
 
-
 def _options(options, allowed):
     options = options or {}
     if set(options) - allowed or any(type(v) is not bool for v in options.values()):
diff --git a/harness/web_search/comparison.py b/harness/web_search/comparison.py
index 9512ee354..7dc2068c2 100644
--- a/harness/web_search/comparison.py
+++ b/harness/web_search/comparison.py
@@ -5,12 +5,20 @@
 import re
 from urllib.parse import urlsplit
 
-from lexbench_websearch.scoring import candidate_identity, evaluate, factual_evidence, in_domain
 from lexbench_websearch.evidence_quality import is_benchmark_page
+from lexbench_websearch.relevance import query_metrics
+from lexbench_websearch.scoring import (
+    candidate_identity,
+    evaluate,
+    factual_evidence,
+    in_domain,
+    task_acceptance,
+)
+
+from harness.web_search.relevance import parse_judgments
 
 
 def acceptance(case, row, metric="source", *, quality_excluded=False, cutoff=5, relevance_grades=None):
-    from lexbench_websearch.scoring import task_acceptance
     return task_acceptance(case, row, metric, quality_excluded=quality_excluded, cutoff=cutoff, relevance_grades=relevance_grades)
 
 
@@ -71,7 +79,6 @@ def comparison_data(profile, selected_rows, zh=False, detail_link=None, relevanc
     judgments = {}
     packets = {p["case_id"]: p for p in (relevance or {}).get("blind_candidate_packets", [])}
     for call in (relevance or {}).get("judge", {}).get("calls", []):
-        from .relevance import parse_judgments
         packet = packets[call["case_id"]]
         judgments[call["case_id"]] = {x["candidate_id"]: x for x in parse_judgments(packet, call["response"])}
     cases = []
@@ -108,7 +115,6 @@ def comparison_data(profile, selected_rows, zh=False, detail_link=None, relevanc
                               "grade": (labels[case["id"]].get(candidate_identity(case["id"], item["url"])) if case["id"] in labels and _url(item.get("url")) and not (quality_excluded or contaminated) else None),
                               "site_match": not (quality_excluded or contaminated) and any(in_domain(item.get("url", ""), d) for d in case.get("domains", [])),
                               **displayed})
-            from lexbench_websearch.relevance import query_metrics
             ranking = [candidate_identity(case["id"], item["url"]) for item in row.get("results", [])[:cutoff] if _url(item.get("url"))]
             query_score = query_metrics(ranking, labels[case["id"]], cutoff) if case["id"] in labels and not quality_excluded else None
             outputs[method["id"]] = {
diff --git a/harness/web_search/core.py b/harness/web_search/core.py
index f87b3f736..2ede1acb2 100644
--- a/harness/web_search/core.py
+++ b/harness/web_search/core.py
@@ -4,9 +4,6 @@
 Unjudged URLs are not irrelevant documents. Provider scores are not judgments.
 """
 
-from lexbench_websearch.contracts import validate_cases
-from lexbench_websearch.scoring import canonical_url
-
 import hashlib
 import json
 import re
@@ -15,6 +12,11 @@
 from urllib.parse import urljoin
 
 from lexbench_webfetch.metrics import percentile
+from lexbench_websearch.contracts import validate_cases
+from lexbench_websearch.scoring import canonical_url, normalize_results
+
+from harness.reporting.publication import atomic_write
+from harness.web_search.adapters import BILLING_POLICIES, Adapter, adapter_for, normalize_envelope
 
 
 def digest(data):
@@ -23,19 +25,15 @@ def digest(data):
 
 def request_cost(method, status, transport_exit, payload):
     """Billing follows the provider response, not local URL acceptance."""
-    from .adapters import Adapter, adapter_for
     adapter = adapter_for(method["adapter"]) if method.get("adapter") else Adapter(None)
     return adapter.cost(method, status, transport_exit, payload)
 
 
 def request_body(adapter, case, protocol, options=None):
-    from .adapters import adapter_for
     return adapter_for(adapter).request(case, protocol, options)
 
 
 def normalize(payload, adapter=None, top_k=None, *, base_url=None, project_slots=False):
-    from .adapters import normalize_envelope
-    from lexbench_websearch.scoring import normalize_results
     payload = normalize_envelope(payload, adapter, top_k=top_k, project_slots=project_slots)
     if isinstance(payload, dict) and isinstance(payload.get("results"), list):
         pages = payload["results"][:top_k] if project_slots and top_k is not None else payload["results"]
@@ -132,7 +130,6 @@ def validate_profile(profile):
     query_limit = 4096 if protocol.get("preserve_publisher_queries") else 128
     for method in methods:
         canonical_url(method["endpoint"])
-        from .adapters import adapter_for, BILLING_POLICIES
         adapter_for(method["adapter"])
         if method.get("billing", "unknown") not in BILLING_POLICIES:
             raise ValueError("Unsupported billing policy")
@@ -166,5 +163,4 @@ def verify_bundle(root, bindings):
 
 
 def write_json(path, value):
-    from harness.reporting.publication import atomic_write
     atomic_write(Path(path), (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode())
diff --git a/harness/web_search/delivery.py b/harness/web_search/delivery.py
index 9382c5f69..3f8861af8 100644
--- a/harness/web_search/delivery.py
+++ b/harness/web_search/delivery.py
@@ -4,16 +4,15 @@
 their generated pages and raw evidence without generating alternative findings.
 """
 
-import json
-import gzip
 import base64
+import gzip
+import json
 from pathlib import Path
 from urllib.parse import quote
 
 from bs4 import BeautifulSoup
 
-from .core import digest, write_json
-
+from harness.web_search.core import digest, write_json
 
 BRIDGE = r"""
 const nativeFetch=window.fetch.bind(window);
diff --git a/harness/web_search/diagnostics.py b/harness/web_search/diagnostics.py
index 34c683a88..c830f9620 100644
--- a/harness/web_search/diagnostics.py
+++ b/harness/web_search/diagnostics.py
@@ -3,6 +3,8 @@
 from collections import Counter
 from statistics import mean
 
+from harness.web_search.adapters import adapter_for
+
 
 def error_evidence(row, payload):
     """Keep collection errors and nested service failure evidence distinguishable."""
@@ -22,7 +24,6 @@ def operational(rows, payloads, methods=None):
     by_method = {m["id"]: m for m in (methods or [])}
     for row in rows:
         payload = payloads.get(row["id"], {})
-        from .adapters import adapter_for
         adapter = by_method.get(row["method"], {}).get("adapter")
         if adapter:
             timing = adapter_for(adapter).timing(payload)
diff --git a/harness/web_search/example.py b/harness/web_search/example.py
index c657eb823..a4ed99ec5 100644
--- a/harness/web_search/example.py
+++ b/harness/web_search/example.py
@@ -4,9 +4,10 @@
 import json
 from pathlib import Path
 
-from .core import digest, normalize, write_json
 from lexbench_websearch.scoring import evaluate
-from .report import build
+
+from harness.web_search.core import digest, normalize, write_json
+from harness.web_search.report import build
 
 
 def synthetic_profile():
diff --git a/harness/web_search/messages.py b/harness/web_search/messages.py
index 21712b43f..67306dfc1 100644
--- a/harness/web_search/messages.py
+++ b/harness/web_search/messages.py
@@ -3,6 +3,7 @@
 from functools import cache
 from pathlib import Path
 
+
 @cache
 def _catalog():
     return json.loads(Path(__file__).with_suffix(".json").read_text(encoding="utf-8"))
diff --git a/harness/web_search/pages.py b/harness/web_search/pages.py
index ede5db8c9..b3b492257 100644
--- a/harness/web_search/pages.py
+++ b/harness/web_search/pages.py
@@ -1,6 +1,7 @@
 """One top-level page registry for navigation, release checks and local captures."""
 
-from .messages import report_text
+from harness.web_search.messages import report_text
+
 _STANDARD = (
     ("home", "index.html", "Overview", report_text('text_fea405f9b01d1416'), True),
     ("dataset", "dataset.html", "Queries", report_text('text_a18713d1bce780df'), True),
diff --git a/harness/web_search/profiles.py b/harness/web_search/profiles.py
index 9d535d967..5a61f7235 100644
--- a/harness/web_search/profiles.py
+++ b/harness/web_search/profiles.py
@@ -2,19 +2,23 @@
 
 import copy
 import json
+import os
+from collections import Counter
 from pathlib import Path
 
-
-from .core import digest, validate_profile
+from lexbench_webfetch.paths import contained_path
 from lexbench_websearch.inputs import resolve_profile
 
+from harness.web_fetch.publication_target import validate_service_base
+from harness.web_search.core import digest, validate_profile
+
+
 def authority_root(path, root=None):
     if path is None:
         raise ValueError("Select an external Search instance with --profile")
     return Path(root).resolve() if root is not None else Path(path).resolve().parent
 
 
-
 def _inside(root, relative):
     relative = Path(relative)
     if relative.is_absolute() or ".." in relative.parts:
@@ -123,7 +127,6 @@ def publication_authority(manifest, path, *, root=None, planned_target=None, exp
             raise ValueError("Publication plan no longer matches the exact live target")
         target = planned_target
 
-    from harness.web_fetch.publication_target import validate_service_base
     validate_service_base(target['base_url'])
     if not target.get("slug") or not target.get("versionId") or target.get("url") != target["base_url"].rstrip("/") + "/s/" + target["slug"]:
         raise ValueError("Incomplete live search publication target")
@@ -152,7 +155,6 @@ def publication_authority(manifest, path, *, root=None, planned_target=None, exp
 
 def publication_metadata(manifest, review, relevance):
     """Derive release populations from the same evidence used by the renderer."""
-    from collections import Counter
     profile = manifest['profile']
     factual = {c['id'] for c in profile['cases'] if 'reference_answer' in c}
     excluded = factual & set(profile.get('factual_quality_exclusions', []))
@@ -168,8 +170,6 @@ def publication_metadata(manifest, review, relevance):
 
 def verify_publication_inputs(manifest, target, *, root=None, data_home=None):
     """Verify pinned source and previous-release files before a replacement upload."""
-    from lexbench_webfetch.paths import contained_path
-    import os
     root = Path(root) if root is not None else None
     data_home = Path(data_home) if data_home is not None else Path(os.environ.get("BROWSER_EVAL_DATA_HOME", Path.home() / ".browser-eval")).expanduser()
     bound_paths = {}
diff --git a/harness/web_search/publication.py b/harness/web_search/publication.py
index 70a96540b..9774df343 100644
--- a/harness/web_search/publication.py
+++ b/harness/web_search/publication.py
@@ -1,19 +1,31 @@
 """Freeze and verify the next release before updating its existing Artifact Site."""
+
 import argparse
-from datetime import UTC, datetime
 import json
 import os
-from pathlib import Path
 import shutil
 import subprocess
 import zipfile
+from datetime import UTC, datetime
+from pathlib import Path
 
 from lexbench_webfetch.paths import contained_path
+
 from harness.reporting.publication import atomic_write
-from harness.web_fetch.report_publish import PublishConfig, publish as publish_package
-from .core import digest
 from harness.web_fetch.config import ROOT
-from .profiles import current_selection, authority_root, publication_metadata, collection_receipts
+from harness.web_fetch.report_publish import PublishConfig
+from harness.web_fetch.report_publish import publish as publish_package
+from harness.web_search import delivery as report_delivery
+from harness.web_search import profiles
+from harness.web_search import report as search_report
+from harness.web_search import visual as report_visual
+from harness.web_search.core import digest
+from harness.web_search.profiles import (
+    authority_root,
+    collection_receipts,
+    current_selection,
+    publication_metadata,
+)
 
 
 def data_home():
@@ -25,11 +37,8 @@ def save(path, value):
 
 
 def prepare(report, visual, content, sources, work, *, profile):
-    from .report import check
-    from .delivery import export, verify as verify_delivery
-    from .visual import verify as verify_visual
-    check(report)
-    verify_visual(report, visual)
+    search_report.check(report)
+    report_visual.verify(report, visual)
     manifest = json.loads((report / "manifest.json").read_bytes())
     if not manifest["receipt"].get("certified") or manifest["receipt"].get("synthetic"):
         raise ValueError("Publication requires certified measured evidence")
@@ -74,8 +83,8 @@ def bind(name, source, filename):
         bindings[cohort + "_profile"] = {k: expected[cohort][k] for k in ("path", "sha256")}
         bind(cohort + "_receipt", source / "receipt.json", cohort + "-receipt.json")
         bind(cohort + "_frozen_profile", source / "profile.json", cohort + "-profile.json")
-    export(report, work / "delivery")
-    verify_delivery(report, work / "delivery")
+    report_delivery.export(report, work / "delivery")
+    report_delivery.verify(report, work / "delivery")
     bind("delivery_index", work / "delivery/delivery-index.json", "delivery-index.json")
     archive = work / "report.zip"
     with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as output:
@@ -100,7 +109,6 @@ def bind(name, source, filename):
 
 
 def verify_plan(report, work):
-    from .profiles import publication_authority, verify_publication_inputs
     plan = json.loads((work / "publication-plan.json").read_bytes())
     manifest = json.loads((report / "manifest.json").read_bytes())
     if digest((work / "report.zip").read_bytes()) != plan["archive_sha256"] or plan["target"]["archive_sha256"] != plan["archive_sha256"]:
@@ -121,7 +129,7 @@ def verify_plan(report, work):
         if live.get("versionId") == activated.get("versionId") and live.get("archive_sha256") == plan["archive_sha256"]:
             expected = {k: live[k] for k in ("base_url", "slug", "versionId", "visibility")}
             candidate = dict(candidate, versionId=live["versionId"])
-    publication_authority(manifest, profile, root=root, planned_target=candidate, expected_target=expected)
+    profiles.publication_authority(manifest, profile, root=root, planned_target=candidate, expected_target=expected)
     visual_binding = plan["target"]["bindings"]["visual_acceptance"]
     visual = contained_path(data_home(), visual_binding["path"], reject_symlinks=True)
     if json.loads(visual.read_bytes())["manifest_sha256"] != digest((report / "manifest.json").read_bytes()):
@@ -130,7 +138,7 @@ def verify_plan(report, work):
     if digest((work / "delivery/delivery-index.json").read_bytes()) != delivery_binding["sha256"]:
         raise ValueError("Publication delivery index changed")
 
-    verify_publication_inputs(manifest, plan["target"], root=root)
+    profiles.verify_publication_inputs(manifest, plan["target"], root=root)
     return plan
 
 
diff --git a/harness/web_search/relevance.py b/harness/web_search/relevance.py
index 686d9bc86..2ea0afc12 100644
--- a/harness/web_search/relevance.py
+++ b/harness/web_search/relevance.py
@@ -1,24 +1,27 @@
 """Blind, pooled top-k preview relevance scoring for Web Search results."""
 
-from lexbench_websearch.rubric import JUDGE_PROMPT, known_prompt
-
-
+import argparse
+import base64
 import hashlib
 import json
+import os
+import re
+import time
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from ipaddress import ip_address
 from pathlib import Path
+from urllib.parse import urlsplit
 
-from .messages import report_text
-
-from lexbench_websearch.scoring import canonical_url
+from dotenv import dotenv_values
 from lexbench_websearch.relevance import candidate_packet, score, validate_source_rule_regressions
+from lexbench_websearch.rubric import JUDGE_PROMPT, known_prompt
+from lexbench_websearch.scoring import canonical_url
 
-
-
-
-
-
-
-
+from harness.web_fetch import config as runtime_config
+from harness.web_fetch.config import load_runtime_env
+from harness.web_search.messages import report_text
 
 
 def verify_protocol(profile, evidence, revision=None, *, authority=None):
@@ -150,8 +153,6 @@ def verify_provenance(profile, rows, evidence, *, authority=None):
         raise ValueError("Cross-check grades are invalid")
 
 
-
-
 def import_annotations(profile, rows, submitted):
     """Validate complete frozen human input, then restore private rank mappings."""
     packet, rankings = candidate_packet(profile, rows)
@@ -180,8 +181,6 @@ def export_annotations(profile, rows, destination):
 ANNOTATION_HTML = report_text('text_38d7f4992b5eb237')
 
 
-
-
 def judge_input(packet):
     if not packet.get("source_checked"):
         return packet
@@ -287,8 +286,6 @@ def legacy_evidence_allowed(evidence, kind, registry=None):
 
 
 def validate_judge_url(value):
-    from ipaddress import ip_address
-    from urllib.parse import urlsplit
     base = urlsplit(value)
     hostname = base.hostname
     try:
@@ -405,7 +402,6 @@ def validate_model_evidence(evidence, *, _depth=0, authority=None):
                 raise ValueError("Model judge configuration differs across calls")
         if strict:
             _validate_judge_field_references(call["response"])
-            import base64
 
             attempts = call.get("attempts", [])
             if not attempts or len(attempts) > configuration["attempt_limit"]:
@@ -449,16 +445,7 @@ def validate_model_evidence(evidence, *, _depth=0, authority=None):
 
 def judge_main(argv=None):
     """Explicit model judging, resumable from immutable per-query receipts."""
-    import argparse
-    import base64
-    import os
-    import time
-    import urllib.error
-    import urllib.request
-    from concurrent.futures import ThreadPoolExecutor, as_completed
-    from pathlib import Path
 
-    from harness.web_fetch.config import load_runtime_env
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--report", type=Path, required=True)
@@ -484,7 +471,6 @@ def judge_main(argv=None):
         "--disable-thinking", action="store_true", help="Pass enable_thinking=false through Bifrost"
     )
     args = parser.parse_args(argv)
-    import re
     if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", args.token_env):
         raise ValueError("Judge credential variable must be a valid environment variable name")
     if args.token_env not in {"SEARCH_JUDGE_API_KEY", "BIFROST_API_KEY"}:
@@ -495,12 +481,10 @@ def judge_main(argv=None):
     token = os.environ.get(args.token_env, "")
     if not token.strip() or "\r" in token or "\n" in token:
         raise ValueError(f"Set a nonempty {args.token_env} in the repository .env")
-    from dotenv import dotenv_values
-    from harness.web_fetch import config as runtime_config
     configured_base = dotenv_values(runtime_config.ROOT / ".env").get("SEARCH_JUDGE_BASE_URL", "")
     if not configured_base or validate_judge_url(configured_base).netloc != validate_judge_url(args.base_url).netloc:
         raise ValueError("Judge endpoint host must match repository SEARCH_JUDGE_BASE_URL")
-    from .report import check
+    from harness.web_search.report import check
 
     profile, rows = check(args.report, return_data=True, evidence_only=True)
     source_requirements = (
diff --git a/harness/web_search/report.py b/harness/web_search/report.py
index 76e43fc9f..f774bd611 100644
--- a/harness/web_search/report.py
+++ b/harness/web_search/report.py
@@ -1,9 +1,5 @@
 """Small bilingual Web Search report built from frozen calls, never invented scores."""
 
-from .messages import report_text
-
-from lexbench_websearch.inputs import definition_path, implementation_sources
-
 import argparse
 import copy
 import html
@@ -17,19 +13,67 @@
 from urllib.parse import parse_qs, urlsplit
 
 from bs4 import BeautifulSoup
+from lexbench_websearch import answer_rules
+from lexbench_websearch.evidence_quality import CONTAMINATION_RULE, factual_sensitivity
+from lexbench_websearch.inputs import definition_path, implementation_sources
+from lexbench_websearch.relevance import score as score_relevance
+from lexbench_websearch.relevance import summary as relevance_summary
+from lexbench_websearch.scoring import (
+    canonical_url,
+    classify,
+    evaluate,
+    factual_evidence,
+    in_domain,
+)
 
 from harness.record_io import same_json_record
+from harness.report_languages import (
+    REPORT_PAGE_MARKER,
+    check_language_contract,
+    generated_languages,
+    resolve_report_language,
+    select_report_language,
+    write_language_contract,
+)
 from harness.reporting.presentation import check_presentation
-
-from .core import aggregate, digest, normalize_response, request_cost, request_body, validate_profile, verify_bundle, write_json
-from lexbench_websearch.scoring import canonical_url, classify, evaluate, in_domain
-from lexbench_websearch.evidence_quality import factual_sensitivity, CONTAMINATION_RULE
-from .diagnostics import error_evidence, operational
-from .runner import reference_available, reference_changes
-from .profiles import load_profile, _bound
-from harness.web_search.revisions import apply_identity_correction
-from .relevance import verify_provenance, verify_protocol
-from lexbench_websearch.relevance import score as score_relevance, summary as relevance_summary
+from harness.reporting.versions import FILENAME, bind_build_version, check_build_version
+from harness.web_fetch.publication_privacy import SENSITIVE_KEY, project_file
+from harness.web_search.comparison import render_comparison
+from harness.web_search.core import (
+    aggregate,
+    digest,
+    normalize_response,
+    request_body,
+    request_cost,
+    validate_profile,
+    verify_bundle,
+    write_json,
+)
+from harness.web_search.diagnostics import error_evidence, operational
+from harness.web_search.messages import report_text
+from harness.web_search.pages import reader_pages
+from harness.web_search.profiles import _bound, load_profile
+from harness.web_search.relevance import (
+    export_annotations,
+    import_annotations,
+    report_authority,
+    verify_protocol,
+    verify_provenance,
+)
+from harness.web_search.review import (
+    coverage_percent,
+    factual_quality_exclusions,
+    render_detail,
+    render_summary,
+    validate_public_review,
+)
+from harness.web_search.revisions import (
+    apply_identity_correction,
+    apply_report_revision,
+    collected_projection,
+    validate_revision_binding,
+)
+from harness.web_search.runner import reference_available, reference_changes
 
 STYLE = """
 .dataset-controls{display:grid;grid-template-columns:repeat(auto-fit,minmax(210px,1fr));gap:14px;align-items:end;margin:20px 0}.dataset-controls label{display:grid;gap:5px}.dataset-controls select,.dataset-controls input{width:100%;min-width:0}
@@ -136,7 +180,6 @@ def replace(match):
 
 def public_calls(profile, rows, source, sources):
     """Keep measured outcomes; redact URL credentials and audit billing corrections."""
-    from harness.web_fetch.publication_privacy import SENSITIVE_KEY, project_file
     rows = copy.deepcopy(rows)
     methods = {m["id"]: m for m in profile["methods"]}
     files, redactions, corrections = {}, [], []
@@ -380,7 +423,6 @@ def load_report_runs(source, cohorts, labels, method_ids=None):
 
 
 def _language_notice(output, languages):
-    from harness.report_languages import write_language_contract
     if languages == ["zh"] and (output / "en").exists():
         for directory in sorted((output / "en").rglob("*"), key=lambda p: len(p.parts), reverse=True):
             if directory.is_dir() and not any(directory.iterdir()):
@@ -390,7 +432,6 @@ def _language_notice(output, languages):
 
 
 def _check_report_languages(output, manifest):
-    from harness.report_languages import check_language_contract, generated_languages
     check_language_contract(output)
     if list(generated_languages(output)) != manifest.get("languages"):
         raise ValueError("Report language metadata differs from the report manifest")
@@ -398,7 +439,6 @@ def _check_report_languages(output, manifest):
 
 def assessment_profile(profile, relevance, review):
     """Describe current assessment without rewriting historical collection metadata."""
-    from .review import factual_quality_exclusions
     protocol = dict(profile["protocol"])
     exclusions = sorted(set(profile.get("factual_quality_exclusions", [])) | set(factual_quality_exclusions(review)))
     if relevance:
@@ -464,8 +504,6 @@ def build(
         }
     assessment_revision = None
     if authored.get("assessment_revision"):
-        from harness.web_search.revisions import apply_report_revision
-        from harness.web_search.revisions import validate_revision_binding
         validate_revision_binding(authored["assessment_revision"], normalizer=normalize_response, authority=authority)
         profile, rows, changes = apply_report_revision(profile, rows, authored["assessment_revision"], artifacts=public_files, normalizer=normalize_response)
         assessment_revision = {"contract": authored["assessment_revision"], "changes": changes, "implementation_sha256": {name: digest(path.read_bytes()) for name, path in implementation_sources().items()}}
@@ -482,13 +520,11 @@ def build(
     relevance_data = json.loads(relevance.read_text()) if relevance else None
     review_percent = None
     if review is not None:
-        from .review import coverage_percent, validate_public_review
         validate_public_review(profile, selected, review_data, authority=authority, relevance=relevance_data)
         review_percent = coverage_percent(review_data)
         write_json(output / "independent-review.json", review_data)
     if relevance_data:
         if "plan" not in relevance_data:
-            from .relevance import import_annotations
             relevance_data = import_annotations(profile, selected, relevance_data)
         verify_provenance(profile, selected, relevance_data, authority=authority)
         verify_protocol(profile, relevance_data, authored.get("relevance_revision"), authority=authority)
@@ -558,7 +594,6 @@ def build(
         def t(en, cn, zh=zh):
             return cn if zh else en
 
-        from .pages import reader_pages
         page_registry = reader_pages(authored.get("presentation"), reviewed=bool(review_data))
         nav = [(p["route"], p[lang]) for p in page_registry if p["nav"]]
         if reading_sections and authored.get("presentation") != "comparison-report":
@@ -656,7 +691,6 @@ def page(route, title, body, prefix=prefix, lang=lang, nav=nav, state=state, t=t
                         link['data-reviewed-outputs'] = f"{review_data['output_reviewed']}/{review_data['output_total']}"
                     footer.append(link)
                 document.main.append(footer)
-            from harness.report_languages import REPORT_PAGE_MARKER
             path.write_text(str(document).replace("", REPORT_PAGE_MARKER + ""))
             routes.append(prefix + route)
 
@@ -1045,7 +1079,6 @@ def page(route, title, body, prefix=prefix, lang=lang, nav=nav, state=state, t=t
             if query_relevance:
                 reason_label = t("Useful results: ", report_text('text_7a4d7a014722e76c')) + str(query_relevance["useful"]) + f"/{cutoff}"
             if review_data:
-                from .review import render_detail
                 detail += render_detail(review_data, r["id"], zh)
             diagnosis = reason_label
             detail += "
    " + diagnosis + "
    "
@@ -1065,7 +1098,6 @@ def page(route, title, body, prefix=prefix, lang=lang, nav=nav, state=state, t=t
             ]
             detail_metrics = r["metrics"]
             if "reference_answer" in c:
-                from lexbench_websearch.scoring import factual_evidence
                 detail_metrics = {**r["metrics"], **factual_evidence(c, r["results"], cutoff=cutoff)}
                 metric_labels += [
                     (f"labeled_source_hit_{cutoff}", t(f"Published source in top {cutoff}", f"{report_text('text_c5a8970d6255caf1')}{cutoff}{report_text('text_c8c82eb0cadfc2bb')}")),
@@ -1245,9 +1277,7 @@ def page(route, title, body, prefix=prefix, lang=lang, nav=nav, state=state, t=t
         else:
             page("performance.html", t("Latency diagnostics", report_text('text_51464dd5dcb1bce3')), authored_sections(authored, "performance", lang, rows=selected) + performance_view(profile, diagnostics, zh, evidence_prefix="../" if prefix else ""))
         if review_data:
-            from .review import render_summary
             page("review.html", t("Independent result review", report_text('text_f51205d6074e15bf')), render_summary(review_data, zh) + authored_sections(authored, "review", lang, rows=selected))
-    from harness.reporting.versions import bind_build_version, FILENAME
     build_version = bind_build_version(output)
     bundle.append({"path": FILENAME, "sha256": build_version["sha256"]})
     write_json(
@@ -1293,24 +1323,19 @@ def page(route, title, body, prefix=prefix, lang=lang, nav=nav, state=state, t=t
             renderer_sha256=digest(Path(__file__).read_bytes()),
         ),
     )
-    from harness.report_languages import select_report_language
     entry = select_report_language(output, language)
     check(output, evidence_only=True)
     return entry
 
 
 def check(output, *, return_data=False, evidence_only=False):
-    from .relevance import report_authority
     authority = report_authority(output)
     manifest = json.loads((output / "manifest.json").read_text())
     if not evidence_only:
-        from harness.reporting.versions import check_build_version
         check_build_version(output, manifest)
     if manifest.get("assessment_revision"):
-        from harness.web_search.revisions import validate_revision_binding
         validate_revision_binding(manifest["assessment_revision"]["contract"], normalizer=normalize_response, authority=authority)
     _check_report_languages(output, manifest)
-    from .pages import reader_pages
     top_routes = reader_pages((manifest.get("authored") or {}).get("presentation"), reviewed=bool(manifest.get("independent_review_sha256")))
     for lang in manifest["languages"]:
         prefix = "" if lang == "zh" else lang + "/"
@@ -1369,7 +1394,6 @@ def check(output, *, return_data=False, evidence_only=False):
                 raise ValueError("Result receipt disagrees with raw response")
             row["results"] = results
     if manifest.get("independent_review_sha256"):
-        from .review import coverage_percent, validate_public_review, render_detail, render_summary
         review_path = output / "independent-review.json"
         if digest(review_path.read_bytes()) != manifest["independent_review_sha256"]:
             raise ValueError("Independent review evidence hash mismatch")
@@ -1522,7 +1546,6 @@ def check(output, *, return_data=False, evidence_only=False):
         if changes != expected_changes:
             raise ValueError("Scoring revision provenance mismatch")
         if manifest.get("assessment_revision"):
-            from harness.web_search.revisions import apply_report_revision
             revised, _, _ = apply_report_revision(revised, [], manifest["assessment_revision"]["contract"], normalizer=normalize_response)
         revised, _ = project_comparison(revised, [], manifest.get("authored") or {})
         if manifest.get("report_assessment"):
@@ -1530,7 +1553,6 @@ def check(output, *, return_data=False, evidence_only=False):
         if revised != manifest["profile"]:
             raise ValueError("Scoring revision profile mismatch")
     elif manifest.get("assessment_revision"):
-        from harness.web_search.revisions import apply_report_revision, collected_projection
         revision = manifest["assessment_revision"]
         if revision["contract"].get("normalization"):
             artifacts = {r["evidence"]["path"]: (output / r["evidence"]["path"]).read_bytes() for r in manifest["rows"]}
@@ -1808,7 +1830,6 @@ def main(argv=None):
     if args.export_annotations:
         if args.source or args.cohort:
             parser.error("Annotation export consumes an existing report")
-        from .relevance import export_annotations
         profile, selected = check(args.output, return_data=True)
         print(export_annotations(profile, selected, args.export_annotations))
         return
@@ -1819,7 +1840,7 @@ def main(argv=None):
             release_check(args.output, args.release_check, publication_plan=args.publication_plan)
         else:
             check(args.output)
-        from .delivery import export, verify
+        from harness.web_search.delivery import export, verify
         export(args.output, args.export_delivery)
         verify(args.output, args.export_delivery)
         print(args.export_delivery)
@@ -1832,7 +1853,6 @@ def main(argv=None):
         check(args.output)
         print("Web Search report checks passed")
         return
-    from harness.report_languages import resolve_report_language
     language = resolve_report_language(args.output, args.language)
     print(
         build(
@@ -1865,11 +1885,11 @@ def release_check(output, visual_receipt, *, publication_plan=None):
         if not report_manifest.get("receipt", {}).get("certified"):
             raise ValueError("A formal release requires certified collection evidence")
         if publication_plan is not None:
-            from .publication import verify_plan
+            from harness.web_search.publication import verify_plan
             verify_plan(output, publication_plan.parent)
         else:
             raise ValueError("A measured release requires an explicit publication plan")
-    from .visual import verify
+    from harness.web_search.visual import verify
     return verify(output, visual_receipt)
 
 
@@ -2198,7 +2218,6 @@ def t(en, cn):
 
 def latency_rounds(profile, rows, zh):
     """Expose repeated-query and reported cache state without inferring cold starts."""
-    from collections import Counter
 
     def t(en, cn):
         return cn if zh else en
@@ -2686,7 +2705,6 @@ def relevance_panel(profile, scored, evidence, zh):
 
 
 def factual_panel(profile, selected, zh, *, compact=False):
-    from lexbench_websearch.scoring import factual_evidence
     cutoff = profile["protocol"]["top_k"]
     excluded = set(profile.get('factual_quality_exclusions', []))
     cases = {c['id']: c for c in profile['cases'] if 'reference_answer' in c and c['id'] not in excluded}
@@ -2764,8 +2782,7 @@ def factual_panel(profile, selected, zh, *, compact=False):
             'This includes returned summaries and bodies after the same explicit-mirror exclusion. Added means absent when disabled and present when enabled; lost is the reverse. Search results can change, so this is the overall option effect rather than a fixed-page body-only gain.'
         ) + '
    ' + table([report_text('text_e5d4c3fe74891d30'), report_text('text_4e114db7ef22ed94'), report_text('text_741c5d468d053162'), report_text('text_f24f7c9c6114f123'), report_text('text_0006d696d8e1ec28'), report_text('text_b3206f118ce9374b')] if zh else
             ['Enabled mode', 'Pairs', 'Disabled hits', 'Enabled hits', 'Added', 'Lost'], extra)
-    from lexbench_websearch.answer_rules import POLICY_NAME
-    if any(c.get('answer_match_rule') == POLICY_NAME for c in cases.values()):
+    if any(c.get('answer_match_rule') == answer_rules.POLICY_NAME for c in cases.values()):
         return body + '
    ' + (
             report_text('text_0f0231d2c5b6ab86')
             if zh else
@@ -3258,7 +3275,6 @@ def reference_retrieval_panel(profile, rows, case_ids, zh):
 
 
 def population_summary(profile, selected, content, zh, *, diagnostics=False, relevance_scores=None):
-    from lexbench_websearch.scoring import factual_evidence
     cutoff = profile["protocol"]["top_k"]
     lang = "zh" if zh else "en"
     independent = content.get("independent_benchmarks")
@@ -3382,7 +3398,6 @@ def population_summary(profile, selected, content, zh, *, diagnostics=False, rel
 
 
 def render_result_comparison(profile, selected, zh, *, prefix="", site_prefix="", content=None, relevance=None):
-    from .comparison import render_comparison
     profile = dict(profile, comparison_populations=metric_populations(profile, content or {}))
     return render_comparison(profile, selected, zh, relevance=relevance, primary_factual_metric="source" if (content or {}).get("summary_metrics") == "retrieval" else "answer", detail_link=lambda row: posixpath.join(site_prefix, prefix, "details", row["id"] + ".html"))
 
diff --git a/harness/web_search/review.py b/harness/web_search/review.py
index 06c5db85f..dd7568773 100644
--- a/harness/web_search/review.py
+++ b/harness/web_search/review.py
@@ -2,26 +2,33 @@
 
 from __future__ import annotations
 
-from .messages import report_text
-
 import argparse
 import hashlib
 import json
 import os
 import re
-import signal
 import shutil
+import signal
+import subprocess
 import tempfile
 import threading
-import subprocess
 import time
+from collections import Counter
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from html import escape
 from pathlib import Path
 from threading import Event
+from urllib.parse import urlsplit
+
+from lexbench_webfetch.paths import contained_path
+from lexbench_websearch.relevance import candidate_packet
+from lexbench_websearch.scoring import candidate_identity, classify
 
-from .core import normalize_response
-from .diagnostics import error_evidence
 from harness.reporting.publication import atomic_write
+from harness.web_search.core import normalize_response
+from harness.web_search.diagnostics import error_evidence
+from harness.web_search.messages import report_text
+from harness.web_search.relevance import legacy_evidence_allowed
 
 
 def sha(data):
@@ -120,7 +127,6 @@ def _input_text(contract, packet, packet_hash):
     )
 
 
-
 def _object(properties):
     return {
         "type": "object",
@@ -189,7 +195,6 @@ def _array(item):
 
 def make_cases(profile, rows, report):
     """Return full, blinded cases and a private mapping to original row IDs."""
-    from lexbench_websearch.relevance import candidate_packet
 
     methods = {method["id"]: method for method in profile["methods"]}
     hydrated, failures = [], {}
@@ -303,7 +308,6 @@ def report_binding(profile, rows):
 
 def row_state(profile, row, relevance=None):
     """Use the report's relevance labels and quarantine rules for review selection."""
-    from lexbench_websearch.scoring import classify, candidate_identity
     case = next(c for c in profile["cases"] if c["id"] == row["case_id"])
     labels = (relevance or {}).get("labels", {}).get(case["id"])
     grades = None if labels is None else [
@@ -331,7 +335,7 @@ def freeze(report, destination, **options):
 
 
 def _freeze(report, destination, *, model=None, budget=180000, scope="failures", authorization=None, source_mode="frozen"):
-    from .report import check
+    from harness.web_search.report import check
 
     if source_mode not in REVIEW_PROMPTS:
         raise ValueError("Review source mode must be frozen or live")
@@ -431,7 +435,6 @@ def _freeze(report, destination, *, model=None, budget=180000, scope="failures",
 
 
 def frozen_path(root, relative):
-    from lexbench_webfetch.paths import contained_path
     path = Path(relative)
     if path.is_absolute():
         try:
@@ -709,8 +712,6 @@ def _successful(root, entry, contract):
         return None
 
 
-
-
 def run(root, *, workers=4, limit=None, executable="codex", timeout=3600):
     contract = load_frozen(root)
     if contract["model"] == "codex-app-inherited":
@@ -995,7 +996,6 @@ def validate_public_review(profile, rows, evidence, *, authority=None, relevance
         raise ValueError("Failure-only review cannot change the quality population")
     if profile.get("independent_result_review_scope") == "all" and scope != "all":
         raise ValueError("This report explicitly requires a complete result review")
-    from .relevance import legacy_evidence_allowed
     proof = evidence.get("provenance")
     if not proof and not legacy_evidence_allowed(evidence, "review", authority):
         raise ValueError("Independent review requires frozen contract, packets and execution receipts")
@@ -1162,8 +1162,6 @@ def validate_public_review(profile, rows, evidence, *, authority=None, relevance
 
 def candidate_packet_for_review(profile, rows):
     # Candidate coverage is a URL pool; no original grades enter this check.
-    from lexbench_websearch.relevance import candidate_packet
-
     packet, rankings = candidate_packet(profile, rows)
     custom = {p["case_id"]: p["candidates"] for p in packet["blind_candidate_packets"]}
     return (
@@ -1184,8 +1182,6 @@ def coverage_percent(evidence):
 
 
 def render_summary(evidence, zh=True):
-    from collections import Counter
-
     counts = Counter(d["resolution"] for d in evidence["adjudications"].values())
     title = (report_text('text_521deef44b4ce505') if zh else "Result audit") + f" · {coverage_percent(evidence)}%"
     text = (
@@ -1221,7 +1217,6 @@ def render_summary(evidence, zh=True):
         + "
    "
     )
     body += '
    ' + (report_text('review_unreviewed') if zh else 'Accepted passing outputs without review: ') + str(evidence.get('accepted_unreviewed_passes', 0)) + '
    '
-    from html import escape
     reference_cases = [c for c in evidence["cases"] if c["dataset_review"]["status"] == "issue"]
     if reference_cases:
         body += '
    ' + (report_text('text_84a8421bb5133e4f') if zh else "Per-query reference decisions") + '
    '
@@ -1245,8 +1240,6 @@ def render_summary(evidence, zh=True):
 
 
 def render_detail(evidence, row_id, zh=True):
-    from html import escape
-
     for case in evidence["cases"]:
         opinion = next((o for o in case["output_reviews"] if o["row_id"] == row_id), None)
         if opinion is None:
@@ -1264,7 +1257,6 @@ def render_detail(evidence, row_id, zh=True):
             ) + " " + escape(decision["reason"]) + '
    '
         body += '
    ' + (report_text('text_73abe0f2d9af6695') if zh else 'Original review opinion, evidence and decisions') + '
    ' + escape(opinion['reason']) + '
    ' + escape(opinion['evidence']) + '
    '
         if case["dataset_review"]["status"] == "issue":
-            from urllib.parse import urlsplit
 
             body += (
                 "
    "
diff --git a/harness/web_search/revisions.py b/harness/web_search/revisions.py
index c6b6e1b81..83f911432 100644
--- a/harness/web_search/revisions.py
+++ b/harness/web_search/revisions.py
@@ -1,12 +1,14 @@
 """Re-evaluate saved evidence with explicit external correction authority."""
-from pathlib import Path
-from copy import deepcopy
+
 import json
-from lexbench_websearch.scoring import evaluate
+from copy import deepcopy
+from pathlib import Path
+
+from lexbench_websearch.answer_rules import POLICY_NAME, RULE_VERSION, reference_rule
 from lexbench_websearch.contracts import validate_cases
 from lexbench_websearch.inputs import digest
-from lexbench_websearch.answer_rules import POLICY_NAME
-from lexbench_websearch.qualification import validate_regressions, _qualified, _equivalent
+from lexbench_websearch.qualification import _equivalent, _qualified, validate_regressions
+from lexbench_websearch.scoring import evaluate
 
 
 def _rescore(profile, rows):
@@ -109,13 +111,11 @@ def apply_report_revision(profile, rows, revision, *, artifacts=None, normalizer
         for case in cases.values():
             if case.get("reference_answer"):
                 case["answer_match_rule"] = revision["answer_match_rule"]
-    from lexbench_websearch.answer_rules import RULE_VERSION
 
     if revision.get("answer_annotation_rule") not in (None, RULE_VERSION):
         raise ValueError("Answer derivation revision uses an unsupported annotation rule")
     derivations = revision.get("answer_derivations", [])
     if derivations:
-        from lexbench_websearch.answer_rules import RULE_VERSION, reference_rule
 
         if (revision.get("answer_annotation_rule") != RULE_VERSION
                 or not {"published-answer-citation-metadata", "literal-bracketed-answer"}.issubset(revision["regression_ids"])):
diff --git a/harness/web_search/runner.py b/harness/web_search/runner.py
index a984e6968..9ec89bf69 100644
--- a/harness/web_search/runner.py
+++ b/harness/web_search/runner.py
@@ -10,12 +10,23 @@
 from datetime import UTC, datetime
 from pathlib import Path
 
-from harness.web_fetch.config import ROOT, load_runtime_env
-from harness.record_io import read_completed_records
-
-from .core import digest, normalize_response, request_body, request_cost, validate_profile, verify_bundle, write_json
+from lexbench_websearch import inputs as benchmark_inputs
 from lexbench_websearch.scoring import evaluate
 
+from harness.record_io import read_completed_records
+from harness.web_fetch.config import ROOT, load_runtime_env
+from harness.web_search.adapters import adapter_for
+from harness.web_search.core import (
+    digest,
+    normalize_response,
+    request_body,
+    request_cost,
+    validate_profile,
+    verify_bundle,
+    write_json,
+)
+from harness.web_search.profiles import load_profile_bytes
+
 
 def call_budgets(profile):
     declared = profile["protocol"].get("max_calls_per_method")
@@ -183,10 +194,6 @@ def prepare_execution_revision(profile_path, output, revision_path, *, cohort="p
     return retained
 
 
-from .adapters import adapter_for
-from .profiles import load_profile_bytes
-
-
 def http_call(url, body=None, header=None, timeout=20):
     """Hard total timeout; credentials enter curl on stdin and never appear in argv."""
     with tempfile.TemporaryDirectory(prefix="search-http-") as temp:
@@ -305,15 +312,13 @@ def collection_implementation():
         "harness/web_search/adapters.py", "harness/web_search/profiles.py",
         "harness/web_fetch/config.py", "harness/record_io.py", "uv.lock", "pyproject.toml",
     )
-    from lexbench_websearch.inputs import implementation_sources
     sources = {name: ROOT / name for name in names}
-    sources.update({"dependencies/lexbench_websearch/" + name: path for name, path in implementation_sources().items()})
+    sources.update({"dependencies/lexbench_websearch/" + name: path for name, path in benchmark_inputs.implementation_sources().items()})
     return {name: digest(source.read_bytes()) for name, source in sources.items()}
 
 
 def implementation_source(relative):
-    from lexbench_websearch.inputs import implementation_sources
-    return implementation_sources()[relative.removeprefix("dependencies/lexbench_websearch/")] if relative.startswith("dependencies/") else ROOT / relative
+    return benchmark_inputs.implementation_sources()[relative.removeprefix("dependencies/lexbench_websearch/")] if relative.startswith("dependencies/") else ROOT / relative
 
 
 def validate_resume_implementation(output, contract):
diff --git a/harness/web_search/visual.py b/harness/web_search/visual.py
index a647281c1..b3bdb1bf5 100644
--- a/harness/web_search/visual.py
+++ b/harness/web_search/visual.py
@@ -5,11 +5,22 @@
 import json
 from pathlib import Path
 
+from lexbench_webfetch.paths import contained_path
+
+from harness.reporting import visual as shared_visual
 from harness.reporting.visual import (
-    browser_executable, file_sha256, layout_passes, local_page,
-    local_report_server, prepare_check, settle_theme, validate_routes, write_receipt,
+    browser_executable,
+    file_sha256,
+    layout_passes,
+    local_page,
+    local_report_server,
+    prepare_check,
+    settle_theme,
+    validate_routes,
+    write_receipt,
 )
-from lexbench_webfetch.paths import contained_path
+from harness.web_search import pages
+from harness.web_search.pages import reader_pages
 
 SCHEMA = 'web-search-reader-capture/2'
 WIDTH, HEIGHT = 1280, 900
@@ -17,8 +28,6 @@
 
 def producer_sources() -> dict[str, str]:
     """Qualify same-named modules so neither producer binding is overwritten."""
-    from harness.reporting import visual as shared_visual
-    from . import pages
 
     return {
         'harness/web_search/visual.py': file_sha256(Path(__file__)),
@@ -29,7 +38,6 @@ def producer_sources() -> dict[str, str]:
 
 def routes(report: Path) -> list[tuple[str, str, str]]:
     manifest = json.loads((report / 'manifest.json').read_text())
-    from .pages import reader_pages
     chapters = [(p["key"], p["route"]) for p in reader_pages((manifest.get("authored") or {}).get("presentation"), reviewed=bool(manifest.get("independent_review_sha256")))]
     languages = manifest.get('languages') or [manifest['primary_language']]
     result = [(lang, name, (lang + '/' if lang != 'zh' else '') + route)
diff --git a/pyproject.toml b/pyproject.toml
index cf55b40d0..412dc586f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,7 +83,10 @@ include = [
 target-version = "py311"
 
 [tool.ruff.lint]
-select = ["B", "E", "F", "I", "UP"]
+select = ["B", "E", "F", "I", "TID252", "UP"]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
 
 [tool.ruff.lint.per-file-ignores]
 "harness/trajectory_replay/report.py" = ["E501"]
diff --git a/scripts/apply_webmainbench_case_correction.py b/scripts/apply_webmainbench_case_correction.py
index dfe21fca3..9e9afc047 100644
--- a/scripts/apply_webmainbench_case_correction.py
+++ b/scripts/apply_webmainbench_case_correction.py
@@ -1,8 +1,9 @@
 """Command-line entry point; implementation lives in harness.web_fetch.webmainbench_case_correction."""
-from pathlib import Path
 import sys
+from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from harness.web_fetch.webmainbench_case_correction import main
 
 if __name__ == "__main__":
diff --git a/scripts/audit_initial_references.py b/scripts/audit_initial_references.py
index d403b0293..f7fbf7ee2 100644
--- a/scripts/audit_initial_references.py
+++ b/scripts/audit_initial_references.py
@@ -16,10 +16,11 @@
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
 
-from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes
-from lexbench_webfetch.reference_quality import clean_reference_text
 from lexbench_webfetch.paths import contained_path, safe_identifier
-from lexbench_webfetch.scoring import score_attempt, SCORING_SOURCE_FILES
+from lexbench_webfetch.reference_quality import clean_reference_text
+from lexbench_webfetch.scoring import SCORING_SOURCE_FILES, score_attempt
+
+from harness.web_fetch.reference_inputs import input_tree, read_dataset_bytes
 from harness.web_fetch.source_capture import extract_html
 
 
diff --git a/scripts/build_native_dns.py b/scripts/build_native_dns.py
index 02b6e55e8..5dd8d39ef 100644
--- a/scripts/build_native_dns.py
+++ b/scripts/build_native_dns.py
@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 """Prepare the process-local native DNS bridge outside Git."""
 import argparse
-from pathlib import Path
 import sys
+from pathlib import Path
+
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from harness.web_fetch.network_dns import build_native_dns_library
 
 if __name__ == "__main__":
diff --git a/scripts/build_report.py b/scripts/build_report.py
index a5fe8d045..7bf36c003 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -1,10 +1,25 @@
 """Build the current measured report or its public-benchmark research companion."""
-import argparse,json,sys
+import argparse
+import json
+import sys
 from pathlib import Path
+
 ROOT=Path(__file__).resolve().parents[1]
 sys.path.insert(0,str(ROOT))
-from harness.report_languages import add_language_argument, resolve_report_language, SUPPORTED_REPORT_LANGUAGES
-from harness.web_fetch.report_build import build_report, build_public_benchmark_research, build_wcxb_report, build_webmainbench_full_report, build_diagnostic_cohorts, refresh_local_presentation
+
+from harness.report_languages import (
+    SUPPORTED_REPORT_LANGUAGES,
+    add_language_argument,
+    resolve_report_language,
+)
+from harness.web_fetch.report_build import (
+    build_diagnostic_cohorts,
+    build_public_benchmark_research,
+    build_report,
+    build_wcxb_report,
+    build_webmainbench_full_report,
+    refresh_local_presentation,
+)
 from harness.web_fetch.storage import (
     current_evidence_dir,
     require_canonical_inputs,
diff --git a/scripts/check_diagnostic_report.py b/scripts/check_diagnostic_report.py
index e9a3ba626..2f5fcdcba 100644
--- a/scripts/check_diagnostic_report.py
+++ b/scripts/check_diagnostic_report.py
@@ -3,9 +3,11 @@
 import json
 import sys
 from pathlib import Path
+
 sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
-from harness.web_fetch.report_diagnostic import check_diagnostic_report
+
 from harness.web_fetch.publication import verify_published_tree
+from harness.web_fetch.report_diagnostic import check_diagnostic_report
 
 if __name__ == '__main__':
     parser=argparse.ArgumentParser(description=__doc__)
diff --git a/scripts/check_track_naming.py b/scripts/check_track_naming.py
index 5a84cfef4..79d646952 100644
--- a/scripts/check_track_naming.py
+++ b/scripts/check_track_naming.py
@@ -6,6 +6,8 @@
 import re
 from pathlib import Path
 
+from harness.web_fetch.reference_inputs import check_reference_input_boundary
+
 ROOT = Path(__file__).resolve().parents[1]
 EXCLUDED_ROOTS = {
     ".git",
@@ -131,7 +133,6 @@ def repository_contract_findings(root: Path) -> list[str]:
 
 
 def main() -> None:
-    from harness.web_fetch.reference_inputs import check_reference_input_boundary
     check_reference_input_boundary(ROOT / 'evaluation')
     findings = repository_contract_findings(ROOT)
     if findings:
diff --git a/scripts/finalize_reference_checks.py b/scripts/finalize_reference_checks.py
index 2d03850ca..9edd6a575 100644
--- a/scripts/finalize_reference_checks.py
+++ b/scripts/finalize_reference_checks.py
@@ -9,8 +9,8 @@
 
 import argparse
 import json
-from pathlib import Path
 import sys
+from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
diff --git a/scripts/run_certified_webmainbench_full.py b/scripts/run_certified_webmainbench_full.py
index a05433f86..17cac895c 100644
--- a/scripts/run_certified_webmainbench_full.py
+++ b/scripts/run_certified_webmainbench_full.py
@@ -1,8 +1,9 @@
 """Command-line entry point; implementation lives in harness.web_fetch.webmainbench_certification."""
-from pathlib import Path
 import sys
+from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from harness.web_fetch.webmainbench_certification import main
 
 if __name__ == "__main__":
diff --git a/scripts/run_webmainbench_benchmark.py b/scripts/run_webmainbench_benchmark.py
index cc0d3dd15..1d70547a2 100644
--- a/scripts/run_webmainbench_benchmark.py
+++ b/scripts/run_webmainbench_benchmark.py
@@ -1,8 +1,9 @@
 """Command-line entry point; implementation lives in harness.web_fetch.webmainbench_runner."""
-from pathlib import Path
 import sys
+from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from harness.web_fetch.webmainbench_runner import main
 
 if __name__ == "__main__":
diff --git a/scripts/score_webmainbench_results.py b/scripts/score_webmainbench_results.py
index 36bb8d139..31df45e2f 100644
--- a/scripts/score_webmainbench_results.py
+++ b/scripts/score_webmainbench_results.py
@@ -1,8 +1,9 @@
 """Command-line entry point; implementation lives in harness.web_fetch.webmainbench_scoring."""
-from pathlib import Path
 import sys
+from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from harness.web_fetch.webmainbench_scoring import main
 
 if __name__ == "__main__":
diff --git a/scripts/select_wcxb_sample.py b/scripts/select_wcxb_sample.py
index be8b128d3..56843e33e 100644
--- a/scripts/select_wcxb_sample.py
+++ b/scripts/select_wcxb_sample.py
@@ -1,8 +1,9 @@
 """Command-line entry point; implementation lives in harness.web_fetch.wcxb_dataset."""
-from pathlib import Path
 import sys
+from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from harness.web_fetch.wcxb_dataset import main
 
 if __name__ == "__main__":
diff --git a/tests/fixtures/native-route-baseline.json b/tests/fixtures/native-route-baseline.json
index a85b41f96..d790c77af 100644
--- a/tests/fixtures/native-route-baseline.json
+++ b/tests/fixtures/native-route-baseline.json
@@ -1,74 +1,74 @@
 {
-  "cloudflare_content:configured": {
-    "cells": 192,
-    "sha256": "c501d817b59dbefc362964386736f67ae4c46a7f2981aebf84dae53f10ae677b"
-  },
-  "cloudflare_content:defaults": {
+  "moli_webfetch:defaults": {
     "cells": 192,
-    "sha256": "287739522a6a6c57e529380e909472a5d515a846a2979bd659a67222b0f9dfd6"
+    "sha256": "6fec640b5795d2ca3feb16f7e37133a4f68268cae0d781a12e7b14fcb31d3161"
   },
-  "cloudflare_markdown:configured": {
+  "moli_webfetch:configured": {
     "cells": 192,
-    "sha256": "4905efb447fcdb4aec57bc73dd26880bf1c0e9084a71fca5374e442416db60ae"
+    "sha256": "dbb8c986b11f620db162a867b52b592cf546efc06190ebe31e6ece5416971be7"
   },
-  "cloudflare_markdown:defaults": {
+  "firecrawl:defaults": {
     "cells": 192,
-    "sha256": "2e9171663d9791377f107794667707836d65572ed9e06077d275b0e0a00c791f"
+    "sha256": "bac3e056691510cb5f88df724fd47c5a1ea781218fd011630a886eec269254ab"
   },
-  "cowagent_builtin:configured": {
+  "firecrawl:configured": {
     "cells": 192,
-    "sha256": "709a83a64fc0b36e9ae446f3c430c42223627d7332c6853e4d30091a9ed0c992"
+    "sha256": "006bffafd9c368cd0bd9596b7b03612939ff67aa73dff08c53f57527e66e3b57"
   },
-  "cowagent_builtin:defaults": {
+  "tavily_extract:defaults": {
     "cells": 192,
-    "sha256": "f952cc5a5f0e59975b18403da67bbbc138d886febdabd817ccd2682a4fdef39b"
+    "sha256": "408ccbcc11fbad17bdbbf3b334d297f12965b9d1cb16a58a2ceb9b527a14be7d"
   },
-  "exa_contents:configured": {
+  "tavily_extract:configured": {
     "cells": 192,
-    "sha256": "671cdcdd4902ec034caee226164861e3503bac84c8ce9832ec8413896f0b16f8"
+    "sha256": "873427fd3e108f7373c2899c4dd248c50fa656596bb971845b073f0a8bd27643"
   },
   "exa_contents:defaults": {
     "cells": 192,
-    "sha256": "d65bc8289d165286e6341568e7009233549cfb641bf6b6ad293a410675fb8c20"
+    "sha256": "31a93b9ba5f115da7baed76635b0640aa0fff9280afa67cd997d445069536f01"
   },
-  "firecrawl:configured": {
+  "exa_contents:configured": {
     "cells": 192,
-    "sha256": "7b3a41efd52dbe26075e1b7fb4525adba0ae2edf1bebb5c84ba6391e0392e079"
+    "sha256": "a78b34793b3093bb5df6f40413c1e93fc84cdcfc28e6fd28bfcc54461cb029f8"
   },
-  "firecrawl:defaults": {
+  "cloudflare_markdown:defaults": {
     "cells": 192,
-    "sha256": "b8d94e892531dc7f5d3ea8a805a96bf556de609e02fb4d50abe8877723186330"
+    "sha256": "b8e77a3eee45420f81af15ef0cdc42147b18b66fded2adb491c04b69e97058c6"
   },
-  "fixture:configured": {
+  "cloudflare_markdown:configured": {
     "cells": 192,
-    "sha256": "c95ef4b78f9df35c12713d7dac41f1c3abc675925653f43a4a22ee09152489e2"
+    "sha256": "39aa7bcf98a1470c5d70c8e37ad34182fb57496f2d91f0083d0954e4c4ad46b2"
   },
-  "fixture:defaults": {
+  "cloudflare_content:defaults": {
     "cells": 192,
-    "sha256": "0ec20a64f8c447a737c303c02eafdb218a94e3f4930680cc34ab9fa33d8e744b"
+    "sha256": "9c7e378c1e711e20484c3202b776c72e562e59c4116a199be257a34d2932be50"
   },
-  "moli_webfetch:configured": {
+  "cloudflare_content:configured": {
     "cells": 192,
-    "sha256": "940e90b1de64c65270d1db7726eec53680c68ca03dc5e43193b5eff4628c6338"
+    "sha256": "c506bcca75ac9aa41e2a9031c131a90a54069b38f2c9b1f9ede037fbc0b8b2f6"
   },
-  "moli_webfetch:defaults": {
+  "subprocess_jsonl:defaults": {
     "cells": 192,
-    "sha256": "551cfed573d065b40f58f53e20e78e5c6a1d7b77fd3808c67e6400c785fe892a"
+    "sha256": "86da2f053e55c6b4340e086b6cc4e41f84b11012deeaf63347ccaac4116d38d3"
   },
   "subprocess_jsonl:configured": {
     "cells": 192,
-    "sha256": "7d03a82682e0ac27139633c071fe119841baebf889f224fc5eea7c745088fcd1"
+    "sha256": "0b4b91b0fa3eb0182dc96fdb7b9a18645b721a061b589e1f688f2d7e5f048282"
   },
-  "subprocess_jsonl:defaults": {
+  "cowagent_builtin:defaults": {
     "cells": 192,
-    "sha256": "c18e2cacbb00b1759156548a62b499fcd9c0dbcffd531fb3f454ef0a1f82900c"
+    "sha256": "6f10daaff92e0fb4961c9af4be060f4ed2468d9fb8d105dc107cb6201e883211"
   },
-  "tavily_extract:configured": {
+  "cowagent_builtin:configured": {
     "cells": 192,
-    "sha256": "aecbb4eacbdefc6a9726e97d30b263d4f94c39012d9829afbab8d936e0c22287"
+    "sha256": "218d41a478b61c9e0a113feea5906eb046a35dde14d10d51d7501f13694f22b4"
   },
-  "tavily_extract:defaults": {
+  "fixture:defaults": {
+    "cells": 192,
+    "sha256": "323c3be505e72c0fb53d8fa100a550518addb09f7d9ed5c7813e1d2701f0a70d"
+  },
+  "fixture:configured": {
     "cells": 192,
-    "sha256": "84a39ad2141dca513b6d4c0e89293d8993facd3f47b449c8094321d171262930"
+    "sha256": "14644ca74d3ad320df58973c8334fc5adb35cba011fab8f42b91b216cec9465b"
   }
 }

From c5ee0a8814de7e84e328f6167e04f273568939e2 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Mon, 26 Oct 2026 20:57:10 +0800
Subject: [PATCH 19/19] fix(webfetch): preserve legacy rule authority and
 standalone worker startup

---
 harness/web_fetch/reference_lifecycle.py     | 15 +++++++--
 harness/web_fetch/workers/cowagent_worker.py |  6 ++--
 tests/test_cowagent_isolation.py             | 19 ++++++++++-
 tests/test_reference_lifecycle.py            | 35 +++++++++++++++++++-
 4 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py
index 462667f0b..00e4dc003 100644
--- a/harness/web_fetch/reference_lifecycle.py
+++ b/harness/web_fetch/reference_lifecycle.py
@@ -759,13 +759,22 @@ def check_case_reference_binding(case, binding: dict, *, original_case=None) ->
 
 def check_case_equivalent_reference_binding(case, binding: dict, *, original_case=None) -> None:
     """Allow distinct independent captures only when reference bytes and scope are identical."""
-    if binding.get('scope_identity', {}).get('schema_version', 1) == 1:
+    version = binding.get('scope_identity', {}).get('schema_version', 1)
+    if version not in (1, 2):
+        raise ReferenceLifecycleError('Unsupported reference scope version', 'reference_changed')
+    if version == 1:
         if original_case is None:
             raise ReferenceLifecycleError('Legacy rules require the hash-bound original dataset', 'reference_changed')
         binding = deepcopy(binding)
+        contract = binding['scope_identity']['evaluation_contract']
         for key in ('acceptance_contract', 'body_requirements', 'composite_acceptance'):
-            if _get(original_case, key, {}):
-                binding['scope_identity']['evaluation_contract'].setdefault(key, _get(original_case, key))
+            original = _get(original_case, key, {})
+            if key in contract and contract[key] != original:
+                raise ReferenceLifecycleError('Legacy rules differ from the hash-bound original dataset', 'reference_changed')
+            if original:
+                contract[key] = original
+            else:
+                contract.pop(key, None)
     reference = _get(case, "ground_truth_main_content", "")
     verification = _get(case, "source_verification", {})
     if (sha256(reference) != binding.get("reference_sha256")
diff --git a/harness/web_fetch/workers/cowagent_worker.py b/harness/web_fetch/workers/cowagent_worker.py
index bc8638b91..c483fabe8 100644
--- a/harness/web_fetch/workers/cowagent_worker.py
+++ b/harness/web_fetch/workers/cowagent_worker.py
@@ -7,6 +7,9 @@
 from contextlib import redirect_stdout
 from pathlib import Path
 
+# Direct execution starts in the upstream checkout, without an installed harness.
+sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+
 from harness.web_fetch.diagnostics import diagnostic_context, redact
 from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode
 
@@ -17,8 +20,7 @@ def main():
     workspace.mkdir(parents=True, exist_ok=True)
     # Only this process imports upstream code. Its prints cannot impersonate a
     # JSONL response, including prints that happen during module initialization.
-    sys.path.insert(0, str(root))
-    sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+    sys.path.insert(1, str(root))
 
     class NativeLogSink:
         def emit(self, source, event, **fields):
diff --git a/tests/test_cowagent_isolation.py b/tests/test_cowagent_isolation.py
index a49f368ac..1284df39b 100644
--- a/tests/test_cowagent_isolation.py
+++ b/tests/test_cowagent_isolation.py
@@ -1,11 +1,13 @@
 """Different upstream checkouts must execute their own code and respect deadlines."""
+import json
+import subprocess
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor
 
 import pytest
-
 from lexbench_webfetch.models import EvalCase
+
 from harness.web_fetch.providers.cowagent import CowAgentBuiltinProvider
 
 
@@ -84,3 +86,18 @@ def test_import_failure_has_a_structured_error(providers):
     assert not answer.ok
     assert answer.error_type == 'ImportError'
     assert answer.error_message == 'missing synthetic dependency'
+
+
+def test_worker_runs_from_source_without_an_installed_harness(providers):
+    provider = providers[0]
+    executable, *arguments = provider.config['command']
+    result = subprocess.run(
+        [executable, '-I', '-S', *arguments],
+        cwd=provider.config['repo_path'],
+        input=json.dumps({'url': case().url, 'timeout_seconds': 5}) + '\n',
+        text=True, capture_output=True, timeout=10,
+    )
+    assert result.returncode == 0, result.stderr
+    response = json.loads(result.stdout)
+    assert response['ok'] and response['content'] == 'first'
+    assert 'spoofed protocol' in result.stderr
diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py
index 5fc083fcd..e0a9efaac 100644
--- a/tests/test_reference_lifecycle.py
+++ b/tests/test_reference_lifecycle.py
@@ -1,5 +1,4 @@
 """Synthetic clocks and captures only: no network, browsers or real providers."""
-import pytest
 import json
 import tempfile
 import unittest
@@ -9,7 +8,9 @@
 from pathlib import Path
 from unittest.mock import Mock, patch
 
+import pytest
 from lexbench_webfetch.models import EvalCase
+
 from harness.web_fetch.models import ProviderResult
 from harness.web_fetch.reference_lifecycle import (
     ReferenceLifecycleError,
@@ -20,6 +21,7 @@
     preflight_reference_checks,
     require_verified_reference_checks,
     resource_source_scope_identity,
+    scope_identity,
     sha256,
     validate_resource_source_evidence,
 )
@@ -913,3 +915,34 @@ def test_original_scope_certificate_survives_unchanged_inputs_but_rejects_rule_c
         with pytest.raises(ReferenceLifecycleError):
             require_verified_reference_checks([manifest], cases=[stripped], source=packaged,
                                               equivalent_capture_bindings=equivalent)
+
+
+@pytest.mark.parametrize('fixture_name', ['legacy-live-reference.json', 'legacy-contract-reference.json'])
+@pytest.mark.parametrize('field', ['acceptance_contract', 'body_requirements', 'composite_acceptance'])
+def test_equivalent_legacy_capture_rejects_conflicting_embedded_rules(fixture_name, field):
+    fixture = json.loads((Path(__file__).parent / 'fixtures' / fixture_name).read_text())
+    original = EvalCase(**json.loads(fixture['files']['dataset.json'])['cases'][0])
+    manifest = json.loads(fixture['files']['manifest.json'])
+    binding = manifest['reference_checks']['cases'][original.id]
+    before = deepcopy(binding)
+    check_case_equivalent_reference_binding(original, binding, original_case=original)
+    assert binding == before
+
+    changed_rule = {'required_fields': ['a different requirement']}
+    changed = replace(original, **{field: changed_rule})
+    conflicting = deepcopy(binding)
+    conflicting['scope_identity']['evaluation_contract'][field] = changed_rule
+    for check in (check_case_reference_binding, check_case_equivalent_reference_binding):
+        with pytest.raises(ReferenceLifecycleError):
+            check(changed, conflicting, original_case=original)
+
+
+@pytest.mark.parametrize('version', [0, 3])
+def test_equivalent_capture_rejects_unknown_scope_versions(version):
+    fixture = json.loads((Path(__file__).parent / 'fixtures/legacy-live-reference.json').read_text())
+    case = EvalCase(**json.loads(fixture['files']['dataset.json'])['cases'][0])
+    binding = json.loads(fixture['files']['manifest.json'])['reference_checks']['cases'][case.id]
+    binding['scope_identity'] = {**scope_identity(case), 'schema_version': version}
+    for check in (check_case_reference_binding, check_case_equivalent_reference_binding):
+        with pytest.raises(ReferenceLifecycleError):
+            check(case, binding, original_case=case)