diff --git a/.env.example b/.env.example index 6174801f4..e311e0a30 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,8 @@ OPENCLAW_REPO= OPENCLAW_NODE=~/.browser-eval/toolchains/openclaw-node/node_modules/node/bin/node OPENCLAW_PNPM=~/.browser-eval/toolchains/openclaw-node/node_modules/.bin/pnpm DEEPSEEK_HARNESS_REPO= +# DeepSeek API (Fetch and Search): https://platform.deepseek.com/api_keys +DEEPSEEK_API_KEY= OSS_PYTHON=.venv/bin/python # Official Moli CLIs; each measured release binary is version- and SHA-256-gated. @@ -72,7 +74,9 @@ FIRECRAWL_API_KEY= TAVILY_API_KEY= EXA_API_KEY= BROWSERBASE_API_KEY= +# Optional for browser sessions; not required by the Fetch adapter. BROWSERBASE_PROJECT_ID= +BROWSER_USE_API_KEY= # Independent SERP comparison: https://serper.dev/ and https://brightdata.com/ SERPER_API_KEY= BRIGHTDATA_API_KEY= @@ -97,9 +101,10 @@ OXYLABS_PASSWORD= # Create it in Rainbond: Personal Center -> Access Token. RAINBOND_API_TOKEN= +# Crawl4AI Cloud: https://crawl4ai.com/dashboard/ (distinct from the local library) +CRAWL4AI_CLOUD_API_KEY= # Web Search providers. Keep evaluation credentials in this repository file. LEXMOUNT_SEARCH_API_KEY= -DEEPSEEK_API_KEY= BOCHA_API_KEY= # Optional, only for explicitly selected model relevance evaluation. SEARCH_JUDGE_API_KEY= diff --git a/README.md b/README.md index 249ec7184..ac9d2305a 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ Open the report path printed in the terminal. The example has one expected pass Next, [try HTTP, Moli or Chromium on a local article](examples/web_fetch/README.md). Optional adapters, service credentials and live evaluation commands are covered in the track guides above. +The reusable ten-method Web Fetch cohort is selected with `--providers configured --provider-config configs/providers.example.json`; shared defaults are 30 seconds per call and one failure retry. See the [provider contract](docs/provider-contract.md#selected-native-cohort). + ## Reports Reports include comparisons, task results, full responses and failure evidence. Runs are saved under `~/.browser-eval/runs/`; use `--output /absolute/path/new-run` to choose another directory. diff --git a/configs/providers.example.json b/configs/providers.example.json index cce838c9d..c376a87d8 100644 --- a/configs/providers.example.json +++ b/configs/providers.example.json @@ -1,6 +1,61 @@ { "schema_version": 1, + "default_providers": [ + "moli-webfetch", + "firecrawl", + "cloudflare-markdown", + "tavily", + "exa", + "brightdata", + "browser-use-fetch", + "browserbase", + "deepseek-fetch-api", + "crawl4ai-cloud" + ], "providers": { + "crawl4ai-cloud": { + "type": "crawl4ai_cloud", + "api_key": "${CRAWL4AI_CLOUD_API_KEY}", + "engine": "browser", + "proxy": "residential", + "output_format": "markdown", + "dns_mode": "system", + "log_level": "debug", + "uniform_native_parameters": true + }, + "deepseek-fetch-api": { + "type": "deepseek_fetch", + "api_key": "${DEEPSEEK_API_KEY}", + "dns_mode": "system", + "log_level": "debug", + "uniform_native_parameters": true + }, + "brightdata": { + "type": "brightdata_unlocker", + "api_key": "${BRIGHTDATA_API_TOKEN}", + "zone": "${BRIGHTDATA_UNLOCKER_ZONE}", + "render": true, + "debug": true, + "log_level": "debug", + "dns_mode": "system", + "uniform_native_parameters": true + }, + "browser-use-fetch": { + "type": "browser_use_fetch", + "api_key": "${BROWSER_USE_API_KEY}", + "proxy_country": "US", + "log_level": "debug", + "dns_mode": "system", + "uniform_native_parameters": true + }, + "browserbase": { + "type": "browserbase_fetch", + "api_key": "${BROWSERBASE_API_KEY}", + "proxies": true, + "log_level": "debug", + "dns_mode": "system", + "uniform_native_parameters": true + }, "moli-webfetch": { "type": "moli_webfetch", "binary": "${MOLI_BINARY:-moli}", @@ -10,14 +65,11 @@ "expected_binary_sha256": "7a53cdd071ae60a2f8ff6d08107f67f6b0f1d6247a9acd970d2bcec77ff15b50", "wait_until": "done", "block_private_networks": true, - "private_network_allowlist_hosts": [ - "browser.lexmount.com", - "react-shopping-cart-67954.firebaseapp.com" - ], "max_bytes": 20000000, "log_level": "debug", "retain_complete_diagnostic_log": true, - "max_diagnostic_bytes": 100000000 + "max_diagnostic_bytes": 100000000, + "uniform_native_parameters": true }, "fixture": { "type": "fixture", @@ -53,6 +105,7 @@ }, "firecrawl": { "type": "firecrawl", + "log_level": "debug", "base_url": "https://api.firecrawl.dev", "api_key": "${FIRECRAWL_API_KEY:-}", "max_age": 0, @@ -60,7 +113,7 @@ "min_interval_seconds": 6.1, "proxy": "auto", "capacity_credits_per_call": 1, - "dynamic_wait_strategy": "actions" + "uniform_native_parameters": true }, "firecrawl-resources": { "type": "firecrawl", @@ -76,11 +129,12 @@ }, "tavily": { "type": "tavily_extract", + "log_level": "debug", "base_url": "https://api.tavily.com", "api_key": "${TAVILY_API_KEY}", "extract_depth": "advanced", - "extract_timeout_seconds": 60.0, - "max_bytes": 20000000 + "max_bytes": 20000000, + "uniform_native_parameters": true }, "tavily-resources": { "type": "tavily_extract", @@ -93,12 +147,14 @@ }, "exa": { "type": "exa_contents", + "log_level": "debug", "base_url": "https://api.exa.ai", "api_key": "${EXA_API_KEY}", "text_verbosity": "full", "max_age_hours": 0, - "livecrawl_timeout_ms": 15000, - "max_bytes": 20000000 + "max_bytes": 20000000, + "uniform_native_parameters": true, + "rich_links_limit": 100 }, "exa-resources": { "type": "exa_contents", @@ -112,11 +168,13 @@ }, "cloudflare-markdown": { "type": "cloudflare_markdown", + "log_level": "debug", "api_token": "${CLOUDFLARE_API_TOKEN}", "account_id": "${CLOUDFLARE_ACCOUNT_ID}", "max_bytes": 20000000, "document_max_bytes": 20000000, - "wait_until": "networkidle2" + "wait_until": "networkidle2", + "uniform_native_parameters": true }, "cloudflare-content": { "type": "cloudflare_content", diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index bfcf21432..3b0ed00e8 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -45,6 +45,8 @@ uv run --frozen browser-harness-eval run --delay 0 接下来可以[用 HTTP、Moli 或 Chromium 读取本地文章](../examples/web_fetch/README.md)。可选适配器、服务凭据和在线评测命令见上面的各流程指南。 +使用 `--providers configured --provider-config configs/providers.example.json` 选择可复用的十种 Web Fetch 方法；默认每次调用 30 秒，失败允许重试一次。详见[适配器规则](provider-contract.md#selected-native-cohort)。 + ## 报告 报告包含结果对比、逐题结果、完整响应和失败证据。运行结果默认保存在 `~/.browser-eval/runs/`，也可用 `--output /absolute/path/new-run` 指定新目录。 diff --git a/docs/provider-contract.md b/docs/provider-contract.md index 8887a3751..892cc84a1 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -14,6 +14,13 @@ fetch(case: EvalCase, timeout: float) -> ProviderResult - provider-specific request ID, engine, server elapsed time and billing data; - a stable error type and actionable error message. +Disclose service API status as `metadata.api_http_status` and the final target-page +status as `metadata.upstream_status`; use `None` when unavailable. Do not copy an +API status into the target status. Before scoring and retry selection, the shared +answer boundary marks a known non-2xx target response as a failed call, preserving +its returned content and existing failure details. Content acceptance and call +success remain separate requirements; historical results are not rewritten. + Credentials are expanded from environment placeholders in the provider config. They must never be added to `metadata`, content, manifest or logs. @@ -25,6 +32,42 @@ They must never be added to `metadata`, content, manifest or logs. Use the existing scorer and report renderer. +### Hosted Markdown fetch services + +Five methods share `providers/hosted_fetch.py` and the existing HTTP client, +runner and scorer. Their credentials and fixed options are declared in +`configs/providers.example.json`; credentials come from the repository `.env`. + +| Method | Credentials | Native route and example options | +| --- | --- | --- | +| Bright Data | `BRIGHTDATA_API_TOKEN`, `BRIGHTDATA_UNLOCKER_ZONE` | `/request`, raw Markdown, rendering and native debug enabled | +| Browser Use Fetch | `BROWSER_USE_API_KEY` | `/fetch`, Markdown, fresh session, US proxy; no Agent | +| Browserbase Fetch | `BROWSERBASE_API_KEY` | `/v1/fetch`, Markdown, redirects and native proxy enabled; no Project ID | +| DeepSeek Fetch API | `DEEPSEEK_API_KEY` | `/v1/tools/fetch`, URL-only request; retain content and title | +| Crawl4AI Cloud | `CRAWL4AI_CLOUD_API_KEY` | `/scrape`, `format=md`, `engine=browser`, `proxy=residential` | + +These adapters implement webpage body/structure tasks. Unsupported document, +media and file tasks fail before transport, without helper downloads or a +substitute browser/Agent. Rendering and internal retries remain service-owned; +offline adapter checks do not establish JavaScript capability or task pass rates. + +Retain complete native responses separately from score-bearing answer fields. +An API HTTP 200 does not prove target retrieval succeeded: explicit service or +origin errors, empty content and malformed responses remain failures. API status +is distinct from target-site status. `log_level: debug` saves redacted caller +requests, response headers/body and errors; it cannot expose undisclosed service +internals. Only Bright Data receives the native debug option. + +`dns_mode: system` uses the local resolver for API hosts; target-site DNS belongs +to the service. No automatic DNS fallback is added. All five services retain their +native internal timeout defaults within the caller's deadline. Historical runs +keep their recorded configuration. + +API references: [Bright Data](https://docs.brightdata.com/api-reference/rest-api/unlocker/unlock-website), +[fetch-use](https://browser-use.com/posts/fetch-use), +[Browserbase Fetch](https://docs.browserbase.com/platform/fetch/overview), +[Crawl4AI Cloud](https://api.crawl4ai.com/docs/). + | Output contract | Comparison scope | | --- | --- | | Cleaned content: `main_text`, Markdown or text | Shared readable-content leaderboard | @@ -91,3 +134,45 @@ The version-2 extraction scope describes text extraction and requires the dynamic text route. Combining it with resource discovery is rejected before a call. Direct file delivery also rejects DOM completion: it cannot claim a rendered-page condition was satisfied while fetching only response bytes. + +## Selected native cohort + +`configs/providers.example.json` owns the ten retained methods in +`default_providers`. Use `--providers configured --provider-config +configs/providers.example.json` with `run`, `validate`, `run_benchmark.py` or +`run_certified_benchmark.py`. Selection is frozen as explicit names in evidence; +other adapters remain selectable, and the no-argument CLI still uses a fixture. + +- The default is 30 seconds per physical call and one failure retry. The final + response supplies the result and primary latency; every call and accumulated + retry cost is retained. CLI overrides remain explicit and recorded. +- Stop timing after the response and required decoding; preparation for scoring, + scoring and evidence persistence follow. Discard late answers and quarantine + still-running workers rather than overlapping abandoned calls. +- Reject per-case hints and never derive requests from grading requirements. + Dynamic cases declare native retrieval mode. Exa uniformly requests 100 rich + links; Cloudflare receives URLs directly, without helper document downloads. +- Keep the 20,000,000-byte HTTP response/Moli output ceiling and debug logs. + Moli retains its pinned binary identity and private-network guard without + per-host exceptions or a caller-imposed subresource download cap. Non-public + DNS answers must be resolved as an environment issue before a public cohort. +- `crawl4ai-cloud` is distinct from local `crawl4ai`: browser versions and + extraction differ, and old local results must not be relabeled as Cloud. + Cloud uses service-cleaned Markdown; retain its engine/cache/usage fields. + Its configured native proxy adds provider charges; never switch proxy policy + in response to scores. + +For diagnostics, supply frozen tasks and a new external output. `--measure-only` +retains scored evidence without building a report; it does not certify a cohort. +Historical unscored collections remain ineligible for scored reports. + +```sh +uv run --frozen python scripts/run_benchmark.py \ + --dataset /path/to/frozen-dataset.json \ + --provider-config configs/providers.example.json --providers configured \ + --timeout 30 --max-failure-retries 1 --workers 1 --measure-only \ + --output /path/to/new-run +``` + +Publishable cohorts use `scripts/run_certified_benchmark.py` with its required +scope, work root and output. Report profiles and measured data stay outside Git. diff --git a/harness/record_io.py b/harness/record_io.py index bbde73baf..fbe341d82 100644 --- a/harness/record_io.py +++ b/harness/record_io.py @@ -4,6 +4,11 @@ from pathlib import Path +def parse_utf8_json(body: bytes): + """Use the same strict wire encoding for acquisition and evidence replay.""" + return json.loads(body.decode('utf-8')) + + def file_sha256(path): """Hash file bytes without loading the entire artifact into memory. diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py index 7b968cd10..304390fae 100644 --- a/harness/web_fetch/artifacts.py +++ b/harness/web_fetch/artifacts.py @@ -164,9 +164,12 @@ def verify_public_call_log(root: Path, binding: dict[str, Any], *, allow_public_ for event in events) or events[0]["stage"] != "call_started" or events[-1]["stage"] != "call_completed" - or not any(event["stage"] in {"response_received", "exception_raised"} + or not any(event["stage"] in {"response_received", "exception_raised", "answer_deadline_exceeded"} for event in events[1:-1])): raise ValueError("Public call log has an unexpected event schema") + if any(event["stage"] == "answer_deadline_exceeded" and event["error_type"] != "timeout" + for event in events): + raise ValueError("Deadline event must identify a timeout") summary = payload["request"]["summary"] if summary is not None and not isinstance(summary, dict): raise ValueError("Public call log has an unexpected request summary") diff --git a/harness/web_fetch/benchmark_runner.py b/harness/web_fetch/benchmark_runner.py index cfff58e4c..ffe527d2f 100644 --- a/harness/web_fetch/benchmark_runner.py +++ b/harness/web_fetch/benchmark_runner.py @@ -15,6 +15,7 @@ load_dataset, load_provider_config, load_runtime_env, + resolve_provider_selection, ) from harness.web_fetch.network_preflight import preflight_guarded_providers from harness.web_fetch.providers import build_provider @@ -33,7 +34,7 @@ def main(argv=None): parser.add_argument('--env-file',type=Path) parser.add_argument('--provider-config',type=Path, help='One provider configuration for all selected tasks; defaults to the repository task-group configurations') - parser.add_argument('--providers',help='Comma-separated names; default: all benchmark methods') + parser.add_argument('--providers',help='Comma-separated names or configured; default: dataset coverage plan') parser.add_argument('--track',help='Run only this task group, such as dynamic; default: all groups') parser.add_argument('--case-ids',help='Comma-separated case IDs; default: every case in the selected task group') parser.add_argument('--validate-only',action='store_true',help='Check selected configuration without creating files or calling providers') @@ -56,6 +57,10 @@ def main(argv=None): if args.runs < 1:parser.error('--runs must be positive') if not args.validate_only and args.output.exists():parser.error('Output exists; choose an empty directory to avoid overwriting measured evidence') load_runtime_env(args.env_file) + try: + args.providers = resolve_provider_selection(args.providers, args.provider_config) + except ValueError as error: + parser.error(str(error)) dataset,cases=load_dataset(args.dataset) if args.track: cases=[c for c in cases if c.track==args.track] diff --git a/harness/web_fetch/call_boundary.py b/harness/web_fetch/call_boundary.py index 6bf53d44e..6549f2e3b 100644 --- a/harness/web_fetch/call_boundary.py +++ b/harness/web_fetch/call_boundary.py @@ -1,8 +1,8 @@ -"""One end-to-end answer deadline for every adapter, including local processing.""" - +"""Bound provider calls and prepare answers outside fixed-native response timing.""" import queue import threading import time +import traceback from lexbench_webfetch.file_acceptance import evaluate_files from lexbench_webfetch.response_input import answer_input @@ -12,7 +12,17 @@ def prepare_answer(result, case): - # This required representation conversion is inside the measured call. + # A successful service request is not proof that its target page succeeded. + # Adapters disclose origin status separately; never infer it from API status. + origin = result.metadata.get('upstream_status') + origin = origin if type(origin) is int and 100 <= origin <= 599 else None + result.metadata['upstream_status'] = origin + result.metadata['upstream_http_status_available'] = origin is not None + if result.ok and origin is not None and not 200 <= origin < 300: + result.ok = False + result.error_type = 'http_error' + result.error_message = f'Target page returned HTTP {origin}' + # Prepare scorer input; fixed-native calls have already stopped their timer. result.prepared_answer = answer_input(result.content.encode('utf-8', errors='replace'), result.metadata, ok=result.ok, status_code=result.status_code) # Persist and fingerprint every returned binary, including incidental @@ -31,6 +41,7 @@ def fetch_with_deadline(provider, case, timeout, *, native_route_ledger=None): stop, the runner quarantines that provider and declares remaining cases unexecuted. It does not silently run concurrent abandoned paid requests. """ + response_only = bool(provider.config.get("uniform_native_parameters")) output = queue.Queue(maxsize=1) provider.begin_call() begin = time.perf_counter() @@ -41,23 +52,34 @@ def work(): fetch_options = ({'native_route_ledger': native_route_ledger} if native_route_ledger is not None else {}) fetched = provider.fetch(case, timeout, **fetch_options) + if response_only: + output.put((time.perf_counter(), fetched)) + return emit("provider_returned", ok=fetched.ok, content_chars=len(fetched.content)) value = prepare_answer(fetched, case) emit("answer_prepared", content_chars=len(value.prepared_answer[0])) except Exception as error: sink=getattr(provider, "_call_diagnostics", None) - if sink: sink.emit("caller", "provider_exception", type=type(error).__name__, message=str(error)) + if sink: sink.emit("caller", "provider_exception", type=type(error).__name__, message=str(error), + traceback=traceback.format_exc()) value = error output.put((time.perf_counter(), value)) worker = threading.Thread(target=work, daemon=True, name='webfetch-answer') worker.start() + caller_deadline_exceeded = False try: completed, value = output.get(timeout=max(0, timeout - (time.perf_counter() - begin))) except queue.Empty: + caller_deadline_exceeded = True completed, value = time.perf_counter(), TimeoutError('End-to-end answer deadline exceeded') if completed - begin > timeout: + caller_deadline_exceeded = True value = TimeoutError('End-to-end answer deadline exceeded') - if isinstance(value, TimeoutError): + if caller_deadline_exceeded: + sink = getattr(provider, "_call_diagnostics", None) + if sink: + sink.emit("caller", "answer_timeout", type=type(value).__name__, message=str(value), + timeout_seconds=timeout, worker_still_running=worker.is_alive()) # Completion/cancellation cleanup is not additional answer budget. answer_elapsed_ms = round((completed - begin) * 1000, 2) provider.cancel() @@ -79,6 +101,7 @@ def work(): if case.task_kind == 'file': metadata['file_evaluation'] = evaluate_files(vars(case), []) metadata['_answer_elapsed_ms']=answer_elapsed_ms + metadata['caller_deadline_exceeded'] = True return ProviderResult( False, '', @@ -88,4 +111,13 @@ def work(): ), worker if isinstance(value, Exception): raise value + value.metadata['caller_deadline_exceeded'] = False + if response_only: + # Stop the user-observed timer before scorer input preparation/storage. + value.metadata['_answer_elapsed_ms'] = round((completed - begin) * 1000, 2) + value.metadata['latency_scope'] = 'native_response_received' + with diagnostic_context(getattr(provider, "_call_diagnostics", None)): + emit("provider_returned", ok=value.ok, content_chars=len(value.content)) + value = prepare_answer(value, case) + emit("answer_prepared", content_chars=len(value.prepared_answer[0])) return value, worker diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py index 5170d1d36..24f373b87 100644 --- a/harness/web_fetch/certification.py +++ b/harness/web_fetch/certification.py @@ -26,7 +26,7 @@ webmainbench_certification, webmainbench_runner, ) -from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS, add_failure_retry_argument +from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS, add_failure_retry_argument, resolve_provider_selection from harness.web_fetch.reference_inputs import read_dataset_bytes from harness.web_fetch.reference_lifecycle import ( requires_browser_observation, @@ -154,7 +154,7 @@ def main(argv: list[str] | None = None) -> None: help="Frozen benchmark edition; checked before any external calls") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--work-root", type=Path, required=True) - parser.add_argument("--providers", required=True) + parser.add_argument("--providers", required=True, help="Comma-separated names or configured with --provider-config") parser.add_argument("--case-ids", required=True) parser.add_argument("--workers", type=int, default=1) parser.add_argument("--timeout", type=float, default=DEFAULT_CALL_TIMEOUT_SECONDS, @@ -190,6 +190,10 @@ def main(argv: list[str] | None = None) -> None: parser.error("--timeout must be positive and finite") if args.provider_config and not args.provider_config.is_file(): parser.error("--provider-config must be an existing file") + try: + args.providers = resolve_provider_selection(args.providers, args.provider_config) + except ValueError as error: + parser.error(str(error)) if args.call_order == "case-alternating" and (args.workers != 1 or not args.provider_config): parser.error("Alternating case calls require --workers 1 and --provider-config") if args.browser_capture_attempts < 1: diff --git a/harness/web_fetch/cli.py b/harness/web_fetch/cli.py index a2466e8c0..2f03d90bb 100644 --- a/harness/web_fetch/cli.py +++ b/harness/web_fetch/cli.py @@ -19,6 +19,7 @@ load_dataset, load_provider_config, load_runtime_env, + resolve_provider_selection, select_diagnostic_scope, ) from harness.web_fetch.providers import build_provider @@ -41,7 +42,8 @@ def command_run(args: argparse.Namespace) -> int: language = resolve_report_language(args.output / "report", args.language) dataset_meta, cases = load_dataset(args.dataset) provider_configs = load_provider_config(args.provider_config) - names = [name.strip() for name in args.providers.split(",") if name.strip()] + selection = resolve_provider_selection(args.providers, args.provider_config) + names = [name.strip() for name in selection.split(",") if name.strip()] unknown = [name for name in names if name not in provider_configs] if unknown: raise SystemExit(f"Unknown providers: {', '.join(unknown)}") @@ -94,7 +96,8 @@ def command_validate(args: argparse.Namespace) -> int: load_runtime_env(args.env_file) _, cases = load_dataset(args.dataset) configs = load_provider_config(args.provider_config) - names = [name.strip() for name in args.providers.split(",") if name.strip()] + selection = resolve_provider_selection(args.providers, args.provider_config) + names = [name.strip() for name in selection.split(",") if name.strip()] errors = [] for name in names: if name not in configs: @@ -127,7 +130,7 @@ def build_parser() -> argparse.ArgumentParser: add_language_argument(run) run.add_argument("--dataset", type=Path, default=EXAMPLES / "fixture-dataset.json") run.add_argument("--provider-config", type=Path, default=EXAMPLES / "fixture-providers.json") - run.add_argument("--providers", default="fixture", help="Comma-separated configured provider names") + run.add_argument("--providers", default="fixture", help="Comma-separated names, or configured for the provider config default cohort") run.add_argument("--case-ids", default="", help="Optional comma-separated case IDs") run.add_argument("--task-kind", choices=TASK_KINDS, help="Select one task kind; intersects with --case-ids") diff --git a/harness/web_fetch/config.py b/harness/web_fetch/config.py index c4b08cdda..60281f99d 100644 --- a/harness/web_fetch/config.py +++ b/harness/web_fetch/config.py @@ -19,7 +19,7 @@ ROOT = Path(__file__).resolve().parents[2] DEFAULT_FAILURE_RETRIES = 1 -DEFAULT_CALL_TIMEOUT_SECONDS = 45 +DEFAULT_CALL_TIMEOUT_SECONDS = 30 def nonnegative_int(value: str) -> int: @@ -188,3 +188,21 @@ def load_provider_settings(path: Path) -> dict[str, Any]: def load_provider_config(path: Path) -> dict[str, dict[str, Any]]: return load_provider_settings(path)["providers"] + + +def resolve_provider_selection(selection: str | None, config_path: Path | None) -> str | None: + """Resolve an explicitly requested configured cohort without changing other defaults.""" + if selection != 'configured': + return selection + if config_path is None: + raise ValueError('--providers configured requires --provider-config') + settings = load_provider_settings(config_path) + names = settings.get('default_providers') + if (not isinstance(names, list) or not names + or any(not isinstance(name, str) or not name.strip() or ',' in name for name in names) + or len(names) != len(set(names))): + raise ValueError('Provider config must declare nonempty, unique default_providers') + unknown = set(names) - set(settings['providers']) + if unknown: + raise ValueError('Configured cohort contains unknown providers: ' + ', '.join(sorted(unknown))) + return ','.join(names) diff --git a/harness/web_fetch/data/report-methods.json b/harness/web_fetch/data/report-methods.json index 0371aca83..540363855 100644 --- a/harness/web_fetch/data/report-methods.json +++ b/harness/web_fetch/data/report-methods.json @@ -404,6 +404,51 @@ "zh": "离线诊断", "en": "Offline diagnostics" } + }, + "brightdata_unlocker": { + "extends": "hosted_fetch_adapter", + "endpoint": "POST https://api.brightdata.com/request", + "interface_method": "brightdata", + "contract": { + "zh": "已知网址交给 Web Unlocker，返回原生 Markdown。", + "en": "Send a known URL to Web Unlocker and retain native Markdown." + } + }, + "browserbase_fetch": { + "extends": "hosted_fetch_adapter", + "endpoint": "POST https://api.browserbase.com/v1/fetch", + "interface_method": "browserbase", + "contract": { + "zh": "通过 Fetch 接口获取网页文字，不替换为浏览器会话。", + "en": "Retrieve webpage text through Fetch without substituting a browser session." + } + }, + "browser_use_fetch": { + "extends": "hosted_fetch_adapter", + "endpoint": "POST https://fetch.browser-use.com/fetch", + "interface_method": "browser-use-fetch", + "contract": { + "zh": "通过 fetch-use 获取网页文字，不启动 Agent。", + "en": "Retrieve webpage text through fetch-use without an Agent." + } + }, + "deepseek_fetch": { + "extends": "hosted_fetch_adapter", + "endpoint": "POST https://api.deepseek.com/v1/tools/fetch", + "interface_method": "deepseek-fetch-api", + "contract": { + "zh": "只发送网址；正文与标题分别保留。", + "en": "Send only the URL; retain content and title separately." + } + }, + "crawl4ai_cloud": { + "extends": "hosted_fetch_adapter", + "endpoint": "POST https://api.crawl4ai.com/scrape", + "interface_method": "crawl4ai-cloud", + "contract": { + "zh": "调用 Cloud 抓取接口，与本机开源库分别登记。", + "en": "Call the Cloud scrape API, registered separately from the local library." + } } }, "route_labels": { @@ -5176,26 +5221,17 @@ "status": "implemented" }, "browserbase": { + "extends": "hosted_fetch", "label": { "zh": "Browserbase Fetch", "en": "Browserbase Fetch" }, - "status": "planned", "source_url": "https://www.browserbase.com/sign-up", "access": { - "type": "hosted", "env_vars": [ - "BROWSERBASE_API_KEY", - "BROWSERBASE_PROJECT_ID" + "BROWSERBASE_API_KEY" ], "links": [ - [ - { - "zh": "注册 / 登录", - "en": "Register / sign in" - }, - "https://www.browserbase.com/sign-up" - ], [ { "zh": "获取凭据", @@ -5203,20 +5239,30 @@ }, "https://www.browserbase.com/overview" ] + ] + }, + "adapter_types": [ + "browserbase_fetch" + ], + "interface": { + "entry": "POST https://api.browserbase.com/v1/fetch", + "auth": "X-BB-API-Key: ", + "fields": [ + [ + "content", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] ], - "account": { - "zh": "官方页面存在契约冲突；打分前必须先以账号内实际接口核验。", - "en": "Official pages describe conflicting contracts; verify the account-specific interface before scoring." - }, - "configuration": { - "zh": "注册后创建或选择项目，从 Overview 控制台复制 API key 与 Project ID。", - "en": "After registering, create or select a project and copy its API key and Project ID from Overview." - }, - "required": true, - "priority": { - "zh": "B · 已知 URL 抓取候选", - "en": "B · Known-URL fetch candidate" + "request": { + "format": "markdown" } + }, + "capabilities": { + "scope": "POST https://api.browserbase.com/v1/fetch" } }, "bocha-web-search": { @@ -5414,26 +5460,18 @@ } }, "brightdata": { + "extends": "hosted_fetch", "label": { "zh": "Bright Data Web Unlocker", "en": "Bright Data Web Unlocker" }, - "status": "planned", "source_url": "https://brightdata.com/cp/start", "access": { - "type": "hosted", "env_vars": [ "BRIGHTDATA_API_TOKEN", "BRIGHTDATA_UNLOCKER_ZONE" ], "links": [ - [ - { - "zh": "注册 / 登录", - "en": "Register / sign in" - }, - "https://brightdata.com/cp/start" - ], [ { "zh": "获取凭据", @@ -5441,20 +5479,32 @@ }, "https://brightdata.com/cp/zones" ] + ] + }, + "adapter_types": [ + "brightdata_unlocker" + ], + "interface": { + "entry": "POST https://api.brightdata.com/request", + "auth": "Authorization: Bearer ", + "fields": [ + [ + "HTTP response body", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] ], - "account": { - "zh": "需要账号、计费或试用资格，以及 Web Unlocker zone。", - "en": "Requires an account, billing or trial eligibility, and a Web Unlocker zone." - }, - "configuration": { - "zh": "注册账号，创建 Web Unlocker zone，再复制 API token 与 zone 名称。", - "en": "Register, create a Web Unlocker zone and copy the API token and zone name." - }, - "required": true, - "priority": { - "zh": "C · 解锁场景数据集", - "en": "C · Access-unblocking dataset" + "request": { + "zone": "", + "format": "raw", + "data_format": "markdown" } + }, + "capabilities": { + "scope": "POST https://api.brightdata.com/request" } }, "scrapingbee": { @@ -5665,6 +5715,138 @@ "en": "C · Fetch API comparison" } } + }, + "browser-use-fetch": { + "extends": "hosted_fetch", + "label": { + "zh": "Browser Use Fetch", + "en": "Browser Use Fetch" + }, + "source_url": "https://cloud.browser-use.com/settings?tab=api-keys", + "adapter_types": [ + "browser_use_fetch" + ], + "access": { + "env_vars": [ + "BROWSER_USE_API_KEY" + ], + "links": [ + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://cloud.browser-use.com/settings?tab=api-keys" + ] + ] + }, + "interface": { + "entry": "POST https://fetch.browser-use.com/fetch", + "auth": "X-Browser-Use-API-Key: ", + "fields": [ + [ + "body", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "output_format": "markdown" + } + }, + "capabilities": { + "scope": "POST https://fetch.browser-use.com/fetch" + } + }, + "deepseek-fetch-api": { + "extends": "hosted_fetch", + "label": { + "zh": "DeepSeek Fetch API", + "en": "DeepSeek Fetch API" + }, + "source_url": "https://platform.deepseek.com/api_keys", + "adapter_types": [ + "deepseek_fetch" + ], + "access": { + "env_vars": [ + "DEEPSEEK_API_KEY" + ], + "links": [ + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://platform.deepseek.com/api_keys" + ] + ] + }, + "interface": { + "entry": "POST https://api.deepseek.com/v1/tools/fetch", + "auth": "Authorization: Bearer ", + "fields": [ + [ + "content; title", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ] + }, + "capabilities": { + "scope": "POST https://api.deepseek.com/v1/tools/fetch" + } + }, + "crawl4ai-cloud": { + "extends": "hosted_fetch", + "label": { + "zh": "Crawl4AI Cloud", + "en": "Crawl4AI Cloud" + }, + "source_url": "https://api.crawl4ai.com/docs/", + "adapter_types": [ + "crawl4ai_cloud" + ], + "access": { + "env_vars": [ + "CRAWL4AI_CLOUD_API_KEY" + ], + "links": [ + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://crawl4ai.com/dashboard/" + ] + ] + }, + "interface": { + "entry": "POST https://api.crawl4ai.com/scrape", + "auth": "Authorization: Bearer ", + "fields": [ + [ + "markdown", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "format": "md" + } + }, + "capabilities": { + "scope": "POST https://api.crawl4ai.com/scrape" + } } }, "sources": { @@ -5970,6 +6152,15 @@ "url": "https://github.com/lexmount/moli/releases/tag/v1.1.5", "checked_at": "2026-09-11", "checked_version": "1.1.5" + }, + "hosted-fetch-adapter": { + "label": { + "zh": "托管 Fetch 接入代码（能力范围）", + "en": "Hosted Fetch adapter (implemented scope)" + }, + "url": "https://github.com/lexmount/browser-eval/blob/1c2f5d310700db97968d5f75e9a89c96f9505840/harness/web_fetch/providers/hosted_fetch.py", + "checked_at": "2026-10-25", + "checked_version": "1c2f5d310700db97968d5f75e9a89c96f9505840" } }, "output_contract_labels": { @@ -6096,6 +6287,155 @@ "en": "The browser handles access and native output retrieval. Embedded media and text use native Markdown; the shared scorer only checks target resource URLs. See the interpretation interface table for the full scope and boundaries." } } + }, + "hosted_fetch": { + "status": "implemented", + "access": { + "type": "hosted", + "account": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "configuration": { + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." + }, + "required": true, + "priority": { + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + } + }, + "kind": "adapter", + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "request": { + "url": "https://example.com/" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." + } + }, + "capabilities": { + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } + } + }, + "hosted_fetch_adapter": { + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } } }, "unregistered_access": { diff --git a/harness/web_fetch/diagnostics.py b/harness/web_fetch/diagnostics.py index 09d5568fd..d195be3ca 100644 --- a/harness/web_fetch/diagnostics.py +++ b/harness/web_fetch/diagnostics.py @@ -9,7 +9,7 @@ from pathlib import Path _LOCAL = threading.local() -_SECRET = {"authorization", "proxy-authorization", "cookie", "set-cookie", "api_key", "apikey", "api-key", "x-api-key", "token", "password", "secret"} +_SECRET = {"authorization", "proxy-authorization", "cookie", "set-cookie", "api_key", "apikey", "api-key", "x-api-key", "x-bb-api-key", "x-browser-use-api-key", "token", "password", "secret"} def redact(value): if isinstance(value, dict): diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py index 90aa92c28..a18048431 100644 --- a/harness/web_fetch/methods.py +++ b/harness/web_fetch/methods.py @@ -16,7 +16,7 @@ def _resolve_catalog_templates(data): - """Expand named defaults once; methods override dictionaries and replace lists.""" + """Expand named defaults once; catalog entries override dictionaries and replace lists.""" def merge(base, override): result = deepcopy(base) for key, value in override.items(): @@ -37,7 +37,8 @@ def resolve(row, chain=()): raise ValueError('Cyclic method template: ' + parent) return merge(resolve(templates[parent], (*chain, parent)), row) - return {**data, 'methods': {name: resolve(row) for name, row in data['methods'].items()}} + return {**data, **{section: {name: resolve(row) for name, row in data[section].items()} + for section in ('methods', 'adapters') if section in data}} @lru_cache(maxsize=1) diff --git a/harness/web_fetch/native_projection.py b/harness/web_fetch/native_projection.py index 0560f0a70..01515a664 100644 --- a/harness/web_fetch/native_projection.py +++ b/harness/web_fetch/native_projection.py @@ -50,6 +50,22 @@ def native_answer_projection(provider_type, case, payload, status_code=None, *, track, task_kind = get('track', 'web'), get('task_kind', 'body') payload = _object(payload) successful_http = type(status_code) is int and 200 <= status_code < 300 + if provider_type == 'brightdata_unlocker': + return {'schema_version': 2, 'source_format': 'bytes', + 'content_format': 'utf8' if successful_http else 'empty'} + if provider_type in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: + field = ('body' if provider_type == 'browser_use_fetch' else + 'markdown' if provider_type == 'crawl4ai_cloud' else 'content') + value = payload.get(field) + fmt = 'text' if isinstance(value, str) else 'empty' + declaration = {'schema_version': 2, 'content_pointer': [field], + 'content_format': fmt if successful_http else 'empty', + 'optional_content': True, 'result_fields': {}} + if provider_type == 'deepseek_fetch': + declaration['result_fields']['title'] = {'pointer': ['title'], 'format': 'text_or_none'} + if provider_type == 'browser_use_fetch': + declaration['result_fields']['final_url'] = {'pointer': ['final_url'], 'format': 'text_or_none'} + return declaration if provider_type in {'tavily_extract', 'exa_contents'}: results = payload.get('results') result = _object(results[0]) if isinstance(results, list) and results else {} @@ -322,7 +338,11 @@ def project_native_answer(payload, declaration, *, historical_scoring_source=Non if not isinstance(payload, bytes): raise ValueError('Native byte projection requires bytes') if kind == 'utf8': - return payload.decode('utf-8'), {} + try: + return payload.decode('utf-8'), {} + except UnicodeDecodeError: + # Invalid text remains raw evidence, never repaired answer text. + return '', {} if kind == 'empty': return '', {} raise ValueError('Unknown native byte projection') @@ -545,7 +565,9 @@ def check_source(spec): if 'legacy' in spec: check_source(spec['legacy']) if declaration.get('source_format') == 'bytes': - required = 'original_bytes' if declaration.get('files') else 'stdout' + required = ('original_bytes' if declaration.get('files') else + 'response_body' if 'response_body' in allowed + else 'stdout') if required not in allowed: raise ValueError('Native byte projection differs from its frozen answer fields') else: diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py index 51b3d226e..ddee6e8b1 100644 --- a/harness/web_fetch/native_routes.py +++ b/harness/web_fetch/native_routes.py @@ -12,7 +12,7 @@ from lexbench_webfetch.file_acceptance import evaluate_files from lexbench_webfetch.paths import contained_path -from harness.record_io import canonical_json_sha256, file_sha256 +from harness.record_io import canonical_json_sha256, file_sha256, parse_utf8_json from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS from harness.web_fetch.native_projection import ( SCORE_ANSWER_FIELDS, @@ -171,6 +171,14 @@ def normalized_http_parameters(case, arguments): body['urls'] = [''] else: raise ValueError('Native HTTP input differs from the frozen task URL') + # Native fetch-use requires a fresh opaque UUID per physical invocation. + # Normalize only this session token, never task or extraction parameters. + if 'session_id' in body and 'X-Browser-Use-API-Key' in arguments.get('headers', {}): + import uuid + token = body['session_id'] + if not isinstance(token, str) or str(uuid.UUID(token)) != token: + raise ValueError('Native fetch session must be a canonical UUID') + body['session_id'] = '' return {'method': arguments.get('method', 'GET'), 'body': body, 'headers_sha256': canonical_json_sha256(arguments.get('headers', {})), 'timeout_seconds': arguments.get('timeout'), 'max_bytes': arguments.get('max_bytes')} @@ -486,13 +494,22 @@ def _expected_native_declaration(root, record, raw, cases_by_id): if case is None or case.get('url') != record.get('url'): raise ValueError('Native answer selection lacks its frozen request identity') try: - payload = json.loads(raw) if record['provider_type'] != 'moli_webfetch' else {} + if record['provider_type'] in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: + payload = parse_utf8_json(raw) + else: + payload = json.loads(raw) if record['provider_type'] not in {'moli_webfetch', 'brightdata_unlocker'} else {} except (ValueError, UnicodeDecodeError): + payload = None + if record['provider_type'] in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: + if not isinstance(payload, dict): + return None # Malformed API envelopes are evidence, never answers. + if payload is None: payload = {} requests = record['provider_metadata']['native_route_receipt'].get('requests', []) interface = requests[0].get('signature', {}).get('interface') if requests else None return native_answer_projection(record['provider_type'], case, payload, - record.get('status_code'), interface=interface) + record['provider_metadata'].get('api_http_status', record.get('status_code')), + interface=record['provider_metadata'].get('output_contract', interface)) def verify_saved_native_answer(root, record, *, cases_by_id=None): @@ -523,7 +540,7 @@ def verify_saved_native_answer(root, record, *, cases_by_id=None): # A timed-out empty answer is deliberately discarded. Network or # pre-transport failures may have no raw response to select from. if record.get('error_type') != 'timeout': - bytes_source = record['provider_type'] == 'moli_webfetch' + bytes_source = record.get('provider_type') == 'moli_webfetch' raw_path = metadata.get('raw_stdout_path' if bytes_source else 'raw_response_path') raw_hash = metadata.get('raw_stdout_sha256' if bytes_source else 'raw_response_sha256') if raw_path or raw_hash: @@ -539,8 +556,10 @@ def verify_saved_native_answer(root, record, *, cases_by_id=None): if any(metadata.get(key) for key in SCORE_ANSWER_FIELDS) or metadata.get('file_evaluation', {}).get('files'): raise ValueError('Discarded late native answer retains score-bearing fields') return - raw_path = metadata.get('raw_stdout_path') if declaration.get('source_format') == 'bytes' else metadata.get('raw_response_path') - raw_hash = metadata.get('raw_stdout_sha256') if declaration.get('source_format') == 'bytes' else metadata.get('raw_response_sha256') + stdout_source = (declaration.get('source_format') == 'bytes' + and record.get('provider_type') != 'brightdata_unlocker') + raw_path = metadata.get('raw_stdout_path' if stdout_source else 'raw_response_path') + raw_hash = metadata.get('raw_stdout_sha256' if stdout_source else 'raw_response_sha256') if not raw_path or not raw_hash: raise ValueError('Declared native answer lacks its retained raw response') raw = read_evidence_binding(root, {'path': raw_path, 'sha256': raw_hash}).read_bytes() diff --git a/harness/web_fetch/providers/base.py b/harness/web_fetch/providers/base.py index 935c7c6ee..1d8fa470c 100644 --- a/harness/web_fetch/providers/base.py +++ b/harness/web_fetch/providers/base.py @@ -83,6 +83,8 @@ def terminate_owned_process_tree(process): class Provider(ABC): + allow_failure_retries = True + # Public configuration diagnostics only. Credentials, commands and local # paths remain covered by the full configuration hash, never displayed. # Adapters may extend this declaration for their own safe options. @@ -118,7 +120,7 @@ def check_native_call(self, case: EvalCase) -> None: return if case.retrieval_hints and not (case.track == "dynamic" and case.retrieval_hints == NATIVE_RETRIEVAL_HINTS): raise ValueError("fixed native calls prohibit per-case retrieval hints") - if case.track not in {"web", "dynamic", "challenge", "risk_control", "document", "image", "video", "file"}: + if case.track not in {"web", "dynamic", "challenge", "risk_control", "articles", "document", "image", "video", "file"}: raise ValueError("fixed native calls require a declared task route") if case.track == "document" and case.task_kind == "resource": raise ValueError("fixed native calls require a declared task route") @@ -198,6 +200,9 @@ def begin_call(self) -> None: def build_provider(name: str, config: dict[str, Any]) -> Provider: provider_type = config.get("type") + if provider_type in {"brightdata_unlocker", "browser_use_fetch", "browserbase_fetch", "deepseek_fetch", "crawl4ai_cloud"}: + from harness.web_fetch.providers.hosted_fetch import HostedFetchProvider + return HostedFetchProvider(name, config) if provider_type in {"headless_html", "headless_markdown"}: from harness.web_fetch.providers.headless import HeadlessProvider return HeadlessProvider(name, config) diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py index 39d747daa..cba8d96c5 100644 --- a/harness/web_fetch/providers/cloudflare.py +++ b/harness/web_fetch/providers/cloudflare.py @@ -461,6 +461,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict | No "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, **({'native_answer_projection': native_answer_projection('cloudflare_markdown', case, payload, response.status, interface='/markdown')} if (envelope_success and isinstance(payload.get('result', ''), str)) or meta else {}), "route": route, @@ -523,7 +524,7 @@ def _fetch_media_resources( error_type, error_message = _interface_error( response_status=response.status, envelope_success=envelope_success, - origin_status=None, + origin_status=_origin_status(meta.get('status')), ) errors = payload.get("errors") errors = errors if isinstance(errors, list) else [] @@ -539,7 +540,9 @@ def _fetch_media_resources( "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, "route": "browser_rendering_scrape_resources", + "upstream_status": _origin_status(meta.get('status')), **({'native_answer_projection': (native_answer_projection('cloudflare_markdown', case, payload, response.status))} if envelope_success or meta else {}), "browser": self.config.get("browser"), @@ -717,6 +720,7 @@ def _fetch_direct_document( "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, **({'native_answer_projection': {'schema_version': 2, 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True, 'result_fields': {'final_url': {'format': 'transport_response_url'}}}} @@ -784,6 +788,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, **({'native_answer_projection': native_answer_projection('cloudflare_content', case, payload, response.status)} if (envelope_success and isinstance(payload.get('result', ''), str)) or meta else {}), "output_contract": "rendered_html", diff --git a/harness/web_fetch/providers/exa.py b/harness/web_fetch/providers/exa.py index 011fc817b..ccde55c34 100644 --- a/harness/web_fetch/providers/exa.py +++ b/harness/web_fetch/providers/exa.py @@ -143,6 +143,8 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, + "upstream_status": None, "empty_content": 200 <= response.status < 300 and not content, "request_id": payload.get("requestId"), "url_status": public_status, diff --git a/harness/web_fetch/providers/firecrawl.py b/harness/web_fetch/providers/firecrawl.py index 504b4d8fb..fd546bdd2 100644 --- a/harness/web_fetch/providers/firecrawl.py +++ b/harness/web_fetch/providers/firecrawl.py @@ -401,6 +401,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, "upstream_status": ( metadata.get("statusCode") if isinstance(metadata.get("statusCode"), int) else None diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py new file mode 100644 index 000000000..cc0aa6eef --- /dev/null +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -0,0 +1,218 @@ +"""URL-only hosted Markdown retrieval; no Agent or browser-session fallback.""" +from __future__ import annotations + +import base64 +import hashlib +import uuid + +from lexbench_webfetch.models import EvalCase +from harness.record_io import parse_utf8_json +from harness.web_fetch.models import ProviderResult +from harness.web_fetch.native_routes import NativeRoute +from harness.web_fetch.native_projection import native_answer_projection +from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode +from harness.web_fetch.providers.base import Provider +from harness.web_fetch.providers.http_client import inner_timeout_budget, request + + +ENDPOINTS = { + "crawl4ai_cloud": "https://api.crawl4ai.com/scrape", + "brightdata_unlocker": "https://api.brightdata.com/request", + "browser_use_fetch": "https://fetch.browser-use.com/fetch", + "browserbase_fetch": "https://api.browserbase.com/v1/fetch", + "deepseek_fetch": "https://api.deepseek.com/v1/tools/fetch", +} + + +def _http_status(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and 100 <= value <= 599 else None + + +class HostedFetchProvider(Provider): + # The shared runner owns declared failure retries; the adapter calls once. + native_configuration_fields = (*Provider.native_configuration_fields, + 'render', 'debug', 'proxies', 'proxy_country', 'engine', 'proxy', 'output_format') + + def __init__(self, name, config): + super().__init__(name, config) + self._session_id = str(uuid.uuid4()) + + def begin_call(self): + self._session_id = str(uuid.uuid4()) + + def native_route(self, case): + fields = {'brightdata_unlocker': ('response_body',), 'browser_use_fetch': ('body', 'final_url'), + 'browserbase_fetch': ('content',), 'deepseek_fetch': ('content', 'title'), + 'crawl4ai_cloud': ('markdown',)} + return NativeRoute('http', ENDPOINTS[self.type], fields[self.type]) + + def validate(self) -> list[str]: + errors = [] if self.config.get("api_key") else ["api_key"] + if self.type == "brightdata_unlocker" and not self.config.get("zone"): + errors.append("zone") + if self.type == "brightdata_unlocker" and "render" in self.config and not isinstance(self.config["render"], bool): + errors.append("render must be boolean") + if self.type == "brightdata_unlocker" and "debug" in self.config and not isinstance(self.config["debug"], bool): + errors.append("debug must be boolean") + try: + validate_dns_mode(self.config.get("dns_mode", "system")) + except ValueError as error: + errors.append(str(error)) + if self.type == "browserbase_fetch" and not isinstance(self.config.get("proxies", False), bool): + errors.append("proxies must be boolean") + if self.type == "crawl4ai_cloud": + if self.config.get("engine", "browser") not in ("auto", "browser-light", "browser"): + errors.append("unsupported engine") + if self.config.get("proxy", "none") not in ("none", "isp", "residential"): + errors.append("unsupported proxy") + output_format = self.config.get("output_format", "markdown") + if output_format != "markdown": + errors.append("unsupported output_format; this adapter requires markdown") + return errors + + def native_http_arguments(self, case, timeout): + budget = inner_timeout_budget(timeout, native_default=True) + output_format = self.config.get("output_format", "markdown") + if output_format != "markdown": + raise ValueError("This adapter requires markdown output") + key = str(self.config.get("api_key", "")) + body: dict = {"url": case.url} + if self.type == "brightdata_unlocker": + headers = {"Authorization": f"Bearer {key}"} + body.update(zone=self.config["zone"], format="raw", data_format="markdown") + # Native Web Unlocker option, applied uniformly from the declared + # config. It adds no caller-side browser, wait, or follow-up fetch. + if "render" in self.config: + body["render"] = "true" if self.config["render"] else "false" + if "debug" in self.config: + body["debug"] = self.config["debug"] + elif self.type == "browserbase_fetch": + headers = {"X-BB-API-Key": key} + body.update(format=output_format, allowRedirects=True, + proxies=self.config.get("proxies", True)) + elif self.type == "crawl4ai_cloud": + headers = {"Authorization": f"Bearer {key}"} + body.update(format="md", engine=self.config.get("engine", "browser"), + proxy=self.config.get("proxy", "none")) + elif self.type == "deepseek_fetch": + headers = {"Authorization": f"Bearer {key}"} + else: + headers = {"X-Browser-Use-API-Key": key} + body.update(method="GET", output_format=output_format, + follow_redirects=True, max_redirects=10, + proxy_country=self.config.get("proxy_country", "US"), + session_id=self._session_id) + return self.config.get("endpoint", ENDPOINTS[self.type]), self.native_http_options(body, headers, budget) + + def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> ProviderResult: + self.check_native_call(case) + if case.task_kind not in {"body", "structure"} or case.track not in {"web", "dynamic", "challenge", "risk_control", "articles"}: + return ProviderResult(False, "", error_type="unsupported_task", + error_message="This adapter supports webpage text tasks only") + # Direct diagnostic invocations also get a fresh session. + if self.type == "browser_use_fetch": + self.begin_call() + budget = inner_timeout_budget(timeout, native_default=True) + endpoint, arguments = self.native_http_arguments(case, budget) + body = arguments['json_body'] + output_format = self.config.get("output_format", "markdown") + dns_mode = self.config.get("dns_mode", "system") + with public_dns_context(dns_mode, budget) as dns_trace: + response = self.native_http_request(case, request, endpoint, + native_route_ledger=native_route_ledger, **arguments) + metadata = { + "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), + "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), + "raw_response_bytes": len(response.body), + "api_http_status": response.status, "output_contract": output_format, + "dns_mode": dns_mode, "dns_trace": dns_trace, + "client_retries": 0, "retrieval_hints_applied": False, + "service_retry_policy": "provider_default_not_overridden", + "request_timeout_seconds": budget, + } + # API envelopes are evidence, never page content. Bright Data's raw + # mode deliberately returns the selected Markdown directly. + api_ok = 200 <= response.status < 300 + content, status, final_url, title = "", None, None, None + malformed = False + service_error = False + if self.type == "brightdata_unlocker": + try: + content = response.body.decode("utf-8") if api_ok else "" + except UnicodeDecodeError: + malformed = True + metadata["native_answer_projection"] = native_answer_projection( + self.type, case, {}, response.status) + metadata["upstream_http_status_available"] = False + else: + try: + payload = parse_utf8_json(response.body) + except ValueError: + payload = None + if not isinstance(payload, dict): + malformed = True + else: + if self.type == "browserbase_fetch": + content = payload.get("content") + status = _http_status(payload.get("statusCode", payload.get("status_code"))) + service_error = payload.get("encoding") == "base64" or bool(payload.get("error")) + elif self.type == "crawl4ai_cloud": + content = payload.get("markdown") + status = _http_status(payload.get("origin_status")) + service_error = (bool(payload.get("error")) or payload.get("success") is False + or payload.get("ok") is False) + metadata["upstream_http_status_available"] = status is not None + elif self.type == "deepseek_fetch": + content = payload.get("content") + title = payload.get("title") + service_error = bool(payload.get("error")) + metadata["upstream_http_status_available"] = False + else: + content = payload.get("body") + status = _http_status(payload.get("status_code")) + final_url = payload.get("final_url") if isinstance(payload.get("final_url"), str) else None + service_error = bool(payload.get("error")) or bool(payload.get("is_binary")) + metadata["request_id"] = payload.get("id") if isinstance(payload.get("id"), str) else None + malformed = (not isinstance(content, str) or + (not isinstance(title, str) if self.type == "deepseek_fetch" else + False if self.type == "crawl4ai_cloud" else status is None)) + content = content if isinstance(content, str) else "" + if self.type == "deepseek_fetch" and isinstance(payload.get("error"), dict): + metadata["service_error"] = payload["error"] + if not api_ok: + # Keep error envelopes in raw evidence, not in scored page text. + content = "" + error_type = ("access_denied" if response.status == 403 and self.type == "crawl4ai_cloud" else + "security_policy" if response.status == 403 and self.type == "deepseek_fetch" else + "authentication" if response.status in {401, 403} else + "rate_limit" if response.status == 429 else + "quota" if response.status == 402 else "api_error") + message = f"Fetch API returned HTTP {response.status}; inspect retained response" + elif service_error: + error_type, message = "api_error", "Fetch service reported an error or binary output; inspect retained response" + elif malformed: + error_type, message = "invalid_response", "Fetch API returned an invalid text response envelope" + elif status is not None and not 200 <= status < 300: + error_type, message = "http_error", f"Target page returned HTTP {status}" + elif not content.strip(): + error_type, message = "empty_content", "Fetch API returned empty Markdown" + else: + error_type = message = None + if self.type != "brightdata_unlocker" and isinstance(payload, dict): + metadata['native_answer_projection'] = native_answer_projection( + self.type, case, payload, response.status, interface=output_format) + if self.type == "crawl4ai_cloud": + metadata["request_parameters"] = body + metadata["native_browser_logs_available"] = False + if self.type == "browserbase_fetch": + metadata["proxies"] = self.config.get("proxies", True) + if self.type == "brightdata_unlocker": + metadata["render"] = self.config.get("render") + if self.type == "browser_use_fetch": + metadata["proxy_country"] = body["proxy_country"] + metadata['upstream_status'] = status + return ProviderResult( + ok=error_type is None, content=content, status_code=status, + final_url=final_url, title=title if isinstance(title, str) else None, + error_type=error_type, error_message=message, metadata=metadata, + ) diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py index 5d1d5fdf1..037093a63 100644 --- a/harness/web_fetch/providers/moli.py +++ b/harness/web_fetch/providers/moli.py @@ -752,6 +752,11 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> error_message="Moli fetch exceeded time budget", metadata=metadata, ) + native_status, readiness_timed_out = _native_fetch_diagnostics(diagnostic_raw, case.url) + metadata['api_http_status'] = None + metadata['upstream_status'] = native_status + if native_status is not None: + metadata['http_status_source'] = 'same_call_native_document_event' if raw_task: self._retain_source(metadata, raw, case) if route == "native_document_bytes": @@ -776,12 +781,9 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> metadata=metadata, ) self._retain_source(metadata, raw, case) - native_status, readiness_timed_out = _native_fetch_diagnostics(diagnostic_raw, case.url) http_context = (replay_native_http_error_context(raw, diagnostic_raw, case.url, process.returncode, {'schema_version': 1, 'format': 'fetch_log_http_error'}) if process.returncode == 0 and native_status is not None and 400 <= native_status < 600 else None) - if native_status is not None: - metadata["http_status_source"] = "same_call_native_document_event" if readiness_timed_out: metadata["native_readiness_timed_out"] = True if process.returncode == 0 and not content.strip() and native_status not in range(400, 600): diff --git a/harness/web_fetch/providers/tavily.py b/harness/web_fetch/providers/tavily.py index dbfd4c021..ef279e9ea 100644 --- a/harness/web_fetch/providers/tavily.py +++ b/harness/web_fetch/providers/tavily.py @@ -92,6 +92,8 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, + "upstream_status": None, "empty_content": 200 <= response.status < 300 and not content, "response_time": ( payload.get("response_time") diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py index 0b0125729..048c63304 100644 --- a/harness/web_fetch/report_build.py +++ b/harness/web_fetch/report_build.py @@ -414,6 +414,9 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, presentation_binding: Path | None = None, profile_path: Path | None = None) -> Path: include_english = includes_english(language, include_english) source, output = source.resolve(), output.resolve() + manifest_path = contained_path(source, 'manifest.json') + if manifest_path.is_file() and json.loads(manifest_path.read_text()).get('collection_only'): + raise ValueError('Collection-only evidence has no scores; explicitly rescore before building a scored report') if source == output or output in source.parents: raise ValueError('Report output must not contain its evidence source') marker = '.generated-report' diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index c815b5177..924234d05 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -33,6 +33,7 @@ PUBLIC_CALL_LOG_ROOT, diagnostic_log_binding, file_scoring_metadata, + materialize_task_references, raw_response_binding, verify_record_artifacts, verify_record_private_artifacts, @@ -55,7 +56,7 @@ from harness.web_fetch.network_preflight import preflight_guarded_providers from harness.web_fetch.providers import Provider from harness.web_fetch.publication_privacy import redact_url_queries -from harness.web_fetch.reference_inputs import benchmark_input_path, read_dataset_bytes +from harness.web_fetch.reference_inputs import benchmark_input_path, read_dataset_bytes, input_tree from harness.web_fetch.reference_lifecycle import ( ReferenceLifecycleError, check_case_reference_binding, @@ -354,7 +355,8 @@ def persist_public_call_log( "events": [ {"stage": "call_started", "at": started_at, "status_code": None, "error_type": None}, - {"stage": "exception_raised" if exception is not None else "response_received", + {"stage": ("exception_raised" if exception is not None else + "answer_deadline_exceeded" if metadata.get("caller_deadline_exceeded") else "response_received"), "at": completed_at, "status_code": result.status_code, "error_type": _public_text(result.error_type, limit=256)}, {"stage": "call_completed", "at": completed_at, @@ -408,7 +410,7 @@ def persist_result_artifacts( require_native_projection_schema(result.metadata['native_answer_projection']) raw = result.metadata.get('_raw_response_b64') if result.metadata.get('native_answer_projection', {}).get('source_format') == 'bytes': - raw = result.metadata.get('_raw_stdout_b64') + raw = result.metadata.get('_raw_stdout_b64', raw) needs_projection = (result.metadata.get('native_answer_required') or result.metadata.get('native_answer_projection') or any(result.metadata.get(key) for key in ('links', 'images', 'videos', 'media', 'extras')) @@ -534,6 +536,8 @@ def run_evaluation( if not cases or not providers: raise ValueError('Execution requires at least one task and provider') for provider in providers: + if max_failure_retries and not provider.allow_failure_retries: + raise ValueError(f"provider {provider.name!r} prohibits caller failure retries") if provider.config.get("uniform_native_parameters"): if runs != 1: raise ValueError("fixed native calls require one run") @@ -570,6 +574,9 @@ def run_evaluation( ) network_audit = preflight_guarded_providers(providers, cases) output_dir.mkdir(parents=True, exist_ok=True) + materialize_task_references( + output_dir, {name: input_tree(dataset_path, name) for name in ("sources", "list-references")}, + [case.id for case in cases]) if network_audit is not None: (output_dir / 'network-preflight.json').write_text( json.dumps(network_audit, ensure_ascii=False, indent=2) + '\n') @@ -688,7 +695,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> raise ReferenceLifecycleError("Frozen rules changed during the run", "reference_changed") else: check_case_reference_fresh(case, dataset_path, binding) - # Admission waits before the answer timer and 45-second budget. + # Admission waits before the per-call answer timer. admission_token = (case_admission.acquire(provider, case) if case_admission is not None else None) pending = None @@ -696,6 +703,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> try: started = utc_now() physical_attempts = [] + sequence_begin = time.monotonic() stopped_provider = False for physical_attempt in range(1, max_failure_retries + 2): retry_throttle_wait_ms = 0.0 @@ -833,6 +841,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> and pending.is_alive()) if physical_passed or stopped_provider or physical_attempt > max_failure_retries: break + result.metadata['call_sequence_elapsed_ms'] = round((time.monotonic() - sequence_begin) * 1000, 2) elapsed_ms = physical_attempts[-1]['elapsed_ms'] result.metadata['call_log'] = physical_attempts[-1]['call_log'] if len(physical_attempts) > 1: diff --git a/tests/fixtures/native-route-baseline.json b/tests/fixtures/native-route-baseline.json index d790c77af..502eb5af5 100644 --- a/tests/fixtures/native-route-baseline.json +++ b/tests/fixtures/native-route-baseline.json @@ -1,74 +1,74 @@ { - "moli_webfetch:defaults": { + "cloudflare_content:configured": { "cells": 192, - "sha256": "6fec640b5795d2ca3feb16f7e37133a4f68268cae0d781a12e7b14fcb31d3161" + "sha256": "ec644a8762031c4c441c8ed14529756f05f0d3be91435ee11695ec3ba6282da3" }, - "moli_webfetch:configured": { + "cloudflare_content:defaults": { "cells": 192, - "sha256": "dbb8c986b11f620db162a867b52b592cf546efc06190ebe31e6ece5416971be7" + "sha256": "c64bd0485c46987d8582d0d053a0a50bc5991c1ff8b21f2b783f9d427aee6b04" }, - "firecrawl:defaults": { + "cloudflare_markdown:configured": { "cells": 192, - "sha256": "bac3e056691510cb5f88df724fd47c5a1ea781218fd011630a886eec269254ab" + "sha256": "04973a0ce1b58c32ff9d5b0694bc3bcdf1f4551b25ecc4cedf92b818f00b511e" }, - "firecrawl:configured": { + "cloudflare_markdown:defaults": { "cells": 192, - "sha256": "006bffafd9c368cd0bd9596b7b03612939ff67aa73dff08c53f57527e66e3b57" + "sha256": "6f9b8c5af5d5e3f95c98649509c765931bfd6fda1f7f7c5c617708f378e6650f" }, - "tavily_extract:defaults": { + "cowagent_builtin:configured": { "cells": 192, - "sha256": "408ccbcc11fbad17bdbbf3b334d297f12965b9d1cb16a58a2ceb9b527a14be7d" + "sha256": "5080a60969a9d03b522a7dbe7ee66987305a8e5e984a42e34fa86ca6369e511c" }, - "tavily_extract:configured": { + "cowagent_builtin:defaults": { "cells": 192, - "sha256": "873427fd3e108f7373c2899c4dd248c50fa656596bb971845b073f0a8bd27643" + "sha256": "38706ec0f43b0fcd87d90ee7d06dbdda7a953510f46b5cc931030aef56755f1e" }, - "exa_contents:defaults": { + "exa_contents:configured": { "cells": 192, - "sha256": "31a93b9ba5f115da7baed76635b0640aa0fff9280afa67cd997d445069536f01" + "sha256": "90a121a621655a984f8d8982e2cd8ddac165ba569f2ecfbb1b8cc2bcadca20b6" }, - "exa_contents:configured": { + "exa_contents:defaults": { "cells": 192, - "sha256": "a78b34793b3093bb5df6f40413c1e93fc84cdcfc28e6fd28bfcc54461cb029f8" + "sha256": "48947f9e9d8ebb330f2a67c97ad4630aca187684de263761eeda715f4be5af32" }, - "cloudflare_markdown:defaults": { + "firecrawl:configured": { "cells": 192, - "sha256": "b8e77a3eee45420f81af15ef0cdc42147b18b66fded2adb491c04b69e97058c6" + "sha256": "a2a8e2410d95952953c99885feb03b8f7b5979b6248314a3c346e747c2300234" }, - "cloudflare_markdown:configured": { + "firecrawl:defaults": { "cells": 192, - "sha256": "39aa7bcf98a1470c5d70c8e37ad34182fb57496f2d91f0083d0954e4c4ad46b2" + "sha256": "216bf864f7084166e9af5bf41161601cddd73fbad05810da73436e73e7258583" }, - "cloudflare_content:defaults": { + "fixture:configured": { "cells": 192, - "sha256": "9c7e378c1e711e20484c3202b776c72e562e59c4116a199be257a34d2932be50" + "sha256": "dd5ce33ed287440013067581375dc1153533c4cea5b3c8333b2e3d9ad481cd3b" }, - "cloudflare_content:configured": { + "fixture:defaults": { "cells": 192, - "sha256": "c506bcca75ac9aa41e2a9031c131a90a54069b38f2c9b1f9ede037fbc0b8b2f6" + "sha256": "78433820449839127a1dd7be17702f9ed5362d549726d90271c7f9d36bf220f4" }, - "subprocess_jsonl:defaults": { + "moli_webfetch:configured": { "cells": 192, - "sha256": "86da2f053e55c6b4340e086b6cc4e41f84b11012deeaf63347ccaac4116d38d3" + "sha256": "b023f9c2858150c61eeba2ed93b3196ff1798089a27b4e827be913ab6ef5a8eb" }, - "subprocess_jsonl:configured": { + "moli_webfetch:defaults": { "cells": 192, - "sha256": "0b4b91b0fa3eb0182dc96fdb7b9a18645b721a061b589e1f688f2d7e5f048282" + "sha256": "808df49bcc4afef466e6dfff35fbca4523917867ff8c68f46707b9b742e02621" }, - "cowagent_builtin:defaults": { + "subprocess_jsonl:configured": { "cells": 192, - "sha256": "6f10daaff92e0fb4961c9af4be060f4ed2468d9fb8d105dc107cb6201e883211" + "sha256": "8f3ff37d6cd1c0f471bbf1b5e938b8ce476ecb71ba17c1ce47ba530e7c119cae" }, - "cowagent_builtin:configured": { + "subprocess_jsonl:defaults": { "cells": 192, - "sha256": "218d41a478b61c9e0a113feea5906eb046a35dde14d10d51d7501f13694f22b4" + "sha256": "a19a5ca05d634724c8e65cb65225810d6cef2ecf6455a67784ac538bfd18e5e8" }, - "fixture:defaults": { + "tavily_extract:configured": { "cells": 192, - "sha256": "323c3be505e72c0fb53d8fa100a550518addb09f7d9ed5c7813e1d2701f0a70d" + "sha256": "c9fc284b3d80875575308c5da96de1b42ec298a4d3cdf32ec9bff929051c0090" }, - "fixture:configured": { + "tavily_extract:defaults": { "cells": 192, - "sha256": "14644ca74d3ad320df58973c8334fc5adb35cba011fab8f42b91b216cec9465b" + "sha256": "c9d62d9acd2b67c66bf4388473457d5a8da1aab2ee0acf024c837b7cbbd810d0" } } diff --git a/tests/test_accounts.py b/tests/test_accounts.py index 593d2bf87..01b1934bb 100644 --- a/tests/test_accounts.py +++ b/tests/test_accounts.py @@ -74,8 +74,9 @@ def test_cloudflare_retains_both_product_permissions(): def test_unimplemented_candidates_remain_marked_planned(): entries = account_checklist() planned = {item['id'] for item in entries if not item['implemented']} - assert 'browserbase' in planned - assert len(planned) == 12 + assert 'scrapingbee' in planned + assert not {'browserbase', 'brightdata', 'browser-use-fetch', 'deepseek-fetch-api'} & planned + assert len(planned) == 10 catalog = report_method_catalog()['methods'] assert all(catalog[name]['status'] == 'planned' for name in planned) diff --git a/tests/test_assembly_evidence.py b/tests/test_assembly_evidence.py index d3ea7d9c8..f0857ba3f 100644 --- a/tests/test_assembly_evidence.py +++ b/tests/test_assembly_evidence.py @@ -521,7 +521,7 @@ def fetch(self, case, timeout): (root / 'execution.json').write_text(json.dumps({'batches': [manifest]})) (root / 'manifest.json').unlink() for folder in ('sources', 'list-references'): - (root / folder).mkdir() + (root / folder).mkdir(exist_ok=True) (root / 'reference-checks/private-note.txt').write_text('synthetic private note') output = tmp_path / 'assembled' relative = manifest['reference_checks']['postcheck']['evidence_path'] diff --git a/tests/test_dynamic_provider_waits.py b/tests/test_dynamic_provider_waits.py index 391ac36fb..1fe98c964 100644 --- a/tests/test_dynamic_provider_waits.py +++ b/tests/test_dynamic_provider_waits.py @@ -35,7 +35,10 @@ def dynamic_case(**completion_overrides): def test_release_configs_select_firecrawl_actions_route(): for filename in ("configs/providers.example.json", "configs/providers.resources.json"): providers = json.loads(Path(filename).read_text())["providers"] - assert providers["firecrawl"]["dynamic_wait_strategy"] == "actions" + if providers["firecrawl"].get("uniform_native_parameters"): + assert "dynamic_wait_strategy" not in providers["firecrawl"] + else: + assert providers["firecrawl"]["dynamic_wait_strategy"] == "actions" assert providers["firecrawl-resources"]["dynamic_wait_strategy"] == "actions" assert providers["firecrawl"]["proxy"] == "auto" assert providers["firecrawl-resources"]["proxy"] == "auto" diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py new file mode 100644 index 000000000..241e7ced8 --- /dev/null +++ b/tests/test_hosted_fetch.py @@ -0,0 +1,393 @@ +import base64 +import hashlib +import json +import tempfile +import unittest +import threading +import urllib.error +from pathlib import Path +from unittest.mock import Mock, patch + +from harness.web_fetch.diagnostics import CallDiagnostics +from harness.web_fetch.methods import method_info +from lexbench_webfetch.models import EvalCase +from harness.web_fetch.providers import build_provider +from harness.web_fetch.providers.http_client import HttpResponse +from harness.web_fetch.providers.hosted_fetch import ENDPOINTS +from harness.web_fetch.runner import run_evaluation +from lexbench_webfetch.metrics import task_passed + + +TYPES = ('brightdata_unlocker', 'browser_use_fetch', 'browserbase_fetch', + 'deepseek_fetch', 'crawl4ai_cloud') + + +class HostedFetchTest(unittest.TestCase): + def setUp(self): + self.case = EvalCase('article', 'Article', 'https://example.com', 'static', ['Required article text']) + + def provider(self, kind, **config): + return build_provider(kind, {'type': kind, 'api_key': 'private-test-key', 'zone': 'test-zone', **config}) + + def response(self, kind, text='Required article text', status=200): + payload = ({'body': text, 'status_code': status, 'final_url': self.case.url} + if kind == 'browser_use_fetch' else {'markdown': text, 'origin_status': status} + if kind == 'crawl4ai_cloud' else {'content': text, 'title': 'Title', 'statusCode': status}) + body = text.encode() if kind == 'brightdata_unlocker' else json.dumps(payload).encode() + return HttpResponse(200, 'https://service.example/', {}, body) + + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_explicit_service_errors_do_not_require_success_fields(self, call): + for kind in TYPES[1:]: + for payload, expected in (({'error': 'blocked'}, 'api_error'), + ({}, 'invalid_response')): + with self.subTest(kind=kind, payload=payload): + raw = json.dumps(payload).encode() + call.return_value = HttpResponse(200, ENDPOINTS[kind], {}, raw) + result = self.provider(kind).fetch(self.case, 30) + self.assertEqual(result.error_type, expected) + self.assertEqual(result.content, '') + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), raw) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_hosted_invalid_utf8_retains_response_evidence(self, call): + self.case.source_benchmark['input_media_kind'] = 'html_page' + variants = [(kind, encoding) for kind in TYPES for encoding in + (['invalid'] if kind == 'brightdata_unlocker' else ['invalid', 'bom', 'utf16'])] + for kind, encoding in variants: + with self.subTest(kind=kind, encoding=encoding): + call.reset_mock() + raw = (b'Invalid UTF-8: \xff' if kind == 'brightdata_unlocker' else + self.response(kind, text='BROKEN').body.replace(b'BROKEN', b'\xff')) + if encoding == 'bom': + raw = b'\xef\xbb\xbf' + self.response(kind).body + elif encoding == 'utf16': + raw = self.response(kind).body.decode().encode('utf-16') + call.return_value = HttpResponse(200, ENDPOINTS[kind], {}, raw) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dataset = {'id': 'invalid-utf8', 'cases': [vars(self.case)]} + dataset_path = root / 'input.json' + dataset_path.write_text(json.dumps(dataset)) + records, _ = run_evaluation( + output_dir=root, dataset_meta=dataset, + dataset_path=dataset_path, cases=[self.case], + providers=[self.provider(kind, uniform_native_parameters=True)], + provider_config_path=Path('providers.json'), runs=1, + timeout=30, delay=0, max_failure_retries=0) + record = records[0] + self.assertFalse(record['ok']) + self.assertEqual(record['error_type'], 'invalid_response') + self.assertEqual((root / record['artifact_path']).read_bytes(), b'') + metadata = record['provider_metadata'] + self.assertEqual((root / metadata['raw_response_path']).read_bytes(), raw) + self.assertEqual(metadata['raw_response_sha256'], hashlib.sha256(raw).hexdigest()) + from harness.web_fetch.artifacts import verify_record_artifacts + verify_record_artifacts(root, record) + call.assert_called_once() + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_crawl4ai_cloud_errors_and_projection(self, call): + from harness.web_fetch.native_projection import require_native_answer_projection + provider = self.provider('crawl4ai_cloud') + self.assertEqual(provider.validate(), []) + for payload, status, expected in [ + ({'markdown':'Native body'}, 200, None), + ({'markdown':'Native body','origin_status':404}, 200, 'http_error'), + ({'markdown':'','error':'blocked'}, 403, 'access_denied'), + ({'content':'Wrong field'}, 200, 'invalid_response'), + ({'markdown':''}, 200, 'empty_content'), + ({'markdown':'body','success':False}, 200, 'api_error'), + ({'markdown':'body','ok':False}, 200, 'api_error'), + ({'success':False,'error':'blocked'}, 200, 'api_error'), + ({'ok':False}, 200, 'api_error'), + ]: + call.reset_mock() + raw=json.dumps(payload).encode() + call.return_value=HttpResponse(status,'https://api.crawl4ai.com/scrape',{},raw) + result=provider.fetch(self.case,45) + self.assertEqual(result.error_type,expected) + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']),raw) + call.assert_called_once() + require_native_answer_projection(result.content,result.metadata,raw, + title=result.title,final_url=result.final_url,request_url=self.case.url) + for option in ({'engine':'invalid'},{'proxy':'invalid'},{'output_format':'html'}): + self.assertTrue(self.provider('crawl4ai_cloud',**option).validate()) + for field in ('engine', 'proxy'): + for value in ([], {}, None, True, 1): + with self.subTest(field=field, value=value): + self.assertIn('unsupported ' + field, + self.provider('crawl4ai_cloud', **{field: value}).validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_latest_article_track_preserves_native_answer_and_rejects_tampering(self, call): + from harness.web_fetch.native_projection import require_native_answer_projection + case = EvalCase('article-new', 'Article', 'https://example.com/article', 'article', [], track='articles') + for kind in TYPES: + with self.subTest(kind=kind): + response = self.response(kind) + call.return_value = response + result = self.provider(kind).fetch(case, 45) + self.assertTrue(result.ok) + self.assertEqual(result.content, 'Required article text') + if kind != 'brightdata_unlocker': + args = dict(title=result.title, final_url=result.final_url, request_url=case.url) + require_native_answer_projection(result.content, result.metadata, response.body, **args) + with self.assertRaises(ValueError): + require_native_answer_projection('Changed answer', result.metadata, response.body, **args) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_routes_and_preserved_evidence(self, call): + for kind in TYPES: + with self.subTest(kind=kind): + call.reset_mock() + response = self.response(kind) + call.return_value = response + result = self.provider(kind).fetch(self.case, 45) + self.assertTrue(result.ok) + self.assertEqual(result.content, 'Required article text') + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), response.body) + self.assertNotIn('private-test-key', json.dumps(result.metadata)) + self.assertNotEqual(method_info(kind)['contract'], '未知') + call.assert_called_once() + body = call.call_args.kwargs['json_body'] + self.assertEqual(body['url'], self.case.url) + self.assertEqual(call.call_args.kwargs['timeout'], 45) + self.assertEqual(result.metadata['request_timeout_seconds'], 45) + self.assertEqual(result.metadata['dns_mode'], 'system') + self.assertEqual(result.metadata['dns_trace'], []) + self.assertEqual(result.metadata['service_retry_policy'], 'provider_default_not_overridden') + if kind == 'brightdata_unlocker': + self.assertEqual(body['data_format'], 'markdown') + self.assertEqual(body['format'], 'raw') + self.assertIsNone(result.status_code) + elif kind == 'browser_use_fetch': + self.assertNotIn('retry', body) + self.assertNotIn('timeout_ms', body) + self.assertEqual(body['output_format'], 'markdown') + session = body['session_id'] + self.provider(kind).fetch(self.case, 45) + self.assertNotEqual(session, call.call_args.kwargs['json_body']['session_id']) + elif kind == 'browserbase_fetch': + self.assertTrue(body['allowRedirects']) + self.assertTrue(body['proxies']) + self.assertNotIn('projectId', body) + elif kind == 'deepseek_fetch': + self.assertEqual(body, {'url': self.case.url}) + self.assertEqual(result.title, 'Title') + self.assertIsNone(result.status_code) + self.assertEqual(call.call_args.kwargs['headers'], {'Authorization': 'Bearer private-test-key'}) + else: + self.assertEqual(body, {'url': self.case.url, 'format': 'md', 'engine': 'browser', 'proxy': 'none'}) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_brightdata_native_render_option_is_explicit(self, call): + call.return_value = self.response('brightdata_unlocker') + for value in (None, True, False): + config = {} if value is None else {'render': value} + result = self.provider('brightdata_unlocker', **config).fetch(self.case, 45) + body = call.call_args.kwargs['json_body'] + self.assertEqual(body.get('render'), None if value is None else str(value).lower()) + self.assertEqual(result.metadata['render'], value) + self.assertNotIn('wait', body) + self.assertNotIn('selector', body) + self.assertIn('render must be boolean', self.provider('brightdata_unlocker', render='true').validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_only_brightdata_receives_documented_native_debug_flag(self, call): + for kind in TYPES: + call.return_value = self.response(kind) + self.provider(kind, log_level='debug', debug=True).fetch(self.case, 45) + body = call.call_args.kwargs['json_body'] + if kind == 'brightdata_unlocker': + self.assertIs(body['debug'], True) + else: + self.assertNotIn('debug', body) + self.assertIn('debug must be boolean', self.provider(TYPES[0], debug='true').validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_unsupported_output_formats_are_rejected_before_transport(self, call): + for kind, output in (('browserbase_fetch', 'json'), ('browser_use_fetch', 'structured')): + provider = self.provider(kind, output_format=output) + self.assertTrue(provider.validate()) + with self.assertRaisesRegex(ValueError, 'requires markdown'): + provider.fetch(self.case, 30) + call.assert_not_called() + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_http_200_is_not_enough(self, call): + for kind in TYPES: + call.return_value = self.response(kind, '') + result = self.provider(kind).fetch(self.case, 45) + self.assertFalse(result.ok) + self.assertEqual(result.error_type, 'empty_content') + for kind in TYPES[1:3]: + call.return_value = self.response(kind, 'Required article text', 403) + self.assertEqual(self.provider(kind).fetch(self.case, 45).error_type, 'http_error') + for raw in (b'not json', b'{"message":"Required article text"}', + b'{"content":{},"statusCode":200}'): + call.return_value = HttpResponse(200, 'https://service.example/', {}, raw) + self.assertFalse(self.provider(kind).fetch(self.case, 45).ok) + for kind, payload in ( + ('browser_use_fetch', {'body':'Required article text','status_code':200,'error':'failed'}), + ('browserbase_fetch', {'content':'Required article text','statusCode':200,'encoding':'base64'}), + ): + call.return_value = HttpResponse(200, 'https://service.example/', {}, json.dumps(payload).encode()) + self.assertFalse(self.provider(kind).fetch(self.case, 45).ok) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_api_errors_are_separate_from_content(self, call): + for kind in TYPES: + for status, label in ((401,'authentication'), (402,'quota'), (429,'rate_limit'), (500,'api_error')): + call.return_value = HttpResponse(status, 'https://service.example/', {}, b'{"error":{"message":"actual error"}}') + result = self.provider(kind).fetch(self.case, 45) + self.assertFalse(result.ok) + self.assertEqual(result.content, '') + self.assertEqual(result.error_type, label) + if kind == 'deepseek_fetch': + self.assertEqual(result.metadata['service_error']['message'], 'actual error') + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_unsupported_delivery_never_calls_service(self, call): + case = EvalCase('file', 'File', 'https://example.com/file.pdf', 'document', [], track='document', task_kind='file') + for kind in TYPES: + self.assertEqual(self.provider(kind).fetch(case, 45).error_type, 'unsupported_task') + self.assertIn('api_key', build_provider(kind, {'type':kind}).validate()) + call.assert_not_called() + + @patch('harness.web_fetch.providers.hosted_fetch.request') + @patch('harness.web_fetch.network_dns.lookup_public_dns') + def test_single_request_without_dns_fallback_or_task_assistance(self, lookup, call): + self.case.retrieval_hints.update(version=1, completion={ + 'selector':'#answer', 'minimum_count':10, 'timeout_ms':30000}) + for kind in TYPES: + with self.subTest(kind=kind): + call.reset_mock() + call.return_value = HttpResponse(503, 'https://service.example/', {}, b'Unavailable') + result = self.provider(kind).fetch(self.case, 45) + self.assertFalse(result.ok) + call.assert_called_once() + body = call.call_args.kwargs['json_body'] + self.assertNotIn('#answer', json.dumps(body)) + self.assertNotIn('retry', body) + self.assertFalse(result.metadata['retrieval_hints_applied']) + lookup.assert_not_called() + + @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') + def test_runner_scores_and_saves_hosted_methods(self, call): + def respond(request, **kwargs): + kind = next(kind for kind in TYPES if ENDPOINTS[kind] == request.full_url) + response = self.response(kind) + return Mock(status=response.status, headers={'x-brd-debug':'req_id=test-native-id; render=true'}, + read=Mock(return_value=response.body), geturl=Mock(return_value=response.url)) + call.return_value.open.side_effect = respond + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + reference = root/'sources/article/reference.txt' + reference.parent.mkdir(parents=True) + reference.write_text('Required article text') + self.case.source_benchmark.update(stable_annotation={ + 'anchor_evidence': 'sources/article/reference.txt', + 'anchor_evidence_sha256': hashlib.sha256(reference.read_bytes()).hexdigest(), + }) + self.case.acceptance_contract.update(version=1, schema='body_text', minimum_records=0, + stable_text_anchors=[{'id':'article','alternatives':['Required article text']}]) + dataset = {'tool':'web_fetch','id':'hosted-test','name':'Test','cases':[vars(self.case)], + 'coverage_plan':{'web':list(TYPES)}, + 'comparison':{'evidence_state':'diagnostic','kind':'web_fetch_diagnostic'}} + dataset_path = root/'dataset.json' + dataset_path.write_text(json.dumps(dataset)) + config_path = root/'providers.json' + config_path.write_text('{}') + records, _ = run_evaluation(output_dir=root/'run', dataset_meta=dataset, + dataset_path=dataset_path, cases=[self.case], providers=[self.provider(k, log_level='debug') for k in TYPES], + provider_config_path=config_path, runs=1, timeout=45, delay=0, provider_concurrency=1,max_failure_retries=0) + self.assertEqual(len(records), len(TYPES)) + self.assertTrue(all(task_passed(r['ok'],r['metrics'],'body') for r in records)) + for record in records: + self.assertTrue((root/'run'/record['provider_metadata']['raw_response_path']).is_file()) + log_path = root/'run'/record['provider_metadata']['diagnostic_log_path'] + raw_log = log_path.read_bytes() + self.assertEqual(hashlib.sha256(raw_log).hexdigest(), record['provider_metadata']['diagnostic_log_sha256']) + self.assertNotIn(b'private-test-key', raw_log) + for event in ('http_request_started', 'http_response_received', 'collection_completed'): + self.assertIn(event, raw_log.decode()) + self.assertIn('req_id=test-native-id', raw_log.decode()) + # Reporting must work from the saved run after original references + # disappear; it must never need another provider call. + reference.unlink() + from harness.web_fetch.report_build import build_report + report = build_report(root/'run', root/'run/report', diagnostic_preview=True, language='en') + self.assertTrue(report.is_file()) + self.assertEqual(call.call_count, len(TYPES)) + + def test_diagnostics_redact_new_auth_headers(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory)/'log.jsonl' + log = CallDiagnostics(path, 'test') + log.emit('caller','request',headers={'X-BB-API-Key':'secret-bb','X-Browser-Use-API-Key':'secret-bu'}) + log.file.close() + self.assertNotIn('secret-bb',path.read_text()) + self.assertNotIn('secret-bu',path.read_text()) + + @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') + def test_runner_retains_http_errors_and_exception_trace(self, opener): + responses = [Mock(status=200, headers={}, read=Mock(return_value=b'{"title":"Title","content":"Body"}'), + geturl=Mock(return_value='https://service.example/')), + Mock(status=500, headers={}, read=Mock(return_value=b'{"error":{"message":"native failure"}}'), + geturl=Mock(return_value='https://service.example/')), + urllib.error.URLError('synthetic DNS failure')] + opener.return_value.open.side_effect = responses + cases = [EvalCase(str(i),'Control',self.case.url,'static',[]) for i in range(3)] + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + rows,_=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(c) for c in cases]}, + dataset_path=root/'dataset.json', cases=cases, + providers=[self.provider('deepseek_fetch',log_level='debug')],provider_config_path=root/'providers.json', + runs=1,timeout=45,delay=0,max_failure_retries=0) + self.assertEqual([r['ok'] for r in rows],[True,False,False]) + self.assertEqual(opener.return_value.open.call_count,3) + for row in rows: + log=(root/'run'/row['provider_metadata']['diagnostic_log_path']).read_text() + self.assertNotIn('private-test-key',log) + self.assertIn('http_request_started',log) + failure_log=(root/'run'/rows[-1]['provider_metadata']['diagnostic_log_path']).read_text() + self.assertIn('synthetic DNS failure',failure_log) + self.assertIn('Traceback',failure_log) + server_log=(root/'run'/rows[1]['provider_metadata']['diagnostic_log_path']).read_text() + self.assertIn('native failure',server_log) + + def test_historical_unscored_collection_cannot_build_report(self): + from harness.web_fetch.report_build import build_report + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root/'manifest.json').write_text('{"collection_only":true}') + with self.assertRaisesRegex(ValueError, 'Collection-only evidence'): + build_report(root, root/'report') + + @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') + def test_runner_timeout_has_actual_deadline_event_and_no_retry(self, opener): + release=threading.Event() + def blocked(*args,**kwargs): + release.wait(1) + raise TimeoutError('simulated socket timeout') + opener.return_value.open.side_effect=blocked + try: + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + rows,_=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(self.case)]}, + dataset_path=root/'dataset.json',cases=[self.case], + providers=[self.provider('deepseek_fetch',log_level='debug')],provider_config_path=root/'providers.json', + runs=1,timeout=0.02,delay=0,max_failure_retries=0) + self.assertEqual(rows[0]['error_type'],'timeout') + log=(root/'run'/rows[0]['provider_metadata']['diagnostic_log_path']).read_text() + self.assertIn('answer_timeout',log) + self.assertIn('End-to-end answer deadline exceeded',log) + self.assertEqual(opener.return_value.open.call_count,1) + finally: + release.set() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_interrupted_evidence.py b/tests/test_interrupted_evidence.py index ade1e9d21..2266b26f3 100644 --- a/tests/test_interrupted_evidence.py +++ b/tests/test_interrupted_evidence.py @@ -98,8 +98,8 @@ def test_adding_a_provider_preserves_the_declared_review_policy(self): run_evaluation(output_dir=output, dataset_meta=dataset, dataset_path=path, cases=[case], providers=[FixtureProvider(provider, {"type": "fixture"})], provider_config_path=config, runs=1, timeout=1, delay=0, max_failure_retries=0) - (output / "sources").mkdir() - (output / "list-references").mkdir() + (output / "sources").mkdir(exist_ok=True) + (output / "list-references").mkdir(exist_ok=True) runs.append(output) target = root / "assembled" assemble(runs[0], root / "unused", [], target, provider_additions=[runs[1]]) @@ -133,7 +133,7 @@ def test_case_addition_retains_every_certified_batch_and_call_log(self): manifest=json.loads((output/'manifest.json').read_text()) (output/'execution.json').write_text(json.dumps({'batches':[manifest]})) (output/'manifest.json').unlink() - (output/'sources').mkdir();(output/'list-references').mkdir();runs.append(output) + (output/'sources').mkdir(exist_ok=True);(output/'list-references').mkdir(exist_ok=True);runs.append(output) target=root/'assembled' assemble(runs[0],root/'unused',[],target,additions=[runs[1]]) rows=parse_jsonl((target/'results.jsonl').read_bytes()) diff --git a/tests/test_native_route_contract.py b/tests/test_native_route_contract.py index d7fbc94d2..17964c69f 100644 --- a/tests/test_native_route_contract.py +++ b/tests/test_native_route_contract.py @@ -71,7 +71,7 @@ def _route_http_fixture(tmp_path): cases = [EvalCase(name, name, 'https://example.test/'+name, 'control', [], source_benchmark={'input_media_kind': 'html_page'}) for name in ('one', 'two')] - ledger = NativeRouteLedger(cases, [provider]) + ledger = NativeRouteLedger(cases, [provider], timeout_seconds=45) binding = ledger.write(tmp_path, dataset_sha256='dataset', runtime_source_sha256='runtime') manifest = {'native_routes_required': True, 'native_route_contract': binding, 'dataset_sha256': 'dataset', 'runtime_source_sha256': 'runtime', @@ -257,7 +257,7 @@ def test_custom_firecrawl_without_auth_still_matches_actual_request(tmp_path): 'uniform_native_parameters': True}) case = EvalCase('case', 'Case', 'https://example.test/', 'control', [], source_benchmark={'input_media_kind': 'html_page'}) - ledger = NativeRouteLedger([case], [provider]) + ledger = NativeRouteLedger([case], [provider], timeout_seconds=45) ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture') with patch('harness.web_fetch.providers.firecrawl.request', return_value=HttpResponse(200, case.url, {}, b'{}')) as transport: @@ -443,7 +443,7 @@ def test_http_native_answers_survive_persistence_and_reject_tampering(tmp_path, 'include_resource_links': True, 'rich_links_limit': 12, 'formats': ['markdown', 'images', 'links']}) case = EvalCase('case', 'Case', 'https://example.test/', 'control', [], track=track, task_kind=kind, source_benchmark={'input_media_kind': 'html_page'}) - ledger = NativeRouteLedger([case], [provider]) + ledger = NativeRouteLedger([case], [provider], timeout_seconds=45) ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture') with patch('harness.web_fetch.providers.'+module+'.request', return_value=HttpResponse(200, case.url, {}, json.dumps(payload).encode())): @@ -935,7 +935,7 @@ def test_explicit_ledgers_do_not_replace_each_others_provider_state(tmp_path): from unittest.mock import Mock from harness.web_fetch.native_routes import NativeRouteLedger provider, cases, first, _, _ = _route_http_fixture(tmp_path) - second = NativeRouteLedger(cases, [provider]) + second = NativeRouteLedger(cases, [provider], timeout_seconds=45) other = tmp_path / 'other' other.mkdir() second.write(other, dataset_sha256='dataset', runtime_source_sha256='runtime') diff --git a/tests/test_providers.py b/tests/test_providers.py index f8978e436..7bf3c0ae4 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -924,11 +924,17 @@ def test_release_configs_bind_strongest_tavily_and_exa_text_routes(self) -> None with self.subTest(filename=filename): self.assertEqual(providers["tavily"]["extract_depth"], "advanced") self.assertEqual(providers["tavily-resources"]["extract_depth"], "advanced") - self.assertEqual(providers["tavily"]["extract_timeout_seconds"], 60.0) + if providers["tavily"].get("uniform_native_parameters"): + self.assertNotIn("extract_timeout_seconds", providers["tavily"]) + else: + self.assertEqual(providers["tavily"]["extract_timeout_seconds"], 60.0) self.assertEqual(providers["tavily-resources"]["extract_timeout_seconds"], 60.0) self.assertEqual(providers["exa"]["text_verbosity"], "full") self.assertEqual(providers["exa-resources"]["text_verbosity"], "full") - self.assertEqual(providers["exa"]["livecrawl_timeout_ms"], 15_000) + if providers["exa"].get("uniform_native_parameters"): + self.assertNotIn("livecrawl_timeout_ms", providers["exa"]) + else: + self.assertEqual(providers["exa"]["livecrawl_timeout_ms"], 15_000) self.assertEqual(providers["exa-resources"]["livecrawl_timeout_ms"], 15_000) @patch("harness.web_fetch.providers.cloudflare.request") diff --git a/tests/test_public_call_logs.py b/tests/test_public_call_logs.py index e3df72116..b454b260e 100644 --- a/tests/test_public_call_logs.py +++ b/tests/test_public_call_logs.py @@ -2,6 +2,7 @@ import hashlib import json import tempfile +import threading import unittest from pathlib import Path @@ -20,11 +21,11 @@ ) -def run(root: Path, provider: Provider, *, retry: int = 0): +def run(root: Path, provider: Provider, *, retry: int = 0, timeout: float = 2): return run_evaluation( output_dir=root, dataset_meta={"id": "call-log-test"}, dataset_path=Path("dataset.json"), cases=[CASE], providers=[provider], - provider_config_path=Path("providers.json"), runs=1, timeout=2, delay=0, + provider_config_path=Path("providers.json"), runs=1, timeout=timeout, delay=0, max_failure_retries=retry, )[0][0] @@ -43,6 +44,56 @@ def fetch(self, case, timeout): class PublicCallLogTests(unittest.TestCase): + def test_native_timeout_is_not_a_caller_deadline(self): + result = ProviderResult(False, "", error_type="timeout", + error_message="Native readiness wait ended") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = run(root, ResultProvider(result), timeout=30) + log = load_log(root, record["provider_metadata"]["call_log"]) + self.assertEqual(log["events"][1]["stage"], "response_received") + self.assertEqual(log["error"]["type"], "timeout") + self.assertFalse(log["timing"]["deadline_overrun"]) + self.assertFalse(record["provider_metadata"]["caller_deadline_exceeded"]) + + def test_outer_deadline_is_recorded_and_late_answer_discarded(self): + class Waiting(Provider): + def __init__(self): + super().__init__("waiting", {"type": "fixture"}) + self.release = threading.Event() + + @property + def cancellation_cleanup_seconds(self): + return 1 + + def cancel(self): + self.release.set() + + def fetch(self, case, timeout): + self.release.wait(5) + return ProviderResult(True, "Expected") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = run(root, Waiting(), timeout=.02) + log = load_log(root, record["provider_metadata"]["call_log"]) + self.assertEqual(log["events"][1]["stage"], "answer_deadline_exceeded") + self.assertTrue(record["provider_metadata"]["caller_deadline_exceeded"]) + self.assertTrue(log["output"]["empty"]) + + def test_native_timeout_exception_keeps_its_origin(self): + class Broken(Provider): + def fetch(self, case, timeout): + raise TimeoutError("Native transport timed out") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = run(root, Broken("broken", {"type": "fixture"}), timeout=30) + log = load_log(root, record["provider_metadata"]["call_log"]) + self.assertEqual(log["events"][1]["stage"], "exception_raised") + self.assertEqual(log["error"]["message"], "Native transport timed out") + self.assertFalse(log["timing"]["deadline_overrun"]) + def test_common_sensitive_fields_are_redacted_from_public_log(self): secret_lines = "\n".join([ 'refresh_token=refresh-secret', diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py index 3708abdeb..92cfd95ac 100644 --- a/tests/test_report_methods.py +++ b/tests/test_report_methods.py @@ -269,3 +269,23 @@ def test_method_model_selects_direct_english_fields_without_reverse_translation( assert '1/1 records' in observed['configuration'] assert not re.search(r'[\u4e00-\u9fff]', observed['configuration']) + + +def test_hosted_interfaces_follow_adapter_identity_in_external_cohorts(): + from harness.web_fetch.report_interfaces import method_spec + from harness.web_fetch.methods import method_info + adapters = ('brightdata_unlocker', 'browser_use_fetch', 'browserbase_fetch', + 'deepseek_fetch', 'crawl4ai_cloud') + for adapter in adapters: + record = {'provider': 'external-participant', 'provider_type': adapter, + 'provider_metadata': {}} + for language in ('zh', 'en'): + spec = method_spec('external-participant', record, language=language) + assert spec['entry'] == method_info(adapter, language=language)['endpoint'] + assert spec['request']['url'] == 'https://example.com/' + assert spec['source'] == 'harness/web_fetch/providers/hosted_fetch.py' + assert spec['fields'] + assert 'API_KEY' in spec['auth'] + english = method_spec('external-participant', record, language='en') + assert 'historical call parameters' in english['notes'] + assert not any('\u3400' <= char <= '\u9fff' for char in str(english)) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index f7eddf2e8..5c2b0b01b 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -949,7 +949,7 @@ def setUp(self): self.path = self.root / "dataset.json" self.path.write_text(json.dumps(self.dataset)) refs = self.root / "list-references" - refs.mkdir() + refs.mkdir(exist_ok=True) for name in ("legacy", "unverified"): (refs / f"{name}.json").write_text(json.dumps({**REFERENCE, "case_id": name})) texts = {"web": "Alpha\r\n", "dynamic": "Catalog\nCategory filter\n" + LINKS, @@ -985,7 +985,7 @@ def live(self): def prepare_script_references(self): refs = self.run / "list-references" - refs.mkdir() + refs.mkdir(exist_ok=True) for name in ("legacy", "unverified"): (refs / f"{name}.json").write_bytes((self.root / "list-references" / f"{name}.json").read_bytes()) diff --git a/tests/test_selected_native_policy.py b/tests/test_selected_native_policy.py new file mode 100644 index 000000000..6b6c6abb4 --- /dev/null +++ b/tests/test_selected_native_policy.py @@ -0,0 +1,246 @@ +"""Offline behavior checks for the selected ten-method configuration.""" +import json +import time +from dataclasses import asdict, replace +from pathlib import Path +from unittest.mock import patch, Mock + +import pytest + +from lexbench_webfetch.models import EvalCase +from harness.web_fetch.cli import build_parser +from harness.web_fetch.providers.base import build_provider +from harness.web_fetch.native_routes import normalized_http_parameters, verify_saved_native_answer +from harness.web_fetch.runner import run_evaluation + +SELECTED = ('moli-webfetch', 'firecrawl', 'cloudflare-markdown', 'tavily', 'exa', + 'brightdata', 'browser-use-fetch', 'browserbase', 'deepseek-fetch-api', 'crawl4ai-cloud') +CONFIG = json.loads(Path('configs/providers.example.json').read_text())['providers'] + + +def provider(name): + return build_provider(name, {**CONFIG[name], 'api_key': 'offline-key', + 'api_token': 'offline-token', 'account_id': 'offline-account', 'zone': 'offline-zone'}) + + +def case(): + return EvalCase('one', 'One', 'https://example.test/', 'static', ['Expected'], + source_benchmark={'input_media_kind': 'html_page'}) + + +def test_defaults(): + args = build_parser().parse_args(['run']) + assert args.timeout == 30 + assert args.max_failure_retries == 1 + + +@pytest.mark.parametrize('name', SELECTED) +def test_selected_methods_reject_case_assistance_before_transport(name): + p = provider(name) + assert p.config['uniform_native_parameters'] is True + hinted = replace(case(), retrieval_hints={'completion': {'selector': '#answer'}}) + with pytest.raises(ValueError, match='per-case'): + p.fetch(hinted, 30) + + +@pytest.mark.parametrize('name', SELECTED[1:]) +def test_request_is_independent_of_grading_requirements(name): + p = provider(name) + first = case() + second = replace(first, expected_any=['Different answer'], acceptance_contract={ + 'schema': 'named_link', 'required_links': [{'url': 'https://answer.test/'}]}) + endpoint, args = p.native_http_arguments(first, 30) + other_endpoint, other = p.native_http_arguments(second, 30) + assert endpoint == other_endpoint + assert normalized_http_parameters(first, args) == normalized_http_parameters(second, other) + assert args['timeout'] == 30 + if name == 'exa': + assert args['json_body']['extras']['richLinks'] == 100 + if name == 'cloudflare-markdown': + document = replace(first, url='https://example.test/file.pdf', track='document') + doc_endpoint, doc_args = p.native_http_arguments(document, 30) + assert doc_endpoint.endswith('/markdown') + assert doc_args['json_body']['url'] == document.url + assert 'multipart' not in doc_args + + +@pytest.mark.parametrize('name', SELECTED[5:]) +@pytest.mark.parametrize('answers', [('Expected',), ('wrong', 'Expected'), ('wrong', 'still wrong'), ('HTTP_ERROR', 'Expected')]) +def test_hosted_native_retry_persists_every_physical_answer(tmp_path, name, answers): + p, c = provider(name), case() + texts = iter(answers) + def respond(request, **kwargs): + text = next(texts) + payload = ({'body': text, 'status_code': 200, 'final_url': c.url} if name == 'browser-use-fetch' + else {'markdown': text} if name == 'crawl4ai-cloud' + else {'content': text, 'title': 'Title', 'statusCode': 200}) + raw = text.encode() if name == 'brightdata' else json.dumps(payload).encode() + assert kwargs['timeout'] <= 30 + return Mock(status=503 if text == 'HTTP_ERROR' else 200, headers={}, read=Mock(return_value=raw), + geturl=Mock(return_value=request.full_url)) + with patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') as opener: + opener.return_value.open.side_effect = respond + rows, _ = run_evaluation(output_dir=tmp_path, dataset_meta={'id':'offline', 'cases':[asdict(c)]}, + dataset_path=tmp_path/'dataset.json', cases=[c], providers=[p], + provider_config_path=tmp_path/'providers.json', runs=1, timeout=30, delay=0) + assert opener.return_value.open.call_count == len(answers) + row = rows[0] + assert (tmp_path/row['artifact_path']).read_text() == answers[-1] + assert row['provider_metadata']['latency_scope'] == 'native_response_received' + assert (tmp_path/row['provider_metadata']['raw_response_path']).is_file() + assert (tmp_path/row['provider_metadata']['diagnostic_log_path']).is_file() + verify_saved_native_answer(tmp_path, row, cases_by_id={c.id: asdict(c)}) + # Raw responses and caller logs remain separate for each paid physical call. + assert len(list(tmp_path.rglob('*.response.bin'))) == len(answers) + + +def test_native_preparation_does_not_consume_response_budget(): + from harness.web_fetch.call_boundary import fetch_with_deadline, prepare_answer + from harness.web_fetch.models import ProviderResult + p = build_provider('fixture', {'type':'fixture', 'uniform_native_parameters':True}) + p.fetch = Mock(return_value=ProviderResult(True, 'Expected')) + def slow_prepare(result, c): + time.sleep(.08) + return prepare_answer(result, c) + with patch('harness.web_fetch.call_boundary.prepare_answer', side_effect=slow_prepare): + result, _ = fetch_with_deadline(p, case(), .05) + assert result.ok + assert result.metadata['_answer_elapsed_ms'] < 50 + + +@pytest.mark.parametrize('name', SELECTED[6:]) +def test_malformed_api_response_is_retained_as_failure(tmp_path, name): + p, c = provider(name), case() + with patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') as opener: + opener.return_value.open.return_value = Mock(status=200, headers={}, + read=Mock(return_value=b'not valid JSON'), geturl=Mock(return_value='https://service.test/')) + rows, _ = run_evaluation(output_dir=tmp_path, dataset_meta={'id':'offline', 'cases':[asdict(c)]}, + dataset_path=tmp_path/'dataset.json', cases=[c], providers=[p], + provider_config_path=tmp_path/'providers.json', runs=1, timeout=30, delay=0) + assert opener.return_value.open.call_count == 2 + row = rows[0] + assert not row['ok'] and row['error_type'] == 'invalid_response' + assert (tmp_path/row['artifact_path']).read_text() == '' + assert (tmp_path/row['provider_metadata']['raw_response_path']).read_bytes() == b'not valid JSON' + verify_saved_native_answer(tmp_path, row, cases_by_id={c.id: asdict(c)}) + + +def test_moli_native_command_has_no_case_hint_or_reduced_timer(): + p = provider('moli-webfetch') + first = case() + second = replace(first, expected_any=['Another answer'], acceptance_contract={'schema':'named_link'}) + assert p.native_cli_command(first) == p.native_cli_command(second) + command = p.native_cli_command(first) + assert '--wait-until' in command # Fixed product-native completion option. + assert not any('timeout' in arg or 'selector' in arg for arg in command) + + +def test_configured_cohort_and_local_configuration_guards(): + from harness.web_fetch.config import resolve_provider_selection + names = resolve_provider_selection('configured', Path('configs/providers.example.json')).split(',') + assert tuple(names) == SELECTED + assert resolve_provider_selection('fixture', None) == 'fixture' + assert resolve_provider_selection(None, None) is None + for name in names: + assert CONFIG[name]['log_level'] == 'debug' + p = provider('moli-webfetch') + assert not p._configuration_errors('/offline/moli') + + +@pytest.mark.parametrize('names', [[], ['missing'], ['fixture', 'fixture']]) +def test_configured_cohort_rejects_invalid_selection(tmp_path, names): + from harness.web_fetch.config import resolve_provider_selection + config = tmp_path/'providers.json' + config.write_text(json.dumps({'providers':{'fixture':{'type':'fixture'}}, 'default_providers':names})) + with pytest.raises(ValueError): + resolve_provider_selection('configured', config) + + +def test_configured_cli_selection_reaches_runner(tmp_path): + from harness.web_fetch.cli import command_run + config = tmp_path/'providers.json' + config.write_text(json.dumps({'providers':{'fixture':{'type':'fixture'}}, 'default_providers':['fixture']})) + args = build_parser().parse_args(['run','--providers','configured','--provider-config',str(config), + '--output',str(tmp_path/'run')]) + with patch('harness.web_fetch.cli.load_runtime_env'), \ + patch('harness.web_fetch.cli.run_evaluation', return_value=([], {})) as run, \ + patch('harness.web_fetch.cli.build_report', return_value=tmp_path/'report'): + command_run(args) + assert [p.name for p in run.call_args.kwargs['providers']] == ['fixture'] + assert run.call_args.kwargs['timeout'] == 30 + assert run.call_args.kwargs['max_failure_retries'] == 1 + + +@pytest.mark.parametrize('name,route', [(name, 'body') for name in SELECTED] + + [('cloudflare-markdown', 'image'), ('cloudflare-markdown', 'video'), + ('moli-webfetch', 'file')]) +@pytest.mark.parametrize('origin_status', [200, 404, None]) +def test_selected_origin_status_policy_controls_acceptance_and_retry(tmp_path, name, route, origin_status): + import hashlib + import sys + from lexbench_webfetch.metrics import task_passed + p, c = provider(name), case() + file_bytes = b'\x89PNG\r\n\x1a\n' + b'fixture bytes' * 8 + if route in {'image', 'video'}: + c = replace(c, track=route, task_kind='resource', expected_any=[], + expected_resources=[{'urls': ['https://example.test/asset.png' if route == 'image' + else 'https://example.test/asset.mp4'], 'kind': route}]) + elif route == 'file': + c = replace(c, task_kind='file', expected_any=[], + source_benchmark={'input_media_kind': 'direct_image'}, + source_verification={'mime': 'image/png', 'file_sha256': hashlib.sha256(file_bytes).hexdigest()}) + exposed = name in {'moli-webfetch', 'firecrawl', 'cloudflare-markdown', + 'browser-use-fetch', 'browserbase', 'crawl4ai-cloud'} + if name == 'moli-webfetch': + binary = tmp_path / 'moli-fixture' + event = (f'DEBUG fetch request complete method=GET url={c.url} ' + f'final_url={c.url} status={origin_status}') if origin_status is not None else '' + binary.write_text(f'#!{sys.executable}\nimport sys\n' + 'if sys.argv[1:] == ["--version"]:\n print("moli fixture"); sys.exit(0)\n' + + (f'sys.stdout.buffer.write({file_bytes!r})\n' if route == 'file' else 'print("Expected")\n') + + f'print({event!r}, file=sys.stderr)\n') + binary.chmod(0o700) + p = build_provider(name, {**p.config, 'binary': str(binary), + 'expected_reported_version': 'moli fixture', + 'expected_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest()}) + payloads = { + 'firecrawl': {'success': True, 'data': {'markdown': 'Expected', 'metadata': {'statusCode': origin_status}}}, + 'cloudflare-markdown': {'success': True, 'result': 'Expected', 'meta': {'status': origin_status}}, + 'tavily': {'results': [{'url': c.url, 'raw_content': 'Expected'}]}, + 'exa': {'results': [{'url': c.url, 'text': 'Expected'}]}, + 'browser-use-fetch': {'body': 'Expected', 'status_code': origin_status, 'final_url': c.url}, + 'browserbase': {'content': 'Expected', 'statusCode': origin_status}, + 'deepseek-fetch-api': {'content': 'Expected', 'title': 'Title'}, + 'crawl4ai-cloud': {'markdown': 'Expected', 'origin_status': origin_status}, + } + if route in {'image', 'video'}: + payloads[name]['result'] = [{'results': [{'attributes': [ + {'name': 'src', 'value': c.expected_resources[0]['urls'][0]}]}]}] + # The selected Browser Use and Browserbase schemas require status on success; + # missing status is malformed rather than an invented origin error. + malformed = origin_status is None and name in {'browser-use-fetch', 'browserbase'} + raw = b'Expected' if name == 'brightdata' else json.dumps(payloads.get(name, {})).encode() + def respond(request, **kwargs): + return Mock(status=200, headers={}, read=Mock(return_value=raw), + geturl=Mock(return_value=request.full_url)) + with patch('harness.web_fetch.network_preflight.socket.getaddrinfo', + return_value=[(2, 1, 6, '', ('93.184.216.34', 443))]), \ + patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') as opener: + opener.return_value.open.side_effect = respond + rows, _ = run_evaluation(output_dir=tmp_path / 'run', + dataset_meta={'id': 'origin-control', 'cases': [asdict(c)]}, + dataset_path=tmp_path / 'dataset.json', cases=[c], providers=[p], + provider_config_path=tmp_path / 'providers.json', runs=1, + timeout=30, delay=0, max_failure_retries=1) + row = rows[0] + expected_pass = not ((exposed and origin_status == 404) or malformed) + assert task_passed(row['ok'], row['metrics'], c.task_kind) == expected_pass + metadata = row['provider_metadata'] + assert metadata['upstream_status'] == (origin_status if exposed else None) + assert metadata['api_http_status'] == (None if name == 'moli-webfetch' else 200) + assert metadata['upstream_http_status_available'] == (exposed and origin_status is not None) + assert len(list((tmp_path / 'run' / 'call-logs').rglob('*.json'))) == (1 if expected_pass else 2) + from harness.web_fetch.artifacts import verify_record_artifacts + verify_record_artifacts(tmp_path / 'run', row) + if route == 'body': + assert 'Expected' in (tmp_path / 'run' / row['artifact_path']).read_text()