fix(records): honor network.tor in records search/fetch/download

CHANGELOG.md

@@ -30,6 +30,13 @@ exhaustive lists.
 
 ### Fixed
 
+- `network.tor: true` was ignored by `records search -s <source>` (every source
+  except cia), `records search --all-sources` (all sources, cia included),
+  `records fetch -s documentcloud`, `records fetch -s nara` without
+  `--objects`/`--list-objects`, and `records download -s muckrock`: they sent
+  your query or identifiers from your real IP with no Tor check. They now route
+  through Tor, refuse to run when Tor is configured but unreachable, and print
+  the route ("via Tor, real IP hidden" or "direct, real IP") before sending.
 - The entity graph now cools and settles instead of jittering forever: forces
   decay over ~300 frames, the animation loop stops once settled, overlapping
   nodes are pushed apart, and dragging a node reheats the layout (#58).

openfoia/cli.py

@@ -5106,14 +5106,18 @@ def _records_search_all(
     from .records import get_adapter, list_sources
 
     sources = list_sources()
+    policy = _records_policy_or_exit()
     data_gov_key = load_config().records.api_data_gov_key
 
     def say(msg: str) -> None:
         # Status goes to stderr in --json mode so stdout stays pure JSON lines.
         typer.echo(msg, err=json_lines)
 
-    say(f"Sending '{query}' to {len(sources)} services: {', '.join(sources)}")
-    if "nara" in sources and getattr(get_adapter("nara"), "has_key", False):
+    say(
+        f"Sending '{query}' to {len(sources)} services ({_route_note(policy)}): "
+        f"{', '.join(sources)}"
+    )
+    if "nara" in sources and getattr(get_adapter("nara", egress=policy), "has_key", False):
         from .records.nara import ATTRIBUTION
 
         say(
@@ -5126,7 +5130,7 @@ async def one(source: str):
             if kwargs.get(key) and source != owner:
                 return source, None, f"skipped ({key} filter not supported)"
         try:
-            adapter = get_adapter(source, api_data_gov_key=data_gov_key)
+            adapter = get_adapter(source, egress=policy, api_data_gov_key=data_gov_key)
             if source == "nara" and not adapter.has_key:
                 return source, None, "skipped (no OPENFOIA_NARA_API_KEY)"
             result = await _search_one(adapter, source, query, limit, kwargs)
@@ -5291,15 +5295,16 @@ def records_search(
         rprint(f"[red]Unknown source '{source}'. Available: {', '.join(available)}[/red]")
         raise typer.Exit(1)
 
-    adapter = get_adapter(source, api_data_gov_key=load_config().records.api_data_gov_key)
+    policy = _records_policy_or_exit()
+    adapter = get_adapter(
+        source, egress=policy, api_data_gov_key=load_config().records.api_data_gov_key
+    )
     # Pipe-safe modes: stdout carries only JSON, so no spinner or Rich markup.
     quiet = raw or json_lines
 
+    if source != "cia":
+        typer.echo(f"Sending {query!r} to {source} ({_route_note(policy)}).", err=True)
     if source == "cia":
-        from .records.cia import CIAReadingRoomAdapter
-
-        policy = _cia_policy_or_exit()
-        adapter = CIAReadingRoomAdapter(egress=policy)
         notice = f"WARNING: This will send the query {query!r} to cia.gov ({_route_note(policy)})."
         if quiet:
             typer.echo(notice, err=True)
@@ -5814,7 +5819,9 @@ def records_fetch(
     from .models import Request as RequestModel
     from .records import get_adapter
 
-    adapter = get_adapter(source)
+    policy = _records_policy_or_exit()
+    typer.echo(f"Fetching from {source} ({_route_note(policy)}).", err=True)
+    adapter = get_adapter(source, egress=policy)
     if source == "nara":
         from .records.nara import ATTRIBUTION, NO_KEY_MESSAGE
 
@@ -5827,10 +5834,6 @@ def records_fetch(
             err=True,
         )
         if list_objects or objects:
-            from .records.nara import NARAAdapter
-
-            policy = _cia_policy_or_exit()  # fails closed when Tor is configured but down
-            adapter = NARAAdapter(egress=policy)
             _records_nara_objects(
                 adapter, ids, download=objects, as_json=as_json, route=_route_note(policy)
             )
@@ -5995,7 +5998,7 @@ def _ingest_nara_objects(entity, naid: str, results) -> int:
     return done
 
 
-def _cia_policy_or_exit() -> EgressPolicy:
+def _records_policy_or_exit() -> EgressPolicy:
     """Egress policy from config.network.tor, fail-closed when Tor is configured but down."""
     from .config import load_config
 
@@ -6023,7 +6026,7 @@ def _records_fetch_cia(identifier: str) -> None:
         rprint(f"[red]{e}[/red]")
         raise typer.Exit(1) from None
 
-    policy = _cia_policy_or_exit()
+    policy = _records_policy_or_exit()
     rprint(
         f"\n[yellow]WARNING: This will send the identifier {identifier!r} to cia.gov "
         f"({_route_note(policy)}).[/yellow]"
@@ -6095,7 +6098,9 @@ def records_download(
 
     from .records.muckrock import MuckRockAdapter
 
-    adapter = MuckRockAdapter()
+    policy = _records_policy_or_exit()
+    rprint(f"[dim]Contacting MuckRock ({_route_note(policy)}).[/dim]")
+    adapter = MuckRockAdapter(egress=policy)
 
     # First fetch to show what we're downloading
     with Progress(

openfoia/records/__init__.py

@@ -10,6 +10,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
+    from ..net import EgressPolicy
     from .base import RecordAdapter
 
 # Adapter registry: source name -> adapter class
@@ -21,23 +22,34 @@ def register(name: str, adapter_cls: type[RecordAdapter]) -> None:
     _REGISTRY[name] = adapter_cls
 
 
-def get_adapter(name: str, *, api_data_gov_key: str | None = None) -> RecordAdapter:
+def get_adapter(
+    name: str,
+    *,
+    egress: EgressPolicy | None = None,
+    api_data_gov_key: str | None = None,
+) -> RecordAdapter:
     """Instantiate and return the adapter for the given source name.
 
+    *egress* routes its requests; None means DIRECT, so CLI callers must pass
+    the policy built from config.network or a configured Tor is bypassed.
+
     *api_data_gov_key* (config ``records.api_data_gov_key``) goes to the
-    api.data.gov sources; None leaves them on DEMO_KEY.
+    api.data.gov sources; None leaves them on DEMO_KEY. Over Tor the literal
+    DEMO_KEY is always used: a personal key identifies the user regardless of
+    route, and None would let GovInfo fall back to its env key.
 
     Raises KeyError if the source is not registered.
     """
-    from .base import DataGovAdapter
+    from .base import DEMO_KEY, DataGovAdapter
 
     if name not in _REGISTRY:
         available = ", ".join(sorted(_REGISTRY.keys()))
         raise KeyError(f"Unknown records source '{name}'. Available: {available}")
     cls = _REGISTRY[name]
     if issubclass(cls, DataGovAdapter):
-        return cls(api_key=api_data_gov_key)
-    return cls()
+        key = DEMO_KEY if egress is not None and egress.is_tor else api_data_gov_key
+        return cls(api_key=key, egress=egress)
+    return cls(egress=egress)
 
 
 def list_sources() -> list[str]:

tests/test_records_nara.py

@@ -477,7 +477,7 @@ def test_objects_use_the_configured_tor_policy(monkeypatch):
     monkeypatch.setenv("OPENFOIA_NARA_API_KEY", "k")
     _patch_request(monkeypatch, _response([_hit_with_files()]), [])
     tor = EgressPolicy(mode=EgressMode.TOR)
-    monkeypatch.setattr(cli, "_cia_policy_or_exit", lambda: tor)
+    monkeypatch.setattr(cli, "_records_policy_or_exit", lambda: tor)
     seen: list = []
 
     async def fake_objects(self, entity, dest_dir):

tests/test_security_egress_cli.py

@@ -498,3 +498,90 @@ def test_threat_model_names_what_tor_does_not_protect():
     text = _threat_model_text().lower()
     assert "who is asking, not what is asked" in text or "hides who is asking" in text
     assert "correlate timing" in text or "timing" in text
+
+
+# ---------------------------------------------------------------------------
+# records search/fetch/download honor network.tor
+# ---------------------------------------------------------------------------
+
+RECORDS_COMMANDS = [
+    ["records", "search", "acme", "-s", "fec"],
+    ["records", "search", "acme", "--all-sources"],
+    ["records", "fetch", "123", "-s", "documentcloud"],
+    ["records", "fetch", "7283224", "-s", "nara"],
+    ["records", "download", "68490", "-s", "muckrock"],
+]
+
+
+@pytest.fixture
+def tor_records(monkeypatch, isolated_data_dir):
+    """Tor on in config.json; records every adapter built and every HTTP request."""
+    import httpx
+
+    from openfoia.config import OpenFOIAConfig, save_config
+    from openfoia.records.base import RecordAdapter
+
+    cfg = OpenFOIAConfig()
+    cfg.network.tor = True
+    save_config(cfg)
+    monkeypatch.setenv("OPENFOIA_NARA_API_KEY", "k")
+
+    built: list[tuple[str, EgressPolicy]] = []
+    requests: list = []
+    original_init = RecordAdapter.__init__
+
+    def capture_init(self, *, egress=None):
+        original_init(self, egress=egress)
+        built.append((type(self).__name__, self._egress))
+
+    async def no_network(self, request):
+        requests.append(request.url)
+        raise httpx.ConnectError("network disabled in tests", request=request)
+
+    monkeypatch.setattr(RecordAdapter, "__init__", capture_init)
+    monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", no_network)
+    return SimpleNamespace(built=built, requests=requests)
+
+
+def _tor_reachable(monkeypatch, up: bool):
+    async def _check(*a, **k):
+        return up
+
+    monkeypatch.setattr("openfoia.net.check_tor", _check)
+
+
+@pytest.mark.parametrize("args", RECORDS_COMMANDS, ids=lambda a: "-".join(a[1:2] + a[-1:]))
+def test_records_commands_build_adapters_on_tor(monkeypatch, tor_records, args):
+    _tor_reachable(monkeypatch, True)
+    r = runner.invoke(app, args)
+    assert tor_records.built, r.output
+    assert all(p.mode is EgressMode.TOR for _, p in tor_records.built), tor_records.built
+    assert "via Tor" in r.output
+
+
+@pytest.mark.parametrize("args", RECORDS_COMMANDS, ids=lambda a: "-".join(a[1:2] + a[-1:]))
+def test_records_commands_fail_closed_when_tor_down(monkeypatch, tor_records, args):
+    _tor_reachable(monkeypatch, False)
+    r = runner.invoke(app, args)
+    assert r.exit_code != 0
+    assert tor_records.requests == []
+    assert "not reachable" in r.output
+
+
+def test_all_sources_sends_cia_over_tor(monkeypatch, tor_records):
+    _tor_reachable(monkeypatch, True)
+    runner.invoke(app, ["records", "search", "acme", "--all-sources"])
+    cia = [p for name, p in tor_records.built if name == "CIAReadingRoomAdapter"]
+    assert cia and all(p.mode is EgressMode.TOR for p in cia)
+
+
+def test_get_adapter_uses_demo_key_over_tor_even_with_env_key(monkeypatch):
+    from openfoia.net import EgressMode, EgressPolicy
+    from openfoia.records import get_adapter
+
+    monkeypatch.setenv("OPENFOIA_GOVINFO_API_KEY", "personal-govinfo")
+    tor = EgressPolicy(mode=EgressMode.TOR)
+    assert get_adapter("govinfo", egress=tor, api_data_gov_key="personal")._api_key == "DEMO_KEY"
+    assert get_adapter("fec", egress=tor, api_data_gov_key="personal")._api_key == "DEMO_KEY"
+    direct = EgressPolicy()
+    assert get_adapter("fec", egress=direct, api_data_gov_key="personal")._api_key == "personal"