diff --git a/CHANGELOG.md b/CHANGELOG.md index e878651..1a35343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,13 @@ exhaustive lists. ### Fixed +- `network.tor: true` was ignored by `records search -s ` (every source + except cia), `records search --all-sources` (all sources, cia included), + `records fetch -s documentcloud`, `records fetch -s nara` without + `--objects`/`--list-objects`, and `records download -s muckrock`: they sent + your query or identifiers from your real IP with no Tor check. They now route + through Tor, refuse to run when Tor is configured but unreachable, and print + the route ("via Tor, real IP hidden" or "direct, real IP") before sending. - The entity graph now cools and settles instead of jittering forever: forces decay over ~300 frames, the animation loop stops once settled, overlapping nodes are pushed apart, and dragging a node reheats the layout (#58). diff --git a/openfoia/cli.py b/openfoia/cli.py index fac5422..4689ea1 100644 --- a/openfoia/cli.py +++ b/openfoia/cli.py @@ -5106,14 +5106,18 @@ def _records_search_all( from .records import get_adapter, list_sources sources = list_sources() + policy = _records_policy_or_exit() data_gov_key = load_config().records.api_data_gov_key def say(msg: str) -> None: # Status goes to stderr in --json mode so stdout stays pure JSON lines. typer.echo(msg, err=json_lines) - say(f"Sending '{query}' to {len(sources)} services: {', '.join(sources)}") - if "nara" in sources and getattr(get_adapter("nara"), "has_key", False): + say( + f"Sending '{query}' to {len(sources)} services ({_route_note(policy)}): " + f"{', '.join(sources)}" + ) + if "nara" in sources and getattr(get_adapter("nara", egress=policy), "has_key", False): from .records.nara import ATTRIBUTION say( @@ -5126,7 +5130,7 @@ async def one(source: str): if kwargs.get(key) and source != owner: return source, None, f"skipped ({key} filter not supported)" try: - adapter = get_adapter(source, api_data_gov_key=data_gov_key) + adapter = get_adapter(source, egress=policy, api_data_gov_key=data_gov_key) if source == "nara" and not adapter.has_key: return source, None, "skipped (no OPENFOIA_NARA_API_KEY)" result = await _search_one(adapter, source, query, limit, kwargs) @@ -5291,15 +5295,16 @@ def records_search( rprint(f"[red]Unknown source '{source}'. Available: {', '.join(available)}[/red]") raise typer.Exit(1) - adapter = get_adapter(source, api_data_gov_key=load_config().records.api_data_gov_key) + policy = _records_policy_or_exit() + adapter = get_adapter( + source, egress=policy, api_data_gov_key=load_config().records.api_data_gov_key + ) # Pipe-safe modes: stdout carries only JSON, so no spinner or Rich markup. quiet = raw or json_lines + if source != "cia": + typer.echo(f"Sending {query!r} to {source} ({_route_note(policy)}).", err=True) if source == "cia": - from .records.cia import CIAReadingRoomAdapter - - policy = _cia_policy_or_exit() - adapter = CIAReadingRoomAdapter(egress=policy) notice = f"WARNING: This will send the query {query!r} to cia.gov ({_route_note(policy)})." if quiet: typer.echo(notice, err=True) @@ -5814,7 +5819,9 @@ def records_fetch( from .models import Request as RequestModel from .records import get_adapter - adapter = get_adapter(source) + policy = _records_policy_or_exit() + typer.echo(f"Fetching from {source} ({_route_note(policy)}).", err=True) + adapter = get_adapter(source, egress=policy) if source == "nara": from .records.nara import ATTRIBUTION, NO_KEY_MESSAGE @@ -5827,10 +5834,6 @@ def records_fetch( err=True, ) if list_objects or objects: - from .records.nara import NARAAdapter - - policy = _cia_policy_or_exit() # fails closed when Tor is configured but down - adapter = NARAAdapter(egress=policy) _records_nara_objects( adapter, ids, download=objects, as_json=as_json, route=_route_note(policy) ) @@ -5995,7 +5998,7 @@ def _ingest_nara_objects(entity, naid: str, results) -> int: return done -def _cia_policy_or_exit() -> EgressPolicy: +def _records_policy_or_exit() -> EgressPolicy: """Egress policy from config.network.tor, fail-closed when Tor is configured but down.""" from .config import load_config @@ -6023,7 +6026,7 @@ def _records_fetch_cia(identifier: str) -> None: rprint(f"[red]{e}[/red]") raise typer.Exit(1) from None - policy = _cia_policy_or_exit() + policy = _records_policy_or_exit() rprint( f"\n[yellow]WARNING: This will send the identifier {identifier!r} to cia.gov " f"({_route_note(policy)}).[/yellow]" @@ -6095,7 +6098,9 @@ def records_download( from .records.muckrock import MuckRockAdapter - adapter = MuckRockAdapter() + policy = _records_policy_or_exit() + rprint(f"[dim]Contacting MuckRock ({_route_note(policy)}).[/dim]") + adapter = MuckRockAdapter(egress=policy) # First fetch to show what we're downloading with Progress( diff --git a/openfoia/records/__init__.py b/openfoia/records/__init__.py index 4f23d33..9a9f1cf 100644 --- a/openfoia/records/__init__.py +++ b/openfoia/records/__init__.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + from ..net import EgressPolicy from .base import RecordAdapter # Adapter registry: source name -> adapter class @@ -21,23 +22,34 @@ def register(name: str, adapter_cls: type[RecordAdapter]) -> None: _REGISTRY[name] = adapter_cls -def get_adapter(name: str, *, api_data_gov_key: str | None = None) -> RecordAdapter: +def get_adapter( + name: str, + *, + egress: EgressPolicy | None = None, + api_data_gov_key: str | None = None, +) -> RecordAdapter: """Instantiate and return the adapter for the given source name. + *egress* routes its requests; None means DIRECT, so CLI callers must pass + the policy built from config.network or a configured Tor is bypassed. + *api_data_gov_key* (config ``records.api_data_gov_key``) goes to the - api.data.gov sources; None leaves them on DEMO_KEY. + api.data.gov sources; None leaves them on DEMO_KEY. Over Tor the literal + DEMO_KEY is always used: a personal key identifies the user regardless of + route, and None would let GovInfo fall back to its env key. Raises KeyError if the source is not registered. """ - from .base import DataGovAdapter + from .base import DEMO_KEY, DataGovAdapter if name not in _REGISTRY: available = ", ".join(sorted(_REGISTRY.keys())) raise KeyError(f"Unknown records source '{name}'. Available: {available}") cls = _REGISTRY[name] if issubclass(cls, DataGovAdapter): - return cls(api_key=api_data_gov_key) - return cls() + key = DEMO_KEY if egress is not None and egress.is_tor else api_data_gov_key + return cls(api_key=key, egress=egress) + return cls(egress=egress) def list_sources() -> list[str]: diff --git a/tests/test_records_nara.py b/tests/test_records_nara.py index 80ea7f8..29cfb9a 100644 --- a/tests/test_records_nara.py +++ b/tests/test_records_nara.py @@ -477,7 +477,7 @@ def test_objects_use_the_configured_tor_policy(monkeypatch): monkeypatch.setenv("OPENFOIA_NARA_API_KEY", "k") _patch_request(monkeypatch, _response([_hit_with_files()]), []) tor = EgressPolicy(mode=EgressMode.TOR) - monkeypatch.setattr(cli, "_cia_policy_or_exit", lambda: tor) + monkeypatch.setattr(cli, "_records_policy_or_exit", lambda: tor) seen: list = [] async def fake_objects(self, entity, dest_dir): diff --git a/tests/test_security_egress_cli.py b/tests/test_security_egress_cli.py index ec2f725..365781b 100644 --- a/tests/test_security_egress_cli.py +++ b/tests/test_security_egress_cli.py @@ -498,3 +498,90 @@ def test_threat_model_names_what_tor_does_not_protect(): text = _threat_model_text().lower() assert "who is asking, not what is asked" in text or "hides who is asking" in text assert "correlate timing" in text or "timing" in text + + +# --------------------------------------------------------------------------- +# records search/fetch/download honor network.tor +# --------------------------------------------------------------------------- + +RECORDS_COMMANDS = [ + ["records", "search", "acme", "-s", "fec"], + ["records", "search", "acme", "--all-sources"], + ["records", "fetch", "123", "-s", "documentcloud"], + ["records", "fetch", "7283224", "-s", "nara"], + ["records", "download", "68490", "-s", "muckrock"], +] + + +@pytest.fixture +def tor_records(monkeypatch, isolated_data_dir): + """Tor on in config.json; records every adapter built and every HTTP request.""" + import httpx + + from openfoia.config import OpenFOIAConfig, save_config + from openfoia.records.base import RecordAdapter + + cfg = OpenFOIAConfig() + cfg.network.tor = True + save_config(cfg) + monkeypatch.setenv("OPENFOIA_NARA_API_KEY", "k") + + built: list[tuple[str, EgressPolicy]] = [] + requests: list = [] + original_init = RecordAdapter.__init__ + + def capture_init(self, *, egress=None): + original_init(self, egress=egress) + built.append((type(self).__name__, self._egress)) + + async def no_network(self, request): + requests.append(request.url) + raise httpx.ConnectError("network disabled in tests", request=request) + + monkeypatch.setattr(RecordAdapter, "__init__", capture_init) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", no_network) + return SimpleNamespace(built=built, requests=requests) + + +def _tor_reachable(monkeypatch, up: bool): + async def _check(*a, **k): + return up + + monkeypatch.setattr("openfoia.net.check_tor", _check) + + +@pytest.mark.parametrize("args", RECORDS_COMMANDS, ids=lambda a: "-".join(a[1:2] + a[-1:])) +def test_records_commands_build_adapters_on_tor(monkeypatch, tor_records, args): + _tor_reachable(monkeypatch, True) + r = runner.invoke(app, args) + assert tor_records.built, r.output + assert all(p.mode is EgressMode.TOR for _, p in tor_records.built), tor_records.built + assert "via Tor" in r.output + + +@pytest.mark.parametrize("args", RECORDS_COMMANDS, ids=lambda a: "-".join(a[1:2] + a[-1:])) +def test_records_commands_fail_closed_when_tor_down(monkeypatch, tor_records, args): + _tor_reachable(monkeypatch, False) + r = runner.invoke(app, args) + assert r.exit_code != 0 + assert tor_records.requests == [] + assert "not reachable" in r.output + + +def test_all_sources_sends_cia_over_tor(monkeypatch, tor_records): + _tor_reachable(monkeypatch, True) + runner.invoke(app, ["records", "search", "acme", "--all-sources"]) + cia = [p for name, p in tor_records.built if name == "CIAReadingRoomAdapter"] + assert cia and all(p.mode is EgressMode.TOR for p in cia) + + +def test_get_adapter_uses_demo_key_over_tor_even_with_env_key(monkeypatch): + from openfoia.net import EgressMode, EgressPolicy + from openfoia.records import get_adapter + + monkeypatch.setenv("OPENFOIA_GOVINFO_API_KEY", "personal-govinfo") + tor = EgressPolicy(mode=EgressMode.TOR) + assert get_adapter("govinfo", egress=tor, api_data_gov_key="personal")._api_key == "DEMO_KEY" + assert get_adapter("fec", egress=tor, api_data_gov_key="personal")._api_key == "DEMO_KEY" + direct = EgressPolicy() + assert get_adapter("fec", egress=direct, api_data_gov_key="personal")._api_key == "personal"