From 46655d0c2610b6677b0378fbfada71c1c462e922 Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 11:47:47 -0500 Subject: [PATCH] fix(cli): browse honors network.tor; Tor-down error on stderr; CSV nits (#132) browse now builds its route from config.network.tor via _egress_policy_from, with --tor/--no-tor as overrides, runs the fail-closed _check_tor_or_exit gate, and prints the same stderr route line as the records commands. tor_browse.py is unchanged; it receives the resolved use_tor. _check_tor_or_exit writes to stderr so --json and --csv - stdout stay clean, and its hint no longer tells users to "drop --tor". --csv PATH writes a header-only file on "no results" for records search, request list, analyze extract and deadlines list, so a stale file from a previous run is overwritten. records search keeps full usaspending descriptions and rank snippets in CSV; the Rich columns ellipsize instead. Closes #132 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- CHANGELOG.md | 13 ++++++++ openfoia/cli.py | 54 ++++++++++++++++++------------- tests/test_records_cli.py | 37 +++++++++++++++++++++ tests/test_security_egress_cli.py | 46 ++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a35343..b1161d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,19 @@ exhaustive lists. your query or identifiers from your real IP with no Tor check. They now route through Tor, refuse to run when Tor is configured but unreachable, and print the route ("via Tor, real IP hidden" or "direct, real IP") before sending. +- `openfoia browse` ignored `network.tor: true` and loaded pages from your real + IP unless you passed `--tor`. It now follows the config (`--tor` / `--no-tor` + override it), refuses to open the page when Tor is configured but + unreachable, and prints the route before loading (#132). +- The "Tor is not reachable" error now goes to stderr, so it no longer lands + in `--json` or `--csv -` output, and tells you to start Tor or set + `network.tor` to false instead of suggesting a `--tor` flag most commands + do not have. +- `--csv PATH` on `records search`, `request list`, `analyze extract` and + `deadlines list` now writes a header-only file when there are no results, + instead of leaving a stale file from an earlier run in place. +- `records search --csv` keeps full USAspending descriptions and ranked-search + snippets; only the on-screen table shortens them. - The entity graph now cools and settles instead of jittering forever: forces decay over ~300 frames, the animation loop stops once settled, overlapping nodes are pushed apart, and dragging a node reheats the layout (#58). diff --git a/openfoia/cli.py b/openfoia/cli.py index 4689ea1..b3edd1b 100644 --- a/openfoia/cli.py +++ b/openfoia/cli.py @@ -925,7 +925,8 @@ def request_list( if not requests: rprint("[yellow]No requests found.[/yellow]") - return + if not csv_path: + return table = Table(title=f"FOIA Requests ({len(requests)} results)") table.add_column("Request #", style="cyan") @@ -3425,6 +3426,8 @@ def analyze_extract( if not result.entities: rprint("[yellow]No entities found in document.[/yellow]") + if csv_path: + _emit_table(Table("Type", "Entity", "Confidence"), csv_path) return # Save entities to database @@ -4528,7 +4531,8 @@ def deadline_list( rows, no_clock, unsent = _collect_clocks(session, show_all) if not rows and not no_clock and not unsent: rprint("[dim]No active requests with deadlines.[/dim]") - return + if not csv_path: + return from .clocks import is_overdue def clock_table(group) -> Table: @@ -4694,12 +4698,12 @@ def _check_tor_or_exit(policy: EgressPolicy) -> None: return if not asyncio.run(check_tor(policy)): - rprint( - f"[red]Tor egress requested but the SOCKS proxy at " - f"{policy.tor_host}:{policy.tor_port} is not reachable.[/red]" - ) - rprint( - "[yellow]Start Tor (e.g. `tor` / `sudo systemctl start tor`) or drop --tor.[/yellow]" + typer.echo( + f"Tor egress requested but the SOCKS proxy at " + f"{policy.tor_host}:{policy.tor_port} is not reachable.\n" + "Start Tor (e.g. `tor` / `sudo systemctl start tor`), or set " + '"network": {"tor": false} in config.json to go direct.', + err=True, ) raise typer.Exit(1) @@ -4752,8 +4756,8 @@ def egress_status( @app.command() def browse( url: str = typer.Argument(..., help="URL to navigate to"), - tor: bool = typer.Option( - False, "--tor", help="Route traffic through Tor (SOCKS5 localhost:9050)" + tor: bool | None = typer.Option( + None, "--tor/--no-tor", help="Route through Tor; default is network.tor in config" ), headless: bool = typer.Option(False, "--headless", help="Run browser without visible window"), save: bool = typer.Option( @@ -4762,9 +4766,9 @@ def browse( ): """Browse a URL with optional Tor routing and fingerprint hardening. - Uses Playwright to launch a hardened Chromium instance. When --tor is - specified, traffic is routed through the Tor SOCKS5 proxy at localhost:9050 - (the Tor daemon must be running). + Uses Playwright to launch a hardened Chromium instance. Traffic goes + through the Tor SOCKS5 proxy when network.tor is set in config or --tor + is passed (the Tor daemon must be running); --no-tor goes direct. Fingerprint hardening is always applied: WebGL disabled, WebRTC disabled, timezone set to UTC, and a common user-agent is used. @@ -4778,13 +4782,18 @@ def browse( """ import asyncio + from .config import load_config from .tor_browse import browse as _browse + policy = _egress_policy_from(load_config(), tor=tor) + _check_tor_or_exit(policy) + typer.echo(f"Opening {url} ({_route_note(policy)}).", err=True) + try: result = asyncio.run( _browse( url, - use_tor=tor, + use_tor=policy.is_tor, headless=headless, save=save, ) @@ -5408,10 +5417,11 @@ def _print_search_result( ) else: rprint(f"[yellow]No results found for '{query}' on {source}.[/yellow]") - return - - rprint(f"\n[bold]{source}[/bold]: {result.total_results} total results for '{query}'") - rprint("=" * 60) + if not csv_path: + return + else: + rprint(f"\n[bold]{source}[/bold]: {result.total_results} total results for '{query}'") + rprint("=" * 60) entities = result.entities[:limit] @@ -5420,10 +5430,10 @@ def _print_search_result( table.add_column("Score", width=5, justify="right") table.add_column("ID", style="dim", width=10, no_wrap=True, overflow="crop") table.add_column("Title", style="cyan", ratio=2) - table.add_column("Snippet", ratio=3, style="yellow") + table.add_column("Snippet", ratio=3, style="yellow", no_wrap=True, overflow="ellipsis") for e in entities: c = _compact_result(e) - table.add_row(str(c["score"]), c["id"] or "-", e.name, c["snippet"][:120]) + table.add_row(str(c["score"]), c["id"] or "-", e.name, c["snippet"]) _emit_table(table, csv_path) rprint(f"\n[dim]Showing {len(entities)} of {result.total_results} results.[/dim]") return @@ -5482,7 +5492,7 @@ def _print_search_result( table.add_column("Amount", width=12, justify="right", style="green") table.add_column("Agency", ratio=2) table.add_column("Type", width=10) - table.add_column("Description", ratio=3) + table.add_column("Description", ratio=3, no_wrap=True, overflow="ellipsis") for e in entities: table.add_row( @@ -5490,7 +5500,7 @@ def _print_search_result( e.extra_data.get("amount_formatted", "-"), e.extra_data.get("awarding_agency", "-") or "-", e.extra_data.get("award_type", "-") or "-", - (e.extra_data.get("description") or "-")[:80], + e.extra_data.get("description") or "-", ) _emit_table(table, csv_path) rprint("\n[dim]View on USAspending.gov: click award URLs in the table[/dim]") diff --git a/tests/test_records_cli.py b/tests/test_records_cli.py index 5e1b4d9..8505b45 100644 --- a/tests/test_records_cli.py +++ b/tests/test_records_cli.py @@ -71,6 +71,43 @@ async def search(self, query, **kwargs): ] +def _one_shot_adapter(monkeypatch, source, entities): + class Adapter: + async def search(self, query, **kwargs): + return SearchResult( + source=source, query=query, total_results=len(entities), entities=entities + ) + + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) + + +def test_records_search_csv_no_results_overwrites_stale_file(monkeypatch, tmp_path): + _one_shot_adapter(monkeypatch, "sec", []) + out = tmp_path / "r.csv" + out.write_text("stale,rows\n") + + result = CliRunner().invoke(app, ["records", "search", "x", "-s", "sec", "--csv", str(out)]) + + assert result.exit_code == 0, result.output + assert out.read_text().splitlines() == ["Company,Filing,Date,CIK,URL"] + + +def test_records_search_csv_keeps_full_usaspending_description(monkeypatch): + desc = "Long award description " * 10 + entity = RecordEntity( + entity_type="ORGANIZATION", + name="Acme", + source="usaspending", + extra_data={"description": desc}, + ) + _one_shot_adapter(monkeypatch, "usaspending", [entity]) + + result = CliRunner().invoke(app, ["records", "search", "x", "-s", "usaspending", "--csv", "-"]) + + assert result.exit_code == 0, result.output + assert list(csv.reader(result.stdout.splitlines()))[1][-1] == desc + + def test_records_search_raw_is_parseable_json_without_terminal_output(monkeypatch): """Raw mode must be pipe-safe even when upstream data has control characters.""" diff --git a/tests/test_security_egress_cli.py b/tests/test_security_egress_cli.py index 365781b..1f45131 100644 --- a/tests/test_security_egress_cli.py +++ b/tests/test_security_egress_cli.py @@ -585,3 +585,49 @@ def test_get_adapter_uses_demo_key_over_tor_even_with_env_key(monkeypatch): assert get_adapter("fec", egress=tor, api_data_gov_key="personal")._api_key == "DEMO_KEY" direct = EgressPolicy() assert get_adapter("fec", egress=direct, api_data_gov_key="personal")._api_key == "personal" + + +# --------------------------------------------------------------------------- +# `openfoia browse` honors network.tor (#132) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_browser(monkeypatch): + calls = [] + + async def _browse(url, *, use_tor, headless, save): + calls.append(use_tor) + return {"title": "t", "url": url} + + monkeypatch.setattr("openfoia.tor_browse.browse", _browse) + return calls + + +@pytest.mark.parametrize( + ("configured", "flags", "want_tor"), + [("1", [], True), ("0", [], False), ("1", ["--no-tor"], False), ("0", ["--tor"], True)], +) +def test_browse_route_follows_config_with_flag_override( + monkeypatch, fake_browser, configured, flags, want_tor +): + monkeypatch.setenv("OPENFOIA_TOR", configured) + _tor_reachable(monkeypatch, True) + + result = runner.invoke(app, ["browse", "https://example.test", *flags]) + + assert result.exit_code == 0, result.output + assert fake_browser == [want_tor] + assert ("via Tor, real IP hidden" if want_tor else "direct, real IP") in result.stderr + + +def test_browse_fails_closed_when_configured_tor_is_down(monkeypatch, fake_browser): + monkeypatch.setenv("OPENFOIA_TOR", "1") + _tor_reachable(monkeypatch, False) + + result = runner.invoke(app, ["browse", "https://example.test"]) + + assert result.exit_code != 0 + assert fake_browser == [] + assert "not reachable" in result.stderr + assert "not reachable" not in result.stdout