fix(cli): browse honors network.tor; Tor-down error on stderr; CSV nits (#132)

CHANGELOG.md

@@ -37,6 +37,19 @@ exhaustive lists.
   your query or identifiers from your real IP with no Tor check. They now route
   through Tor, refuse to run when Tor is configured but unreachable, and print
   the route ("via Tor, real IP hidden" or "direct, real IP") before sending.
+- `openfoia browse` ignored `network.tor: true` and loaded pages from your real
+  IP unless you passed `--tor`. It now follows the config (`--tor` / `--no-tor`
+  override it), refuses to open the page when Tor is configured but
+  unreachable, and prints the route before loading (#132).
+- The "Tor is not reachable" error now goes to stderr, so it no longer lands
+  in `--json` or `--csv -` output, and tells you to start Tor or set
+  `network.tor` to false instead of suggesting a `--tor` flag most commands
+  do not have.
+- `--csv PATH` on `records search`, `request list`, `analyze extract` and
+  `deadlines list` now writes a header-only file when there are no results,
+  instead of leaving a stale file from an earlier run in place.
+- `records search --csv` keeps full USAspending descriptions and ranked-search
+  snippets; only the on-screen table shortens them.
 - The entity graph now cools and settles instead of jittering forever: forces
   decay over ~300 frames, the animation loop stops once settled, overlapping
   nodes are pushed apart, and dragging a node reheats the layout (#58).

openfoia/cli.py

@@ -925,7 +925,8 @@ def request_list(
 
         if not requests:
             rprint("[yellow]No requests found.[/yellow]")
-            return
+            if not csv_path:
+                return
 
         table = Table(title=f"FOIA Requests ({len(requests)} results)")
         table.add_column("Request #", style="cyan")
@@ -3425,6 +3426,8 @@ def analyze_extract(
 
         if not result.entities:
             rprint("[yellow]No entities found in document.[/yellow]")
+            if csv_path:
+                _emit_table(Table("Type", "Entity", "Confidence"), csv_path)
             return
 
         # Save entities to database
@@ -4528,7 +4531,8 @@ def deadline_list(
         rows, no_clock, unsent = _collect_clocks(session, show_all)
         if not rows and not no_clock and not unsent:
             rprint("[dim]No active requests with deadlines.[/dim]")
-            return
+            if not csv_path:
+                return
         from .clocks import is_overdue
 
         def clock_table(group) -> Table:
@@ -4694,12 +4698,12 @@ def _check_tor_or_exit(policy: EgressPolicy) -> None:
         return
 
     if not asyncio.run(check_tor(policy)):
-        rprint(
-            f"[red]Tor egress requested but the SOCKS proxy at "
-            f"{policy.tor_host}:{policy.tor_port} is not reachable.[/red]"
-        )
-        rprint(
-            "[yellow]Start Tor (e.g. `tor` / `sudo systemctl start tor`) or drop --tor.[/yellow]"
+        typer.echo(
+            f"Tor egress requested but the SOCKS proxy at "
+            f"{policy.tor_host}:{policy.tor_port} is not reachable.\n"
+            "Start Tor (e.g. `tor` / `sudo systemctl start tor`), or set "
+            '"network": {"tor": false} in config.json to go direct.',
+            err=True,
         )
         raise typer.Exit(1)
 
@@ -4752,8 +4756,8 @@ def egress_status(
 @app.command()
 def browse(
     url: str = typer.Argument(..., help="URL to navigate to"),
-    tor: bool = typer.Option(
-        False, "--tor", help="Route traffic through Tor (SOCKS5 localhost:9050)"
+    tor: bool | None = typer.Option(
+        None, "--tor/--no-tor", help="Route through Tor; default is network.tor in config"
     ),
     headless: bool = typer.Option(False, "--headless", help="Run browser without visible window"),
     save: bool = typer.Option(
@@ -4762,9 +4766,9 @@ def browse(
 ):
     """Browse a URL with optional Tor routing and fingerprint hardening.
 
-    Uses Playwright to launch a hardened Chromium instance. When --tor is
-    specified, traffic is routed through the Tor SOCKS5 proxy at localhost:9050
-    (the Tor daemon must be running).
+    Uses Playwright to launch a hardened Chromium instance. Traffic goes
+    through the Tor SOCKS5 proxy when network.tor is set in config or --tor
+    is passed (the Tor daemon must be running); --no-tor goes direct.
 
     Fingerprint hardening is always applied: WebGL disabled, WebRTC disabled,
     timezone set to UTC, and a common user-agent is used.
@@ -4778,13 +4782,18 @@ def browse(
     """
     import asyncio
 
+    from .config import load_config
     from .tor_browse import browse as _browse
 
+    policy = _egress_policy_from(load_config(), tor=tor)
+    _check_tor_or_exit(policy)
+    typer.echo(f"Opening {url} ({_route_note(policy)}).", err=True)
+
     try:
         result = asyncio.run(
             _browse(
                 url,
-                use_tor=tor,
+                use_tor=policy.is_tor,
                 headless=headless,
                 save=save,
             )
@@ -5408,10 +5417,11 @@ def _print_search_result(
             )
         else:
             rprint(f"[yellow]No results found for '{query}' on {source}.[/yellow]")
-        return
-
-    rprint(f"\n[bold]{source}[/bold]: {result.total_results} total results for '{query}'")
-    rprint("=" * 60)
+        if not csv_path:
+            return
+    else:
+        rprint(f"\n[bold]{source}[/bold]: {result.total_results} total results for '{query}'")
+        rprint("=" * 60)
 
     entities = result.entities[:limit]
 
@@ -5420,10 +5430,10 @@ def _print_search_result(
         table.add_column("Score", width=5, justify="right")
         table.add_column("ID", style="dim", width=10, no_wrap=True, overflow="crop")
         table.add_column("Title", style="cyan", ratio=2)
-        table.add_column("Snippet", ratio=3, style="yellow")
+        table.add_column("Snippet", ratio=3, style="yellow", no_wrap=True, overflow="ellipsis")
         for e in entities:
             c = _compact_result(e)
-            table.add_row(str(c["score"]), c["id"] or "-", e.name, c["snippet"][:120])
+            table.add_row(str(c["score"]), c["id"] or "-", e.name, c["snippet"])
         _emit_table(table, csv_path)
         rprint(f"\n[dim]Showing {len(entities)} of {result.total_results} results.[/dim]")
         return
@@ -5482,15 +5492,15 @@ def _print_search_result(
         table.add_column("Amount", width=12, justify="right", style="green")
         table.add_column("Agency", ratio=2)
         table.add_column("Type", width=10)
-        table.add_column("Description", ratio=3)
+        table.add_column("Description", ratio=3, no_wrap=True, overflow="ellipsis")
 
         for e in entities:
             table.add_row(
                 e.name,
                 e.extra_data.get("amount_formatted", "-"),
                 e.extra_data.get("awarding_agency", "-") or "-",
                 e.extra_data.get("award_type", "-") or "-",
-                (e.extra_data.get("description") or "-")[:80],
+                e.extra_data.get("description") or "-",
             )
         _emit_table(table, csv_path)
         rprint("\n[dim]View on USAspending.gov: click award URLs in the table[/dim]")

tests/test_records_cli.py

@@ -71,6 +71,43 @@ async def search(self, query, **kwargs):
     ]
 
 
+def _one_shot_adapter(monkeypatch, source, entities):
+    class Adapter:
+        async def search(self, query, **kwargs):
+            return SearchResult(
+                source=source, query=query, total_results=len(entities), entities=entities
+            )
+
+    monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter())
+
+
+def test_records_search_csv_no_results_overwrites_stale_file(monkeypatch, tmp_path):
+    _one_shot_adapter(monkeypatch, "sec", [])
+    out = tmp_path / "r.csv"
+    out.write_text("stale,rows\n")
+
+    result = CliRunner().invoke(app, ["records", "search", "x", "-s", "sec", "--csv", str(out)])
+
+    assert result.exit_code == 0, result.output
+    assert out.read_text().splitlines() == ["Company,Filing,Date,CIK,URL"]
+
+
+def test_records_search_csv_keeps_full_usaspending_description(monkeypatch):
+    desc = "Long award description " * 10
+    entity = RecordEntity(
+        entity_type="ORGANIZATION",
+        name="Acme",
+        source="usaspending",
+        extra_data={"description": desc},
+    )
+    _one_shot_adapter(monkeypatch, "usaspending", [entity])
+
+    result = CliRunner().invoke(app, ["records", "search", "x", "-s", "usaspending", "--csv", "-"])
+
+    assert result.exit_code == 0, result.output
+    assert list(csv.reader(result.stdout.splitlines()))[1][-1] == desc
+
+
 def test_records_search_raw_is_parseable_json_without_terminal_output(monkeypatch):
     """Raw mode must be pipe-safe even when upstream data has control characters."""
 

tests/test_security_egress_cli.py

@@ -585,3 +585,49 @@ def test_get_adapter_uses_demo_key_over_tor_even_with_env_key(monkeypatch):
     assert get_adapter("fec", egress=tor, api_data_gov_key="personal")._api_key == "DEMO_KEY"
     direct = EgressPolicy()
     assert get_adapter("fec", egress=direct, api_data_gov_key="personal")._api_key == "personal"
+
+
+# ---------------------------------------------------------------------------
+# `openfoia browse` honors network.tor (#132)
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def fake_browser(monkeypatch):
+    calls = []
+
+    async def _browse(url, *, use_tor, headless, save):
+        calls.append(use_tor)
+        return {"title": "t", "url": url}
+
+    monkeypatch.setattr("openfoia.tor_browse.browse", _browse)
+    return calls
+
+
+@pytest.mark.parametrize(
+    ("configured", "flags", "want_tor"),
+    [("1", [], True), ("0", [], False), ("1", ["--no-tor"], False), ("0", ["--tor"], True)],
+)
+def test_browse_route_follows_config_with_flag_override(
+    monkeypatch, fake_browser, configured, flags, want_tor
+):
+    monkeypatch.setenv("OPENFOIA_TOR", configured)
+    _tor_reachable(monkeypatch, True)
+
+    result = runner.invoke(app, ["browse", "https://example.test", *flags])
+
+    assert result.exit_code == 0, result.output
+    assert fake_browser == [want_tor]
+    assert ("via Tor, real IP hidden" if want_tor else "direct, real IP") in result.stderr
+
+
+def test_browse_fails_closed_when_configured_tor_is_down(monkeypatch, fake_browser):
+    monkeypatch.setenv("OPENFOIA_TOR", "1")
+    _tor_reachable(monkeypatch, False)
+
+    result = runner.invoke(app, ["browse", "https://example.test"])
+
+    assert result.exit_code != 0
+    assert fake_browser == []
+    assert "not reachable" in result.stderr
+    assert "not reachable" not in result.stdout