feat(config): api.data.gov key and retry count in config (#56)

CHANGELOG.md

@@ -21,6 +21,12 @@ exhaustive lists.
   `analyze extract`, `crossref` and `deadlines list` writes the table's columns
   as CSV for spreadsheets. Cells that would run as a spreadsheet formula are
   prefixed with `'` (#53).
+- One api.data.gov key for `fec`, `govinfo` and `regulations` in `records search`
+  and `crossref`: `OPENFOIA_API_DATA_GOV_KEY` or `records._api_data_gov_key` in
+  config.json (the shared `DEMO_KEY` stays the default). Errors from these
+  sources no longer echo the key. `network.max_attempts`
+  (`OPENFOIA_NETWORK_MAX_ATTEMPTS`, default 3, 1 disables retry) sets the retry
+  count for commands that read `network` config (#56).
 
 ### Fixed
 

openfoia/cli.py

@@ -588,7 +588,11 @@ def _emit_table(table: Table, csv_path: str | None) -> None:
 jurisdiction_app = typer.Typer(help="Which public-records law applies: federal FOIA or a state act")
 analyze_app = typer.Typer(help="Analyze documents")
 template_app = typer.Typer(help="Request templates")
-records_app = typer.Typer(help="Search public records")
+records_app = typer.Typer(
+    help="Search public records. fec, govinfo and regulations share the rate-limited "
+    "DEMO_KEY unless you set a free api.data.gov key: OPENFOIA_API_DATA_GOV_KEY, or "
+    '"records": {"_api_data_gov_key": "..."} in config.json.'
+)
 
 entities_app = typer.Typer(help="Manage custom entity types")
 deadline_app = typer.Typer(help="Track FOIA deadlines")
@@ -4671,6 +4675,7 @@ def _egress_policy_from(config, *, tor: bool | None = None) -> EgressPolicy:
         tor_host=config.network.tor_host,
         tor_port=config.network.tor_port,
         isolate_streams=config.network.isolate_streams,
+        max_attempts=config.network.max_attempts,
     )
 
 
@@ -5097,9 +5102,11 @@ def _records_search_all(
 ) -> None:
     import asyncio
 
+    from .config import load_config
     from .records import get_adapter, list_sources
 
     sources = list_sources()
+    data_gov_key = load_config().records.api_data_gov_key
 
     def say(msg: str) -> None:
         # Status goes to stderr in --json mode so stdout stays pure JSON lines.
@@ -5119,7 +5126,7 @@ async def one(source: str):
             if kwargs.get(key) and source != owner:
                 return source, None, f"skipped ({key} filter not supported)"
         try:
-            adapter = get_adapter(source)
+            adapter = get_adapter(source, api_data_gov_key=data_gov_key)
             if source == "nara" and not adapter.has_key:
                 return source, None, "skipped (no OPENFOIA_NARA_API_KEY)"
             result = await _search_one(adapter, source, query, limit, kwargs)
@@ -5250,6 +5257,7 @@ def records_search(
     import asyncio
     import math
 
+    from .config import load_config
     from .records import get_adapter, list_sources
 
     if (ancestor or all_pages) and source != "nara":
@@ -5283,7 +5291,7 @@ def records_search(
         rprint(f"[red]Unknown source '{source}'. Available: {', '.join(available)}[/red]")
         raise typer.Exit(1)
 
-    adapter = get_adapter(source)
+    adapter = get_adapter(source, api_data_gov_key=load_config().records.api_data_gov_key)
     # Pipe-safe modes: stdout carries only JSON, so no spinner or Rich markup.
     quiet = raw or json_lines
 
@@ -7047,6 +7055,7 @@ def _progress(event: str, msg: str) -> None:
                 on_progress=_progress,
                 allow_network=True,  # user was warned and confirmed above
                 egress=policy,
+                api_data_gov_key=cfg.records.api_data_gov_key,
             )
         )
     except TorUnavailableError:

openfoia/config.py

@@ -181,6 +181,16 @@ class NetworkConfig:
     tor_port: int = 9050
     # Unique SOCKS credentials per request -> a fresh Tor circuit each time.
     isolate_streams: bool = True
+    # Attempts per idempotent request on 429/503/connect errors; 1 disables retry.
+    max_attempts: int = 3
+
+
+@dataclass
+class RecordsConfig:
+    """Public records adapters (openfoia/records)."""
+
+    # One api.data.gov key for fec, govinfo and regulations. None = shared DEMO_KEY.
+    api_data_gov_key: str | None = None
 
 
 @dataclass
@@ -196,6 +206,7 @@ class OpenFOIAConfig:
     encryption: EncryptionConfig = field(default_factory=EncryptionConfig)
     server: ServerConfig = field(default_factory=ServerConfig)
     network: NetworkConfig = field(default_factory=NetworkConfig)
+    records: RecordsConfig = field(default_factory=RecordsConfig)
 
     # Data directory — override with OPENFOIA_DATA_DIR env var for portable / air-gapped installs
     data_dir: Path = field(
@@ -337,6 +348,14 @@ def _merge_config(config: OpenFOIAConfig, data: dict[str, Any]) -> OpenFOIAConfi
         config.network.tor_host = net.get("tor_host", config.network.tor_host)
         config.network.tor_port = net.get("tor_port", config.network.tor_port)
         config.network.isolate_streams = net.get("isolate_streams", config.network.isolate_streams)
+        with contextlib.suppress(TypeError, ValueError):
+            config.network.max_attempts = int(net.get("max_attempts", config.network.max_attempts))
+
+    if "records" in data:
+        rec = data["records"]
+        config.records.api_data_gov_key = rec.get("api_data_gov_key") or rec.get(
+            "_api_data_gov_key"
+        )
 
     return config
 
@@ -418,6 +437,15 @@ def _apply_env_overrides(config: OpenFOIAConfig, prefix: str) -> OpenFOIAConfig:
             config.network.tor_port = int(v)
         except ValueError:
             print(f"Warning: {prefix}TOR_PORT={v!r} is not a valid integer, ignoring")
+    if v := os.environ.get(f"{prefix}NETWORK_MAX_ATTEMPTS"):
+        try:
+            config.network.max_attempts = int(v)
+        except ValueError:
+            print(f"Warning: {prefix}NETWORK_MAX_ATTEMPTS={v!r} is not a valid integer, ignoring")
+
+    # Records
+    if v := os.environ.get(f"{prefix}API_DATA_GOV_KEY"):
+        config.records.api_data_gov_key = v
 
     return config
 
@@ -426,6 +454,7 @@ def _apply_env_overrides(config: OpenFOIAConfig, prefix: str) -> OpenFOIAConfig:
 _SECRET_KEY_MARKERS = (
     "password",
     "api_key",
+    "api_data_gov_key",
     "apikey",
     "token",
     "secret",
@@ -496,6 +525,7 @@ def save_config(config: OpenFOIAConfig, config_path: Path | str | None = None) -
             "tor_host": config.network.tor_host,
             "tor_port": config.network.tor_port,
             "isolate_streams": config.network.isolate_streams,
+            "max_attempts": config.network.max_attempts,
         },
     }
 

openfoia/crossref.py

@@ -171,6 +171,7 @@ async def crossref_entities(
     allow_network: bool = False,
     egress: EgressPolicy | None = None,
     opensanctions_data_dir: str | None = None,
+    api_data_gov_key: str | None = None,
 ) -> CrossRefReport:
     """Cross-reference extracted entities against all available sources.
 
@@ -193,6 +194,8 @@ async def crossref_entities(
             allow_network — allow_network gates WHETHER to contact remote
             sources at all; egress governs HOW (direct vs Tor). Defaults to
             a plain DIRECT policy.
+        api_data_gov_key: Key for govinfo, fec and regulations (config
+            records.api_data_gov_key); None uses the shared DEMO_KEY.
 
     Returns:
         CrossRefReport with all hits
@@ -202,7 +205,9 @@ async def crossref_entities(
     """
     import asyncio
 
-    available_sources = _get_available_sources(icij_data_dir, egress, opensanctions_data_dir)
+    available_sources = _get_available_sources(
+        icij_data_dir, egress, opensanctions_data_dir, api_data_gov_key
+    )
     if sources:
         available_sources = {k: v for k, v in available_sources.items() if k in sources}
 
@@ -328,6 +333,7 @@ def _get_available_sources(
     icij_data_dir: str | None = None,
     egress: EgressPolicy | None = None,
     opensanctions_data_dir: str | None = None,
+    api_data_gov_key: str | None = None,
 ) -> dict[str, Any]:
     """Discover which cross-reference sources are available.
 
@@ -337,6 +343,17 @@ def _get_available_sources(
     """
     sources: dict[str, Any] = {}
 
+    # A personal api.data.gov key is tied to the signup email, so sending it
+    # over Tor identifies the user anyway. Withhold it and use DEMO_KEY.
+    # Pass the literal DEMO_KEY, not None, so no adapter-level env fallback
+    # (GovInfo's OPENFOIA_GOVINFO_API_KEY) can re-inject a personal key.
+    if egress is not None and egress.is_tor:
+        from .records.base import DEMO_KEY
+
+        if api_data_gov_key:
+            logger.info("Tor mode: not sending the personal api.data.gov key; using DEMO_KEY")
+        api_data_gov_key = DEMO_KEY
+
     # MuckRock — always available (public API, no key)
     sources["muckrock"] = lambda name, etype: _check_muckrock(name, etype, egress)
 
@@ -363,13 +380,15 @@ def _get_available_sources(
     sources["nonprofits"] = lambda name, etype: _check_nonprofits(name, etype, egress)
 
     # GovInfo — court opinions, congressional reports, Federal Register (DEMO_KEY)
-    sources["govinfo"] = lambda name, etype: _check_govinfo(name, etype, egress)
+    sources["govinfo"] = lambda name, etype: _check_govinfo(name, etype, egress, api_data_gov_key)
 
     # FEC — campaign finance contributions (DEMO_KEY)
-    sources["fec"] = lambda name, etype: _check_fec(name, etype, egress)
+    sources["fec"] = lambda name, etype: _check_fec(name, etype, egress, api_data_gov_key)
 
     # Regulations.gov — federal rulemaking documents (DEMO_KEY)
-    sources["regulations"] = lambda name, etype: _check_regulations(name, etype, egress)
+    sources["regulations"] = lambda name, etype: _check_regulations(
+        name, etype, egress, api_data_gov_key
+    )
 
     # OpenSanctions API — keyed by OPENFOIA_OPENSANCTIONS_API_KEY when set
     sources["opensanctions"] = lambda name, etype: _check_opensanctions(name, etype, egress)
@@ -598,12 +617,15 @@ async def _check_icij(name: str, entity_type: EntityType, data_dir: str) -> list
 
 
 async def _check_fec(
-    name: str, entity_type: EntityType, egress: EgressPolicy | None = None
+    name: str,
+    entity_type: EntityType,
+    egress: EgressPolicy | None = None,
+    api_key: str | None = None,
 ) -> list[CrossRefHit]:
     """Search FEC for campaign finance contributions involving this entity."""
     from .records.fec import FECAdapter
 
-    adapter = FECAdapter(egress=egress)
+    adapter = FECAdapter(api_key, egress=egress)
     try:
         result = await adapter.search(name, page_size=5)
         _check_rate_limit(result)
@@ -635,12 +657,15 @@ async def _check_fec(
 
 
 async def _check_regulations(
-    name: str, entity_type: EntityType, egress: EgressPolicy | None = None
+    name: str,
+    entity_type: EntityType,
+    egress: EgressPolicy | None = None,
+    api_key: str | None = None,
 ) -> list[CrossRefHit]:
     """Search Regulations.gov for federal rulemaking mentioning this entity."""
     from .records.regulations import RegulationsGovAdapter
 
-    adapter = RegulationsGovAdapter(egress=egress)
+    adapter = RegulationsGovAdapter(api_key, egress=egress)
     try:
         result = await adapter.search(name, page_size=5)
         _check_rate_limit(result)
@@ -674,12 +699,15 @@ async def _check_regulations(
 
 
 async def _check_govinfo(
-    name: str, entity_type: EntityType, egress: EgressPolicy | None = None
+    name: str,
+    entity_type: EntityType,
+    egress: EgressPolicy | None = None,
+    api_key: str | None = None,
 ) -> list[CrossRefHit]:
     """Search GovInfo for court opinions, congressional reports, and federal rules."""
     from .records.govinfo import GovInfoAdapter
 
-    adapter = GovInfoAdapter(egress=egress)
+    adapter = GovInfoAdapter(api_key, egress=egress)
     try:
         result = await adapter.search(name, page_size=5)
         _check_rate_limit(result)

openfoia/net.py

@@ -88,7 +88,7 @@ class RetryTransport(httpx.AsyncBaseTransport):
 
     def __init__(self, inner: httpx.AsyncBaseTransport, max_attempts: int = MAX_ATTEMPTS):
         self._inner = inner
-        self._max_attempts = max_attempts
+        self._max_attempts = max(1, max_attempts)
 
     async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
         attempts = self._max_attempts if request.method in RETRY_METHODS else 1
@@ -146,6 +146,7 @@ class EgressPolicy:
     tor_port: int = 9050
     isolate_streams: bool = True  # unique SOCKS creds -> separate Tor circuit
     user_agent: str = DEFAULT_USER_AGENT
+    max_attempts: int = MAX_ATTEMPTS  # RetryTransport attempts; 1 disables retry
 
     @property
     def is_tor(self) -> bool:
@@ -211,8 +212,10 @@ def egress_client(
         if caller_transport is None and kwargs.get("trust_env", True):
             for pattern, url in get_environment_proxies().items():
                 inner = httpx.AsyncHTTPTransport(proxy=url) if url else httpx.AsyncHTTPTransport()
-                mounts[pattern] = RetryTransport(inner)
-        default = RetryTransport(caller_transport or httpx.AsyncHTTPTransport())
+                mounts[pattern] = RetryTransport(inner, policy.max_attempts)
+        default = RetryTransport(
+            caller_transport or httpx.AsyncHTTPTransport(), policy.max_attempts
+        )
         return httpx.AsyncClient(
             transport=default, mounts=mounts, timeout=timeout, headers=merged_headers, **kwargs
         )
@@ -221,7 +224,7 @@ def egress_client(
     try:
         # proxy= keeps env proxies from mounting; the "all://" mount then
         # replaces the transport httpx built from it with the same proxy, wrapped.
-        proxied = RetryTransport(httpx.AsyncHTTPTransport(proxy=proxy_url))
+        proxied = RetryTransport(httpx.AsyncHTTPTransport(proxy=proxy_url), policy.max_attempts)
         return httpx.AsyncClient(
             proxy=proxy_url,
             mounts={"all://": proxied},

openfoia/records/__init__.py

@@ -21,15 +21,23 @@ def register(name: str, adapter_cls: type[RecordAdapter]) -> None:
     _REGISTRY[name] = adapter_cls
 
 
-def get_adapter(name: str) -> RecordAdapter:
+def get_adapter(name: str, *, api_data_gov_key: str | None = None) -> RecordAdapter:
     """Instantiate and return the adapter for the given source name.
 
+    *api_data_gov_key* (config ``records.api_data_gov_key``) goes to the
+    api.data.gov sources; None leaves them on DEMO_KEY.
+
     Raises KeyError if the source is not registered.
     """
+    from .base import DataGovAdapter
+
     if name not in _REGISTRY:
         available = ", ".join(sorted(_REGISTRY.keys()))
         raise KeyError(f"Unknown records source '{name}'. Available: {available}")
-    return _REGISTRY[name]()
+    cls = _REGISTRY[name]
+    if issubclass(cls, DataGovAdapter):
+        return cls(api_key=api_data_gov_key)
+    return cls()
 
 
 def list_sources() -> list[str]:

openfoia/records/base.py

@@ -249,3 +249,27 @@ async def fetch(self, identifier: str, **kwargs: Any) -> RecordEntity | None:
             RecordEntity if found, None otherwise.
         """
         ...
+
+
+DEMO_KEY = "DEMO_KEY"
+
+
+class DataGovAdapter(RecordAdapter):
+    """Base for api.data.gov sources (fec, govinfo, regulations), which share one key.
+
+    Without an explicit *api_key* these use the shared, rate-limited DEMO_KEY. Get a
+    free key at https://api.data.gov/signup/ and set OPENFOIA_API_DATA_GOV_KEY, or
+    ``"records": {"_api_data_gov_key": "..."}`` in config.json. The key is tied to the
+    email you signed up with, so it identifies you to api.data.gov even over Tor.
+    """
+
+    def __init__(self, api_key: str | None = None, *, egress: EgressPolicy | None = None):
+        super().__init__(egress=egress)
+        self._api_key = api_key or DEMO_KEY
+
+    def _redact(self, err: object) -> str:
+        """Error text with the key removed, wherever httpx or a URL may have put it."""
+        text = re.sub(r"api_key=[^&\s'\"]+", "api_key=<redacted>", str(err))
+        if self._api_key != DEMO_KEY:
+            text = text.replace(self._api_key, "<redacted>")
+        return text

openfoia/records/documentcloud.py

@@ -145,7 +145,7 @@ async def search(
                         "DocumentCloud rate limit hit (500/day anonymous). "
                         "Register at https://accounts.muckrock.com/ for unlimited access."
                     )
-                    logger.warning(msg)
+                    logger.info(msg)
                     return SearchResult(
                         source="documentcloud",
                         query=query,
@@ -157,7 +157,7 @@ async def search(
                     )
                 if resp.status_code == 503:
                     msg = "DocumentCloud rate limit (10/sec) — slow down."
-                    logger.warning(msg)
+                    logger.info(msg)
                     return SearchResult(
                         source="documentcloud",
                         query=query,
@@ -174,7 +174,7 @@ async def search(
                         "or region. Try the document in a browser, then use `docs ingest` "
                         "for local processing."
                     )
-                    logger.warning(msg)
+                    logger.info(msg)
                     return SearchResult(
                         source="documentcloud",
                         query=query,