From c031737b22aeb227add208e0369dd5e43fd6e631 Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 11:03:38 -0500 Subject: [PATCH 1/3] feat(config): api.data.gov key and retry count in config (#56) records.api_data_gov_key (OPENFOIA_API_DATA_GOV_KEY, _api_data_gov_key accepted) replaces the shared DEMO_KEY for fec, govinfo and regulations. The three adapters now share a DataGovAdapter base (DEMO_KEY default, key redaction). get_adapter() takes the key and hands it to DataGovAdapter subclasses; records search and crossref pass it from config. FEC and Regulations.gov send the key as a query parameter, so their error text (httpx puts the URL in HTTPStatusError) is now redacted the way GovInfo's already was; with DEMO_KEY that never mattered, with a personal key it would have reached logs and terminal output. The key is also masked by `config --show`. network.max_attempts (OPENFOIA_NETWORK_MAX_ATTEMPTS, default 3) is carried on EgressPolicy, set in _egress_policy_from, and handed to RetryTransport by egress_client, so net.py still does not import config. Values below 1 are clamped to 1. Log levels: rate limits and rejected queries that already return a clear message to the user (FEC/Regulations/GovInfo 429, DocumentCloud 429/503 and Cloudflare block, USAspending 422) drop from warning to info. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- CHANGELOG.md | 6 ++ openfoia/cli.py | 15 ++++- openfoia/config.py | 29 +++++++++ openfoia/crossref.py | 37 ++++++++---- openfoia/net.py | 11 ++-- openfoia/records/__init__.py | 12 +++- openfoia/records/base.py | 24 ++++++++ openfoia/records/documentcloud.py | 6 +- openfoia/records/fec.py | 19 +++--- openfoia/records/govinfo.py | 26 +++----- openfoia/records/regulations.py | 19 +++--- openfoia/records/usaspending.py | 2 +- tests/test_case.py | 4 +- tests/test_config_network_keys.py | 99 +++++++++++++++++++++++++++++++ tests/test_crossref_status.py | 8 +-- tests/test_records_cli.py | 12 ++-- tests/test_records_nara.py | 2 +- tests/test_records_search.py | 12 ++-- 18 files changed, 259 insertions(+), 84 deletions(-) create mode 100644 tests/test_config_network_keys.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e053799..e878651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ exhaustive lists. `analyze extract`, `crossref` and `deadlines list` writes the table's columns as CSV for spreadsheets. Cells that would run as a spreadsheet formula are prefixed with `'` (#53). +- One api.data.gov key for `fec`, `govinfo` and `regulations` in `records search` + and `crossref`: `OPENFOIA_API_DATA_GOV_KEY` or `records._api_data_gov_key` in + config.json (the shared `DEMO_KEY` stays the default). Errors from these + sources no longer echo the key. `network.max_attempts` + (`OPENFOIA_NETWORK_MAX_ATTEMPTS`, default 3, 1 disables retry) sets the retry + count for commands that read `network` config (#56). ### Fixed diff --git a/openfoia/cli.py b/openfoia/cli.py index 976af85..fac5422 100644 --- a/openfoia/cli.py +++ b/openfoia/cli.py @@ -588,7 +588,11 @@ def _emit_table(table: Table, csv_path: str | None) -> None: jurisdiction_app = typer.Typer(help="Which public-records law applies: federal FOIA or a state act") analyze_app = typer.Typer(help="Analyze documents") template_app = typer.Typer(help="Request templates") -records_app = typer.Typer(help="Search public records") +records_app = typer.Typer( + help="Search public records. fec, govinfo and regulations share the rate-limited " + "DEMO_KEY unless you set a free api.data.gov key: OPENFOIA_API_DATA_GOV_KEY, or " + '"records": {"_api_data_gov_key": "..."} in config.json.' +) entities_app = typer.Typer(help="Manage custom entity types") deadline_app = typer.Typer(help="Track FOIA deadlines") @@ -4671,6 +4675,7 @@ def _egress_policy_from(config, *, tor: bool | None = None) -> EgressPolicy: tor_host=config.network.tor_host, tor_port=config.network.tor_port, isolate_streams=config.network.isolate_streams, + max_attempts=config.network.max_attempts, ) @@ -5097,9 +5102,11 @@ def _records_search_all( ) -> None: import asyncio + from .config import load_config from .records import get_adapter, list_sources sources = list_sources() + data_gov_key = load_config().records.api_data_gov_key def say(msg: str) -> None: # Status goes to stderr in --json mode so stdout stays pure JSON lines. @@ -5119,7 +5126,7 @@ async def one(source: str): if kwargs.get(key) and source != owner: return source, None, f"skipped ({key} filter not supported)" try: - adapter = get_adapter(source) + adapter = get_adapter(source, api_data_gov_key=data_gov_key) if source == "nara" and not adapter.has_key: return source, None, "skipped (no OPENFOIA_NARA_API_KEY)" result = await _search_one(adapter, source, query, limit, kwargs) @@ -5250,6 +5257,7 @@ def records_search( import asyncio import math + from .config import load_config from .records import get_adapter, list_sources if (ancestor or all_pages) and source != "nara": @@ -5283,7 +5291,7 @@ def records_search( rprint(f"[red]Unknown source '{source}'. Available: {', '.join(available)}[/red]") raise typer.Exit(1) - adapter = get_adapter(source) + adapter = get_adapter(source, api_data_gov_key=load_config().records.api_data_gov_key) # Pipe-safe modes: stdout carries only JSON, so no spinner or Rich markup. quiet = raw or json_lines @@ -7047,6 +7055,7 @@ def _progress(event: str, msg: str) -> None: on_progress=_progress, allow_network=True, # user was warned and confirmed above egress=policy, + api_data_gov_key=cfg.records.api_data_gov_key, ) ) except TorUnavailableError: diff --git a/openfoia/config.py b/openfoia/config.py index e34f548..42392f1 100644 --- a/openfoia/config.py +++ b/openfoia/config.py @@ -181,6 +181,16 @@ class NetworkConfig: tor_port: int = 9050 # Unique SOCKS credentials per request -> a fresh Tor circuit each time. isolate_streams: bool = True + # Attempts per idempotent request on 429/503/connect errors; 1 disables retry. + max_attempts: int = 3 + + +@dataclass +class RecordsConfig: + """Public records adapters (openfoia/records).""" + + # One api.data.gov key for fec, govinfo and regulations. None = shared DEMO_KEY. + api_data_gov_key: str | None = None @dataclass @@ -196,6 +206,7 @@ class OpenFOIAConfig: encryption: EncryptionConfig = field(default_factory=EncryptionConfig) server: ServerConfig = field(default_factory=ServerConfig) network: NetworkConfig = field(default_factory=NetworkConfig) + records: RecordsConfig = field(default_factory=RecordsConfig) # Data directory — override with OPENFOIA_DATA_DIR env var for portable / air-gapped installs data_dir: Path = field( @@ -337,6 +348,13 @@ def _merge_config(config: OpenFOIAConfig, data: dict[str, Any]) -> OpenFOIAConfi config.network.tor_host = net.get("tor_host", config.network.tor_host) config.network.tor_port = net.get("tor_port", config.network.tor_port) config.network.isolate_streams = net.get("isolate_streams", config.network.isolate_streams) + config.network.max_attempts = net.get("max_attempts", config.network.max_attempts) + + if "records" in data: + rec = data["records"] + config.records.api_data_gov_key = rec.get("api_data_gov_key") or rec.get( + "_api_data_gov_key" + ) return config @@ -418,6 +436,15 @@ def _apply_env_overrides(config: OpenFOIAConfig, prefix: str) -> OpenFOIAConfig: config.network.tor_port = int(v) except ValueError: print(f"Warning: {prefix}TOR_PORT={v!r} is not a valid integer, ignoring") + if v := os.environ.get(f"{prefix}NETWORK_MAX_ATTEMPTS"): + try: + config.network.max_attempts = int(v) + except ValueError: + print(f"Warning: {prefix}NETWORK_MAX_ATTEMPTS={v!r} is not a valid integer, ignoring") + + # Records + if v := os.environ.get(f"{prefix}API_DATA_GOV_KEY"): + config.records.api_data_gov_key = v return config @@ -426,6 +453,7 @@ def _apply_env_overrides(config: OpenFOIAConfig, prefix: str) -> OpenFOIAConfig: _SECRET_KEY_MARKERS = ( "password", "api_key", + "api_data_gov_key", "apikey", "token", "secret", @@ -496,6 +524,7 @@ def save_config(config: OpenFOIAConfig, config_path: Path | str | None = None) - "tor_host": config.network.tor_host, "tor_port": config.network.tor_port, "isolate_streams": config.network.isolate_streams, + "max_attempts": config.network.max_attempts, }, } diff --git a/openfoia/crossref.py b/openfoia/crossref.py index 959915d..572d4c2 100644 --- a/openfoia/crossref.py +++ b/openfoia/crossref.py @@ -171,6 +171,7 @@ async def crossref_entities( allow_network: bool = False, egress: EgressPolicy | None = None, opensanctions_data_dir: str | None = None, + api_data_gov_key: str | None = None, ) -> CrossRefReport: """Cross-reference extracted entities against all available sources. @@ -193,6 +194,8 @@ async def crossref_entities( allow_network — allow_network gates WHETHER to contact remote sources at all; egress governs HOW (direct vs Tor). Defaults to a plain DIRECT policy. + api_data_gov_key: Key for govinfo, fec and regulations (config + records.api_data_gov_key); None uses the shared DEMO_KEY. Returns: CrossRefReport with all hits @@ -202,7 +205,9 @@ async def crossref_entities( """ import asyncio - available_sources = _get_available_sources(icij_data_dir, egress, opensanctions_data_dir) + available_sources = _get_available_sources( + icij_data_dir, egress, opensanctions_data_dir, api_data_gov_key + ) if sources: available_sources = {k: v for k, v in available_sources.items() if k in sources} @@ -328,6 +333,7 @@ def _get_available_sources( icij_data_dir: str | None = None, egress: EgressPolicy | None = None, opensanctions_data_dir: str | None = None, + api_data_gov_key: str | None = None, ) -> dict[str, Any]: """Discover which cross-reference sources are available. @@ -363,13 +369,15 @@ def _get_available_sources( sources["nonprofits"] = lambda name, etype: _check_nonprofits(name, etype, egress) # GovInfo — court opinions, congressional reports, Federal Register (DEMO_KEY) - sources["govinfo"] = lambda name, etype: _check_govinfo(name, etype, egress) + sources["govinfo"] = lambda name, etype: _check_govinfo(name, etype, egress, api_data_gov_key) # FEC — campaign finance contributions (DEMO_KEY) - sources["fec"] = lambda name, etype: _check_fec(name, etype, egress) + sources["fec"] = lambda name, etype: _check_fec(name, etype, egress, api_data_gov_key) # Regulations.gov — federal rulemaking documents (DEMO_KEY) - sources["regulations"] = lambda name, etype: _check_regulations(name, etype, egress) + sources["regulations"] = lambda name, etype: _check_regulations( + name, etype, egress, api_data_gov_key + ) # OpenSanctions API — keyed by OPENFOIA_OPENSANCTIONS_API_KEY when set sources["opensanctions"] = lambda name, etype: _check_opensanctions(name, etype, egress) @@ -598,12 +606,15 @@ async def _check_icij(name: str, entity_type: EntityType, data_dir: str) -> list async def _check_fec( - name: str, entity_type: EntityType, egress: EgressPolicy | None = None + name: str, + entity_type: EntityType, + egress: EgressPolicy | None = None, + api_key: str | None = None, ) -> list[CrossRefHit]: """Search FEC for campaign finance contributions involving this entity.""" from .records.fec import FECAdapter - adapter = FECAdapter(egress=egress) + adapter = FECAdapter(api_key, egress=egress) try: result = await adapter.search(name, page_size=5) _check_rate_limit(result) @@ -635,12 +646,15 @@ async def _check_fec( async def _check_regulations( - name: str, entity_type: EntityType, egress: EgressPolicy | None = None + name: str, + entity_type: EntityType, + egress: EgressPolicy | None = None, + api_key: str | None = None, ) -> list[CrossRefHit]: """Search Regulations.gov for federal rulemaking mentioning this entity.""" from .records.regulations import RegulationsGovAdapter - adapter = RegulationsGovAdapter(egress=egress) + adapter = RegulationsGovAdapter(api_key, egress=egress) try: result = await adapter.search(name, page_size=5) _check_rate_limit(result) @@ -674,12 +688,15 @@ async def _check_regulations( async def _check_govinfo( - name: str, entity_type: EntityType, egress: EgressPolicy | None = None + name: str, + entity_type: EntityType, + egress: EgressPolicy | None = None, + api_key: str | None = None, ) -> list[CrossRefHit]: """Search GovInfo for court opinions, congressional reports, and federal rules.""" from .records.govinfo import GovInfoAdapter - adapter = GovInfoAdapter(egress=egress) + adapter = GovInfoAdapter(api_key, egress=egress) try: result = await adapter.search(name, page_size=5) _check_rate_limit(result) diff --git a/openfoia/net.py b/openfoia/net.py index 8e2f0ec..565172b 100644 --- a/openfoia/net.py +++ b/openfoia/net.py @@ -88,7 +88,7 @@ class RetryTransport(httpx.AsyncBaseTransport): def __init__(self, inner: httpx.AsyncBaseTransport, max_attempts: int = MAX_ATTEMPTS): self._inner = inner - self._max_attempts = max_attempts + self._max_attempts = max(1, max_attempts) async def handle_async_request(self, request: httpx.Request) -> httpx.Response: attempts = self._max_attempts if request.method in RETRY_METHODS else 1 @@ -146,6 +146,7 @@ class EgressPolicy: tor_port: int = 9050 isolate_streams: bool = True # unique SOCKS creds -> separate Tor circuit user_agent: str = DEFAULT_USER_AGENT + max_attempts: int = MAX_ATTEMPTS # RetryTransport attempts; 1 disables retry @property def is_tor(self) -> bool: @@ -211,8 +212,10 @@ def egress_client( if caller_transport is None and kwargs.get("trust_env", True): for pattern, url in get_environment_proxies().items(): inner = httpx.AsyncHTTPTransport(proxy=url) if url else httpx.AsyncHTTPTransport() - mounts[pattern] = RetryTransport(inner) - default = RetryTransport(caller_transport or httpx.AsyncHTTPTransport()) + mounts[pattern] = RetryTransport(inner, policy.max_attempts) + default = RetryTransport( + caller_transport or httpx.AsyncHTTPTransport(), policy.max_attempts + ) return httpx.AsyncClient( transport=default, mounts=mounts, timeout=timeout, headers=merged_headers, **kwargs ) @@ -221,7 +224,7 @@ def egress_client( try: # proxy= keeps env proxies from mounting; the "all://" mount then # replaces the transport httpx built from it with the same proxy, wrapped. - proxied = RetryTransport(httpx.AsyncHTTPTransport(proxy=proxy_url)) + proxied = RetryTransport(httpx.AsyncHTTPTransport(proxy=proxy_url), policy.max_attempts) return httpx.AsyncClient( proxy=proxy_url, mounts={"all://": proxied}, diff --git a/openfoia/records/__init__.py b/openfoia/records/__init__.py index 9604b96..4f23d33 100644 --- a/openfoia/records/__init__.py +++ b/openfoia/records/__init__.py @@ -21,15 +21,23 @@ def register(name: str, adapter_cls: type[RecordAdapter]) -> None: _REGISTRY[name] = adapter_cls -def get_adapter(name: str) -> RecordAdapter: +def get_adapter(name: str, *, api_data_gov_key: str | None = None) -> RecordAdapter: """Instantiate and return the adapter for the given source name. + *api_data_gov_key* (config ``records.api_data_gov_key``) goes to the + api.data.gov sources; None leaves them on DEMO_KEY. + Raises KeyError if the source is not registered. """ + from .base import DataGovAdapter + if name not in _REGISTRY: available = ", ".join(sorted(_REGISTRY.keys())) raise KeyError(f"Unknown records source '{name}'. Available: {available}") - return _REGISTRY[name]() + cls = _REGISTRY[name] + if issubclass(cls, DataGovAdapter): + return cls(api_key=api_data_gov_key) + return cls() def list_sources() -> list[str]: diff --git a/openfoia/records/base.py b/openfoia/records/base.py index 6c316c2..4b46d1b 100644 --- a/openfoia/records/base.py +++ b/openfoia/records/base.py @@ -249,3 +249,27 @@ async def fetch(self, identifier: str, **kwargs: Any) -> RecordEntity | None: RecordEntity if found, None otherwise. """ ... + + +DEMO_KEY = "DEMO_KEY" + + +class DataGovAdapter(RecordAdapter): + """Base for api.data.gov sources (fec, govinfo, regulations), which share one key. + + Without an explicit *api_key* these use the shared, rate-limited DEMO_KEY. Get a + free key at https://api.data.gov/signup/ and set OPENFOIA_API_DATA_GOV_KEY, or + ``"records": {"_api_data_gov_key": "..."}`` in config.json. The key is tied to the + email you signed up with, so it identifies you to api.data.gov even over Tor. + """ + + def __init__(self, api_key: str | None = None, *, egress: EgressPolicy | None = None): + super().__init__(egress=egress) + self._api_key = api_key or DEMO_KEY + + def _redact(self, err: object) -> str: + """Error text with the key removed, wherever httpx or a URL may have put it.""" + text = re.sub(r"api_key=[^&\s'\"]+", "api_key=", str(err)) + if self._api_key != DEMO_KEY: + text = text.replace(self._api_key, "") + return text diff --git a/openfoia/records/documentcloud.py b/openfoia/records/documentcloud.py index 722ddb7..74a3587 100644 --- a/openfoia/records/documentcloud.py +++ b/openfoia/records/documentcloud.py @@ -145,7 +145,7 @@ async def search( "DocumentCloud rate limit hit (500/day anonymous). " "Register at https://accounts.muckrock.com/ for unlimited access." ) - logger.warning(msg) + logger.info(msg) return SearchResult( source="documentcloud", query=query, @@ -157,7 +157,7 @@ async def search( ) if resp.status_code == 503: msg = "DocumentCloud rate limit (10/sec) — slow down." - logger.warning(msg) + logger.info(msg) return SearchResult( source="documentcloud", query=query, @@ -174,7 +174,7 @@ async def search( "or region. Try the document in a browser, then use `docs ingest` " "for local processing." ) - logger.warning(msg) + logger.info(msg) return SearchResult( source="documentcloud", query=query, diff --git a/openfoia/records/fec.py b/openfoia/records/fec.py index 3e0dc63..ffe1da4 100644 --- a/openfoia/records/fec.py +++ b/openfoia/records/fec.py @@ -3,7 +3,8 @@ Who donated to whom. PAC spending. Candidate financials. Contribution records going back decades. This is how you follow the political money. -Uses the OpenFEC API v1. DEMO_KEY works, free key at api.data.gov for higher limits. +Uses the OpenFEC API v1. DEMO_KEY works; for higher limits set an api.data.gov key +(OPENFOIA_API_DATA_GOV_KEY or records._api_data_gov_key, see DataGovAdapter). API docs: https://api.open.fec.gov/developers/ """ @@ -15,15 +16,15 @@ import httpx -from ..net import EgressPolicy, egress_client -from .base import RecordAdapter, RecordEntity, SearchResult +from ..net import egress_client +from .base import DataGovAdapter, RecordEntity, SearchResult logger = logging.getLogger(__name__) API_BASE = "https://api.open.fec.gov/v1" -class FECAdapter(RecordAdapter): +class FECAdapter(DataGovAdapter): """Search FEC campaign finance data — candidates, committees, contributions. Usage:: @@ -35,10 +36,6 @@ class FECAdapter(RecordAdapter): source_name = "fec" - def __init__(self, api_key: str | None = None, *, egress: EgressPolicy | None = None): - super().__init__(egress=egress) - self._api_key = api_key or "DEMO_KEY" - async def search( self, query: str, @@ -66,7 +63,7 @@ async def search( ) if resp.status_code == 429: msg = "FEC rate limit hit. Get a free key at https://api.data.gov/signup/" - logger.warning(msg) + logger.info(msg) return SearchResult( source="fec", query=query, @@ -91,7 +88,7 @@ async def search( error=msg, ) except Exception as e: - msg = f"FEC search failed: {e}" + msg = f"FEC search failed: {self._redact(e)}" logger.warning(msg) return SearchResult( source="fec", @@ -163,7 +160,7 @@ async def fetch(self, identifier: str, **kwargs: Any) -> RecordEntity | None: return None data = resp.json() except Exception as e: - logger.warning("FEC candidate search failed for %s: %s", identifier, e) + logger.warning("FEC candidate search failed for %s: %s", identifier, self._redact(e)) return None results = data.get("results", []) diff --git a/openfoia/records/govinfo.py b/openfoia/records/govinfo.py index f1c6c1e..113227c 100644 --- a/openfoia/records/govinfo.py +++ b/openfoia/records/govinfo.py @@ -4,7 +4,8 @@ congressional hearings, CFR, and more. 50+ collections, millions of documents. Uses the GovInfo API v2. Requires a free API key from api.data.gov -(DEMO_KEY works for testing). Rate limit: 1,200 req/min. +(DEMO_KEY works for testing; set OPENFOIA_API_DATA_GOV_KEY or +records._api_data_gov_key, see DataGovAdapter). Rate limit: 1,200 req/min. API docs: https://api.govinfo.gov/docs/ GitHub: https://github.com/usgpo/api """ @@ -18,7 +19,7 @@ import httpx from ..net import EgressPolicy, egress_client -from .base import RecordAdapter, RecordEntity, SearchResult +from .base import DataGovAdapter, RecordEntity, SearchResult logger = logging.getLogger(__name__) @@ -46,7 +47,7 @@ } -class GovInfoAdapter(RecordAdapter): +class GovInfoAdapter(DataGovAdapter): """Search documents from all three branches of the US government. Court opinions, congressional reports, Federal Register rules, GAO reports, @@ -62,23 +63,10 @@ class GovInfoAdapter(RecordAdapter): source_name = "govinfo" def __init__(self, api_key: str | None = None, *, egress: EgressPolicy | None = None): - """Initialize. Without *api_key*, OPENFOIA_GOVINFO_API_KEY is used, else DEMO_KEY. - - DEMO_KEY is shared and rate-limited; get a free key at https://api.data.gov/signup/ - """ + """Initialize. Without *api_key*, OPENFOIA_GOVINFO_API_KEY is used, else DEMO_KEY.""" import os - super().__init__(egress=egress) - self._api_key = api_key or os.environ.get("OPENFOIA_GOVINFO_API_KEY") or "DEMO_KEY" - - def _redact(self, err: object) -> str: - """Error text with the key removed, wherever httpx or a URL may have put it.""" - import re - - text = re.sub(r"api_key=[^&\s'\"]+", "api_key=", str(err)) - if self._api_key and self._api_key != "DEMO_KEY": - text = text.replace(self._api_key, "") - return text + super().__init__(api_key or os.environ.get("OPENFOIA_GOVINFO_API_KEY"), egress=egress) async def search( self, @@ -118,7 +106,7 @@ async def search( "bad key. Get a free key at https://api.data.gov/signup/ and set " "OPENFOIA_GOVINFO_API_KEY." ) - logger.warning(msg) + logger.info(msg) return SearchResult( source="govinfo", query=query, diff --git a/openfoia/records/regulations.py b/openfoia/records/regulations.py index 181298a..4c78a9d 100644 --- a/openfoia/records/regulations.py +++ b/openfoia/records/regulations.py @@ -3,7 +3,8 @@ Every federal proposed rule gets public comments. Who's commenting tells you who's trying to influence policy. Search documents, comments, and dockets. -Uses the Regulations.gov API v4. DEMO_KEY works, free key at api.data.gov. +Uses the Regulations.gov API v4. DEMO_KEY works; for higher limits set an api.data.gov key +(OPENFOIA_API_DATA_GOV_KEY or records._api_data_gov_key, see DataGovAdapter). API docs: https://open.gsa.gov/api/regulationsgov/ """ @@ -15,15 +16,15 @@ import httpx -from ..net import EgressPolicy, egress_client -from .base import RecordAdapter, RecordEntity, SearchResult +from ..net import egress_client +from .base import DataGovAdapter, RecordEntity, SearchResult logger = logging.getLogger(__name__) API_BASE = "https://api.regulations.gov/v4" -class RegulationsGovAdapter(RecordAdapter): +class RegulationsGovAdapter(DataGovAdapter): """Search federal rulemaking documents and public comments. Usage:: @@ -34,10 +35,6 @@ class RegulationsGovAdapter(RecordAdapter): source_name = "regulations" - def __init__(self, api_key: str | None = None, *, egress: EgressPolicy | None = None): - super().__init__(egress=egress) - self._api_key = api_key or "DEMO_KEY" - async def search( self, query: str, @@ -71,7 +68,7 @@ async def search( msg = ( "Regulations.gov rate limit. Get a free key at https://api.data.gov/signup/" ) - logger.warning(msg) + logger.info(msg) return SearchResult( source="regulations", query=query, @@ -96,7 +93,7 @@ async def search( error=msg, ) except Exception as e: - msg = f"Regulations.gov search failed: {e}" + msg = f"Regulations.gov search failed: {self._redact(e)}" logger.warning(msg) return SearchResult( source="regulations", @@ -161,7 +158,7 @@ async def fetch(self, identifier: str, **kwargs: Any) -> RecordEntity | None: resp.raise_for_status() data = resp.json() except Exception as e: - logger.warning("Regulations.gov fetch failed for %s: %s", identifier, e) + logger.warning("Regulations.gov fetch failed for %s: %s", identifier, self._redact(e)) return None doc = data.get("data", {}) diff --git a/openfoia/records/usaspending.py b/openfoia/records/usaspending.py index ad03532..8b38179 100644 --- a/openfoia/records/usaspending.py +++ b/openfoia/records/usaspending.py @@ -98,7 +98,7 @@ async def search( ) if resp.status_code == 422: msg = f"USAspending rejected query: {resp.text[:200]}" - logger.warning(msg) + logger.info(msg) return SearchResult( source="usaspending", query=query, diff --git a/tests/test_case.py b/tests/test_case.py index 52091ca..b949e09 100644 --- a/tests/test_case.py +++ b/tests/test_case.py @@ -47,7 +47,7 @@ class Adapter: async def search(self, query, **kwargs): return SearchResult(source="sec", query=query, total_results=total, entities=[]) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) def test_case_lifecycle_attaches_fetches_and_queries(monkeypatch, data_dir): @@ -111,7 +111,7 @@ class Adapter: async def search(self, query, **kwargs): return SearchResult(source="sec", query=query, total_results=0, entities=[], error="x") - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) assert runner.invoke(app, ["records", "search", "PAS", "-s", "sec"]).exit_code == 1 with get_session() as session: diff --git a/tests/test_config_network_keys.py b/tests/test_config_network_keys.py new file mode 100644 index 0000000..c318e25 --- /dev/null +++ b/tests/test_config_network_keys.py @@ -0,0 +1,99 @@ +"""records.api_data_gov_key and network.max_attempts: loading and wiring (#56).""" + +from __future__ import annotations + +import asyncio +import json + +import httpx +import pytest +from typer.testing import CliRunner + +from openfoia import net +from openfoia.cli import _egress_policy_from, app +from openfoia.config import load_config, redact_secrets + +ENV_VARS = ("OPENFOIA_API_DATA_GOV_KEY", "OPENFOIA_NETWORK_MAX_ATTEMPTS") + + +@pytest.fixture(autouse=True) +def data_dir(tmp_path, monkeypatch): + monkeypatch.setenv("OPENFOIA_DATA_DIR", str(tmp_path)) + for var in ENV_VARS: + monkeypatch.delenv(var, raising=False) + return tmp_path + + +def _write(data_dir, data): + (data_dir / "config.json").write_text(json.dumps(data)) + + +def test_defaults(): + cfg = load_config() + assert cfg.records.api_data_gov_key is None + assert cfg.network.max_attempts == 3 + + +@pytest.mark.parametrize("key", ["api_data_gov_key", "_api_data_gov_key"]) +def test_api_data_gov_key_from_file(data_dir, key): + _write(data_dir, {"records": {key: "file-key"}}) + assert load_config().records.api_data_gov_key == "file-key" + + +def test_env_overrides_file(data_dir, monkeypatch): + _write(data_dir, {"records": {"_api_data_gov_key": "file-key"}, "network": {"max_attempts": 5}}) + cfg = load_config() + assert cfg.network.max_attempts == 5 + + monkeypatch.setenv("OPENFOIA_API_DATA_GOV_KEY", "env-key") + monkeypatch.setenv("OPENFOIA_NETWORK_MAX_ATTEMPTS", "1") + cfg = load_config() + assert cfg.records.api_data_gov_key == "env-key" + assert cfg.network.max_attempts == 1 + + monkeypatch.setenv("OPENFOIA_NETWORK_MAX_ATTEMPTS", "lots") + assert load_config().network.max_attempts == 5 + + +def test_key_redacted_in_config_show(): + shown = redact_secrets({"records": {"_api_data_gov_key": "real-key"}}) + assert "real-key" not in json.dumps(shown) + + +def test_records_search_sends_configured_key_and_redacts_it(data_dir, monkeypatch): + _write(data_dir, {"records": {"_api_data_gov_key": "real-key"}}) + seen = [] + + def handler(request): + seen.append(request.url.params.get("api_key")) + return httpx.Response(500, request=request) + + real_client = net.egress_client + monkeypatch.setattr( + "openfoia.records.fec.egress_client", + lambda policy, **kw: real_client(policy, transport=httpx.MockTransport(handler), **kw), + ) + + result = CliRunner().invoke(app, ["records", "search", "Acme", "-s", "fec", "--json"]) + + assert seen == ["real-key"] + assert "real-key" not in result.output + assert "api_key=" in result.output + + +def test_max_attempts_one_sends_a_single_request_on_429(monkeypatch): + cfg = load_config() + cfg.network.max_attempts = 1 + policy = _egress_policy_from(cfg) + calls = [] + + def handler(request): + calls.append(request) + return httpx.Response(429) + + async def go(): + async with net.egress_client(policy, transport=httpx.MockTransport(handler)) as client: + return await client.get("https://api.example.org/x") + + assert asyncio.run(go()).status_code == 429 + assert len(calls) == 1 diff --git a/tests/test_crossref_status.py b/tests/test_crossref_status.py index cb4c0af..5536d63 100644 --- a/tests/test_crossref_status.py +++ b/tests/test_crossref_status.py @@ -34,7 +34,7 @@ async def no_sleep(duration): monkeypatch.setattr( crossref_mod, "_get_available_sources", - lambda icij_data_dir, egress, opensanctions_data_dir=None: { + lambda *_args: { "broken": failing_checker, "working": matching_checker, }, @@ -73,7 +73,7 @@ async def no_sleep(duration): monkeypatch.setattr( crossref_mod, "_get_available_sources", - lambda icij_data_dir, egress, opensanctions_data_dir=None: { + lambda *_args: { "limited": rate_limited_checker, "working": working_checker, }, @@ -160,9 +160,7 @@ async def no_sleep(duration): monkeypatch.setattr( crossref_mod, "_get_available_sources", - lambda icij_data_dir, egress, opensanctions_data_dir=None: { - "icij": partially_failing_checker - }, + lambda *_args: {"icij": partially_failing_checker}, ) monkeypatch.setattr(asyncio, "sleep", no_sleep) diff --git a/tests/test_records_cli.py b/tests/test_records_cli.py index b50942f..5e1b4d9 100644 --- a/tests/test_records_cli.py +++ b/tests/test_records_cli.py @@ -60,7 +60,7 @@ async def search(self, query, **kwargs): ], ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke(app, ["records", "search", "Uranium", "-s", "sec", "--csv", "-"]) @@ -90,7 +90,7 @@ async def search(self, query, **kwargs): ], ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, @@ -137,7 +137,7 @@ async def search(self, query, **kwargs): error="AdapterRequestError: HTTP 503", ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, @@ -166,7 +166,7 @@ async def search(self, query, **kwargs): ], ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, @@ -184,7 +184,7 @@ class Adapter: async def search(self, query, **kwargs): raise RuntimeError("boom") - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, @@ -218,7 +218,7 @@ def ent(name, snippet, ident): ], ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) def test_records_search_rank_scores_and_sorts(monkeypatch): diff --git a/tests/test_records_nara.py b/tests/test_records_nara.py index 77109ff..80ea7f8 100644 --- a/tests/test_records_nara.py +++ b/tests/test_records_nara.py @@ -295,7 +295,7 @@ async def search(self, query, **kwargs): return SearchResult(source="sec", query=query, total_results=1, entities=[]) monkeypatch.setattr("openfoia.records.list_sources", lambda: ["nara", "sec"]) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Quiet()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Quiet()) r = runner.invoke(app, ["records", "search", "x", "--all-sources"]) assert r.exit_code == 0, r.output assert "skipped (no OPENFOIA_NARA_API_KEY)" in r.output and "sec" in r.output diff --git a/tests/test_records_search.py b/tests/test_records_search.py index 0546667..63ec9af 100644 --- a/tests/test_records_search.py +++ b/tests/test_records_search.py @@ -29,7 +29,7 @@ async def search(self, query, **kwargs): assert kwargs == {"filing_type": "F-3"} return SearchResult(source="sec", query=query, total_results=0, entities=[]) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, @@ -51,7 +51,7 @@ class Adapter: async def search(self, query, **kwargs): return SearchResult(source="opencorporates", query=query, total_results=0, entities=[]) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, @@ -97,7 +97,7 @@ async def search(self, query, page=1, page_size=25, **kwargs): source="documentcloud", query=query, total_results=230, entities=ents ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, ["records", "search", "PAS", "-s", "documentcloud", "-n", "250", "--json"] @@ -241,7 +241,7 @@ async def search(self, query, **kwargs): monkeypatch.setattr("openfoia.records.list_sources", lambda: ["govinfo", "muckrock", "sec"]) monkeypatch.setattr( - "openfoia.records.get_adapter", lambda s: Broken() if s == "muckrock" else Ok() + "openfoia.records.get_adapter", lambda s, **_: Broken() if s == "muckrock" else Ok() ) result = CliRunner().invoke(app, ["records", "search", "PAS", "--all-sources"]) @@ -269,7 +269,7 @@ async def search(self, query, page=1, page_size=25, **kwargs): source="documentcloud", query=query, total_results=500, entities=ents ) - monkeypatch.setattr("openfoia.records.get_adapter", lambda source: Adapter()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda source, **_: Adapter()) result = CliRunner().invoke( app, ["records", "search", "PAS", "-s", "documentcloud", "-n", "300", "--json"] @@ -286,7 +286,7 @@ async def search(self, query, **kwargs): raise AdapterRequestError("down") monkeypatch.setattr("openfoia.records.list_sources", lambda: ["muckrock"]) - monkeypatch.setattr("openfoia.records.get_adapter", lambda s: Broken()) + monkeypatch.setattr("openfoia.records.get_adapter", lambda s, **_: Broken()) result = CliRunner().invoke(app, ["records", "search", "PAS", "--all-sources"]) From 32e947d994067e49ac16b8debeef1e9825823c0a Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 11:08:29 -0500 Subject: [PATCH 2/3] fix(config): withhold personal api.data.gov key over Tor; coerce max_attempts from file A personal key is tied to the signup email, so sending it over Tor identifies the user anyway. crossref now falls back to DEMO_KEY in Tor mode and logs that once. A string max_attempts in config.json no longer raises TypeError in every egress client. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- openfoia/config.py | 3 ++- openfoia/crossref.py | 6 ++++++ tests/test_config_network_keys.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/openfoia/config.py b/openfoia/config.py index 42392f1..f14df9b 100644 --- a/openfoia/config.py +++ b/openfoia/config.py @@ -348,7 +348,8 @@ def _merge_config(config: OpenFOIAConfig, data: dict[str, Any]) -> OpenFOIAConfi config.network.tor_host = net.get("tor_host", config.network.tor_host) config.network.tor_port = net.get("tor_port", config.network.tor_port) config.network.isolate_streams = net.get("isolate_streams", config.network.isolate_streams) - config.network.max_attempts = net.get("max_attempts", config.network.max_attempts) + with contextlib.suppress(TypeError, ValueError): + config.network.max_attempts = int(net.get("max_attempts", config.network.max_attempts)) if "records" in data: rec = data["records"] diff --git a/openfoia/crossref.py b/openfoia/crossref.py index 572d4c2..c41cb6a 100644 --- a/openfoia/crossref.py +++ b/openfoia/crossref.py @@ -343,6 +343,12 @@ def _get_available_sources( """ sources: dict[str, Any] = {} + # A personal api.data.gov key is tied to the signup email, so sending it + # over Tor identifies the user anyway. Withhold it and use DEMO_KEY. + if api_data_gov_key and egress is not None and egress.is_tor: + logger.info("Tor mode: not sending the personal api.data.gov key; using DEMO_KEY") + api_data_gov_key = None + # MuckRock — always available (public API, no key) sources["muckrock"] = lambda name, etype: _check_muckrock(name, etype, egress) diff --git a/tests/test_config_network_keys.py b/tests/test_config_network_keys.py index c318e25..cc47833 100644 --- a/tests/test_config_network_keys.py +++ b/tests/test_config_network_keys.py @@ -97,3 +97,32 @@ async def go(): assert asyncio.run(go()).status_code == 429 assert len(calls) == 1 + + +def test_max_attempts_from_file_is_coerced_and_bad_values_ignored(data_dir): + _write(data_dir, {"network": {"max_attempts": "5"}}) + assert load_config().network.max_attempts == 5 + _write(data_dir, {"network": {"max_attempts": "lots"}}) + assert load_config().network.max_attempts == 3 + + +def test_personal_data_gov_key_is_withheld_over_tor(monkeypatch): + from openfoia import crossref + from openfoia.net import EgressMode, EgressPolicy + + seen = {} + + async def fake_fec(name, etype, egress, api_key=None): + seen["fec"] = api_key + return None + + monkeypatch.setattr(crossref, "_check_fec", fake_fec) + tor = EgressPolicy(mode=EgressMode.TOR) + sources = crossref._get_available_sources(egress=tor, api_data_gov_key="personal") + asyncio.run(sources["fec"]("Acme", "ORGANIZATION")) + assert seen["fec"] is None + + direct = EgressPolicy() + sources = crossref._get_available_sources(egress=direct, api_data_gov_key="personal") + asyncio.run(sources["fec"]("Acme", "ORGANIZATION")) + assert seen["fec"] == "personal" From 2afa308f9ba96c75f4f792cf9875ae8bf363d6ff Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 11:11:22 -0500 Subject: [PATCH 3/3] fix(crossref): pass literal DEMO_KEY over Tor so env fallbacks cannot re-inject a personal key Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- openfoia/crossref.py | 11 ++++++++--- tests/test_config_network_keys.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/openfoia/crossref.py b/openfoia/crossref.py index c41cb6a..a96ad05 100644 --- a/openfoia/crossref.py +++ b/openfoia/crossref.py @@ -345,9 +345,14 @@ def _get_available_sources( # A personal api.data.gov key is tied to the signup email, so sending it # over Tor identifies the user anyway. Withhold it and use DEMO_KEY. - if api_data_gov_key and egress is not None and egress.is_tor: - logger.info("Tor mode: not sending the personal api.data.gov key; using DEMO_KEY") - api_data_gov_key = None + # Pass the literal DEMO_KEY, not None, so no adapter-level env fallback + # (GovInfo's OPENFOIA_GOVINFO_API_KEY) can re-inject a personal key. + if egress is not None and egress.is_tor: + from .records.base import DEMO_KEY + + if api_data_gov_key: + logger.info("Tor mode: not sending the personal api.data.gov key; using DEMO_KEY") + api_data_gov_key = DEMO_KEY # MuckRock — always available (public API, no key) sources["muckrock"] = lambda name, etype: _check_muckrock(name, etype, egress) diff --git a/tests/test_config_network_keys.py b/tests/test_config_network_keys.py index cc47833..79e3079 100644 --- a/tests/test_config_network_keys.py +++ b/tests/test_config_network_keys.py @@ -116,11 +116,21 @@ async def fake_fec(name, etype, egress, api_key=None): seen["fec"] = api_key return None + async def fake_govinfo(name, etype, egress, api_key=None): + from openfoia.records.govinfo import GovInfoAdapter + + seen["govinfo"] = GovInfoAdapter(api_key=api_key, egress=egress)._api_key + return None + monkeypatch.setattr(crossref, "_check_fec", fake_fec) + monkeypatch.setattr(crossref, "_check_govinfo", fake_govinfo) + monkeypatch.setenv("OPENFOIA_GOVINFO_API_KEY", "personal-govinfo") tor = EgressPolicy(mode=EgressMode.TOR) sources = crossref._get_available_sources(egress=tor, api_data_gov_key="personal") asyncio.run(sources["fec"]("Acme", "ORGANIZATION")) - assert seen["fec"] is None + asyncio.run(sources["govinfo"]("Acme", "ORGANIZATION")) + assert seen["fec"] == "DEMO_KEY" + assert seen["govinfo"] == "DEMO_KEY" direct = EgressPolicy() sources = crossref._get_available_sources(egress=direct, api_data_gov_key="personal")