diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 24b36f7..984c735 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -28,8 +28,10 @@ openoutfind/ # config.py (the environment) + readiness.py, llm.py, conf.py, # ML (qualifier/embeddings), discovery+qualify pipeline, # the lead export, db/ helpers, geo, management commands - enrichment/ # the one paid step (not an app — no models) — the BetterContact - # client and the two-step buy/check lookup + enrichment/ # the one paid step (not an app, no models): the finder clients + # (Enrow, BetterContact, FullEnrich, and LeadMagic, the + # last synchronous), the price-aware selection seam + # (provider.py), and the buy/check lookup crm/ # app (label: outfind_crm) — Lead, Company, Deal contacts/ # central contacts-store client (service.py only — no models, not an app) ``` @@ -196,9 +198,15 @@ What a program depends on, and the reason both exist as one place rather than a `bad_config: OPENOUTFIND_LLM_API_KEY`. It used to catch `Exception`, so when `anthropic` 1.0.0 dropped `temperature` the resulting `TypeError` in our own process read as a rejected key and sent the operator after credentials that were fine. A broken install is a bug and gets its traceback. -- **The provider's refusals are three different things**, typed at the HTTP boundary in - `bettercontact._request`: 401 → `provider_auth`, 402 → `provider_out_of_credits`, an exhausted 429 - backoff → `provider_rate_limited`. Everything else stays `provider_unavailable`. +- **A provider's refusals are three different things**, typed at the HTTP boundary in each adapter's + `_request` (`bettercontact`, `enrow`, `fullenrich` and `leadmagic`): 401 → `provider_auth`, 402 → + `provider_out_of_credits`, an exhausted 429 backoff → `provider_rate_limited`. Everything else + stays `provider_unavailable`, which is also what a finder that cannot run at all raises (each + adapter's own `BetterContactUnavailable` / `EnrowUnavailable` / `FullEnrichUnavailable` / + `LeadMagicUnavailable`, all subclasses of `provider.ProviderUnavailable`, carrying the vendor's + message). Every one of them is a + *couldn't-run*, and that is the distinction `lookup.buy_address` acts on: it steps to the next + finder in the chain, where a terminal miss ends the lookup rather than spending a second credit. - **A 429 is backed off, never retried at speed** — their docs warn that a client which keeps firing can get the *account* blocked. The backoff is `urllib3.Retry` mounted on the session (`_RETRY`: 5 attempts, `backoff_factor=5` doubling, `backoff_max=120`, `respect_retry_after_header`, @@ -282,7 +290,7 @@ the job. |:----|:----------------| | `config` | `complete`, the satisfied groups, and per-group the variables that would satisfy the rest | | `totals` | the pipeline counts | -| `credits` | `balance` + `error` — `GET /api/v2/account` → `credits_left` | +| `credits` | `balance` + `error` for the finder `active()` names, with `provider` beside them — Enrow's read is the free `GET /account/info` → `credits`, BetterContact's is `GET /api/v2/account` → `credits_left`, LeadMagic's is the free `GET /v1/credits`; FullEnrich has none, so it is *unknown* | | `hub` | `balance` + `known` — the give-to-get counter (`contacts.service.hub_balance`), a different number on a different service than `credits` | | `blocked` | what stands between now and more qualified rows, typed from `core/errors.py` | | `next_action` | the one thing to do next, with what it unlocks and the command or URL that does it | @@ -301,13 +309,17 @@ Three decisions inside it: not break the *never before value* rule — `ranked_for_lookup > 0` is itself the proof that qualified leads with written reasons exist, and an install that has qualified nobody is asked for no money, only told to go and find some (`find_leads`). -- **The hub balance is not the `Credits:` line.** `credits` is BetterContact's own prepaid balance - (`GET /api/v2/account`); `hub` is the give-to-get counter on the contacts store — a different - number on a different service, so it gets its own key rather than being folded into or shown - beside `credits` under one label. Read via `hub_balance()`, which piggybacks on `register` (a +- **The hub balance is not the `Credits:` line.** `credits` is whichever finder `provider.active()` + names, which is the cheapest configured one, reported with a `provider` key beside its balance + (`GET /account/info` for Enrow, `GET /api/v2/account` for BetterContact, `GET /v1/credits` for + LeadMagic); `hub` is the give-to-get counter on the contacts store, a different number on a + different service, so it gets its own key rather than being folded into or shown beside `credits` + under one label. Read via `hub_balance()`, which piggybacks on `register` (a record-less call is idempotent server-side and already returns `credits`), so checking it spends nothing. `known: False` (no token yet, or a hub outage) must render as *unknown*, never as zero — - the same rule the provider balance already follows. + the same rule the provider balance already follows, and a FullEnrich install lives under that rule + permanently because FullEnrich exposes no balance read, so `credit_balance()` raises rather than + guess. ## Configuration (`core/config.py`) and the readiness check (`core/readiness.py`) @@ -326,6 +338,9 @@ in its own `SiteConfig` and exports these names to both children. campaign OPENOUTFIND_PRODUCT_DOCS, OPENOUTFIND_CAMPAIGN_TARGET (+ MAX_PER_COMPANY, optional) llm OPENOUTFIND_AI_MODEL, OPENOUTFIND_LLM_API_KEY (+ LLM_API_BASE, required for openai_compatible:*) bettercontact OPENOUTFIND_BETTERCONTACT_API_KEY +fullenrich OPENOUTFIND_FULLENRICH_API_KEY +enrow OPENOUTFIND_ENROW_API_KEY +leadmagic OPENOUTFIND_LEADMAGIC_API_KEY account OPENOUTFIND_OPERATOR_EMAIL, OPENOUTFIND_OPERATOR_COUNTRY (optional) hub OPENOUTFIND_CONTACTS_API_TOKEN (optional) ``` @@ -377,7 +392,8 @@ stops. `core/export.py` reads it from there. - **`FINDING_EMAIL`** — a provider job is in flight; the deal is excluded from the candidate pool (so the next cycle can't re-select it and double-charge) while `check_lookup` polls to termination. The job handle (`lookup_request_id`) and the poll backoff (`lookup_attempt` + `not_before`) live **on the deal**, so an in-flight lookup survives a restart and its wait gates that one row and nothing else. - **`RESOLVED`** — an address is in hand. Where a fully-enriched deal comes to rest, at no cost, since nothing iterates it. -**The paid lookup runs behind a provider seam** (`enrichment/provider.py`): `active()` names the finder from whichever key is configured — BetterContact is the only one upstream ships — and nothing above the package knows the vendor. `buy_address` resolves free-hub-first (hit → `RESOLVED` with no job/credit), then calls the finder's `start`. **The transport is the provider's, and the interface spans both**: a synchronous finder (a match-style API a fork might add) comes back already terminated and lands on `RESOLVED`/`NO_EMAIL_FOUND` without ever entering `FINDING_EMAIL`; an async one (BetterContact's waterfall) returns a handle and parks there. A couldn't-run (no key / API down) stays `READY_TO_FIND_EMAIL`. The handle records **which** finder minted it (`Deal.lookup_provider`), so swapping keys mid-flight cannot hand one vendor's `request_id` to another. `check_lookup` (poll) is then **tri-state**: hit → `RESOLVED` (address given back to the hub); **miss** (job terminated, no address) → `NO_EMAIL_FOUND` — its own terminal, with **no reason and blank outcome**, critically not `FAILED+wrong_fit`, which the ML labeler reads as a negative: a lead we simply couldn't reach was still an LLM fit positive and stays label=1; **still running** → chains the next poll with doubled backoff on the same `request_id`, with no deadline and no attempt limit. +**The paid lookup runs behind a provider seam** (`enrichment/provider.py`): `active()` names the cheapest finder by `UNIT_COST_USD` among those holding a key, and `chain()` is that one followed by every other configured finder, cheapest first. Nothing above the package knows the vendor. `buy_address` resolves free-hub-first (hit → `RESOLVED` with no job/credit), then walks the chain. **The transport is each provider's, and the interface spans both**: a synchronous finder (LeadMagic's one-call finder, or a match-style API a fork adds) comes back already terminated and lands on `RESOLVED`/`NO_EMAIL_FOUND` without ever entering `FINDING_EMAIL`; an async one (Enrow's single search, BetterContact's waterfall, FullEnrich's bulk enrich) returns a +handle and parks there. A finder that cannot run — every `ProviderUnavailable`, whether a typed refusal or a missing key or an API that is down — steps to the next finder in the chain, and only when the whole chain has refused does the deal stay `READY_TO_FIND_EMAIL`. A finder that *does* run is authoritative: a terminal miss ends the lookup there rather than spending a second credit. The handle records **which** finder minted it (`Deal.lookup_provider`), so swapping keys mid-flight cannot hand one vendor's `request_id` to another and `check_lookup` dispatches by name. The poll is then **tri-state**: hit → `RESOLVED` (address given back to the hub); **miss** (job terminated, no address) → `NO_EMAIL_FOUND` — its own terminal, with **no reason and blank outcome**, critically not `FAILED+wrong_fit`, which the ML labeler reads as a negative: a lead we simply couldn't reach was still an LLM fit positive and stays label=1; **still running** → chains the next poll with doubled backoff on the same `request_id`, with no deadline and no attempt limit. `crm/models/deal.py:Outcome` is down to **two** values: `wrong_fit` and `unknown`. The reply outcomes it used to carry — converted, not_interested, no_budget, has_solution, bad_timing, @@ -527,7 +543,7 @@ end honestly: a row that waits stops being due, so `run_one_action` can return ` `HALTING_ERRORS` (today `ModelHTTPError`) is the exception: a misconfigured LLM key ends the job with a typed `bad_config` line rather than being retried forever behind an `alive` log line. -1. **`buy_address`** (`enrichment/lookup.py`) — resolves cheapest-first: an address already on the lead → `RESOLVED`; the free hub cache (`contacts.resolve`) → `RESOLVED`; else `bettercontact.submit` fires a job, stores `lookup_request_id`, and parks at `FINDING_EMAIL`. Couldn't-submit (no key, API down) stays in `READY_TO_FIND_EMAIL` — no credit spent, no handle to poll — but **backs the deal off first** (`_back_off`, the same doubling the poll uses). Without that the row is still due on the next pass, so the same deal is re-picked immediately and forever: noise every few seconds under the daemon, an endless job now that a run stops only when nothing can advance. *Queued* and *due* are different things. +1. **`buy_address`** (`enrichment/lookup.py`) — resolves cheapest-first: an address already on the lead → `RESOLVED`; the free hub cache (`contacts.resolve`) → `RESOLVED`; else the chain's leading finder (cheapest configured first) fires a job through its `start`, `Deal.lookup_provider` records which one, and the handle lands in `lookup_request_id` as the deal parks at `FINDING_EMAIL`. Couldn't-submit (no finder key, API down) stays in `READY_TO_FIND_EMAIL` — no credit spent, no handle to poll — but **backs the deal off first** (`_back_off`, the same doubling the poll uses). Without that the row is still due on the next pass, so the same deal is re-picked immediately and forever: noise every few seconds under the daemon, an endless job now that a run stops only when nothing can advance. *Queued* and *due* are different things. 2. **`check_lookup`** (same module) — polls `lookup_request_id` once: hit → `RESOLVED` + hub give-back; miss → `NO_EMAIL_FOUND` (no reason, blank outcome); still-running → double `not_before`. **The only terminal outcomes are the provider's own** — no deadline, no attempt limit. Both were tried: past the deadline the leg abandoned the job and reverted the deal, where the buy step bought a *second* job for the same lead, so a provider outage became a hot resubmit loop (418 submits and 4,512 polls in a week for ~40 leads, none terminating). Doubling makes waiting nearly free (a week costs 17 polls) and refuses to mislabel — a timeout is evidence about the provider, not about the lead. The interval rails at `COLLECT_BACKOFF_MAX_S` (a month) only so `datetime` can still express it. A deal parked here with an **empty** `lookup_request_id` has no job and never cost a credit, so row 1 routes it to **`reclaim_lookup`** → `READY_TO_FIND_EMAIL` instead. The row used to `.exclude(lookup_request_id="")` and skip it, which stranded it in a state no other row claims — measured on a live install: two deals stuck for 206 hours. A third and fourth step stood here — `send_first_email` and `answer_reply` — and both left with @@ -707,7 +723,7 @@ rather than pinned to a single hallucination. - **`core`** — Engine: the `Search` / `Page` / `PageLead` walk models; the cycle, operator lookup, LLM factory, the environment config + readiness check, the ML/discovery/qualify pipeline, the lead export, geo. - **`crm`** — `Lead` (identity + embedding + email), `Company` (the shared employer row) and `Deal` (`crm/models/lead.py`, `crm/models/company.py`, `crm/models/deal.py`); also defines `DealState` and `Outcome`. -- **`enrichment`** — **not an app** (no models). `bettercontact.py` (paid finder: the two-leg `submit(query)→request_id` + `poll_once(request_id)→PollOutcome`, the blocking `search_and_poll` transport used by discovery, the shared `RequestBudget`, `is_configured`, `BetterContactQuery`/`Result`/`PollOutcome`/`Unavailable`); `lookup.py` (`buy_address`/`check_lookup`/`reclaim_lookup` — one entity in, the next `DealState` or `None` out). It sat under `emails/` while a resolved address existed to be written to; that coupling is exactly what made a mailbox-less install produce nothing. +- **`enrichment`** — **not an app** (no models). `bettercontact.py` (paid finder: the two-leg `submit(query)→request_id` + `poll_once(request_id)→PollOutcome`, the blocking `search_and_poll` transport used by discovery, the shared `RequestBudget`, `is_configured`, `BetterContactQuery`/`Result`/`PollOutcome`/`Unavailable`); `fullenrich.py` (the second finder, and a different shape of one: the bare `professional_network_url` is a complete query, `submit` → `enrichment_id` and `poll_once` → `PollOutcome` once `FINISHED`, the vendor's statuses mapped through, its own `RequestBudget`, and no balance read because the vendor exposes none); `leadmagic.py` (the third finder and the only synchronous one: `start` is the whole lookup and returns a finished `PollOutcome`, so it has no `poll_once` and leaves no handle to strand, one POST to `/v1/people/email-finder` with nothing but the profile URL, a `valid` status or a miss, and a real balance read from the free `GET /v1/credits`); `enrow.py` (the fourth finder added, and the cheapest: `start(profile_url)` POSTs one `linkedin_url` and returns the vendor's `id`, `poll_once(id)` → `PollOutcome` while it runs at HTTP 202, `valid`/`invalid` mapped through, a `lookup_failed` LinkedIn status deferred rather than read as a miss because the vendor bills nothing for it, a `RequestBudget` of 120 a minute with the free `GET /account/info` exempt, and a real balance read from that same endpoint); `provider.py` (`configured`/`active`/`chain`/`by_name` — which finder is cheapest among those holding a key, and the order the rest are tried in; no HTTP of its own); `lookup.py` (`buy_address`/`check_lookup`/`reclaim_lookup` — one entity in, the next `DealState` or `None` out). It sat under `emails/` while a resolved address existed to be written to; that coupling is exactly what made a mailbox-less install produce nothing. - **`contacts`** — the central contacts-store client (`service.py`, no models, **not** an installed app) — "the hub" (`hub.openoutreach.app`), logged under the `hub:` prefix. `resolve(lead)` (free read-back before the paid finder) `contribute(lead, emails, origin)` (give-back, non-EEA only, registers on first use), `share_profiles(rows, country_code)` (every discovery page, non-EEA only, no credit) and `share_companies(rows, country_code)` (that page's distinct employers, same gates, no person attached). All best-effort; an outage or missing token degrades to a no-op. **Three apps are gone.** `emails` (Mailbox, the mail log, sender, the send guards, two steps) went @@ -954,6 +970,10 @@ Paths relative to `openoutfind/`. - **`core/llm.py`** — `get_llm_model()` factory (reads the run's `SiteConfig`, `split_model_id` parses the provider out of `ai_model`, dispatches to the per-provider builder), `build_llm_model` (from explicit creds), `verify_llm_credentials` (one live ping, tenacity-retried, run before every pass by `readiness.check_ready` — it returns only the failures a *configuration* causes and lets anything else propagate, since its caller blames `LLM_API_KEY` for whatever comes back), and `run_agent_sync(coro)` — the sync boundary that drives async pydantic-ai on a dedicated long-lived worker-thread loop (never `Agent.run_sync`, whose anyio portal poisons the caller thread's loop slot; never per-call `asyncio.run`, which closes loops the SDK HTTP clients still reference). - **`core/geo.py`** — jurisdiction sets + predicates: `is_gdpr_protected` (broad opt-in set, read only by OpenOutreach's wizard for its newsletter default) and `is_eea_located` / `EEA_UK_CH` (narrow EEA/UK/CH collection-regime set — the client-side pre-gate for contacts-store contribution; the server re-gates authoritatively). The operator's country is `OPENOUTFIND_OPERATOR_COUNTRY`; a lead's is the one its query node searched. Neither ever comes from a scrape. - **`enrichment/bettercontact.py`** — the provider client. The paid finder is the two-leg `submit(query) → request_id` + `poll_once(request_id) → PollOutcome`, so a run never blocks on a poll; the free Lead Finder index uses the blocking `search_and_poll` transport from the same module, since `discovery.search` genuinely wants a page back: its first poll goes at the measured ready time, the rest a second apart (enrichment keeps `_POLL_INTERVAL_S`, 5s). Every request but `GET /account` first takes a place in `_BUDGET` (`RequestBudget`, `REQUESTS_PER_MINUTE` = 15 in any 60 seconds, thread-safe), one process's share of the key's 100/min. `is_configured()` reads `OPENOUTFIND_BETTERCONTACT_API_KEY` — one key, two endpoints, and only one of them bills. +- **`enrichment/fullenrich.py`** — the second finder, and a different shape of one: a bare `professional_network_url` is a complete query, so `submit(url) → enrichment_id` + `poll_once(enrichment_id) → PollOutcome` mirror the first finder's two legs without the name/company fields. The vendor's job statuses are mapped through (`CREDITS_INSUFFICIENT` → out of credits, `RATE_LIMIT` → rate limited, `CANCELED` → a miss, anything unfinished → still running), only `DELIVERABLE` / `HIGH_PROBABILITY` count as a usable address, and `credit_balance()` raises instead of guessing because the vendor exposes no balance read at all (so a FullEnrich install renders its balance as *unknown*). It carries its own `_BUDGET` with no exempt account read, since there is no account read to exempt. +- **`enrichment/leadmagic.py`** — the third finder and the only synchronous one: `start(profile_url)` is the whole lookup, a single POST to `/v1/people/email-finder` carrying nothing but the URL, so the answer comes back already finished (`valid` and an address, or a free miss) and there is no `poll_once` here at all, no handle for a vendor to strand. It keeps its own `RequestBudget` (60 a minute, a courtesy fifth of the 300 the entry plan publishes per organisation) with the free `GET /v1/credits` exempt, and unlike FullEnrich it reads a real balance, floor-don't-round, since `status` shows it. +- **`enrichment/enrow.py`** — the fourth finder added, and the cheapest: `start(profile_url)` POSTs a single `linkedin_url` to `/email/find/single` and returns the vendor's handle (`id`), `poll_once(id)` reads it back as a `PollOutcome` while the search runs at HTTP 202, and `valid`/`invalid` are the only two verdicts that settle a lookup (`valid` carries the address). A `lookup_failed` LinkedIn status is transient and free, so it defers rather than recording a miss, and an unknown or expired handle comes back as a 500 rather than a 404, which also defers. It keeps its own `RequestBudget` (120 a minute, a courtesy fifth of the documented 10 a second) with the free `GET /account/info` exempt, and unlike FullEnrich it reads a real balance, floor-don't-round, since `status` shows it. +- **`enrichment/provider.py`** — the selection seam and no HTTP of its own: `configured()` (the finders holding a key), `active()` (the cheapest of them by `UNIT_COST_USD`), `chain()` (that one first, then the rest cheapest-first), and `by_name()` (dispatch for a parked lookup, from `Deal.lookup_provider`). Also home to `ProviderUnavailable` (the couldn't-run base every adapter subclasses), `PollOutcome`, and `Lookup`. - **`enrichment/lookup.py`** — the two pipeline steps, `buy_address` / `check_lookup` / `reclaim_lookup`, plus `_store_identity` (the name parts the provider echoes back with the address) and the backoff helpers. The enrichment query is **URL-only by decision** — the provider accepts name and company and resolves better with them, but the less of a lead's record leaves for a third party the better, and URL-only measures ~42% usable. The docstring says not to widen it without a decision to widen it. - **`core/business_time.py`** — `business_days_between(start, end)`: whole Mon–Fri days elapsed. It was the agent's only sense of a thread's age; with no agent it is now unused by the pipeline and kept as a small, correct utility. Public holidays are not modelled (per-country data we don't carry). - **`core/logging.py`** — `configure_logging` + `print_banner`; `SILENCED_LOGGERS` quiets urllib3/httpx/pydantic_ai/openai/fastembed/etc.; `format_elapsed` (`52s` / `4m09s` / `1h04m`) for the milestones. diff --git a/CLAUDE.md b/CLAUDE.md index 5cca494..28060bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ OpenOutFind — a self-hosted **lead finder that qualifies for you**, with **zer - **Discovery** is a licensed source — BetterContact **Lead Finder** (ICP search returns firmographic profiles, no emails, billed nothing). The walk is a best-first search: the LLM writes the first 4, and each later round fires the one search — a next page, or a search composed from the qualified profiles' keywords — that one draw of the belief puts first among the ten worth the most at the mean (Thompson sampling), on a belief per keyword and per fired search measured by the people searches brought, five of whom are judged at random per search. - **Qualification** is the crown jewel — **GPR + BALD active learning** over 384-dim embeddings, decided by an LLM, which also **writes down why**. That reason is the product. -- **Enrichment** is the one paid step — a **finder** resolves a work email for the top-ranked leads only (one credit per verified hit), fronted by a free cross-operator cache (the hub). Optional: a lead with no address still exports. **The resolver sits behind a seam** (`enrichment/provider.py`), which picks a finder from whichever key is configured — BetterContact (async waterfall) is the only one upstream ships, and the seam's `Lookup` still spans a synchronous finder so a fork can add one without touching the pipeline. **Only the resolver leg is behind it** — discovery is BetterContact's Lead Finder and nothing else. +- **Enrichment** is the one paid step — a **finder** resolves a work email for the top-ranked leads only (one credit per verified hit), fronted by a free cross-operator cache (the hub). Optional: a lead with no address still exports. **The resolver sits behind a seam** (`enrichment/provider.py`), which picks a finder from whichever key is configured: BetterContact and FullEnrich are the two upstream ships, both async waterfalls, and each declares its `UNIT_COST_USD` so price decides the order rather than a hardcoded list. `active()` is the cheapest finder holding a key; `chain()` is that one followed by every other configured finder, cheapest first. `lookup.buy_address` walks the chain: a finder that cannot run (`ProviderUnavailable`) steps to the next, a terminal miss does not. The seam's `Lookup` still spans a synchronous finder so a fork can add one without touching the pipeline. **Only the resolver leg is behind it**: discovery is BetterContact's Lead Finder and nothing else. - **The output is the command's own stdout.** `outfind find 10 emails > leads.csv` prints **every lead in the store** in the column names Instantly and Smartlead already read, so the newest file supersedes every earlier one. No export command, and no file the tool manages. See *The lead export* below. Django + Django Admin own the CRM/ORM (models are this project's); pydantic-ai drives the LLM calls. @@ -68,16 +68,16 @@ For detailed module docs, see `ARCHITECTURE.md`. - **Entry**: `openoutfind/__main__.py:main` — the `outfind` console script, and the one entry point; it strips the global `--db` flag before Django's per-command parsing. A bare invocation prints the command list — it used to default to `run`, and with the work verb bounded there is nothing sensible to default *to*, since `find` needs a number and choosing one would spend the operator's credits on a guess. `manage.py` is a thin shim over it for work inside a checkout. **Three verbs, chosen rather than accumulated**: `check [--product-docs F] [--target F] [--json]` · `find N [emails] [--emails] [--new] [--json] [--batch] [--open] [--debug] [--exclude PATH] [--brief PATH] [--labels PATH] [--agent-qualify [--verdict fit|no-fit --reason MSG] [--population JSON] [--keyword-guesses JSON]]` · `status [--json]`. **`--labels PATH`** (`core/pipeline/labels.py`) takes a customer's marks — the hub's `/ratings/`, a JSON array or JSON Lines — in as verdicts on a fresh store: each becomes a `Lead` with `marked=True` and a `Deal` (`good` and *already a customer or contact* `QUALIFIED`, any other *not a fit* but *gone* `FAILED` + `wrong_fit` with the customer's reason; *not sure*, blank and *gone* left out, a mark with no `profile_text` listed). The GP and the walk's term pools read them as verdicts (a mark is on no page, so it never scores a search); the export, the promote pool (`deals._deals_at_state`), `pipeline_summary` and `status` leave marked leads out, so one is never judged, printed or looked up. A URL already holding a deal is skipped, so the flag rides every call; new marks on a store with verdicts of its own are `bad_config`. The run object under `--json` carries what it did as `labels`. **`--brief PATH`** replaces the store's brief (`core/pipeline/brief.py`, one `core.Brief` row): our reading of the customer's two texts — who fits, who does not, which companies look like buyers and are not — rendered as `rules` into `qualify_lead.j2`, `icp_filters.j2` and `anchor_profiles.j2`. It is *produced*, so it lives in the store rather than the environment, which holds only what a person typed; `AI_MODEL` writes it (`write_brief.j2`) the first time a prompt needs one, and a stronger model than the judge hands one in with `--brief`. Under `--agent-qualify` nothing writes it, and the `qualify_pending`/`population_pending` payloads carry it as `rules`. `status` reports it under `brief`. **`--exclude PATH`** (`core/exclude.py`, a contextvar like `agent_qualify`) is a file of LinkedIn URLs whose leads are never put up for a verdict (`fetch_qualification_candidates`, and a pending `--agent-qualify` row naming one is dropped) and never exported (`lead_records`, which the goal also counts through); `normalize_linkedin_url` is the hub's, verbatim. **`--agent-qualify` is the opt-in, additional way into the qualify step** for a caller that is itself an LLM (Claude Code driving the CLI) and would otherwise pay for a second, redundant model call: it stops at the first candidate needing a verdict instead of calling `AI_MODEL`, raising `qualify_pending` (`core/pipeline/qualify.py:QualifyPending`) with the candidate's own fields riding the error's `payload` (`OpenOutFindError.payload`, merged into the `--json` error object alongside `type`/`message` — the one error type that needs more than those two to be *acted on*; in plain mode `find._narrate_pending_candidate` logs the `profile_url` and `profile_text` above the one-line error, so a verdict never needs `--json` just to see the profile). **The cold start is answered the same way**: on a store with no searches or anchors yet, `icp.agent_population_answer` stops the run with `population_pending` (`qualify.PopulationPending`, a `QualifyPending` subclass so every layer already lets it through) carrying `product_docs`, `campaign_target`, the brief as `rules`, `population_size`, the field types (`icp.FIELD_TYPES`), the seniority vocabulary, `anchor_count` and `PopulationAnswer`'s JSON schema; the agent re-runs with `--population ''` — one object holding the 4 written searches, the market and size band, and the anchor profiles, validated before any work (`bad_config` if it does not fit), read (not consumed) by `write_initial_population` and `generate_anchors` alike. The model path writes the same searches (`InitialPopulation`) in one call and the anchors in another. Without this a fresh store under `--agent-qualify` crashed on the missing model key. At most one candidate is ever pending — `crm.PendingQualification`, a one-row table a second, separate CLI invocation reads to resume *that* candidate rather than asking the qualifier's balance-driven pick to choose again — and the verdict itself is threaded from the CLI flag down to `run_qualification` via a contextvar (`core/agent_qualify.py`), not a parameter, because none of `job → cycle → top_up` has any other reason to know an agent is answering instead of a model. **`reset` was deleted 2026-08-21 to keep the surface lean** — everything an install writes hangs off `settings.state_dir()`, so `rm -rf ~/.openoutfind` plus a `find` is the reset and configuration comes back from the environment. The real cost: the initial population is only written when there are **no** searches yet, so there is no way to re-seed a walk while keeping its leads and verdicts. **There is no `--campaign` flag any more** — this install has never run more than one campaign, so the `Campaign` model was folded away (one implicit campaign, no selection) and every `campaign=` scoping FK went with it; its text is two environment variables and its walk is the `Search`/`Page` rows. **Spending is opt-in at every layer** (2026-08-21): `buy_addresses` defaults to `False` in `run_job` and `run_one_action`, `--emails` turns it on, and the `emails` unit implies it — the noun says what to *count*, the flag says what may be *paid for*. It used to be the other way round (`--no-emails` to opt out), so a run counting leads quietly bought addresses for whatever an earlier run left past the confidence gate; a flag you forget should cost a feature, never money. `--debug` is the shorthand for `--log-level debug` (same dest, so they cannot disagree). **Neither the CSV nor the daemon is a verb** — `export_leads` was deleted when the run started writing the file itself, and then the file went too when `run` became `find` and the rows became its stdout. `setup_crm` lost its verb — it was an implementation detail, and `find` calls the function; `reset_data` and then `reset` itself went the same way. **`--json` belongs on every verb with a result**: `check`, `find` and `status`, which is all of them — and **a failure answers in the caller's format too**, so `--json` turns the `error:` line into `{"error": {"type", "message"}}`, still on stderr (stdout stays result-only, or `find --json > leads.json` would collect an error object). The `find` command (`core/management/commands/find.py`) does: migrate → CRM bootstrap → `check_ready()` → `run_job` → print. **There is no wizard and no onboarding here** (`core/readiness.py` replaced `core/onboarding.py` + the vendored `onboarding_wizard.py`, and `init` became `check`): the environment is the *only* way in, so a run that lacks something raises **one** error naming every variable that would have satisfied it. Required: `OPENOUTFIND_PRODUCT_DOCS`, `CAMPAIGN_TARGET`, `BETTERCONTACT_API_KEY`, plus `AI_MODEL`/`LLM_API_KEY` unless `--agent-qualify`. Optional: `LLM_API_BASE` (openai_compatible only), `OPERATOR_EMAIL`, `OPERATOR_COUNTRY`, `CONTACTS_API_TOKEN`. Grouped as `campaign`/`llm`/`bettercontact` because that is how a person goes and finds the values. Three rules hold it honest: **the model is pinged before any work**, since a check rather than a row is what makes reading fresh safe; a set-but-invalid value is `bad_config` rather than falling through to "missing"; and the operator `User` is written **once** (identity is a record — it is the contacts-store key, and a renamed variable must not rename the person). Without `OPERATOR_EMAIL` the row is `operator` and the install has no hub identity (no store reads, no give-back); an email given later fills that blank once. **There is no newsletter here** — subscribing a human is the OpenOutreach wizard's job. *(The `mailbox` and `signature` steps went with the sending leg — that was most of the install path, and behind the mailbox step stood the disclosure that the tool would send the maintainer's promo campaign from the operator's own mailbox.)* - **The run's narrative**: what `find` prints on stderr while it works **is** the product surface, and it follows a script written down first — `openoutreach-docs/docs/first-run-design.md` §6. **Two levels, and nothing is deleted between them**: INFO is addressed to the operator (every line answers a question they would ask), DEBUG carries *everything INFO ever said plus the engine's own reasoning* — the posterior behind the spend gate, each round's search with its expected rate, count, seen and new people, its count against the provider's, what became of the last page, dead searches, the initial population, the anchor top-up, and the cycle's per-row decisions and timings. A line that stops being operator-facing is **demoted, never removed**. **The GP's acquisition strategy and class counts moved the other way, to INFO** (`qualify.run_qualification`, `top_up._advance`) — *which regime picked this candidate and why* (`cold phase`, `exploit (p)`, `explore (BALD)`, with the neg/pos count that decided it) is a question an operator watching the run asks, not just the engine reasoning about itself. The beats: minute 0 states campaign + goal + whether the run may spend (`find._announce_the_run`); the **ICP echo** prints the synthetic ideal profiles as *"Looking for people like:"* (`icp.log_icp_echo` — already computed, previously shown only as a count, and the earliest chance to notice the product description was misread; a first run has none until the job writes them); **both verdicts are printed** with their reason, `✗` against `✓` (`qualify._verdict_line` — watching it turn people down is what makes the acceptances credible, and `logblock.step_line`'s fixed-width label column is the wrong primitive for a person's name); progress counts **toward the goal the operator typed** with `★ first lead · 52s` as its own milestone (`job._log_progress`, `JobResult.elapsed` — reported, never enforced; there is still no timeout); and the gate that is actually holding is named with **every consequence it has** (`cycle.pipeline_summary` — the finder key is one key doing two jobs, so losing it stops discovery *and* the paid lookup while leaving the free address sources; `buy_addresses` is passed in because *addresses not requested* is the gate most likely to be holding on a bare `find`). `logblock.py`'s grammar was **not** replaced — the redesign happened inside it. - **The output contract**: `core/management/base.py:OpenOutFindCommand` + `core/errors.py`. **stdout is result-only** — logs, the banner and Django's own `migrate` narration all go to **stderr** (`core/logging.py`, and `core/management/bootstrap.py:ensure_database` passes `stdout=stderr` into `call_command("migrate")`), so a redirected stdout yields data and nothing else. **An expected failure is one line**: `error: : ` on stderr + exit 1, `type` from `ErrorType`, and *no traceback* — a rejected key is an answer, not a bug (an unexpected exception still raises). `OpenOutFindError` deliberately does **not** subclass `CommandError`, because Django prefixes that with the class name and would put noise in front of the line an agent parses. Provider refusals are typed at the HTTP boundary: 401 → `provider_auth`, 402 → `provider_out_of_credits`, exhausted 429 backoff → `provider_rate_limited`. **The 429 backoff is `urllib3.Retry` mounted on the session** (`bettercontact._RETRY`: 5 attempts, `backoff_factor=5` doubling, `backoff_max=120`, `respect_retry_after_header`, and **only** 429 in the forcelist — a 401/402 is final and retrying it is noise). Hand-rolling that loop, or bending `tenacity` (which retries on exceptions, not statuses) into the shape, would both be worse copies of what the HTTP stack already does. -- **`status`**: `core/status.py` (builds one dict, reads only) + `core/management/commands/status.py` (renders it — human by default, `--json` for a program). **`find` ends with the same `next_action`**, read once from `build_status()` after the work and rendered by `status.render_next_action` — on stderr beside the counts, or as a `next_action` key under `--json`. That is what makes a run stopping with ranked leads and an empty wallet say so, at the one moment it can. The renderer lives in `core/status.py` rather than in the command because two callers print the sentence and it must have one spelling; the arithmetic is derived once and **not** carried through the run — two earlier attempts put the balance read and a deal count in `core/job.py` (an HTTP call to a payment provider inside the bounded-goal loop) and in `enrichment/lookup.py` (per-deal, so *read once* needed module-level mutable state). The standing state of the database, for a reader who did not run the job: what is configured, what is **blocked and why** (stable strings from `core/errors.py`), the counts, the credit balance, `walk` (every fired page: the `N(q)` and `θ(q)` the walk chose it on, kept on `core.Page`, beside `leads_found` and the page's verdicts and fits), and `next_action`. **It is smaller than it was, on purpose** — it began as the verb an agent asked *instead of tailing a log*, because a daemon could not answer for itself; with `find` returning its own result, that job is gone. Three things it gets right on purpose: **a balance that could not be read is not a balance of zero** (`no_credential` / `provider_auth` / `provider_unavailable` are reported as *why*, so a rejected key never renders like an empty run); **`exportable` is not `mailable`** — the export excludes only the two rejections, so a `QUALIFIED` lead exports with a blank `email`, and the counts split `exportable_with_email` / `exportable_without_email` from the records themselves rather than from a state standing in for them; and **`next_action` is ordered by what blocks progress** — `add_credits` sits above `print_leads` because a ranked lead cannot advance without credits, and `ranked_for_lookup > 0` is itself the proof that value exists, so the *never before value* rule holds without a special case (a campaign with nothing qualified gets `find_leads`, which costs nothing). The balance is `GET /api/v2/account` → `credits_left` (`bettercontact.credit_balance`, `X-API-Key`). +- **`status`**: `core/status.py` (builds one dict, reads only) + `core/management/commands/status.py` (renders it — human by default, `--json` for a program). **`find` ends with the same `next_action`**, read once from `build_status()` after the work and rendered by `status.render_next_action` — on stderr beside the counts, or as a `next_action` key under `--json`. That is what makes a run stopping with ranked leads and an empty wallet say so, at the one moment it can. The renderer lives in `core/status.py` rather than in the command because two callers print the sentence and it must have one spelling; the arithmetic is derived once and **not** carried through the run — two earlier attempts put the balance read and a deal count in `core/job.py` (an HTTP call to a payment provider inside the bounded-goal loop) and in `enrichment/lookup.py` (per-deal, so *read once* needed module-level mutable state). The standing state of the database, for a reader who did not run the job: what is configured, what is **blocked and why** (stable strings from `core/errors.py`), the counts, the credit balance, `walk` (every fired page: the `N(q)` and `θ(q)` the walk chose it on, kept on `core.Page`, beside `leads_found` and the page's verdicts and fits), and `next_action`. **It is smaller than it was, on purpose** — it began as the verb an agent asked *instead of tailing a log*, because a daemon could not answer for itself; with `find` returning its own result, that job is gone. Three things it gets right on purpose: **a balance that could not be read is not a balance of zero** (`no_credential` / `provider_auth` / `provider_unavailable` are reported as *why*, so a rejected key never renders like an empty run); **`exportable` is not `mailable`** — the export excludes only the two rejections, so a `QUALIFIED` lead exports with a blank `email`, and the counts split `exportable_with_email` / `exportable_without_email` from the records themselves rather than from a state standing in for them; and **`next_action` is ordered by what blocks progress** — `add_credits` sits above `print_leads` because a ranked lead cannot advance without credits, and `ranked_for_lookup > 0` is itself the proof that value exists, so the *never before value* rule holds without a special case (a campaign with nothing qualified gets `find_leads`, which costs nothing). The balance reported is whichever finder `provider.active()` names, and the vendor's own read is BetterContact's: `GET /api/v2/account` → `credits_left` (`bettercontact.credit_balance`, `X-API-Key`). FullEnrich exposes no balance endpoint at all, so `fullenrich.credit_balance` raises rather than guess and a FullEnrich install renders *unknown* permanently. - **State machine**: `crm/models/deal.py:DealState` — `QUALIFIED → READY_TO_FIND_EMAIL → FINDING_EMAIL → RESOLVED`, with `NO_EMAIL_FOUND` and `FAILED` as the other terminals. **Every state below `RESOLVED` is terminal**, and `QUALIFIED` is already exportable: an address is an enrichment on top, never a precondition. The GP confidence gate promotes `QUALIFIED → READY_TO_FIND_EMAIL` (rationing the paid lookup — a **spend** gate and nothing else, never a quality score). The paid lookup runs **whichever finder `provider.active()` names**, and the transport is the provider's own: `buy_address` resolves free-hub-first (hit → `RESOLVED` directly), then calls `start`, which either returns a **terminated** outcome (a synchronous finder a fork might add — straight to `RESOLVED`/`NO_EMAIL_FOUND`, never entering `FINDING_EMAIL`) or a **handle** (BetterContact's async job — park at `FINDING_EMAIL`); `check_lookup` then polls the job's `request_id` — hit → `RESOLVED`, miss → `NO_EMAIL_FOUND` (its own terminal, **no reason and blank outcome** — the lead was an LLM fit positive and only *reachability* failed, so the ML labeler keeps it at label=1), couldn't-run → back to `READY_TO_FIND_EMAIL`. **A still-running job is never abandoned**: there is no deadline and no attempt limit, the step just doubles `not_before` on the same `request_id`. The deadline used to revert the deal instead, which put it back in the pool for a *second* paid job — a hot resubmit loop that turned a provider outage into 418 submits and 4,512 polls in a week for ~40 leads, none terminating. Doubling makes waiting cheap (a week = 17 polls) and refuses to mislabel a provider outage as an unreachable lead. `COLLECT_BACKOFF_MAX_S` (a month) rails the *interval* only, so `datetime` can still represent the schedule. The job handle + poll backoff live **on the deal** (`lookup_request_id`, `lookup_attempt`, `not_before`), so the wait gates that one row. `Outcome` is down to `wrong_fit`/`unknown`; `Lead.disqualified=True` = permanent account-level exclusion (a **different column** from `FAILED` — the export filters both); LLM rejections = `FAILED` + `wrong_fit` (campaign-scoped). *(Two legs have been removed: the connect leg with the browser channel, and the send leg — `READY_TO_EMAIL`/`EMAILED`/`COMPLETED`/`UNSUBSCRIBED` plus the reply outcomes — with `emails/*`. Dropping the reply outcomes also retired a live mislabelling: the reply step moved a deal to `COMPLETED`, not `FAILED`, so a "not interested" reply trained the GP as a **positive**.)* - **The job**: `core/job.py`. `Goal(count, unit)` where the count is a **delta** — "ten more than you had" — and the loop is four lines: `while produced < goal: if not run_one_action(campaign): break`. **No timeout, by decision**: every unit of work already carries its own bound (`deal.not_before`, the doubling lookup backoff, `urllib3.Retry`), so a job-level clock would answer the same question worse — it knows nothing about *why* the run is waiting. The terminal condition is real state: *nothing can advance right now*. **Progress is a set, not a subtraction**, so a lead rejected mid-run cannot cancel out one that was found, and `--new` stays exact for `emails` (whose goal is usually met by leads that were already exportable and merely gained an address). Stopping short is `goal_unreached` — including `Ctrl-C`, the operator's own deadline for the one unbounded case, a campaign whose leads are all rejected — `bad_config` from `HALTING_ERRORS`, or `qualify_pending` (`QualifyPending`, caught the same way as `ProviderUnavailable`) from `--agent-qualify` opting a candidate out of the LLM call. **Exit 0 means the goal was met and nothing else**; the rows print either way, so *the code says how much you got and the type says why it stopped*. - **The cycle**: `core/cycle.py`, which owns the hierarchy and **no loop at all**. **A queue is a status, not a table** — work is found by asking the deals what they need, so nothing is minted and no row's timestamp can gate another. One action per call, with **no periodic side-effects** (the mail pass and the daily warmth re-measure both left with the sending leg). `run_one_action` walks one ordered hierarchy and stops at the first thing it can do, so priority *is* that order: **1** `FINDING_EMAIL` → `check_lookup` (`not_before` elapsed), or `reclaim_lookup` → `READY_TO_FIND_EMAIL` when the deal carries no `lookup_request_id` — there is no job to poll and no credit was spent, and the row used to *skip* those, stranding them in a state nothing else claims (two deals, 206h) · **2** `QUALIFIED` → `promote_to_ready` · **3** `READY_TO_FIND_EMAIL` → `buy_address` (a provider is configured) · **4** the install itself → `top_up`. Unlisted states are terminal and cost nothing. Steps live in `enrichment/lookup.py` and `core/pipeline/top_up.py`; each takes one entity and returns the next `DealState` or `None`, and `cycle._apply` does one `deal.save()` so a transition and its audit fields commit together. Steps are **total** — the loop's `try/except` is a bug backstop, not a retry policy; a step that wants to wait writes `deal.not_before`, which is the only retry mechanism there is — and is what lets a job end honestly, since a row that waits stops being due. **Queued is not due**: `buy_address` on an unreachable provider backs the deal off rather than leaving it eligible, or the next pass re-picks the same one forever (noise under the daemon, an endless job now). `HALTING_ERRORS` (`ModelHTTPError`) ends the job with a typed line instead, or a bad LLM key would retry forever behind an `alive` line. **Rows 2 and 4 have no gate at all, and that is the pivot**: they used to share `room_to_send_today` (*never buy an address, and never spend an LLM call qualifying, for someone there is no room to email today*), which was right while every lead ended in a send and was also the single line that made a mailbox-less install produce **nothing** — zero pool headroom, comparison never true, discovery and qualification silently stopped. Nothing replaced it: discovery is free, qualification costs the operator's own LLM key, and what caps the paid step is the number the operator typed, since one credit is one verified address. **Do not reintroduce a gate on a free step derived from a paid or absent one.** **There is no rotation any more** — `_rotate` is gone, because there is one install and one job at a time, and fairness has nobody to be fair between. **The hierarchy is data** (`ROWS`, name + function), so each action logs the row that fired and its duration, every row logs its decision time at `debug`, and when nothing fires `pipeline_summary` prints the counts — which is also the *reason* a job stopped short, since *no work* otherwise looks exactly like *work behind a gate*. *(Replaces the `Task` model, `core/scheduler.py`, `core/quota.py` and `core/session.py` — a token queue inherited from the Playwright era. On 2026-08-05 it slept 34 hours: two never-terminating polls had backed off 45h, were the only rows, and 55 ready deals had no token and so were not work.)* - **ML pipeline**: GPR (sklearn) + BALD active learning + LLM qualification, over 384-dim FastEmbed vectors cached on `Lead.embedding`. **The GP+BALD engine itself is `openoutlearn.GPBaldQualifier`** (github.com/eracle/OpenOutLearn), a shared dependency also used by OpenOutNews's `EngagementQualifier` — extracted once the two had converged on identical code under two domain labels. `BayesianQualifier` (`core/ml/qualifier.py`) subclasses it, adding only what's lead-gen specific: cold-start anchors, majority-class balancing in the early window between the cold phase and 10 real fits, and the Django-backed `rank_profiles`/`explain`. `core/ml/embeddings.py` is likewise a thin wrapper over `openoutlearn.embeddings`, supplying this install's model name (`CAMPAIGN_CONFIG["embedding_model"]`) and cache dir (`settings.FASTEMBED_CACHE_DIR`). **The fit is never persisted** — `qualifier_for` holds it in memory under `_label_fingerprint()` (a digest of the verdicts plus the anchor set) and refits the moment a verdict lands. **A refit keeps the last searched kernel** (`openoutlearn.qualifier.gpr_for`, called from `BayesianQualifier._fit_if_needed`): the hyperparameter search is ~60s at 2,000 labels and lands on the same kernel one verdict later, so it reruns only once the labels have grown 25% since the last search, and a refit in between is ~1s. The `ranking model trained` line names which one ran. The old `model_blob` column was joblib-dumped on every fit and read back by nothing, a cache of a value already derived from rows that are still there. **It trains on the LLM's fit verdict and nothing else** (`get_labeled_arrays`: label 1 = any non-`FAILED` deal, 0 = `FAILED` + `wrong_fit`) — no market signal was ever in that loop, which is why handing sending away cost the model nothing. The correction loop is the human: read `reason`, edit the product description and the ICP. -- **Discovery + enrichment**: `openoutfind/discovery.py` (Lead Finder `search`, free) blocks on `bettercontact.search_and_poll`, which polls first at the store's measured ready time (`meter.SEARCH_READY_SECONDS`, `walk.FIRST_POLL_SECONDS` = 1.5s until measured) and then every second — a search is ready 1.1–3.6s after the submit, and the old 5s interval was most of its time; `enrichment/bettercontact.py` (paid finder) splits the handshake into `submit(query) → request_id` + `poll_once(request_id) → PollOutcome` so a run never blocks on a poll (`check_lookup` owns the backoff, on the deal's own `not_before`), and keeps its 5s poll. **Every request the key sends, polls included, passes one per-process budget** (`bettercontact.RequestBudget`, `REQUESTS_PER_MINUTE` = 15 over 60s, `GET /account` exempt as the provider exempts it): the key is on 100/min, a client firing through 429s can get the account blocked, so the limit is taken from the docs and never probed. A process cannot see another's requests, so 15 is a share of the key: it holds the first round's 4 searches, about 12, and the fulfil skill runs at most six `find`s on the key at once. **What the provider can express** (probe table above `discovery.SEARCH_FIELDS`): no field takes query-string syntax (`AND`/`OR`/`+`/`*`/quotes are words or nothing); titles are a `match` with every word required in any order, tags a `match_phrase` inside one tag (adjacent, in order), place a parsed `state, country` (country required), seniority and industry a closed-list value; nothing is stemmed (`manager` and `managers` are different terms); a title drops stopwords while a tag keeps them, a hyphen splits a title word and also matches the joined form while in a tag it only splits; two entries in one include list OR on every field, so **no query can AND two tags**. `company_industry` is left out on measurement, `company_description` is the same field as `company_keywords`, `company_name` and `lead_industry` are ignored, `lead_function` is `lead_department` under a second name, and `lead_department` has no source field on a lead row. **Discovery is a best-first search over every search the store's keywords compose** (`core/pipeline/walk.py`) — the card `p1-e3-the-walk-explores-the-vocabulary-best-first`. **A search** (`query.Terms`, stored as `core.Search`) is a set of `field:term` pairs, each field holding what its match type allows: title words (any number, ANDed), at most one tag phrase (a word or a phrase a company's tags hold, **never without a title**: alone the provider pages one company's whole staff), one place (the market's country or a state in it), at most one of the 12 seniorities; a search holds a title or a seniority, since a place alone is the market's famous-company head. `Terms.to_filters` is the only place a search becomes provider JSON, and `Terms.matches` checks a person against every term **in its own field**, from `Lead.source_fields` (`discovery.SOURCE_FIELDS`: `contact_job_title`, `company_keywords`, `contact_location_state/country`, `contact_seniority`) with each field's analyzer copied from the provider's (`core/pipeline/analyzers.py`: ICU's Unicode word split via `icu4py`, accents kept, and the measured rules on top — a straight apostrophe, a dot or an underscore joins, a Thai run is one word, a Chinese character one each). The size band and the market's country code are the **shape** the customer fixed: every `Search` carries the initial population's, never varied. **An arm is a search at a page** (`core.Page` once fired, `PAGE_SIZE` 200 via `limit`/`offset` — never `max_leads`, which misreports the count — and `LAST_PAGE` 50, where the 10,000-row reach ends). **The initial population is 4 whole searches written by the LLM** (`icp.write_initial_population`, `initial_population.j2`) — "a head of procurement at a cannabis extraction company in California" is `{title: head, procurement; tag: cannabis extraction; place: California, United States}` — or by the calling agent (`--population`); the marks are never fired as searches, their terms join the vocabulary searches are composed from and they train the GP. **Round 0** fires the written population whole — nothing is known yet to choose among its searches by. **Every later round** (`walk.run_round`, one blocking step where a search was taken) reads the **frontier** (`search_arms.Frontier`) as the store stands, after the run has judged what the last round brought in, and fires one search. **Every store is anchored** (`walk._anchor`): the anchors are written with the initial population, so a store seeded with `--labels` (fits from the start, so never cold) has them, and a store from before them gets them once before its next round; when none can be written the round raises `unanchored` and fires nothing. The arms are every fired search's next page, every unfired written search, and the searches **composed from the qualified profiles' keywords** (`core/pipeline/compose.py`) — the `COMPOSED` (200) worth the most in fits on a page at the mean belief — never per dollar, which ranked on the rate alone and picked searches of 1 to 6 people. **No term is cut before it is valued**: the vocabulary (`Frontier.vocabulary`) is every title word the qualified profiles hold on its own (never a piece of a hyphenated word: `co` alone reaches co-owners and co-CEOs), every whole tag they hold (never a phrase inside one, which a search on it would still match: `pre` from `pre-rolls` reached 6,512 founders of pre-seed funds and pre-k schools; `analyzers.sub_phrases` is for matching only), the written searches' terms, all 12 seniorities and the market's places. A keyword's lift (`logit θ_t − logit π`) and share (`log N_t / N₀`) sum over a search, so the composition adds the fields one at a time keeping the `PER_BIN` (4) highest lifts per 0.1 bin of summed share — every title word alone and each one that lifts added where it pays, then a seniority or none, the place, every tag — and values each finished search per dollar of its page; the frontier then values them exactly. **A composed search always holds a title and a tag** — a tag-less search is the whole market's head for its title (on batch 5 two of them brought 334 of a round's 426 new people, none in the industry) — and a store with no written search composes nothing. No LLM. **The titles and tags come from the qualified profiles** (`judged_fits`: the customer's good marks and the run's fits), never from everyone the store holds: with every lift centred on the market's rate, the store's 10,000 keywords would all read that rate and be ranked on size alone — the fits are how the marks reach the walk. **No composed search holds a seniority a title word already names** (title and seniority `founder` are near one keyword; 19 of batch 6's 23 searches held both, doubling the lift). **Searches are composed at the mean only**: composed on each draw under the GP prior this replaced, only 9 of 50 draws stayed in CaleafTech's cannabis market. **The walk keeps a belief per keyword** (`core/pipeline/belief.py`, the card `p1-e3-the-belief-is-one-bayesian-model`): `N_t`, the people holding it, and `θ_t`, the share who fit. **A fired search is one sample of the AND of its keywords**, and both halves are **one hierarchical model sampled whole by NUTS** (PyMC, compiled by nutpie — numba, no C++ compiler): `log N(q) = log N₀ + Σ (μ_field + γ·x_t + u_t)`, `logit θ(q) = α + Σ (p_t + w_t − u_t) + e_q`, each keyword's `(u_t, w_t)` — how far its log people and its log fits stray from their guesses — a correlated Normal pair whose spreads `τ`, `s_f` and correlation `ρ` are learned, as are `s_e` and the count's dispersion. **A keyword's lift is its fits less its size**, so a count far above the guess lowers its rate before any verdict (batch 6's `processing`, guessed at cannabis processors only, counted 61× that, and with the lift free of the size took 14 no-fits to come down). **The count** is a negative binomial around `N(q)` — inside this model it covered 16 of batch 6's 20 held-out counts in its 80% interval against 14 for a log-normal on `log(y + ½)`. **The prior is the LLM's guess, in plain counts** (`core/pipeline/keyword_guess.py`, `guess_market.j2`, `guess_keywords.j2`): the market first and once (`core.MarketGuess`: its people and how many of them fit), then per keyword how many people in the market hold it, in the customer's trade or any other, and how many of those fit — every count over what a search runs on, everyone on LinkedIn in the customer's country and not the customer's buyers (`keyword_guess.COUNTS_MEAN`, in both prompts and the pending payload; without it batch 6's session guessed a market of 60,000) — (`core.KeywordGuess`, its field read off the keyword's own prefix), every keyword against the stored market so all share its scale, each field (titles, tags, seniorities, places) in one structured answer beside the ones of that field already guessed. Fits above people do not parse, an unknown key does not parse, and a count past the market's or a keyword not asked for is refused and asked again. Only means are asked: how far keywords stray from their guesses is the learned `τ` and `s_f`. `belief.Guesses` turns the counts into the model's terms: `x_t = log(people / market people)`, evidence whose weight `γ` the counts learn, and the lift's centre `p_t = logit(fits / people) − logit(market fits / market people)` with half a count on each side (`belief.count_log_odds` — `clip_rate`'s 0.001 floor sits above a guessed market's rate); with no guess a keyword reads its log share of the store and `p_t` 0, the market's rate. A keyword no fired search holds reads `p_t` at the mean and on a draw `p_t` plus a pair from the draw's own `MVNormal(0, Σ)`. **When:** `search_arms.next_round` has the market and every keyword of the vocabulary and the fired searches guessed before the belief is read (`Frontier.guess_keywords`; `Frontier.beliefs` is fitted on first read and is handed `keyword_guess.load()`) — the whole vocabulary before the first composed round, then the titles and tags new fits bring, one `AI_MODEL` call for the market and one per field holding unguessed keywords. A keyword is guessed once; a replaced brief (`brief.adopt`) clears every guess, the market's too. Under `--agent-qualify` the run stops with `keyword_guess_pending` (`qualify.KeywordGuessPending`), the payload carrying the market (`null` to be guessed), each field's keywords, what was refused and why, and the schema, answered with `--keyword-guesses JSON` (`KeywordGuesses`: `market` and a flat `keywords` list). Not judge-or-search: its frontier reads the guesses already stored. `log N₀` is anchored broadly at the guessed market's people (else `MARKET_PEOPLE`, 10M), since no search counts the market. Sampling has no mode to reach: the Newton fit this replaced fell into a 2-cycle on batch 6 and stopped at `α` = +17.86, every arm reading `θ` = 1. **The posterior is refitted only when a page or a verdict lands** (`belief.fitted`, keyed on the fired pages and the judged set, held for the process) and the model is compiled once per size (searches and keywords padded to a power of two, `belief.COMPILED`), so a refit is 1–2 s after the first compile. A `Draw` is one stored posterior draw chosen by its seed, both halves from the same draw. **The fit rate's evidence**: each fired search's people are a binomial sample of its `θ(q)`, each adding a verdict at full weight, an unjudged person's fit rate — the GP's score through the isotonic **fit-rate map** (`store_snapshot.fit_rate_map`, fitted on the store's verdicts at their out-of-fold scores — the GP ranks but is not calibrated) — at a quarter times the GP's **confidence** at them (`BayesianQualifier.confidence`, the share of its prior variance the verdicts removed: 0.90 on the judged, ~0.05 on batch 6's 780 strangers, who read the base rate whoever they were), each company once at the mean of its people. **`α = logit π` is the searched market's rate**, centred on the store's and learned from the searched people only — read off the store, batch 6 put every unknown keyword at its marks' 0.21, and a page of food founders at 40 fits; learned, `π` reads 0.06 on its 23 searches. **The store is not the market**: people the store holds that no search brought move no lift — counted as evidence, Bill's cannabis-wholesaler marks read founder · `wholesale` across the US at 0.35 ± 0.08 after its first page brought 170 and no fit. **`e_q` is a fired search's own effect**, so its next page is valued on what its own pages brought (founder · `food` reads 0.09 on batch 6, against 0.36 when the store's rate stood in for `π`). **Thompson sampling over the top `k`** (`Frontier.pick`): the `TOP_K` (10) arms whose page brings the most fits at the mean (`top_at_mean`), each arm's fits on its page `new · (F − held) / (N − seen)`, are valued on one draw of the belief seeded by the round (`beliefs.draw((round, 0), …)`), and the one with the most fits on that draw is fired; `log_chosen` logs its fits on the draw and at the mean and its rank among the `k`. The greedy pick it replaced (mean plus one σ over 200 draws) fired one idea round after round: 46 of batch 6's rounds 16–66 were `ceo`/`chief`/`executive`/`officer` in another combination. Composing on the draw is not done: maximising over 504 drawn lifts overvalued its own pick (a median ~75 fits on a page where real pages bring 10–20%). **The probe settles the draw** (`core/pipeline/probe.py`): past the cold phase each `top_up` pass first judges one person drawn at random from the latest fired search whose people hold fewer than `PROBE_VERDICTS` (5) verdicts, since no other move reaches them (batch 6's founder · `food` brought 186 and none was judged); five without a fit take a search from 0.22 ± 0.21 to about 0.08 ± 0.09. `N` is the provider's count where the search fired, else the estimate, capped by any fired search it narrows and kept a tenth above the store's people it matches (`store_snapshot.People`, a per-field index whose `match` equals `Terms.matches`); `F` is `N · θ(q)`; `seen` and `held` the store's people it matches and the fits expected among them, each at their verdict or fit rate. `new` is what the next page of a fired search was last first to bring in (the page–lead table), or for an unfired search a page of those still out there. The cost is the measured ready time, and for each new person the measured ingest time (`meter.INGEST_SECONDS_PER_LEAD`) and a verdict, all in dollars. **The walk stops** when no arm is expected to bring a new fit (`FITS_WORTH_A_SEARCH`), and in the cold phase only when no arm brings anybody new. **A search holding all the terms of a drained one** (its pages returned everything it counts) can only re-find people and is never an arm. **Nor, where every written search names a state, is a search of the whole country** (`Frontier.in_market`). **A row with no state matches any state of its own country**: the provider filters on a state outside the US but returns one on US rows alone. Nothing is ranked when it lands: a page is evidence the next round reads, and an unfired arm is never cut — it stays on the frontier, recounted every round. A round is fired by `walk._fire_all` (a thread pool; the threads make the API calls and nothing else), then the main thread records each page and takes every page in together (`discover.harvest`: rows with no headline or a capped company dropped, the rest stored via **one** `db.leads.store_rows` call and written to the **page–lead table** `core.PageLead` — every person the page returned, new to the store or not, except marked leads and anchors). One call means one embedding pass over the whole round, and `openoutlearn.embed_texts` embeds shortest profile first in batches of 16 and hands the vectors back in order: a batch pads to its longest text, so one 2,681-character profile padded a whole page, and embedding is most of a round's time (15 pages page by page: 14m35s; 10 pages in one sorted pass: 2m42s). **Do not cap `profile_text` to speed it up** — it loses what the GP reads. **A search whose page 1 is empty with a count of 0 is dead** (`Search.dead`), and so is every search holding all its terms (`Terms.holds_all_of`); an empty page 1 with a positive count is the **transport artifact** (a burst once answered a 71M-lead query with nothing in 0.0s) and leaves no trace; an outage leaves no trace; a refusal (401/402/exhausted 429) is re-raised. A page past its count, at page 50, or empty is **at the bottom** and is not paged again, but narrower and sibling searches can still be composed. No other pruning. A lead's embedding is its `profile_text` alone — no query terms ride it, since an arm is valued on its people's fit rates. Then `qualify.py` (`run_qualification` — the balance-driven pick, which on a cold campaign runs against the anchored GP rather than against no model at all) → `ready_pool.py` (GP gate; `min_gp_confidence` is the paid-lookup spend gate **and nothing else**) → `top_up.py` (**one** unit of work per call — the cycle is the loop that used to live here. `top_up` is the *labelling* steering, the qualifier's own **explore/exploit** split: **cold phase** (`is_cold` — `n_real_positives < ANCHOR_COUNT`, a clock independent of whether the anchors themselves are still standing) judges the strongest lead every pass and fires a round only when the pool is empty — a round is up to 800 people, and with rankings still leaning on the anchors' guess nothing observed says a second one beats judging what the first brought; a round adding nobody is still a move, and only a walk with nothing left to fire stalls. The anchors are permanent — `set_anchors`/`update` never trim them — so what ends the cold phase is real acceptances reaching `ANCHOR_COUNT`, not the anchors running out. **the cold phase always exploits** — while real positives are still short of `ANCHOR_COUNT` the one goal is *more real positives*, and the highest-P lead is the one most like the ideal profile, where BALD spends each call on the lead the model is most *confused* about, i.e. the one least like the ICP; the balance could not have chosen it anyway, since while `is_cold` held, `n_neg > n_pos` was false by construction. Once real positives reach `ANCHOR_COUNT` the phase ends and balancing (`_balance`, the majority subsampled to twice the minority) takes over **for the early window only** — from 10 real fits (`_BALANCE_UNTIL_POSITIVES`, a placeholder until a replay measures it) the GP trains on every verdict, since subsampling there drops paid-for verdicts (batch 5 at 54 fits kept 108 of 375 no-fits, so a fresh one was usually left out and its lookalike topped the pool) and the scores need only rank, `store_snapshot.fit_rate_map` turning them into fit rates — the anchors keep contributing to every fit after that too, a small constant addition next to a growing real count. Past the phase: **probe** comes first (above), then **calibrate** — while the store holds fewer than √N calibration verdicts (N = the unjudged pool, ≥ 1,600 to switch on) it judges one lead drawn at random from the thinnest score band, and once √N are in it measures the bar, once — pool growth does not reopen it (`core/ml/calibration.py`: the fit rate at each score as an isotonic regression on out-of-fold scores, then the lowest score where a verdict pays more than a search — the break-even weighs a verdict's run time and LLM bill against a search's run time and the leads it adds on this store, all in dollars with time priced as the 2 GiB droplet a run needs — stored as a `CalibrationBar`); calibration verdicts train the GP but are left out of the mode balance (`balance_counts`). Then **explore** (`neg ≤ pos`) BALD-labels the pool with **no gate**, **exploit** (`neg > pos`) qualifies the strongest lead clearing `calibration.current_threshold()` (the measured bar, `min_gp_confidence` until there is one — the same getter the promote gate reads); failing that, **judge or search, whichever buys a fit for less on this run's measured costs** (`core/pipeline/judge_or_search.py:decide`): judging the best lead waiting costs `c_verdict / rate(best waiting)`, firing the arm worth the most at the mean belief (`search_arms.best_expected`) `cost(arm) / expected fits(arm)`. **Both are fit rates on one scale** — the GP's score through `store_snapshot.fit_rate_map`, since the GP ranks but is not calibrated (0.15 where the truth was 0.28). Before the first round the pool is judged, and after a round that added leads it is judged at least once before the next. Each round's lead count after its harvest less before is `meter.ROUND_LEADS` (read by `calibration.leads_per_search`), and the pool's fit rate (`meter.POOL_FIT`, logged beside the decision) starts over once per round that added leads. The times are **measured** (`core/meter.py`: a `RunningMean` row per quantity, sum and count in the store, since an `--agent-qualify` run is many processes — every round's wall time, its ingest time per new lead, every search's ready time, every model call's, and for a calling agent the time from the hand-over to the answer); the settings are only starting values. The bills are not measured: a verdict's tokens at the model's rates, or `verdict_dollars_agent`. With nothing left to search the pool is judged to its end. It replaced a fixed floor on BALD (`min_bald_gain`, 0.04 nats), which judged any lead the model was unsure of before searching — and off-industry profiles are the ones it is most unsure of: CaleafTech's batch 5 spent 42 verdicts, none a fit, on soap makers and lenders while 6 queries fired in the whole store. The case the floor was built for still holds: a cold posterior tops out low (a live 26,737-lead pool at 3 real positives reached **P=0.37**, and discovering every pass spent 14h33m on 295 pages and 19 verdicts), and the rule judges a fresh pool rather than discovering past it). +- **Discovery + enrichment**: `openoutfind/discovery.py` (Lead Finder `search`, free) blocks on `bettercontact.search_and_poll`, which polls first at the store's measured ready time (`meter.SEARCH_READY_SECONDS`, `walk.FIRST_POLL_SECONDS` = 1.5s until measured) and then every second — a search is ready 1.1–3.6s after the submit, and the old 5s interval was most of its time; `enrichment/bettercontact.py` (paid finder) splits the handshake into `submit(query) → request_id` + `poll_once(request_id) → PollOutcome` so a run never blocks on a poll (`check_lookup` owns the backoff, on the deal's own `not_before`), and keeps its 5s poll. `enrichment/fullenrich.py` is the second finder behind the same seam; it takes a bare profile URL (`submit(profile_url) → enrichment_id`) where BetterContact takes a structured query, carries its own per-process `RequestBudget` (15 over 60s, nothing exempt because it has no account read), and maps its statuses onto the same `PollOutcome` (only `DELIVERABLE`/`HIGH_PROBABILITY` count as a hit, `CANCELED` is a miss, `RATE_LIMIT`/`CREDITS_INSUFFICIENT` the same typed refusals). It exposes no balance read, so `credit_balance()` raises rather than guess and `status` reports the balance as unknown. `enrichment/leadmagic.py` is the third finder and the only synchronous one: `start(profile_url)` is the whole lookup, so there is no `poll_once` and no handle left to strand, one POST to `/v1/people/email-finder` carrying only the profile URL, and unlike FullEnrich it does expose a balance read (`GET /v1/credits`, free), so `status` shows a real number. `enrichment/enrow.py` is the fourth finder added, and the cheapest: `start(profile_url)` POSTs one `linkedin_url` and returns the vendor's handle (`id`), `poll_once(id)` reads it back as a `PollOutcome` while the search runs at HTTP 202, the `valid`/`invalid` qualification maps straight through, a `lookup_failed` LinkedIn status defers rather than reading as a miss (the vendor bills nothing for it), and unlike FullEnrich it does expose a balance read (`GET /account/info`, free), so `status` shows a real number too. **Every request each adapter sends, polls included, passes that adapter's own per-process budget** (`bettercontact.RequestBudget`, sized by each module's `REQUESTS_PER_MINUTE` and exempting the free account/credits read where there is one: 15 over 60s for BetterContact and FullEnrich, 60 for LeadMagic, whose `GET /v1/credits` sits outside it, and 120 for Enrow, whose free `GET /account/info` does too): the key is on 100/min, a client firing through 429s can get the account blocked, so the limit is taken from the docs and never probed. A process cannot see another's requests, so 15 is a share of the key: it holds the first round's 4 searches, about 12, and the fulfil skill runs at most six `find`s on the key at once. **What the provider can express** (probe table above `discovery.SEARCH_FIELDS`): no field takes query-string syntax (`AND`/`OR`/`+`/`*`/quotes are words or nothing); titles are a `match` with every word required in any order, tags a `match_phrase` inside one tag (adjacent, in order), place a parsed `state, country` (country required), seniority and industry a closed-list value; nothing is stemmed (`manager` and `managers` are different terms); a title drops stopwords while a tag keeps them, a hyphen splits a title word and also matches the joined form while in a tag it only splits; two entries in one include list OR on every field, so **no query can AND two tags**. `company_industry` is left out on measurement, `company_description` is the same field as `company_keywords`, `company_name` and `lead_industry` are ignored, `lead_function` is `lead_department` under a second name, and `lead_department` has no source field on a lead row. **Discovery is a best-first search over every search the store's keywords compose** (`core/pipeline/walk.py`) — the card `p1-e3-the-walk-explores-the-vocabulary-best-first`. **A search** (`query.Terms`, stored as `core.Search`) is a set of `field:term` pairs, each field holding what its match type allows: title words (any number, ANDed), at most one tag phrase (a word or a phrase a company's tags hold, **never without a title**: alone the provider pages one company's whole staff), one place (the market's country or a state in it), at most one of the 12 seniorities; a search holds a title or a seniority, since a place alone is the market's famous-company head. `Terms.to_filters` is the only place a search becomes provider JSON, and `Terms.matches` checks a person against every term **in its own field**, from `Lead.source_fields` (`discovery.SOURCE_FIELDS`: `contact_job_title`, `company_keywords`, `contact_location_state/country`, `contact_seniority`) with each field's analyzer copied from the provider's (`core/pipeline/analyzers.py`: ICU's Unicode word split via `icu4py`, accents kept, and the measured rules on top — a straight apostrophe, a dot or an underscore joins, a Thai run is one word, a Chinese character one each). The size band and the market's country code are the **shape** the customer fixed: every `Search` carries the initial population's, never varied. **An arm is a search at a page** (`core.Page` once fired, `PAGE_SIZE` 200 via `limit`/`offset` — never `max_leads`, which misreports the count — and `LAST_PAGE` 50, where the 10,000-row reach ends). **The initial population is 4 whole searches written by the LLM** (`icp.write_initial_population`, `initial_population.j2`) — "a head of procurement at a cannabis extraction company in California" is `{title: head, procurement; tag: cannabis extraction; place: California, United States}` — or by the calling agent (`--population`); the marks are never fired as searches, their terms join the vocabulary searches are composed from and they train the GP. **Round 0** fires the written population whole — nothing is known yet to choose among its searches by. **Every later round** (`walk.run_round`, one blocking step where a search was taken) reads the **frontier** (`search_arms.Frontier`) as the store stands, after the run has judged what the last round brought in, and fires one search. **Every store is anchored** (`walk._anchor`): the anchors are written with the initial population, so a store seeded with `--labels` (fits from the start, so never cold) has them, and a store from before them gets them once before its next round; when none can be written the round raises `unanchored` and fires nothing. The arms are every fired search's next page, every unfired written search, and the searches **composed from the qualified profiles' keywords** (`core/pipeline/compose.py`) — the `COMPOSED` (200) worth the most in fits on a page at the mean belief — never per dollar, which ranked on the rate alone and picked searches of 1 to 6 people. **No term is cut before it is valued**: the vocabulary (`Frontier.vocabulary`) is every title word the qualified profiles hold on its own (never a piece of a hyphenated word: `co` alone reaches co-owners and co-CEOs), every whole tag they hold (never a phrase inside one, which a search on it would still match: `pre` from `pre-rolls` reached 6,512 founders of pre-seed funds and pre-k schools; `analyzers.sub_phrases` is for matching only), the written searches' terms, all 12 seniorities and the market's places. A keyword's lift (`logit θ_t − logit π`) and share (`log N_t / N₀`) sum over a search, so the composition adds the fields one at a time keeping the `PER_BIN` (4) highest lifts per 0.1 bin of summed share — every title word alone and each one that lifts added where it pays, then a seniority or none, the place, every tag — and values each finished search per dollar of its page; the frontier then values them exactly. **A composed search always holds a title and a tag** — a tag-less search is the whole market's head for its title (on batch 5 two of them brought 334 of a round's 426 new people, none in the industry) — and a store with no written search composes nothing. No LLM. **The titles and tags come from the qualified profiles** (`judged_fits`: the customer's good marks and the run's fits), never from everyone the store holds: with every lift centred on the market's rate, the store's 10,000 keywords would all read that rate and be ranked on size alone — the fits are how the marks reach the walk. **No composed search holds a seniority a title word already names** (title and seniority `founder` are near one keyword; 19 of batch 6's 23 searches held both, doubling the lift). **Searches are composed at the mean only**: composed on each draw under the GP prior this replaced, only 9 of 50 draws stayed in CaleafTech's cannabis market. **The walk keeps a belief per keyword** (`core/pipeline/belief.py`, the card `p1-e3-the-belief-is-one-bayesian-model`): `N_t`, the people holding it, and `θ_t`, the share who fit. **A fired search is one sample of the AND of its keywords**, and both halves are **one hierarchical model sampled whole by NUTS** (PyMC, compiled by nutpie — numba, no C++ compiler): `log N(q) = log N₀ + Σ (μ_field + γ·x_t + u_t)`, `logit θ(q) = α + Σ (p_t + w_t − u_t) + e_q`, each keyword's `(u_t, w_t)` — how far its log people and its log fits stray from their guesses — a correlated Normal pair whose spreads `τ`, `s_f` and correlation `ρ` are learned, as are `s_e` and the count's dispersion. **A keyword's lift is its fits less its size**, so a count far above the guess lowers its rate before any verdict (batch 6's `processing`, guessed at cannabis processors only, counted 61× that, and with the lift free of the size took 14 no-fits to come down). **The count** is a negative binomial around `N(q)` — inside this model it covered 16 of batch 6's 20 held-out counts in its 80% interval against 14 for a log-normal on `log(y + ½)`. **The prior is the LLM's guess, in plain counts** (`core/pipeline/keyword_guess.py`, `guess_market.j2`, `guess_keywords.j2`): the market first and once (`core.MarketGuess`: its people and how many of them fit), then per keyword how many people in the market hold it, in the customer's trade or any other, and how many of those fit — every count over what a search runs on, everyone on LinkedIn in the customer's country and not the customer's buyers (`keyword_guess.COUNTS_MEAN`, in both prompts and the pending payload; without it batch 6's session guessed a market of 60,000) — (`core.KeywordGuess`, its field read off the keyword's own prefix), every keyword against the stored market so all share its scale, each field (titles, tags, seniorities, places) in one structured answer beside the ones of that field already guessed. Fits above people do not parse, an unknown key does not parse, and a count past the market's or a keyword not asked for is refused and asked again. Only means are asked: how far keywords stray from their guesses is the learned `τ` and `s_f`. `belief.Guesses` turns the counts into the model's terms: `x_t = log(people / market people)`, evidence whose weight `γ` the counts learn, and the lift's centre `p_t = logit(fits / people) − logit(market fits / market people)` with half a count on each side (`belief.count_log_odds` — `clip_rate`'s 0.001 floor sits above a guessed market's rate); with no guess a keyword reads its log share of the store and `p_t` 0, the market's rate. A keyword no fired search holds reads `p_t` at the mean and on a draw `p_t` plus a pair from the draw's own `MVNormal(0, Σ)`. **When:** `search_arms.next_round` has the market and every keyword of the vocabulary and the fired searches guessed before the belief is read (`Frontier.guess_keywords`; `Frontier.beliefs` is fitted on first read and is handed `keyword_guess.load()`) — the whole vocabulary before the first composed round, then the titles and tags new fits bring, one `AI_MODEL` call for the market and one per field holding unguessed keywords. A keyword is guessed once; a replaced brief (`brief.adopt`) clears every guess, the market's too. Under `--agent-qualify` the run stops with `keyword_guess_pending` (`qualify.KeywordGuessPending`), the payload carrying the market (`null` to be guessed), each field's keywords, what was refused and why, and the schema, answered with `--keyword-guesses JSON` (`KeywordGuesses`: `market` and a flat `keywords` list). Not judge-or-search: its frontier reads the guesses already stored. `log N₀` is anchored broadly at the guessed market's people (else `MARKET_PEOPLE`, 10M), since no search counts the market. Sampling has no mode to reach: the Newton fit this replaced fell into a 2-cycle on batch 6 and stopped at `α` = +17.86, every arm reading `θ` = 1. **The posterior is refitted only when a page or a verdict lands** (`belief.fitted`, keyed on the fired pages and the judged set, held for the process) and the model is compiled once per size (searches and keywords padded to a power of two, `belief.COMPILED`), so a refit is 1–2 s after the first compile. A `Draw` is one stored posterior draw chosen by its seed, both halves from the same draw. **The fit rate's evidence**: each fired search's people are a binomial sample of its `θ(q)`, each adding a verdict at full weight, an unjudged person's fit rate — the GP's score through the isotonic **fit-rate map** (`store_snapshot.fit_rate_map`, fitted on the store's verdicts at their out-of-fold scores — the GP ranks but is not calibrated) — at a quarter times the GP's **confidence** at them (`BayesianQualifier.confidence`, the share of its prior variance the verdicts removed: 0.90 on the judged, ~0.05 on batch 6's 780 strangers, who read the base rate whoever they were), each company once at the mean of its people. **`α = logit π` is the searched market's rate**, centred on the store's and learned from the searched people only — read off the store, batch 6 put every unknown keyword at its marks' 0.21, and a page of food founders at 40 fits; learned, `π` reads 0.06 on its 23 searches. **The store is not the market**: people the store holds that no search brought move no lift — counted as evidence, Bill's cannabis-wholesaler marks read founder · `wholesale` across the US at 0.35 ± 0.08 after its first page brought 170 and no fit. **`e_q` is a fired search's own effect**, so its next page is valued on what its own pages brought (founder · `food` reads 0.09 on batch 6, against 0.36 when the store's rate stood in for `π`). **Thompson sampling over the top `k`** (`Frontier.pick`): the `TOP_K` (10) arms whose page brings the most fits at the mean (`top_at_mean`), each arm's fits on its page `new · (F − held) / (N − seen)`, are valued on one draw of the belief seeded by the round (`beliefs.draw((round, 0), …)`), and the one with the most fits on that draw is fired; `log_chosen` logs its fits on the draw and at the mean and its rank among the `k`. The greedy pick it replaced (mean plus one σ over 200 draws) fired one idea round after round: 46 of batch 6's rounds 16–66 were `ceo`/`chief`/`executive`/`officer` in another combination. Composing on the draw is not done: maximising over 504 drawn lifts overvalued its own pick (a median ~75 fits on a page where real pages bring 10–20%). **The probe settles the draw** (`core/pipeline/probe.py`): past the cold phase each `top_up` pass first judges one person drawn at random from the latest fired search whose people hold fewer than `PROBE_VERDICTS` (5) verdicts, since no other move reaches them (batch 6's founder · `food` brought 186 and none was judged); five without a fit take a search from 0.22 ± 0.21 to about 0.08 ± 0.09. `N` is the provider's count where the search fired, else the estimate, capped by any fired search it narrows and kept a tenth above the store's people it matches (`store_snapshot.People`, a per-field index whose `match` equals `Terms.matches`); `F` is `N · θ(q)`; `seen` and `held` the store's people it matches and the fits expected among them, each at their verdict or fit rate. `new` is what the next page of a fired search was last first to bring in (the page–lead table), or for an unfired search a page of those still out there. The cost is the measured ready time, and for each new person the measured ingest time (`meter.INGEST_SECONDS_PER_LEAD`) and a verdict, all in dollars. **The walk stops** when no arm is expected to bring a new fit (`FITS_WORTH_A_SEARCH`), and in the cold phase only when no arm brings anybody new. **A search holding all the terms of a drained one** (its pages returned everything it counts) can only re-find people and is never an arm. **Nor, where every written search names a state, is a search of the whole country** (`Frontier.in_market`). **A row with no state matches any state of its own country**: the provider filters on a state outside the US but returns one on US rows alone. Nothing is ranked when it lands: a page is evidence the next round reads, and an unfired arm is never cut — it stays on the frontier, recounted every round. A round is fired by `walk._fire_all` (a thread pool; the threads make the API calls and nothing else), then the main thread records each page and takes every page in together (`discover.harvest`: rows with no headline or a capped company dropped, the rest stored via **one** `db.leads.store_rows` call and written to the **page–lead table** `core.PageLead` — every person the page returned, new to the store or not, except marked leads and anchors). One call means one embedding pass over the whole round, and `openoutlearn.embed_texts` embeds shortest profile first in batches of 16 and hands the vectors back in order: a batch pads to its longest text, so one 2,681-character profile padded a whole page, and embedding is most of a round's time (15 pages page by page: 14m35s; 10 pages in one sorted pass: 2m42s). **Do not cap `profile_text` to speed it up** — it loses what the GP reads. **A search whose page 1 is empty with a count of 0 is dead** (`Search.dead`), and so is every search holding all its terms (`Terms.holds_all_of`); an empty page 1 with a positive count is the **transport artifact** (a burst once answered a 71M-lead query with nothing in 0.0s) and leaves no trace; an outage leaves no trace; a refusal (401/402/exhausted 429) is re-raised. A page past its count, at page 50, or empty is **at the bottom** and is not paged again, but narrower and sibling searches can still be composed. No other pruning. A lead's embedding is its `profile_text` alone — no query terms ride it, since an arm is valued on its people's fit rates. Then `qualify.py` (`run_qualification` — the balance-driven pick, which on a cold campaign runs against the anchored GP rather than against no model at all) → `ready_pool.py` (GP gate; `min_gp_confidence` is the paid-lookup spend gate **and nothing else**) → `top_up.py` (**one** unit of work per call — the cycle is the loop that used to live here. `top_up` is the *labelling* steering, the qualifier's own **explore/exploit** split: **cold phase** (`is_cold` — `n_real_positives < ANCHOR_COUNT`, a clock independent of whether the anchors themselves are still standing) judges the strongest lead every pass and fires a round only when the pool is empty — a round is up to 800 people, and with rankings still leaning on the anchors' guess nothing observed says a second one beats judging what the first brought; a round adding nobody is still a move, and only a walk with nothing left to fire stalls. The anchors are permanent — `set_anchors`/`update` never trim them — so what ends the cold phase is real acceptances reaching `ANCHOR_COUNT`, not the anchors running out. **the cold phase always exploits** — while real positives are still short of `ANCHOR_COUNT` the one goal is *more real positives*, and the highest-P lead is the one most like the ideal profile, where BALD spends each call on the lead the model is most *confused* about, i.e. the one least like the ICP; the balance could not have chosen it anyway, since while `is_cold` held, `n_neg > n_pos` was false by construction. Once real positives reach `ANCHOR_COUNT` the phase ends and balancing (`_balance`, the majority subsampled to twice the minority) takes over **for the early window only** — from 10 real fits (`_BALANCE_UNTIL_POSITIVES`, a placeholder until a replay measures it) the GP trains on every verdict, since subsampling there drops paid-for verdicts (batch 5 at 54 fits kept 108 of 375 no-fits, so a fresh one was usually left out and its lookalike topped the pool) and the scores need only rank, `store_snapshot.fit_rate_map` turning them into fit rates — the anchors keep contributing to every fit after that too, a small constant addition next to a growing real count. Past the phase: **probe** comes first (above), then **calibrate** — while the store holds fewer than √N calibration verdicts (N = the unjudged pool, ≥ 1,600 to switch on) it judges one lead drawn at random from the thinnest score band, and once √N are in it measures the bar, once — pool growth does not reopen it (`core/ml/calibration.py`: the fit rate at each score as an isotonic regression on out-of-fold scores, then the lowest score where a verdict pays more than a search — the break-even weighs a verdict's run time and LLM bill against a search's run time and the leads it adds on this store, all in dollars with time priced as the 2 GiB droplet a run needs — stored as a `CalibrationBar`); calibration verdicts train the GP but are left out of the mode balance (`balance_counts`). Then **explore** (`neg ≤ pos`) BALD-labels the pool with **no gate**, **exploit** (`neg > pos`) qualifies the strongest lead clearing `calibration.current_threshold()` (the measured bar, `min_gp_confidence` until there is one — the same getter the promote gate reads); failing that, **judge or search, whichever buys a fit for less on this run's measured costs** (`core/pipeline/judge_or_search.py:decide`): judging the best lead waiting costs `c_verdict / rate(best waiting)`, firing the arm worth the most at the mean belief (`search_arms.best_expected`) `cost(arm) / expected fits(arm)`. **Both are fit rates on one scale** — the GP's score through `store_snapshot.fit_rate_map`, since the GP ranks but is not calibrated (0.15 where the truth was 0.28). Before the first round the pool is judged, and after a round that added leads it is judged at least once before the next. Each round's lead count after its harvest less before is `meter.ROUND_LEADS` (read by `calibration.leads_per_search`), and the pool's fit rate (`meter.POOL_FIT`, logged beside the decision) starts over once per round that added leads. The times are **measured** (`core/meter.py`: a `RunningMean` row per quantity, sum and count in the store, since an `--agent-qualify` run is many processes — every round's wall time, its ingest time per new lead, every search's ready time, every model call's, and for a calling agent the time from the hand-over to the answer); the settings are only starting values. The bills are not measured: a verdict's tokens at the model's rates, or `verdict_dollars_agent`. With nothing left to search the pool is judged to its end. It replaced a fixed floor on BALD (`min_bald_gain`, 0.04 nats), which judged any lead the model was unsure of before searching — and off-industry profiles are the ones it is most unsure of: CaleafTech's batch 5 spent 42 verdicts, none a fit, on soap makers and lenders while 6 queries fired in the whole store. The case the floor was built for still holds: a cold posterior tops out low (a live 26,737-lead pool at 3 real positives reached **P=0.37**, and discovering every pass spent 14h33m on 295 pages and 19 verdicts), and the rule judges a fresh pool rather than discovering past it). - **The lead export**: `core/export.py`, printed by `find` — the finder's **public output**, and Tier 0 of the boundary described by the `openoutreach-docs` roadmap card `p1-e3-leadfinder-sequencer-boundary`. The governing rule is that **our own sender gets no privileged path**: a sequencer, a CRM and a spreadsheet all read the same rows. **The column names are the importers', not ours** — Instantly and Smartlead both *require* `email`/`first_name`/`last_name` and recognise `company`/`title`/`website`/`linkedin_url`, mapping anything else to a custom variable, so the record is exactly `email, first_name, last_name, company, title, website, linkedin_url, reason, lead_id, qualified_at, full_name` and a file imports without column mapping. `full_name` is discovery's one reported name, verbatim — the name parts stay null until the paid lookup returns them, so without it a leads-only file names nobody; it is last because the record only ever adds. `qualified_at` (`Deal.creation_date`, ISO 8601) is provenance the file keeps — a caller tells which rows its own call produced by comparing against its start time, and a sequencer imports only what is newer than last time; a `new` flag would be invocation-relative state in a file that outlives the invocation, and therefore a lie the second time it is read. The internal model keeps its own names (`profile_url` is provider-agnostic on purpose; a `domain` is not a `website`) and `lead_record()` is the one translation — one schema cannot match N importers, so the mapping is a function, never a migration. **One record, two serialisations**: `JSON_FIELDS` is the record — those eleven plus `profile_text`, the raw firmographic string the qualifier judged on, `source_fields` (`Lead.source_fields`, each searchable field kept apart under its Lead Finder name) and `employees_min`/`employees_max` (the employer's LinkedIn band) — the last three so a record carries the whole person a store holds, and a marked lead can seed a fresh store as a verdict without the store it was found in — and `RECORD_FIELDS` is the importer-shaped **projection** of it, so `write_csv` projects rather than defining a second schema (`None` → empty cell; `profile_text` is `""` and `source_fields` `{}`, never absent). `--json` is **JSON Lines** — one record per line on stdout, run metadata as one JSON object on stderr and nothing else there (no banner, no logs — unless `--debug`/`--log-level` was given, which keeps the log lines beside the objects for an agent that needs the walk's reasoning too), because a truncated stream stays usable and a caller should not need `tail -1` over prose. **Output is progressive by default** (either format): `find` writes what the campaign already has immediately, before any work, then writes each new lead's record the moment its deal settles — `find 20 emails | outsend` starts feeding the sender within seconds of the first resolved address instead of after the whole run. It rides the same `on_new_lead` hook `--open` already used (`_combine` in `find.py` chains the two, via `core/export.py:IncrementalWriter`, which owns the CSV-header-on-first-row/JSON-line timing and flushes after every write). `--new` skips the opening bulk and only ever streams what this run produces. `--batch` is the escape hatch back to the old shape — nothing is written until the job ends, then the whole thing prints once, for a consumer that cannot take a partial stream. This is safe where a daemon-managed live CSV was not: that file was deleted because rows kept changing state under an unbounded process running for days; a single `find` run's deals settle once and stay settled for the rest of that run. **Summarising for a message is the sender's job**: `profile_text` crosses raw and the receiver extracts what an opener needs, which is why `core/db/summaries.py`, `Deal.profile_summary` and the vendored mem0 are gone — they had no consumer here once the sending leg left. `reason` is **operator-facing** evidence, never text for the prospect. The compatibility rule replaces the shared schema package we are not building: a receiver ignores unknown keys, and this side only ever adds. **There is no score column and the export touches nothing but the DB** — exporting the GP posterior was a category error (`min_gp_confidence` is the paid-lookup *spend* gate and nothing else; the fit verdict is the LLM's, already present as `reason`), and it cost an O(n³) `qualifier_for` fit — minutes on the live 2,538-deal campaign — which also calls `ensure_anchors`, so a cold campaign would have spent LLM calls and mutated state from a read-only command. The **Deal** is the unit (the `reason` is per-campaign). **A Deal is not an endorsement**: `DealState.FAILED` (the LLM's campaign-scoped rejection) and `Lead.disqualified` (the permanent opt-out) are *different columns*, and the first shipped version filtered only the second — exporting 1,944 rows whose `reason` read "does not align well with the target market". Both are excluded, always, with no flag to include them. **There is no command and no file — it is `find`'s stdout**, and the whole campaign every time, so `> leads.csv` is correct by construction (the newest file supersedes every earlier one, and a lead whose address resolved since last time comes back with it filled in). It is one file to overwrite, not a batch per run; `--new` narrows to this run's rows, `find 0` prints without working. This took two attempts in one day: a command nobody discovered → a CSV the daemon kept current (naming rule, collision tie-break, rewrite-vs-append, atomic temp-and-rename) → nothing at all. **Every one of those problems existed only because a background process can change a row after it was written.** **The boundary is one-way** (2026-08-19): leads leave, nothing comes back. There is no inbound endpoint and no event vocabulary — reply outcomes are conversation states belonging to whoever sent the mail, and a finder that never contacts anyone is not the sender under CAN-SPAM/GDPR, so suppression stays with the sequencer (Instantly and Smartlead both block a suppressed address at import). The GP therefore keeps training on the LLM's fit verdict alone; `get_labeled_arrays` has never seen a market signal, so nothing was lost. `lead_id` keeps the door open if that is revisited — the one candidate is `bounced`, which grades the row we emitted rather than the message. Import dedupe is **opt-in** on Smartlead and undocumented on Instantly, so every adapter's docs must tell the operator to enable it. - **Lead identity + Company**: `Lead.full_name`/`first_name`/`last_name`/`job_title` (all nullable — `NULL` means the provider never told us) and `Lead.company` → `crm.Company`, a shared employer row keyed on `Company.key` (the lowercased domain, or `name:` when no domain was reported) and carrying LinkedIn's employee band (`employees_min`/`employees_max`, which the verdict reads as its own *Company size* line — `qualify.verdict_fields`, `qualify_lead.j2`, the `qualify_pending` payload — and never through `profile_text`, whose embedding it would move). They serve the export and the record the product keeps, and are deliberately **out of** `profile_text` and the embedding: a name carries no ICP signal and would only give the GP noise. **Nothing splits a name in-house** — discovery reports one `contact_full_name`, and the paid enrichment response reports the real `contact_first_name`/`contact_last_name` at no extra call or credit (`enrichment/lookup.py` writes them on a hit). A hub-cache resolution never reaches that provider, so its name parts stay `NULL`; they feed a sequencer's `{{first_name}}` merge tag, where a guess lands in someone's cold email. The **enrichment query stays URL-only** by decision — the provider accepts name/company and resolves better with them, but the less of a lead's record leaves for a third party the better, and URL-only measures ~42% usable. `Company` rows are *what the provider said*, not verified truth (Lead Finder fuzzy-matches them), and a firm seen once with and once without a domain produces two rows — a known cost of the simple key. **A placeholder employer is no employer**: `Company.from_row` returns `None` for the names in `PLACEHOLDER_NAMES` ("Self employed", "Stealth startup", "None at this time", …), because the provider staples a real, unrelated domain to them and hundreds of leads then share one row; `export.lead_record` drops the same names from older stores. - **Contacts store (hub)**: `contacts/service.py` — an optional free `profile_url → email` cache at `hub.openoutreach.app`, tried *before* the paid finder and given back to on a fresh paid hit. **Every discovery page is given back too** (`share_profiles`, called from `discover.harvest` on each whole page): each row's `discovery.SHARED_FIELDS` kept apart — never the stapled `company_description`/`company_keywords` — plus the searched country and the **profile-only** vector, tagged with the model. **Nothing is embedded twice**: `harvest` hands the sharers the round's vectors keyed on text, so a stored person's `Lead.embedding` goes out as it is (it is `profile_text` alone), and what only the hub needs (a dropped row, a person whose text changed, an employer) is embedded once a round, however many pages carry it (`service._vectors_for`). `contribute` sends the address and no vector. No credit for it, and the same EEA/UK/CH gates on operator and page. **The page's employers go too** (`share_companies`, beside it in `harvest`): one record per distinct `company_linkedin_url` (`discovery.company_records` — `COMPANY_RECORD_FIELDS`, description, tags and LinkedIn size band *included*, a placeholder employer skipped) with the vector of the company's own text (`company_text_for`), and **no person attached** — the staple can name the wrong employer for a person, but it is still a true record of the company it names. The hub keeps them for a private company index it does not serve. Same gates (`_page_token`), no credit. All calls are best-effort (outage/no-token → no-op). The give-back is off for an operator who declares an EEA/UK/CH country in the optional `OPENOUTFIND_OPERATOR_COUNTRY` (`is_eea_located`) and on for any other country **or none** — never a stored toggle; the server re-gates authoritatively. **Identity is minted, not earned by contributing** (`token_in_hand` → `register_operator`): the token says *which install this is*, the balance says what it may read, and folding them into one act meant an install that cannot contribute had no identity at all and stayed invisible to the hub for life. It is minted from the operator's email — no new question, and no email means no identity and no hub calls at all — **regardless of jurisdiction**, since the EEA/UK/CH rule governs *contributing records*, a different act still gated in `contribute`. Best-effort and idempotent; `_register` remains as the folded-into-a-contribution path for a hub that still requires a record. **The token is never written down**: `OPENOUTFIND_CONTACTS_API_TOKEN` supplies it, or a run registers for one and holds it in a module global for the length of the process — `register` is idempotent server-side, so forgetting costs one round trip and nothing else. A read-only caller passes `mint=False` (`hub_balance`, so `status` cannot register an install as a side effect of reporting). **The give-to-get balance is visible in `status`, as its own `hub` key** — a different number on a different service than the `credits` key (BetterContact's own prepaid balance); `hub_balance()` reads it via a spend-free record-less `register` call, and `known: False` (no token, or an outage) must render as *unknown*, never as a balance of zero. A `resolve` miss also tells the two zeros apart: `credits <= 0` logs as *no balance* (the store was never even checked — `resolve_lead` short-circuits at zero balance server-side), anything else logs as *no stored email*. The hub URL is fixed (`contacts.service.HUB_URL`). **Tests may never reach the hub, or a real LLM**: `tests/conftest.py::_no_live_writes_to_our_own_services` refuses `contacts.service` POSTs (a readiness check in a test would otherwise register a fake operator against production), and `_no_live_model_ping` stubs `verify_llm_credentials` — the ping is unconditional now, so any fixture that sets a key would otherwise be a live 401 (`no_llm_mock` opts the tests *of* that function out). -- **Config**: **the environment, and nothing else** (`core/config.py`) — a frozen `SiteConfig` dataclass read from `OPENOUTFIND_*` on every run: `ai_model` (a pydantic-ai `provider:model` id, e.g. `anthropic:claude-sonnet-4-5-20250929`; bare `gpt-*`/`o1`/`o3`/`claude-*`/`gemini-*` auto-prefixed), `llm_api_key`, `llm_api_base` (only for `openai_compatible:*`), `bettercontact_api_key` (blank disables discovery + enrichment), `product_docs`/`campaign_target`, `max_per_company` (optional — at most N fits from one company, read at discovery and at the qualification scan; `core/company_cap.py`), `operator_country_code` (optional — **the operator's jurisdiction**, not a target market; unset means the install contributes), `contacts_api_token`. **There is no config row and no wizard** — remembering what a human typed is a human feature, and this program is a library, a pipe stage and a scripted command; the wizard lives in OpenOutreach, which holds the answers and exports these names. What makes reading fresh safe is `readiness.check_ready()`, which names every missing variable at once and pings the model **before** any work starts, so a key rotated under a running timer fails at second one instead of mid-pass. Identity is the exception and stays a row: `OPENOUTFIND_OPERATOR_EMAIL` seeds the Django `User` **once** and is never re-read, because a renamed variable must not rename the person a campaign belongs to. `core/conf.py`: `CAMPAIGN_CONFIG` (ML defaults, incl. `min_gp_confidence`, the paid-step rank gate's fallback until the calibrate arm measures one, and the arm's costs (`run_hour_dollars`, `round_seconds`, `ingest_seconds_per_lead`, `verdict_seconds_*` — starting values only, each replaced by the store's measured running mean once it has one — `verdict_input_tokens`/`verdict_output_tokens` priced by `MODEL_PRICES` for the configured `AI_MODEL`, `verdict_dollars_*` for an unpriced model and an agent's turn), which `judge_or_search` reads too, and `calibration_min_band_verdicts` — the gate is **only** a spend gate; there is no discovery-interleave threshold and **no discovery cadence knob**), `WALK_CONFIG` (the walk's `population`), `COLLECT_BACKOFF_BASE_S`/`COLLECT_BACKOFF_MAX_S` (the lookup poll backoff — doubling, no deadline, MAX rails the interval only), `CYCLE_SECONDS` in `core/cycle.py`. **There is no spend cap on the paid lookup**: it used to ride on send headroom, and with no sending leg nothing replaced it — what bounds the spend is the operator's prepaid provider balance, which we cannot see. *(Gone with the sending leg: `WARM_*`, `MIN_SEND_INTERVAL_SECONDS`/`SEND_INTERVAL_JITTER_*`, `SEND_WINDOW_*`, `COLLECT_TODAY_HORIZON_S`, `MAIL_PASS_INTERVAL_S`.)* `core/business_time.py` has **no knob** and only *measures* (Mon–Fri, no public holidays); with the agent gone the pipeline no longer calls it. +- **Config**: **the environment, and nothing else** (`core/config.py`) — a frozen `SiteConfig` dataclass read from `OPENOUTFIND_*` on every run: `ai_model` (a pydantic-ai `provider:model` id, e.g. `anthropic:claude-sonnet-4-5-20250929`; bare `gpt-*`/`o1`/`o3`/`claude-*`/`gemini-*` auto-prefixed), `llm_api_key`, `llm_api_base` (only for `openai_compatible:*`), `bettercontact_api_key` (blank disables discovery, so the funnel stops; it is also one of the four finder keys), `fullenrich_api_key` (one of the finder keys, optional: the resolver needs one, not all, and `provider.active()` takes the cheaper), `leadmagic_api_key` (the third finder, the only synchronous one, also optional), `enrow_api_key` (the fourth finder and the cheapest, async, also optional), `product_docs`/`campaign_target`, `max_per_company` (optional — at most N fits from one company, read at discovery and at the qualification scan; `core/company_cap.py`), `operator_country_code` (optional — **the operator's jurisdiction**, not a target market; unset means the install contributes), `contacts_api_token`. **There is no config row and no wizard** — remembering what a human typed is a human feature, and this program is a library, a pipe stage and a scripted command; the wizard lives in OpenOutreach, which holds the answers and exports these names. What makes reading fresh safe is `readiness.check_ready()`, which names every missing variable at once and pings the model **before** any work starts, so a key rotated under a running timer fails at second one instead of mid-pass. Identity is the exception and stays a row: `OPENOUTFIND_OPERATOR_EMAIL` seeds the Django `User` **once** and is never re-read, because a renamed variable must not rename the person a campaign belongs to. `core/conf.py`: `CAMPAIGN_CONFIG` (ML defaults, incl. `min_gp_confidence`, the paid-step rank gate's fallback until the calibrate arm measures one, and the arm's costs (`run_hour_dollars`, `round_seconds`, `ingest_seconds_per_lead`, `verdict_seconds_*` — starting values only, each replaced by the store's measured running mean once it has one — `verdict_input_tokens`/`verdict_output_tokens` priced by `MODEL_PRICES` for the configured `AI_MODEL`, `verdict_dollars_*` for an unpriced model and an agent's turn), which `judge_or_search` reads too, and `calibration_min_band_verdicts` — the gate is **only** a spend gate; there is no discovery-interleave threshold and **no discovery cadence knob**), `WALK_CONFIG` (the walk's `population`), `COLLECT_BACKOFF_BASE_S`/`COLLECT_BACKOFF_MAX_S` (the lookup poll backoff — doubling, no deadline, MAX rails the interval only), `CYCLE_SECONDS` in `core/cycle.py`. **There is no spend cap on the paid lookup**: it used to ride on send headroom, and with no sending leg nothing replaced it — what bounds the spend is the operator's prepaid provider balance, which we cannot see. *(Gone with the sending leg: `WARM_*`, `MIN_SEND_INTERVAL_SECONDS`/`SEND_INTERVAL_JITTER_*`, `SEND_WINDOW_*`, `COLLECT_TODAY_HORIZON_S`, `MAIL_PASS_INTERVAL_S`.)* `core/business_time.py` has **no knob** and only *measures* (Mon–Fri, no public holidays); with the agent gone the pipeline no longer calls it. - **Django apps** (nested under `openoutfind/`, dotted `AppConfig.name`): **just two** — `outfind_core` (engine — the cycle, operator lookup, the walk's `Search`/`Page`/`PageLead` models, the environment config + readiness check, llm, ML, discovery/qualify pipeline, the lead export) and `outfind_crm` (Lead + Company + Deal, the anchors among them). `enrichment` and `contacts` are service-only modules (no models, not installed apps). *(`emails`, `chat` and `legacy` are gone — see* The cut *above.)* - **Labels are namespaced (`outfind_*`) because these apps are hosted twice.** OpenOutreach installs them in one registry beside OpenOutSend's, whose own `core` would otherwise collide. Never write a table name as a literal — ask the model (`Lead._meta.db_table`) — and write cross-app model references label-first (`"outfind_core.QueryNode"`). - **`defaults.py` is what a host must splat.** `APPS` (the two `AppConfig` paths, in order), `state_dir()`, `database_path()`, `allow_async_unsafe()` and `app_settings()` — the settings names the *apps* read, today `DATABASE_PATH` and `FASTEMBED_CACHE_DIR`. This repo's `settings.py` splats it and so does OpenOutreach's, so a new requirement lands in one place. `MEDIA_ROOT`/`MEDIA_URL` are **not** in it: only `openoutfind/urls.py` reads them, and a host with its own URLconf owns them. diff --git a/README.md b/README.md index bc19d85..ca5f079 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ There is no promotional campaign sent from your mailbox and no "Sent with" line | # | What | Example | |---|------|---------| | 1 | **An LLM API key** | OpenAI, Anthropic, or any OpenAI-compatible endpoint | -| 2 | **An email-finder API key** ([BetterContact](https://bettercontact.rocks?fpr=openoutreach)) | **Free account: 40 credits, no card.** Powers **both** discovery (Lead Finder, billed nothing) and enrichment (one credit per verified work email, only with `--emails`) | +| 2 | **An email-finder API key** ([BetterContact](https://bettercontact.rocks?fpr=openoutreach)) | **Free account: 50 credits, no card.** Powers **both** discovery (Lead Finder, billed nothing) and enrichment (one credit per verified work email, only with `--emails`) | | 3 | **A product description + target market** | "We sell cloud cost optimization for DevOps teams at mid-market SaaS companies" | That's it. **No mailbox**, no social-network account, no spreadsheets, no lead databases, no scraping setup. The BetterContact key is required because it drives discovery *and* enrichment — note the barrier is an **account, not a bill**, since searching the index costs nothing and only resolving an address spends a credit. diff --git a/docs/configuration.md b/docs/configuration.md index b7c81f4..6110fbc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -25,7 +25,7 @@ things before it starts working. | `OPENOUTFIND_AI_MODEL` | llm | `provider:model`, e.g. `anthropic:claude-sonnet-4-5-20250929`; bare `gpt-*`/`claude-*`/`gemini-*` are auto-prefixed. Providers: openai/anthropic/google/groq/mistral/cohere/openai_compatible | | `OPENOUTFIND_LLM_API_KEY` | llm | **verified by one live ping before every run** — a key rotated out from under a timer fails before a lead is chosen, not mid-pass | | `OPENOUTFIND_LLM_API_BASE` | llm | required for `openai_compatible:*`, ignored otherwise | -| `OPENOUTFIND_BETTERCONTACT_API_KEY` | bettercontact | [free account, 40 credits, no card](https://bettercontact.rocks?fpr=openoutreach) (affiliate link, no markup to you). Powers **both** Lead Finder discovery (billed nothing) **and** work-email enrichment (one credit per verified address, only with `--emails`) | +| `OPENOUTFIND_BETTERCONTACT_API_KEY` | bettercontact | [free account, 50 credits, no card](https://bettercontact.rocks?fpr=openoutreach) (affiliate link, no markup to you). Powers **both** Lead Finder discovery (billed nothing) **and** work-email enrichment (one credit per verified address, only with `--emails`) | | `OPENOUTFIND_OPERATOR_EMAIL` | account | optional. Your own inbox — the contacts-store key. Unset, the operator row is named `operator` and the install never talks to the hub (no free cached addresses, no give-back). Fills a blank row once; never replaces an email already there | | `OPENOUTFIND_OPERATOR_COUNTRY` | account | optional. ISO 3166 alpha-2, e.g. `US` — **your jurisdiction**, not your target market. Unset, the install contributes to the contacts store | | `OPENOUTFIND_CONTACTS_API_TOKEN` | hub | optional. Without a token a run registers for one and keeps it for the length of the process; `register` is idempotent, so nothing is lost by not storing it | diff --git a/openoutfind/core/config.py b/openoutfind/core/config.py index fe1d9d8..a3aa586 100644 --- a/openoutfind/core/config.py +++ b/openoutfind/core/config.py @@ -77,6 +77,22 @@ class SiteConfig: # paid email lookup behind `--emails`. bettercontact_api_key: str = "" + # FullEnrich's key powers the paid email lookup behind `--emails` and nothing else. + # Discovery stays on BetterContact's Lead Finder, so a FullEnrich-only install can + # resolve an address but cannot search (core/readiness.py: REQUIRED_DISCOVERY_FIELDS). + fullenrich_api_key: str = "" + + # LeadMagic's key powers the paid email lookup behind `--emails` and nothing else. + # A single synchronous call where the others run a job, so a hit lands without the + # lead ever parking on a vendor-held handle. + leadmagic_api_key: str = "" + + # Enrow's key powers the paid email lookup behind `--emails` and nothing else. + # Asynchronous like the waterfalls, and the cheapest address per hit on the entry + # plan ($17 buys 1,000 credits), so `provider.active()` reaches for it first when + # it is keyed. + enrow_api_key: str = "" + # The operator's own ISO-3166 alpha-2 jurisdiction — not to be confused with # `Lead.country_code`, the per-lead target country an ICP search surfaced someone # under. Optional: an EEA/UK/CH country turns the contacts-store give-back off diff --git a/openoutfind/core/status.py b/openoutfind/core/status.py index 6c7363e..856d3ec 100644 --- a/openoutfind/core/status.py +++ b/openoutfind/core/status.py @@ -170,7 +170,12 @@ def _credits() -> dict: return {"balance": finder.credit_balance(), "provider": finder.NAME, "error": None} except provider.ProviderUnavailable as exc: logger.debug("Could not read the credit balance: %s", exc) - return {"balance": None, "error": exc.error_type, "detail": str(exc)} + return { + "balance": None, + "provider": finder.NAME, + "error": exc.error_type, + "detail": str(exc), + } def _signup_url() -> str: @@ -203,25 +208,30 @@ def _blocked(config: dict, credits: dict, totals: dict) -> list[dict]: "message": "this run was not given: " + ", ".join(config["missing"]), }) + # The finder that actually ran is the one an operator has to act on. When no key is + # set there is no finder to name, so the sentence describes the missing piece instead + # of picking a vendor the install never configured. + finder = credits.get("provider") or "the email finder" + if credits["error"] == ErrorType.NO_CREDENTIAL: blocked.append({ "type": ErrorType.NO_CREDENTIAL, - "message": "no BetterContact key — discovery and email finding are both off", + "message": "no email-finder key set: discovery and email finding are both off", }) elif credits["error"] == ErrorType.PROVIDER_AUTH: blocked.append({ "type": ErrorType.PROVIDER_AUTH, - "message": "BetterContact rejected the API key", + "message": f"{finder} rejected the API key", }) elif credits["error"] == ErrorType.PROVIDER_OUT_OF_CREDITS: blocked.append({ "type": ErrorType.PROVIDER_OUT_OF_CREDITS, - "message": "BetterContact reports the credits are exhausted", + "message": f"{finder} reports the credits are exhausted", }) elif credits["error"] == ErrorType.PROVIDER_RATE_LIMITED: blocked.append({ "type": ErrorType.PROVIDER_RATE_LIMITED, - "message": "BetterContact is rate-limiting this client — the run is backing off", + "message": f"{finder} is rate-limiting this client, the run is backing off", }) elif credits["balance"] == 0 and totals["ranked_for_lookup"]: blocked.append({ diff --git a/openoutfind/enrichment/bettercontact.py b/openoutfind/enrichment/bettercontact.py index f29698b..a854c96 100644 --- a/openoutfind/enrichment/bettercontact.py +++ b/openoutfind/enrichment/bettercontact.py @@ -76,6 +76,14 @@ REQUESTS_PER_MINUTE = 15 _BUDGET_WINDOW_S = 60.0 +# **What one credit costs on the entry plan, used only to rank finders against each +# other.** Starter is $15 for 200 credits, so $0.075 a hit; the rate falls as the plan +# grows (their ladder runs to $0.02 at volume), and a bigger plan reorders the waterfall. +# So this is an anchor for the cheapest paid tier, not a measurement of what an +# operator's own account charges — the seam picks the cheapest *listed* finder and the +# prices are re-checked when a plan changes. +UNIT_COST_USD = 0.075 + # **A 429 is backed off, never retried at speed** — their docs warn that a client which # keeps firing through one can get the *account* blocked, which costs the operator far # more than the throttled call was worth. urllib3 already does this correctly: waits of @@ -108,11 +116,19 @@ class RequestBudget: spend about 12, a later round's one about 3, so a run rarely waits. """ - def __init__(self, limit: int, window: float, clock=time.monotonic, sleep=time.sleep): + def __init__( + self, + limit: int, + window: float, + clock=time.monotonic, + sleep=time.sleep, + label: str = NAME, + ): self._limit = limit self._window = window self._clock = clock self._sleep = sleep + self._label = label self._sent: deque[float] = deque() self._lock = threading.Lock() @@ -126,7 +142,9 @@ def take(self) -> None: self._sent.append(now) return wait = self._window - (now - self._sent[0]) - logger.debug("bettercontact: request budget full — waiting %.1fs", wait) + logger.debug( + "%s: request budget full, waiting %.1fs", self._label, wait + ) self._sleep(wait) diff --git a/openoutfind/enrichment/enrow.py b/openoutfind/enrichment/enrow.py new file mode 100644 index 0000000..8aeee53 --- /dev/null +++ b/openoutfind/enrichment/enrow.py @@ -0,0 +1,278 @@ +"""Enrow: the fourth paid finder, and a waterfall behind the same seam as the rest. + +Same two-leg handshake as FullEnrich: ``start`` fires one single search and hands back +Enrow's own ``id``; ``poll_once`` checks it once and returns a ``PollOutcome``. Nothing +here blocks on a poll: the collect step owns the wait, on the deal's own ``not_before``, +so a run parked at FINDING_EMAIL costs this process nothing. + +Why a fourth adapter: the seam's job is to pick the finder that is cheapest on the +operator's plan (``provider.py``), and the waterfalls quote different prices per credit at +different tiers. Enrow is the cheapest of the four at entry, so a keyed install picks it +first and the others stay as the fallback chain. + +Discovery is **not** here. Lead Finder is BetterContact's free index and stays on +BetterContact's key, so an Enrow-only install can resolve an address but cannot search, +by decision (see ``REQUIRED_DISCOVERY_FIELDS`` in ``core/config.py``). +""" + +from __future__ import annotations + +import logging + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from openoutfind.core.errors import ErrorType +from openoutfind.enrichment.bettercontact import RequestBudget +from openoutfind.enrichment.provider import Lookup, PollOutcome, ProviderUnavailable + +logger = logging.getLogger(__name__) + +NAME = "enrow" + +# The one path to an account. Enrow runs no affiliate redirect of its own, so this is +# their plain signup page rather than a parameterised one: inventing a tracking query +# here would be a link that claims an attribution nobody granted. +SIGNUP_URL = "https://enrow.io" + +# One credit on the entry plan ($17/month for 1,000 credits), which is what an operator +# starting today pays: the per-credit price falls on larger plans, so this is an anchor +# for the cheapest-first pick and not a claim about volume. Kept beside the adapter, as +# the others' is, so the selector reads it from the provider rather than a table that +# goes stale without the vendor. +UNIT_COST_USD = 0.017 + +_SINGLE_URL = "https://api.enrow.io/email/find/single" +_ACCOUNT_URL = "https://api.enrow.io/account/info" + +# One HTTP call's ceiling, not a job's. A single search walks Enrow's own waterfall for +# a while, so the gap between polls belongs to the collect step's cadence rather than to +# a constant here. +_HTTP_TIMEOUT_S = 30 + +# Enrow documents 10 requests per second on every POST endpoint, for every plan, and +# asks for backoff on a 429. This is a courtesy fifth of that figure, per process, so +# several concurrent runs stay well inside anything they would call abuse. It is a share +# of the key, not a measurement of the account, and their GET endpoints are not limited +# at all, so the poll leg could climb higher safely if a run ever needed it. +REQUESTS_PER_MINUTE = 120 +_BUDGET_WINDOW_S = 60.0 +_RATE_LIMIT_ATTEMPTS = 5 + +# The one verdict that means an address was found. Enrow keeps the vocabulary to three +# values, so anything else is either still running or a failure we must not record. +_FOUND = "valid" +_NOT_FOUND = "invalid" +_RUNNING = "ongoing" + +# A LinkedIn search that failed on Enrow's side costs nothing and is explicitly +# retryable, so it must defer the lookup rather than settle it as a miss. +_TRANSIENT_LINKEDIN_STATUSES = frozenset({"lookup_failed"}) + +_RETRY = Retry( + total=_RATE_LIMIT_ATTEMPTS, + status_forcelist=(429,), + allowed_methods=frozenset({"GET", "POST"}), + backoff_factor=5, + backoff_max=120, + respect_retry_after_header=True, +) + +_BUDGET = RequestBudget(REQUESTS_PER_MINUTE, _BUDGET_WINDOW_S, label=NAME) + + +class EnrowUnavailable(ProviderUnavailable): + """Enrow could not answer, as distinct from answering that there is no address. + + The seam catches ``ProviderUnavailable`` and never names a vendor, so this exists + for tests and readers rather than for callers to branch on. A 5xx and a transient + LinkedIn status both read as this: losing a real lead to a provider hiccup is worse + than deferring its lookup by one backoff step. + """ + + def __init__( + self, message: str, error_type: ErrorType = ErrorType.PROVIDER_UNAVAILABLE + ) -> None: + super().__init__(message, error_type) + + +def is_configured() -> bool: + """True when the Enrow paid finder is configured (an API key is set).""" + from openoutfind.core.config import SiteConfig + + return bool(SiteConfig.load().enrow_api_key) + + +def credit_balance() -> int: + """The credits left on the account, as Enrow counts them. + + Like LeadMagic and unlike FullEnrich, Enrow has a balance endpoint, + ``GET /account/info``, so this is a real number rather than a raised refusal and the + run can warn before a lead fails instead of after a 402. The response is + ``{"credits": 36000, "webhooks": [...]}``, so it goes through the same + floor-don't-round reading LeadMagic's balance does: a fraction of a credit buys + nothing, and rounding it up would report a balance that cannot pay for the next + lookup. The read is free per Enrow's own docs, so it sits outside the budget. + """ + api_key = _require_key() + with _session(api_key) as session: + try: + body = _request(session, "GET", _ACCOUNT_URL).json() + except (requests.RequestException, TimeoutError) as exc: + raise EnrowUnavailable(f"Enrow unreachable: {exc}") from exc + return _as_credit_count(body) + + +def start(profile_url: str) -> Lookup: + """Fire one single search and hand back Enrow's handle for it. + + **Only the profile URL is sent.** The API accepts a name and a company too, and + resolves better with them, but the query stays as narrow as the other adapters' for + the same reason: the less of a lead's record leaves for a third party, the better. + Their docs treat ``linkedin_url`` as sufficient on its own. + """ + api_key = _require_key() + with _session(api_key) as session: + try: + body = _request( + session, "POST", _SINGLE_URL, json={"linkedin_url": profile_url} + ).json() + except (requests.RequestException, TimeoutError) as exc: + raise EnrowUnavailable(f"Enrow unreachable: {exc}") from exc + request_id = _clean(body.get("id")) + if not request_id: + raise EnrowUnavailable("Enrow returned no search id") + return Lookup(request_id=request_id) + + +def poll_once(request_id: str) -> PollOutcome: + """Check one search once. A completed search is a hit or a miss; else still running.""" + api_key = _require_key() + url = f"{_SINGLE_URL}?id={request_id}" + with _session(api_key) as session: + try: + response = _request(session, "GET", url) + except (requests.RequestException, TimeoutError) as exc: + raise EnrowUnavailable(f"Enrow unreachable: {exc}") from exc + # A running search answers 202 with the same body a completed one carries, so the + # status code is the cheapest signal and the qualification is the confirmation. + if response.status_code == 202: + return PollOutcome(running=True) + try: + body = response.json() + except ValueError as exc: + raise EnrowUnavailable("Enrow returned a response that is not JSON") from exc + return _outcome(body) + + +def _outcome(body: dict) -> PollOutcome: + """Turn one search document into a hit, a miss, or a still-running outcome. + + Only ``valid`` with an address is a hit and only ``invalid`` is a miss. Anything + else, including a status this adapter does not recognise, reads as still running: an + unrecognised answer must never be recorded as a lead with no address, because the + deal would then settle as NO_EMAIL_FOUND and never be retried. + """ + linkedin_status = _clean(body.get("linkedin_status")) + if linkedin_status in _TRANSIENT_LINKEDIN_STATUSES: + raise EnrowUnavailable( + f"Enrow could not read the LinkedIn profile yet ({linkedin_status})" + ) + + qualification = _clean(body.get("qualification")) + if qualification == _RUNNING: + return PollOutcome(running=True) + if qualification == _FOUND: + email = _clean(body.get("email")) + if not email: + return PollOutcome(running=True) + info = body.get("info") or {} + return PollOutcome( + running=False, + email=email, + first_name=_clean(info.get("firstname")), + last_name=_clean(info.get("lastname")), + ) + if qualification == _NOT_FOUND: + return PollOutcome(running=False) + return PollOutcome(running=True) + + +def _clean(value) -> str | None: + """A stripped string, or None when the field is absent or blank.""" + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _as_credit_count(body: dict) -> int: + """Read ``credits`` as a whole number, flooring any fraction. + + Enrow answers with an integer today, but the same reader BetterContact and LeadMagic + use keeps one shape for all three, and a float in a JSON number is the kind of thing + an API adds without warning. + """ + raw = body.get("credits") + try: + credits = int(float(raw)) + except (TypeError, ValueError): + raise EnrowUnavailable(f"Enrow returned no credit count: {body!r}") from None + if credits < 0: + raise EnrowUnavailable(f"Enrow returned a negative balance: {body!r}") + return credits + + +def _require_key() -> str: + from openoutfind.core.config import SiteConfig + + api_key = SiteConfig.load().enrow_api_key + if not api_key: + raise EnrowUnavailable("no Enrow API key configured") + return api_key + + +def _session(api_key: str) -> requests.Session: + """A session with Enrow's key and 429 backoff mounted on it. + + Enrow authenticates with ``x-api-key`` on every request and asks for exponential + backoff on a 429, so the retry lives in the transport where it belongs rather than in + a hand-rolled loop here. + """ + session = requests.Session() + session.headers.update({"x-api-key": api_key, "Accept": "application/json"}) + adapter = HTTPAdapter(max_retries=_RETRY) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _request(session: requests.Session, method: str, url: str, **kwargs): + """One HTTP call, with the budget spent and errors typed at the boundary.""" + kwargs.setdefault("timeout", _HTTP_TIMEOUT_S) + # The account read is free per Enrow's docs and outside their limiter, so it does + # not spend the budget the way a search does. + if url != _ACCOUNT_URL: + _BUDGET.take() + try: + response = session.request(method, url, **kwargs) + except requests.exceptions.RetryError as exc: + raise EnrowUnavailable( + "Enrow rate limit not cleared after retries", + ErrorType.PROVIDER_RATE_LIMITED, + ) from exc + if response.status_code == 401: + raise EnrowUnavailable("Enrow rejected the API key", ErrorType.PROVIDER_AUTH) + if response.status_code == 402: + raise EnrowUnavailable( + "Enrow reports insufficient credits", ErrorType.PROVIDER_OUT_OF_CREDITS + ) + # The API never answers 404: an unknown or expired id comes back as a 500 on the + # single endpoints, which is indistinguishable from a transient server error. Both + # defer the lookup rather than settle it as a miss. + if response.status_code >= 500: + raise EnrowUnavailable( + f"Enrow returned {response.status_code}", ErrorType.PROVIDER_UNAVAILABLE + ) + response.raise_for_status() + return response diff --git a/openoutfind/enrichment/fullenrich.py b/openoutfind/enrichment/fullenrich.py new file mode 100644 index 0000000..2b1f235 --- /dev/null +++ b/openoutfind/enrichment/fullenrich.py @@ -0,0 +1,315 @@ +"""FullEnrich: the paid finder's second waterfall, behind the same seam as BetterContact. + +Same two-leg handshake: ``start`` fires one bulk job and hands back FullEnrich's +``enrichment_id``; ``poll_once`` checks it once and returns a ``PollOutcome``. Nothing +here blocks on a poll: the collect step owns the wait, on the deal's own ``not_before``, +so a run parked at FINDING_EMAIL costs this process nothing. + +Why a second adapter at all: the seam's job is to pick the finder that is cheapest on +the operator's plan (``provider.py``), and the two waterfalls quote different prices per +credit at different tiers: FullEnrich is cheaper at entry and BetterContact is cheaper +at the top, so which one wins flips with volume. That is a pricing decision, not an +architecture change, which is the point of the seam. + +Discovery is **not** here. Lead Finder is BetterContact's free index and stays on +BetterContact's key, so a FullEnrich-only install can resolve an address but cannot +search, by decision (see ``REQUIRED_DISCOVERY_FIELDS`` in ``core/config.py``). +""" + +from __future__ import annotations + +import logging + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from openoutfind.core.errors import ErrorType +from openoutfind.enrichment.bettercontact import RequestBudget +from openoutfind.enrichment.provider import Lookup, PollOutcome, ProviderUnavailable + +logger = logging.getLogger(__name__) + +NAME = "fullenrich" + +# The one path to an account. FullEnrich runs no affiliate redirect of its own yet, so +# this is their plain signup page rather than a parameterised one: inventing a tracking +# query here would be a link that claims an attribution nobody granted. +SIGNUP_URL = "https://fullenrich.com" + +# One credit on the entry plan ($55/month for 1,000 credits), which is what an operator +# starting today pays: the per-credit price falls on larger plans, so this is an anchor +# for the cheapest-first pick and not a claim about volume. Kept beside the adapter, as +# BetterContact's is, so the selector reads it from the provider rather than a table +# that goes stale without the vendor. +UNIT_COST_USD = 0.055 + +_BULK_URL = "https://app.fullenrich.com/api/v2/contact/enrich/bulk" + +# One HTTP call's ceiling, not a job's. A bulk job walks FullEnrich's own waterfall for +# minutes, so the gap between polls belongs to the collect step's cadence rather than +# to a constant here. +_HTTP_TIMEOUT_S = 30 + +# FullEnrich publishes no per-minute figure on the API surface we can verify (their +# rate limit shows up as a job status, `RATE_LIMIT`), so this is our own courtesy +# budget rather than a quoted limit: the same 15/minute BetterContact's key lives +# under, per process, so six concurrent runs stay well inside anything either vendor +# would call abuse. It is a share of the key, not a measurement of the account. +REQUESTS_PER_MINUTE = 15 +_BUDGET_WINDOW_S = 60.0 +_RATE_LIMIT_ATTEMPTS = 5 + +# The statuses that actually mean "this address is one you can mail". FullEnrich's +# `CATCH_ALL` is **not** here on purpose: it only says the domain accepts everything, +# which is a domain-level fact with no claim about the mailbox, whereas BetterContact's +# `catch_all_safe` is their assertion that a specific mailbox was safely verified. The +# two providers' usable sets therefore are not interchangeable, and treating them as +# if they were would import unverified addresses into someone's sender reputation. +_USABLE_STATUSES = frozenset({"DELIVERABLE", "HIGH_PROBABILITY"}) + +_RUNNING_STATUSES = frozenset({"CREATED", "IN_PROGRESS"}) + +_RETRY = Retry( + total=_RATE_LIMIT_ATTEMPTS, + status_forcelist=(429,), + allowed_methods=frozenset({"GET", "POST"}), + backoff_factor=5, + backoff_max=120, + respect_retry_after_header=True, +) + +# The limiter is provider-agnostic and already tested where BetterContact defines it; +# FullEnrich gets its own instance because it is a different key on a different account, +# and labels its log line with its own name. +_BUDGET = RequestBudget(REQUESTS_PER_MINUTE, _BUDGET_WINDOW_S, label="fullenrich") + + +class FullEnrichUnavailable(ProviderUnavailable): + """FullEnrich could not run: no API key configured, or the service was unreachable. + + Distinct from a genuine miss (it ran, found no usable work email), exactly as + ``BetterContactUnavailable`` is. Carries a stable ``error_type`` from + ``core.errors.ErrorType``; the default is ``provider_unavailable`` and the HTTP and + job-status layers narrow it to auth, out-of-credits or rate-limited. + + ``lookup.py`` catches the base class and never names a vendor, so this name exists + for the tests and for a reader, not for a caller to branch on. + """ + + +def start(profile_url: str) -> Lookup: + """Fire one lookup job and hand back its handle: the async half of the interface. + + Never terminates here: a bulk job runs FullEnrich's waterfall, so the deal parks at + FINDING_EMAIL on the ``enrichment_id`` and ``poll_once`` finishes it later. + + **Only the profile URL is sent.** The API accepts a name and a company too, and + resolves better with them, but the query stays as narrow as BetterContact's for the + same reason: the less of a lead's record leaves for a third party, the better. Their + docs list ``professional_network_url`` as sufficient on its own. + """ + return Lookup(request_id=submit(profile_url)) + + +def is_configured() -> bool: + """True when the FullEnrich paid finder is configured (an API key is set).""" + from openoutfind.core.config import SiteConfig + + return bool(SiteConfig.load().fullenrich_api_key) + + +def credit_balance() -> int: + """FullEnrich exposes no balance endpoint, so this is not a balance of zero. + + BetterContact answers ``GET /account`` with ``credits_left``; FullEnrich's documented + surface reports only what a finished job cost (``cost.credits`` on the enrichment + itself), and no credits, account or team read exists to ask (checked 2026-11-11 + against their OpenAPI document). The seam's contract is *raise rather than guess*, so + the run reports why it cannot see a balance instead of reading it as zero and sending + the operator to top up an account that is already funded. + """ + raise FullEnrichUnavailable( + "FullEnrich does not expose a credit balance through its API; " + "check the account dashboard" + ) + + +def submit(profile_url: str) -> str: + """One bulk job for one profile URL; returns FullEnrich's own job id.""" + api_key = _require_key() + with _session(api_key) as session: + try: + payload = _request( + session, "POST", _BULK_URL, json=_enrich_body(profile_url) + ).json() + except (requests.RequestException, TimeoutError) as exc: + raise FullEnrichUnavailable(f"FullEnrich unreachable: {exc}") from exc + enrichment_id = payload.get("enrichment_id") + if not enrichment_id: + raise FullEnrichUnavailable("FullEnrich returned no enrichment id") + return enrichment_id + + +def poll_once(enrichment_id: str) -> PollOutcome: + """Ask once. No wait, no retry loop: the collect step decides when to ask again. + + FullEnrich runs a bulk job in minutes, so the gap between asks is the engine's + cadence rather than anything here. Returns ``running`` for a job FullEnrich has not + finished, a hit when it has and the address is one it vouched for, and a miss when + it has and it is not. A finished job that says the wallet was empty or the client + was throttled is a refusal, not a result, and raises: neither may ever be recorded + as an unreachable lead. + """ + api_key = _require_key() + url = f"{_BULK_URL}/{enrichment_id}" + with _session(api_key) as session: + try: + body = _request(session, "GET", url).json() + except (requests.RequestException, TimeoutError) as exc: + raise FullEnrichUnavailable(f"FullEnrich unreachable: {exc}") from exc + return _outcome(body) + + +def _enrich_body(profile_url: str) -> dict: + """The smallest bulk body that asks for a work email. + + ``enrich_fields`` is required and closed: asking for nothing parses to + ``error.enrichment.enrich_fields.empty``, and only ``contact.work_emails`` is wanted, + since a personal address costs three credits and a phone ten for a field this tool + does not send to. The job name is a label that makes the account's own log readable. + """ + return { + "name": "outfind", + "data": [ + { + "professional_network_url": profile_url, + "enrich_fields": ["contact.work_emails"], + } + ], + } + + +def _outcome(body: dict) -> PollOutcome: + """Turn one status document into the seam's three answers.""" + status = body.get("status") + + if status in _RUNNING_STATUSES: + return PollOutcome(running=True) + if status == "RATE_LIMIT": + raise FullEnrichUnavailable( + "FullEnrich rate-limited the job", ErrorType.PROVIDER_RATE_LIMITED + ) + if status == "CREDITS_INSUFFICIENT": + raise FullEnrichUnavailable( + "FullEnrich account is out of credits", ErrorType.PROVIDER_OUT_OF_CREDITS + ) + if status == "CANCELED": + return PollOutcome(running=False) + if status != "FINISHED": + # `UNKNOWN` is FullEnrich's own catch-all, and an unrecognised value is the same + # kind of answer: the job exists but its state was not resolved. Read as still + # running, as BetterContact's non-terminated states are, so a transient + # server-side answer cannot be recorded as a lead with no address. + return PollOutcome(running=True) + + row = (body.get("data") or [{}])[0] + email = _work_email(row.get("contact_info") or {}) + if not email: + return PollOutcome(running=False) + given = row.get("input") or {} + return PollOutcome( + running=False, + email=email, + first_name=given.get("first_name") or None, + last_name=given.get("last_name") or None, + ) + + +def _work_email(contact: dict) -> str: + """The first work address FullEnrich vouched for, or an empty string. + + The list is the evidence: each entry carries its own status. ``most_probable_work_email`` + is read as a fallback because the documented response shape lets it be either the + address itself or the same object, and a finished job that returned an address only + there is still an answer we paid for. + """ + for candidate in ( + *(contact.get("work_emails") or ()), + contact.get("most_probable_work_email"), + ): + address = _usable(candidate) + if address: + return address + return "" + + +def _usable(candidate: object) -> str: + """The address when the entry vouched for it. A bare string carries no status. + + A bare ``most_probable_work_email`` is the one exception: it is FullEnrich's own + most-probable address for a finished job, and with no status beside it there is + nothing to check it against, so discarding it would spend a credit and throw the + answer away. + """ + if isinstance(candidate, dict): + address = candidate.get("email") + if address and candidate.get("status") in _USABLE_STATUSES: + return address + return "" + return candidate if isinstance(candidate, str) else "" + + +def _require_key() -> str: + from openoutfind.core.config import SiteConfig + + api_key = SiteConfig.load().fullenrich_api_key + if not api_key: + raise FullEnrichUnavailable("no FullEnrich API key configured") + return api_key + + +def _session(api_key: str) -> requests.Session: + """A session that backs off on 429 before any of our code sees the response.""" + session = requests.Session() + session.headers.update( + {"Authorization": f"Bearer {api_key}", "Accept": "application/json"} + ) + adapter = HTTPAdapter(max_retries=_RETRY) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _request(session: requests.Session, method: str, url: str, **kwargs) -> requests.Response: + """One HTTP call, with the answers FullEnrich actually gives. + + The 429 backoff lives in the transport adapter, not here: a hand-rolled loop would be + a worse copy of what ``requests`` and ``urllib3`` already do. What is left is naming + the refusals a reader has to tell apart: a bad key and an empty wallet are final + answers, not things to retry. + + Any other error status is raised as ``HTTPError`` and surfaces as unavailable at the + seam. Without that, a 500 body carries no ``status``, ``_outcome`` reads it as a job + still running, and the lead polls a broken job forever. + """ + kwargs.setdefault("timeout", _HTTP_TIMEOUT_S) + _BUDGET.take() + try: + resp = session.request(method, url, **kwargs) + except requests.exceptions.RetryError as exc: + raise FullEnrichUnavailable( + f"FullEnrich rate-limited this client through {_RATE_LIMIT_ATTEMPTS} " + "backed-off attempts", + ErrorType.PROVIDER_RATE_LIMITED, + ) from exc + if resp.status_code == 401: + raise FullEnrichUnavailable( + "FullEnrich rejected the API key", ErrorType.PROVIDER_AUTH + ) + if resp.status_code == 402: + raise FullEnrichUnavailable( + "FullEnrich account is out of credits", ErrorType.PROVIDER_OUT_OF_CREDITS + ) + resp.raise_for_status() + return resp diff --git a/openoutfind/enrichment/leadmagic.py b/openoutfind/enrichment/leadmagic.py new file mode 100644 index 0000000..b2774b9 --- /dev/null +++ b/openoutfind/enrichment/leadmagic.py @@ -0,0 +1,275 @@ +"""LeadMagic: the paid finder that answers in a single call. + +The other two adapters speak a two-leg handshake because their APIs run a job: ``start`` +returns a handle and ``poll_once`` finishes it later. LeadMagic's email finder is +synchronous, so ``start`` is the whole lookup and returns a terminated ``PollOutcome``. +The seam already allows that (``Lookup.outcome`` rather than ``Lookup.request_id``), so +there is no ``poll_once`` here at all: a hit lands on RESOLVED without the deal ever +entering FINDING_EMAIL, and nothing ever parks on a handle that a vendor could strand. + +Why a third adapter: the seam's job is to pick the finder that is cheapest on the +operator's plan (``provider.py``), and every vendor quotes a different price per credit +at a different tier. LeadMagic answers in one call and undercuts both waterfalls at +entry ($99 buys 5,000 credits), though Enrow's entry plan is cheaper again. That is a +pricing decision, not an architecture change, which is the point of the seam. + +Discovery is **not** here either. Lead Finder is BetterContact's free index and stays on +BetterContact's key, so a LeadMagic-only install can resolve an address but cannot +search, by decision (see ``REQUIRED_DISCOVERY_FIELDS`` in ``core/config.py``). +""" + +from __future__ import annotations + +import logging + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from openoutfind.core.errors import ErrorType +from openoutfind.enrichment.bettercontact import RequestBudget +from openoutfind.enrichment.provider import Lookup, PollOutcome, ProviderUnavailable + +logger = logging.getLogger(__name__) + +NAME = "leadmagic" + +# The one path to an account. LeadMagic runs no affiliate redirect of its own, so this is +# their plain signup page rather than a parameterised one: inventing a tracking query here +# would be a link that claims an attribution nobody granted. +SIGNUP_URL = "https://leadmagic.io" + +# One credit on the entry plan ($99/month for 5,000 credits), which is what an operator +# starting today pays: the per-credit price falls on larger plans, so this is an anchor +# for the cheapest-first pick and not a claim about volume. Kept beside the adapter, as +# the other two are, so the selector reads it from the provider rather than a table that +# goes stale without the vendor. +UNIT_COST_USD = 0.0198 + +_FINDER_URL = "https://api.leadmagic.io/v1/people/email-finder" + +# The balance read, and the one call the request budget does not count. LeadMagic +# documents it as free and outside the rate limiter, which is what lets ``status`` +# ask for it without spending the operator's allowance. +_CREDITS_URL = "https://api.leadmagic.io/v1/credits" + +# One HTTP call's ceiling. A sync finder answers or refuses inside a single request, so +# there is no job clock to hand out. +_HTTP_TIMEOUT_S = 30 + +# LeadMagic publishes a per-plan rate limit (300 requests/minute on the entry plan) but +# that allowance is per organisation and shared by every key and credit bank, so a process +# taking the whole of it would starve the operator's other tools. This is our courtesy +# fifth of the published figure, per process, and it is deliberately not presented as a +# vendor number: the limiter is a share of the key, not a measurement of the account. +REQUESTS_PER_MINUTE = 60 +_BUDGET_WINDOW_S = 60.0 +_RATE_LIMIT_ATTEMPTS = 5 + +# LeadMagic's own word for an address its pipeline verified. Anything else (including +# ``null``) is a miss, and the vendor bills a miss at zero credits, so recording it as a +# miss costs the operator nothing. +_VALID_STATUS = "valid" + +_RETRY = Retry( + total=_RATE_LIMIT_ATTEMPTS, + status_forcelist=(429,), + allowed_methods=frozenset({"GET", "POST"}), + backoff_factor=5, + backoff_max=120, + respect_retry_after_header=True, +) + +# The limiter is provider-agnostic and already tested where BetterContact defines it; +# LeadMagic gets its own instance because it is a different key on a different account, +# and labels its log line with its own name. +_BUDGET = RequestBudget(REQUESTS_PER_MINUTE, _BUDGET_WINDOW_S, label="leadmagic") + + +class LeadMagicUnavailable(ProviderUnavailable): + """LeadMagic could not run: no API key configured, or the service was unreachable. + + Distinct from a genuine miss (it ran, found no verified work email), exactly as + ``BetterContactUnavailable`` is. Carries a stable ``error_type`` from + ``core.errors.ErrorType``; the default is ``provider_unavailable`` and the HTTP layer + narrows it to auth, out-of-credits or rate-limited. + + ``lookup.py`` catches the base class and never names a vendor, so this name exists + for the tests and for a reader, not for a caller to branch on. + """ + + +def start(profile_url: str) -> Lookup: + """Resolve one address and hand back a finished answer: the synchronous half. + + This is the whole lookup. LeadMagic's finder returns the result in the same response, + so the deal moves straight to RESOLVED or NO_EMAIL_FOUND and never occupies a poll + slot, which also means it can never be left holding a handle the vendor forgot. + + **Only the profile URL is sent.** The API accepts a name and a company too, and + resolves better with them, but the query stays as narrow as the other two adapters' + for the same reason: the less of a lead's record leaves for a third party, the + better. Their docs list ``profile_url`` as the strongest single input, sufficient on + its own without either name or domain. + """ + return Lookup(outcome=find_email(profile_url)) + + +def is_configured() -> bool: + """True when the LeadMagic paid finder is configured (an API key is set).""" + from openoutfind.core.config import SiteConfig + + return bool(SiteConfig.load().leadmagic_api_key) + + +def credit_balance() -> int: + """The credits left across the whole organisation, as LeadMagic counts them. + + Unlike FullEnrich, LeadMagic has a balance endpoint, ``GET /v1/credits``, so this is + a real number rather than a raised refusal and the run can warn before a lead fails + instead of after a 402. The response is ``{"credits": 15432.50}``, a float, so it goes + through the same floor-don't-round reading BetterContact's balance does: a fraction + of a credit buys nothing, and rounding it up would report a balance that cannot pay + for the next lookup. + """ + api_key = _require_key() + with _session(api_key) as session: + try: + body = _request(session, "GET", _CREDITS_URL).json() + except (requests.RequestException, TimeoutError) as exc: + raise LeadMagicUnavailable(f"LeadMagic unreachable: {exc}") from exc + return _as_credit_count(body) + + +def find_email(profile_url: str) -> PollOutcome: + """One synchronous lookup for one profile URL, as a terminated outcome.""" + api_key = _require_key() + with _session(api_key) as session: + try: + body = _request( + session, "POST", _FINDER_URL, json={"profile_url": profile_url} + ).json() + except (requests.RequestException, TimeoutError) as exc: + raise LeadMagicUnavailable(f"LeadMagic unreachable: {exc}") from exc + return _outcome(body) + + +def _outcome(body: dict) -> PollOutcome: + """Turn one finder document into a hit or a miss. Nothing here is still running. + + ``status`` is LeadMagic's own verdict on the address: ``valid`` means the pipeline + verified the work email, and ``null`` means it found nothing and billed nothing. A + domain-mismatched address arrives as a free not-found with ``domain_mismatch: true`` + unless the operator opts into accepting it, so it reads as a miss here without a + special case. + + The name parts ride the same response the email does, which is what lets a hit carry + them into the export with no second call and no second credit. + """ + if body.get("status") != _VALID_STATUS: + return PollOutcome(running=False) + email = body.get("email") + if not isinstance(email, str) or not email: + return PollOutcome(running=False) + return PollOutcome( + running=False, + email=email, + first_name=_clean(body.get("first_name")), + last_name=_clean(body.get("last_name")), + ) + + +def _clean(value: object) -> str | None: + """A name part as a non-empty string, or ``None`` when the vendor said nothing.""" + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _as_credit_count(body: dict) -> int: + """The credit count as an int, refusing anything that is not one. + + The provider answers with a JSON number holding a float (``15432.50``), so an int + check alone would reject a real answer and report ``provider_unavailable`` against a + 200 that carried the number. The floor is deliberate: a fraction of a credit buys + nothing, and a negative count is a broken answer rather than an empty wallet, so it + is refused instead of clamped to zero. + """ + raw = body.get("credits") + try: + credits = int(float(raw)) + except (TypeError, ValueError): + raise LeadMagicUnavailable( + f"LeadMagic returned no credit count: {body!r}" + ) from None + if credits < 0: + raise LeadMagicUnavailable( + f"LeadMagic returned a negative balance: {body!r}" + ) + return credits + + +def _require_key() -> str: + from openoutfind.core.config import SiteConfig + + api_key = SiteConfig.load().leadmagic_api_key + if not api_key: + raise LeadMagicUnavailable("no LeadMagic API key configured") + return api_key + + +def _session(api_key: str) -> requests.Session: + """A session that backs off on 429 before any of our code sees the response. + + No browser User-Agent is spoofed here, unlike BetterContact's Cloudflare front: a + probe of this API with no key answered 401 with and without a Chrome UA, so there is + no gate to work around and pretending to be a browser would only be a lie in the log. + """ + session = requests.Session() + session.headers.update( + {"X-API-Key": api_key, "Accept": "application/json"} + ) + adapter = HTTPAdapter(max_retries=_RETRY) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _request(session: requests.Session, method: str, url: str, **kwargs) -> requests.Response: + """One HTTP call, with the answers LeadMagic actually gives. + + The 429 backoff lives in the transport adapter, not here: a hand-rolled loop would be + a worse copy of what ``requests`` and ``urllib3`` already do. What is left is naming + the refusals a reader has to tell apart: a bad key and an empty wallet are final + answers, not things to retry. LeadMagic documents 402 as insufficient credits and + reuses it for a spent budget, so it maps to out-of-credits rather than to a generic + failure the caller might retry. + + Any other error status is raised as ``HTTPError`` and surfaces as unavailable at the + seam. Without that, a 500 body carries no ``status``, ``_outcome`` reads it as a miss, + and the operator is told a reachable person has no address. + + The balance read skips the request budget because the vendor documents it as free and + outside the limiter, the same exemption BetterContact gives its own account read. + """ + kwargs.setdefault("timeout", _HTTP_TIMEOUT_S) + if url != _CREDITS_URL: + _BUDGET.take() + try: + resp = session.request(method, url, **kwargs) + except requests.exceptions.RetryError as exc: + raise LeadMagicUnavailable( + f"LeadMagic rate-limited this client through {_RATE_LIMIT_ATTEMPTS} " + "backed-off attempts", + ErrorType.PROVIDER_RATE_LIMITED, + ) from exc + if resp.status_code == 401: + raise LeadMagicUnavailable( + "LeadMagic rejected the API key", ErrorType.PROVIDER_AUTH + ) + if resp.status_code == 402: + raise LeadMagicUnavailable( + "LeadMagic account is out of credits", ErrorType.PROVIDER_OUT_OF_CREDITS + ) + resp.raise_for_status() + return resp diff --git a/openoutfind/enrichment/lookup.py b/openoutfind/enrichment/lookup.py index 4959234..8e19c2d 100644 --- a/openoutfind/enrichment/lookup.py +++ b/openoutfind/enrichment/lookup.py @@ -79,52 +79,62 @@ def buy_address(deal) -> DealState | None: def _submit(deal) -> DealState | None: - """Run the configured finder — resolving outright, or parking on a job handle. - - Which vendor answers is ``provider.active()``'s decision and nothing here knows the - name; the branch that matters is *sync or async*, and the provider reports that by - what ``start`` returns. A synchronous hit lands on RESOLVED without the deal ever - entering FINDING_EMAIL, so it never occupies a poll slot and never risks the - stranded-handle case ``reclaim_lookup`` exists for. - - A couldn't-run (no key, no credits, API down) leaves the deal in - `READY_TO_FIND_EMAIL` — no credit was spent and there is no handle to poll — but - **backs it off first**. Without that the row is still due on the very next pass, - so the same deal is re-picked - immediately and forever: noise every few seconds under the old daemon, and a job that - never returns now that a bounded run stops only when nothing can advance. Writing - ``not_before`` is the architecture's one waiting mechanism, and an unreachable - provider is exactly the case it exists for. + """Walk the configured finders in price order until one can take the lookup. + + Which vendor answers is ``provider.chain()``'s decision and nothing here knows the + names: it hands back every configured provider, cheapest first, and each is tried in + turn. A provider that cannot run (no key, no credits, API down) raises + ``ProviderUnavailable`` and the walk moves to the next; only after every one has + refused does the deal go back to waiting. A provider that does run is authoritative, + miss included: a terminal miss is the answer to the lookup, not a reason to spend a + second credit at the next vendor. + + Inside a single provider the branch that matters is *sync or async*, and the provider + reports it by what ``start`` returns. A synchronous hit lands on RESOLVED without the + deal ever entering FINDING_EMAIL, so it never occupies a poll slot and never risks + the stranded-handle case ``reclaim_lookup`` exists for. + + A couldn't-run leaves the deal in `READY_TO_FIND_EMAIL` (no credit spent, no handle + to poll) but **backs it off first**. Without that the row is still due on the very + next pass, so the same deal is re-picked immediately and forever: noise every few + seconds under the old daemon, and a job that never returns now that a bounded run + stops only when nothing can advance. Writing ``not_before`` is the architecture's one + waiting mechanism, and an unreachable provider is exactly the case it exists for. """ from openoutfind.enrichment import provider - finder = provider.active() - if finder is None: + finders = provider.chain() + if not finders: logger.info("%s", step_line( - "finder", "unconfigured — left queued", glyph="⚠", color="yellow")) + "finder", "unconfigured: left queued", glyph="⚠", color="yellow")) _back_off(deal, advance=True) return None - try: - lookup = finder.start(deal.lead.profile_url) - except provider.ProviderUnavailable as exc: + for finder in finders: + try: + lookup = finder.start(deal.lead.profile_url) + except provider.ProviderUnavailable as exc: + logger.info("%s", step_line( + finder.NAME, f"unavailable ({exc}), trying next", + glyph="⚠", color="yellow")) + continue + + if not lookup.pending: + return _finish(deal, finder, lookup.outcome) + + deal.lookup_request_id = lookup.request_id + deal.lookup_provider = finder.NAME + deal.lookup_attempt = 0 + deal.not_before = timezone.now() + timedelta(seconds=COLLECT_BACKOFF_BASE_S) logger.info("%s", step_line( - finder.NAME, f"unavailable ({exc}) — left queued", - glyph="⚠", color="yellow")) - _back_off(deal, advance=True) - return None - - if not lookup.pending: - return _finish(deal, finder, lookup.outcome) + finder.NAME, f"submitted · req {lookup.request_id[:12]}… → FINDING_EMAIL · polling", + glyph="✓", color="green")) + return DealState.FINDING_EMAIL - deal.lookup_request_id = lookup.request_id - deal.lookup_provider = finder.NAME - deal.lookup_attempt = 0 - deal.not_before = timezone.now() + timedelta(seconds=COLLECT_BACKOFF_BASE_S) logger.info("%s", step_line( - finder.NAME, f"submitted · req {lookup.request_id[:12]}… → FINDING_EMAIL · polling", - glyph="✓", color="green")) - return DealState.FINDING_EMAIL + "finder", "no provider could run: left queued", glyph="⚠", color="yellow")) + _back_off(deal, advance=True) + return None def reclaim_lookup(deal) -> DealState: diff --git a/openoutfind/enrichment/provider.py b/openoutfind/enrichment/provider.py index cec1fbb..66af9a4 100644 --- a/openoutfind/enrichment/provider.py +++ b/openoutfind/enrichment/provider.py @@ -1,9 +1,13 @@ # openoutfind/enrichment/provider.py -"""The email-finder seam: one interface, one configured provider, two transports. +"""The email-finder seam: one interface, a ranked set of providers, two transports. ``lookup.py`` drives *a* provider, never a named one. Which module answers is decided -here by ``active()`` — today that is BetterContact whenever its key is set, the only -finder upstream ships. +here, and it is a price rather than an order: the configured finders sort by what one +verified address costs on each vendor's entry plan, and the cheapest leads. Four +finders ship, BetterContact and FullEnrich (waterfalls), LeadMagic (synchronous) and +Enrow (asynchronous, cheapest per hit), and which of them is cheaper flips with volume +(see ``UNIT_COST_USD`` in each), so the ranking is computed every run instead of being +written down here. **The interface spans sync and async, because finders differ.** BetterContact is a waterfall that takes seconds to minutes: ``start`` fires a job and returns a handle, the @@ -25,19 +29,20 @@ poll_once(id) PollOutcome — async providers only; sync ones never see a call **Adding a provider (including in a fork).** Write the module against the six names -above, add it to ``_providers()`` below and give it a key in ``core/config.py`` — that -is the whole integration; nothing else in the codebase names a vendor. With more than -one key set, the first configured entry in ``_providers()`` wins. +above, give it a ``UNIT_COST_USD`` saying what one hit costs on its entry plan, add it to +``_providers()`` below and give it a key in ``core/config.py``. That is the whole +integration; nothing else in the codebase names a vendor. With more than one key set the +cheapest configured entry leads and the rest follow it in price order (``chain()``). ``NAME`` is the part that leaves the machine. ``lookup`` passes it straight to ``contacts.contribute`` as the give-back's ``origin``, and it travels to the hub as -that **string**, never as a number — so pick a short lowercase slug for your vendor +that **string**, never as a number. Pick a short lowercase slug for your vendor (``"hunter"``, ``"dropcontact"``) and keep it stable, because it is what your contributions are grouped by forever after. The hub stores it verbatim whether or not it has ever heard of you, so a fork's own finder stays legible on the analytics side with no change needed there; only the small-int fast path in the hub's ``Contribution.Origin`` is limited to the vendors upstream ships. Do not reuse another -vendor's ``NAME`` to "fit in" — that silently merges your spend into theirs. +vendor's ``NAME`` to "fit in": that silently merges your spend into theirs. """ from __future__ import annotations @@ -99,15 +104,43 @@ def pending(self) -> bool: def _providers() -> tuple: - """Every finder upstream ships, in the order ``active`` prefers them.""" - from openoutfind.enrichment import bettercontact + """Every finder upstream ships. Order here is not preference: ``_cost`` decides.""" + from openoutfind.enrichment import bettercontact, enrow, fullenrich, leadmagic - return (bettercontact,) + return (bettercontact, enrow, fullenrich, leadmagic) + + +def _cost(module) -> float: + """What one verified address costs on *module*'s entry plan. + + Read from the module rather than kept in a table here so a finder's price lives + beside the finder, and a fork that adds a vendor states its own. A module that + carries no ``UNIT_COST_USD`` sorts last instead of failing: an unpriced finder is + still usable, just never preferred. + """ + return getattr(module, "UNIT_COST_USD", float("inf")) def active(): - """The provider module this install resolves with, or ``None`` if none is configured.""" - return next((module for module in _providers() if module.is_configured()), None) + """The cheapest configured provider module, or ``None`` if none is configured. + + Cheapest, not first-listed: the finders trade places with volume, so the ranking + is computed per run from each module's current ``UNIT_COST_USD``. + """ + configured = [module for module in _providers() if module.is_configured()] + return min(configured, key=_cost) if configured else None + + +def chain() -> tuple: + """Every configured provider, cheapest first: the order a lookup tries them. + + ``active()`` leads, so a caller that pins one provider for a test or a retry still + sees the same head, and everything after it is the fallback order. Empty when + nothing is configured. + """ + head = active() + rest = [module for module in _providers() if module.is_configured() and module is not head] + return ((head,) if head is not None else ()) + tuple(sorted(rest, key=_cost)) def by_name(name: str): diff --git a/skills/find-leads/SKILL.md b/skills/find-leads/SKILL.md index 1f9bf87..8a95d70 100644 --- a/skills/find-leads/SKILL.md +++ b/skills/find-leads/SKILL.md @@ -329,7 +329,7 @@ is a stable string worth branching on: | `goal_unreached` | Ran, produced fewer than asked. The rows are on stdout. | Read the message: a drained index is a dead end, addresses on order are a reason to run again later. | | `not_initialized` | No pipeline at this database yet. | `outfind check` | | `onboarding_incomplete` | The run was not given everything it needs. | The message names every missing variable; `outfind status` does too. | -| `no_credential` | No BetterContact key. | Configure one; discovery needs it too, and the free tier has 40 credits. | +| `no_credential` | No BetterContact key. | Configure one; discovery needs it too, and the free tier has 50 credits. | | `provider_auth` | The key was rejected. | Do not retry; the key is wrong. | | `provider_out_of_credits` | Credits exhausted. | Free `find N` still works; addresses do not. | | `provider_rate_limited` | 429. | Back off. **Never retry at speed** — the provider's docs say that can block the account. | diff --git a/tests/test_bettercontact.py b/tests/test_bettercontact.py index 57f99f8..bb35ed6 100644 --- a/tests/test_bettercontact.py +++ b/tests/test_bettercontact.py @@ -31,6 +31,17 @@ def unkeyed(db, configure): return configure(bettercontact_api_key="") +@pytest.fixture(autouse=True) +def _fresh_request_window(): + """The request budget is one per process, so the tests share its window. + + ``RequestBudget.take`` waits for real once the minute is full, which turns a + long file into real minutes of sleep. Emptying the window per test keeps the + real ``take`` in every path and the suite quick. + """ + bettercontact._BUDGET._sent.clear() + + def _response(body, error=None, status_code=200, headers=None): resp = MagicMock() resp.json.return_value = body diff --git a/tests/test_enrow.py b/tests/test_enrow.py new file mode 100644 index 0000000..5bf9db5 --- /dev/null +++ b/tests/test_enrow.py @@ -0,0 +1,366 @@ +# tests/test_enrow.py +"""Enrow slice, mocked at the HTTP boundary (`enrow._session`). + +The same two-leg async handshake as FullEnrich: ``start`` fires one single search and +returns Enrow's ``id``; ``poll_once`` checks it once and reports running / hit / miss. A +running search answers 202, so the status code is the cheap signal and the qualification +is the confirmation. + +The refusals carry more weight here than elsewhere. Enrow's LinkedIn lookup reports +``lookup_failed`` as retryable and free, and the API answers an unknown id with a 500 +rather than a 404, so both must defer the lookup rather than settle it as a miss: losing a +reachable person to a provider hiccup is worse than one more poll. +""" +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from openoutfind.core.errors import ErrorType +from openoutfind.enrichment import enrow +from openoutfind.enrichment.enrow import EnrowUnavailable + +URL = "https://www.linkedin.com/in/alice/" + + +@pytest.fixture +def keyed(configure): + return configure(enrow_api_key="secret") + + +@pytest.fixture +def unkeyed(configure): + return configure(enrow_api_key="") + + +@pytest.fixture(autouse=True) +def _fresh_request_window(): + """The request budget is one per process, so the tests share its window. + + Emptying the window per test keeps the real ``take`` in every path and stops a later + test inheriting a full minute of sleep from an earlier one. + """ + enrow._BUDGET._sent.clear() + + +def _response(body, error=None, status_code=200, headers=None): + resp = MagicMock() + resp.json.return_value = body + resp.raise_for_status.side_effect = error + resp.status_code = status_code + resp.headers = headers or {} + return resp + + +def _unreadable(): + """A 200 whose body is not JSON, which is a refusal and never a miss.""" + resp = _response(None) + resp.json.side_effect = ValueError("not JSON") + return resp + + +def _fake_session(post=None, get=None): + """A ``requests.Session`` stand-in usable as a context manager. + + ``_request`` goes through ``session.request(method, url, ...)``, the one place the + status refusals are typed, so the fake dispatches by method and the tests mock the + same boundary the real session presents. + """ + session = MagicMock() + session.__enter__.return_value = session + session.post = post or MagicMock() + session.get = get or MagicMock() + + def request(method, url, **kwargs): + handler = session.post if method.upper() == "POST" else session.get + return handler(url, **kwargs) + + session.request = MagicMock(side_effect=request) + return session + + +def _patch_session(post=None, get=None): + return patch.object(enrow, "_session", return_value=_fake_session(post, get)) + + +def _hit(email="alice@acme.com", **fields): + return _response({"qualification": "valid", "email": email, **fields}) + + +# enrow.start + +class TestStart: + def test_returns_enrows_own_handle(self, keyed): + post = MagicMock( + return_value=_response( + {"message": "Single search operating", "id": "e1", "credits_used": 1} + ) + ) + with _patch_session(post=post): + lookup = enrow.start(URL) + + assert lookup.request_id == "e1" + assert lookup.pending + + def test_sends_only_the_profile_url(self, keyed): + """The API takes a name and a company and resolves better with them, but the + query stays as narrow as the other adapters': the less of a lead's record leaves + for a third party, the better. + """ + post = MagicMock(return_value=_response({"id": "e1"})) + with _patch_session(post=post): + enrow.start(URL) + + assert post.call_args.kwargs["json"] == {"linkedin_url": URL} + + def test_no_key_is_unavailable(self, unkeyed): + post = MagicMock() + with _patch_session(post=post), pytest.raises(EnrowUnavailable): + enrow.start(URL) + post.assert_not_called() + + def test_a_response_with_no_id_is_unavailable(self, keyed): + post = MagicMock(return_value=_response({"message": "queued"})) + with _patch_session(post=post), pytest.raises(EnrowUnavailable): + enrow.start(URL) + + def test_network_error_is_unavailable(self, keyed): + post = MagicMock(side_effect=requests.ConnectionError("boom")) + with _patch_session(post=post), pytest.raises(EnrowUnavailable): + enrow.start(URL) + + +# enrow.poll_once + +class TestPollOnce: + def test_a_running_search_is_202(self, keyed): + """The status code is the signal, even when the body looks finished.""" + get = MagicMock( + return_value=_response(_hit(), status_code=202) + ) + with _patch_session(get=get): + outcome = enrow.poll_once("e1") + + assert outcome.running + assert not outcome.hit + + def test_the_handle_is_read_by_id(self, keyed): + get = MagicMock(return_value=_response({"qualification": "ongoing"})) + with _patch_session(get=get): + enrow.poll_once("e1") + + assert get.call_args.args[0] == f"{enrow._SINGLE_URL}?id=e1" + + def test_a_valid_qualification_with_an_address_is_a_hit(self, keyed): + with _patch_session(get=MagicMock(return_value=_hit())): + outcome = enrow.poll_once("e1") + + assert outcome.hit + assert outcome.email == "alice@acme.com" + + def test_the_address_carries_the_name_enrow_resolved(self, keyed): + body = _hit(info={"firstname": "Alice", "lastname": "Anders"}) + with _patch_session(get=MagicMock(return_value=body)): + outcome = enrow.poll_once("e1") + + assert outcome.first_name == "Alice" + assert outcome.last_name == "Anders" + + def test_a_valid_qualification_with_no_address_is_still_running(self, keyed): + """A half-filled document is not a lead, and settling it would drop the person.""" + with _patch_session(get=MagicMock(return_value=_response({"qualification": "valid"}))): + assert enrow.poll_once("e1").running + + def test_an_invalid_qualification_is_a_miss(self, keyed): + with _patch_session(get=MagicMock(return_value=_response({"qualification": "invalid"}))): + outcome = enrow.poll_once("e1") + + assert outcome.miss + assert not outcome.running + + def test_ongoing_is_running(self, keyed): + with _patch_session(get=MagicMock(return_value=_response({"qualification": "ongoing"}))): + assert enrow.poll_once("e1").running + + def test_an_unrecognised_qualification_reads_as_running(self, keyed): + """Recording it as a miss would settle the deal as NO_EMAIL_FOUND and never retry.""" + with _patch_session(get=MagicMock(return_value=_response({"qualification": "queued"}))): + assert enrow.poll_once("e1").running + + def test_a_non_json_body_is_unavailable(self, keyed): + with _patch_session(get=MagicMock(return_value=_unreadable())), pytest.raises( + EnrowUnavailable + ): + enrow.poll_once("e1") + + def test_no_key_is_unavailable(self, unkeyed): + get = MagicMock() + with _patch_session(get=get), pytest.raises(EnrowUnavailable): + enrow.poll_once("e1") + get.assert_not_called() + + def test_the_search_is_checked_exactly_once(self, keyed): + get = MagicMock(return_value=_response({"qualification": "ongoing"})) + with _patch_session(get=get): + enrow.poll_once("e1") + + get.assert_called_once() + + +# a transient LinkedIn status defers, even alongside a verdict + +class TestTransientLinkedInStatus: + def test_lookup_failed_defers_even_when_the_qualification_says_invalid(self, keyed): + """The status is checked first: ``lookup_failed`` costs nothing and is retryable, + so it must outrank a qualification that would otherwise settle the lookup. + """ + body = _response({"qualification": "invalid", "linkedin_status": "lookup_failed"}) + with _patch_session(get=MagicMock(return_value=body)), pytest.raises(EnrowUnavailable): + enrow.poll_once("e1") + + def test_a_settled_status_falls_through_to_the_miss(self, keyed): + body = _response({"qualification": "invalid", "linkedin_status": "email_not_found"}) + with _patch_session(get=MagicMock(return_value=body)): + outcome = enrow.poll_once("e1") + + assert outcome.miss + + +# enrow._request, the typed refusals + +class TestHttpRefusals: + def test_401_is_auth(self, keyed): + post = MagicMock(return_value=_response({}, status_code=401)) + with _patch_session(post=post), pytest.raises(EnrowUnavailable) as caught: + enrow.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_AUTH + + def test_402_is_out_of_credits(self, keyed): + """A single search that finds no credits answers 402, not a generic failure.""" + post = MagicMock(return_value=_response({}, status_code=402)) + with _patch_session(post=post), pytest.raises(EnrowUnavailable) as caught: + enrow.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_OUT_OF_CREDITS + + def test_a_server_error_on_a_search_is_unavailable(self, keyed): + post = MagicMock( + return_value=_response({}, error=requests.HTTPError("500"), status_code=500) + ) + with _patch_session(post=post), pytest.raises(EnrowUnavailable) as caught: + enrow.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_UNAVAILABLE + + def test_an_unknown_id_on_poll_is_unavailable_not_a_miss(self, keyed): + """The API answers an expired or unknown id with a 500, indistinguishable from a + transient server error, so the lookup defers rather than recording a miss. + """ + get = MagicMock( + return_value=_response({}, error=requests.HTTPError("500"), status_code=500) + ) + with _patch_session(get=get), pytest.raises(EnrowUnavailable) as caught: + enrow.poll_once("stale") + assert caught.value.error_type == ErrorType.PROVIDER_UNAVAILABLE + + def test_exhausted_backoff_is_rate_limited(self, keyed): + post = MagicMock(side_effect=requests.exceptions.RetryError("too many")) + with _patch_session(post=post), pytest.raises(EnrowUnavailable) as caught: + enrow.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_RATE_LIMITED + + +# enrow.credit_balance + +class TestCreditBalance: + def test_reads_the_whole_credit_count(self, keyed): + """The account document carries the balance beside the webhooks.""" + get = MagicMock(return_value=_response({"credits": 36000, "webhooks": []})) + with _patch_session(get=get): + assert enrow.credit_balance() == 36000 + + def test_a_float_balance_is_floored(self, keyed): + """A fraction of a credit buys nothing, and rounding up would claim a balance that + cannot pay for the next lookup. + """ + get = MagicMock(return_value=_response({"credits": 15432.50})) + with _patch_session(get=get): + assert enrow.credit_balance() == 15432 + + def test_a_missing_count_is_a_refusal(self, keyed): + get = MagicMock(return_value=_response({})) + with _patch_session(get=get), pytest.raises(EnrowUnavailable): + enrow.credit_balance() + + def test_a_non_numeric_count_is_a_refusal(self, keyed): + get = MagicMock(return_value=_response({"credits": "plenty"})) + with _patch_session(get=get), pytest.raises(EnrowUnavailable): + enrow.credit_balance() + + def test_a_negative_balance_is_a_refusal(self, keyed): + get = MagicMock(return_value=_response({"credits": -1})) + with _patch_session(get=get), pytest.raises(EnrowUnavailable): + enrow.credit_balance() + + def test_no_key_is_unavailable(self, unkeyed): + get = MagicMock() + with _patch_session(get=get), pytest.raises(EnrowUnavailable): + enrow.credit_balance() + get.assert_not_called() + + def test_an_unreachable_service_is_unavailable(self, keyed): + get = MagicMock(side_effect=requests.ConnectionError("boom")) + with _patch_session(get=get), pytest.raises(EnrowUnavailable): + enrow.credit_balance() + + +# the per-process request budget + +class TestRequestBudget: + def test_a_search_takes_a_place(self, keyed): + session = _fake_session(post=MagicMock(return_value=_response({"id": "e1"}))) + with patch.object(enrow._BUDGET, "take") as take: + enrow._request(session, "POST", enrow._SINGLE_URL, json={}) + take.assert_called_once() + + def test_a_poll_takes_a_place(self, keyed): + session = _fake_session(get=MagicMock(return_value=_response({}))) + with patch.object(enrow._BUDGET, "take") as take: + enrow._request(session, "GET", f"{enrow._SINGLE_URL}?id=e1") + take.assert_called_once() + + def test_the_balance_read_does_not_take_a_place(self): + """Enrow documents the account read as free and outside its limiter, the same + exemption BetterContact gives its own account read. + """ + session = _fake_session(get=MagicMock(return_value=_response({"credits": 1}))) + with patch.object(enrow._BUDGET, "take") as take: + enrow._request(session, "GET", enrow._ACCOUNT_URL) + take.assert_not_called() + + +# enrow's published price and setup + +class TestIsConfigured: + def test_false_when_key_blank(self, unkeyed): + assert enrow.is_configured() is False + + def test_true_when_key_set(self, keyed): + assert enrow.is_configured() is True + + +class TestSignupUrl: + def test_is_the_plain_signup_page_with_no_invented_attribution(self): + """Enrow runs no affiliate redirect of its own, so a query string here would be a + link claiming an attribution nobody granted. + """ + assert enrow.SIGNUP_URL == "https://enrow.io" + assert "?" not in enrow.SIGNUP_URL + + +class TestThePublishedPrice: + def test_the_entry_plan_is_cheaper_than_the_other_three(self): + """The cheapest-first pick depends on this, so the anchor is pinned here.""" + from openoutfind.enrichment import bettercontact, fullenrich, leadmagic + + assert enrow.UNIT_COST_USD < min( + bettercontact.UNIT_COST_USD, fullenrich.UNIT_COST_USD, leadmagic.UNIT_COST_USD + ) diff --git a/tests/test_fullenrich.py b/tests/test_fullenrich.py new file mode 100644 index 0000000..62c43eb --- /dev/null +++ b/tests/test_fullenrich.py @@ -0,0 +1,350 @@ +# tests/test_fullenrich.py +"""FullEnrich slice, mocked at the HTTP boundary (`fullenrich._session`). + +The second paid finder is the same two-leg async handshake as BetterContact: +``submit`` fires one bulk job and returns FullEnrich's ``enrichment_id``; +``poll_once`` checks that job exactly once and reports running / hit / miss. A +missing key or an unreachable service raises FullEnrichUnavailable rather than a +bare error, and a finished job that says the wallet was empty or the client was +throttled raises too, since neither is ever an unreachable lead. +""" +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from openoutfind.core.errors import ErrorType +from openoutfind.enrichment import fullenrich +from openoutfind.enrichment.fullenrich import FullEnrichUnavailable + +URL = "https://www.linkedin.com/in/alice/" + + +@pytest.fixture +def keyed(db, configure): + return configure(fullenrich_api_key="secret") + + +@pytest.fixture +def unkeyed(db, configure): + return configure(fullenrich_api_key="") + + +@pytest.fixture(autouse=True) +def _fresh_request_window(): + """The request budget is one per process, so the tests share its window. + + ``RequestBudget.take`` waits for real once the minute is full, which turned + the fifteenth filer in this file into a real minute of sleep. Emptying the + window per test keeps the real ``take`` in every path and the suite quick. + """ + fullenrich._BUDGET._sent.clear() + + +def _response(body, error=None, status_code=200, headers=None): + resp = MagicMock() + resp.json.return_value = body + resp.raise_for_status.side_effect = error + resp.status_code = status_code + resp.headers = headers or {} + return resp + + +def _fake_session(post=None, get=None): + """A requests.Session stand-in usable as a context manager. + + ``_request`` goes through ``session.request(method, url, ...)``, the one place + the status refusals are typed, so the fake dispatches by method and the tests + mock the same boundary the real session presents. + """ + session = MagicMock() + session.__enter__.return_value = session + session.post = post or MagicMock() + session.get = get or MagicMock() + + def request(method, url, **kwargs): + handler = session.post if method.upper() == "POST" else session.get + return handler(url, **kwargs) + + session.request = MagicMock(side_effect=request) + return session + + +def _patch_session(post=None, get=None): + return patch.object(fullenrich, "_session", return_value=_fake_session(post, get)) + + +def _finished(contact_info, input_=None): + """One FINISHED status document carrying a single record.""" + row = {"contact_info": contact_info} + if input_ is not None: + row["input"] = input_ + return _response({"status": "FINISHED", "data": [row]}) + + +# fullenrich.submit + +class TestSubmit: + def test_returns_enrichment_id(self, keyed): + post = MagicMock(return_value=_response({"enrichment_id": "e1"})) + with _patch_session(post): + assert fullenrich.submit(URL) == "e1" + + def test_sends_only_the_profile_url(self, keyed): + """The API takes a name and a company and resolves better with them, but + the query stays as narrow as BetterContact's: the less of a lead's record + leaves for a third party, the better. + """ + post = MagicMock(return_value=_response({"enrichment_id": "e1"})) + with _patch_session(post): + fullenrich.submit(URL) + + row = post.call_args.kwargs["json"]["data"][0] + assert row["professional_network_url"] == URL + assert row["enrich_fields"] == ["contact.work_emails"] + assert set(row) == {"professional_network_url", "enrich_fields"} + + def test_no_key_is_unavailable(self, unkeyed): + post = MagicMock() + with _patch_session(post), pytest.raises(FullEnrichUnavailable): + fullenrich.submit(URL) + post.assert_not_called() + + def test_missing_enrichment_id_is_unavailable(self, keyed): + post = MagicMock(return_value=_response({})) + with _patch_session(post), pytest.raises(FullEnrichUnavailable): + fullenrich.submit(URL) + + def test_http_error_is_unavailable(self, keyed): + post = MagicMock( + return_value=_response({}, error=requests.HTTPError("500"), status_code=500) + ) + with _patch_session(post), pytest.raises(FullEnrichUnavailable): + fullenrich.submit(URL) + + def test_network_error_is_unavailable(self, keyed): + post = MagicMock(side_effect=requests.ConnectionError("boom")) + with _patch_session(post), pytest.raises(FullEnrichUnavailable): + fullenrich.submit(URL) + + +# fullenrich.poll_once + +class TestPollOnce: + def test_created_is_running(self, keyed): + get = MagicMock(return_value=_response({"status": "CREATED"})) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.running and not outcome.hit and not outcome.miss + + def test_in_progress_is_running_and_polls_once(self, keyed): + get = MagicMock(return_value=_response({"status": "IN_PROGRESS"})) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.running + get.assert_called_once() # a single poll, no retry loop + + def test_polls_the_jobs_own_url(self, keyed): + get = MagicMock(return_value=_response({"status": "CREATED"})) + with _patch_session(get=get): + fullenrich.poll_once("e1") + assert get.call_args.args[0].endswith("/e1") + + def test_finished_deliverable_is_a_hit(self, keyed): + get = MagicMock( + return_value=_finished( + {"work_emails": [{"email": "alice@acme.com", "status": "DELIVERABLE"}]} + ) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.hit and outcome.email == "alice@acme.com" + + def test_high_probability_is_a_hit(self, keyed): + get = MagicMock( + return_value=_finished( + {"work_emails": [{"email": "alice@acme.com", "status": "HIGH_PROBABILITY"}]} + ) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.hit + + def test_catch_all_is_a_miss(self, keyed): + """CATCH_ALL only says the domain accepts everything, a domain-level fact + with no claim about the mailbox, so it is not a usable status here. + """ + get = MagicMock( + return_value=_finished( + {"work_emails": [{"email": "alice@acme.com", "status": "CATCH_ALL"}]} + ) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.miss and not outcome.hit + + def test_a_later_usable_entry_wins_when_the_first_is_not(self, keyed): + get = MagicMock( + return_value=_finished( + { + "work_emails": [ + {"email": "alice@acme.com", "status": "UNVERIFIED"}, + {"email": "a.smith@acme.com", "status": "DELIVERABLE"}, + ] + } + ) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.email == "a.smith@acme.com" + + def test_a_bare_most_probable_email_is_a_hit(self, keyed): + """With no status beside it there is nothing to check it against, so + discarding it would spend a credit and throw the answer away. + """ + get = MagicMock( + return_value=_finished({"most_probable_work_email": "alice@acme.com"}) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.hit and outcome.email == "alice@acme.com" + + def test_a_hit_carries_the_identity_the_job_echoed(self, keyed): + get = MagicMock( + return_value=_finished( + {"work_emails": [{"email": "elon@tesla.com", "status": "DELIVERABLE"}]}, + input_={"first_name": "Elon", "last_name": "Musk"}, + ) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.hit + assert (outcome.first_name, outcome.last_name) == ("Elon", "Musk") + + def test_a_hit_without_identity_fields_is_still_a_hit(self, keyed): + get = MagicMock( + return_value=_finished( + {"work_emails": [{"email": "alice@acme.com", "status": "DELIVERABLE"}]} + ) + ) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.hit + assert outcome.first_name is None and outcome.last_name is None + + def test_finished_with_no_usable_email_is_a_miss(self, keyed): + get = MagicMock(return_value=_finished({"work_emails": []})) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.miss and not outcome.hit + + def test_finished_with_no_rows_is_a_miss(self, keyed): + get = MagicMock(return_value=_response({"status": "FINISHED", "data": []})) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.miss + + def test_canceled_is_a_miss(self, keyed): + get = MagicMock(return_value=_response({"status": "CANCELED"})) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.miss and not outcome.running + + def test_unknown_reads_as_still_running(self, keyed): + """UNKNOWN is FullEnrich's own catch-all, so a transient server-side + answer cannot be recorded as a lead with no address. + """ + get = MagicMock(return_value=_response({"status": "UNKNOWN"})) + with _patch_session(get=get): + outcome = fullenrich.poll_once("e1") + assert outcome.running and not outcome.miss + + def test_rate_limited_job_is_typed(self, keyed): + get = MagicMock(return_value=_response({"status": "RATE_LIMIT"})) + with _patch_session(get=get), pytest.raises(FullEnrichUnavailable) as caught: + fullenrich.poll_once("e1") + assert caught.value.error_type == ErrorType.PROVIDER_RATE_LIMITED + + def test_out_of_credits_job_is_typed(self, keyed): + get = MagicMock(return_value=_response({"status": "CREDITS_INSUFFICIENT"})) + with _patch_session(get=get), pytest.raises(FullEnrichUnavailable) as caught: + fullenrich.poll_once("e1") + assert caught.value.error_type == ErrorType.PROVIDER_OUT_OF_CREDITS + + def test_no_key_is_unavailable(self, unkeyed): + get = MagicMock() + with _patch_session(get=get), pytest.raises(FullEnrichUnavailable): + fullenrich.poll_once("e1") + get.assert_not_called() + + def test_transport_error_is_unavailable(self, keyed): + get = MagicMock( + return_value=_response({}, error=requests.HTTPError("500"), status_code=500) + ) + with _patch_session(get=get), pytest.raises(FullEnrichUnavailable): + fullenrich.poll_once("e1") + + +# the typed refusals the HTTP boundary owes the caller + +class TestHttpRefusals: + def test_401_is_auth(self, keyed): + post = MagicMock(return_value=_response({}, status_code=401)) + with _patch_session(post), pytest.raises(FullEnrichUnavailable) as caught: + fullenrich.submit(URL) + assert caught.value.error_type == ErrorType.PROVIDER_AUTH + + def test_402_is_out_of_credits(self, keyed): + post = MagicMock(return_value=_response({}, status_code=402)) + with _patch_session(post), pytest.raises(FullEnrichUnavailable) as caught: + fullenrich.submit(URL) + assert caught.value.error_type == ErrorType.PROVIDER_OUT_OF_CREDITS + + def test_exhausted_backoff_is_rate_limited(self, keyed): + post = MagicMock(side_effect=requests.exceptions.RetryError("too many")) + with _patch_session(post), pytest.raises(FullEnrichUnavailable) as caught: + fullenrich.submit(URL) + assert caught.value.error_type == ErrorType.PROVIDER_RATE_LIMITED + + +# fullenrich.credit_balance + +class TestCreditBalance: + def test_raises_rather_than_reporting_a_balance_of_zero(self): + """FullEnrich exposes no balance endpoint, so a zero read here would send + an operator to top up an account that is already funded. + """ + with patch.object(fullenrich, "_session") as session, pytest.raises( + FullEnrichUnavailable + ): + fullenrich.credit_balance() + session.assert_not_called() + + +# the per-process request budget + +class TestRequestBudget: + def test_every_request_takes_a_place(self): + session = _fake_session(get=MagicMock(return_value=_response({}))) + with patch.object(fullenrich._BUDGET, "take") as take: + fullenrich._request(session, "GET", "https://app.fullenrich.com/api/v2/x") + take.assert_called_once() + + +# fullenrich.is_configured + +class TestIsConfigured: + def test_false_when_key_blank(self, unkeyed): + assert fullenrich.is_configured() is False + + def test_true_when_key_set(self, keyed): + assert fullenrich.is_configured() is True + + +class TestSignupUrl: + def test_is_the_plain_signup_page_with_no_invented_attribution(self): + """FullEnrich runs no affiliate redirect of its own, so a query string here + would be a link claiming an attribution nobody granted. + """ + assert fullenrich.SIGNUP_URL == "https://fullenrich.com" + assert "?" not in fullenrich.SIGNUP_URL diff --git a/tests/test_leadmagic.py b/tests/test_leadmagic.py new file mode 100644 index 0000000..b2c5d46 --- /dev/null +++ b/tests/test_leadmagic.py @@ -0,0 +1,326 @@ +"""LeadMagic resolves an address inside the ``start`` call. + +The other two adapters speak a two-leg handshake because their APIs run a job. LeadMagic's +finder answers in the same response, so ``start`` is the whole lookup and every outcome is +already finished: there is no ``poll_once`` here to be left holding a handle. + +The refusals matter more than usual in this file. A provider that cannot run must raise, so +``lookup.py`` steps over it to the next finder; a provider that does run is authoritative, +miss included. A missing key, an unreachable service, an empty wallet and a throttle are all +"could not run", never "this lead has no address". +""" + +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from openoutfind.core.errors import ErrorType +from openoutfind.enrichment import leadmagic +from openoutfind.enrichment.leadmagic import LeadMagicUnavailable + +URL = "https://www.linkedin.com/in/alice/" + + +@pytest.fixture +def keyed(configure): + return configure(leadmagic_api_key="secret") + + +@pytest.fixture +def unkeyed(configure): + return configure(leadmagic_api_key="") + + +@pytest.fixture(autouse=True) +def _fresh_request_window(): + """The request budget is one per process, so the tests share its window. + + Emptying the window per test keeps the real ``take`` in every path and stops a later + test inheriting a full minute of sleep from an earlier one. + """ + leadmagic._BUDGET._sent.clear() + + +def _response(body, error=None, status_code=200, headers=None): + resp = MagicMock() + resp.json.return_value = body + resp.raise_for_status.side_effect = error + resp.status_code = status_code + resp.headers = headers or {} + return resp + + +def _fake_session(post=None, get=None): + """A ``requests.Session`` stand-in usable as a context manager. + + ``_request`` goes through ``session.request(method, url, ...)``, the one place the + status refusals are typed, so the fake dispatches by method and the tests mock the same + boundary the real session presents. + """ + session = MagicMock() + session.__enter__.return_value = session + session.post = post or MagicMock() + session.get = get or MagicMock() + + def request(method, url, **kwargs): + handler = session.post if method.upper() == "POST" else session.get + return handler(url, **kwargs) + + session.request = MagicMock(side_effect=request) + return session + + +def _patch_session(post=None, get=None): + return patch.object(leadmagic, "_session", return_value=_fake_session(post, get)) + + +def _hit(email="alice@acme.com", **fields): + return _response({"status": "valid", "email": email, **fields}) + + +# leadmagic.start + +class TestStart: + def test_start_is_the_whole_lookup(self, keyed): + """The answer is already terminated, so the deal never enters FINDING_EMAIL.""" + post = MagicMock(return_value=_hit()) + with _patch_session(post=post): + lookup = leadmagic.start(URL) + + assert lookup.outcome is not None + assert not lookup.pending + assert lookup.outcome.hit + assert lookup.outcome.email == "alice@acme.com" + + def test_start_carries_no_request_id(self, keyed): + """Nothing is submitted to be collected later, so there is no handle to record.""" + with _patch_session(post=MagicMock(return_value=_hit())): + lookup = leadmagic.start(URL) + + assert lookup.request_id == "" + + def test_a_miss_is_also_finished(self, keyed): + post = MagicMock(return_value=_response({"status": None})) + with _patch_session(post=post): + lookup = leadmagic.start(URL) + + assert not lookup.pending + assert lookup.outcome.miss + + def test_there_is_no_poll_once(self): + """A synchronous finder must not offer a poll, or the seam would call one.""" + assert not hasattr(leadmagic, "poll_once") + + def test_sends_only_the_profile_url(self, keyed): + """The API takes a name and a company and resolves better with them, but the query + stays as narrow as the other two adapters': the less of a lead's record leaves for + a third party, the better. + """ + post = MagicMock(return_value=_hit()) + with _patch_session(post=post): + leadmagic.start(URL) + + assert post.call_args.kwargs["json"] == {"profile_url": URL} + + def test_no_key_is_unavailable(self, unkeyed): + post = MagicMock() + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable): + leadmagic.start(URL) + post.assert_not_called() + + def test_http_error_is_unavailable(self, keyed): + post = MagicMock( + return_value=_response({}, error=requests.HTTPError("500"), status_code=500) + ) + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable): + leadmagic.start(URL) + + def test_network_error_is_unavailable(self, keyed): + post = MagicMock(side_effect=requests.ConnectionError("boom")) + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable): + leadmagic.start(URL) + + +# leadmagic._outcome, driven through find_email + +class TestOutcome: + def test_valid_status_is_a_hit(self, keyed): + with _patch_session(post=MagicMock(return_value=_hit())): + outcome = leadmagic.find_email(URL) + assert outcome.hit and outcome.email == "alice@acme.com" + + def test_a_hit_carries_the_names_from_the_same_response(self, keyed): + post = MagicMock( + return_value=_hit("elon@tesla.com", first_name="Elon", last_name="Musk") + ) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.hit + assert (outcome.first_name, outcome.last_name) == ("Elon", "Musk") + + def test_names_are_trimmed(self, keyed): + post = MagicMock(return_value=_hit(first_name=" El on ", last_name=" Musk\n")) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert (outcome.first_name, outcome.last_name) == ("El on", "Musk") + + def test_blank_names_are_absent(self, keyed): + post = MagicMock(return_value=_hit(first_name=" ", last_name="")) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.first_name is None and outcome.last_name is None + + def test_a_hit_without_names_is_still_a_hit(self, keyed): + with _patch_session(post=MagicMock(return_value=_hit())): + outcome = leadmagic.find_email(URL) + assert outcome.hit + assert outcome.first_name is None and outcome.last_name is None + + def test_null_status_is_a_miss(self, keyed): + """``null`` is LeadMagic finding nothing, and it bills nothing for it.""" + post = MagicMock(return_value=_response({"status": None, "credits_consumed": 0})) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.miss and not outcome.hit + + def test_a_missing_status_is_a_miss(self, keyed): + post = MagicMock(return_value=_response({"email": "alice@acme.com"})) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.miss + + def test_an_empty_email_is_a_miss(self, keyed): + post = MagicMock(return_value=_response({"status": "valid", "email": ""})) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.miss + + def test_a_non_string_email_is_a_miss(self, keyed): + post = MagicMock(return_value=_response({"status": "valid", "email": 42})) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.miss + + def test_a_domain_mismatch_is_a_free_miss(self, keyed): + """LeadMagic returns the unrelated-domain address as a not-found with a flag, so + it reads as a miss without a special case here. + """ + post = MagicMock( + return_value=_response( + { + "status": None, + "email": "alice@elsewhere.com", + "domain_mismatch": True, + "credits_consumed": 0, + } + ) + ) + with _patch_session(post=post): + outcome = leadmagic.find_email(URL) + assert outcome.miss and not outcome.hit + + +# the typed refusals the HTTP boundary owes the caller + +class TestHttpRefusals: + def test_401_is_auth(self, keyed): + post = MagicMock(return_value=_response({}, status_code=401)) + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable) as caught: + leadmagic.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_AUTH + + def test_402_is_out_of_credits(self, keyed): + """LeadMagic documents 402 as insufficient credits, not a generic failure.""" + post = MagicMock(return_value=_response({}, status_code=402)) + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable) as caught: + leadmagic.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_OUT_OF_CREDITS + + def test_exhausted_backoff_is_rate_limited(self, keyed): + post = MagicMock(side_effect=requests.exceptions.RetryError("too many")) + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable) as caught: + leadmagic.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_RATE_LIMITED + + def test_a_server_error_is_unavailable_not_a_miss(self, keyed): + """A 500 body carries no ``status``, so without the raise it would read as a miss + and tell the operator a reachable person has no address. + """ + post = MagicMock( + return_value=_response({}, error=requests.HTTPError("500"), status_code=500) + ) + with _patch_session(post=post), pytest.raises(LeadMagicUnavailable) as caught: + leadmagic.start(URL) + assert caught.value.error_type == ErrorType.PROVIDER_UNAVAILABLE + + +# leadmagic.credit_balance + +class TestCreditBalance: + def test_a_float_balance_is_floored(self, keyed): + """The provider answers ``{"credits": 15432.50}``, a float, and a fraction of a + credit buys nothing. + """ + get = MagicMock(return_value=_response({"credits": 15432.50})) + with _patch_session(get=get): + assert leadmagic.credit_balance() == 15432 + + def test_a_missing_count_is_a_refusal(self, keyed): + get = MagicMock(return_value=_response({})) + with _patch_session(get=get), pytest.raises(LeadMagicUnavailable): + leadmagic.credit_balance() + + def test_a_negative_balance_is_a_refusal(self, keyed): + get = MagicMock(return_value=_response({"credits": -1})) + with _patch_session(get=get), pytest.raises(LeadMagicUnavailable): + leadmagic.credit_balance() + + def test_no_key_is_unavailable(self, unkeyed): + get = MagicMock() + with _patch_session(get=get), pytest.raises(LeadMagicUnavailable): + leadmagic.credit_balance() + get.assert_not_called() + + def test_an_unreachable_service_is_unavailable(self, keyed): + get = MagicMock(side_effect=requests.ConnectionError("boom")) + with _patch_session(get=get), pytest.raises(LeadMagicUnavailable): + leadmagic.credit_balance() + + +# the per-process request budget + +class TestRequestBudget: + def test_a_finder_call_takes_a_place(self, keyed): + session = _fake_session(post=MagicMock(return_value=_hit())) + with patch.object(leadmagic._BUDGET, "take") as take: + leadmagic._request(session, "POST", leadmagic._FINDER_URL, json={}) + take.assert_called_once() + + def test_the_balance_read_does_not_take_a_place(self): + """The vendor documents the balance as free and outside the limiter, the same + exemption BetterContact gives its own account read. + """ + session = _fake_session(get=MagicMock(return_value=_response({"credits": 1}))) + with patch.object(leadmagic._BUDGET, "take") as take: + leadmagic._request(session, "GET", leadmagic._CREDITS_URL) + take.assert_not_called() + + +# leadmagic.is_configured + +class TestIsConfigured: + def test_false_when_key_blank(self, unkeyed): + assert leadmagic.is_configured() is False + + def test_true_when_key_set(self, keyed): + assert leadmagic.is_configured() is True + + +class TestSignupUrl: + def test_is_the_plain_signup_page_with_no_invented_attribution(self): + """LeadMagic runs no affiliate redirect of its own, so a query string here would + be a link claiming an attribution nobody granted. + """ + assert leadmagic.SIGNUP_URL == "https://leadmagic.io" + assert "?" not in leadmagic.SIGNUP_URL diff --git a/tests/test_provider.py b/tests/test_provider.py index 7357429..0e9b1b9 100644 --- a/tests/test_provider.py +++ b/tests/test_provider.py @@ -13,9 +13,9 @@ import pytest from openoutfind.crm.models import DealState -from openoutfind.enrichment import bettercontact, provider +from openoutfind.enrichment import bettercontact, enrow, fullenrich, leadmagic, provider from openoutfind.enrichment.lookup import buy_address -from openoutfind.enrichment.provider import Lookup, PollOutcome +from openoutfind.enrichment.provider import Lookup, PollOutcome, ProviderUnavailable from tests.factories import DealFactory, LeadFactory @@ -44,6 +44,25 @@ def _sync_finder(outcome: PollOutcome): ) +def _finder(name: str, outcome=None, cost=None, fails: bool = False): + """A configured finder with an optional price tag, able to refuse the call. + + Price is left off by default so a test can prove the unpriced case sorts last + rather than winning on a fluke. Refusal is a side effect, not a return value, + because that is how a real adapter reports it cannot run. + """ + module = SimpleNamespace( + NAME=name, + is_configured=lambda: True, + start=MagicMock(return_value=Lookup(outcome=outcome)), + ) + if cost is not None: + module.UNIT_COST_USD = cost + if fails: + module.start.side_effect = ProviderUnavailable(f"{name} is down") + return module + + # ── selection ───────────────────────────────────────────────────── @@ -118,3 +137,113 @@ def test_the_free_sources_still_come_first(self, config, site_config): assert buy_address(deal) == DealState.RESOLVED start.assert_not_called() + + +# the chain steps over a refusal, and only a refusal + + +class TestTheChainStepsOverARefusal: + def test_a_provider_that_cannot_run_hands_the_lookup_to_the_next(self, config, site_config): + """Availability outranks price: a vendor that cannot run does not get paid.""" + deal = _ready_to_find(site_config) + down = _finder("down", fails=True) + up = _finder("up", outcome=PollOutcome(running=False, email="alice@acme.com")) + + with patch.object(provider, "_providers", return_value=(down, up)), \ + patch("openoutfind.contacts.service.contribute"): + assert buy_address(deal) == DealState.RESOLVED + + assert deal.lead.email == "alice@acme.com" + up.start.assert_called_once() + + def test_a_terminal_miss_does_not_pay_a_second_vendor(self, config, site_config): + """A miss is an answer. Paying a second vendor to hear it again is waste.""" + deal = _ready_to_find(site_config) + first = _finder("first", outcome=PollOutcome(running=False)) + second = _finder("second", outcome=PollOutcome(running=False, email="late@acme.com")) + + with patch.object(provider, "_providers", return_value=(first, second)): + assert buy_address(deal) == DealState.NO_EMAIL_FOUND + + second.start.assert_not_called() + + +# the chain is price ordered, and unpriced sorts last + + +class TestTheChainIsPriceOrdered: + def test_the_cheapest_configured_provider_leads(self): + cheap = _finder("cheap", cost=0.05) + dear = _finder("dear", cost=0.09) + + with patch.object(provider, "_providers", return_value=(dear, cheap)): + assert [module.NAME for module in provider.chain()] == ["cheap", "dear"] + assert provider.active() is cheap + + def test_a_provider_with_no_published_price_sorts_last(self): + priced = _finder("priced", cost=0.2) + unpriced = _finder("unpriced") + + with patch.object(provider, "_providers", return_value=(unpriced, priced)): + assert [module.NAME for module in provider.chain()] == ["priced", "unpriced"] + + def test_an_unconfigured_provider_is_not_in_the_chain(self): + keyed = _finder("keyed", cost=0.1) + blank = SimpleNamespace( + NAME="blank", + is_configured=lambda: False, + start=MagicMock(), + ) + + with patch.object(provider, "_providers", return_value=(keyed, blank)): + assert [module.NAME for module in provider.chain()] == ["keyed"] + assert provider.active() is keyed + + +# the shipped roster, with no stand-ins + + +class TestTheRealRoster: + """The real modules, not doubles: the order the pricing actually justifies. + + A mocked chain proves the sort. This proves the roster, so a registered finder + whose price or key wiring drifted fails here instead of quietly overspending. + """ + + def test_the_shipped_finders_run_cheapest_first(self, config): + config( + bettercontact_api_key="bc", + enrow_api_key="enrow", + fullenrich_api_key="fr", + leadmagic_api_key="lm", + ) + + assert [module.NAME for module in provider.chain()] == [ + "enrow", + "leadmagic", + "fullenrich", + "bettercontact", + ] + + def test_only_the_keyed_finders_are_in_the_chain(self, config): + config(enrow_api_key="enrow") + + assert [module.NAME for module in provider.chain()] == ["enrow"] + assert provider.active() is enrow + + +# a stored handle names the vendor that minted it + + +class TestByName: + def test_a_fullenrich_handle_is_polled_by_fullenrich(self): + assert provider.by_name("fullenrich") is fullenrich + + def test_an_enrow_handle_is_polled_by_enrow(self): + assert provider.by_name("enrow") is enrow + + def test_a_leadmagic_handle_is_polled_by_leadmagic(self): + assert provider.by_name("leadmagic") is leadmagic + + def test_an_unknown_handle_yields_nothing(self): + assert provider.by_name("no-such-finder") is None