From 321f9fc34012e2d531478601fc8d19f4471394ee Mon Sep 17 00:00:00 2001 From: namirali Date: Mon, 2 Nov 2026 23:57:23 +0300 Subject: [PATCH 1/2] Add opt-in CompanyEnrich discovery with bounded spend and local provenance --- ARCHITECTURE.md | 45 +- CLAUDE.md | 33 +- LEGAL_NOTICE.md | 3 +- README.md | 25 +- docs/companyenrich.md | 126 +++ openoutfind/companyenrich.py | 160 ++++ openoutfind/companyenrich_query.schema.json | 747 ++++++++++++++++++ openoutfind/contacts/service.py | 42 +- openoutfind/core/config.py | 3 + openoutfind/core/db/leads.py | 14 +- openoutfind/core/discovery_source.py | 30 + openoutfind/core/export.py | 4 +- openoutfind/core/job.py | 5 + openoutfind/core/management/commands/find.py | 30 +- .../core/migrations/0009_companyenrichpage.py | 25 + openoutfind/core/models.py | 16 + openoutfind/core/pipeline/companyenrich.py | 107 +++ openoutfind/core/pipeline/discover.py | 6 + openoutfind/core/pipeline/judge_or_search.py | 6 + openoutfind/core/readiness.py | 7 +- openoutfind/core/status.py | 9 +- .../migrations/0007_lead_discovery_source.py | 18 + openoutfind/crm/models/lead.py | 3 + pyproject.toml | 1 + skills/find-leads/SKILL.md | 39 +- tests/test_companyenrich.py | 369 +++++++++ 26 files changed, 1853 insertions(+), 20 deletions(-) create mode 100644 docs/companyenrich.md create mode 100644 openoutfind/companyenrich.py create mode 100644 openoutfind/companyenrich_query.schema.json create mode 100644 openoutfind/core/discovery_source.py create mode 100644 openoutfind/core/migrations/0009_companyenrichpage.py create mode 100644 openoutfind/core/pipeline/companyenrich.py create mode 100644 openoutfind/crm/migrations/0007_lead_discovery_source.py create mode 100644 tests/test_companyenrich.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d6ee144..0ced3e3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -10,6 +10,49 @@ export** — off licensed data, with no social-network account and no scraping. reason the LLM chose it. Everything downstream of that file — the message, the mailbox, the sequence, the opt-out — belongs to whatever tool the operator already sends with. +## Explicit paid source: CompanyEnrich + +CompanyEnrich is a B2B company/people data API. Its integration is an optional, +provider-selected page source (`core/discovery_source.py`), separate from the default +free BetterContact lattice and the email-resolver seam. The CLI requires +`--companyenrich-query PATH`, `--companyenrich-request-id ID` and +`--discovery-credit-budget N` together. Only that invocation reads +`OPENOUTFIND_COMPANYENRICH_API_KEY` for discovery; readiness uses that key instead of +BetterContact's. Selection resets at the command boundary, even on errors. + +`companyenrich.py` validates direct people-search filters against a bundled OpenAPI +subset (no `companyFilter`, expansions or unknown fields), defaults to page 1/size 10, +and caps size at 100. The reservation is `2 * pageSize` credits at the current published +rate. It sends one POST to the fixed HTTPS host with timeout/body bounds, no redirects, +no retries and redacted errors. A larger budget never causes more requests. `find 0` +never calls it; existing pool work may meet the goal before any purchase. + +`core.CompanyEnrichPage` is a durable reservation keyed by the operator-supplied ID, +created in autocommit before HTTP. The same ID cannot submit again, including after a +crash or concurrent invocation. An unknown response blocks reuse until the operator +inspects usage and explicitly authorizes a new ID. A successful full response is saved +before local embedding, enabling no-spend recovery. Query, maximum credits and import +state persist; keys do not. It does not create free-lattice Search/Page observations. + +The adapter accepts a usable professional profile URL and current title, copies reported +name parts, country and current employment, and applies the existing company cap. +The free provider's headline heuristic is not applied. The qualifier consumes local +candidates before considering this one page; exhausted authorized data gives +`goal_unreached` and no implicit further spending. The full raw response keeps exact +IDs, nested fields and pagination; existing CSV/JSON lead columns remain unchanged. +`--json` run metadata gains a `discovery` object; existing error types cover refusal. + +`crm.Lead.discovery_source` records CompanyEnrich provenance, including duplicate +identities whose existing profile stays unchanged. Later discovery cannot clear that +marker. Contacts-hub resolution and contribution skip those leads permanently; raw +CompanyEnrich profiles/companies never enter the hub sharing path. Qualification still +uses the operator's LLM, and exports still go wherever the operator directs stdout. +See [setup and the complete scope/cost contract](docs/companyenrich.md). +Later free-provider pages also filter identities already marked CompanyEnrich before +sharing raw profiles or employers; alternate host/trailing-slash/query spellings are +matched conservatively. A new source cannot undo the persisted local-only choice. + + ## Project Layout All source lives in the single `openoutfind/` package; Django apps are nested inside it @@ -28,7 +71,7 @@ openoutfind/ # config.py (the environment) + readiness.py, llm.py, conf.py, # ML (qualifier/embeddings), discovery+qualify pipeline, # the lead export, db/ helpers, geo, management commands - enrichment/ # the one paid step (not an app — no models) — the BetterContact + enrichment/ # paid email resolution (not an app — no models) — the BetterContact # client and the two-step buy/check lookup crm/ # app (label: outfind_crm) — Lead, Company, Deal contacts/ # central contacts-store client (service.py only — no models, not an app) diff --git a/CLAUDE.md b/CLAUDE.md index bf35344..5fdd932 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,9 +20,9 @@ OpenOutFind — a self-hosted **lead finder that qualifies for you**, with **zer **define ICP → discover → qualify → rank → (optionally) resolve an address → export CSV.** -- **Discovery** is a licensed source — BetterContact **Lead Finder** (ICP search returns firmographic profiles, no emails, billed nothing). The walk is a best-first search over the lattice of searches: the LLM writes the first 10, and each round fires at once the 10 searches — next pages, neighbours on the lattice — expected to add the most people beating the best lead waiting, per cost. +- **Default discovery** is a licensed source — BetterContact **Lead Finder** (ICP search returns firmographic profiles, no emails, billed nothing). The walk is a best-first search over the lattice of searches: the LLM writes the first 10, and each round fires at once the 10 searches — next pages, neighbours on the lattice — expected to add the most people beating the best lead waiting, per cost. - **Qualification** is the crown jewel — **GPR + BALD active learning** over 384-dim embeddings, decided by an LLM, which also **writes down why**. That reason is the product. -- **Enrichment** is the one paid step — a **finder** resolves a work email for the top-ranked leads only (one credit per verified hit), fronted by a free cross-operator cache (the hub). Optional: a lead with no address still exports. **The resolver sits behind a seam** (`enrichment/provider.py`), which picks a finder from whichever key is configured — BetterContact (async waterfall) is the only one upstream ships, and the seam's `Lookup` still spans a synchronous finder so a fork can add one without touching the pipeline. **Only the resolver leg is behind it** — discovery is BetterContact's Lead Finder and nothing else. +- **Email enrichment** is a separate paid step — a **finder** resolves a work email for the top-ranked leads only (one credit per verified hit), fronted by a free cross-operator cache (the hub). Optional: a lead with no address still exports. **The resolver sits behind a seam** (`enrichment/provider.py`), which picks a finder from whichever key is configured — BetterContact (async waterfall) is the only one upstream ships, and the seam's `Lookup` still spans a synchronous finder so a fork can add one without touching the pipeline. **Only the resolver leg is behind that seam.** Default discovery is BetterContact Lead Finder; an explicitly selected CompanyEnrich page uses the separate `core/discovery_source.py` seam. - **The output is the command's own stdout.** `outfind find 10 emails > leads.csv` prints **every lead in the store** in the column names Instantly and Smartlead already read, so the newest file supersedes every earlier one. No export command, and no file the tool manages. See *The lead export* below. Django + Django Admin own the CRM/ORM (models are this project's); pydantic-ai drives the LLM calls. @@ -37,6 +37,35 @@ Three consequences worth knowing before changing anything: 2. **The opt-out duty belongs to whoever sends.** A finder that contacts nobody is not the sender under CAN-SPAM/GDPR/CASL. `Lead.disqualified` is the export's exclusion filter; nothing sets it automatically. The hub store's own suppression is separate and unaffected. 3. **The GP never trains on a market signal.** `get_labeled_arrays` reads the LLM's fit verdict alone — no reply or bounce data ever enters that loop, by construction, since OpenOutFind never sees a reply or a bounce. +## Optional paid CompanyEnrich discovery + +`find --companyenrich-query PATH --companyenrich-request-id ID --discovery-credit-budget N` +selects one bounded people-search page. All three flags are required; a configured +`OPENOUTFIND_COMPANYENRICH_API_KEY` alone is inert. `N >= 2 * pageSize` reserves the +maximum at the current published 2-credit/profile rate, independently of `--emails`. +No automatic next page, compound company lookup, expansion or free-walk fallback. +The default bare `find N` retains its free BetterContact discovery contract. + +`core/discovery_source.py` scopes selection to one command and restores it on failure. +`companyenrich.py` validates the query against the checked-in official schema subset, +makes one bounded/no-retry/no-redirect HTTPS request and adapts current professional +profiles. `core/pipeline/companyenrich.py` commits `CompanyEnrichPage` before HTTP, +stores the full response before embedding, and reuses it across resumed invocations. +An ambiguous outcome never automatically replays; a different query cannot reuse its +ID. The paid source does not fabricate free lattice searches or change their estimates. + +Persist `Lead.discovery_source=companyenrich` even for previously known URLs. Do not +clear it on later BetterContact discovery: hub `resolve` and `contribute` must stay +blocked, and raw CompanyEnrich pages/companies never go to `share_*`. Existing LLM +qualification and operator-directed exports still consume these profiles. CLI JSON +metadata adds `discovery`; export columns and error vocabulary remain unchanged. +See [the complete contract](docs/companyenrich.md). +Later free-provider pages also filter identities already marked CompanyEnrich before +sharing raw profiles or employers; alternate host/trailing-slash/query spellings are +matched conservatively. A new source cannot undo the persisted local-only choice. + Keep these flags and paid/local +boundaries synchronized in README, ARCHITECTURE and `skills/find-leads/SKILL.md`. + ## Commands ```bash diff --git a/LEGAL_NOTICE.md b/LEGAL_NOTICE.md index 0b8340a..46e6a91 100644 --- a/LEGAL_NOTICE.md +++ b/LEGAL_NOTICE.md @@ -11,7 +11,7 @@ OpenOutFind is a self-hosted, open-source **lead finder**. It discovers B2B lead > This notice describes how the software behaves and is **not legal advice**. You are responsible for your own compliance; where the stakes warrant it, consult a lawyer. Material aspects of the data model below are still pending a formal legal review. ### 1. No Platform Scraping or Automation -OpenOutFind performs **no** automated access to any social or professional network — no login, no browser session, no bot, no scraping, no messaging on such a platform. Lead **discovery** comes from a licensed data provider (currently BetterContact **Lead Finder**), and **enrichment** (resolving a work email) comes from a paid email-finder — both third-party services **you** sign up for and configure with **your own** API key, used under **that provider's** terms. +OpenOutFind performs **no** automated access to any social or professional network — no login, no browser session, no bot, no scraping, no messaging on such a platform. Lead **discovery** comes from a licensed data provider (BetterContact **Lead Finder** by default, or explicitly selected paid CompanyEnrich discovery), and **enrichment** (resolving a work email) comes from a paid email-finder — both third-party services **you** sign up for and configure with **your own** API key, used under **that provider's** terms. - **Profile URLs are identifiers, not fetch targets.** A discovered lead may carry a professional-network profile URL as an opaque identifier. OpenOutFind **stores it and never visits it** — it is a lookup/dedup key, nothing more. - **You accept the third-party terms.** You are responsible for using the data provider and email-finder in line with each provider's terms of service and acceptable-use policy. @@ -51,6 +51,7 @@ Address resolution runs through a **third-party email-finder** (e.g. BetterConta ### 6. Central Contacts Store (Contribution and Resolution) OpenOutFind connects to a **central contacts store operated by the project maintainer** (`hub.openoutreach.app`). It pools three things across the OpenOutreach network: work email addresses, so a contact one operator has already paid to resolve can be served — for free — to another; the professional profiles discovery returns, so the pool of people outlives the free index they came from; and the company records those results carry, from which the maintainer builds a company index. By running the software you participate as described here. +- **CompanyEnrich exception:** CompanyEnrich discovery pages, employers and embeddings are not contributed to the central store. Persisted CompanyEnrich identities also bypass hub resolution, later email give-back and sharing from later free-provider pages. The operator's existing LLM qualification and directed exports still apply. See [the source contract](docs/companyenrich.md). - **Resolved addresses — what is sent, and when**: **after a paid email-finder returns a verified work email**, OpenOutFind sends the person's **profile identifier** (the stored, never-fetched profile URL), the **country code** searched, and the **work email address(es)** resolved. - **Discovery profiles — what is sent, and when**: **every page of results a Lead Finder search returns** is sent as one batch, including people your campaign discards and people you already hold. Each row carries the **profile identifier**, the **country code** searched, and the row's professional fields as the provider returned them — **name, headline, job title, seniority, industry, state/region and country, employer name and domain, employer industry** — plus a **384-dimension numeric vector** computed locally from those fields and the name of the model that produced it. The employer's free-text **description and keywords are never sent with the person**: the provider fuzzy-matches them and they often describe a different company. **No email address is part of this record, and it earns no store credit.** - **Company records — what is sent, and when**: with every such page, OpenOutFind also sends **one record per distinct company** the page carried that has a LinkedIn company page: that **page's URL**, and the company's **name, domain, industry, keywords, description, LinkedIn size band and headquarters city and country** as the provider returned them, plus a **384-dimension numeric vector** computed locally from the company's name, industry, keywords and description, and the name of the model that produced it. **No person is attached to a company record** — not who was employed there, not the search that found it — and it earns no store credit. The same gates apply as for discovery profiles: nothing is sent from an EEA/UK/CH-based operator or for a page searched there. The maintainer keeps these records in a **private company index** for its own use; the store does not serve them to operators. diff --git a/README.md b/README.md index 8e885d8..62f1f45 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,29 @@ It has **zero platform-ToS surface**: browserless, no social-network account, no 4. **An LLM qualifies each candidate** against your ICP and **writes down why**. A model of your ICP (Gaussian Process over profile embeddings) learns from those verdicts and picks who to qualify next 5. **Every lead you have prints as CSV when the run ends** — name, title, company, website, profile URL, and the `reason` — so `> leads.csv` is the only file there is. Ask for addresses (`find 10 emails`, or `--emails`) and the best-fit leads get a **paid email lookup** first, one credit per verified hit, gated on the model's confidence so the spend goes to the leads most likely to fit -Searching the licensed source is free, so the system can afford to look at a lot and spend paid lookups only on the best fits. *(The learning loop is an active experiment — it is not yet shown to beat picking at random, and no claim is made that it does.)* +Searching the default BetterContact source is free, so the system can afford to look at a lot and spend paid lookups only on the best fits. *(The learning loop is an active experiment — it is not yet shown to beat picking at random, and no claim is made that it does.)* --- +## Optional CompanyEnrich source + +[CompanyEnrich](https://companyenrich.com/) provides B2B company and people data APIs. +Its people search can supply a bounded page of professional profiles for the existing +qualification and export workflow. It is **paid and explicitly opt-in**: setting +`OPENOUTFIND_COMPANYENRICH_API_KEY` alone changes nothing. + +```bash +outfind find 5 --companyenrich-query people.json \ + --companyenrich-request-id founders-page-1 --discovery-credit-budget 20 +``` + +The query file controls filters and page size; ten requested profiles reserve at most +20 CompanyEnrich credits at the current published rate. Reuse the request ID when +resuming to avoid duplicate purchases. CompanyEnrich data is excluded from the shared +contacts hub, including later email contributions. People search supplies no emails. +See [setup, query example, spending and data handling](docs/companyenrich.md). +The default free BetterContact discovery and separate `--emails` opt-in remain unchanged. + ## 📤 What You Get Out The deliverable is a file, and it is shaped for the tools you already send with. You ask for an @@ -76,7 +95,7 @@ outfind status --json # the same thing, for a script or an agent email, first_name, last_name, company, title, website, linkedin_url, reason, lead_id, qualified_at, full_name ``` -Those column names are **the importers', not ours**. Instantly and Smartlead both require `email`/`first_name`/`last_name` and recognise `company`/`title`/`website`/`linkedin_url` as standard fields, so an exported file imports **without column mapping**. Anything else — including `reason` — arrives as a custom variable you can merge into a template. `first_name`/`last_name` come from the email lookup, so a leads-only file has them blank; `full_name` always names the person. +Those column names are **the importers', not ours**. Instantly and Smartlead both require `email`/`first_name`/`last_name` and recognise `company`/`title`/`website`/`linkedin_url` as standard fields, so an exported file imports **without column mapping**. Anything else — including `reason` — arrives as a custom variable you can merge into a template. `first_name`/`last_name` come from the provider when supplied (CompanyEnrich discovery or the email lookup), never a guessed split; `full_name` always names the person. - **`reason` is the point.** Everybody exports rows; almost nobody exports *why this lead*. - **There is no score column, on purpose.** The model's confidence is a spend gate for the paid lookup, not a quality signal, and thresholding on it would be reading a number that was never calibrated to mean "good lead". The fit verdict is the LLM's, and it is already in the file as a sentence. @@ -125,7 +144,7 @@ Two things that are *not* privileges, and that is the point: - 📝 **A stated reason per lead** — read exactly why the agent picked someone, and fix the description when it is wrong - 🔍 **Nothing decides in the dark** — the ICP, the verdicts and the whole pipeline are on your machine and open to read - 🛡️ **Zero platform-ToS surface** — browserless, no social-network account, no scraping — nothing to get banned -- 💸 **Pay only for what resolves** — searching is free; a paid lookup is rationed and billed on a verified hit +- 💸 **Default discovery is free** — BetterContact searching is free; a paid lookup is rationed and billed on a verified hit - 📤 **Exports where you already work** — CSV in the shape the sequencer importers expect - ⚡ **One-command setup** — `uvx --from openoutfind outfind find 10`, configured by four environment variables, no container required diff --git a/docs/companyenrich.md b/docs/companyenrich.md new file mode 100644 index 0000000..5051d44 --- /dev/null +++ b/docs/companyenrich.md @@ -0,0 +1,126 @@ +# Optional CompanyEnrich discovery + +[CompanyEnrich](https://companyenrich.com/) is a B2B company and people data API. +It supports company enrichment, account search, similar companies, professional +profiles and workforce research. OpenOutFind uses its people-search endpoint to +bring a deliberately chosen, bounded page of profiles into local qualification. + +**CompanyEnrich discovery is paid.** Merely setting its key does nothing. The +existing BetterContact free discovery remains the default. To authorize one paid +page, supply all three flags and the key: + +```bash +export OPENOUTFIND_COMPANYENRICH_API_KEY='your-api-key' +outfind find 5 --companyenrich-query people.json \ + --companyenrich-request-id founders-page-1 --discovery-credit-budget 20 +``` + +`people.json`: + +```json +{ + "domains": ["example.com"], + "positionQuery": ["Founder", "CEO"], + "countries": ["US"], + "page": 1, + "pageSize": 10 +} +``` + +The normal product/target and LLM configuration still apply. `--agent-qualify` +continues to work, including the existing `population_pending` cold-start and +`qualify_pending` verdict handoff. Keep the same query, request ID and budget when +answering those handoffs. A CompanyEnrich-only discovery run does not require a +BetterContact key; the separate `--emails` option still needs an email finder. +The `check` and standalone `status` verbs describe the default BetterContact path. + +## What can spend + +People search currently costs **2 CompanyEnrich credits per returned profile**. +The local reservation is conservative: `2 × pageSize` credits, even if the API +returns fewer profiles. `pageSize` defaults to 10 and is limited to 1–100; the +budget must cover that entire page before any request. The integration makes at +most **one** request, even with a larger budget. It never spends the unused budget +on another page. Pricing was checked on 2026-11-02 against the +[provider pricing](https://companyenrich.com/pricing); the provider controls actual +billing and this local cap assumes that published rate remains in effect. + +The goal (`find 5`) counts qualified leads, not paid results. Ten purchased profiles +may produce fewer than five fits. If the pool cannot advance, the command prints +what it has and exits `goal_unreached`. A fresh run may satisfy its goal from an +existing pool without buying the selected page at all. `find 0` never purchases it. +No automatic free-source expansion runs while the CompanyEnrich source is selected. + +Email purchase is a separate authorization: the discovery flags do not imply +`--emails`, and `--emails` does not authorize CompanyEnrich discovery. People search +returns **no email addresses or phone numbers**. Existing email lookup remains a +separate provider workflow; CompanyEnrich-sourced identities bypass the shared hub. + +## Resuming without duplicate purchases + +The request ID is an operator-chosen 1–64 character identifier using letters, +digits, `_`, `.` or `-`. It identifies one paid request in this database: + +- The reservation is committed before HTTP. A repeated or concurrent invocation + with the same ID cannot submit it again. +- The full successful response is stored before embedding or qualification. A + local interruption can resume using that response without another purchase. +- A reused ID with different query parameters fails. Choose a **new ID** to + explicitly buy a different page; keep `pageSize` fixed while incrementing `page`. +- HTTP failures, rate limits, redirects, invalid responses and timeouts are never + retried. If the outcome is unknown, the existing ID remains blocked. Check + provider usage before authorizing a retry using a new ID: it could charge again. + +The request ID is local replay protection, **not provider-side idempotency**. A new +ID or different database creates a separate authorization. The stored +`core.CompanyEnrichPage` contains query, maximum credits, raw response and import +state. It contains no API key. Under `--json`, run metadata adds a `discovery` +object with source, ID, page, size, maximum credits, selected budget and hub policy. +Provider refusal uses existing `provider_auth`, `provider_out_of_credits`, +`provider_rate_limited` or `provider_unavailable` errors; configuration uses +`bad_config`/`onboarding_incomplete`. Raw error bodies and credentials are not echoed. + +## Filters and profile fields + +Supported request fields follow the checked-in official OpenAPI subset: +`domains`, `query` (company name/domain text), `positionQuery`, `countries`, +`seniority`, `department`, current-company/current-position UTC date bounds, +`education`, `exclude`, `page` and `pageSize`. At least one targeting filter is +required. Unknown fields, invalid enum values, page bounds and malformed UTC dates +fail before spending. See the [API reference](https://docs.companyenrich.com/reference). + +Current title, reported names, country, current employer/domain/industry and +seniority feed the existing local lead/qualification fields. Names are copied +only when supplied, never split in-house. Historical employment is not treated +as current. Profiles without a usable LinkedIn URL or current title are retained +in the raw page but skipped as local qualification candidates; company caps also +apply. The BetterContact-specific missing-headline heuristic is not applied to a +provider that does not supply that field. Nested history and page metadata remain +in the stored response; they are not represented as new export columns. + +## Data handling and scope + +CompanyEnrich pages are never sent to the shared contacts hub: neither profiles, +companies nor embeddings. `Lead.discovery_source` persists across runs and known +profile deduplication. A later email resolution cannot send that identity or email +to the hub, and hub lookups are skipped too. Other sources cannot clear that marker. +An existing row's profile is not overwritten when the same person appears again. +Later free-provider pages also filter identities already marked CompanyEnrich before +sharing raw profiles or employers; alternate host/trailing-slash/query spellings are +matched conservatively. A new source cannot undo the persisted local-only choice. + + +“Local” here means excluded from the shared hub. The existing qualification path +still sends profile text to the operator's configured LLM, and stdout/CSV/JSON +still exports qualified leads to the operator's chosen tools. Operator registration +and aggregate account-balance reads retain their existing behavior. CompanyEnrich +is a data source, not permission to redistribute its records to a shared data pool. + +Company search/lookalikes/enrichment, workforce, person reverse lookup, compound +`companyFilter` searches, async jobs, scroll, exports and beta email/mobile +operations are deliberately outside this finder contribution. They need separate +inputs and spend contracts; none runs behind a people lookup. In particular, +`companyFilter` could add a company-search step, so it is rejected by the bounded +people-only schema. This source does not add sending capabilities. + +Tests use synthetic fixtures only; no paid API calls or real credentials are needed. diff --git a/openoutfind/companyenrich.py b/openoutfind/companyenrich.py new file mode 100644 index 0000000..c75b343 --- /dev/null +++ b/openoutfind/companyenrich.py @@ -0,0 +1,160 @@ +"""CompanyEnrich's bounded, paid people search; no email discovery or hub calls. + +Request fields are validated against a checked-in subset of the official OpenAPI. +Compound companyFilter lookups and expansions are deliberately excluded from this +fixed 2-credit/person contract. See docs/companyenrich.md for scope and pricing. +""" +from __future__ import annotations + +import json +from datetime import datetime, timedelta +from importlib.resources import files +from pathlib import Path +from urllib.parse import urlsplit + +import requests +from jsonschema import Draft202012Validator, FormatChecker + +from openoutfind.core.errors import ErrorType, OpenOutFindError +from openoutfind.enrichment.provider import ProviderUnavailable + +API_URL = "https://api.companyenrich.com/people/search" +CREDITS_PER_PERSON = 2 +MAX_RESPONSE_BYTES = 4 * 1024 * 1024 +_SCHEMA = json.loads(files("openoutfind").joinpath("companyenrich_query.schema.json").read_text()) +_VALIDATOR = Draft202012Validator(_SCHEMA, format_checker=FormatChecker()) + + +def read_query(path: str) -> dict: + try: + with Path(path).open("rb") as stream: + raw = stream.read(65537) + if len(raw) > 65536: + raise OpenOutFindError(ErrorType.BAD_CONFIG, "CompanyEnrich query exceeds 64 KiB") + query = json.loads(raw) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + raise OpenOutFindError(ErrorType.BAD_CONFIG, "Cannot read CompanyEnrich query JSON") from None + return validate_query(query) + + +def validate_query(query: dict) -> dict: + if not isinstance(query, dict): + raise OpenOutFindError(ErrorType.BAD_CONFIG, "CompanyEnrich query must be an object") + query = {"page": 1, "pageSize": 10, **query} + error = next(_VALIDATOR.iter_errors(query), None) + if error is not None: + # The validator's full message can echo user input. Report the field only. + field = ".".join(str(part) for part in error.absolute_path) or "query" + raise OpenOutFindError(ErrorType.BAD_CONFIG, + f"Invalid CompanyEnrich {field}; see docs/companyenrich.md") + for key in ("atCurrentCompanyAfter", "atCurrentCompanyBefore", + "atCurrentPositionAfter", "atCurrentPositionBefore"): + value = query.get(key) + if value is not None: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + valid = "T" in value and parsed.utcoffset() == timedelta(0) + except ValueError: + valid = False + if not valid: + raise OpenOutFindError(ErrorType.BAD_CONFIG, + f"CompanyEnrich {key} needs an ISO 8601 UTC timestamp") + if not any(value for key, value in query.items() if key not in ("page", "pageSize")): + raise OpenOutFindError(ErrorType.BAD_CONFIG, "CompanyEnrich requires a targeting filter") + return query + + +def search(api_key: str, query: dict) -> dict: + """Exactly one HTTP request; errors never become empty results or automatic retries.""" + query = validate_query(query) + if not api_key: + raise ProviderUnavailable("Set OPENOUTFIND_COMPANYENRICH_API_KEY", ErrorType.NO_CREDENTIAL) + try: + with requests.Session() as session: + # requests' default adapter has retries disabled. A dedicated session also + # avoids inheriting the BetterContact polling/retry transport. + with session.post(API_URL, json=query, + headers={"Authorization": f"Bearer {api_key}", + "Accept": "application/json"}, + timeout=(5, 30), allow_redirects=False, stream=True) as response: + if response.status_code != 200: + error_type = {401: ErrorType.PROVIDER_AUTH, 403: ErrorType.PROVIDER_AUTH, + 402: ErrorType.PROVIDER_OUT_OF_CREDITS, + 429: ErrorType.PROVIDER_RATE_LIMITED}.get( + response.status_code, ErrorType.PROVIDER_UNAVAILABLE) + raise ProviderUnavailable( + f"CompanyEnrich HTTP {response.status_code}; not retried", error_type) + body = bytearray() + for chunk in response.iter_content(65536): + body.extend(chunk) + if len(body) > MAX_RESPONSE_BYTES: + raise ProviderUnavailable("CompanyEnrich response exceeded 4 MiB; not retried") + except requests.RequestException: + raise ProviderUnavailable("CompanyEnrich request failed; outcome unknown, not retried") from None + try: + data = json.loads(body) + except (ValueError, UnicodeDecodeError): + raise ProviderUnavailable("Invalid CompanyEnrich JSON; not retried") from None + if (not isinstance(data, dict) or not isinstance(data.get("items"), list) + or len(data["items"]) > query["pageSize"] + or any(not isinstance(item, dict) or type(item.get("id")) is not int + for item in data["items"])): + raise ProviderUnavailable("Invalid CompanyEnrich people page; not retried") + return data + + +def _object(value) -> dict: + return value if isinstance(value, dict) else {} + + +def _text(value, limit: int = 200) -> str: + return value[:limit].strip() if isinstance(value, str) else "" + + +def rows_for(data: dict) -> list[dict]: + """Adapt current professional profiles to the existing local qualification fields. + + Current employment is never inferred from the first historical job. Missing emails, + employee counts, headlines and names are not invented. No URL is fetched. + """ + rows = [] + for person in data["items"]: + url = _text(_object(person.get("socials")).get("linkedin_url"), 2048) + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError: + continue + if (len(url) > 200 or parsed.scheme != "https" + or parsed.hostname not in ("linkedin.com", "www.linkedin.com") + or parsed.username or port is not None + or not parsed.path.startswith("/in/") or not parsed.path[4:].strip("/")): + continue + url = "https://www.linkedin.com" + parsed.path.rstrip("/") + experiences = person.get("experiences") + current = next((job for job in experiences if isinstance(job, dict) + and job.get("isCurrent") is True), {}) if isinstance(experiences, list) else {} + company = _object(current.get("company")) if current.get("type") == "company" else {} + location = _object(person.get("location")) + country = _object(location.get("country")) + code = _text(country.get("code")).upper() + row = { + "contact_linkedin_profile_url": url, + "contact_full_name": _text(person.get("name")), + "contact_job_title": _text(person.get("position")), + "contact_seniority": _text(person.get("seniority")), + "contact_location_country": _text(country.get("name")), + "company_name": _text(company.get("name") or (current.get("companyName") + if current.get("type") == "unknown" else "")), + "company_domain": _text(company.get("domain")), + "company_industry": _text(company.get("industry")), + "_discovery_source": "companyenrich", + "_country_code": code if len(code) == 2 and code.isascii() and code.isalpha() else "", + "_first_name": _text(person.get("first_name"), 100) or None, + "_last_name": _text(person.get("last_name"), 100) or None, + } + # People search has no headline field. The BetterContact-specific headline + # completeness heuristic does not apply; a position is useful qualification text. + if row["contact_job_title"]: + rows.append(row) + return rows diff --git a/openoutfind/companyenrich_query.schema.json b/openoutfind/companyenrich_query.schema.json new file mode 100644 index 0000000..3bfc4fc --- /dev/null +++ b/openoutfind/companyenrich_query.schema.json @@ -0,0 +1,747 @@ +{ + "type": "object", + "properties": { + "page": { + "maximum": 1000000, + "minimum": 1, + "type": "integer" + }, + "pageSize": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, + "query": { + "anyOf": [ + { + "maxLength": 250, + "type": "string" + }, + { + "type": "null" + } + ] + }, + "positionQuery": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "atCurrentCompanyAfter": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "atCurrentCompanyBefore": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "atCurrentPositionAfter": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "atCurrentPositionBefore": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "exclude": { + "anyOf": [ + { + "type": "object", + "properties": { + "positionQuery": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "countries": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2}$" + } + }, + { + "type": "null" + } + ] + }, + "domains": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 2500 + }, + { + "type": "null" + } + ] + }, + "seniority": { + "anyOf": [ + { + "type": "array", + "items": { + "enum": [ + "owner", + "founder", + "c-suite", + "partner", + "vp", + "head", + "director", + "manager", + "senior", + "entry", + "intern" + ] + } + }, + { + "type": "null" + } + ] + }, + "department": { + "anyOf": [ + { + "type": "array", + "items": { + "enum": [ + "c-suite", + "c-suite/executive", + "c-suite/finance-executive", + "c-suite/founder", + "c-suite/human-resources-executive", + "c-suite/information-technology-executive", + "c-suite/legal-executive", + "c-suite/marketing-executive", + "c-suite/medical-health-executive", + "c-suite/operations-executive", + "c-suite/sales-executive", + "product-management", + "product-management/product-development", + "product-management/product-management", + "engineering-technical", + "engineering-technical/artificial-intelligence-machine-learning", + "engineering-technical/bioengineering", + "engineering-technical/biometrics", + "engineering-technical/business-intelligence", + "engineering-technical/chemical-engineering", + "engineering-technical/cloud-mobility", + "engineering-technical/data-science", + "engineering-technical/devops", + "engineering-technical/digital-transformation", + "engineering-technical/emerging-technology-innovation", + "engineering-technical/engineering-technical", + "engineering-technical/industrial-engineering", + "engineering-technical/mechanic", + "engineering-technical/mobile-development", + "engineering-technical/project-management", + "engineering-technical/research-development", + "engineering-technical/scrum-master-agile-coach", + "engineering-technical/software-development", + "engineering-technical/support-technical-services", + "engineering-technical/technician", + "engineering-technical/technology-operations", + "engineering-technical/test-quality-assurance", + "engineering-technical/ui-ux", + "engineering-technical/web-development", + "design", + "design/all-design", + "design/product-ui-ux-design", + "design/graphic-design", + "education", + "education/teacher", + "education/principal", + "education/superintendent", + "education/professor", + "finance", + "finance/accounting", + "finance/finance", + "finance/financial-planning-analysis", + "finance/financial-reporting", + "finance/financial-strategy", + "finance/financial-systems", + "finance/internal-audit-control", + "finance/investor-relations", + "finance/mergers-acquisitions", + "finance/real-estate-finance", + "finance/financial-risk", + "finance/shared-services", + "finance/sourcing-procurement", + "finance/tax", + "finance/treasury", + "human-resources", + "human-resources/compensation-benefits", + "human-resources/culture-diversity-inclusion", + "human-resources/employee-labor-relations", + "human-resources/health-safety", + "human-resources/human-resource-information-system", + "human-resources/human-resources", + "human-resources/hr-business-partner", + "human-resources/learning-development", + "human-resources/organizational-development", + "human-resources/recruiting-talent-acquisition", + "human-resources/talent-management", + "human-resources/workforce-management", + "human-resources/people-operations", + "information-technology", + "information-technology/application-development", + "information-technology/business-service-management-itsm", + "information-technology/collaboration-web-app", + "information-technology/data-center", + "information-technology/data-warehouse", + "information-technology/database-administration", + "information-technology/ecommerce-development", + "information-technology/enterprise-architecture", + "information-technology/help-desk-desktop-services", + "information-technology/hr-financial-erp-systems", + "information-technology/information-security", + "information-technology/information-technology", + "information-technology/infrastructure", + "information-technology/it-asset-management", + "information-technology/it-audit-it-compliance", + "information-technology/it-operations", + "information-technology/it-procurement", + "information-technology/it-strategy", + "information-technology/it-training", + "information-technology/networking", + "information-technology/project-program-management", + "information-technology/quality-assurance", + "information-technology/retail-store-systems", + "information-technology/servers", + "information-technology/storage-disaster-recovery", + "information-technology/telecommunications", + "information-technology/virtualization", + "legal", + "legal/acquisitions", + "legal/compliance", + "legal/contracts", + "legal/corporate-secretary", + "legal/ediscovery", + "legal/ethics", + "legal/governance", + "legal/governmental-affairs-regulatory-law", + "legal/intellectual-property-patent", + "legal/labor-employment", + "legal/lawyer-attorney", + "legal/legal", + "legal/legal-counsel", + "legal/legal-operations", + "legal/litigation", + "legal/privacy", + "marketing", + "marketing/advertising", + "marketing/brand-management", + "marketing/content-marketing", + "marketing/customer-experience", + "marketing/customer-marketing", + "marketing/demand-generation", + "marketing/digital-marketing", + "marketing/ecommerce-marketing", + "marketing/event-marketing", + "marketing/field-marketing", + "marketing/lead-generation", + "marketing/marketing", + "marketing/marketing-analytics-insights", + "marketing/marketing-communications", + "marketing/marketing-operations", + "marketing/product-marketing", + "marketing/public-relations", + "marketing/search-engine-optimization-pay-per-click", + "marketing/social-media-marketing", + "marketing/strategic-communications", + "marketing/technical-marketing", + "medical-health", + "medical-health/anesthesiology", + "medical-health/chiropractics", + "medical-health/clinical-systems", + "medical-health/dentistry", + "medical-health/dermatology", + "medical-health/doctors-physicians", + "medical-health/epidemiology", + "medical-health/first-responder", + "medical-health/infectious-disease", + "medical-health/medical-administration", + "medical-health/medical-education-training", + "medical-health/medical-research", + "medical-health/medicine", + "medical-health/neurology", + "medical-health/nursing", + "medical-health/nutrition-dietetics", + "medical-health/obstetrics-gynecology", + "medical-health/oncology", + "medical-health/opthalmology", + "medical-health/optometry", + "medical-health/orthopedics", + "medical-health/pathology", + "medical-health/pediatrics", + "medical-health/pharmacy", + "medical-health/physical-therapy", + "medical-health/psychiatry", + "medical-health/psychology", + "medical-health/public-health", + "medical-health/radiology", + "medical-health/social-work", + "operations", + "operations/call-center", + "operations/construction", + "operations/corporate-strategy", + "operations/customer-service-support", + "operations/enterprise-resource-planning", + "operations/facilities-management", + "operations/leasing", + "operations/logistics", + "operations/office-operations", + "operations/operations", + "operations/physical-security", + "operations/project-development", + "operations/quality-management", + "operations/real-estate", + "operations/safety", + "operations/store-operations", + "operations/supply-chain", + "sales", + "sales/account-management", + "sales/business-development", + "sales/channel-sales", + "sales/customer-retention-development", + "sales/customer-success", + "sales/field-outside-sales", + "sales/inside-sales", + "sales/partnerships", + "sales/revenue-operations", + "sales/sales", + "sales/sales-enablement", + "sales/sales-engineering", + "sales/sales-operations", + "sales/sales-training", + "consulting", + "consulting/consultant" + ] + } + }, + { + "type": "null" + } + ] + }, + "education": { + "anyOf": [ + { + "type": "object", + "properties": { + "institutionLinkedinHandles": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "institutionNameQuery": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "countries": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2}$" + } + }, + { + "type": "null" + } + ] + }, + "domains": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 2500 + }, + { + "type": "null" + } + ] + }, + "seniority": { + "anyOf": [ + { + "type": "array", + "items": { + "enum": [ + "owner", + "founder", + "c-suite", + "partner", + "vp", + "head", + "director", + "manager", + "senior", + "entry", + "intern" + ] + } + }, + { + "type": "null" + } + ] + }, + "department": { + "anyOf": [ + { + "type": "array", + "items": { + "enum": [ + "c-suite", + "c-suite/executive", + "c-suite/finance-executive", + "c-suite/founder", + "c-suite/human-resources-executive", + "c-suite/information-technology-executive", + "c-suite/legal-executive", + "c-suite/marketing-executive", + "c-suite/medical-health-executive", + "c-suite/operations-executive", + "c-suite/sales-executive", + "product-management", + "product-management/product-development", + "product-management/product-management", + "engineering-technical", + "engineering-technical/artificial-intelligence-machine-learning", + "engineering-technical/bioengineering", + "engineering-technical/biometrics", + "engineering-technical/business-intelligence", + "engineering-technical/chemical-engineering", + "engineering-technical/cloud-mobility", + "engineering-technical/data-science", + "engineering-technical/devops", + "engineering-technical/digital-transformation", + "engineering-technical/emerging-technology-innovation", + "engineering-technical/engineering-technical", + "engineering-technical/industrial-engineering", + "engineering-technical/mechanic", + "engineering-technical/mobile-development", + "engineering-technical/project-management", + "engineering-technical/research-development", + "engineering-technical/scrum-master-agile-coach", + "engineering-technical/software-development", + "engineering-technical/support-technical-services", + "engineering-technical/technician", + "engineering-technical/technology-operations", + "engineering-technical/test-quality-assurance", + "engineering-technical/ui-ux", + "engineering-technical/web-development", + "design", + "design/all-design", + "design/product-ui-ux-design", + "design/graphic-design", + "education", + "education/teacher", + "education/principal", + "education/superintendent", + "education/professor", + "finance", + "finance/accounting", + "finance/finance", + "finance/financial-planning-analysis", + "finance/financial-reporting", + "finance/financial-strategy", + "finance/financial-systems", + "finance/internal-audit-control", + "finance/investor-relations", + "finance/mergers-acquisitions", + "finance/real-estate-finance", + "finance/financial-risk", + "finance/shared-services", + "finance/sourcing-procurement", + "finance/tax", + "finance/treasury", + "human-resources", + "human-resources/compensation-benefits", + "human-resources/culture-diversity-inclusion", + "human-resources/employee-labor-relations", + "human-resources/health-safety", + "human-resources/human-resource-information-system", + "human-resources/human-resources", + "human-resources/hr-business-partner", + "human-resources/learning-development", + "human-resources/organizational-development", + "human-resources/recruiting-talent-acquisition", + "human-resources/talent-management", + "human-resources/workforce-management", + "human-resources/people-operations", + "information-technology", + "information-technology/application-development", + "information-technology/business-service-management-itsm", + "information-technology/collaboration-web-app", + "information-technology/data-center", + "information-technology/data-warehouse", + "information-technology/database-administration", + "information-technology/ecommerce-development", + "information-technology/enterprise-architecture", + "information-technology/help-desk-desktop-services", + "information-technology/hr-financial-erp-systems", + "information-technology/information-security", + "information-technology/information-technology", + "information-technology/infrastructure", + "information-technology/it-asset-management", + "information-technology/it-audit-it-compliance", + "information-technology/it-operations", + "information-technology/it-procurement", + "information-technology/it-strategy", + "information-technology/it-training", + "information-technology/networking", + "information-technology/project-program-management", + "information-technology/quality-assurance", + "information-technology/retail-store-systems", + "information-technology/servers", + "information-technology/storage-disaster-recovery", + "information-technology/telecommunications", + "information-technology/virtualization", + "legal", + "legal/acquisitions", + "legal/compliance", + "legal/contracts", + "legal/corporate-secretary", + "legal/ediscovery", + "legal/ethics", + "legal/governance", + "legal/governmental-affairs-regulatory-law", + "legal/intellectual-property-patent", + "legal/labor-employment", + "legal/lawyer-attorney", + "legal/legal", + "legal/legal-counsel", + "legal/legal-operations", + "legal/litigation", + "legal/privacy", + "marketing", + "marketing/advertising", + "marketing/brand-management", + "marketing/content-marketing", + "marketing/customer-experience", + "marketing/customer-marketing", + "marketing/demand-generation", + "marketing/digital-marketing", + "marketing/ecommerce-marketing", + "marketing/event-marketing", + "marketing/field-marketing", + "marketing/lead-generation", + "marketing/marketing", + "marketing/marketing-analytics-insights", + "marketing/marketing-communications", + "marketing/marketing-operations", + "marketing/product-marketing", + "marketing/public-relations", + "marketing/search-engine-optimization-pay-per-click", + "marketing/social-media-marketing", + "marketing/strategic-communications", + "marketing/technical-marketing", + "medical-health", + "medical-health/anesthesiology", + "medical-health/chiropractics", + "medical-health/clinical-systems", + "medical-health/dentistry", + "medical-health/dermatology", + "medical-health/doctors-physicians", + "medical-health/epidemiology", + "medical-health/first-responder", + "medical-health/infectious-disease", + "medical-health/medical-administration", + "medical-health/medical-education-training", + "medical-health/medical-research", + "medical-health/medicine", + "medical-health/neurology", + "medical-health/nursing", + "medical-health/nutrition-dietetics", + "medical-health/obstetrics-gynecology", + "medical-health/oncology", + "medical-health/opthalmology", + "medical-health/optometry", + "medical-health/orthopedics", + "medical-health/pathology", + "medical-health/pediatrics", + "medical-health/pharmacy", + "medical-health/physical-therapy", + "medical-health/psychiatry", + "medical-health/psychology", + "medical-health/public-health", + "medical-health/radiology", + "medical-health/social-work", + "operations", + "operations/call-center", + "operations/construction", + "operations/corporate-strategy", + "operations/customer-service-support", + "operations/enterprise-resource-planning", + "operations/facilities-management", + "operations/leasing", + "operations/logistics", + "operations/office-operations", + "operations/operations", + "operations/physical-security", + "operations/project-development", + "operations/quality-management", + "operations/real-estate", + "operations/safety", + "operations/store-operations", + "operations/supply-chain", + "sales", + "sales/account-management", + "sales/business-development", + "sales/channel-sales", + "sales/customer-retention-development", + "sales/customer-success", + "sales/field-outside-sales", + "sales/inside-sales", + "sales/partnerships", + "sales/revenue-operations", + "sales/sales", + "sales/sales-enablement", + "sales/sales-engineering", + "sales/sales-operations", + "sales/sales-training", + "consulting", + "consulting/consultant" + ] + } + }, + { + "type": "null" + } + ] + }, + "education": { + "anyOf": [ + { + "type": "object", + "properties": { + "institutionLinkedinHandles": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "institutionNameQuery": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CompanyEnrich bounded people search (OpenAPI snapshot 2026-11-02)", + "$comment": "https://api.companyenrich.com/openapi/v1.json; companyFilter omitted because compound lookup has a separate spend contract. Unknown keys fail closed." +} diff --git a/openoutfind/contacts/service.py b/openoutfind/contacts/service.py index 0db75b1..0b429ae 100644 --- a/openoutfind/contacts/service.py +++ b/openoutfind/contacts/service.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging +from urllib.parse import urlsplit import requests @@ -71,9 +72,44 @@ def token_in_hand(config: SiteConfig, *, mint: bool = True) -> str: return _minted_token or "" +def _profile_key(value: str) -> str: + """Match provider URL variants without fetching a profile.""" + try: + parsed = urlsplit(value.strip()) + if parsed.hostname in ("linkedin.com", "www.linkedin.com"): + return "https://www.linkedin.com" + parsed.path.rstrip("/") + except ValueError: + pass + return value + + +def _local_profile(lead) -> bool: + from openoutfind.crm.models import Lead + + if getattr(lead, "discovery_source", "") == "companyenrich": + return True + # Another source may have stored the same identity with a trailing slash or + # alternate host. Its independently fetched page must not undo the local choice. + return Lead.objects.filter(discovery_source="companyenrich", + profile_url=_profile_key(lead.profile_url)).exists() + + +def _shareable_rows(rows: list[dict]) -> list[dict]: + from openoutfind.crm.models import Lead + + rows = [row for row in rows if row.get("_discovery_source") != "companyenrich"] + keys = {_profile_key(str(row.get("contact_linkedin_profile_url") or "")) for row in rows} + private = set(Lead.objects.filter(discovery_source="companyenrich", profile_url__in=keys) + .values_list("profile_url", flat=True)) + return [row for row in rows + if _profile_key(str(row.get("contact_linkedin_profile_url") or "")) not in private] + + def resolve(lead) -> str | None: """A stored email for *lead*, or ``None`` — a miss, no token yet, or an outage all return ``None``, so the caller falls back to BetterContact.""" + if _local_profile(lead): + return None config = SiteConfig.load() token = token_in_hand(config) if not token: @@ -123,6 +159,8 @@ def contribute(lead, emails: list[str], origin: str) -> None: with their discovery page (``share_profiles``). An operator who declares none contributes; the server re-gates every record authoritatively. """ + if _local_profile(lead): + return operator_country = SiteConfig.load().operator_country_code if operator_country and is_eea_located(operator_country): logger.debug("hub: operator in EEA/UK/CH — skipping give-back for %s", lead.profile_url) @@ -169,7 +207,7 @@ def share_profiles(rows: list[dict], country_code: str) -> None: The same gates as ``contribute``: an operator in the EEA/UK/CH sends nothing, and a page searched in one — or in no named country — is not sent. The hub gates again. """ - rows = [row for row in rows if row.get("contact_linkedin_profile_url")] + rows = [row for row in _shareable_rows(rows) if row.get("contact_linkedin_profile_url")] token = _page_token(country_code) if rows else "" if not token: return @@ -188,6 +226,8 @@ def share_companies(rows: list[dict], country_code: str) -> None: """ from openoutfind.discovery import company_records + rows = _shareable_rows(rows) + records = company_records(rows) token = _page_token(country_code) if records else "" if not token: diff --git a/openoutfind/core/config.py b/openoutfind/core/config.py index fe1d9d8..3c1c555 100644 --- a/openoutfind/core/config.py +++ b/openoutfind/core/config.py @@ -77,6 +77,9 @@ class SiteConfig: # paid email lookup behind `--emails`. bettercontact_api_key: str = "" + # Inert unless find explicitly selects a bounded CompanyEnrich page and budget. + companyenrich_api_key: str = "" + # The operator's own ISO-3166 alpha-2 jurisdiction — not to be confused with # `Lead.country_code`, the per-lead target country an ICP search surfaced someone # under. Optional: an EEA/UK/CH country turns the contacts-store give-back off diff --git a/openoutfind/core/db/leads.py b/openoutfind/core/db/leads.py index 09b2781..2871214 100644 --- a/openoutfind/core/db/leads.py +++ b/openoutfind/core/db/leads.py @@ -32,7 +32,8 @@ def promote_lead_to_deal(profile_url: str, reason: str = ""): return deal -def store_rows(rows: list[dict], country_code: str = "") -> tuple[list, int]: +def store_rows(rows: list[dict], country_code: str = "", *, + discovery_source: str = "bettercontact") -> tuple[list, int]: """Persist Lead Finder rows as embedded Leads; return every row's Lead and how many were new. @@ -58,6 +59,12 @@ def store_rows(rows: list[dict], country_code: str = "") -> tuple[list, int]: by_url = {row["contact_linkedin_profile_url"]: row for row in rows if row.get("contact_linkedin_profile_url")} known = {lead.profile_url: lead for lead in Lead.objects.filter(profile_url__in=by_url)} + if discovery_source == "companyenrich": + # Restrict combined provenance too; never undo this on a later free-source hit. + Lead.objects.filter(pk__in=[lead.pk for lead in known.values()]).update( + discovery_source="companyenrich") + for lead in known.values(): + lead.discovery_source = "companyenrich" fresh = [url for url in by_url if url not in known] texts = [profile_text_for(by_url[url]) for url in fresh] @@ -69,7 +76,10 @@ def store_rows(rows: list[dict], country_code: str = "") -> tuple[list, int]: embedding=np.asarray(vector, dtype=np.float32).tobytes(), profile_text=text, source_fields=source_fields_for(row), - country_code=country_code, + country_code=row.get("_country_code", country_code), + discovery_source=discovery_source, + first_name=row.get("_first_name"), + last_name=row.get("_last_name"), company=company_for(row), **person_for(row), ) diff --git a/openoutfind/core/discovery_source.py b/openoutfind/core/discovery_source.py new file mode 100644 index 0000000..4238a7f --- /dev/null +++ b/openoutfind/core/discovery_source.py @@ -0,0 +1,30 @@ +"""Per-invocation discovery selection. No configured key opts a run into spending.""" +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Protocol + + +class Source(Protocol): + required_field: str + + def discover(self, site_config) -> bool: ... + + def report(self) -> dict: ... + + +_current: ContextVar[Source | None] = ContextVar("discovery_source", default=None) + + +def current() -> Source | None: + return _current.get() + + +@contextmanager +def selected(source: Source | None): + token = _current.set(source) + try: + yield + finally: + _current.reset(token) diff --git a/openoutfind/core/export.py b/openoutfind/core/export.py index b9926e6..c9f70cb 100644 --- a/openoutfind/core/export.py +++ b/openoutfind/core/export.py @@ -61,7 +61,7 @@ # flag would have been the obvious alternative and is wrong: invocation-relative state # written into a file that outlives the invocation is a lie the second time it is read. # Ours: full_name — the one name discovery reports, verbatim. ``first_name``/``last_name`` -# stay null until the paid lookup returns the provider's own split, so without this a +# are supplied by CompanyEnrich discovery or a paid lookup, never guessed, so without this a # leads-only file (no addresses bought) names nobody. Last, because the contract only # ever adds: a reader indexing the older columns by position is not moved. RECORD_FIELDS = ( @@ -119,7 +119,7 @@ def lead_record(deal) -> dict: company = None return { "email": lead.email, - # From the enrichment provider's own response, never split in-house. Null for a + # From a discovery/enrichment provider response, never split in-house. Null for a # lead resolved through the free hub cache, which never calls BetterContact. "first_name": lead.first_name, "last_name": lead.last_name, diff --git a/openoutfind/core/job.py b/openoutfind/core/job.py index 7a14956..4ddc40e 100644 --- a/openoutfind/core/job.py +++ b/openoutfind/core/job.py @@ -263,7 +263,12 @@ def _why_idle(site_config, result: JobResult, buy_addresses: bool) -> str: a reason to run again in an hour. """ from openoutfind.core.cycle import pipeline_summary + from openoutfind.core.discovery_source import current + if current() is not None: + return (f"{result.produced} of {result.goal} — the authorized CompanyEnrich page " + "and available pool cannot advance. No additional page was purchased. " + "Inspect the results before authorizing another request ID/page.") return ( f"{result.produced} of {result.goal} — nothing left to do right now. " f"{pipeline_summary(site_config, buy_addresses=buy_addresses)}" diff --git a/openoutfind/core/management/commands/find.py b/openoutfind/core/management/commands/find.py index 23a0264..cee59f4 100644 --- a/openoutfind/core/management/commands/find.py +++ b/openoutfind/core/management/commands/find.py @@ -11,8 +11,8 @@ outfind find 10 --exclude F # ...never judging or printing the URLs listed in F outfind find 10 --labels F # ...a fresh store starting from a customer's marks -**Finding is free; buying an address is not, and the free thing is the default.** -Discovery and qualification cost only the operator's own LLM key, so a bare ``find 10`` +**Default discovery is free; paid source selection and addresses are separately opt-in.** +Default BetterContact discovery and qualification cost only the operator's own LLM key, so a bare ``find 10`` cannot spend a credit however many deals have queued up past the confidence gate. The address lookup is opt-in — ``--emails`` permits it, and the ``emails`` unit implies it. @@ -113,7 +113,7 @@ def add_arguments(self, parser): parser.add_argument("--emails", action="store_true", dest="buy_emails", help="Also buy an address for any lead that has cleared the " "confidence gate — one credit per verified hit. Without " - "this the run cannot spend. Implied by the `emails` unit.") + "this no email credits are spent. Implied by the `emails` unit.") parser.add_argument("--json", action="store_true", dest="as_json", help="Emit the rows as JSON Lines, one record per line, " "profile text included. The run's own metadata goes to " @@ -169,7 +169,21 @@ def add_arguments(self, parser): "lead is never judged, printed or looked up. Marks already in the " "store are skipped, so the flag can ride every call.") + parser.add_argument("--companyenrich-query", metavar="PATH", + help="Explicitly use one paid CompanyEnrich people-search page from JSON.") + parser.add_argument("--companyenrich-request-id", metavar="ID", + help="Stable request ID; reuse it when resuming to avoid buying a page twice.") + parser.add_argument("--discovery-credit-budget", type=int, metavar="CREDITS", + help="Maximum CompanyEnrich discovery credits authorized; requires its query and ID.") + def handle(self, *args, **options): + from openoutfind.core.discovery_source import selected + from openoutfind.core.pipeline.companyenrich import from_options + + with selected(from_options(options)): + return self._handle(*args, **options) + + def _handle(self, *args, **options): if options["count"] < 0: raise OpenOutFindError(ErrorType.BAD_CONFIG, "count cannot be negative") if options["verdict"] and not options["agent_qualify"]: @@ -338,6 +352,10 @@ def _write_run_object(result: JobResult, action, rows: int, labels: dict | None) } if labels is not None: run["labels"] = labels + from openoutfind.core.discovery_source import current + + if current() is not None: + run["discovery"] = current().report() sys.stderr.write(json.dumps(run) + "\n") @@ -364,6 +382,12 @@ def _announce_the_run(site_config, goal: Goal, buy_addresses: bool) -> None: else "finding only, no addresses bought") work = f"goal: {goal}" if goal.count else "no work — printing what is already there" logger.info("%s · %s", work, spending) + from openoutfind.core.discovery_source import current + + if current() is not None: + details = current().report() + logger.info("CompanyEnrich selected: one page, up to %d discovery credits; no hub sharing", + details["maximum_credits"]) log_icp_echo() log_brief_echo() diff --git a/openoutfind/core/migrations/0009_companyenrichpage.py b/openoutfind/core/migrations/0009_companyenrichpage.py new file mode 100644 index 0000000..1372992 --- /dev/null +++ b/openoutfind/core/migrations/0009_companyenrichpage.py @@ -0,0 +1,25 @@ +# Generated by Django 6.1.2 on 2026-11-02 20:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('outfind_core', '0008_the_walk_searches_best_first'), + ] + + operations = [ + migrations.CreateModel( + name='CompanyEnrichPage', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('request_id', models.CharField(max_length=64, unique=True)), + ('query', models.JSONField()), + ('maximum_credits', models.PositiveIntegerField()), + ('response', models.JSONField(blank=True, null=True)), + ('imported', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/openoutfind/core/models.py b/openoutfind/core/models.py index 7bd2549..b7d44ab 100644 --- a/openoutfind/core/models.py +++ b/openoutfind/core/models.py @@ -205,3 +205,19 @@ class Brief(models.Model): def __str__(self): return f"brief by {self.written_by}, {self.updated_at:%Y-%m-%d %H:%M}" + + +class CompanyEnrichPage(models.Model): + """One explicitly authorized paid request, retained to prevent accidental replay. + + A reservation is committed before HTTP. A response is saved before embedding, so + local work can resume without buying the page again. An ambiguous request is never + repeated under the same operator-supplied request ID. + """ + + request_id = models.CharField(max_length=64, unique=True) + query = models.JSONField() + maximum_credits = models.PositiveIntegerField() + response = models.JSONField(null=True, blank=True) + imported = models.BooleanField(default=False) + created_at = models.DateTimeField(auto_now_add=True) diff --git a/openoutfind/core/pipeline/companyenrich.py b/openoutfind/core/pipeline/companyenrich.py new file mode 100644 index 0000000..af5e1c9 --- /dev/null +++ b/openoutfind/core/pipeline/companyenrich.py @@ -0,0 +1,107 @@ +"""A single paid source page, separately authorized from the free discovery walk.""" +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field + +from django.db import transaction + +from openoutfind import companyenrich +from openoutfind.core.errors import ErrorType, OpenOutFindError +from openoutfind.enrichment.provider import ProviderUnavailable + +logger = logging.getLogger(__name__) + + +@dataclass +class CompanyEnrichSource: + query: dict + request_id: str + credit_budget: int + required_field: str = field(default="companyenrich_api_key", init=False) + _done: bool = field(default=False, init=False) + _used: bool = field(default=False, init=False) + + def __post_init__(self): + self.query = companyenrich.validate_query(self.query) + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", self.request_id): + raise OpenOutFindError(ErrorType.BAD_CONFIG, + "--companyenrich-request-id needs 1–64 letters, digits, _, . or -") + if type(self.credit_budget) is not int or self.credit_budget < self.maximum_credits: + raise OpenOutFindError(ErrorType.BAD_CONFIG, + f"--discovery-credit-budget must be at least {self.maximum_credits} " + "(2 CompanyEnrich credits per requested profile)") + + @property + def maximum_credits(self) -> int: + return companyenrich.CREDITS_PER_PERSON * self.query["pageSize"] + + def report(self) -> dict: + return {"source": "companyenrich", "request_id": self.request_id, + "maximum_credits": self.maximum_credits, "credit_budget": self.credit_budget, + "page": self.query["page"], "page_size": self.query["pageSize"], + "used": self._used, "hub_sharing": False} + + def discover(self, site_config) -> bool: + from openoutfind.core.company_cap import full_company_keys + from openoutfind.core.db.leads import store_rows + from openoutfind.core.models import CompanyEnrichPage + from openoutfind.discovery import company_key_for + + if self._done: + return False + if not site_config.companyenrich_api_key: + raise ProviderUnavailable("Set OPENOUTFIND_COMPANYENRICH_API_KEY", ErrorType.NO_CREDENTIAL) + # get_or_create's unique request ID arbitrates concurrent invocations. The + # reservation is committed before HTTP; only its creator may submit the request. + if transaction.get_connection().in_atomic_block: + raise RuntimeError("CompanyEnrich discovery requires an autocommit boundary") + page, created = CompanyEnrichPage.objects.get_or_create( + request_id=self.request_id, + defaults={"query": self.query, "maximum_credits": self.maximum_credits}) + if page.query != self.query: + raise ProviderUnavailable("CompanyEnrich request ID already belongs to a different query", + ErrorType.BAD_CONFIG) + if created: + logger.info("CompanyEnrich: one page, at most %d credits; provider data stays local", + self.maximum_credits) + data = companyenrich.search(site_config.companyenrich_api_key, self.query) + page.response = data + page.save(update_fields=["response"]) + elif page.response is None: + raise ProviderUnavailable( + "CompanyEnrich request was already submitted or its outcome is unknown; " + "not retried. Inspect provider usage before explicitly authorizing a new request ID.") + self._used = True + self._done = True + if page.imported: + return False + full = full_company_keys() + rows = [row for row in companyenrich.rows_for(page.response) + if company_key_for(row) not in full] + # No share_profiles/share_companies. Source provenance persists even for a + # previously known URL, so a later email resolver cannot give that identity back. + with transaction.atomic(): + page = CompanyEnrichPage.objects.select_for_update().get(pk=page.pk) + if page.imported: + return False + leads, created_count = store_rows(rows, discovery_source="companyenrich") + page.imported = True + page.save(update_fields=["imported"]) + logger.info("CompanyEnrich: %d profiles returned, %d usable, %d new; no emails supplied", + len(page.response["items"]), len(leads), created_count) + return True + + +def from_options(options) -> CompanyEnrichSource | None: + path = options.get("companyenrich_query") + request_id = options.get("companyenrich_request_id") + budget = options.get("discovery_credit_budget") + if not path and not request_id and budget is None: + return None + if not path or not request_id or budget is None: + raise OpenOutFindError(ErrorType.BAD_CONFIG, + "CompanyEnrich requires --companyenrich-query, " + "--companyenrich-request-id and --discovery-credit-budget together") + return CompanyEnrichSource(companyenrich.read_query(path), request_id, budget) diff --git a/openoutfind/core/pipeline/discover.py b/openoutfind/core/pipeline/discover.py index 8b88d8d..0b5798f 100644 --- a/openoutfind/core/pipeline/discover.py +++ b/openoutfind/core/pipeline/discover.py @@ -87,6 +87,12 @@ def discover(site_config, qualifier=None, floor: float = 0.0) -> bool: Two gates: no finder key means the index can't be reached, and no product or target means there is nothing to write the initial population from. """ + from openoutfind.core.discovery_source import current + + source = current() + if source is not None: + return source.discover(site_config) + from openoutfind.core.pipeline.walk import run_round from openoutfind.enrichment import bettercontact diff --git a/openoutfind/core/pipeline/judge_or_search.py b/openoutfind/core/pipeline/judge_or_search.py index 9a92006..e9b411e 100644 --- a/openoutfind/core/pipeline/judge_or_search.py +++ b/openoutfind/core/pipeline/judge_or_search.py @@ -70,7 +70,13 @@ def decide(qualifier, candidates) -> Decision: from openoutfind.core.models import Page from openoutfind.core.pipeline import frontier + from openoutfind.core.discovery_source import current + floor = best_waiting(qualifier, candidates, frontier.fit_rate_map()) + if current() is not None: + # The selected paid source is one bounded page, not a free frontier arm. + # Exhaust useful local qualification work before asking it for anything. + return Decision(True, floor) if not Page.objects.exists(): return Decision(True, floor) fits, verdicts = meter.totals(meter.POOL_FIT) diff --git a/openoutfind/core/readiness.py b/openoutfind/core/readiness.py index 83069ea..0703821 100644 --- a/openoutfind/core/readiness.py +++ b/openoutfind/core/readiness.py @@ -80,14 +80,19 @@ def missing_variables() -> dict[str, list[str]]: The groups are the questions a person is asked, kept because that is how the failure reads to somebody who has to go and find the values. An empty dict is a ready run. """ + from openoutfind.core.discovery_source import current + config = SiteConfig.load() + source = current() + discovery_group = "companyenrich" if source is not None else "bettercontact" + discovery_fields = (source.required_field,) if source is not None else REQUIRED_DISCOVERY_FIELDS groups = { "campaign": missing(config, REQUIRED_ICP_FIELDS), # `--agent-qualify` never calls AI_MODEL for a verdict — the whole point is a # calling agent answering instead of a second, separately-keyed model — so an # install running only that way is not asked for a key it will never spend. "llm": [] if _agent_qualify_active() else missing(config, REQUIRED_LLM_FIELDS), - "bettercontact": missing(config, REQUIRED_DISCOVERY_FIELDS), + discovery_group: missing(config, discovery_fields), } return {group: names for group, names in groups.items() if names} diff --git a/openoutfind/core/status.py b/openoutfind/core/status.py index 490547e..2177736 100644 --- a/openoutfind/core/status.py +++ b/openoutfind/core/status.py @@ -55,10 +55,13 @@ def _config_state() -> dict: """Which groups this run was given, and the variables that would give it the rest.""" from openoutfind.core.readiness import GROUPS, missing_variables + from openoutfind.core.discovery_source import current + missing = missing_variables() + groups = ("campaign", "llm", "companyenrich") if current() else GROUPS return { "complete": not missing, - "done": [group for group in GROUPS if group not in missing], + "done": [group for group in groups if group not in missing], "missing": missing, } @@ -169,7 +172,9 @@ def _blocked(config: dict, credits: dict, totals: dict) -> list[dict]: if credits["error"] == ErrorType.NO_CREDENTIAL: blocked.append({ "type": ErrorType.NO_CREDENTIAL, - "message": "no BetterContact key — discovery and email finding are both off", + "message": ("no BetterContact key — email finding is off" + if "companyenrich" in config.get("done", []) else + "no BetterContact key — discovery and email finding are both off"), }) elif credits["error"] == ErrorType.PROVIDER_AUTH: blocked.append({ diff --git a/openoutfind/crm/migrations/0007_lead_discovery_source.py b/openoutfind/crm/migrations/0007_lead_discovery_source.py new file mode 100644 index 0000000..a276c4c --- /dev/null +++ b/openoutfind/crm/migrations/0007_lead_discovery_source.py @@ -0,0 +1,18 @@ +# Generated by Django 6.1.2 on 2026-11-02 20:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('outfind_crm', '0006_remove_lead_discovered_by'), + ] + + operations = [ + migrations.AddField( + model_name='lead', + name='discovery_source', + field=models.CharField(default='bettercontact', max_length=32), + ), + ] diff --git a/openoutfind/crm/models/lead.py b/openoutfind/crm/models/lead.py index a8c20cc..13e3f3e 100644 --- a/openoutfind/crm/models/lead.py +++ b/openoutfind/crm/models/lead.py @@ -34,6 +34,9 @@ class Meta: # walk's moves can draw terms from them. See ``discovery.SOURCE_FIELDS``. A lead # stored without one of them matches no search term on that field. source_fields = models.JSONField(default=dict, blank=True) + # CompanyEnrich-sourced identities must never reach the shared contacts hub, + # including a later email lookup or a duplicate seen by another source. + discovery_source = models.CharField(max_length=32, default="bettercontact") # Who this person is and where they work — the record the product accumulates, and # what the lead export hands to whatever the operator sends with. Deliberately # absent from ``profile_text`` and the embedding: a name carries no ICP signal, and diff --git a/pyproject.toml b/pyproject.toml index 5f317f5..13877f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ # empty `griffelib` meta-package and reorganized symbols. Pin until upstream fixes. "griffe>=1,<2", "jsonpath-ng>=1.6,<2", + "jsonschema>=4.23,<5", "pandas>=2.2,<3", "jinja2>=3.1,<4", "termcolor>=2.4,<4", diff --git a/skills/find-leads/SKILL.md b/skills/find-leads/SKILL.md index 18d3233..78fb059 100644 --- a/skills/find-leads/SKILL.md +++ b/skills/find-leads/SKILL.md @@ -84,8 +84,8 @@ Three things about `N` that are easy to get wrong: ### What costs money -Discovery and qualification are free (they cost only the user's own LLM key). **The address lookup -is the only paid step**, and it is opt-in: +Default BetterContact discovery and qualification cost only the user's own LLM key. +**Email lookup and optional CompanyEnrich discovery are separately opt-in paid steps**: - a bare `find N` **cannot** spend a credit, however many leads are queued past the confidence gate; - `--emails` permits buying for whatever is ready; @@ -96,6 +96,41 @@ is the only paid step**, and it is opt-in: "find me leads", run the free form and tell them the paid form exists. A lead with no address still exports — the row carries the person, the company and the reason with a blank `email`. +### Optional CompanyEnrich discovery (paid) + +CompanyEnrich supplies B2B professional profiles, without emails or phone numbers. +Never select it merely because `OPENOUTFIND_COMPANYENRICH_API_KEY` is present. Only +use it when the user authorizes paid discovery and its maximum credit reservation: + +```bash +outfind find 5 --companyenrich-query people.json \ + --companyenrich-request-id founders-page-1 --discovery-credit-budget 20 +``` + +All three flags are required. The JSON query uses the direct people-search filters: +domains/company-name query, titles, countries, seniority, department, UTC join/start +dates, education, exclusions, page and pageSize. Size defaults to 10, maximum 100; +reserve **2 credits per requested profile**, at the currently published rate. Ten +profiles cost at most 20 credits at that rate, regardless of how many qualify. This +makes at most one request; a larger budget never buys additional pages. No +`companyFilter`, compound search, expansions or automatic pagination. A bare `find N` +still uses the free BetterContact path. Discovery authorization never implies emails. + +Reuse the exact query and **same request ID** when answering `population_pending` or +`qualify_pending` or resuming local failures. The response is saved before embedding. +A timeout/HTTP error is not retried; an ambiguous reserved ID cannot be submitted +again. Check provider usage before authorizing a new ID for a retry, which may charge +again. A new ID/different database is a new purchase, not a resume. A page without +enough fits ends `goal_unreached`; do not silently add pages to meet the goal. + +CompanyEnrich data is excluded from the shared contacts hub, including later hub +lookups/email give-back. The existing qualifier still sends text to the user's LLM, +and CSV/JSON still goes to the user's chosen tools. API key configuration is required +instead of BetterContact for this source; email lookup still requires an email finder. +Standalone `check`/`status` describe the default source. `--json` run metadata adds +`discovery`; exported columns and `ErrorType` values stay the same. Full setup, +query example and provider pricing caveat: [docs/companyenrich.md](../../docs/companyenrich.md). + ### Other flags | Flag | What it does | diff --git a/tests/test_companyenrich.py b/tests/test_companyenrich.py new file mode 100644 index 0000000..1469ad4 --- /dev/null +++ b/tests/test_companyenrich.py @@ -0,0 +1,369 @@ +"""Synthetic paid-source fixtures: no provider, model or hub network calls.""" +import json +from unittest.mock import MagicMock + +import pytest +import requests + +from openoutfind import companyenrich as api +from openoutfind.core import discovery_source +from openoutfind.core.config import SiteConfig +from openoutfind.core.errors import OpenOutFindError +from openoutfind.core.models import CompanyEnrichPage +from openoutfind.core.pipeline.companyenrich import CompanyEnrichSource, from_options +from openoutfind.enrichment.provider import ProviderUnavailable + +pytestmark = pytest.mark.django_db(transaction=True) +QUERY = {"domains": ["example.com"], "positionQuery": ["Founder"], "page": 1, "pageSize": 10} + + +@pytest.fixture +def page(): + return {"items": [{"id": 9223372036854775807, "name": "Alex Example", + "first_name": "Alex", "last_name": "Example", "position": "Founder", + "seniority": "founder", "socials": {"linkedin_url": "https://www.linkedin.com/in/alex-example/"}, + "location": {"country": {"code": "US", "name": "United States"}}, + "experiences": [ + {"type": "company", "isCurrent": False, + "company": {"id": "old", "name": "Old Employer"}}, + {"type": "company", "isCurrent": True, + "company": {"id": "new", "name": "Example", "domain": "example.com", "industry": "Software"}}]}], + "page": 1, "pageSize": 10, "totalItems": 23, "totalPages": 3} + + +@pytest.fixture +def http(mocker, page): + response = MagicMock() + response.status_code = 200 + response.iter_content.return_value = [json.dumps(page).encode()] + response.__enter__.return_value = response + session = MagicMock() + session.__enter__.return_value = session + session.post.return_value = response + mocker.patch.object(api.requests, "Session", return_value=session) + return session, response + + +def test_single_request_exact_schema_and_full_page(http, page): + session, _ = http + result = api.search("test-secret", QUERY) + assert result == page + assert result["items"][0]["id"] == 9223372036854775807 + session.post.assert_called_once_with( + api.API_URL, json=QUERY, + headers={"Authorization": "Bearer test-secret", "Accept": "application/json"}, + timeout=(5, 30), allow_redirects=False, stream=True) + row = api.rows_for(result)[0] + assert row["company_name"] == "Example" + assert row["_first_name"] == "Alex" + assert row["_country_code"] == "US" + assert row["_discovery_source"] == "companyenrich" + assert "email" not in row + assert "contact_headline" not in row + + +@pytest.mark.parametrize("status,error", [(301, "provider_unavailable"), (400, "provider_unavailable"), + (401, "provider_auth"), (402, "provider_out_of_credits"), (403, "provider_auth"), + (429, "provider_rate_limited"), (503, "provider_unavailable")]) +def test_http_errors_are_not_empty_results_or_retried(http, status, error): + session, response = http + response.status_code = status + response.text = "test-secret raw private response" + with pytest.raises(ProviderUnavailable) as caught: + api.search("test-secret", QUERY) + assert caught.value.error_type == error + assert "test-secret" not in str(caught.value) + assert "private response" not in str(caught.value) + session.post.assert_called_once() + response.iter_content.assert_not_called() + + +def test_transport_failure_is_ambiguous_and_redacted(http): + session, _ = http + session.post.side_effect = requests.Timeout("test-secret") + with pytest.raises(ProviderUnavailable, match="outcome unknown") as caught: + api.search("test-secret", QUERY) + assert "test-secret" not in str(caught.value) + session.post.assert_called_once() + + +@pytest.mark.parametrize("body", [b"not-json", b"null", b'{"items":{}}', + b'{"items":[{"id":"wrong"}]}', b'{"items":[{"id":true}]}']) +def test_invalid_response(http, body): + _, response = http + response.iter_content.return_value = [body] + with pytest.raises(ProviderUnavailable, match="Invalid"): + api.search("test-secret", QUERY) + + +def test_response_size_bound(http, monkeypatch): + _, response = http + monkeypatch.setattr(api, "MAX_RESPONSE_BYTES", 4) + response.iter_content.return_value = [b"12345"] + with pytest.raises(ProviderUnavailable, match="exceeded"): + api.search("test-secret", QUERY) + + +@pytest.mark.parametrize("query", [{}, {"domains": ["example.com"], "pageSize": 101}, + {"domains": ["example.com"], "page": True}, {"companyFilter": {"search": {"query": "software"}}}, + {"countries": ["USA"]}, {"seniority": ["invented"]}, {"department": ["invented"]}, + {"query": "x", "expand": "email"}, {"domains": "example.com"}, + {"atCurrentCompanyAfter": "yesterday"}]) +def test_bad_queries_never_reach_http(http, query): + session, _ = http + with pytest.raises(OpenOutFindError): + api.search("test-secret", query) + session.post.assert_not_called() + + +def test_optional_exclusion_education_and_date_filters(tmp_path): + query = {**QUERY, "exclude": {"seniority": ["intern"]}, + "education": {"institutionNameQuery": ["Example University"]}, + "atCurrentCompanyAfter": "2024-01-01T00:00:00Z"} + path = tmp_path / "query.json" + path.write_text(json.dumps(query)) + assert api.read_query(str(path)) == query + + +@pytest.mark.parametrize("url", ["https://evil.example/in/person", "https://www.linkedin.com/company/person", + "https://www.linkedin.com:bad/in/person", "https://user@www.linkedin.com/in/person", "http://www.linkedin.com/in/person", ""]) +def test_unusable_profile_urls_are_skipped(page, url): + page["items"][0]["socials"]["linkedin_url"] = url + assert api.rows_for(page) == [] + + +def test_historical_job_is_not_current_company(page): + page["items"][0]["experiences"][1]["isCurrent"] = False + row = api.rows_for(page)[0] + assert row["company_name"] == "" + assert row["company_domain"] == "" + + +def test_all_three_flags_and_sufficient_budget_required(tmp_path): + path = tmp_path / "query.json" + path.write_text(json.dumps(QUERY)) + assert from_options({}) is None + with pytest.raises(OpenOutFindError, match="together"): + from_options({"companyenrich_query": str(path)}) + with pytest.raises(OpenOutFindError, match="at least 20"): + CompanyEnrichSource(QUERY, "page-one", 19) + with pytest.raises(OpenOutFindError): + CompanyEnrichSource(QUERY, "", 20) + assert CompanyEnrichSource(QUERY, "page-one", 20).maximum_credits == 20 + + +def test_reserved_page_reused_across_invocations_and_names_preserved(http): + from openoutfind.crm.models import Lead + + session, _ = http + config = SiteConfig(companyenrich_api_key="test-secret") + first = CompanyEnrichSource(QUERY, "page-one", 20) + assert first.discover(config) + assert not first.discover(config) + assert not CompanyEnrichSource(QUERY, "page-one", 20).discover(config) + session.post.assert_called_once() + record = CompanyEnrichPage.objects.get(request_id="page-one") + assert record.imported and record.response["totalPages"] == 3 + lead = Lead.objects.get(synthetic=False) + assert lead.first_name == "Alex" and lead.last_name == "Example" + assert lead.email is None + assert lead.country_code == "US" + assert lead.discovery_source == "companyenrich" + assert lead.company.domain == "example.com" + + +def test_ambiguous_failure_never_replays(http): + session, _ = http + session.post.side_effect = requests.Timeout("private") + config = SiteConfig(companyenrich_api_key="test-secret") + with pytest.raises(ProviderUnavailable): + CompanyEnrichSource(QUERY, "uncertain", 20).discover(config) + assert CompanyEnrichPage.objects.get(request_id="uncertain").response is None + with pytest.raises(ProviderUnavailable, match="already submitted"): + CompanyEnrichSource(QUERY, "uncertain", 20).discover(config) + session.post.assert_called_once() + with pytest.raises(ProviderUnavailable, match="different query"): + CompanyEnrichSource({**QUERY, "page": 2}, "uncertain", 20).discover(config) + session.post.assert_called_once() + + +def test_saved_response_resumes_after_local_embedding_failure(http, mocker): + session, _ = http + config = SiteConfig(companyenrich_api_key="test-secret") + embedding = mocker.patch("openoutfind.core.ml.embeddings.embed_texts", side_effect=RuntimeError("local failure")) + with pytest.raises(RuntimeError, match="local failure"): + CompanyEnrichSource(QUERY, "resume", 20).discover(config) + assert CompanyEnrichPage.objects.get(request_id="resume").response is not None + import numpy as np + embedding.side_effect = lambda texts: np.ones((len(texts), 384)) + assert CompanyEnrichSource(QUERY, "resume", 20).discover(config) + session.post.assert_called_once() + + +def test_source_selection_is_scoped_and_default_walk_unchanged(mocker): + from openoutfind.core.pipeline.discover import discover + + source = CompanyEnrichSource(QUERY, "page-one", 20) + paid = mocker.patch.object(source, "discover", return_value=True) + walk = mocker.patch("openoutfind.core.pipeline.walk.run_round", return_value=object()) + mocker.patch("openoutfind.enrichment.bettercontact.is_configured", return_value=True) + config = SiteConfig(product_docs="example", companyenrich_api_key="unused") + with discovery_source.selected(source): + assert discover(config) + assert discovery_source.current() is None + paid.assert_called_once_with(config) + walk.assert_not_called() + assert discover(config) + walk.assert_called_once() + + +def test_provenance_survives_duplicate_and_blocks_all_hub_paths(http, mocker): + from openoutfind.contacts import service + from openoutfind.core.db.leads import store_rows + from openoutfind.crm.models import Lead + + known = Lead.objects.create(profile_url="https://www.linkedin.com/in/alex-example", + full_name="Original", country_code="US") + CompanyEnrichSource(QUERY, "page-one", 20).discover(SiteConfig(companyenrich_api_key="test")) + known.refresh_from_db() + assert known.discovery_source == "companyenrich" + assert known.full_name == "Original" + store_rows([{"contact_linkedin_profile_url": known.profile_url}]) + known.refresh_from_db() + assert known.discovery_source == "companyenrich" + token = mocker.patch.object(service, "token_in_hand") + page_token = mocker.patch.object(service, "_page_token") + send = mocker.patch.object(service, "_send") + assert service.resolve(known) is None + service.contribute(known, ["alex@example.com"], "bettercontact") + row = {"contact_linkedin_profile_url": known.profile_url, + "company_linkedin_url": "https://www.linkedin.com/company/example", + "_discovery_source": "companyenrich"} + service.share_profiles([row], "US") + service.share_companies([row], "US") + token.assert_not_called() + page_token.assert_not_called() + send.assert_not_called() + + +def test_readiness_uses_only_explicit_source_key(configure): + from openoutfind.core.readiness import missing_variables + + configure(product_docs="product", campaign_target="target", ai_model="openai:test", llm_api_key="fake") + with discovery_source.selected(CompanyEnrichSource(QUERY, "page-one", 20)): + assert missing_variables() == {"companyenrich": ["OPENOUTFIND_COMPANYENRICH_API_KEY"]} + assert missing_variables() == {"bettercontact": ["OPENOUTFIND_BETTERCONTACT_API_KEY"]} + + +def test_cli_selection_resets_on_exception(tmp_path, mocker): + from openoutfind.core.management.commands.find import Command + + path = tmp_path / "query.json" + path.write_text(json.dumps(QUERY)) + command = Command() + options = vars(command.create_parser("outfind", "find").parse_args([ + "1", "--companyenrich-query", str(path), "--companyenrich-request-id", "page-one", + "--discovery-credit-budget", "20"])) + def interrupted(**options): + assert discovery_source.current().maximum_credits == 20 + raise RuntimeError("interrupted") + mocker.patch.object(command, "_handle", side_effect=interrupted) + with pytest.raises(RuntimeError, match="interrupted"): + command.handle(**options) + assert discovery_source.current() is None + + +def test_cli_to_local_lead_and_export_with_one_paid_request(http, tmp_path, configure, mocker): + """Exercise the real CLI, readiness, bounded job, ingestion and CSV. Only the + qualification verdict is supplied locally instead of paying an LLM in a test.""" + import csv + import io + from django.core.management import call_command + from openoutfind.core.db.leads import promote_lead_to_deal + from openoutfind.core.pipeline.discover import discover + from openoutfind.crm.models import Lead + + configure(product_docs="Example product", campaign_target="Founders", + ai_model="openai:test", llm_api_key="test", companyenrich_api_key="test") + path = tmp_path / "query.json" + path.write_text(json.dumps(QUERY)) + mocker.patch("openoutfind.core.management.commands.find.ensure_database") + def qualify_one(config, **kwargs): + assert not kwargs["buy_addresses"] + discover(config) + lead = Lead.objects.filter(discovery_source="companyenrich").first() + promote_lead_to_deal(lead.profile_url, "Fixture: matches the target") + return True + mocker.patch("openoutfind.core.cycle.run_one_action", side_effect=qualify_one) + output = io.StringIO() + call_command("find", "1", companyenrich_query=str(path), + companyenrich_request_id="cli-page", discovery_credit_budget=20, + stdout=output, stderr=io.StringIO(), batch=True) + rows = list(csv.DictReader(io.StringIO(output.getvalue()))) + assert len(rows) == 1 + assert rows[0]["first_name"] == "Alex" + assert rows[0]["email"] == "" + assert rows[0]["website"] == "example.com" + assert rows[0]["reason"] == "Fixture: matches the target" + http[0].post.assert_called_once() + assert discovery_source.current() is None + + +def test_find_zero_with_paid_flags_never_purchases(http, tmp_path, configure, mocker): + import io + from django.core.management import call_command + + configure(product_docs="Example product", campaign_target="Founders", + ai_model="openai:test", llm_api_key="test", companyenrich_api_key="test") + path = tmp_path / "query.json" + path.write_text(json.dumps(QUERY)) + mocker.patch("openoutfind.core.management.commands.find.ensure_database") + call_command("find", "0", companyenrich_query=str(path), + companyenrich_request_id="zero-page", discovery_credit_budget=20, + stdout=io.StringIO(), stderr=io.StringIO(), batch=True) + assert not CompanyEnrichPage.objects.exists() + http[0].post.assert_not_called() + + +def test_source_rejects_uncommitted_outer_transaction(http): + from django.db import transaction + + with transaction.atomic(), pytest.raises(RuntimeError, match="autocommit"): + CompanyEnrichSource(QUERY, "uncommitted", 20).discover( + SiteConfig(companyenrich_api_key="test")) + assert not CompanyEnrichPage.objects.exists() + http[0].post.assert_not_called() + + +def test_later_free_pages_cannot_clear_local_only_identity(http, mocker): + from openoutfind.contacts import service + from openoutfind.crm.models import Lead + + CompanyEnrichSource(QUERY, "private", 20).discover(SiteConfig(companyenrich_api_key="test")) + token = mocker.patch.object(service, "token_in_hand") + page_token = mocker.patch.object(service, "_page_token") + send = mocker.patch.object(service, "_send") + # Independently fetched free rows have no CompanyEnrich marker and may use a + # different URL spelling. The persisted local-only identity still wins. + row = {"contact_linkedin_profile_url": "https://linkedin.com/in/alex-example/?tracking=1", + "company_linkedin_url": "https://www.linkedin.com/company/example", + "company_name": "Example"} + service.share_profiles([row], "US") + service.share_companies([row], "US") + duplicate = Lead.objects.create(profile_url=row["contact_linkedin_profile_url"], + discovery_source="bettercontact", country_code="US") + assert service.resolve(duplicate) is None + service.contribute(duplicate, ["fixture@example.com"], "bettercontact") + token.assert_not_called() + page_token.assert_not_called() + send.assert_not_called() + + +def test_unknown_current_employer_and_invalid_country_are_not_guessed(page): + page["items"][0]["experiences"] = [{"type": "unknown", "isCurrent": True, + "companyName": "Unmatched employer"}] + page["items"][0]["location"]["country"]["code"] = "USA" + row = api.rows_for(page)[0] + assert row["company_name"] == "Unmatched employer" + assert row["company_domain"] == "" + assert row["_country_code"] == "" From ec3cffec450c7b0d47c430fcb1d3be7ed08a1446 Mon Sep 17 00:00:00 2001 From: namirali Date: Mon, 2 Nov 2026 23:57:45 +0300 Subject: [PATCH 2/2] Document CompanyEnrich exclusion from shared contacts data --- PRIVACY_NOTICE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/PRIVACY_NOTICE.md b/PRIVACY_NOTICE.md index 184da13..58400a8 100644 --- a/PRIVACY_NOTICE.md +++ b/PRIVACY_NOTICE.md @@ -43,6 +43,8 @@ Data reaches the store from OpenOutreach operators, at two moments: - **Email records** arrive **after an operator's paid email-finder lookup returns a verified work email.** - **Profile records** arrive **whenever an operator's prospect search returns a page of results.** The source of those results is the operator's own account with the B2B data provider **BetterContact** (its Lead Finder search), which compiles publicly available professional information. +CompanyEnrich discovery is excluded from this shared store by the client, including later contributions for identities marked as originating from that source. + The maintainer does **not** scrape any website, buy data, or run searches of its own to populate the store; it is filled only by operators' own searches and lookups, subject to the geographic exclusion above. ## How data is used and disclosed