From 29c869d1b4bf90dddf72bbd9363314d8999e9aee Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 11:54:13 -0500 Subject: [PATCH 1/3] fix(extract): compact prompt for small models, junk mention scoring (#59, #57) #59: _build_extraction_prompt claimed a compact prompt for models of 8B or fewer but built one prompt for all. _is_compact_model parses the parameter count from the model tag (qwen3:8b, llama3.1:8b, qwen2.5:1.5b; 8x7b MoE tags are skipped) and falls back to a short list of known small tags; unknown sizes keep the full prompt. The compact prompt lists core types plus custom types and drops the relationship section. _call_ollama and _call_openai strip ... before returning. Local A/B on the FOIA letter fixture: qwen3:8b full prompt ran 123s and returned truncated JSON (0/14); compact returned valid JSON with 12-13/14 in 27s. llama3.1:8b went 12/14 full to 9-10/14 compact in a third of the time. #57: _mention_score now drops mentions over 80 characters, NER mentions under 3, sentences (two sentence-ending periods, or a terminal one with 6+ words; "U.S." and "Dr." do not count), and NER mentions with two or more garbled tokens. One garbled token, a non-alphanumeric ratio above 0.25, and PERSON over 5 tokens are penalties, not drops. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- CHANGELOG.md | 2 + openfoia/pipeline/extract.py | 84 ++++++++++++++++++++--- tests/test_extraction_quality.py | 111 +++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 tests/test_extraction_quality.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a35343..b070549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ exhaustive lists. ### Fixed +- Local models of 8B or smaller now get a short extraction prompt (core entity types, no relationships), and `` blocks from reasoning models like qwen3 are stripped before JSON parsing; qwen3:8b went from unparseable output to 13 of 14 entities on the test letter (#59). +- Entity extraction drops full sentences, run-on lines over 80 characters, and OCR garbage like `EC .Orchestrating a or g-?te' 22`, and down-weights punctuation-heavy text and PERSON hits over 5 words (#57). - `network.tor: true` was ignored by `records search -s ` (every source except cia), `records search --all-sources` (all sources, cia included), `records fetch -s documentcloud`, `records fetch -s nara` without diff --git a/openfoia/pipeline/extract.py b/openfoia/pipeline/extract.py index 54fed95..f2feb49 100644 --- a/openfoia/pipeline/extract.py +++ b/openfoia/pipeline/extract.py @@ -198,6 +198,12 @@ class ExtractionResult: _NER_TYPES = {EntityType.PERSON, EntityType.ORGANIZATION, EntityType.LOCATION} +_MAX_MENTION_LEN = 80 +# A period ending a lowercase word of 2+ letters; "U.S." and "Dr." don't count, "Inc." does once +_SENTENCE_PERIOD = re.compile(r"[a-z]{2,}\.(?=\s|$)") +# One plausible name token in any script: "U.S.", "AT&T", "O'Brien", "Smith-Jones", "Müller" +_NAME_TOKEN = re.compile(r"^[^\W_]+(?:[.&'\u2019\-][^\W_]+)*\.?['\u2019]?$") + def _surface_clean(text: str) -> str: """Normalize whitespace and strip trailing punctuation.""" @@ -234,9 +240,24 @@ def _mention_score(m: Mention) -> float: return 0.0 # NER types with ≤2 letters (not an acronym) - if m.entity_type in _NER_TYPES and len(re.sub(r"[^A-Za-z]", "", s)) <= 2: + if m.entity_type in _NER_TYPES and (len(s) < 3 or len(re.sub(r"[^A-Za-z]", "", s)) <= 2): + return 0.0 + + # Names are short; anything this long is a sentence or a run-on line + if len(s) > _MAX_MENTION_LEN: + return 0.0 + + words = s.split() + sentence_periods = len(_SENTENCE_PERIOD.findall(s)) + if sentence_periods > 1 or (s.endswith(".") and sentence_periods and len(words) >= 6): return 0.0 + garbled = 0 + if m.entity_type in _NER_TYPES: + garbled = sum(1 for w in words if not _NAME_TOKEN.match(w.strip(',;:()"'))) + if garbled >= 2: + return 0.0 + # Calibrate backend scores if m.backend_name == "spacy": base = 0.55 # spaCy's fixed 0.85 is inflated @@ -260,8 +281,14 @@ def _mention_score(m: Mention) -> float: if m.entity_type in _NER_TYPES and not re.search(r"[A-Z]", s): base -= 0.35 # Single short word (non-acronym) - if m.entity_type in _NER_TYPES and len(s.split()) == 1 and len(s) <= 3: + if m.entity_type in _NER_TYPES and len(words) == 1 and len(s) <= 3: base -= 0.45 + if garbled: + base -= 0.30 + if sum(1 for c in s if not c.isalnum() and not c.isspace()) / len(s) > 0.25: + base -= 0.50 + if m.entity_type == EntityType.PERSON and len(words) > 5: + base -= 0.40 return max(0.0, min(0.99, base)) @@ -761,17 +788,38 @@ def _find_entity_match(text: str, entities: list[ExtractedEntity]) -> ExtractedE # --------------------------------------------------------------------------- +# "8b", "1.5b", "qwen3:8b"; the lookbehind skips MoE tags like "8x7b" +_PARAM_COUNT = re.compile(r"(? bool: + """True for models of 8B parameters or fewer; unknown sizes get the full prompt.""" + name = (model or "").lower() + m = _PARAM_COUNT.search(name) + if m: + return float(m.group(1)) <= 8 + return any(tag in name for tag in _KNOWN_SMALL_MODELS) + + +def _strip_think(text: str) -> str: + """Drop ... blocks that reasoning models emit before their JSON.""" + return re.sub(r"[\s\S]*?", "", text).strip() + + def _build_extraction_prompt( text: str, context: str | None, page_number: int | None, entity_config: EntityConfig, + compact: bool = False, ) -> str: """Build the structured prompt for LLM entity extraction. - Uses a compact prompt for smaller models (≤8B) and a detailed prompt - for larger models. Smaller models choke on long instructions with - many entity types and relationship definitions. + compact=True (models of 8B or fewer, see _is_compact_model) asks for the + core entity types only, with no relationship section, because small + models lose the JSON format under long instructions. """ type_descriptions = { @@ -786,8 +834,12 @@ def _build_extraction_prompt( "ADDRESS": "Street addresses, mailing addresses", } + builtin = entity_config.builtin_types + if compact: + builtin = [t for t in builtin if t in _COMPACT_TYPES] + type_lines = [] - for t in entity_config.builtin_types: + for t in builtin: desc = type_descriptions.get(t, t) type_lines.append(f"- {t}: {desc}") @@ -802,6 +854,18 @@ def _build_extraction_prompt( if entity_config.extraction_prompt_suffix: suffix = f"\n\nADDITIONAL INSTRUCTIONS:\n{entity_config.extraction_prompt_suffix}" + if compact: + return f"""Extract EVERY named entity from this FOIA document. Be thorough. + +TEXT: +{text} + +Types: +{types_block} + +Return ONLY JSON: +{{"entities": [{{"raw_text": "...", "normalized": "...", "type": "PERSON", "confidence": 0.9, "context": "..."}}]}}{suffix}""" + return f"""Analyze this document and extract ALL entities and relationships. This is a government/legal document obtained through FOIA. Be thorough. @@ -868,7 +932,7 @@ def _call_ollama( ) with urllib.request.urlopen(req, timeout=300) as resp: body = json.loads(resp.read()) - return body.get("response", "") + return _strip_think(body.get("response", "")) def _call_anthropic( @@ -902,7 +966,7 @@ def _call_openai( max_tokens=max_tokens, response_format={"type": "json_object"}, ) - return response.choices[0].message.content or "" + return _strip_think(response.choices[0].message.content or "") def _llm_available(provider: str, api_key: str | None, base_url: str | None) -> bool: @@ -1291,7 +1355,9 @@ async def _extract_with_llm( def _extract_chunk_llm( self, text: str, context: str | None, page_number: int | None ) -> dict[str, Any]: - prompt = _build_extraction_prompt(text, context, page_number, self.entity_config) + prompt = _build_extraction_prompt( + text, context, page_number, self.entity_config, compact=_is_compact_model(self.model) + ) if self.provider == "ollama": content = _call_ollama( diff --git a/tests/test_extraction_quality.py b/tests/test_extraction_quality.py new file mode 100644 index 0000000..34bf03f --- /dev/null +++ b/tests/test_extraction_quality.py @@ -0,0 +1,111 @@ +"""Junk-mention scoring (#57) and small-model prompts (#59).""" + +import json + +import pytest + +from openfoia.config import EntityConfig +from openfoia.pipeline.extract import ( + EntityType, + Mention, + _build_extraction_prompt, + _is_compact_model, + _mention_score, + _strip_think, +) + +ORG, PERSON, LOC = EntityType.ORGANIZATION, EntityType.PERSON, EntityType.LOCATION + + +def _m(text: str, etype: EntityType, backend: str = "regex", score: float = 0.9) -> Mention: + return Mention(etype, text, text, -1, -1, None, "", backend, score) + + +@pytest.mark.parametrize( + "text,etype", + [ + ("To:", ORG), + ("Bill To:", ORG), + ("APPROVED", ORG), + ("EC .Orchestrating a or g-?te' 22", ORG), + ( + "Justification of request: Clcorwcw will assist the department with " + "identifying suspects through facial recognition.", + PERSON, + ), + ("The officer filed the report. Then he left the building.", PERSON), + ("Ab", ORG), + ("x" * 81, ORG), + ], +) +def test_junk_scores_zero(text, etype): + for backend in ("regex", "spacy", "gliner", "llm"): + assert _mention_score(_m(text, etype, backend)) == 0.0, (text, backend) + + +@pytest.mark.parametrize( + "text,etype", + [ + ("U.S. Department of Justice", ORG), + ("AT&T", ORG), + ("Meridian Defense Systems Inc.", ORG), + ("Clearview AI", ORG), + ("FBI", ORG), + ("Sarah Chen", PERSON), + ("Patrick O'Brien", PERSON), + ("Dr. John A. Smith", PERSON), + ("Mrs. Jane Smith-Jones", PERSON), + ("José Müller", PERSON), + ("Washington, DC", LOC), + ], +) +def test_legitimate_names_kept(text, etype): + for backend in ("regex", "spacy", "gliner"): + assert _mention_score(_m(text, etype, backend)) > 0.3, (text, backend) + + +def test_long_person_penalized_not_dropped(): + long_name = _mention_score(_m("Sgt. Lakca Gaither For FMU Use Only", PERSON, "spacy")) + assert 0.0 < long_name < _mention_score(_m("Lakca Gaither", PERSON, "spacy")) + + +@pytest.mark.parametrize( + "model,compact", + [ + ("qwen3:8b", True), + ("llama3.1:8b", True), + ("llama3.1:8b-instruct-q4_K_M", True), + ("mistral:7b", True), + ("qwen2.5:1.5b", True), + ("llama3.2:3b", True), + ("llama3.2", True), + ("tinyllama", True), + ("qwen2.5:14b", False), + ("gemma3:27b", False), + ("llama3.1:70b", False), + ("mixtral:8x7b", False), + ("claude-sonnet-4-5", False), + ("gpt-4o", False), + ("", False), + (None, False), + ], +) +def test_model_size_classification(model, compact): + assert _is_compact_model(model) is compact + + +def test_compact_prompt_drops_relationships(): + cfg = EntityConfig(custom_types=[{"name": "badge", "description": "Badge numbers"}]) + full = _build_extraction_prompt("Some text.", None, 1, cfg) + compact = _build_extraction_prompt("Some text.", None, 1, cfg, compact=True) + assert "RELATIONSHIP" in full.upper() + assert "relationship" not in compact.lower() + assert "DOCUMENT_ID" not in compact and "BADGE" in compact + assert len(compact) < len(full) / 2 + assert compact.isascii() + + +def test_strip_think_yields_json(): + raw = '\nThe doc mentions {Acme}...\n\n{"entities": []}' + assert json.loads(_strip_think(raw)) == {"entities": []} + assert _strip_think('{"entities": []}') == '{"entities": []}' From 6589677508312828274f633967f3bfce1d856da1 Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 12:07:46 -0500 Subject: [PATCH 2/3] fix(extract): keep connector and abbreviated names, compact cutoff under 8B Review fixes for #135: - A standalone "&" no longer counts as a garbled token, so "Kirkland & Ellis LLP" and "Wells Fargo & Co." keep their score. - The sentence check ignores periods after abbreviations (org suffixes, Dr., Rev., Jr., Dept., Div., ...). The PERSON over-5-tokens penalty counts name tokens only, so titles don't push a name over the limit. - The punctuation-ratio penalty applies to PERSON, ORGANIZATION and LOCATION only; MONEY like "$1,250,000.00" keeps its score. - Model sizes in "m" (gemma3:270m) are compact. - Auto cutoff is now under 8B plus the qwen3 family, so llama3.1:8b keeps the full prompt. qwen3:8b on the full prompt with think disabled still returned truncated JSON at 4000 and 8000 tokens. New ai.compact_prompt (auto|true|false, env OPENFOIA_AI_COMPACT_PROMPT) overrides the rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- CHANGELOG.md | 2 +- openfoia/config.py | 5 +++ openfoia/pipeline/extract.py | 55 ++++++++++++++++++++++++-------- tests/test_extraction_quality.py | 47 +++++++++++++++++++++++++-- 4 files changed, 92 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b070549..70b2b9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ exhaustive lists. ### Fixed -- Local models of 8B or smaller now get a short extraction prompt (core entity types, no relationships), and `` blocks from reasoning models like qwen3 are stripped before JSON parsing; qwen3:8b went from unparseable output to 13 of 14 entities on the test letter (#59). +- Local models under 8B and the qwen3 family now get a short extraction prompt (core entity types, no relationships; override with `ai.compact_prompt: true|false` or `OPENFOIA_AI_COMPACT_PROMPT`), and `` blocks from reasoning models like qwen3 are stripped before JSON parsing; qwen3:8b went from unparseable output to 13 of 14 entities on the test letter (#59). - Entity extraction drops full sentences, run-on lines over 80 characters, and OCR garbage like `EC .Orchestrating a or g-?te' 22`, and down-weights punctuation-heavy text and PERSON hits over 5 words (#57). - `network.tor: true` was ignored by `records search -s ` (every source except cia), `records search --all-sources` (all sources, cia included), diff --git a/openfoia/config.py b/openfoia/config.py index f14df9b..685787b 100644 --- a/openfoia/config.py +++ b/openfoia/config.py @@ -41,6 +41,7 @@ class AIConfig: model: str = "llama3.2" base_url: str | None = None api_key: str | None = None + compact_prompt: str = "auto" # auto, true, false (env OPENFOIA_AI_COMPACT_PROMPT) # For entity extraction extraction_temperature: float = 0.1 @@ -265,6 +266,7 @@ def _merge_config(config: OpenFOIAConfig, data: dict[str, Any]) -> OpenFOIAConfi or ai.get("_api_key") or ai.get(ai.get("provider", ""), {}).get("api_key") ) + config.ai.compact_prompt = str(ai.get("compact_prompt", config.ai.compact_prompt)).lower() # Handle ollama specifically if config.ai.provider == "ollama" and "ollama" in ai: @@ -372,6 +374,8 @@ def _apply_env_overrides(config: OpenFOIAConfig, prefix: str) -> OpenFOIAConfig: config.ai.api_key = v if v := os.environ.get(f"{prefix}AI_BASE_URL"): config.ai.base_url = v + if v := os.environ.get(f"{prefix}AI_COMPACT_PROMPT"): + config.ai.compact_prompt = v.lower() # Specific providers if v := os.environ.get(f"{prefix}ANTHROPIC_API_KEY"): @@ -497,6 +501,7 @@ def save_config(config: OpenFOIAConfig, config_path: Path | str | None = None) - "ai": { "provider": config.ai.provider, "model": config.ai.model, + "compact_prompt": config.ai.compact_prompt, }, "ocr": { "backend": config.ocr.backend, diff --git a/openfoia/pipeline/extract.py b/openfoia/pipeline/extract.py index f2feb49..9cecc6c 100644 --- a/openfoia/pipeline/extract.py +++ b/openfoia/pipeline/extract.py @@ -199,8 +199,12 @@ class ExtractionResult: _NER_TYPES = {EntityType.PERSON, EntityType.ORGANIZATION, EntityType.LOCATION} _MAX_MENTION_LEN = 80 -# A period ending a lowercase word of 2+ letters; "U.S." and "Dr." don't count, "Inc." does once -_SENTENCE_PERIOD = re.compile(r"[a-z]{2,}\.(?=\s|$)") +# A period after 2+ lowercase letters ends a sentence unless the word is an abbreviation +_SENTENCE_PERIOD = re.compile(r"[a-z]{2}\.$") +_ABBREVIATIONS = _ORG_SUFFIXES | { + "mr", "mrs", "ms", "dr", "rev", "jr", "sr", "st", "hon", "prof", "gen", "sgt", "lt", "col", + "capt", "dept", "div", "assn", "bros", "intl", "natl", "govt", "ave", "blvd", "no", "vs", +} # fmt: skip # One plausible name token in any script: "U.S.", "AT&T", "O'Brien", "Smith-Jones", "Müller" _NAME_TOKEN = re.compile(r"^[^\W_]+(?:[.&'\u2019\-][^\W_]+)*\.?['\u2019]?$") @@ -248,13 +252,17 @@ def _mention_score(m: Mention) -> float: return 0.0 words = s.split() - sentence_periods = len(_SENTENCE_PERIOD.findall(s)) - if sentence_periods > 1 or (s.endswith(".") and sentence_periods and len(words) >= 6): + sentence_ends = [ + i + for i, w in enumerate(words) + if _SENTENCE_PERIOD.search(w) and w.rstrip(".").lower() not in _ABBREVIATIONS + ] + if len(sentence_ends) > 1 or (sentence_ends == [len(words) - 1] and len(words) >= 6): return 0.0 garbled = 0 if m.entity_type in _NER_TYPES: - garbled = sum(1 for w in words if not _NAME_TOKEN.match(w.strip(',;:()"'))) + garbled = sum(1 for w in words if w != "&" and not _NAME_TOKEN.match(w.strip(',;:()"'))) if garbled >= 2: return 0.0 @@ -285,9 +293,15 @@ def _mention_score(m: Mention) -> float: base -= 0.45 if garbled: base -= 0.30 - if sum(1 for c in s if not c.isalnum() and not c.isspace()) / len(s) > 0.25: + if ( + m.entity_type in _NER_TYPES + and sum(1 for c in s if not c.isalnum() and not c.isspace()) / len(s) > 0.25 + ): base -= 0.50 - if m.entity_type == EntityType.PERSON and len(words) > 5: + if ( + m.entity_type == EntityType.PERSON + and sum(w.rstrip(".").lower() not in _ABBREVIATIONS for w in words) > 5 + ): base -= 0.40 return max(0.0, min(0.99, base)) @@ -788,18 +802,28 @@ def _find_entity_match(text: str, entities: list[ExtractedEntity]) -> ExtractedE # --------------------------------------------------------------------------- -# "8b", "1.5b", "qwen3:8b"; the lookbehind skips MoE tags like "8x7b" -_PARAM_COUNT = re.compile(r"(? bool: - """True for models of 8B parameters or fewer; unknown sizes get the full prompt.""" +def _is_compact_model(model: str | None, setting: str | bool = "auto") -> bool: + """Whether to use the compact prompt. setting is ai.compact_prompt: auto, true or false. + + auto: models under 8B parameters and the qwen3 family; unknown sizes get the full prompt. + """ + choice = str(setting).lower() + if choice in ("true", "false"): + return choice == "true" name = (model or "").lower() + if any(name.startswith(f) for f in _COMPACT_FAMILIES): + return True m = _PARAM_COUNT.search(name) if m: - return float(m.group(1)) <= 8 + return m.group(2) == "m" or float(m.group(1)) < 8 return any(tag in name for tag in _KNOWN_SMALL_MODELS) @@ -1041,6 +1065,7 @@ def __init__( self.base_url = base_url or cfg.ai.base_url self.temperature = cfg.ai.extraction_temperature self.max_tokens = cfg.ai.extraction_max_tokens + self.compact_prompt = cfg.ai.compact_prompt self.entity_config = cfg.entities self._backend: str | None = None # resolved lazily @@ -1356,7 +1381,11 @@ def _extract_chunk_llm( self, text: str, context: str | None, page_number: int | None ) -> dict[str, Any]: prompt = _build_extraction_prompt( - text, context, page_number, self.entity_config, compact=_is_compact_model(self.model) + text, + context, + page_number, + self.entity_config, + compact=_is_compact_model(self.model, self.compact_prompt), ) if self.provider == "ollama": diff --git a/tests/test_extraction_quality.py b/tests/test_extraction_quality.py index 34bf03f..ce9f0d5 100644 --- a/tests/test_extraction_quality.py +++ b/tests/test_extraction_quality.py @@ -4,7 +4,7 @@ import pytest -from openfoia.config import EntityConfig +from openfoia.config import EntityConfig, load_config from openfoia.pipeline.extract import ( EntityType, Mention, @@ -64,6 +64,29 @@ def test_legitimate_names_kept(text, etype): assert _mention_score(_m(text, etype, backend)) > 0.3, (text, backend) +@pytest.mark.parametrize( + "text,etype", + [ + ("Kirkland & Ellis LLP", ORG), + ("Ernst & Young", ORG), + ("Wells Fargo & Co.", ORG), + ("J.P. Morgan Chase & Co.", ORG), + ("Smith, Jones & Associates, P.C.", ORG), + ("Acme Global Defense Solutions Group Inc.", ORG), + ("Johnson and Johnson Holdings Group Inc.", ORG), + ("U.S. Dept. of Justice Civil Rights Div.", ORG), + ("Rev. Dr. Martin Luther King Jr.", PERSON), + ], +) +def test_review_regressions_clear_crossref_cutoff(text, etype): + assert _mention_score(_m(text, etype, "regex")) > 0.5 + + +def test_punctuation_ratio_spares_money(): + money = _m("$1,250,000.00", EntityType.MONEY, "regex") + assert _mention_score(money) == 0.78 + + def test_long_person_penalized_not_dropped(): long_name = _mention_score(_m("Sgt. Lakca Gaither For FMU Use Only", PERSON, "spacy")) assert 0.0 < long_name < _mention_score(_m("Lakca Gaither", PERSON, "spacy")) @@ -73,13 +96,15 @@ def test_long_person_penalized_not_dropped(): "model,compact", [ ("qwen3:8b", True), - ("llama3.1:8b", True), - ("llama3.1:8b-instruct-q4_K_M", True), + ("llama3.1:8b", False), + ("llama3.1:8b-instruct-q4_K_M", False), + ("qwen3:4b", True), ("mistral:7b", True), ("qwen2.5:1.5b", True), ("llama3.2:3b", True), ("llama3.2", True), ("tinyllama", True), + ("gemma3:270m", True), ("qwen2.5:14b", False), ("gemma3:27b", False), ("llama3.1:70b", False), @@ -94,6 +119,22 @@ def test_model_size_classification(model, compact): assert _is_compact_model(model) is compact +@pytest.mark.parametrize("setting,compact", [("true", True), (True, True), ("false", False)]) +def test_compact_setting_overrides_model(setting, compact): + for model in ("llama3.1:70b", "qwen3:8b"): + assert _is_compact_model(model, setting) is compact + + +def test_compact_prompt_config(tmp_path, monkeypatch): + monkeypatch.setenv("OPENFOIA_DATA_DIR", str(tmp_path)) + monkeypatch.delenv("OPENFOIA_AI_COMPACT_PROMPT", raising=False) + assert load_config().ai.compact_prompt == "auto" + (tmp_path / "config.json").write_text(json.dumps({"ai": {"compact_prompt": True}})) + assert load_config().ai.compact_prompt == "true" + monkeypatch.setenv("OPENFOIA_AI_COMPACT_PROMPT", "FALSE") + assert load_config().ai.compact_prompt == "false" + + def test_compact_prompt_drops_relationships(): cfg = EntityConfig(custom_types=[{"name": "badge", "description": "Badge numbers"}]) full = _build_extraction_prompt("Some text.", None, 1, cfg) From 3b5118830dafc7a451b95d83ad2c9b9d2a6dd5e6 Mon Sep 17 00:00:00 2001 From: Jordan Coin Jackson Date: Fri, 6 Nov 2026 12:15:04 -0500 Subject: [PATCH 3/3] fix(extract): drop lowercase prose even when its period sits on an abbreviation; match qwen3 anywhere in model name Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FWQw8DDrDZ2tFzswSSEtce --- openfoia/pipeline/extract.py | 16 ++++++++++++++-- tests/test_extraction_quality.py | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/openfoia/pipeline/extract.py b/openfoia/pipeline/extract.py index 9cecc6c..3fa5e0f 100644 --- a/openfoia/pipeline/extract.py +++ b/openfoia/pipeline/extract.py @@ -205,6 +205,11 @@ class ExtractionResult: "mr", "mrs", "ms", "dr", "rev", "jr", "sr", "st", "hon", "prof", "gen", "sgt", "lt", "col", "capt", "dept", "div", "assn", "bros", "intl", "natl", "govt", "ave", "blvd", "no", "vs", } # fmt: skip + +# Function words inside names, ignored when judging prose vs. name +_CONNECTORS = frozenset( + {"and", "of", "the", "de", "du", "da", "del", "la", "le", "van", "von", "y", "for"} +) # One plausible name token in any script: "U.S.", "AT&T", "O'Brien", "Smith-Jones", "Müller" _NAME_TOKEN = re.compile(r"^[^\W_]+(?:[.&'\u2019\-][^\W_]+)*\.?['\u2019]?$") @@ -259,6 +264,12 @@ def _mention_score(m: Mention) -> float: ] if len(sentence_ends) > 1 or (sentence_ends == [len(words) - 1] and len(words) >= 6): return 0.0 + # Six or more words that are mostly lowercase is prose, not a name, even + # when the only period sits on an abbreviation ("... by Acme Inc."). + if m.entity_type in _NER_TYPES and len(words) >= 6: + content = [w for w in words if w.lower() not in _CONNECTORS and w != "&"] + if content and sum(w[:1].islower() for w in content) * 2 > len(content): + return 0.0 garbled = 0 if m.entity_type in _NER_TYPES: @@ -805,7 +816,8 @@ def _find_entity_match(text: str, entities: list[ExtractedEntity]) -> ExtractedE # "8b", "1.5b", "270m"; the lookbehind skips MoE tags like "8x7b" _PARAM_COUNT = re.compile(r"(? bool: if choice in ("true", "false"): return choice == "true" name = (model or "").lower() - if any(name.startswith(f) for f in _COMPACT_FAMILIES): + if any(f in name for f in _COMPACT_FAMILIES): return True m = _PARAM_COUNT.search(name) if m: diff --git a/tests/test_extraction_quality.py b/tests/test_extraction_quality.py index ce9f0d5..70af659 100644 --- a/tests/test_extraction_quality.py +++ b/tests/test_extraction_quality.py @@ -34,6 +34,9 @@ def _m(text: str, etype: EntityType, backend: str = "regex", score: float = 0.9) PERSON, ), ("The officer filed the report. Then he left the building.", PERSON), + ("The contractor will deliver all software to the department by Acme Inc.", ORG), + ("The department will pay the vendor for services rendered under contract no.", ORG), + ("The officer filed the report. Then he left Inc.", ORG), ("Ab", ORG), ("x" * 81, ORG), ], @@ -105,6 +108,7 @@ def test_long_person_penalized_not_dropped(): ("llama3.2", True), ("tinyllama", True), ("gemma3:270m", True), + ("hf.co/unsloth/Qwen3-8B-GGUF", True), ("qwen2.5:14b", False), ("gemma3:27b", False), ("llama3.1:70b", False),