From 7664dec67b213377779580c4819cb9a5c4b3d486 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Fri, 16 Oct 2026 17:06:43 +0530 Subject: [PATCH] feat(pii_remover): default NER to IndicBERTv2 Naamapadam model spaCy en_core_web_lg is English-only, so PERSON/LOCATION in Hindi, Marathi, Kannada, Telugu text were never detected and Hinglish was over-redacted. Switch the default engine to transformers with neuronsbyisshu/indicbert-hi-ner-naamapadam (CC0, ~0.3B params). spaCy en_core_web_sm stays for tokenization; nlp_engine_type="spacy" still works as a fallback. Regex recognizers are unchanged. Co-Authored-By: Claude Fable 5.1 --- backend/Dockerfile | 4 ++-- backend/app/core/validators/pii_remover.py | 13 ++++++++----- backend/app/tests/validators/test_pii_remover.py | 6 +++--- docs/DEVELOPER_GUIDE.md | 4 ++-- docs/architecture/ARCHITECTURE.md | 4 ++-- docs/validators/pii-remover.md | 12 ++++++------ 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 9197ec7..82f7668 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -64,8 +64,8 @@ AutoModelForSequenceClassification.from_pretrained('michellejieli/NSFW_text_clas RUN --mount=type=secret,id=HF_TOKEN \ HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \ /app/.venv/bin/python -c "from transformers import AutoTokenizer, AutoModelForTokenClassification; \ -AutoTokenizer.from_pretrained('dslim/bert-base-NER-uncased', cache_dir='/app/hf_cache'); \ -AutoModelForTokenClassification.from_pretrained('dslim/bert-base-NER-uncased', cache_dir='/app/hf_cache')" +AutoTokenizer.from_pretrained('neuronsbyisshu/indicbert-hi-ner-naamapadam', cache_dir='/app/hf_cache'); \ +AutoModelForTokenClassification.from_pretrained('neuronsbyisshu/indicbert-hi-ner-naamapadam', cache_dir='/app/hf_cache')" # ------------------------------- # Entrypoint (runtime setup) diff --git a/backend/app/core/validators/pii_remover.py b/backend/app/core/validators/pii_remover.py index 9b71877..23732e2 100644 --- a/backend/app/core/validators/pii_remover.py +++ b/backend/app/core/validators/pii_remover.py @@ -34,8 +34,9 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false" -# NER-based types (PERSON, LOCATION, NRP) are opt-in: the NER model is English-only -# and noisy on Indic names. Everything else is pattern/regex based. +# NER-based types (PERSON, LOCATION, NRP) stay opt-in. The default NER model now +# handles Indic scripts, but NRP has no equivalent label in it. Everything else is +# pattern/regex based. NER_ENTITY_TYPES = {"PERSON", "LOCATION", "NRP"} DEFAULT_ENTITY_TYPES = [ e.value for e in PIIEntityType if e.value not in NER_ENTITY_TYPES @@ -53,7 +54,7 @@ } # Shared label mapping for models using CoNLL-style PER/ORG/LOC/MISC labels -# (e.g. dslim/bert-base-NER, Davlan/xlm-roberta-base-ner-hrl) +# (IndicBERTv2 Naamapadam finetune, dslim/bert-base-NER, Davlan/xlm-roberta-*-ner-hrl) CONLL_NER_LABEL_MAPPING = { "PER": "PERSON", "ORG": "ORGANIZATION", @@ -61,7 +62,9 @@ "MISC": "MISC", } -DEFAULT_TRANSFORMERS_MODEL = "dslim/bert-base-NER-uncased" +# IndicBERTv2 fine-tuned on Hindi Naamapadam NER; transfers to other Indic scripts and +# Latin-script names. Eval vs spaCy en_core_web_lg: docs/validators/pii-remover.md. +DEFAULT_TRANSFORMERS_MODEL = "neuronsbyisshu/indicbert-hi-ner-naamapadam" DEFAULT_TRANSFORMERS_THRESHOLD = 0.7 _NLP_ENGINE_CACHE: dict = {} @@ -187,7 +190,7 @@ def __init__( self, entity_types=None, threshold: float | None = None, - nlp_engine_type: str = "spacy", + nlp_engine_type: str = "transformers", model_name: str | None = None, on_fail: Optional[Callable] = OnFailAction.FIX, ): diff --git a/backend/app/tests/validators/test_pii_remover.py b/backend/app/tests/validators/test_pii_remover.py index 58f8413..ae6aaa4 100644 --- a/backend/app/tests/validators/test_pii_remover.py +++ b/backend/app/tests/validators/test_pii_remover.py @@ -141,11 +141,11 @@ def test_transformers_engine_accepts_custom_model(mock_presidio): assert v.model_name == "dslim/bert-base-NER-uncased" -def test_config_defaults_to_spacy_engine(mock_presidio): +def test_config_defaults_to_transformers_engine(mock_presidio): config = PIIRemoverSafetyValidatorConfig(type="pii_remover") v = config.build() - assert v.nlp_engine_type == "spacy" - assert v.model_name == "en_core_web_lg" + assert v.nlp_engine_type == "transformers" + assert v.model_name == DEFAULT_TRANSFORMERS_MODEL def test_cached_analyzer_registers_only_requested_indian_recognizers(): diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index f3f4bd2..bdb6cdf 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -471,7 +471,7 @@ Detects and anonymizes Personally Identifiable Information using [Microsoft Pres | Property | Value | |----------|-------| | Type | `pii_remover` | -| Implementation | Local (Presidio + spaCy `en_core_web_lg`) | +| Implementation | Local (Presidio + IndicBERTv2 NER `neuronsbyisshu/indicbert-hi-ner-naamapadam`) | | Speed | Fast–Medium (50–200ms depending on text length) | | Languages | English (NLP engine); India-specific patterns via regex | | `fix_value` | Yes — returns redacted text | @@ -523,7 +523,7 @@ Output: "[REDACTED_PERSON_1] and [REDACTED_PERSON_2] called [REDACTED_PHONE_NUMB #### How It Works Internally -1. **Analyzer initialization** — On first use, a spaCy `en_core_web_lg` NLP engine is built and cached globally. A Presidio `AnalyzerEngine` is constructed with this engine. India-specific recognizers are registered based on the configured `entity_types`. +1. **Analyzer initialization** — On first use, a Presidio `AnalyzerEngine` with a HuggingFace NER recognizer (`neuronsbyisshu/indicbert-hi-ner-naamapadam`) is built and cached globally; spaCy `en_core_web_sm` handles tokenization. A Presidio `AnalyzerEngine` is constructed with this engine. India-specific recognizers are registered based on the configured `entity_types`. 2. **Analyzer caching** — Analyzer instances are cached by the set of India-specific recognizers in use. 3. **Analysis** — Presidio analyzes the text for the requested entity types. Each detection returns an entity type, position, and confidence score. 4. **Threshold filtering** — Entities with confidence below the threshold are not redacted. diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index b0a728e..c38a368 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -126,7 +126,7 @@ Each request carries a list of `ValidatorConfigItem` objects (a Pydantic discrim | Validator | Type key | Source | Description | |---|---|---|---| | **Lexical Slur** | `uli_slur_match` | Local (custom) | Detects slurs from a curated bilingual (Hindi/English) CSV. Normalises text (emoji removal, unicode NFKC, `ftfy` encoding fix) before matching. Supports severity filtering: `low`/`medium`/`high`/`all`. Redacts matches with `[REDACTED_SLUR]`. | -| **PII Remover** | `pii_remover` | Local (custom) | Uses Microsoft Presidio (`presidio_analyzer` + `presidio_anonymizer`) with a spaCy NLP engine (`en_core_web_lg`). Detects and redacts 15 entity types including India-specific identifiers (Aadhaar, PAN, Passport, Vehicle Registration, Voter ID). Replaces entities with labelled placeholders, e.g. `[REDACTED_PERSON_1]`. | +| **PII Remover** | `pii_remover` | Local (custom) | Uses Microsoft Presidio (`presidio_analyzer` + `presidio_anonymizer`) with a HuggingFace NER recognizer (`neuronsbyisshu/indicbert-hi-ner-naamapadam`, IndicBERTv2 fine-tuned on Naamapadam) and spaCy `en_core_web_sm` for tokenization. Detects and redacts 15 entity types including India-specific identifiers (Aadhaar, PAN, Passport, Vehicle Registration, Voter ID). Replaces entities with labelled placeholders, e.g. `[REDACTED_PERSON_1]`. | | **Ban List** | `ban_list` | Hub (`guardrails/ban_list`) | Blocks exact words from a caller-supplied list or a stored `BanList` DB record (referenced by UUID). Words are fetched from the database in `_resolve_validator_configs()` before the guard runs. | | **Topic Relevance** | `topic_relevance` | Local (custom) | Wraps the Hub's `LLMCritic` validator. Given a freeform topic-scope description (stored as a `TopicRelevance` DB record or supplied inline), constructs a scoring prompt and calls an LLM (default: `gpt-4o-mini`) via LiteLLM. Scores 1–3; score ≤ 1 is a fail. Supports versioned prompt templates (`v1.md`–`v3.md`). | | **Profanity Free** | `profanity_free` | Hub (`guardrails/profanity_free`) | Detects general profanity in text. | @@ -212,7 +212,7 @@ Tenant scope is never read from the query string or request body, so the depende | Database | PostgreSQL (psycopg3) | | Migrations | Alembic | | Validation engine | guardrails-ai (pinned to v0.10.0) | -| NLP (PII) | spaCy `en_core_web_lg` via Presidio | +| NLP (PII) | IndicBERTv2 NER `neuronsbyisshu/indicbert-hi-ner-naamapadam` via Presidio (spaCy `en_core_web_sm` tokenizer) | | ML (toxicity) | PyTorch CPU, `textdetox/xlmr-large-toxicity-classifier` via Transformers | | LLM calls | LiteLLM (used internally by `LLMCritic` / `TopicRelevance`) | | Package manager | uv | diff --git a/docs/validators/pii-remover.md b/docs/validators/pii-remover.md index 4d86d32..da1d3f6 100644 --- a/docs/validators/pii-remover.md +++ b/docs/validators/pii-remover.md @@ -9,9 +9,9 @@ Detects and anonymizes Personally Identifiable Information using [Microsoft Pres | Property | Value | |----------|-------| | Type | `pii_remover` | -| Implementation | Local (Presidio + spaCy `en_core_web_lg`) | +| Implementation | Local (Presidio + IndicBERTv2 NER `neuronsbyisshu/indicbert-hi-ner-naamapadam`) | | Speed | Fast–Medium (50–200ms depending on text length) | -| Languages | English (NLP engine); India-specific patterns via regex | +| Languages | English, Hinglish, Hindi, Marathi, Kannada, Telugu + other Indic scripts (NER); India-specific patterns via regex | | `fix_value` | Yes — returns redacted text | | Requires API key | No | @@ -76,7 +76,7 @@ This format is: ## How It Works Internally -1. **Analyzer initialization** — On first use, a spaCy `en_core_web_lg` NLP engine is built and cached globally. A Presidio `AnalyzerEngine` is constructed with this engine. India-specific recognizers (`InAadhaarRecognizer`, `InPanRecognizer`, etc.) are registered based on the configured `entity_types`. +1. **Analyzer initialization** — On first use, a Presidio `AnalyzerEngine` is built and cached globally. NER runs through a HuggingFace token-classification model (`neuronsbyisshu/indicbert-hi-ner-naamapadam`, IndicBERTv2 fine-tuned on Naamapadam); spaCy `en_core_web_sm` is used only for tokenization. Set `nlp_engine_type="spacy"` on `PIIRemover` to fall back to spaCy `en_core_web_lg` NER. India-specific recognizers (`InAadhaarRecognizer`, `InPanRecognizer`, etc.) are registered based on the configured `entity_types`. 2. **Analyzer caching** — Analyzer instances are cached by the set of India-specific recognizers in use. This means different `entity_types` combinations that share the same India recognizer set reuse the same analyzer. @@ -140,13 +140,13 @@ Output: "What is the weather in Mumbai today?" (PassResult, no change) "Mumbai" or "India" may be detected as `LOCATION` and redacted, even in benign context. If you don't need location protection, omit `LOCATION` from `entity_types`. ### Short names and ambiguous tokens -Very short or common names (e.g., "Ali", "Sam") may not be detected by spaCy NER with high confidence. They'll pass through if their score is below threshold. +Lone Latin-script first names inside Indic text (e.g. "डॉ Aiswarya") are the weakest case for the NER model (~0.45 recall in eval). Multi-token names and native-script names do better. ### Non-English text -The NLP engine is `en_core_web_lg` — English only. Hindi or other language names will not be detected by the NER component. India-specific regex recognizers (`IN_AADHAAR`, `IN_PAN`, etc.) work independently of language. +The NER model is IndicBERTv2 fine-tuned on Hindi Naamapadam. In eval it detects names and locations in Hindi, Marathi, Kannada, Telugu, Bengali and Tamil scripts, plus English and Hinglish. `NRP` (nationality / religion / political group) is not a label in this model and will not be detected. India-specific regex recognizers (`IN_AADHAAR`, `IN_PAN`, etc.) work independently of language. ### First-run latency -The first request loads the spaCy model into memory. This can take 2–5 seconds. Subsequent requests use the cached model and are fast. +The first request loads the NER model into memory. This can take 5–15 seconds. Subsequent requests use the cached model and are fast. ---