feat(pii_remover): default NER to IndicBERTv2 Naamapadam model

backend/Dockerfile

@@ -64,8 +64,8 @@ AutoModelForSequenceClassification.from_pretrained('michellejieli/NSFW_text_clas
 RUN --mount=type=secret,id=HF_TOKEN \
     HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
     /app/.venv/bin/python -c "from transformers import AutoTokenizer, AutoModelForTokenClassification; \
-AutoTokenizer.from_pretrained('dslim/bert-base-NER-uncased', cache_dir='/app/hf_cache'); \
-AutoModelForTokenClassification.from_pretrained('dslim/bert-base-NER-uncased', cache_dir='/app/hf_cache')"
+AutoTokenizer.from_pretrained('neuronsbyisshu/indicbert-hi-ner-naamapadam', cache_dir='/app/hf_cache'); \
+AutoModelForTokenClassification.from_pretrained('neuronsbyisshu/indicbert-hi-ner-naamapadam', cache_dir='/app/hf_cache')"
 
 # -------------------------------
 # Entrypoint (runtime setup)

backend/app/core/validators/pii_remover.py

@@ -34,8 +34,9 @@
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
-# NER-based types (PERSON, LOCATION, NRP) are opt-in: the NER model is English-only
-# and noisy on Indic names. Everything else is pattern/regex based.
+# NER-based types (PERSON, LOCATION, NRP) stay opt-in. The default NER model now
+# handles Indic scripts, but NRP has no equivalent label in it. Everything else is
+# pattern/regex based.
 NER_ENTITY_TYPES = {"PERSON", "LOCATION", "NRP"}
 DEFAULT_ENTITY_TYPES = [
     e.value for e in PIIEntityType if e.value not in NER_ENTITY_TYPES
@@ -53,15 +54,17 @@
 }
 
 # Shared label mapping for models using CoNLL-style PER/ORG/LOC/MISC labels
-# (e.g. dslim/bert-base-NER, Davlan/xlm-roberta-base-ner-hrl)
+# (IndicBERTv2 Naamapadam finetune, dslim/bert-base-NER, Davlan/xlm-roberta-*-ner-hrl)
 CONLL_NER_LABEL_MAPPING = {
     "PER": "PERSON",
     "ORG": "ORGANIZATION",
     "LOC": "LOCATION",
     "MISC": "MISC",
 }
 
-DEFAULT_TRANSFORMERS_MODEL = "dslim/bert-base-NER-uncased"
+# IndicBERTv2 fine-tuned on Hindi Naamapadam NER; transfers to other Indic scripts and
+# Latin-script names. Eval vs spaCy en_core_web_lg: docs/validators/pii-remover.md.
+DEFAULT_TRANSFORMERS_MODEL = "neuronsbyisshu/indicbert-hi-ner-naamapadam"
 DEFAULT_TRANSFORMERS_THRESHOLD = 0.7
 
 _NLP_ENGINE_CACHE: dict = {}
@@ -187,7 +190,7 @@ def __init__(
         self,
         entity_types=None,
         threshold: float | None = None,
-        nlp_engine_type: str = "spacy",
+        nlp_engine_type: str = "transformers",
         model_name: str | None = None,
         on_fail: Optional[Callable] = OnFailAction.FIX,
     ):

backend/app/tests/validators/test_pii_remover.py

@@ -141,11 +141,11 @@ def test_transformers_engine_accepts_custom_model(mock_presidio):
     assert v.model_name == "dslim/bert-base-NER-uncased"
 
 
-def test_config_defaults_to_spacy_engine(mock_presidio):
+def test_config_defaults_to_transformers_engine(mock_presidio):
     config = PIIRemoverSafetyValidatorConfig(type="pii_remover")
     v = config.build()
-    assert v.nlp_engine_type == "spacy"
-    assert v.model_name == "en_core_web_lg"
+    assert v.nlp_engine_type == "transformers"
+    assert v.model_name == DEFAULT_TRANSFORMERS_MODEL
 
 
 def test_cached_analyzer_registers_only_requested_indian_recognizers():

docs/DEVELOPER_GUIDE.md

@@ -471,7 +471,7 @@ Detects and anonymizes Personally Identifiable Information using [Microsoft Pres
 | Property | Value |
 |----------|-------|
 | Type | `pii_remover` |
-| Implementation | Local (Presidio + spaCy `en_core_web_lg`) |
+| Implementation | Local (Presidio + IndicBERTv2 NER `neuronsbyisshu/indicbert-hi-ner-naamapadam`) |
 | Speed | Fast–Medium (50–200ms depending on text length) |
 | Languages | English (NLP engine); India-specific patterns via regex |
 | `fix_value` | Yes — returns redacted text |
@@ -523,7 +523,7 @@ Output: "[REDACTED_PERSON_1] and [REDACTED_PERSON_2] called [REDACTED_PHONE_NUMB
 
 #### How It Works Internally
 
-1. **Analyzer initialization** — On first use, a spaCy `en_core_web_lg` NLP engine is built and cached globally. A Presidio `AnalyzerEngine` is constructed with this engine. India-specific recognizers are registered based on the configured `entity_types`.
+1. **Analyzer initialization** — On first use, a Presidio `AnalyzerEngine` with a HuggingFace NER recognizer (`neuronsbyisshu/indicbert-hi-ner-naamapadam`) is built and cached globally; spaCy `en_core_web_sm` handles tokenization. A Presidio `AnalyzerEngine` is constructed with this engine. India-specific recognizers are registered based on the configured `entity_types`.
 2. **Analyzer caching** — Analyzer instances are cached by the set of India-specific recognizers in use.
 3. **Analysis** — Presidio analyzes the text for the requested entity types. Each detection returns an entity type, position, and confidence score.
 4. **Threshold filtering** — Entities with confidence below the threshold are not redacted.

docs/architecture/ARCHITECTURE.md

@@ -126,7 +126,7 @@ Each request carries a list of `ValidatorConfigItem` objects (a Pydantic discrim
 | Validator | Type key | Source | Description |
 |---|---|---|---|
 | **Lexical Slur** | `uli_slur_match` | Local (custom) | Detects slurs from a curated bilingual (Hindi/English) CSV. Normalises text (emoji removal, unicode NFKC, `ftfy` encoding fix) before matching. Supports severity filtering: `low`/`medium`/`high`/`all`. Redacts matches with `[REDACTED_SLUR]`. |
-| **PII Remover** | `pii_remover` | Local (custom) | Uses Microsoft Presidio (`presidio_analyzer` + `presidio_anonymizer`) with a spaCy NLP engine (`en_core_web_lg`). Detects and redacts 15 entity types including India-specific identifiers (Aadhaar, PAN, Passport, Vehicle Registration, Voter ID). Replaces entities with labelled placeholders, e.g. `[REDACTED_PERSON_1]`. |
+| **PII Remover** | `pii_remover` | Local (custom) | Uses Microsoft Presidio (`presidio_analyzer` + `presidio_anonymizer`) with a HuggingFace NER recognizer (`neuronsbyisshu/indicbert-hi-ner-naamapadam`, IndicBERTv2 fine-tuned on Naamapadam) and spaCy `en_core_web_sm` for tokenization. Detects and redacts 15 entity types including India-specific identifiers (Aadhaar, PAN, Passport, Vehicle Registration, Voter ID). Replaces entities with labelled placeholders, e.g. `[REDACTED_PERSON_1]`. |
 | **Ban List** | `ban_list` | Hub (`guardrails/ban_list`) | Blocks exact words from a caller-supplied list or a stored `BanList` DB record (referenced by UUID). Words are fetched from the database in `_resolve_validator_configs()` before the guard runs. |
 | **Topic Relevance** | `topic_relevance` | Local (custom) | Wraps the Hub's `LLMCritic` validator. Given a freeform topic-scope description (stored as a `TopicRelevance` DB record or supplied inline), constructs a scoring prompt and calls an LLM (default: `gpt-4o-mini`) via LiteLLM. Scores 1–3; score ≤ 1 is a fail. Supports versioned prompt templates (`v1.md`–`v3.md`). |
 | **Profanity Free** | `profanity_free` | Hub (`guardrails/profanity_free`) | Detects general profanity in text. |
@@ -212,7 +212,7 @@ Tenant scope is never read from the query string or request body, so the depende
 | Database | PostgreSQL (psycopg3) |
 | Migrations | Alembic |
 | Validation engine | guardrails-ai (pinned to v0.10.0) |
-| NLP (PII) | spaCy `en_core_web_lg` via Presidio |
+| NLP (PII) | IndicBERTv2 NER `neuronsbyisshu/indicbert-hi-ner-naamapadam` via Presidio (spaCy `en_core_web_sm` tokenizer) |
 | ML (toxicity) | PyTorch CPU, `textdetox/xlmr-large-toxicity-classifier` via Transformers |
 | LLM calls | LiteLLM (used internally by `LLMCritic` / `TopicRelevance`) |
 | Package manager | uv |

docs/validators/pii-remover.md

@@ -9,9 +9,9 @@ Detects and anonymizes Personally Identifiable Information using [Microsoft Pres
 | Property | Value |
 |----------|-------|
 | Type | `pii_remover` |
-| Implementation | Local (Presidio + spaCy `en_core_web_lg`) |
+| Implementation | Local (Presidio + IndicBERTv2 NER `neuronsbyisshu/indicbert-hi-ner-naamapadam`) |
 | Speed | Fast–Medium (50–200ms depending on text length) |
-| Languages | English (NLP engine); India-specific patterns via regex |
+| Languages | English, Hinglish, Hindi, Marathi, Kannada, Telugu + other Indic scripts (NER); India-specific patterns via regex |
 | `fix_value` | Yes — returns redacted text |
 | Requires API key | No |
 
@@ -76,7 +76,7 @@ This format is:
 
 ## How It Works Internally
 
-1. **Analyzer initialization** — On first use, a spaCy `en_core_web_lg` NLP engine is built and cached globally. A Presidio `AnalyzerEngine` is constructed with this engine. India-specific recognizers (`InAadhaarRecognizer`, `InPanRecognizer`, etc.) are registered based on the configured `entity_types`.
+1. **Analyzer initialization** — On first use, a Presidio `AnalyzerEngine` is built and cached globally. NER runs through a HuggingFace token-classification model (`neuronsbyisshu/indicbert-hi-ner-naamapadam`, IndicBERTv2 fine-tuned on Naamapadam); spaCy `en_core_web_sm` is used only for tokenization. Set `nlp_engine_type="spacy"` on `PIIRemover` to fall back to spaCy `en_core_web_lg` NER. India-specific recognizers (`InAadhaarRecognizer`, `InPanRecognizer`, etc.) are registered based on the configured `entity_types`.
 
 2. **Analyzer caching** — Analyzer instances are cached by the set of India-specific recognizers in use. This means different `entity_types` combinations that share the same India recognizer set reuse the same analyzer.
 
@@ -140,13 +140,13 @@ Output: "What is the weather in Mumbai today?" (PassResult, no change)
 "Mumbai" or "India" may be detected as `LOCATION` and redacted, even in benign context. If you don't need location protection, omit `LOCATION` from `entity_types`.
 
 ### Short names and ambiguous tokens
-Very short or common names (e.g., "Ali", "Sam") may not be detected by spaCy NER with high confidence. They'll pass through if their score is below threshold.
+Lone Latin-script first names inside Indic text (e.g. "डॉ Aiswarya") are the weakest case for the NER model (~0.45 recall in eval). Multi-token names and native-script names do better.
 
 ### Non-English text
-The NLP engine is `en_core_web_lg` — English only. Hindi or other language names will not be detected by the NER component. India-specific regex recognizers (`IN_AADHAAR`, `IN_PAN`, etc.) work independently of language.
+The NER model is IndicBERTv2 fine-tuned on Hindi Naamapadam. In eval it detects names and locations in Hindi, Marathi, Kannada, Telugu, Bengali and Tamil scripts, plus English and Hinglish. `NRP` (nationality / religion / political group) is not a label in this model and will not be detected. India-specific regex recognizers (`IN_AADHAAR`, `IN_PAN`, etc.) work independently of language.
 
 ### First-run latency
-The first request loads the spaCy model into memory. This can take 2–5 seconds. Subsequent requests use the cached model and are fast.
+The first request loads the NER model into memory. This can take 5–15 seconds. Subsequent requests use the cached model and are fast.
 
 ---