From ac69ea7872c0e1db2886c8d7a38cd5bcee38eb32 Mon Sep 17 00:00:00 2001
From: Harsh Raj Singhania
 <40535627+HarshRajSinghania@users.noreply.github.com>
Date: Tue, 13 Oct 2026 10:08:12 +0530
Subject: [PATCH 1/2] Add retrieval example suite exercising recall@k, MRR and
 nDCG

Addresses #2 by providing a small labelled corpus and two deterministic retrievers (good and noisy) so the retrieval scorers, paired comparison and gate can be exercised end-to-end.
---
 examples/retrieval/README.md     | 24 ++++++++++++
 examples/retrieval/cases.jsonl   | 20 ++++++++++
 examples/retrieval/retrievers.py | 64 ++++++++++++++++++++++++++++++++
 examples/retrieval/suite.yaml    | 26 +++++++++++++
 4 files changed, 134 insertions(+)
 create mode 100644 examples/retrieval/README.md
 create mode 100644 examples/retrieval/cases.jsonl
 create mode 100644 examples/retrieval/retrievers.py
 create mode 100644 examples/retrieval/suite.yaml

diff --git a/examples/retrieval/README.md b/examples/retrieval/README.md
new file mode 100644
index 0000000..cdc8b8f
--- /dev/null
+++ b/examples/retrieval/README.md
@@ -0,0 +1,24 @@
+# Retrieval example
+
+Offline example that exercises the retrieval scorers (`recall_at_k`, `mrr`, `ndcg`) end-to-end with a small labelled corpus and two deterministic retrievers.
+
+No models or network required. Suitable for CI.
+
+## Files
+
+- `cases.jsonl` — 20 queries with graded relevance judgements.
+- `retrievers.py` — two systems:
+  - `good_retriever`: ranks by simple term overlap with the query.
+  - `noisy_retriever`: same as good but randomly demotes the top relevant document on half the queries (simulating a regression).
+- `suite.yaml` — configures the good retriever and gates on nDCG@5.
+
+## Run
+
+```bash
+uv run evalproof run examples/retrieval/suite.yaml --out runs/retrieval-good.json --quiet
+# For the noisy (regressed) version, temporarily change the system target in suite.yaml
+# or run with a modified suite.
+uv run evalproof compare runs/retrieval-good.json runs/retrieval-noisy.json --suite examples/retrieval/suite.yaml --gate
+```
+
+The gate fails on a real drop in nDCG and passes when the change is only noise.
diff --git a/examples/retrieval/cases.jsonl b/examples/retrieval/cases.jsonl
new file mode 100644
index 0000000..ac600c7
--- /dev/null
+++ b/examples/retrieval/cases.jsonl
@@ -0,0 +1,20 @@
+{"id": "q1", "input": "machine learning tutorial", "relevant": {"doc1": 3, "doc2": 2, "doc5": 1}}
+{"id": "q2", "input": "python data science", "relevant": {"doc3": 3, "doc4": 1}}
+{"id": "q3", "input": "neural networks deep learning", "relevant": {"doc1": 2, "doc6": 3}}
+{"id": "q4", "input": "web development javascript", "relevant": {"doc7": 3, "doc8": 2}}
+{"id": "q5", "input": "database sql query", "relevant": {"doc9": 3, "doc10": 1}}
+{"id": "q6", "input": "cloud computing aws", "relevant": {"doc11": 3, "doc12": 2}}
+{"id": "q7", "input": "mobile app android", "relevant": {"doc13": 3}}
+{"id": "q8", "input": "security encryption", "relevant": {"doc14": 3, "doc15": 2}}
+{"id": "q9", "input": "devops ci cd", "relevant": {"doc16": 3, "doc17": 1}}
+{"id": "q10", "input": "natural language processing", "relevant": {"doc18": 3, "doc1": 1}}
+{"id": "q11", "input": "computer vision opencv", "relevant": {"doc19": 3}}
+{"id": "q12", "input": "reinforcement learning", "relevant": {"doc20": 3, "doc6": 2}}
+{"id": "q13", "input": "frontend react", "relevant": {"doc7": 2, "doc21": 3}}
+{"id": "q14", "input": "backend nodejs", "relevant": {"doc22": 3, "doc8": 1}}
+{"id": "q15", "input": "data visualization matplotlib", "relevant": {"doc23": 3, "doc3": 1}}
+{"id": "q16", "input": "kubernetes docker", "relevant": {"doc24": 3, "doc16": 2}}
+{"id": "q17", "input": "git version control", "relevant": {"doc25": 3}}
+{"id": "q18", "input": "api design rest", "relevant": {"doc26": 3, "doc22": 1}}
+{"id": "q19", "input": "testing pytest", "relevant": {"doc27": 3, "doc17": 2}}
+{"id": "q20", "input": "open source contribution", "relevant": {"doc28": 3, "doc25": 1}}
diff --git a/examples/retrieval/retrievers.py b/examples/retrieval/retrievers.py
new file mode 100644
index 0000000..5484e0c
--- /dev/null
+++ b/examples/retrieval/retrievers.py
@@ -0,0 +1,64 @@
+"""Simple deterministic retrievers for the retrieval example.
+
+Documents are hardcoded for reproducibility. Relevance is graded 0-3.
+"""
+
+from __future__ import annotations
+
+import random
+
+# Tiny corpus: id -> text
+CORPUS = {
+    "doc1": "introduction to machine learning and neural networks",
+    "doc2": "machine learning basics tutorial for beginners",
+    "doc3": "python for data science and pandas",
+    "doc4": "data analysis with python",
+    "doc5": "supervised learning overview",
+    "doc6": "deep learning neural networks tutorial",
+    "doc7": "javascript web development fundamentals",
+    "doc8": "frontend javascript and html",
+    "doc9": "sql database queries tutorial",
+    "doc10": "relational databases explained",
+    "doc11": "aws cloud computing services",
+    "doc12": "introduction to cloud infrastructure",
+    "doc13": "android mobile app development",
+    "doc14": "encryption and security best practices",
+    "doc15": "cryptography basics",
+    "doc16": "devops ci cd pipelines",
+    "doc17": "continuous integration tools",
+    "doc18": "natural language processing techniques",
+    "doc19": "computer vision with opencv",
+    "doc20": "reinforcement learning algorithms",
+    "doc21": "react frontend framework",
+    "doc22": "nodejs backend development",
+    "doc23": "data visualization with matplotlib",
+    "doc24": "kubernetes container orchestration",
+    "doc25": "git version control tutorial",
+    "doc26": "rest api design principles",
+    "doc27": "pytest testing framework",
+    "doc28": "contributing to open source projects",
+}
+
+
+def _score(query: str, doc: str) -> int:
+    q_terms = set(query.lower().split())
+    d_terms = set(doc.lower().split())
+    return len(q_terms & d_terms)
+
+
+def good_retriever(query: str) -> dict:
+    """Rank all documents by term overlap with the query."""
+    ranked = sorted(CORPUS.items(), key=lambda item: -_score(query, item[1]))
+    retrieved = [doc_id for doc_id, _ in ranked[:10]]
+    return {"output": f"Retrieved {len(retrieved)} docs", "retrieved": retrieved}
+
+
+def noisy_retriever(query: str) -> dict:
+    """Like good_retriever but demotes the top relevant doc on even-length queries (simulating regression)."""
+    ranked = sorted(CORPUS.items(), key=lambda item: -_score(query, item[1]))
+    retrieved = [doc_id for doc_id, _ in ranked[:10]]
+    # Deterministic "noise": if query length even, move first doc to position 5
+    if len(query) % 2 == 0 and len(retrieved) > 5:
+        top = retrieved.pop(0)
+        retrieved.insert(4, top)
+    return {"output": f"Retrieved {len(retrieved)} docs (noisy)", "retrieved": retrieved}
diff --git a/examples/retrieval/suite.yaml b/examples/retrieval/suite.yaml
new file mode 100644
index 0000000..ede8baf
--- /dev/null
+++ b/examples/retrieval/suite.yaml
@@ -0,0 +1,26 @@
+# Retrieval example: exercises recall@k, MRR and nDCG with a tiny corpus.
+# Offline, deterministic, suitable for CI.
+name: retrieval-example
+dataset: cases.jsonl
+
+system:
+  target: python:retrievers:good_retriever
+
+scorers:
+  - name: recall
+    type: recall_at_k
+    params: {k: 5}
+  - name: mrr
+    type: mrr
+    params: {k: 10}
+  - name: ndcg
+    type: ndcg
+    params: {k: 5, pass_threshold: 0.5}
+
+gate:
+  metric: ndcg
+  max_drop: 0.1
+  alpha: 0.05
+  min_cases: 15
+
+cache_dir: .evalproof/cache

From 05eb3d69e1796fc3a933616bac4e602c4285fdb3 Mon Sep 17 00:00:00 2001
From: Basant Bhattarai <32445173+technobasant@users.noreply.github.com>
Date: Fri, 16 Oct 2026 23:53:35 +0545
Subject: [PATCH 2/2] Fix lint and describe the retrieval example as it runs

Drop the unused random import and shorten the noisy retriever's docstring so
ruff passes. The README now says the regression is deterministic (the top hit
drops to rank 5 on even-length queries), switches retrievers with --system
instead of editing the suite, and states what the run shows: recall@5 is
unchanged, MRR and nDCG@5 fall, 9 of 20 queries regress, p = 0.0039, exit 1.
---
 examples/retrieval/README.md     | 33 +++++++++++++++++++-------------
 examples/retrieval/retrievers.py |  4 +---
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/examples/retrieval/README.md b/examples/retrieval/README.md
index cdc8b8f..b09e4f4 100644
--- a/examples/retrieval/README.md
+++ b/examples/retrieval/README.md
@@ -1,24 +1,31 @@
 # Retrieval example
 
-Offline example that exercises the retrieval scorers (`recall_at_k`, `mrr`, `ndcg`) end-to-end with a small labelled corpus and two deterministic retrievers.
-
-No models or network required. Suitable for CI.
+Offline example that exercises the retrieval scorers (`recall_at_k`, `mrr`, `ndcg`) end to
+end on a small labelled corpus and two deterministic retrievers. No models or network.
 
 ## Files
 
-- `cases.jsonl` — 20 queries with graded relevance judgements.
-- `retrievers.py` — two systems:
-  - `good_retriever`: ranks by simple term overlap with the query.
-  - `noisy_retriever`: same as good but randomly demotes the top relevant document on half the queries (simulating a regression).
-- `suite.yaml` — configures the good retriever and gates on nDCG@5.
+- `cases.jsonl`: 20 queries with graded relevance judgements (0 to 3).
+- `retrievers.py`: two systems.
+  - `good_retriever` ranks the corpus by term overlap with the query.
+  - `noisy_retriever` is the same ranking with a deliberate regression: on queries with an
+    even number of characters, the top hit drops to rank 5.
+- `suite.yaml`: recall@5, MRR@10 and nDCG@5, gating on nDCG@5.
 
 ## Run
 
 ```bash
-uv run evalproof run examples/retrieval/suite.yaml --out runs/retrieval-good.json --quiet
-# For the noisy (regressed) version, temporarily change the system target in suite.yaml
-# or run with a modified suite.
-uv run evalproof compare runs/retrieval-good.json runs/retrieval-noisy.json --suite examples/retrieval/suite.yaml --gate
+for v in good noisy; do
+  uv run evalproof run examples/retrieval/suite.yaml \
+    --system "python:retrievers:${v}_retriever" --out "runs/retrieval-$v.json" --quiet
+done
+uv run evalproof compare runs/retrieval-good.json runs/retrieval-noisy.json \
+  --suite examples/retrieval/suite.yaml --gate; echo "exit $?"   # 1: real regression
 ```
 
-The gate fails on a real drop in nDCG and passes when the change is only noise.
+## What it shows
+
+Moving the top hit to rank 5 leaves recall@5 unchanged at 0.775, because recall ignores
+order inside the cut-off. MRR falls from 1.000 to 0.642 and the nDCG@5 pass rate from
+100% to 55%: nine of twenty queries regress, none improve, and the exact McNemar test gives
+p = 0.0039, so the gate fails the change with exit 1.
diff --git a/examples/retrieval/retrievers.py b/examples/retrieval/retrievers.py
index 5484e0c..37d9292 100644
--- a/examples/retrieval/retrievers.py
+++ b/examples/retrieval/retrievers.py
@@ -5,8 +5,6 @@
 
 from __future__ import annotations
 
-import random
-
 # Tiny corpus: id -> text
 CORPUS = {
     "doc1": "introduction to machine learning and neural networks",
@@ -54,7 +52,7 @@ def good_retriever(query: str) -> dict:
 
 
 def noisy_retriever(query: str) -> dict:
-    """Like good_retriever but demotes the top relevant doc on even-length queries (simulating regression)."""
+    """good_retriever with a regression: on even-length queries the top hit drops to rank 5."""
     ranked = sorted(CORPUS.items(), key=lambda item: -_score(query, item[1]))
     retrieved = [doc_id for doc_id, _ in ranked[:10]]
     # Deterministic "noise": if query length even, move first doc to position 5