diff --git a/examples/retrieval/README.md b/examples/retrieval/README.md new file mode 100644 index 0000000..b09e4f4 --- /dev/null +++ b/examples/retrieval/README.md @@ -0,0 +1,31 @@ +# Retrieval example + +Offline example that exercises the retrieval scorers (`recall_at_k`, `mrr`, `ndcg`) end to +end on a small labelled corpus and two deterministic retrievers. No models or network. + +## Files + +- `cases.jsonl`: 20 queries with graded relevance judgements (0 to 3). +- `retrievers.py`: two systems. + - `good_retriever` ranks the corpus by term overlap with the query. + - `noisy_retriever` is the same ranking with a deliberate regression: on queries with an + even number of characters, the top hit drops to rank 5. +- `suite.yaml`: recall@5, MRR@10 and nDCG@5, gating on nDCG@5. + +## Run + +```bash +for v in good noisy; do + uv run evalproof run examples/retrieval/suite.yaml \ + --system "python:retrievers:${v}_retriever" --out "runs/retrieval-$v.json" --quiet +done +uv run evalproof compare runs/retrieval-good.json runs/retrieval-noisy.json \ + --suite examples/retrieval/suite.yaml --gate; echo "exit $?" # 1: real regression +``` + +## What it shows + +Moving the top hit to rank 5 leaves recall@5 unchanged at 0.775, because recall ignores +order inside the cut-off. MRR falls from 1.000 to 0.642 and the nDCG@5 pass rate from +100% to 55%: nine of twenty queries regress, none improve, and the exact McNemar test gives +p = 0.0039, so the gate fails the change with exit 1. diff --git a/examples/retrieval/cases.jsonl b/examples/retrieval/cases.jsonl new file mode 100644 index 0000000..ac600c7 --- /dev/null +++ b/examples/retrieval/cases.jsonl @@ -0,0 +1,20 @@ +{"id": "q1", "input": "machine learning tutorial", "relevant": {"doc1": 3, "doc2": 2, "doc5": 1}} +{"id": "q2", "input": "python data science", "relevant": {"doc3": 3, "doc4": 1}} +{"id": "q3", "input": "neural networks deep learning", "relevant": {"doc1": 2, "doc6": 3}} +{"id": "q4", "input": "web development javascript", "relevant": {"doc7": 3, "doc8": 2}} +{"id": "q5", "input": "database sql query", "relevant": {"doc9": 3, "doc10": 1}} +{"id": "q6", "input": "cloud computing aws", "relevant": {"doc11": 3, "doc12": 2}} +{"id": "q7", "input": "mobile app android", "relevant": {"doc13": 3}} +{"id": "q8", "input": "security encryption", "relevant": {"doc14": 3, "doc15": 2}} +{"id": "q9", "input": "devops ci cd", "relevant": {"doc16": 3, "doc17": 1}} +{"id": "q10", "input": "natural language processing", "relevant": {"doc18": 3, "doc1": 1}} +{"id": "q11", "input": "computer vision opencv", "relevant": {"doc19": 3}} +{"id": "q12", "input": "reinforcement learning", "relevant": {"doc20": 3, "doc6": 2}} +{"id": "q13", "input": "frontend react", "relevant": {"doc7": 2, "doc21": 3}} +{"id": "q14", "input": "backend nodejs", "relevant": {"doc22": 3, "doc8": 1}} +{"id": "q15", "input": "data visualization matplotlib", "relevant": {"doc23": 3, "doc3": 1}} +{"id": "q16", "input": "kubernetes docker", "relevant": {"doc24": 3, "doc16": 2}} +{"id": "q17", "input": "git version control", "relevant": {"doc25": 3}} +{"id": "q18", "input": "api design rest", "relevant": {"doc26": 3, "doc22": 1}} +{"id": "q19", "input": "testing pytest", "relevant": {"doc27": 3, "doc17": 2}} +{"id": "q20", "input": "open source contribution", "relevant": {"doc28": 3, "doc25": 1}} diff --git a/examples/retrieval/retrievers.py b/examples/retrieval/retrievers.py new file mode 100644 index 0000000..37d9292 --- /dev/null +++ b/examples/retrieval/retrievers.py @@ -0,0 +1,62 @@ +"""Simple deterministic retrievers for the retrieval example. + +Documents are hardcoded for reproducibility. Relevance is graded 0-3. +""" + +from __future__ import annotations + +# Tiny corpus: id -> text +CORPUS = { + "doc1": "introduction to machine learning and neural networks", + "doc2": "machine learning basics tutorial for beginners", + "doc3": "python for data science and pandas", + "doc4": "data analysis with python", + "doc5": "supervised learning overview", + "doc6": "deep learning neural networks tutorial", + "doc7": "javascript web development fundamentals", + "doc8": "frontend javascript and html", + "doc9": "sql database queries tutorial", + "doc10": "relational databases explained", + "doc11": "aws cloud computing services", + "doc12": "introduction to cloud infrastructure", + "doc13": "android mobile app development", + "doc14": "encryption and security best practices", + "doc15": "cryptography basics", + "doc16": "devops ci cd pipelines", + "doc17": "continuous integration tools", + "doc18": "natural language processing techniques", + "doc19": "computer vision with opencv", + "doc20": "reinforcement learning algorithms", + "doc21": "react frontend framework", + "doc22": "nodejs backend development", + "doc23": "data visualization with matplotlib", + "doc24": "kubernetes container orchestration", + "doc25": "git version control tutorial", + "doc26": "rest api design principles", + "doc27": "pytest testing framework", + "doc28": "contributing to open source projects", +} + + +def _score(query: str, doc: str) -> int: + q_terms = set(query.lower().split()) + d_terms = set(doc.lower().split()) + return len(q_terms & d_terms) + + +def good_retriever(query: str) -> dict: + """Rank all documents by term overlap with the query.""" + ranked = sorted(CORPUS.items(), key=lambda item: -_score(query, item[1])) + retrieved = [doc_id for doc_id, _ in ranked[:10]] + return {"output": f"Retrieved {len(retrieved)} docs", "retrieved": retrieved} + + +def noisy_retriever(query: str) -> dict: + """good_retriever with a regression: on even-length queries the top hit drops to rank 5.""" + ranked = sorted(CORPUS.items(), key=lambda item: -_score(query, item[1])) + retrieved = [doc_id for doc_id, _ in ranked[:10]] + # Deterministic "noise": if query length even, move first doc to position 5 + if len(query) % 2 == 0 and len(retrieved) > 5: + top = retrieved.pop(0) + retrieved.insert(4, top) + return {"output": f"Retrieved {len(retrieved)} docs (noisy)", "retrieved": retrieved} diff --git a/examples/retrieval/suite.yaml b/examples/retrieval/suite.yaml new file mode 100644 index 0000000..ede8baf --- /dev/null +++ b/examples/retrieval/suite.yaml @@ -0,0 +1,26 @@ +# Retrieval example: exercises recall@k, MRR and nDCG with a tiny corpus. +# Offline, deterministic, suitable for CI. +name: retrieval-example +dataset: cases.jsonl + +system: + target: python:retrievers:good_retriever + +scorers: + - name: recall + type: recall_at_k + params: {k: 5} + - name: mrr + type: mrr + params: {k: 10} + - name: ndcg + type: ndcg + params: {k: 5, pass_threshold: 0.5} + +gate: + metric: ndcg + max_drop: 0.1 + alpha: 0.05 + min_cases: 15 + +cache_dir: .evalproof/cache