diff --git a/README.md b/README.md index 8475c25..5629b39 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Blog and experiment write-ups: [swinecoder101.github.io/hvec](https://swinecoder > connectors, seven codec families (`f32`, `int8`, `binary`, `binary-centred`, `pq`, `mrl` > and `mrl+`), `bench recall` against the f32 ranking and against relevance judgments, a > BEIR importer, `bench run` across the matrix with a payload stage, `report`, an LLM judge and an -> agent memory-loop benchmark. Four retrieval experiments are published. The answer-level +> agent memory-loop benchmark. Ten retrieval experiments are published. The answer-level > experiments (two chat models) are scripted and waiting on API access. Expect APIs and results to > change without notice. Contributions and experiments are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/ROADMAP.md b/ROADMAP.md index 7734718..e10ddd2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -181,6 +181,17 @@ decide how deep a hybrid re-score must go, and to decide where an adaptive code cached buffer is 5.1× faster than the original row scan. Every asymmetric codec costs about what f32 costs per vector. `binary-hamming` is 20× cheaper, and with an f32 re-score it reaches 95% recall@10 on FiQA about 19× faster than exact f32: the first speed claim hvec makes. +- [x] **Text-format experiment** (`experiments/2026-11-text-format`), with `ingest --whole-files` + (one chunk per file, layout kept). The same WikiBio records as prose, bullets, key-value, JSON + and a table row: format moves f32 nDCG@10 by up to 2×, and templated prose is last in every + context. The best format depends on the model (JSON on bge-small, a table row on MiniLM), not + the query type. There is no accuracy-for-speed trade-off: a `binary-hamming` scan plus 100 + f32 re-scores keeps 0.96–0.99 of the best format's nDCG at 1/19 of the time. Noise ÷ band + predicts the depth on a held-out format within a factor of 1.16. +- [ ] Embed one format and return another (match on key-value, hand the model prose): a payload + text per chunk, separate from the text embedded. +- [ ] Per-query scores from `bench recall`, so format and codec comparisons carry confidence + intervals. - [ ] Choose the codec automatically at ingest from noise ÷ band, then measure the re-score depth for a target recall on a sample of self-queries. The routing experiment showed that the curve chooses the codec well but predicts the depth badly. diff --git a/crates/hvec-bench/src/chunk.rs b/crates/hvec-bench/src/chunk.rs index 53b1990..5290e98 100644 --- a/crates/hvec-bench/src/chunk.rs +++ b/crates/hvec-bench/src/chunk.rs @@ -42,6 +42,17 @@ pub fn chunk_text(text: &str, opts: ChunkOptions) -> Vec { out } +/// The whole of `text` as one chunk, line breaks and spacing kept; only the +/// leading and trailing whitespace is trimmed. Empty input yields no chunks. +/// +/// For corpora whose files are already one record each, where the layout is +/// the thing under test: `chunk_text` would join every line with single spaces. +#[must_use] +pub fn chunk_whole(text: &str) -> Vec { + let t = text.trim(); + if t.is_empty() { Vec::new() } else { vec![t.to_owned()] } +} + #[cfg(test)] mod tests { use super::*; @@ -69,4 +80,11 @@ mod tests { let chunks = chunk_text("a b c d e", ChunkOptions { words: 2, overlap: 5 }); assert_eq!(chunks, vec!["a b", "b c", "c d", "d e"]); } + + #[test] + fn whole_keeps_layout_and_trims_the_ends() { + let text = "\n- name: ada\n- born: 1815\n\n"; + assert_eq!(chunk_whole(text), vec!["- name: ada\n- born: 1815"]); + assert!(chunk_whole(" \n\t").is_empty()); + } } diff --git a/crates/hvec-bench/src/lib.rs b/crates/hvec-bench/src/lib.rs index 002089c..b3f5481 100644 --- a/crates/hvec-bench/src/lib.rs +++ b/crates/hvec-bench/src/lib.rs @@ -14,7 +14,7 @@ pub mod record; pub mod report; pub mod rescore; -pub use chunk::{ChunkOptions, chunk_text}; +pub use chunk::{ChunkOptions, chunk_text, chunk_whole}; pub use judge::{JUDGE_PROMPT_VERSION, JUDGE_SYSTEM_PROMPT, Verdict, build_judge_prompt, parse_verdict}; pub use payload::{Payload, PayloadStats}; pub use prompt::{RAG_SYSTEM_PROMPT, build_user_prompt}; diff --git a/crates/hvec/src/cli.rs b/crates/hvec/src/cli.rs index 79f72b3..77a6ac8 100644 --- a/crates/hvec/src/cli.rs +++ b/crates/hvec/src/cli.rs @@ -416,6 +416,10 @@ pub struct IngestArgs { /// Overlapping words between consecutive chunks. #[arg(long, default_value_t = 40)] pub overlap_words: usize, + /// Store each file as one chunk, verbatim (line breaks kept), instead of word windows. + /// For corpora of one record per file, where the layout is under test. + #[arg(long, conflicts_with_all = ["chunk_words", "overlap_words"])] + pub whole_files: bool, /// File extensions to include when walking directories. #[arg(long, value_delimiter = ',', default_value = "md,txt,markdown,rst")] pub ext: Vec, diff --git a/crates/hvec/src/commands/ingest.rs b/crates/hvec/src/commands/ingest.rs index 240bb19..b287bb8 100644 --- a/crates/hvec/src/commands/ingest.rs +++ b/crates/hvec/src/commands/ingest.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use std::time::Instant; use anyhow::{Context as _, Result, bail}; -use hvec_bench::{ChunkOptions, chunk_text}; +use hvec_bench::{ChunkOptions, chunk_text, chunk_whole}; use hvec_connect::Embedder; use hvec_core::{Metric, codec_by_name}; use hvec_store::{NewChunk, NewCollection}; @@ -72,10 +72,10 @@ pub async fn run(config: &Option, args: IngestArgs) -> Result<()> { codec = existing.codec()?; } - let opts = ChunkOptions { + let opts = (!args.whole_files).then_some(ChunkOptions { words: args.chunk_words, overlap: args.overlap_words, - }; + }); let batch = args.batch.max(1); let started = Instant::now(); let mut total_chunks = 0usize; @@ -189,11 +189,15 @@ pub async fn run(config: &Option, args: IngestArgs) -> Result<()> { Ok(()) } -async fn read_chunks(file: &Path, opts: ChunkOptions) -> Result> { +/// Word windows under `opts`, or the whole file as one chunk when `opts` is `None`. +async fn read_chunks(file: &Path, opts: Option) -> Result> { let text = tokio::fs::read_to_string(file) .await .with_context(|| format!("reading {}", file.display()))?; - Ok(chunk_text(&text, opts)) + Ok(match opts { + Some(opts) => chunk_text(&text, opts), + None => chunk_whole(&text), + }) } async fn embed_all(embedder: &dyn Embedder, pieces: &[String], batch: usize) -> Result>> { diff --git a/crates/hvec/src/commands/shell.rs b/crates/hvec/src/commands/shell.rs index adf8c20..1220416 100644 --- a/crates/hvec/src/commands/shell.rs +++ b/crates/hvec/src/commands/shell.rs @@ -294,6 +294,7 @@ impl Shell { .unwrap_or_else(|| "f32".to_owned()), chunk_words: 200, overlap_words: 40, + whole_files: false, ext: ["md", "txt", "markdown", "rst"] .iter() .map(|s| (*s).to_owned()) diff --git a/crates/hvec/tests/pipeline.rs b/crates/hvec/tests/pipeline.rs index 6dce31c..ddd7b58 100644 --- a/crates/hvec/tests/pipeline.rs +++ b/crates/hvec/tests/pipeline.rs @@ -1247,3 +1247,45 @@ fn bench_scan_times_paths_and_codecs_and_keeps_f32_exact() { assert!(codecs.iter().all(|c| c["ms_per_query"].as_f64().unwrap() >= 0.0)); assert_eq!(r["bandwidth"].as_array().unwrap().len(), 2); } + +#[test] +fn whole_files_stores_one_chunk_per_file() { + let env = Env::new(); + // 450 words: three overlapping word windows by default, one chunk with --whole-files. + let long = (0..150) + .map(|i| format!("- field{i}: value{i}")) + .collect::>() + .join("\n"); + let path = env.write_doc("record.md", &long); + let path = path.to_str().unwrap(); + + let out = env.run(&["ingest", path, "--collection", "windows"]); + assert_ok(&out, "ingest windows"); + assert!( + stdout(&out).contains("ingested 3 chunks from 1 files"), + "{}", + stdout(&out) + ); + + let out = env.run(&["ingest", path, "--collection", "whole", "--whole-files"]); + assert_ok(&out, "ingest whole"); + assert!( + stdout(&out).contains("ingested 1 chunks from 1 files"), + "{}", + stdout(&out) + ); + + let out = env.run(&[ + "ingest", + path, + "--collection", + "both", + "--whole-files", + "--chunk-words", + "50", + ]); + assert!( + !out.status.success(), + "--whole-files with --chunk-words should be refused" + ); +} diff --git a/docs/_data/references.yml b/docs/_data/references.yml index 3d9c23a..e6a9f84 100644 --- a/docs/_data/references.yml +++ b/docs/_data/references.yml @@ -314,6 +314,14 @@ year: 2024 url: https://arxiv.org/abs/2407.02883 note: Code retrieval datasets in BEIR's layout. The source of hvec's CoSQA copy. +- key: lebret2016wikibio + topic: Embeddings and RAG + authors: Rémi Lebret, David Grangier, Michael Auli + title: Neural Text Generation from Structured Data with Application to the Biography Domain + venue: Conference on Empirical Methods in Natural Language Processing (EMNLP) + year: 2016 + url: https://arxiv.org/abs/1603.07771 + note: WikiBio, Wikipedia infoboxes paired with each article's opening paragraph. The source of the text-format experiment's records. - key: morris2023vec2text topic: Embeddings and RAG diff --git a/docs/_experiments/text-format.md b/docs/_experiments/text-format.md new file mode 100644 index 0000000..1d19511 --- /dev/null +++ b/docs/_experiments/text-format.md @@ -0,0 +1,90 @@ +--- +title: "Text format: is there a best layout for a record?" +date: 2026-11-05 21:00:00 +0000 +result: "With the facts held fixed, layout moves f32 nDCG@10 by up to 2×, and templated prose is last in every context. The best format depends on the embedding model (JSON on bge-small, a table row on MiniLM), not the query type. Its advantage survives compression: a binary-hamming scan plus 100 f32 re-scores keeps 0.96–0.99 of it at 1/19 of the time." +references: [lebret2016wikibio, xiao2023bge, reimers2019sbert, charikar2002simhash, jegou2011pq] +--- + +*Full write-up, raw results and a one-command reproduction are in the repository:* +[experiments/2026-11-text-format](https://github.com/SwineCoder101/hvec/tree/main/experiments/2026-11-text-format). +Follow-up to [neighbour spacing]({{ "/experiments/neighbour-spacing/" | relative_url }}) and +[zero-copy scan]({{ "/experiments/zero-copy-scan/" | relative_url }}). + +Every earlier experiment held the text fixed and changed the codec. This one holds the facts +fixed and changes only their layout. The question: for a given embedding model and kind of +query, which format and fast path give the best nDCG@10 within a time budget? + +**Format can only change speed indirectly.** The scan never sees text. With one record per chunk, +every format has the same vector count and the same scan cost. Format can only move time through +neighbour spacing, which sets how many f32 re-scores a compressed scan needs. It also changes how +many tokens a retrieved record costs the chat model. + +**Hypothesis.** +1. The best format depends on the query type. Field-labelled layouts (key-value, JSON, table) + win attribute lookups at f32. But they repeat the same keys in every record, which packs + neighbours closer, so they lose at a tight time budget. +2. Format acts on compressed retrieval only through f32 nDCG and noise ÷ band. The depth curve + fitted on some formats predicts the depth on a held-out one. + +**Setup.** 20,000 WikiBio [1] biographies, each written seven ways. +- **Five controlled formats** with identical facts, the cleaned infobox: templated prose, bullets, + key-value, JSON, and a two-row table. +- **Two references:** the article's own opening paragraph, and a key-value header above it. + +There are 500 lookup queries ("who was born on 25 april 1985 and played as guard / offensive +tackle?") and 500 descriptive ones ("people who died in columbus , ohio"), the same for every +format and judged from the infobox. Each record is one chunk, verbatim, using the new +`ingest --whole-files`. The models are bge-small [2] and MiniLM [3], 384 dims. The fast path is +a `binary-hamming` scan [4] (`pq48` [5] for comparison) with an f32 re-score, timed from measured +per-vector costs on one thread. Apple M4 Pro. + +**Result.** Hypothesis 1 is refuted on both counts. Hypothesis 2 is confirmed. + +Exact f32 nDCG@10: + +| format | tokens | bge lookup | bge descriptive | MiniLM lookup | MiniLM descriptive | +|---|---:|---:|---:|---:|---:| +| templated prose (baseline) | 114 | 0.370 | 0.174 | 0.179 | 0.130 | +| bullets | 75 | 0.402 | 0.180 | 0.336 | 0.161 | +| key-value | 74 | 0.435 | 0.191 | 0.276 | 0.143 | +| JSON | 114 | **0.441** | **0.224** | 0.240 | 0.142 | +| table row | 119 | 0.415 | 0.211 | **0.353** | **0.181** | +| Wikipedia paragraph (reference) | 74 | 0.181 | 0.113 | 0.098 | 0.091 | +| key-value header + paragraph (reference) | 147 | 0.370 | 0.158 | 0.208 | 0.136 | + +Every structured layout beats prose in every context. The model, not the query type, decides +which one wins: JSON on bge-small, the table row on MiniLM, where JSON is only fourth. + +The predicted speed penalty is too small to matter. Within a context, format changes the +neighbour band by at most 1.3×, while the model changes it by about 2×. A `binary-hamming` +scan plus 100 f32 re-scores takes 0.22 ms against about 4.1 ms for exact f32 over 20,000 +vectors. + +| context | best format | f32 nDCG | hybrid nDCG (share kept) | prose f32 nDCG | prose hybrid (share kept) | +|---|---|---:|---:|---:|---:| +| bge-small, lookup | JSON | 0.441 | 0.423 (0.959) | 0.370 | 0.335 (0.906) | +| bge-small, descriptive | JSON | 0.224 | 0.219 (0.977) | 0.174 | 0.169 (0.976) | +| MiniLM, lookup | table | 0.353 | 0.349 (0.987) | 0.179 | 0.170 (0.948) | +| MiniLM, descriptive | table | 0.181 | 0.179 (0.992) | 0.130 | 0.126 (0.976) | + +The fast path on the best format beats exact f32 on prose in every context, at 1/19 of the time. + +With one format left out at a time, noise ÷ band ranks the re-score depth at Spearman 0.985. It +predicts the held-out format's depth for 95% recall within a factor of 1.16, tighter than across +corpora (1.25–1.41). Together with f32 nDCG, it predicts a hybrid's nDCG on the held-out format +within 0.003–0.007. + +**Reading.** Choose the format for the model. Measure f32 nDCG on a sample in two or three +layouts; the compressed cost follows from spacing. Key-value and bullets are about 35% cheaper in +tokens than JSON and table, and within 0.02 of the best on lookups. Putting a key-value header +above a prose document doubles its nDCG. + +Caveats: +- With 500 queries per type, gaps under about 0.02 are ties. +- The queries are templated, and the judgments match infobox values exactly, so absolute nDCG is + low. +- The domain is biographies only. +- Only retrieval is measured, not answers. +- Hybrid times are computed from per-vector costs, with the candidates in memory. + +{% include references.html %} diff --git a/experiments/2026-11-text-format/README.md b/experiments/2026-11-text-format/README.md new file mode 100644 index 0000000..73c2344 --- /dev/null +++ b/experiments/2026-11-text-format/README.md @@ -0,0 +1,216 @@ +# Text format: is there a best layout for a record, given the model and the time budget? + +Every earlier experiment held the text fixed and changed the codec. Neighbour spacing found +that the kind of content changes what a codec costs, but only through the score spacing of a +corpus's neighbours. This experiment holds the content fixed and changes only its layout: the +same facts written as prose, bullets, key-value pairs, JSON or a table row. + +**The question.** For a given context (an embedding model and a kind of query), which format and +fast path (first-stage codec, re-score depth) give the best nDCG@10 within a fixed time per query? + +**Format can only change speed indirectly.** The scan never sees text. With one record per chunk, +every format has the same number of vectors and so the same scan cost per codec. Format moves +time in two ways: +1. **Spacing.** A format that spreads neighbours further apart lowers a codec's noise ÷ band, so a + `binary-hamming` scan needs fewer f32 re-scores for the same recall + ([two-stage re-scoring](../2026-11-two-stage-rescoring/)). +2. **Tokens.** What a retrieved record costs the chat model to read. Measured as size per record; + no model is called. + +**Hypothesis.** +1. The best format depends on the query type. Templated, field-labelled layouts (key-value, + JSON, table) match attribute lookups best at f32. They also repeat the same keys in every + record, which should pack neighbours closer together. Their compressed scans would then need + deeper re-scores, so a format can win at f32 and lose at a tight time budget. +2. Format acts on compressed retrieval only through two numbers: f32 nDCG (how well the query + matches the text) and noise ÷ band (how well the ranking survives the codec). If so, the + noise ÷ band curve fitted on some formats predicts the re-score depth on a held-out format + about as well as it did across corpora (within a factor of 1.25–1.41). + +**Result.** Hypothesis 1 is refuted on both counts. Hypothesis 2 is confirmed. +- **Format matters more than any codec here.** With the facts held fixed, it moves exact f32 + nDCG@10 by up to 2×: on MiniLM lookups, templated prose scores 0.179 and a table row 0.353. + Templated prose is the worst of the five controlled formats in all four contexts. +- **The best format depends on the embedding model, not the query type.** On bge-small, JSON is + best for both query types (0.441 lookup, 0.224 descriptive), with key-value and table close + behind on lookups. On MiniLM, the table row is best for both (0.353, 0.181), and JSON is only + fourth (0.240 on lookups). +- **There is no accuracy-for-speed trade-off.** Within a context, format changes the neighbour + band by at most 1.3×, against 1.7–2× between the two models. A `binary-hamming` scan plus 100 f32 + re-scores (0.22 ms against 4.1 ms for exact f32, 19× less) keeps 0.96–0.99 of the winning + format's f32 nDCG, and 0.91–0.98 of prose's. So the fast path on the best format beats + exact f32 on prose in every context, at 1/19 of the time. +- **Noise ÷ band predicts depth on a held-out format** within a factor of 1.16 at 95% recall + (Spearman 0.985). That is tighter than across corpora. Together with f32 nDCG, it predicts a + hybrid's nDCG on a held-out format within 0.003–0.007. In practice this means you can pick a + format by measuring f32 nDCG on a sample, and the codec cost follows from spacing. + +## Setup + +| | | +|---|---| +| Corpus | WikiBio [1] test split (`thoughtworks/wiki_bio` on Hugging Face, parquet, 72,831 biographies): infobox plus the article's opening paragraph. 20,000 records sampled with seed 2026 | +| Cleaning | Image, caption, website and update-stamp fields dropped. Values over 25 words or empty or "unknown" dropped. At most 16 fields per record (median 12). PTB bracket tokens (`-lrb-`) restored. Text stays lowercased, as released | +| Formats | Seven, below. One record per chunk, verbatim, line breaks kept (`ingest --whole-files`) | +| Queries | 500 `lookup` and 500 `descriptive` queries from the infobox, the same for every format | +| Embedding models | `bge-small-en-v1.5` and `all-MiniLM-L6-v2`, 384 dims, fastembed on CPU, no query instruction | +| Measurements | `bench recall` (f32 nDCG@10, `neighbour_band`, `rank_noise` for `int8`, `binary`, `binary-hamming`, `pq48`). `bench rescore` (`binary-hamming` and `pq48` first stages, f32 second stage, depths 10–1000 and adaptive margins). `bench scan` (one thread, cost per vector) | +| Time per query | Scan + mean re-score depth × the f32 cost per vector, on one thread. The candidates are assumed to be in memory, as in [zero-copy scan](../2026-11-zero-copy-scan/) | +| Machine | Apple M4 Pro, 26 GB, release build | +| Code | hvec commit `cb5ffd1` | +| Command | `./run.sh` (about an hour, mostly embedding). Results in `results/`: `recall/`, `rescore/`, `scan/`, `stats.json`, `examples.md`, `summary.txt` | + +### Formats + +`results/examples.md` shows one record in all seven. The first five hold exactly the same facts +(the cleaned infobox). Only the layout differs. + +| format | layout | approx. tokens / record | +|---|---|---:| +| `paragraph` | one templated sentence per field: "ruth born's birth place is bay city , michigan." | 114 | +| `bullets` | `- birth place: bay city , michigan`, one field per line | 75 | +| `kv` | `name: ruth born; birth date: 08 august 1925; ...` on one line | 74 | +| `json` | `{"name": "ruth born", "birth_date": "08 august 1925", ...}` | 114 | +| `table` | a two-row Markdown table, header then values | 119 | +| `wiki` | the article's own opening paragraph, up to 150 words. Different facts: the natural-prose reference, not a controlled comparison | 74 | +| `header-prose` | `kv` as a header, a blank line, then `wiki`. More facts than either | 147 | + +Token counts are a word-and-punctuation split from `stats.json`, not a model's tokenizer. + +### Queries + +- **`lookup`**: two of a record's attributes, for example "who was born on 25 april 1985 and + played as guard / offensive tackle?". Relevant to every record with both values, and kept only + when that is at most three records. +- **`descriptive`**: one attribute value shared by 5 to 50 records, for example "people who died + in columbus , ohio". Relevant to all of them (5,860 judgments in all). + +The queries use their own phrasing ("was born in", "played as"), not the field labels the +documents use ("birth place", "position"). That way no infobox format gets the query's words for +free. + +## Results + +### Exact f32 nDCG@10 by format + +| format | tokens | bge lookup | bge descriptive | MiniLM lookup | MiniLM descriptive | +|---|---:|---:|---:|---:|---:| +| `paragraph` (baseline prose) | 114 | 0.370 | 0.174 | 0.179 | 0.130 | +| `bullets` | 75 | 0.402 | 0.180 | 0.336 | 0.161 | +| `kv` | 74 | 0.435 | 0.191 | 0.276 | 0.143 | +| `json` | 114 | **0.441** | **0.224** | 0.240 | 0.142 | +| `table` | 119 | 0.415 | 0.211 | **0.353** | **0.181** | +| `wiki` (reference) | 74 | 0.181 | 0.113 | 0.098 | 0.091 | +| `header-prose` (reference) | 147 | 0.370 | 0.158 | 0.208 | 0.136 | + +Every structured layout beats templated prose in every context, by 4% (bullets, bge +descriptive) to 97% (table, MiniLM lookup). The model decides which one wins. JSON is best on +bge-small and middling on MiniLM. The table row is best on MiniLM and within 0.03 of the top on +bge-small. + +### Spacing and re-score depth + +| | bge-small | MiniLM | +|---|---|---| +| neighbour band, range across formats | 0.0082–0.0116 | 0.0157–0.0213 | +| `binary-hamming` noise ÷ band | 4.47–5.39 | 2.12–2.80 | +| `pq48` noise ÷ band | 1.08–1.41 | 0.87–1.18 | +| f32 re-scores for 95% recall@10 behind `binary-hamming` | 476–966 | 225–441 | + +Within a context (one model and one query type), format changes the band by at most 1.3×. The +model changes it by about 2×. The +spacing penalty predicted for structured layouts appears once: JSON on bge-small has the tightest +band (0.0082 on lookups) and the highest `binary-hamming` noise ÷ band (5.39). It is too small +to matter, though. JSON still needs fewer re-scores for 95% recall than prose (760 against 966). + +### The fast path: `binary-hamming` scan, 100 f32 re-scores (0.22 ms against about 4.1 ms) + +| context | best format | its f32 nDCG | its hybrid nDCG (share kept) | `paragraph` f32 nDCG | `paragraph` hybrid (share kept) | +|---|---|---:|---:|---:|---:| +| bge-small, lookup | `json` | 0.441 | 0.423 (0.959) | 0.370 | 0.335 (0.906) | +| bge-small, descriptive | `json` | 0.224 | 0.219 (0.977) | 0.174 | 0.169 (0.976) | +| MiniLM, lookup | `table` | 0.353 | 0.349 (0.987) | 0.179 | 0.170 (0.948) | +| MiniLM, descriptive | `table` | 0.181 | 0.179 (0.992) | 0.130 | 0.126 (0.976) | + +Scan cost does not depend on format. Across all 14 collections an exact f32 scan takes +3.87–4.25 ms and a `binary-hamming` scan 0.19–0.22 ms (20,000 vectors, one thread). At 100 +candidates, agreement with the exact top 10 is only 0.64–0.87, but most of the neighbours lost +are not relevant, so nDCG barely moves. In every context, the best format and operating point +within 10% of an exact scan's time (0.41 ms) is the same as within 100%: JSON on bge-small and +the table row on MiniLM. + +### Cross-format checks, leaving one format out at a time + +56 (format, model, query type, first stage) points. + +| target recall@10 | Spearman ρ (noise ÷ band, depth) | fitted depth | held-out format, mean factor error | +|---|---:|---|---:| +| 90% | 0.985 | 3.08·k·r^1.70 | 1.14 | +| 95% | 0.985 | 4.74·k·r^1.83 | 1.16 | +| 99% | 0.968 | 10.59·k·r^1.98 | 1.24 | + +The share of f32 nDCG a hybrid keeps, predicted from log noise ÷ band and fitted without the +held-out format, has these errors: + +| N | error | f32 nDCG × the mean share kept | +|---:|---:|---:| +| 10 | 0.0067 | 0.0181 | +| 30 | 0.0047 | 0.0117 | +| 100 | 0.0026 | 0.0048 | + +## Discussion + +**The answer to the question is mostly "pick the format for the model".** The fast path costs +every format about the same, so the format that wins at f32 also wins at a tenth of the time. You +can choose a format by embedding a sample in two or three layouts and measuring f32 nDCG. The +compressed cost then follows from noise ÷ band, which held on formats it was not fitted on. + +**Prose is the weakest layout for attribute queries, even with identical facts.** The templated +sentences repeat the person's name in every sentence ("ruth born's ... is ..."), which may dilute +the fields. That makes `paragraph` a pessimistic stand-in for prose. The real Wikipedia paragraph +(`wiki`) is lower still, but it lacks many of the facts asked about. + +**A metadata header is a cheap fix for prose documents.** Putting the key-value fields above the +Wikipedia paragraph doubles its nDCG (bge-small lookups 0.181 → 0.370, MiniLM 0.098 → 0.208). The +record alone still scores higher, and the header costs the most tokens (147). + +**Tokens: key-value and bullets are about 35% cheaper than JSON and table** (74–75 against +114–119 per record). On bge-small lookups, key-value is within 0.006 of JSON. On MiniLM lookups, +bullets are within 0.017 of the table. For descriptive queries the cheaper format gives up more +(0.033 and 0.020). + +**Why JSON on bge-small and a table on MiniLM?** Not tested. Finding out would take an ablation, +for example JSON without its quotes and braces, or a table without its pipes. + +**The depth curve is steeper here than across corpora** (r^1.83 at 95% against r^1.29). Most of +the ratio's range in this experiment comes from the model and the codec, not the format. So the +check shows that the curve carries across formats, not that its exponent is universal. + +**Caveats.** +- **No confidence intervals.** With 500 queries per type, treat gaps under about 0.02 nDCG as + ties: JSON against key-value on bge-small lookups, table against bullets on MiniLM lookups. + The per-query scores are not saved, so there is no bootstrap. +- **Absolute nDCG is low** (0.09–0.44). Three causes: the queries are templated, the + judgments match infobox values exactly (so "new york city" and "new york city , new york , + u.s." are separate categories), and no query instruction is used for bge-small. All seven + formats share these, but the ratios between formats may not hold for realistic queries. +- **`wiki` and `header-prose` are not controlled comparisons.** Their facts differ from the + infobox's. +- **One domain.** Biographies are short, field-heavy records. Long documents, code, or tables + with many rows may rank the formats differently. +- **Retrieval only.** Whether a format helps a chat model answer needs a chat model and a judge, + as Experiments 1 and 3 do. +- **Time is computed from per-vector costs**, not timed end to end, and assumes the f32 + candidates are in memory. + +**Next.** +- Embed one format and return another (for example, match on `kv` and hand the model `wiki`). + Retrieval and reading may want different layouts. `ingest` stores the text it embeds, so this + needs a separate payload text per chunk. +- A second structured source, and realistic queries, to see whether "the model decides" holds. +- Save per-query scores in `bench recall` so format comparisons can carry confidence intervals. + +## References + +1. Lebret, Grangier and Auli, "Neural Text Generation from Structured Data with Application to + the Biography Domain", EMNLP 2016. [arXiv:1603.07771](https://arxiv.org/abs/1603.07771) diff --git a/experiments/2026-11-text-format/results/config.toml b/experiments/2026-11-text-format/results/config.toml new file mode 100644 index 0000000..74c511e --- /dev/null +++ b/experiments/2026-11-text-format/results/config.toml @@ -0,0 +1,17 @@ +db_path = "/Users/liam/dev/hvec/experiments/2026-11-text-format/work/hvec.db" +default_chat = "anthropic" +default_embedder = "bge-small" +metric = "cosine" + +[chat.anthropic] +provider = "anthropic" +model = "claude-opus-5-5" +api_key_env = "ANTHROPIC_API_KEY" + +[embed.bge-small] +provider = "local" +model = "BGESmallENV15" + +[embed.minilm] +provider = "local" +model = "AllMiniLML6V2" diff --git a/experiments/2026-11-text-format/results/examples.md b/experiments/2026-11-text-format/results/examples.md new file mode 100644 index 0000000..987eef9 --- /dev/null +++ b/experiments/2026-11-text-format/results/examples.md @@ -0,0 +1,62 @@ +# One record (wb05959) in every format + +## paragraph + +``` +ruth born. ruth born's throws is right. ruth born's birth date is 08 august 1925. ruth born's position is pitcher. ruth born's birth place is bay city , michigan. ruth born's bats is right. ruth born's team is all-american girls professional baseball league. +``` + +## bullets + +``` +- name: ruth born +- throws: right +- birth date: 08 august 1925 +- position: pitcher +- birth place: bay city , michigan +- bats: right +- team: all-american girls professional baseball league +``` + +## kv + +``` +name: ruth born; throws: right; birth date: 08 august 1925; position: pitcher; birth place: bay city , michigan; bats: right; team: all-american girls professional baseball league +``` + +## json + +``` +{"name": "ruth born", "throws": "right", "birth_date": "08 august 1925", "position": "pitcher", "birth_place": "bay city , michigan", "bats": "right", "team": "all-american girls professional baseball league"} +``` + +## table + +``` +| name | throws | birth date | position | birth place | bats | team | +|---|---|---|---|---|---|---| +| ruth born | right | 08 august 1925 | pitcher | bay city , michigan | right | all-american girls professional baseball league | +``` + +## wiki + +``` +ruth born ( born august 8 , 1925 ) is a former pitcher who played in the all-american girls professional baseball league . listed at , 125 lb , she batted and threw right-handed . ruth born was one of the original south bend blue sox founding members of the all-american girls professional baseball league for its inaugural season in 1943 . a native of bay city , michigan , born grew up playing sandlot ball with her neighborhood kids at an early age , most of them boys , but did not start participating in an organized league until she was 12 years old , playing at school and in park softball leagues . at the age of 13 , she put on her first uniform with a moose lodge team . +``` + +## header-prose + +``` +name: ruth born; throws: right; birth date: 08 august 1925; position: pitcher; birth place: bay city , michigan; bats: right; team: all-american girls professional baseball league + +ruth born ( born august 8 , 1925 ) is a former pitcher who played in the all-american girls professional baseball league . listed at , 125 lb , she batted and threw right-handed . ruth born was one of the original south bend blue sox founding members of the all-american girls professional baseball league for its inaugural season in 1943 . a native of bay city , michigan , born grew up playing sandlot ball with her neighborhood kids at an early age , most of them boys , but did not start participating in an organized league until she was 12 years old , playing at school and in park softball leagues . at the age of 13 , she put on her first uniform with a moose lodge team . +``` + +## Queries + +- lookup: who was born on 3 february 1956 and played for medellín atlético nacional? +- lookup: who was born on 25 april 1985 and played as guard / offensive tackle? +- lookup: who was born on 04 october 1942 in hollymount , near claremorris , county mayo , ireland? +- descriptive: people who died in columbus , ohio +- descriptive: people who died in new york city , new york , u.s. +- descriptive: people who played as pivot diff --git a/experiments/2026-11-text-format/results/machine.txt b/experiments/2026-11-text-format/results/machine.txt new file mode 100644 index 0000000..db304bd --- /dev/null +++ b/experiments/2026-11-text-format/results/machine.txt @@ -0,0 +1,3 @@ +hvec cb5ffd1 +Apple M4 Pro +memory: 26 GB diff --git a/experiments/2026-11-text-format/results/recall/bullets-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/bullets-bge-small-descriptive.json new file mode 100644 index 0000000..6212395 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/bullets-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 3, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.17969501434099042, + "doc_recall_at_k": 0.15695345841272806, + "boundary_gap": 0.0009827613830566406, + "neighbour_band": 0.01013973355293274, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9769999999999983, + "top1_agreement": 0.974, + "mean_abs_score_error": 0.0004987362382352352, + "max_abs_score_error": 0.0035058557987213135, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2501, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.17882628925827088, + "doc_recall_at_k": 0.1561142729454467, + "boundary_gap": 0.0009827613830566406, + "neighbour_band": 0.01013973355293274, + "rank_noise": 0.0005157020294026582 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.40840000000000015, + "top1_agreement": 0.31, + "mean_abs_score_error": 0.2853380107817233, + "max_abs_score_error": 0.44112491607666016, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2261, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.15168815299897204, + "doc_recall_at_k": 0.12595201442081372, + "boundary_gap": 0.0009827613830566406, + "neighbour_band": 0.01013973355293274, + "rank_noise": 0.021664264176572003 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.2842, + "top1_agreement": 0.2, + "mean_abs_score_error": 0.258451657642594, + "max_abs_score_error": 0.6197579503059387, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 1999, + "labelled_queries": 500, + "ndcg_at_k": 0.12166128663735999, + "doc_recall_at_k": 0.0981435737515116, + "boundary_gap": 0.0009827613830566406, + "neighbour_band": 0.01013973355293274, + "rank_noise": 0.04885937247046683 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5419999999999995, + "top1_agreement": 0.452, + "mean_abs_score_error": 0.03525727904258966, + "max_abs_score_error": 0.147859126329422, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 350, + "scan_ms": 1995, + "baseline_scan_ms": 1995, + "labelled_queries": 500, + "ndcg_at_k": 0.15724523921045508, + "doc_recall_at_k": 0.13266368133956502, + "boundary_gap": 0.0009827613830566406, + "neighbour_band": 0.01013973355293274, + "rank_noise": 0.012701630639716221 + } +] diff --git a/experiments/2026-11-text-format/results/recall/bullets-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/bullets-bge-small-lookup.json new file mode 100644 index 0000000..8da2fbc --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/bullets-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 9, + "scan_ms": 2002, + "baseline_scan_ms": 2003, + "labelled_queries": 500, + "ndcg_at_k": 0.40178147080319865, + "doc_recall_at_k": 0.49733333333333335, + "boundary_gap": 0.0010104477405548096, + "neighbour_band": 0.009650647640228271, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.978199999999998, + "top1_agreement": 0.97, + "mean_abs_score_error": 0.000532852728164196, + "max_abs_score_error": 0.0036312341690063477, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2500, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.4003450978888609, + "doc_recall_at_k": 0.49733333333333335, + "boundary_gap": 0.0010104477405548096, + "neighbour_band": 0.009650647640228271, + "rank_noise": 0.0005623826488021243 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.39820000000000044, + "top1_agreement": 0.41, + "mean_abs_score_error": 0.22838297591176332, + "max_abs_score_error": 0.40268176794052124, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2312, + "baseline_scan_ms": 2003, + "labelled_queries": 500, + "ndcg_at_k": 0.3118627860087583, + "doc_recall_at_k": 0.4116666666666667, + "boundary_gap": 0.0010104477405548096, + "neighbour_band": 0.009650647640228271, + "rank_noise": 0.02312587601342484 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.2605999999999999, + "top1_agreement": 0.252, + "mean_abs_score_error": 0.21368755620237143, + "max_abs_score_error": 0.5942977070808411, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.23784049426639223, + "doc_recall_at_k": 0.3296666666666666, + "boundary_gap": 0.0010104477405548096, + "neighbour_band": 0.009650647640228271, + "rank_noise": 0.04810842078904391 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5301999999999993, + "top1_agreement": 0.5, + "mean_abs_score_error": 0.03072551447017491, + "max_abs_score_error": 0.14822229743003845, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 372, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.33911710299556397, + "doc_recall_at_k": 0.41966666666666663, + "boundary_gap": 0.0010104477405548096, + "neighbour_band": 0.009650647640228271, + "rank_noise": 0.013291631288552421 + } +] diff --git a/experiments/2026-11-text-format/results/recall/bullets-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/bullets-minilm-descriptive.json new file mode 100644 index 0000000..2a27060 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/bullets-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 2050, + "baseline_scan_ms": 2004, + "labelled_queries": 500, + "ndcg_at_k": 0.1613105898711521, + "doc_recall_at_k": 0.14068256993196904, + "boundary_gap": 0.0017380714416503906, + "neighbour_band": 0.0194818377494812, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9927999999999992, + "top1_agreement": 0.994, + "mean_abs_score_error": 0.000269356801425689, + "max_abs_score_error": 0.00196877121925354, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2555, + "baseline_scan_ms": 1844, + "labelled_queries": 500, + "ndcg_at_k": 0.16153744388643593, + "doc_recall_at_k": 0.1411122236116227, + "boundary_gap": 0.0017380714416503906, + "neighbour_band": 0.0194818377494812, + "rank_noise": 0.0002853310502315725 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5347999999999996, + "top1_agreement": 0.39, + "mean_abs_score_error": 0.04360414190521798, + "max_abs_score_error": 0.21623218059539795, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 2047, + "baseline_scan_ms": 1706, + "labelled_queries": 500, + "ndcg_at_k": 0.1362047839029237, + "doc_recall_at_k": 0.11573467870275221, + "boundary_gap": 0.0017380714416503906, + "neighbour_band": 0.0194818377494812, + "rank_noise": 0.025394864826891173 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.40259999999999985, + "top1_agreement": 0.258, + "mean_abs_score_error": 0.04779376356004747, + "max_abs_score_error": 0.3231307864189148, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 0, + "baseline_scan_ms": 1553, + "labelled_queries": 500, + "ndcg_at_k": 0.11159863165280018, + "doc_recall_at_k": 0.09445272278756318, + "boundary_gap": 0.0017380714416503906, + "neighbour_band": 0.0194818377494812, + "rank_noise": 0.04512499398028013 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6350000000000006, + "top1_agreement": 0.51, + "mean_abs_score_error": 0.027256031842254662, + "max_abs_score_error": 0.1902400255203247, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 362, + "scan_ms": 1520, + "baseline_scan_ms": 1526, + "labelled_queries": 500, + "ndcg_at_k": 0.1417507386014969, + "doc_recall_at_k": 0.12335244337562007, + "boundary_gap": 0.0017380714416503906, + "neighbour_band": 0.0194818377494812, + "rank_noise": 0.019104849747577746 + } +] diff --git a/experiments/2026-11-text-format/results/recall/bullets-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/bullets-minilm-lookup.json new file mode 100644 index 0000000..325762d --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/bullets-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 2002, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.3358602332038654, + "doc_recall_at_k": 0.461, + "boundary_gap": 0.0016207098960876465, + "neighbour_band": 0.015997886657714844, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9917999999999992, + "top1_agreement": 0.986, + "mean_abs_score_error": 0.0002644035335748531, + "max_abs_score_error": 0.0023258626461029053, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2503, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.3352685264643603, + "doc_recall_at_k": 0.461, + "boundary_gap": 0.0016207098960876465, + "neighbour_band": 0.015997886657714844, + "rank_noise": 0.0002746861518686122 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.49040000000000006, + "top1_agreement": 0.404, + "mean_abs_score_error": 0.058469569155130185, + "max_abs_score_error": 0.24339960515499115, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 28, + "scan_ms": 2381, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.27739005841138475, + "doc_recall_at_k": 0.386, + "boundary_gap": 0.0016207098960876465, + "neighbour_band": 0.015997886657714844, + "rank_noise": 0.024756123213476786 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.3489999999999998, + "top1_agreement": 0.29, + "mean_abs_score_error": 0.04806116922290908, + "max_abs_score_error": 0.34698373079299927, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.2279194797381081, + "doc_recall_at_k": 0.333, + "boundary_gap": 0.0016207098960876465, + "neighbour_band": 0.015997886657714844, + "rank_noise": 0.044314074597070555 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6157999999999996, + "top1_agreement": 0.506, + "mean_abs_score_error": 0.03231637099426329, + "max_abs_score_error": 0.2160383015871048, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 358, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.29135438658795565, + "doc_recall_at_k": 0.401, + "boundary_gap": 0.0016207098960876465, + "neighbour_band": 0.015997886657714844, + "rank_noise": 0.018190333912995394 + } +] diff --git a/experiments/2026-11-text-format/results/recall/header-prose-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/header-prose-bge-small-descriptive.json new file mode 100644 index 0000000..a1dc674 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/header-prose-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 8, + "scan_ms": 3946, + "baseline_scan_ms": 3922, + "labelled_queries": 500, + "ndcg_at_k": 0.1574662864514507, + "doc_recall_at_k": 0.13234730201822387, + "boundary_gap": 0.0011032521724700928, + "neighbour_band": 0.010821998119354248, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9803999999999983, + "top1_agreement": 0.98, + "mean_abs_score_error": 0.00047987080247402194, + "max_abs_score_error": 0.003235042095184326, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 3, + "scan_ms": 5906, + "baseline_scan_ms": 4747, + "labelled_queries": 500, + "ndcg_at_k": 0.15812228942684456, + "doc_recall_at_k": 0.1327245278883548, + "boundary_gap": 0.0011032521724700928, + "neighbour_band": 0.010821998119354248, + "rank_noise": 0.0005043166256131755 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.44400000000000034, + "top1_agreement": 0.33, + "mean_abs_score_error": 0.27164905194077044, + "max_abs_score_error": 0.4200870990753174, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 64, + "scan_ms": 6752, + "baseline_scan_ms": 5383, + "labelled_queries": 500, + "ndcg_at_k": 0.13533880972950085, + "doc_recall_at_k": 0.112484805096376, + "boundary_gap": 0.0011032521724700928, + "neighbour_band": 0.010821998119354248, + "rank_noise": 0.022240066526000454 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.2949999999999997, + "top1_agreement": 0.216, + "mean_abs_score_error": 0.2585722618516728, + "max_abs_score_error": 0.6173372268676758, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 53, + "scan_ms": 0, + "baseline_scan_ms": 3209, + "labelled_queries": 500, + "ndcg_at_k": 0.10222367937432461, + "doc_recall_at_k": 0.08149448694453044, + "boundary_gap": 0.0011032521724700928, + "neighbour_band": 0.010821998119354248, + "rank_noise": 0.05150047371425209 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5413999999999992, + "top1_agreement": 0.402, + "mean_abs_score_error": 0.0407490034885183, + "max_abs_score_error": 0.1433349847793579, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 832, + "scan_ms": 4036, + "baseline_scan_ms": 4218, + "labelled_queries": 500, + "ndcg_at_k": 0.1382769425677855, + "doc_recall_at_k": 0.11173204810523511, + "boundary_gap": 0.0011032521724700928, + "neighbour_band": 0.010821998119354248, + "rank_noise": 0.013822674048947345 + } +] diff --git a/experiments/2026-11-text-format/results/recall/header-prose-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/header-prose-bge-small-lookup.json new file mode 100644 index 0000000..fc4a69b --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/header-prose-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 15, + "scan_ms": 3814, + "baseline_scan_ms": 3799, + "labelled_queries": 500, + "ndcg_at_k": 0.3701990448749839, + "doc_recall_at_k": 0.5053333333333334, + "boundary_gap": 0.0009646415710449219, + "neighbour_band": 0.010596781969070435, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9825999999999983, + "top1_agreement": 0.978, + "mean_abs_score_error": 0.0005045342218860984, + "max_abs_score_error": 0.003439873456954956, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 4, + "scan_ms": 5297, + "baseline_scan_ms": 4060, + "labelled_queries": 500, + "ndcg_at_k": 0.36887863004825777, + "doc_recall_at_k": 0.5053333333333334, + "boundary_gap": 0.0009646415710449219, + "neighbour_band": 0.010596781969070435, + "rank_noise": 0.0005225530592222067 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.40740000000000054, + "top1_agreement": 0.37, + "mean_abs_score_error": 0.21853790369511694, + "max_abs_score_error": 0.37187665700912476, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 48, + "scan_ms": 4579, + "baseline_scan_ms": 3913, + "labelled_queries": 500, + "ndcg_at_k": 0.28922307854566026, + "doc_recall_at_k": 0.4003333333333333, + "boundary_gap": 0.0009646415710449219, + "neighbour_band": 0.010596781969070435, + "rank_noise": 0.022892387349592733 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.2511999999999998, + "top1_agreement": 0.22, + "mean_abs_score_error": 0.20611319427951946, + "max_abs_score_error": 0.5583034753799438, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 45, + "scan_ms": 154, + "baseline_scan_ms": 5768, + "labelled_queries": 500, + "ndcg_at_k": 0.2036848401911748, + "doc_recall_at_k": 0.29733333333333334, + "boundary_gap": 0.0009646415710449219, + "neighbour_band": 0.010596781969070435, + "rank_noise": 0.04954050211877942 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5237999999999995, + "top1_agreement": 0.484, + "mean_abs_score_error": 0.0367683074901849, + "max_abs_score_error": 0.1390877366065979, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 833, + "scan_ms": 4904, + "baseline_scan_ms": 4330, + "labelled_queries": 500, + "ndcg_at_k": 0.28452712551551, + "doc_recall_at_k": 0.39866666666666667, + "boundary_gap": 0.0009646415710449219, + "neighbour_band": 0.010596781969070435, + "rank_noise": 0.014511191165207184 + } +] diff --git a/experiments/2026-11-text-format/results/recall/header-prose-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/header-prose-minilm-descriptive.json new file mode 100644 index 0000000..f630b7b --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/header-prose-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 4, + "scan_ms": 1882, + "baseline_scan_ms": 1857, + "labelled_queries": 500, + "ndcg_at_k": 0.1359142550509522, + "doc_recall_at_k": 0.11018328183045886, + "boundary_gap": 0.0017317533493041992, + "neighbour_band": 0.021202847361564636, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9945999999999996, + "top1_agreement": 0.994, + "mean_abs_score_error": 0.00027030059827753175, + "max_abs_score_error": 0.0018450692296028137, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2463, + "baseline_scan_ms": 1734, + "labelled_queries": 500, + "ndcg_at_k": 0.135956193976593, + "doc_recall_at_k": 0.11021891639550518, + "boundary_gap": 0.0017317533493041992, + "neighbour_band": 0.021202847361564636, + "rank_noise": 0.0002960139970208632 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5525999999999994, + "top1_agreement": 0.43, + "mean_abs_score_error": 0.034165254697297816, + "max_abs_score_error": 0.21777576208114624, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 2027, + "baseline_scan_ms": 1598, + "labelled_queries": 500, + "ndcg_at_k": 0.11324703859886479, + "doc_recall_at_k": 0.09196026563981309, + "boundary_gap": 0.0017317533493041992, + "neighbour_band": 0.021202847361564636, + "rank_noise": 0.026779620029337852 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.40560000000000057, + "top1_agreement": 0.25, + "mean_abs_score_error": 0.048601934252947995, + "max_abs_score_error": 0.3019031882286072, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 0, + "baseline_scan_ms": 1618, + "labelled_queries": 500, + "ndcg_at_k": 0.09261660823159618, + "doc_recall_at_k": 0.07745067740614438, + "boundary_gap": 0.0017317533493041992, + "neighbour_band": 0.021202847361564636, + "rank_noise": 0.04895294478025753 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6301999999999991, + "top1_agreement": 0.522, + "mean_abs_score_error": 0.026514292918776924, + "max_abs_score_error": 0.19208519160747528, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 375, + "scan_ms": 1557, + "baseline_scan_ms": 1595, + "labelled_queries": 500, + "ndcg_at_k": 0.12156668081764381, + "doc_recall_at_k": 0.09712727416725812, + "boundary_gap": 0.0017317533493041992, + "neighbour_band": 0.021202847361564636, + "rank_noise": 0.02180369779438052 + } +] diff --git a/experiments/2026-11-text-format/results/recall/header-prose-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/header-prose-minilm-lookup.json new file mode 100644 index 0000000..8507736 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/header-prose-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 1996, + "baseline_scan_ms": 1991, + "labelled_queries": 500, + "ndcg_at_k": 0.20822388928003513, + "doc_recall_at_k": 0.30266666666666664, + "boundary_gap": 0.0018087327480316162, + "neighbour_band": 0.018535256385803223, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9925999999999993, + "top1_agreement": 0.992, + "mean_abs_score_error": 0.00026924784449626634, + "max_abs_score_error": 0.0017986074090003967, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2501, + "baseline_scan_ms": 1985, + "labelled_queries": 500, + "ndcg_at_k": 0.2094944117179786, + "doc_recall_at_k": 0.30466666666666664, + "boundary_gap": 0.0018087327480316162, + "neighbour_band": 0.018535256385803223, + "rank_noise": 0.00029480581931797185 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5099999999999998, + "top1_agreement": 0.37, + "mean_abs_score_error": 0.04008872548254303, + "max_abs_score_error": 0.2135579138994217, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2097, + "baseline_scan_ms": 1978, + "labelled_queries": 500, + "ndcg_at_k": 0.162403375633159, + "doc_recall_at_k": 0.248, + "boundary_gap": 0.0018087327480316162, + "neighbour_band": 0.018535256385803223, + "rank_noise": 0.02643741872798408 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.3497999999999997, + "top1_agreement": 0.252, + "mean_abs_score_error": 0.04859456138452831, + "max_abs_score_error": 0.3348715007305145, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 1975, + "labelled_queries": 500, + "ndcg_at_k": 0.14305307340712115, + "doc_recall_at_k": 0.219, + "boundary_gap": 0.0018087327480316162, + "neighbour_band": 0.018535256385803223, + "rank_noise": 0.04829896583523255 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6155999999999997, + "top1_agreement": 0.45, + "mean_abs_score_error": 0.030202564056479267, + "max_abs_score_error": 0.17772284150123596, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 1848, + "baseline_scan_ms": 1943, + "labelled_queries": 500, + "ndcg_at_k": 0.18205625038313994, + "doc_recall_at_k": 0.2696666666666666, + "boundary_gap": 0.0018087327480316162, + "neighbour_band": 0.018535256385803223, + "rank_noise": 0.020787784400134623 + } +] diff --git a/experiments/2026-11-text-format/results/recall/json-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/json-bge-small-descriptive.json new file mode 100644 index 0000000..a5401ed --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/json-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 3, + "scan_ms": 1998, + "baseline_scan_ms": 1999, + "labelled_queries": 500, + "ndcg_at_k": 0.22384205850421124, + "doc_recall_at_k": 0.18462644561923947, + "boundary_gap": 0.0009859204292297363, + "neighbour_band": 0.009328246116638184, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9755999999999984, + "top1_agreement": 0.964, + "mean_abs_score_error": 0.0005435402622073888, + "max_abs_score_error": 0.003852248191833496, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2500, + "baseline_scan_ms": 1997, + "labelled_queries": 500, + "ndcg_at_k": 0.22389084578541615, + "doc_recall_at_k": 0.1847483157022937, + "boundary_gap": 0.0009859204292297363, + "neighbour_band": 0.009328246116638184, + "rank_noise": 0.0005737336875606049 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.44000000000000034, + "top1_agreement": 0.304, + "mean_abs_score_error": 0.3148651443380162, + "max_abs_score_error": 0.46090155839920044, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2296, + "baseline_scan_ms": 1991, + "labelled_queries": 500, + "ndcg_at_k": 0.1778737095325779, + "doc_recall_at_k": 0.14535953360978812, + "boundary_gap": 0.0009859204292297363, + "neighbour_band": 0.009328246116638184, + "rank_noise": 0.020918450918612032 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.33360000000000006, + "top1_agreement": 0.196, + "mean_abs_score_error": 0.2882455023302957, + "max_abs_score_error": 0.6468000411987305, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2, + "baseline_scan_ms": 1995, + "labelled_queries": 500, + "ndcg_at_k": 0.1481000584218245, + "doc_recall_at_k": 0.11945747342210479, + "boundary_gap": 0.0009859204292297363, + "neighbour_band": 0.009328246116638184, + "rank_noise": 0.04460908965790574 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5861999999999995, + "top1_agreement": 0.402, + "mean_abs_score_error": 0.024403287403663994, + "max_abs_score_error": 0.1628301441669464, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 1980, + "baseline_scan_ms": 1989, + "labelled_queries": 500, + "ndcg_at_k": 0.18908401341979986, + "doc_recall_at_k": 0.15061555613481642, + "boundary_gap": 0.0009859204292297363, + "neighbour_band": 0.009328246116638184, + "rank_noise": 0.01006400530963398 + } +] diff --git a/experiments/2026-11-text-format/results/recall/json-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/json-bge-small-lookup.json new file mode 100644 index 0000000..1e527c3 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/json-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 14, + "scan_ms": 2001, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.44072600267149176, + "doc_recall_at_k": 0.5616666666666666, + "boundary_gap": 0.0008085668087005615, + "neighbour_band": 0.008194446563720703, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9709999999999976, + "top1_agreement": 0.97, + "mean_abs_score_error": 0.0005699185646042227, + "max_abs_score_error": 0.004046738147735596, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 3, + "scan_ms": 2501, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.4425245448922486, + "doc_recall_at_k": 0.5656666666666667, + "boundary_gap": 0.0008085668087005615, + "neighbour_band": 0.008194446563720703, + "rank_noise": 0.0005900646366037159 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.39819999999999983, + "top1_agreement": 0.404, + "mean_abs_score_error": 0.2518099122897923, + "max_abs_score_error": 0.4358070492744446, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2381, + "baseline_scan_ms": 2003, + "labelled_queries": 500, + "ndcg_at_k": 0.3454249360613998, + "doc_recall_at_k": 0.46, + "boundary_gap": 0.0008085668087005615, + "neighbour_band": 0.008194446563720703, + "rank_noise": 0.021633961835105416 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.2724, + "top1_agreement": 0.284, + "mean_abs_score_error": 0.23012764450895934, + "max_abs_score_error": 0.6121545433998108, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 2003, + "labelled_queries": 500, + "ndcg_at_k": 0.26683980433509313, + "doc_recall_at_k": 0.369, + "boundary_gap": 0.0008085668087005615, + "neighbour_band": 0.008194446563720703, + "rank_noise": 0.0441365510434023 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5427999999999991, + "top1_agreement": 0.53, + "mean_abs_score_error": 0.022319330208605527, + "max_abs_score_error": 0.15075385570526123, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 372, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.35536840571637485, + "doc_recall_at_k": 0.45766666666666667, + "boundary_gap": 0.0008085668087005615, + "neighbour_band": 0.008194446563720703, + "rank_noise": 0.010636494620821063 + } +] diff --git a/experiments/2026-11-text-format/results/recall/json-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/json-minilm-descriptive.json new file mode 100644 index 0000000..a8fe204 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/json-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 2003, + "baseline_scan_ms": 2001, + "labelled_queries": 500, + "ndcg_at_k": 0.14217403827387157, + "doc_recall_at_k": 0.11688222541731294, + "boundary_gap": 0.0018673241138458252, + "neighbour_band": 0.020144686102867126, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9919999999999991, + "top1_agreement": 0.992, + "mean_abs_score_error": 0.0002681720210633411, + "max_abs_score_error": 0.0019618719816207886, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2502, + "baseline_scan_ms": 1998, + "labelled_queries": 500, + "ndcg_at_k": 0.14135595674365684, + "doc_recall_at_k": 0.11643936827445579, + "boundary_gap": 0.0018673241138458252, + "neighbour_band": 0.020144686102867126, + "rank_noise": 0.0002920790327229209 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5565999999999998, + "top1_agreement": 0.412, + "mean_abs_score_error": 0.045635988619242465, + "max_abs_score_error": 0.21436604857444763, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 29, + "scan_ms": 2337, + "baseline_scan_ms": 1995, + "labelled_queries": 500, + "ndcg_at_k": 0.11942875376406797, + "doc_recall_at_k": 0.10144384400728709, + "boundary_gap": 0.0018673241138458252, + "neighbour_band": 0.020144686102867126, + "rank_noise": 0.025073297148414518 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.44039999999999935, + "top1_agreement": 0.302, + "mean_abs_score_error": 0.047202809526784344, + "max_abs_score_error": 0.320909708738327, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1988, + "labelled_queries": 500, + "ndcg_at_k": 0.10903977707801757, + "doc_recall_at_k": 0.09182177302900196, + "boundary_gap": 0.0018673241138458252, + "neighbour_band": 0.020144686102867126, + "rank_noise": 0.04268366874841095 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6810000000000002, + "top1_agreement": 0.504, + "mean_abs_score_error": 0.02119770432708542, + "max_abs_score_error": 0.18869158625602722, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 361, + "scan_ms": 1983, + "baseline_scan_ms": 1990, + "labelled_queries": 500, + "ndcg_at_k": 0.12197343548835846, + "doc_recall_at_k": 0.10086089813434157, + "boundary_gap": 0.0018673241138458252, + "neighbour_band": 0.020144686102867126, + "rank_noise": 0.01758386962810289 + } +] diff --git a/experiments/2026-11-text-format/results/recall/json-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/json-minilm-lookup.json new file mode 100644 index 0000000..1ffd956 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/json-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 19, + "scan_ms": 2001, + "baseline_scan_ms": 2002, + "labelled_queries": 500, + "ndcg_at_k": 0.24007849790754515, + "doc_recall_at_k": 0.33466666666666667, + "boundary_gap": 0.0015345662832260132, + "neighbour_band": 0.016464248299598694, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9943999999999994, + "top1_agreement": 0.99, + "mean_abs_score_error": 0.00026641507460700974, + "max_abs_score_error": 0.002117753028869629, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2500, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.2399883369050486, + "doc_recall_at_k": 0.33466666666666667, + "boundary_gap": 0.0015345662832260132, + "neighbour_band": 0.016464248299598694, + "rank_noise": 0.0002833349702738367 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5052000000000002, + "top1_agreement": 0.418, + "mean_abs_score_error": 0.05720474677743775, + "max_abs_score_error": 0.23511794209480286, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2438, + "baseline_scan_ms": 2001, + "labelled_queries": 500, + "ndcg_at_k": 0.18290750958136065, + "doc_recall_at_k": 0.265, + "boundary_gap": 0.0015345662832260132, + "neighbour_band": 0.016464248299598694, + "rank_noise": 0.02449174088693281 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.3667999999999999, + "top1_agreement": 0.264, + "mean_abs_score_error": 0.04964396642853981, + "max_abs_score_error": 0.33861005306243896, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.16082108801678177, + "doc_recall_at_k": 0.235, + "boundary_gap": 0.0015345662832260132, + "neighbour_band": 0.016464248299598694, + "rank_noise": 0.0423662223916466 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6163999999999995, + "top1_agreement": 0.484, + "mean_abs_score_error": 0.022806261390840007, + "max_abs_score_error": 0.20051142573356628, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 2002, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.19358658187262145, + "doc_recall_at_k": 0.269, + "boundary_gap": 0.0015345662832260132, + "neighbour_band": 0.016464248299598694, + "rank_noise": 0.017009062579887112 + } +] diff --git a/experiments/2026-11-text-format/results/recall/kv-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/kv-bge-small-descriptive.json new file mode 100644 index 0000000..a6e306e --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/kv-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 3, + "scan_ms": 1939, + "baseline_scan_ms": 1930, + "labelled_queries": 500, + "ndcg_at_k": 0.19070521998037948, + "doc_recall_at_k": 0.1627244828161221, + "boundary_gap": 0.0008940994739532471, + "neighbour_band": 0.009874284267425537, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9761999999999983, + "top1_agreement": 0.972, + "mean_abs_score_error": 0.0004952428338557481, + "max_abs_score_error": 0.0037380456924438477, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2489, + "baseline_scan_ms": 1944, + "labelled_queries": 500, + "ndcg_at_k": 0.19109550430820366, + "doc_recall_at_k": 0.16382115188951535, + "boundary_gap": 0.0008940994739532471, + "neighbour_band": 0.009874284267425537, + "rank_noise": 0.000497853896572756 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.4281999999999997, + "top1_agreement": 0.288, + "mean_abs_score_error": 0.2879964896052733, + "max_abs_score_error": 0.44385653734207153, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 2148, + "baseline_scan_ms": 1929, + "labelled_queries": 500, + "ndcg_at_k": 0.15221632173095534, + "doc_recall_at_k": 0.12590942385973974, + "boundary_gap": 0.0008940994739532471, + "neighbour_band": 0.009874284267425537, + "rank_noise": 0.0206149263815978 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.3025999999999999, + "top1_agreement": 0.168, + "mean_abs_score_error": 0.24993872631068675, + "max_abs_score_error": 0.6021743416786194, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1935, + "labelled_queries": 500, + "ndcg_at_k": 0.11848966594854614, + "doc_recall_at_k": 0.09511220015601164, + "boundary_gap": 0.0008940994739532471, + "neighbour_band": 0.009874284267425537, + "rank_noise": 0.046493915656692544 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5553999999999992, + "top1_agreement": 0.42, + "mean_abs_score_error": 0.03343380576819777, + "max_abs_score_error": 0.13612496852874756, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 368, + "scan_ms": 1939, + "baseline_scan_ms": 1943, + "labelled_queries": 500, + "ndcg_at_k": 0.15580102729524237, + "doc_recall_at_k": 0.1326244822305499, + "boundary_gap": 0.0008940994739532471, + "neighbour_band": 0.009874284267425537, + "rank_noise": 0.011901817115159499 + } +] diff --git a/experiments/2026-11-text-format/results/recall/kv-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/kv-bge-small-lookup.json new file mode 100644 index 0000000..97343c7 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/kv-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 4, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.43525936923790515, + "doc_recall_at_k": 0.559, + "boundary_gap": 0.0010011792182922363, + "neighbour_band": 0.009523779153823853, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.978199999999998, + "top1_agreement": 0.968, + "mean_abs_score_error": 0.0005330730384334922, + "max_abs_score_error": 0.0036900341510772705, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 3, + "scan_ms": 2504, + "baseline_scan_ms": 1992, + "labelled_queries": 500, + "ndcg_at_k": 0.4355725087771692, + "doc_recall_at_k": 0.561, + "boundary_gap": 0.0010011792182922363, + "neighbour_band": 0.009523779153823853, + "rank_noise": 0.00055696620579668 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.3930000000000005, + "top1_agreement": 0.402, + "mean_abs_score_error": 0.231981233538571, + "max_abs_score_error": 0.40691423416137695, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2204, + "baseline_scan_ms": 1982, + "labelled_queries": 500, + "ndcg_at_k": 0.3314435222901498, + "doc_recall_at_k": 0.4256666666666667, + "boundary_gap": 0.0010011792182922363, + "neighbour_band": 0.009523779153823853, + "rank_noise": 0.022729397407600466 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.25260000000000005, + "top1_agreement": 0.246, + "mean_abs_score_error": 0.2151144156746052, + "max_abs_score_error": 0.5725215673446655, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1977, + "labelled_queries": 500, + "ndcg_at_k": 0.2590545766537737, + "doc_recall_at_k": 0.3586666666666667, + "boundary_gap": 0.0010011792182922363, + "neighbour_band": 0.009523779153823853, + "rank_noise": 0.0471105978607601 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5289999999999997, + "top1_agreement": 0.476, + "mean_abs_score_error": 0.028586182317483427, + "max_abs_score_error": 0.13806307315826416, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 1969, + "baseline_scan_ms": 1970, + "labelled_queries": 500, + "ndcg_at_k": 0.35718302201402347, + "doc_recall_at_k": 0.4713333333333334, + "boundary_gap": 0.0010011792182922363, + "neighbour_band": 0.009523779153823853, + "rank_noise": 0.012737702651780227 + } +] diff --git a/experiments/2026-11-text-format/results/recall/kv-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/kv-minilm-descriptive.json new file mode 100644 index 0000000..ec17d79 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/kv-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 1966, + "baseline_scan_ms": 1969, + "labelled_queries": 500, + "ndcg_at_k": 0.14328363464109778, + "doc_recall_at_k": 0.1261704381573801, + "boundary_gap": 0.0019334405660629272, + "neighbour_band": 0.019775748252868652, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9917999999999992, + "top1_agreement": 0.998, + "mean_abs_score_error": 0.0002672070327238, + "max_abs_score_error": 0.0018847882747650146, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2497, + "baseline_scan_ms": 1973, + "labelled_queries": 500, + "ndcg_at_k": 0.14313078517023775, + "doc_recall_at_k": 0.12633551752245945, + "boundary_gap": 0.0019334405660629272, + "neighbour_band": 0.019775748252868652, + "rank_noise": 0.00028826959057240295 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5269999999999998, + "top1_agreement": 0.398, + "mean_abs_score_error": 0.04272191333427277, + "max_abs_score_error": 0.22007735073566437, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2037, + "baseline_scan_ms": 1958, + "labelled_queries": 500, + "ndcg_at_k": 0.1309206947199622, + "doc_recall_at_k": 0.10807908660364796, + "boundary_gap": 0.0019334405660629272, + "neighbour_band": 0.019775748252868652, + "rank_noise": 0.026023889352178312 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.39699999999999974, + "top1_agreement": 0.282, + "mean_abs_score_error": 0.04752058138117371, + "max_abs_score_error": 0.313972145318985, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 0, + "baseline_scan_ms": 1958, + "labelled_queries": 500, + "ndcg_at_k": 0.10947132568262843, + "doc_recall_at_k": 0.08998031292890955, + "boundary_gap": 0.0019334405660629272, + "neighbour_band": 0.019775748252868652, + "rank_noise": 0.04598879890536952 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6322000000000002, + "top1_agreement": 0.51, + "mean_abs_score_error": 0.026779508753023082, + "max_abs_score_error": 0.21329949796199799, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 1911, + "baseline_scan_ms": 1957, + "labelled_queries": 500, + "ndcg_at_k": 0.12322216903858094, + "doc_recall_at_k": 0.10755841087952485, + "boundary_gap": 0.0019334405660629272, + "neighbour_band": 0.019775748252868652, + "rank_noise": 0.018839253033153817 + } +] diff --git a/experiments/2026-11-text-format/results/recall/kv-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/kv-minilm-lookup.json new file mode 100644 index 0000000..ffd166b --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/kv-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 11, + "scan_ms": 2003, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.2754988710342938, + "doc_recall_at_k": 0.385, + "boundary_gap": 0.0015018284320831299, + "neighbour_band": 0.016394585371017456, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9919999999999993, + "top1_agreement": 0.992, + "mean_abs_score_error": 0.00026427087001888684, + "max_abs_score_error": 0.002007514238357544, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2503, + "baseline_scan_ms": 2001, + "labelled_queries": 500, + "ndcg_at_k": 0.27473186077919326, + "doc_recall_at_k": 0.385, + "boundary_gap": 0.0015018284320831299, + "neighbour_band": 0.016394585371017456, + "rank_noise": 0.00028164664316732054 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.47980000000000034, + "top1_agreement": 0.39, + "mean_abs_score_error": 0.05602198107575474, + "max_abs_score_error": 0.22760508954524994, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2222, + "baseline_scan_ms": 1999, + "labelled_queries": 500, + "ndcg_at_k": 0.2308607617508999, + "doc_recall_at_k": 0.327, + "boundary_gap": 0.0015018284320831299, + "neighbour_band": 0.016394585371017456, + "rank_noise": 0.025233883203666853 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.33500000000000024, + "top1_agreement": 0.272, + "mean_abs_score_error": 0.04802140353554399, + "max_abs_score_error": 0.32455965876579285, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 2001, + "labelled_queries": 500, + "ndcg_at_k": 0.18525895646799123, + "doc_recall_at_k": 0.267, + "boundary_gap": 0.0015018284320831299, + "neighbour_band": 0.016394585371017456, + "rank_noise": 0.04495338965963287 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.608799999999999, + "top1_agreement": 0.55, + "mean_abs_score_error": 0.03126968467234038, + "max_abs_score_error": 0.22697967290878296, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 366, + "scan_ms": 1974, + "baseline_scan_ms": 1986, + "labelled_queries": 500, + "ndcg_at_k": 0.24175039331467338, + "doc_recall_at_k": 0.343, + "boundary_gap": 0.0015018284320831299, + "neighbour_band": 0.016394585371017456, + "rank_noise": 0.017943018844851544 + } +] diff --git a/experiments/2026-11-text-format/results/recall/paragraph-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/paragraph-bge-small-descriptive.json new file mode 100644 index 0000000..fa0caf9 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/paragraph-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 3, + "scan_ms": 2001, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.1735093954631112, + "doc_recall_at_k": 0.15061751636920612, + "boundary_gap": 0.001015782356262207, + "neighbour_band": 0.010970264673233032, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9817999999999986, + "top1_agreement": 0.98, + "mean_abs_score_error": 0.0004594226680621505, + "max_abs_score_error": 0.0033572912216186523, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2502, + "baseline_scan_ms": 1998, + "labelled_queries": 500, + "ndcg_at_k": 0.17382128644062186, + "doc_recall_at_k": 0.14955266349494145, + "boundary_gap": 0.001015782356262207, + "neighbour_band": 0.010970264673233032, + "rank_noise": 0.0004927718980100807 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.4306000000000002, + "top1_agreement": 0.332, + "mean_abs_score_error": 0.2664718560379609, + "max_abs_score_error": 0.42562732100486755, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2422, + "baseline_scan_ms": 2001, + "labelled_queries": 500, + "ndcg_at_k": 0.12984229944057193, + "doc_recall_at_k": 0.10535419383311452, + "boundary_gap": 0.001015782356262207, + "neighbour_band": 0.010970264673233032, + "rank_noise": 0.0226056131570953 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.28460000000000013, + "top1_agreement": 0.208, + "mean_abs_score_error": 0.24924789639427364, + "max_abs_score_error": 0.6189075112342834, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 1999, + "labelled_queries": 500, + "ndcg_at_k": 0.10343436813972624, + "doc_recall_at_k": 0.08208201927830254, + "boundary_gap": 0.001015782356262207, + "neighbour_band": 0.010970264673233032, + "rank_noise": 0.05022904805776396 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5421999999999992, + "top1_agreement": 0.408, + "mean_abs_score_error": 0.037533751673950254, + "max_abs_score_error": 0.14963555335998535, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 372, + "scan_ms": 1984, + "baseline_scan_ms": 1994, + "labelled_queries": 500, + "ndcg_at_k": 0.14246509036752983, + "doc_recall_at_k": 0.11319278636023211, + "boundary_gap": 0.001015782356262207, + "neighbour_band": 0.010970264673233032, + "rank_noise": 0.013679213780810105 + } +] diff --git a/experiments/2026-11-text-format/results/recall/paragraph-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/paragraph-bge-small-lookup.json new file mode 100644 index 0000000..50ee475 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/paragraph-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 11, + "scan_ms": 2001, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.36996333649320917, + "doc_recall_at_k": 0.473, + "boundary_gap": 0.0010963976383209229, + "neighbour_band": 0.009995579719543457, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9819999999999984, + "top1_agreement": 0.982, + "mean_abs_score_error": 0.0004885301106587052, + "max_abs_score_error": 0.003483802080154419, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2501, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.36945957118851513, + "doc_recall_at_k": 0.473, + "boundary_gap": 0.0010963976383209229, + "neighbour_band": 0.009995579719543457, + "rank_noise": 0.0005136322458118983 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.38880000000000026, + "top1_agreement": 0.386, + "mean_abs_score_error": 0.21364501472176908, + "max_abs_score_error": 0.42058998346328735, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2398, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.2760048787103795, + "doc_recall_at_k": 0.36366666666666664, + "boundary_gap": 0.0010963976383209229, + "neighbour_band": 0.009995579719543457, + "rank_noise": 0.023746090173843656 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.24119999999999997, + "top1_agreement": 0.218, + "mean_abs_score_error": 0.19919907793466896, + "max_abs_score_error": 0.5734659433364868, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.2091705478062366, + "doc_recall_at_k": 0.289, + "boundary_gap": 0.0010963976383209229, + "neighbour_band": 0.009995579719543457, + "rank_noise": 0.049126464803224155 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5148, + "top1_agreement": 0.498, + "mean_abs_score_error": 0.03338085960514396, + "max_abs_score_error": 0.1533043086528778, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.3119751071250867, + "doc_recall_at_k": 0.4103333333333334, + "boundary_gap": 0.0010963976383209229, + "neighbour_band": 0.009995579719543457, + "rank_noise": 0.014088010248940087 + } +] diff --git a/experiments/2026-11-text-format/results/recall/paragraph-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/paragraph-minilm-descriptive.json new file mode 100644 index 0000000..226fb64 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/paragraph-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 2000, + "baseline_scan_ms": 1999, + "labelled_queries": 500, + "ndcg_at_k": 0.129510184751575, + "doc_recall_at_k": 0.10634366384959042, + "boundary_gap": 0.0021651387214660645, + "neighbour_band": 0.021275341510772705, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9935999999999993, + "top1_agreement": 0.998, + "mean_abs_score_error": 0.0002728225421706127, + "max_abs_score_error": 0.001973673701286316, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2500, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.12879261984749196, + "doc_recall_at_k": 0.10538652099244755, + "boundary_gap": 0.0021651387214660645, + "neighbour_band": 0.021275341510772705, + "rank_noise": 0.00029926536127772226 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5711999999999993, + "top1_agreement": 0.422, + "mean_abs_score_error": 0.035559407217586166, + "max_abs_score_error": 0.22260519862174988, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 28, + "scan_ms": 2454, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.1138602027822289, + "doc_recall_at_k": 0.0926564760791805, + "boundary_gap": 0.0021651387214660645, + "neighbour_band": 0.021275341510772705, + "rank_noise": 0.026219276763255993 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.4110000000000005, + "top1_agreement": 0.316, + "mean_abs_score_error": 0.04848074346177126, + "max_abs_score_error": 0.32461804151535034, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 28, + "scan_ms": 0, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.09966956222481498, + "doc_recall_at_k": 0.08158369312854236, + "boundary_gap": 0.0021651387214660645, + "neighbour_band": 0.021275341510772705, + "rank_noise": 0.048614573070958664 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6515999999999998, + "top1_agreement": 0.51, + "mean_abs_score_error": 0.025813125376351772, + "max_abs_score_error": 0.17724627256393433, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 372, + "scan_ms": 1996, + "baseline_scan_ms": 1994, + "labelled_queries": 500, + "ndcg_at_k": 0.11187079359034093, + "doc_recall_at_k": 0.09317237174039228, + "boundary_gap": 0.0021651387214660645, + "neighbour_band": 0.021275341510772705, + "rank_noise": 0.021389494657683458 + } +] diff --git a/experiments/2026-11-text-format/results/recall/paragraph-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/paragraph-minilm-lookup.json new file mode 100644 index 0000000..9fe39fa --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/paragraph-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 2002, + "baseline_scan_ms": 2002, + "labelled_queries": 500, + "ndcg_at_k": 0.17925052920584064, + "doc_recall_at_k": 0.26366666666666666, + "boundary_gap": 0.001577615737915039, + "neighbour_band": 0.016923844814300537, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9925999999999995, + "top1_agreement": 0.99, + "mean_abs_score_error": 0.00026938465329518965, + "max_abs_score_error": 0.0019117295742034912, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2500, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.17911219457306232, + "doc_recall_at_k": 0.26366666666666666, + "boundary_gap": 0.001577615737915039, + "neighbour_band": 0.016923844814300537, + "rank_noise": 0.0002964437348832117 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.4946000000000003, + "top1_agreement": 0.384, + "mean_abs_score_error": 0.04727289083232513, + "max_abs_score_error": 0.23304878175258636, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2295, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.138016115095559, + "doc_recall_at_k": 0.21666666666666667, + "boundary_gap": 0.001577615737915039, + "neighbour_band": 0.016923844814300537, + "rank_noise": 0.02533522541298662 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.33540000000000025, + "top1_agreement": 0.242, + "mean_abs_score_error": 0.049153846027801734, + "max_abs_score_error": 0.3436708450317383, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.11722213376054388, + "doc_recall_at_k": 0.18533333333333332, + "boundary_gap": 0.001577615737915039, + "neighbour_band": 0.016923844814300537, + "rank_noise": 0.04740697834644542 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.601, + "top1_agreement": 0.466, + "mean_abs_score_error": 0.03083107079474022, + "max_abs_score_error": 0.2084849774837494, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 371, + "scan_ms": 2000, + "baseline_scan_ms": 2000, + "labelled_queries": 500, + "ndcg_at_k": 0.15695914322927154, + "doc_recall_at_k": 0.22666666666666668, + "boundary_gap": 0.001577615737915039, + "neighbour_band": 0.016923844814300537, + "rank_noise": 0.01977754710449133 + } +] diff --git a/experiments/2026-11-text-format/results/recall/table-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/table-bge-small-descriptive.json new file mode 100644 index 0000000..7adc4c8 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/table-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 1586, + "baseline_scan_ms": 1573, + "labelled_queries": 500, + "ndcg_at_k": 0.21135857285197254, + "doc_recall_at_k": 0.17234242879277611, + "boundary_gap": 0.0009588301181793213, + "neighbour_band": 0.009452968835830688, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9753999999999977, + "top1_agreement": 0.962, + "mean_abs_score_error": 0.0005152514118134976, + "max_abs_score_error": 0.004035264253616333, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2219, + "baseline_scan_ms": 1671, + "labelled_queries": 500, + "ndcg_at_k": 0.21134050849152922, + "doc_recall_at_k": 0.17269618925363597, + "boundary_gap": 0.0009588301181793213, + "neighbour_band": 0.009452968835830688, + "rank_noise": 0.0005236131192798008 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.45480000000000037, + "top1_agreement": 0.314, + "mean_abs_score_error": 0.2949775212333724, + "max_abs_score_error": 0.4439131021499634, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2040, + "baseline_scan_ms": 1747, + "labelled_queries": 500, + "ndcg_at_k": 0.168469645046794, + "doc_recall_at_k": 0.13645654106015437, + "boundary_gap": 0.0009588301181793213, + "neighbour_band": 0.009452968835830688, + "rank_noise": 0.020003218934109397 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.3192, + "top1_agreement": 0.176, + "mean_abs_score_error": 0.25207647221589236, + "max_abs_score_error": 0.6232516765594482, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 24, + "scan_ms": 0, + "baseline_scan_ms": 1561, + "labelled_queries": 500, + "ndcg_at_k": 0.13447418626192612, + "doc_recall_at_k": 0.10372338160060557, + "boundary_gap": 0.0009588301181793213, + "neighbour_band": 0.009452968835830688, + "rank_noise": 0.04422727695990099 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5727999999999996, + "top1_agreement": 0.426, + "mean_abs_score_error": 0.02981007047688663, + "max_abs_score_error": 0.14673680067062378, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 364, + "scan_ms": 1523, + "baseline_scan_ms": 1535, + "labelled_queries": 500, + "ndcg_at_k": 0.17126515983769813, + "doc_recall_at_k": 0.138005463346744, + "boundary_gap": 0.0009588301181793213, + "neighbour_band": 0.009452968835830688, + "rank_noise": 0.01107405841104134 + } +] diff --git a/experiments/2026-11-text-format/results/recall/table-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/table-bge-small-lookup.json new file mode 100644 index 0000000..b35fb63 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/table-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 3, + "scan_ms": 1979, + "baseline_scan_ms": 2109, + "labelled_queries": 500, + "ndcg_at_k": 0.41537201086747644, + "doc_recall_at_k": 0.515, + "boundary_gap": 0.000929713249206543, + "neighbour_band": 0.009497731924057007, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9765999999999985, + "top1_agreement": 0.984, + "mean_abs_score_error": 0.000549076775175333, + "max_abs_score_error": 0.0038572847843170166, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2353, + "baseline_scan_ms": 1828, + "labelled_queries": 500, + "ndcg_at_k": 0.4139853147485782, + "doc_recall_at_k": 0.513, + "boundary_gap": 0.000929713249206543, + "neighbour_band": 0.009497731924057007, + "rank_noise": 0.0005517841459123048 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.4202000000000004, + "top1_agreement": 0.398, + "mean_abs_score_error": 0.23735994950928985, + "max_abs_score_error": 0.4058400094509125, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 2122, + "baseline_scan_ms": 1689, + "labelled_queries": 500, + "ndcg_at_k": 0.3187737011972607, + "doc_recall_at_k": 0.431, + "boundary_gap": 0.000929713249206543, + "neighbour_band": 0.009497731924057007, + "rank_noise": 0.02157090519570898 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.27899999999999986, + "top1_agreement": 0.264, + "mean_abs_score_error": 0.2130840907354608, + "max_abs_score_error": 0.5618352890014648, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 24, + "scan_ms": 106, + "baseline_scan_ms": 1650, + "labelled_queries": 500, + "ndcg_at_k": 0.24162884599723902, + "doc_recall_at_k": 0.331, + "boundary_gap": 0.000929713249206543, + "neighbour_band": 0.009497731924057007, + "rank_noise": 0.04448227541575279 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5395999999999999, + "top1_agreement": 0.484, + "mean_abs_score_error": 0.026287386504681407, + "max_abs_score_error": 0.14301413297653198, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 351, + "scan_ms": 1669, + "baseline_scan_ms": 1783, + "labelled_queries": 500, + "ndcg_at_k": 0.32894921591160287, + "doc_recall_at_k": 0.433, + "boundary_gap": 0.000929713249206543, + "neighbour_band": 0.009497731924057007, + "rank_noise": 0.011659371496484832 + } +] diff --git a/experiments/2026-11-text-format/results/recall/table-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/table-minilm-descriptive.json new file mode 100644 index 0000000..783ca2d --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/table-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 1944, + "baseline_scan_ms": 1935, + "labelled_queries": 500, + "ndcg_at_k": 0.18076097829078805, + "doc_recall_at_k": 0.15366247766624214, + "boundary_gap": 0.0018536150455474854, + "neighbour_band": 0.018120557069778442, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9927999999999993, + "top1_agreement": 0.988, + "mean_abs_score_error": 0.0002681179234600851, + "max_abs_score_error": 0.002000570297241211, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2477, + "baseline_scan_ms": 1920, + "labelled_queries": 500, + "ndcg_at_k": 0.1812952897921115, + "doc_recall_at_k": 0.15431009671386117, + "boundary_gap": 0.0018536150455474854, + "neighbour_band": 0.018120557069778442, + "rank_noise": 0.0002853450993420364 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5385999999999995, + "top1_agreement": 0.382, + "mean_abs_score_error": 0.04566624835349771, + "max_abs_score_error": 0.22033829987049103, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 2031, + "baseline_scan_ms": 1904, + "labelled_queries": 500, + "ndcg_at_k": 0.15949839643600236, + "doc_recall_at_k": 0.13570417880679075, + "boundary_gap": 0.0018536150455474854, + "neighbour_band": 0.018120557069778442, + "rank_noise": 0.024268563265693193 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.42720000000000014, + "top1_agreement": 0.282, + "mean_abs_score_error": 0.04869412343917792, + "max_abs_score_error": 0.3039182424545288, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 0, + "baseline_scan_ms": 1874, + "labelled_queries": 500, + "ndcg_at_k": 0.1404309676201316, + "doc_recall_at_k": 0.11848974553292915, + "boundary_gap": 0.0018536150455474854, + "neighbour_band": 0.018120557069778442, + "rank_noise": 0.04086612698867343 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6463999999999992, + "top1_agreement": 0.51, + "mean_abs_score_error": 0.023832039168116045, + "max_abs_score_error": 0.21161049604415894, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 355, + "scan_ms": 1690, + "baseline_scan_ms": 1862, + "labelled_queries": 500, + "ndcg_at_k": 0.1631618959707789, + "doc_recall_at_k": 0.1370714488804315, + "boundary_gap": 0.0018536150455474854, + "neighbour_band": 0.018120557069778442, + "rank_noise": 0.01687235458795014 + } +] diff --git a/experiments/2026-11-text-format/results/recall/table-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/table-minilm-lookup.json new file mode 100644 index 0000000..757b7f1 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/table-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 1996, + "baseline_scan_ms": 1995, + "labelled_queries": 500, + "ndcg_at_k": 0.35331564081249656, + "doc_recall_at_k": 0.4796666666666667, + "boundary_gap": 0.0015510618686676025, + "neighbour_band": 0.0157146155834198, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9925999999999995, + "top1_agreement": 0.992, + "mean_abs_score_error": 0.00026534606312768994, + "max_abs_score_error": 0.002113819122314453, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2500, + "baseline_scan_ms": 1984, + "labelled_queries": 500, + "ndcg_at_k": 0.3540760085897339, + "doc_recall_at_k": 0.4816666666666667, + "boundary_gap": 0.0015510618686676025, + "neighbour_band": 0.0157146155834198, + "rank_noise": 0.000286645264613586 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5049999999999999, + "top1_agreement": 0.44, + "mean_abs_score_error": 0.05640793391287704, + "max_abs_score_error": 0.22977596521377563, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2081, + "baseline_scan_ms": 1980, + "labelled_queries": 500, + "ndcg_at_k": 0.28452039412340707, + "doc_recall_at_k": 0.4006666666666666, + "boundary_gap": 0.0015510618686676025, + "neighbour_band": 0.0157146155834198, + "rank_noise": 0.02377959652353164 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.36780000000000007, + "top1_agreement": 0.322, + "mean_abs_score_error": 0.0478550285847857, + "max_abs_score_error": 0.3100501298904419, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1977, + "labelled_queries": 500, + "ndcg_at_k": 0.24876125488605277, + "doc_recall_at_k": 0.36066666666666664, + "boundary_gap": 0.0015510618686676025, + "neighbour_band": 0.0157146155834198, + "rank_noise": 0.04160117208060811 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6235999999999999, + "top1_agreement": 0.498, + "mean_abs_score_error": 0.025873560158215226, + "max_abs_score_error": 0.21738746762275696, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 367, + "scan_ms": 1950, + "baseline_scan_ms": 1953, + "labelled_queries": 500, + "ndcg_at_k": 0.30146762849091324, + "doc_recall_at_k": 0.4206666666666667, + "boundary_gap": 0.0015510618686676025, + "neighbour_band": 0.0157146155834198, + "rank_noise": 0.016603972949732794 + } +] diff --git a/experiments/2026-11-text-format/results/recall/wiki-bge-small-descriptive.json b/experiments/2026-11-text-format/results/recall/wiki-bge-small-descriptive.json new file mode 100644 index 0000000..b2c6bb4 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/wiki-bge-small-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 1975, + "baseline_scan_ms": 1967, + "labelled_queries": 500, + "ndcg_at_k": 0.11307960608351139, + "doc_recall_at_k": 0.08904027986652643, + "boundary_gap": 0.001223057508468628, + "neighbour_band": 0.011578291654586792, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9837999999999981, + "top1_agreement": 0.992, + "mean_abs_score_error": 0.00046042774815410373, + "max_abs_score_error": 0.0034506618976593018, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 3, + "scan_ms": 2495, + "baseline_scan_ms": 1957, + "labelled_queries": 500, + "ndcg_at_k": 0.11378035026963701, + "doc_recall_at_k": 0.09001400359025016, + "boundary_gap": 0.001223057508468628, + "neighbour_band": 0.011578291654586792, + "rank_noise": 0.0005044747326514189 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.454, + "top1_agreement": 0.344, + "mean_abs_score_error": 0.2664824763668686, + "max_abs_score_error": 0.4148102402687073, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2013, + "baseline_scan_ms": 1962, + "labelled_queries": 500, + "ndcg_at_k": 0.10696321254725462, + "doc_recall_at_k": 0.08279815657450529, + "boundary_gap": 0.001223057508468628, + "neighbour_band": 0.011578291654586792, + "rank_noise": 0.02300975487801418 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.2910000000000001, + "top1_agreement": 0.186, + "mean_abs_score_error": 0.26110077909897866, + "max_abs_score_error": 0.6076166033744812, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1959, + "labelled_queries": 500, + "ndcg_at_k": 0.08442677380636439, + "doc_recall_at_k": 0.06419716843598916, + "boundary_gap": 0.001223057508468628, + "neighbour_band": 0.011578291654586792, + "rank_noise": 0.05171884692660382 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5485999999999996, + "top1_agreement": 0.416, + "mean_abs_score_error": 0.04118831550203562, + "max_abs_score_error": 0.16241183876991272, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 366, + "scan_ms": 1950, + "baseline_scan_ms": 1963, + "labelled_queries": 500, + "ndcg_at_k": 0.10929737880628104, + "doc_recall_at_k": 0.08544062048005659, + "boundary_gap": 0.001223057508468628, + "neighbour_band": 0.011578291654586792, + "rank_noise": 0.014603027522061725 + } +] diff --git a/experiments/2026-11-text-format/results/recall/wiki-bge-small-lookup.json b/experiments/2026-11-text-format/results/recall/wiki-bge-small-lookup.json new file mode 100644 index 0000000..b09d6cb --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/wiki-bge-small-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 2009, + "baseline_scan_ms": 1998, + "labelled_queries": 500, + "ndcg_at_k": 0.1810291137100681, + "doc_recall_at_k": 0.2846666666666667, + "boundary_gap": 0.0009933710098266602, + "neighbour_band": 0.010829746723175049, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9805999999999985, + "top1_agreement": 0.986, + "mean_abs_score_error": 0.00047990632974505423, + "max_abs_score_error": 0.0034662187099456787, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2503, + "baseline_scan_ms": 1990, + "labelled_queries": 500, + "ndcg_at_k": 0.17942990116468763, + "doc_recall_at_k": 0.2826666666666667, + "boundary_gap": 0.0009933710098266602, + "neighbour_band": 0.010829746723175049, + "rank_noise": 0.0005068958308651954 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.4050000000000005, + "top1_agreement": 0.33, + "mean_abs_score_error": 0.21690763631436677, + "max_abs_score_error": 0.3923826813697815, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2275, + "baseline_scan_ms": 1991, + "labelled_queries": 500, + "ndcg_at_k": 0.13960324011962483, + "doc_recall_at_k": 0.21466666666666667, + "boundary_gap": 0.0009933710098266602, + "neighbour_band": 0.010829746723175049, + "rank_noise": 0.022891357746151964 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.25639999999999996, + "top1_agreement": 0.204, + "mean_abs_score_error": 0.19921519030424728, + "max_abs_score_error": 0.5767011642456055, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 0, + "baseline_scan_ms": 1986, + "labelled_queries": 500, + "ndcg_at_k": 0.09038446533807218, + "doc_recall_at_k": 0.137, + "boundary_gap": 0.0009933710098266602, + "neighbour_band": 0.010829746723175049, + "rank_noise": 0.04906278077779547 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5313999999999997, + "top1_agreement": 0.496, + "mean_abs_score_error": 0.03954165779297352, + "max_abs_score_error": 0.16590426862239838, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 370, + "scan_ms": 1966, + "baseline_scan_ms": 1982, + "labelled_queries": 500, + "ndcg_at_k": 0.15096341335277594, + "doc_recall_at_k": 0.23, + "boundary_gap": 0.0009933710098266602, + "neighbour_band": 0.010829746723175049, + "rank_noise": 0.014215374780827485 + } +] diff --git a/experiments/2026-11-text-format/results/recall/wiki-minilm-descriptive.json b/experiments/2026-11-text-format/results/recall/wiki-minilm-descriptive.json new file mode 100644 index 0000000..eaef822 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/wiki-minilm-descriptive.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 3, + "scan_ms": 1788, + "baseline_scan_ms": 1752, + "labelled_queries": 500, + "ndcg_at_k": 0.09094107841990236, + "doc_recall_at_k": 0.06956664599419451, + "boundary_gap": 0.0021550804376602173, + "neighbour_band": 0.021074384450912476, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9943999999999994, + "top1_agreement": 0.99, + "mean_abs_score_error": 0.0002710422176581739, + "max_abs_score_error": 0.0019910335540771484, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2463, + "baseline_scan_ms": 1805, + "labelled_queries": 500, + "ndcg_at_k": 0.09093592948811073, + "doc_recall_at_k": 0.06923768976523828, + "boundary_gap": 0.0021550804376602173, + "neighbour_band": 0.021074384450912476, + "rank_noise": 0.0002990782029378114 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5417999999999997, + "top1_agreement": 0.402, + "mean_abs_score_error": 0.034853628605729686, + "max_abs_score_error": 0.20929047465324402, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 25, + "scan_ms": 2027, + "baseline_scan_ms": 1805, + "labelled_queries": 500, + "ndcg_at_k": 0.07454770310307464, + "doc_recall_at_k": 0.0590044341323899, + "boundary_gap": 0.0021550804376602173, + "neighbour_band": 0.021074384450912476, + "rank_noise": 0.026701573445971412 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.38280000000000036, + "top1_agreement": 0.266, + "mean_abs_score_error": 0.04901610241112195, + "max_abs_score_error": 0.337940514087677, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1873, + "labelled_queries": 500, + "ndcg_at_k": 0.06298463521735145, + "doc_recall_at_k": 0.050704437537062705, + "boundary_gap": 0.0021550804376602173, + "neighbour_band": 0.021074384450912476, + "rank_noise": 0.04901172234731914 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6255999999999999, + "top1_agreement": 0.522, + "mean_abs_score_error": 0.027086041633640038, + "max_abs_score_error": 0.1989678591489792, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 361, + "scan_ms": 1773, + "baseline_scan_ms": 1873, + "labelled_queries": 500, + "ndcg_at_k": 0.07779229234579542, + "doc_recall_at_k": 0.06029499219501658, + "boundary_gap": 0.0021550804376602173, + "neighbour_band": 0.021074384450912476, + "rank_noise": 0.02206189584306906 + } +] diff --git a/experiments/2026-11-text-format/results/recall/wiki-minilm-lookup.json b/experiments/2026-11-text-format/results/recall/wiki-minilm-lookup.json new file mode 100644 index 0000000..47b1150 --- /dev/null +++ b/experiments/2026-11-text-format/results/recall/wiki-minilm-lookup.json @@ -0,0 +1,112 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 5, + "scan_ms": 1948, + "baseline_scan_ms": 1954, + "labelled_queries": 500, + "ndcg_at_k": 0.09802736676401405, + "doc_recall_at_k": 0.15466666666666665, + "boundary_gap": 0.0017703473567962646, + "neighbour_band": 0.01845613121986389, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.9929999999999995, + "top1_agreement": 0.992, + "mean_abs_score_error": 0.0002694998121557963, + "max_abs_score_error": 0.0018947869539260864, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2499, + "baseline_scan_ms": 1975, + "labelled_queries": 500, + "ndcg_at_k": 0.09799699192328287, + "doc_recall_at_k": 0.15266666666666664, + "boundary_gap": 0.0017703473567962646, + "neighbour_band": 0.01845613121986389, + "rank_noise": 0.00028823884119537525 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.5104000000000003, + "top1_agreement": 0.356, + "mean_abs_score_error": 0.03962393288567214, + "max_abs_score_error": 0.2114839255809784, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 2025, + "baseline_scan_ms": 1956, + "labelled_queries": 500, + "ndcg_at_k": 0.06981485392816121, + "doc_recall_at_k": 0.12466666666666666, + "boundary_gap": 0.0017703473567962646, + "neighbour_band": 0.01845613121986389, + "rank_noise": 0.026674094353510185 + }, + { + "codec": "binary-hamming", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.342, + "top1_agreement": 0.22, + "mean_abs_score_error": 0.04854779517600671, + "max_abs_score_error": 0.338224858045578, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 26, + "scan_ms": 0, + "baseline_scan_ms": 1961, + "labelled_queries": 500, + "ndcg_at_k": 0.05396964236887714, + "doc_recall_at_k": 0.096, + "boundary_gap": 0.0017703473567962646, + "neighbour_band": 0.01845613121986389, + "rank_noise": 0.049314385762989105 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20000, + "queries": 500, + "recall_at_k": 0.6005999999999996, + "top1_agreement": 0.472, + "mean_abs_score_error": 0.030260667684609357, + "max_abs_score_error": 0.18758553266525269, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 378, + "scan_ms": 1863, + "baseline_scan_ms": 1920, + "labelled_queries": 500, + "ndcg_at_k": 0.08752228278292437, + "doc_recall_at_k": 0.14066666666666666, + "boundary_gap": 0.0017703473567962646, + "neighbour_band": 0.01845613121986389, + "rank_noise": 0.021754727283730162 + } +] diff --git a/experiments/2026-11-text-format/results/rescore/bullets-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/bullets-bge-small-descriptive.json new file mode 100644 index 0000000..6606188 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/bullets-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.0981435737515116, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.12166128663735999, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.2842, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.2, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.0981435737515116, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.13709719309434176, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.2842, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.558, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.12587270317682464, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.16072556953134076, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4013999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.662, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.13982197099277832, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.171228700792134, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4841999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.728, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.1475776080199204, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.17741557990836473, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5822000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.794, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15567081649185033, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.18185453427360343, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7081999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.878, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.15437181515446177, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17998629880071865, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8145999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.928, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.1572274049568546, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.18108655851646555, + "p10_recall_at_k": 0.7, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9141999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.1564255081021691, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1799752798540762, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9619999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1047140.352, + "depth": 57, + "doc_recall_at_k": 0.1459063334483363, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 56.732, + "metric": "cosine", + "ndcg_at_k": 0.17444540604632017, + "p10_recall_at_k": 0.3, + "p90_depth": 82, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5938, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.812, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1315246.08, + "depth": 231, + "doc_recall_at_k": 0.1580894708505701, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 231.28, + "metric": "cosine", + "ndcg_at_k": 0.18210991595329387, + "p10_recall_at_k": 0.6, + "p90_depth": 378, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8344000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.95, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2033104.896, + "depth": 699, + "doc_recall_at_k": 0.15629858763538795, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 698.636, + "metric": "cosine", + "ndcg_at_k": 0.1805594761745208, + "p10_recall_at_k": 0.8, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9449999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.13266368133956502, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.15724523921045508, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5419999999999995, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.452, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.13266368133956502, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.16250393348391262, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5419999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.908, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.14741990296817994, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.172975180408961, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7376000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.956, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.1531285726780672, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.17761436657208154, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8255999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.15496007709543358, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1789025039098836, + "p10_recall_at_k": 0.7, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9087999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15568939519546796, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.17916587836651612, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9723999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.15662012507939474, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17953686904981206, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9919999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.15695345841272806, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.17969501434099042, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9989999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.15695345841272806, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.17969501434099042, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 999926.784, + "depth": 26, + "doc_recall_at_k": 0.14606497200854948, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 25.994, + "metric": "cosine", + "ndcg_at_k": 0.17286541390602986, + "p10_recall_at_k": 0.6, + "p90_depth": 37, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7892000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1057751.04, + "depth": 64, + "doc_recall_at_k": 0.15670834968954633, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 63.64, + "metric": "cosine", + "ndcg_at_k": 0.17952288981250106, + "p10_recall_at_k": 0.8, + "p90_depth": 101, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9385999999999975, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1185005.568, + "depth": 146, + "doc_recall_at_k": 0.1571491791334488, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 146.488, + "metric": "cosine", + "ndcg_at_k": 0.17995311626101987, + "p10_recall_at_k": 0.9, + "p90_depth": 242, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9887999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 436 + ], + [ + 0.95, + 825 + ], + [ + 0.99, + 2691 + ], + [ + 1.0, + 10693 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 458.0, + "metric": "cosine", + "neighbour_band": 0.01013973355293274, + "queries": 500, + "rank_noise": 0.04885937247046683, + "recall_at_k": 0.2842, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 46 + ], + [ + 0.95, + 74 + ], + [ + 0.99, + 174 + ], + [ + 1.0, + 857 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 54.0, + "metric": "cosine", + "neighbour_band": 0.01013973355293274, + "queries": 500, + "rank_noise": 0.012701630639716221, + "recall_at_k": 0.5419999999999995, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/bullets-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/bullets-bge-small-lookup.json new file mode 100644 index 0000000..2a36eaf --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/bullets-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.3296666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.23784049426639223, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.2605999999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.252, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.3296666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.30244346494672014, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.2605999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.588, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.39166666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.34593080710691015, + "p10_recall_at_k": 0.1, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.36480000000000007, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.686, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.4086666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.35431163971540997, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4380000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.756, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.4326666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.3673531695175525, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.542, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.808, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.45266666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.38206510106893776, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6662000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.4683333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.3883663919623061, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7850000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.932, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.49133333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.39799440875289593, + "p10_recall_at_k": 0.7, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9011999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.49533333333333335, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.40020689286632893, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9597999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1053812.736, + "depth": 61, + "doc_recall_at_k": 0.4366666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 61.076, + "metric": "cosine", + "ndcg_at_k": 0.369900861810114, + "p10_recall_at_k": 0.4, + "p90_depth": 89, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5731999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.85, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1349695.488, + "depth": 254, + "doc_recall_at_k": 0.4766666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 253.708, + "metric": "cosine", + "ndcg_at_k": 0.39212105499332245, + "p10_recall_at_k": 0.6, + "p90_depth": 385, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8182000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.962, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2130444.288, + "depth": 762, + "doc_recall_at_k": 0.49333333333333335, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 762.008, + "metric": "cosine", + "ndcg_at_k": 0.39849670241706836, + "p10_recall_at_k": 0.8, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.943799999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.41966666666666663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.33911710299556397, + "p10_recall_at_k": 0.3, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5301999999999993, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.5, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.41966666666666663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.36853603230826215, + "p10_recall_at_k": 0.3, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5301999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.888, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.47966666666666663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.39280325513495457, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7173999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.942, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.48166666666666663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.39356427811371875, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7999999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.962, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.49066666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.39946601046797076, + "p10_recall_at_k": 0.7, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8905999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.49533333333333335, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.40129187816459383, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9627999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.49733333333333335, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.40178147080319865, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9929999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.49733333333333335, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.40178147080319865, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9993999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.49733333333333335, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.40178147080319865, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1000829.952, + "depth": 27, + "doc_recall_at_k": 0.48366666666666663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 26.582, + "metric": "cosine", + "ndcg_at_k": 0.3953087715085757, + "p10_recall_at_k": 0.6, + "p90_depth": 39, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7767999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1062770.688, + "depth": 67, + "doc_recall_at_k": 0.49866666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 66.908, + "metric": "cosine", + "ndcg_at_k": 0.4021340200594797, + "p10_recall_at_k": 0.8, + "p90_depth": 112, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9281999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1196282.88, + "depth": 154, + "doc_recall_at_k": 0.49466666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 153.83, + "metric": "cosine", + "ndcg_at_k": 0.400907259239597, + "p10_recall_at_k": 0.9, + "p90_depth": 276, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.988799999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 493 + ], + [ + 0.95, + 860 + ], + [ + 0.99, + 2216 + ], + [ + 1.0, + 6588 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 597.0, + "metric": "cosine", + "neighbour_band": 0.009650647640228271, + "queries": 500, + "rank_noise": 0.04810842078904391, + "recall_at_k": 0.2605999999999999, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 53 + ], + [ + 0.95, + 85 + ], + [ + 0.99, + 180 + ], + [ + 1.0, + 761 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 60.0, + "metric": "cosine", + "neighbour_band": 0.009650647640228271, + "queries": 500, + "rank_noise": 0.013291631288552421, + "recall_at_k": 0.5301999999999993, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/bullets-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/bullets-minilm-descriptive.json new file mode 100644 index 0000000..e0b8380 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/bullets-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.09445272278756318, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.11159863165280018, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.40259999999999985, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.258, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.09445272278756318, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.12047441388554014, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.40259999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.708, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.1152962082867068, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.13779999503850027, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5585999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.804, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.12455022451575679, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.1468325123832713, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6479999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.854, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.13047680056184788, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.15327540989442476, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7457999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.904, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.1363439100479884, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.1588415116053903, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.847, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.13756962230058523, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.15904567595809432, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9151999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.13863421828361738, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.15991030079200053, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9683999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.13973495088434998, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1604212208391406, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9912000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1015698.432, + "depth": 36, + "doc_recall_at_k": 0.12372613825271882, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 36.262, + "metric": "cosine", + "ndcg_at_k": 0.14877944000620033, + "p10_recall_at_k": 0.4, + "p90_depth": 54, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.677, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.884, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1134203.904, + "depth": 113, + "doc_recall_at_k": 0.13503184131871168, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 113.414, + "metric": "cosine", + "ndcg_at_k": 0.15781492722380172, + "p10_recall_at_k": 0.7, + "p90_depth": 187, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8699999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.976, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1429619.712, + "depth": 306, + "doc_recall_at_k": 0.13829368532906003, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 305.742, + "metric": "cosine", + "ndcg_at_k": 0.15945725113398534, + "p10_recall_at_k": 0.9, + "p90_depth": 566, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9637999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.12335244337562007, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.1417507386014969, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6350000000000006, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.51, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.12335244337562007, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.14836438443681363, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6350000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.934, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.1372419836614165, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.15887100264520385, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8365999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.1389081188279713, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.16016972760842527, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9099999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.13896130024330808, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1596274620939713, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.963599999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.14025832750772663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.16103503080554293, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9901999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.14068256993196904, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1613105898711521, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.14068256993196904, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1613105898711521, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9995999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.14068256993196904, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1613105898711521, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 992216.064, + "depth": 21, + "doc_recall_at_k": 0.13642118935397748, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.974, + "metric": "cosine", + "ndcg_at_k": 0.15762854291795492, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.839399999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1025728.512, + "depth": 43, + "doc_recall_at_k": 0.1390924411009173, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 42.792, + "metric": "cosine", + "ndcg_at_k": 0.16062651715718032, + "p10_recall_at_k": 0.9, + "p90_depth": 73, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9609999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1089432.576, + "depth": 84, + "doc_recall_at_k": 0.14003610528550442, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 84.266, + "metric": "cosine", + "ndcg_at_k": 0.16083275332640615, + "p10_recall_at_k": 1.0, + "p90_depth": 157, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9939999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 167 + ], + [ + 0.95, + 315 + ], + [ + 0.99, + 924 + ], + [ + 1.0, + 3447 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 152.0, + "metric": "cosine", + "neighbour_band": 0.0194818377494812, + "queries": 500, + "rank_noise": 0.04512499398028013, + "recall_at_k": 0.40259999999999985, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 29 + ], + [ + 0.95, + 42 + ], + [ + 0.99, + 99 + ], + [ + 1.0, + 965 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 29.0, + "metric": "cosine", + "neighbour_band": 0.0194818377494812, + "queries": 500, + "rank_noise": 0.019104849747577746, + "recall_at_k": 0.6350000000000006, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/bullets-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/bullets-minilm-lookup.json new file mode 100644 index 0000000..9999066 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/bullets-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.2279194797381081, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.3489999999999998, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.29, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.271192032950085, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3489999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.654, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.378, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.2965201604136898, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4878000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.77, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.423, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.31899594878498155, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.575, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.832, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.441, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.3262084246659815, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6721999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.876, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.456, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.3337704590904655, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7967999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.928, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.452, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.3323929701016853, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8873999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.459, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.3354100687243437, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9615999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.459, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.3352470048639693, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9867999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1021483.008, + "depth": 40, + "doc_recall_at_k": 0.421, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 40.028, + "metric": "cosine", + "ndcg_at_k": 0.3190999352041769, + "p10_recall_at_k": 0.4, + "p90_depth": 59, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6227999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.862, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1182249.984, + "depth": 145, + "doc_recall_at_k": 0.466, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 144.694, + "metric": "cosine", + "ndcg_at_k": 0.33716565007584354, + "p10_recall_at_k": 0.7, + "p90_depth": 241, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.858799999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1575324.672, + "depth": 401, + "doc_recall_at_k": 0.459, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 400.602, + "metric": "cosine", + "ndcg_at_k": 0.33532162233376217, + "p10_recall_at_k": 0.9, + "p90_depth": 728, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9597999999999978, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.401, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.29135438658795565, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6157999999999996, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.506, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.401, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3142380993185598, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6157999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.938, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.447, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3309954990358962, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8155999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.459, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.3362378038176712, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8955999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.456, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.3350696475628062, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9585999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.46, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.33555237305055924, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9891999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.46, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.33555237305055924, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9981999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.461, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.3358602332038654, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.461, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.3358602332038654, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 995229.696, + "depth": 23, + "doc_recall_at_k": 0.449, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 22.936, + "metric": "cosine", + "ndcg_at_k": 0.33248366144081004, + "p10_recall_at_k": 0.7, + "p90_depth": 33, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8461999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1039128.576, + "depth": 52, + "doc_recall_at_k": 0.461, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 51.516, + "metric": "cosine", + "ndcg_at_k": 0.3359547123015118, + "p10_recall_at_k": 0.9, + "p90_depth": 80, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.961799999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1126514.688, + "depth": 108, + "doc_recall_at_k": 0.461, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 108.408, + "metric": "cosine", + "ndcg_at_k": 0.3359454547828106, + "p10_recall_at_k": 1.0, + "p90_depth": 183, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9941999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 229 + ], + [ + 0.95, + 407 + ], + [ + 0.99, + 1159 + ], + [ + 1.0, + 7160 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 246.0, + "metric": "cosine", + "neighbour_band": 0.015997886657714844, + "queries": 500, + "rank_noise": 0.044314074597070555, + "recall_at_k": 0.3489999999999998, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 31 + ], + [ + 0.95, + 47 + ], + [ + 0.99, + 107 + ], + [ + 1.0, + 418 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 36.0, + "metric": "cosine", + "neighbour_band": 0.015997886657714844, + "queries": 500, + "rank_noise": 0.018190333912995394, + "recall_at_k": 0.6157999999999996, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/header-prose-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/header-prose-bge-small-descriptive.json new file mode 100644 index 0000000..6473619 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/header-prose-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.08149448694453044, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.10222367937432461, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.2949999999999997, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.216, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.08149448694453044, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.11503824367309977, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.2949999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.53, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.10554706469812362, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.13900305084007752, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.41480000000000045, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.638, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.11067662352218074, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.14255496870876444, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.49299999999999966, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.694, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.11956988415581146, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.15114461232882662, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5923999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.77, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.12664837003153057, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.15498014879843752, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7305999999999998, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.866, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.1290810557858723, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1567795183089273, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8371999999999991, + "rescore_ms": 6, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.1308863379403182, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1564865965316842, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9357999999999983, + "rescore_ms": 87, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.13077866069785837, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.15656488690542864, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9791999999999986, + "rescore_ms": 72, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1043020.8, + "depth": 54, + "doc_recall_at_k": 0.12118535408304996, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 54.05, + "metric": "cosine", + "ndcg_at_k": 0.1522715052243603, + "p10_recall_at_k": 0.4, + "p90_depth": 77, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6077999999999995, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.788, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1357980.672, + "depth": 259, + "doc_recall_at_k": 0.12915212790957606, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 259.102, + "metric": "cosine", + "ndcg_at_k": 0.15624614072232562, + "p10_recall_at_k": 0.7, + "p90_depth": 410, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8717999999999984, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.95, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2167474.176, + "depth": 786, + "doc_recall_at_k": 0.13044700046619814, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 786.116, + "metric": "cosine", + "ndcg_at_k": 0.1563695398618337, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9699999999999983, + "rescore_ms": 20, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.11173204810523511, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.1382769425677855, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5413999999999992, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.402, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.11173204810523511, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.14349657245801756, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5413999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.886, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.129403856443531, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1548537242026822, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7434000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.944, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.12799505156843563, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.15457498570681097, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8331999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.13178356492396492, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.15729621014060227, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9189999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.13141895714627286, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.1570363586976447, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9733999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.13234730201822387, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1574251106224156, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9963999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.13226034549648474, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.15732742400757713, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.13234730201822387, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1574662864514507, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1003370.496, + "depth": 28, + "doc_recall_at_k": 0.12918326863647372, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 28.236, + "metric": "cosine", + "ndcg_at_k": 0.15496179751986178, + "p10_recall_at_k": 0.6, + "p90_depth": 39, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8219999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1074456.576, + "depth": 75, + "doc_recall_at_k": 0.13069709770919602, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 74.516, + "metric": "cosine", + "ndcg_at_k": 0.15659172170567442, + "p10_recall_at_k": 0.9, + "p90_depth": 108, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.963599999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1234759.68, + "depth": 179, + "doc_recall_at_k": 0.13187145660759586, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 178.88, + "metric": "cosine", + "ndcg_at_k": 0.15703848780064356, + "p10_recall_at_k": 1.0, + "p90_depth": 277, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9969999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 332 + ], + [ + 0.95, + 606 + ], + [ + 0.99, + 1478 + ], + [ + 1.0, + 4746 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 419.5, + "metric": "cosine", + "neighbour_band": 0.010821998119354248, + "queries": 500, + "rank_noise": 0.05150047371425209, + "recall_at_k": 0.2949999999999997, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 44 + ], + [ + 0.95, + 69 + ], + [ + 0.99, + 141 + ], + [ + 1.0, + 758 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 51.0, + "metric": "cosine", + "neighbour_band": 0.010821998119354248, + "queries": 500, + "rank_noise": 0.013822674048947345, + "recall_at_k": 0.5413999999999992, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/header-prose-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/header-prose-bge-small-lookup.json new file mode 100644 index 0000000..a8a6e39 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/header-prose-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.29733333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.2036848401911748, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.2511999999999998, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.22, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.29733333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.25521925164865744, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.2511999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.582, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.36133333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.28453949816605806, + "p10_recall_at_k": 0.1, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3586, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.678, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.3713333333333333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.2914026319153473, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4338000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.734, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.42333333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.3272184285327726, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5276, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.812, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.4463333333333333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.34318725595818145, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6595999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.876, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.4563333333333333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.3460919022697865, + "p10_recall_at_k": 0.5, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7847999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.924, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.4833333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.3589168347726154, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9047999999999993, + "rescore_ms": 2, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.49333333333333335, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.36348455309981315, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9621999999999987, + "rescore_ms": 3, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1056666.624, + "depth": 63, + "doc_recall_at_k": 0.42333333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 62.934, + "metric": "cosine", + "ndcg_at_k": 0.32892804448097285, + "p10_recall_at_k": 0.3, + "p90_depth": 92, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5629999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.836, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1445907.456, + "depth": 316, + "doc_recall_at_k": 0.4743333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 316.346, + "metric": "cosine", + "ndcg_at_k": 0.35395427400937707, + "p10_recall_at_k": 0.7, + "p90_depth": 467, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8525999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2308657.152, + "depth": 878, + "doc_recall_at_k": 0.49333333333333335, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 878.032, + "metric": "cosine", + "ndcg_at_k": 0.36365537903917006, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9555999999999983, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.39866666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.28452712551551, + "p10_recall_at_k": 0.3, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5237999999999995, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.484, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.39866666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3247523437122374, + "p10_recall_at_k": 0.3, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5237999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.896, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.46166666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3501661255642696, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.711, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.966, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.4716666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.3569171188585361, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8001999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.4913333333333333, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.36569946946962545, + "p10_recall_at_k": 0.7, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8871999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.5033333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.36961090442789396, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9599999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.5053333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.3701990448749839, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9903999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.5053333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.3701990448749839, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9992000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.5053333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.3701990448749839, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006113.792, + "depth": 30, + "doc_recall_at_k": 0.4646666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.022, + "metric": "cosine", + "ndcg_at_k": 0.35614497653023214, + "p10_recall_at_k": 0.6, + "p90_depth": 42, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7956, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1085687.808, + "depth": 82, + "doc_recall_at_k": 0.5033333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 81.828, + "metric": "cosine", + "ndcg_at_k": 0.36996979208838504, + "p10_recall_at_k": 0.8, + "p90_depth": 125, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.947599999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1263667.2, + "depth": 198, + "doc_recall_at_k": 0.5053333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 197.7, + "metric": "cosine", + "ndcg_at_k": 0.3701990448749839, + "p10_recall_at_k": 1.0, + "p90_depth": 327, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9909999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 485 + ], + [ + 0.95, + 835 + ], + [ + 0.99, + 1955 + ], + [ + 1.0, + 6774 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 560.5, + "metric": "cosine", + "neighbour_band": 0.010596781969070435, + "queries": 500, + "rank_noise": 0.04954050211877942, + "recall_at_k": 0.2511999999999998, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 56 + ], + [ + 0.95, + 88 + ], + [ + 0.99, + 199 + ], + [ + 1.0, + 571 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 66.0, + "metric": "cosine", + "neighbour_band": 0.010596781969070435, + "queries": 500, + "rank_noise": 0.014511191165207184, + "recall_at_k": 0.5237999999999995, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/header-prose-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/header-prose-minilm-descriptive.json new file mode 100644 index 0000000..aa43959 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/header-prose-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.07745067740614438, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.09261660823159618, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.40560000000000057, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.25, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.07745067740614438, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.10216180688126161, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.40560000000000057, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.718, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.09596401465563348, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.11878875801326434, + "p10_recall_at_k": 0.3, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5581999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.832, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.09826920597897582, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.12165403348574165, + "p10_recall_at_k": 0.4, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6467999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.888, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.10364207475227721, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.12787485969799797, + "p10_recall_at_k": 0.5, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7566000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.938, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.10667119736599401, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.13064029315728296, + "p10_recall_at_k": 0.7, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8717999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.962, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.10837889925350445, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1338683885354315, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9421999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.10927855619433784, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.13528397562399191, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9815999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.10988677020255189, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.13581400336507282, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9931999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1018312.704, + "depth": 38, + "doc_recall_at_k": 0.10318338633619314, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 37.964, + "metric": "cosine", + "ndcg_at_k": 0.12714105642829482, + "p10_recall_at_k": 0.5, + "p90_depth": 54, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6966000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.922, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1145017.344, + "depth": 120, + "doc_recall_at_k": 0.1082323584576049, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 120.454, + "metric": "cosine", + "ndcg_at_k": 0.13328761899082914, + "p10_recall_at_k": 0.8, + "p90_depth": 187, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8997999999999978, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1483625.472, + "depth": 341, + "doc_recall_at_k": 0.10961607423185589, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 340.902, + "metric": "cosine", + "ndcg_at_k": 0.13589332029777407, + "p10_recall_at_k": 0.9, + "p90_depth": 606, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9775999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.09712727416725812, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.12156668081764381, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6301999999999991, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.522, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.09712727416725812, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.12496557570739285, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6301999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.10619393658373064, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.13292929183553504, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8335999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.10827763219455724, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.13439513414579218, + "p10_recall_at_k": 0.8, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9061999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.10958210860403367, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.13495796385124587, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9627999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.10992639066819684, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.13568317232478697, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9925999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.11018328183045886, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1359142550509522, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9983999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.11018328183045886, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1359142550509522, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.11018328183045886, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1359142550509522, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 995807.232, + "depth": 23, + "doc_recall_at_k": 0.10675634470555054, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 23.312, + "metric": "cosine", + "ndcg_at_k": 0.13392416224457912, + "p10_recall_at_k": 0.7, + "p90_depth": 33, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8609999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1037368.32, + "depth": 50, + "doc_recall_at_k": 0.11024710642818507, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.37, + "metric": "cosine", + "ndcg_at_k": 0.13583558260154474, + "p10_recall_at_k": 0.9, + "p90_depth": 78, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9689999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1121295.36, + "depth": 105, + "doc_recall_at_k": 0.1105244583010471, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 105.01, + "metric": "cosine", + "ndcg_at_k": 0.13598309204621703, + "p10_recall_at_k": 1.0, + "p90_depth": 174, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9955999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 127 + ], + [ + 0.95, + 225 + ], + [ + 0.99, + 768 + ], + [ + 1.0, + 4592 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 136.5, + "metric": "cosine", + "neighbour_band": 0.021202847361564636, + "queries": 500, + "rank_noise": 0.04895294478025753, + "recall_at_k": 0.40560000000000057, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 29 + ], + [ + 0.95, + 44 + ], + [ + 0.99, + 91 + ], + [ + 1.0, + 794 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 32.0, + "metric": "cosine", + "neighbour_band": 0.021202847361564636, + "queries": 500, + "rank_noise": 0.02180369779438052, + "recall_at_k": 0.6301999999999991, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/header-prose-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/header-prose-minilm-lookup.json new file mode 100644 index 0000000..5b66738 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/header-prose-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.219, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.14305307340712115, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.3497999999999997, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.252, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.219, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.16839127499970316, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3497999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.656, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.25266666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.18155283452762166, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4976000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.76, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.2666666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.189935382654253, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5922000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.824, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.2816666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.19902430943318805, + "p10_recall_at_k": 0.5, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6979999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.886, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.29066666666666663, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.20402163267248072, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8244000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.95, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.29266666666666663, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.20417853473276631, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9131999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.30266666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.20825275904227863, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9737999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.30266666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.20822388928003513, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9923999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1029479.424, + "depth": 45, + "doc_recall_at_k": 0.2736666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 45.234, + "metric": "cosine", + "ndcg_at_k": 0.19418511980952036, + "p10_recall_at_k": 0.5, + "p90_depth": 65, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6751999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.876, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1198101.504, + "depth": 155, + "doc_recall_at_k": 0.30066666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 155.014, + "metric": "cosine", + "ndcg_at_k": 0.2075923100463281, + "p10_recall_at_k": 0.7, + "p90_depth": 238, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8883999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1646567.424, + "depth": 447, + "doc_recall_at_k": 0.29866666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 446.984, + "metric": "cosine", + "ndcg_at_k": 0.20654034466806256, + "p10_recall_at_k": 0.9, + "p90_depth": 721, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9739999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.2696666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.18205625038313994, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6155999999999997, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.45, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.2696666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.1965302916776615, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6155999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.938, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.2876666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.2031401095865601, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8125999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.29333333333333333, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.20507366682392, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8979999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.29866666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.20679903526412052, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9579999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.30466666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.20887874937229875, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9905999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.30266666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.2082766893809708, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9983999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.30266666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.20822388928003513, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.30266666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.20822388928003513, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 999091.2, + "depth": 25, + "doc_recall_at_k": 0.29566666666666663, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 25.45, + "metric": "cosine", + "ndcg_at_k": 0.20629172566593537, + "p10_recall_at_k": 0.7, + "p90_depth": 35, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.866599999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1053471.744, + "depth": 61, + "doc_recall_at_k": 0.3033333333333333, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 60.854, + "metric": "cosine", + "ndcg_at_k": 0.20857879081737962, + "p10_recall_at_k": 0.9, + "p90_depth": 92, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9749999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1168054.2719999999, + "depth": 135, + "doc_recall_at_k": 0.30466666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 135.452, + "metric": "cosine", + "ndcg_at_k": 0.2088020189326709, + "p10_recall_at_k": 1.0, + "p90_depth": 221, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9979999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 177 + ], + [ + 0.95, + 330 + ], + [ + 0.99, + 882 + ], + [ + 1.0, + 3016 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 204.0, + "metric": "cosine", + "neighbour_band": 0.018535256385803223, + "queries": 500, + "rank_noise": 0.04829896583523255, + "recall_at_k": 0.3497999999999997, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 31 + ], + [ + 0.95, + 46 + ], + [ + 0.99, + 95 + ], + [ + 1.0, + 528 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 35.0, + "metric": "cosine", + "neighbour_band": 0.018535256385803223, + "queries": 500, + "rank_noise": 0.020787784400134623, + "recall_at_k": 0.6155999999999997, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/json-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/json-bge-small-descriptive.json new file mode 100644 index 0000000..651e06f --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/json-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.11945747342210479, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.1481000584218245, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.33360000000000006, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.196, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.11945747342210479, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.16381469151464653, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.33360000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.604, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.14894522165563498, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1938748581024718, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4734, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.71, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.15868111586341002, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.20095380491558154, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5612, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.756, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.167413315852711, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.21078849566707814, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6630000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.82, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.17703038222419146, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.21862991519413338, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7864000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.904, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.17977775241646332, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.2207358623817346, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8755999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.956, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.18203855707759117, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.22184174587844777, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9485999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.18388659891200143, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.22263592545518926, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9797999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1013286.912, + "depth": 35, + "doc_recall_at_k": 0.16044014495818476, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 34.692, + "metric": "cosine", + "ndcg_at_k": 0.20511006498785472, + "p10_recall_at_k": 0.3, + "p90_depth": 51, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5837999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.798, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1133564.928, + "depth": 113, + "doc_recall_at_k": 0.17735754211420968, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 112.998, + "metric": "cosine", + "ndcg_at_k": 0.218358199633952, + "p10_recall_at_k": 0.6, + "p90_depth": 189, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8020000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.924, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1412987.904, + "depth": 295, + "doc_recall_at_k": 0.1825261016838277, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 294.914, + "metric": "cosine", + "ndcg_at_k": 0.22265429376229529, + "p10_recall_at_k": 0.8, + "p90_depth": 517, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9173999999999979, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.15061555613481642, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.18908401341979986, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5861999999999995, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.402, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.15061555613481642, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.19825547900805415, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5861999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.914, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.1701757922232262, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.2140447126101148, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7844000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.966, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.17758452379211467, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.21851450486800977, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8621999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.1816850295026739, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.22142134853032236, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9305999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.18401875201260973, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.2231763898363219, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.975599999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.18462644561923947, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.22387556839495704, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9951999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.18462644561923947, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.22384205850421124, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.18462644561923947, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.22384205850421124, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990959.616, + "depth": 20, + "doc_recall_at_k": 0.16830669524747768, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.156, + "metric": "cosine", + "ndcg_at_k": 0.2130507474842056, + "p10_recall_at_k": 0.6, + "p90_depth": 28, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7792000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1020045.312, + "depth": 39, + "doc_recall_at_k": 0.18055705383499465, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 39.092, + "metric": "cosine", + "ndcg_at_k": 0.22072985779763266, + "p10_recall_at_k": 0.8, + "p90_depth": 60, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9043999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1072899.072, + "depth": 74, + "doc_recall_at_k": 0.18360319002204775, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 73.502, + "metric": "cosine", + "ndcg_at_k": 0.22299848855060853, + "p10_recall_at_k": 0.9, + "p90_depth": 114, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9683999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 256 + ], + [ + 0.95, + 505 + ], + [ + 0.99, + 1614 + ], + [ + 1.0, + 10388 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 232.0, + "metric": "cosine", + "neighbour_band": 0.009328246116638184, + "queries": 500, + "rank_noise": 0.04460908965790574, + "recall_at_k": 0.33360000000000006, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 40 + ], + [ + 0.95, + 62 + ], + [ + 0.99, + 155 + ], + [ + 1.0, + 828 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 40.0, + "metric": "cosine", + "neighbour_band": 0.009328246116638184, + "queries": 500, + "rank_noise": 0.01006400530963398, + "recall_at_k": 0.5861999999999995, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/json-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/json-bge-small-lookup.json new file mode 100644 index 0000000..5896f84 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/json-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.369, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.26683980433509313, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.2724, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.284, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.369, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.33674375849882177, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.2724, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.59, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.427, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3734852607610577, + "p10_recall_at_k": 0.1, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3778000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.678, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.456, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.38829566471189364, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.45120000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.734, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.479, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.4023513144179158, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5464, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.796, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.524, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.42266103995828797, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6773999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.858, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.5356666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.43064070846706526, + "p10_recall_at_k": 0.5, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7927999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.926, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.5536666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.43892429261742855, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9143999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.5576666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.44111935535181895, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9639999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1021636.608, + "depth": 40, + "doc_recall_at_k": 0.467, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 40.128, + "metric": "cosine", + "ndcg_at_k": 0.4009131326389756, + "p10_recall_at_k": 0.3, + "p90_depth": 58, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.49480000000000024, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.768, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1183048.704, + "depth": 145, + "doc_recall_at_k": 0.532, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 145.214, + "metric": "cosine", + "ndcg_at_k": 0.4293857325825374, + "p10_recall_at_k": 0.5, + "p90_depth": 230, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.7369999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.92, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1575763.9679999999, + "depth": 401, + "doc_recall_at_k": 0.555, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 400.888, + "metric": "cosine", + "ndcg_at_k": 0.43951322985292074, + "p10_recall_at_k": 0.8, + "p90_depth": 679, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.8991999999999979, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.45766666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.35536840571637485, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5427999999999991, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.53, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.45766666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3917774108781422, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5427999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.5196666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.4221879780149283, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7321999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.956, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.5396666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.4314989172755087, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8213999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.5596666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.43993891794597756, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.907799999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.5616666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.4408376848451904, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9695999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.5616666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.44078729391174476, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9931999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.5616666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.44072600267149176, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9995999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.5616666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.44072600267149176, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 993205.248, + "depth": 22, + "doc_recall_at_k": 0.5146666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 21.618, + "metric": "cosine", + "ndcg_at_k": 0.4183094120991957, + "p10_recall_at_k": 0.5, + "p90_depth": 31, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7425999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.962, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1030155.264, + "depth": 46, + "doc_recall_at_k": 0.5496666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 45.674, + "metric": "cosine", + "ndcg_at_k": 0.436526487609876, + "p10_recall_at_k": 0.7, + "p90_depth": 81, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8827999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1101726.72, + "depth": 92, + "doc_recall_at_k": 0.5656666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 92.27, + "metric": "cosine", + "ndcg_at_k": 0.4422556464427708, + "p10_recall_at_k": 0.9, + "p90_depth": 177, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9657999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 437 + ], + [ + 0.95, + 760 + ], + [ + 0.99, + 1754 + ], + [ + 1.0, + 5044 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 460.5, + "metric": "cosine", + "neighbour_band": 0.008194446563720703, + "queries": 500, + "rank_noise": 0.0441365510434023, + "recall_at_k": 0.2724, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 48 + ], + [ + 0.95, + 75 + ], + [ + 0.99, + 170 + ], + [ + 1.0, + 564 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 52.0, + "metric": "cosine", + "neighbour_band": 0.008194446563720703, + "queries": 500, + "rank_noise": 0.010636494620821063, + "recall_at_k": 0.5427999999999991, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/json-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/json-minilm-descriptive.json new file mode 100644 index 0000000..57e4d59 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/json-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.09182177302900196, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.10903977707801757, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.44039999999999935, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.302, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.09182177302900196, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.11568654944937141, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.44039999999999935, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.748, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.10491546841730293, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1286917413144237, + "p10_recall_at_k": 0.3, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6010000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.842, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.1070232768169426, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.13249404302412943, + "p10_recall_at_k": 0.4, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6928, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.882, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.11207296440990554, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1362646839640393, + "p10_recall_at_k": 0.5, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7802000000000004, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.92, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.11415176672648346, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.13874399154078057, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8677999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.952, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.11660349595825349, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.14103074109355862, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9259999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.11618383265476662, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1411288233693939, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9799999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.11636085789594541, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.14154467677743449, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9957999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 994827.264, + "depth": 23, + "doc_recall_at_k": 0.10546157230204276, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 22.674, + "metric": "cosine", + "ndcg_at_k": 0.12949328109193728, + "p10_recall_at_k": 0.3, + "p90_depth": 33, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6188000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.87, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1042593.792, + "depth": 54, + "doc_recall_at_k": 0.11144370280315305, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 53.772, + "metric": "cosine", + "ndcg_at_k": 0.1363708306619129, + "p10_recall_at_k": 0.6, + "p90_depth": 91, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.7868000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.928, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1131396.096, + "depth": 112, + "doc_recall_at_k": 0.11327941386456866, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 111.586, + "metric": "cosine", + "ndcg_at_k": 0.13953226623848639, + "p10_recall_at_k": 0.7, + "p90_depth": 199, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.890199999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.964, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.10086089813434157, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.12197343548835846, + "p10_recall_at_k": 0.5, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6810000000000002, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.504, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.10086089813434157, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.12797835927074094, + "p10_recall_at_k": 0.5, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6810000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.11115079341904864, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.13728164151656624, + "p10_recall_at_k": 0.7, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8669999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.11492461872228568, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.1400345722225628, + "p10_recall_at_k": 0.8, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9297999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.11600651512355104, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.14106392609908514, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.972799999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.11696793970302721, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.14202731042967284, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9947999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.11688222541731294, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.14217403827387157, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.11688222541731294, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.14217403827387157, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.11688222541731294, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.14217403827387157, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 986655.744, + "depth": 17, + "doc_recall_at_k": 0.10951445458131313, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 17.354, + "metric": "cosine", + "ndcg_at_k": 0.1357102907351855, + "p10_recall_at_k": 0.7, + "p90_depth": 24, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8357999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1005447.1680000001, + "depth": 30, + "doc_recall_at_k": 0.11357035494302188, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 29.588, + "metric": "cosine", + "ndcg_at_k": 0.13952941185146073, + "p10_recall_at_k": 0.8, + "p90_depth": 45, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9351999999999973, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036234.752, + "depth": 50, + "doc_recall_at_k": 0.11608694883128778, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 49.632, + "metric": "cosine", + "ndcg_at_k": 0.1416577091226089, + "p10_recall_at_k": 0.9, + "p90_depth": 85, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.977199999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 147 + ], + [ + 0.95, + 291 + ], + [ + 0.99, + 720 + ], + [ + 1.0, + 3918 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 127.5, + "metric": "cosine", + "neighbour_band": 0.020144686102867126, + "queries": 500, + "rank_noise": 0.04268366874841095, + "recall_at_k": 0.44039999999999935, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 25 + ], + [ + 0.95, + 36 + ], + [ + 0.99, + 80 + ], + [ + 1.0, + 313 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 27.0, + "metric": "cosine", + "neighbour_band": 0.020144686102867126, + "queries": 500, + "rank_noise": 0.01758386962810289, + "recall_at_k": 0.6810000000000002, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/json-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/json-minilm-lookup.json new file mode 100644 index 0000000..db49624 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/json-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.235, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.16082108801678177, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.3667999999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.264, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.235, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.1867395592193948, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3667999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.676, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.276, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.21194529207417084, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5170000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.796, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.29866666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.21925314007142704, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6056000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.834, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.3156666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.22816815381941755, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7055999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.884, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.32866666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.2333151926859954, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8263999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.948, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.33266666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.2363573262422714, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9113999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.33466666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.23942363781528148, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9715999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.33666666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.2406854973224244, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9915999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 999038.976, + "depth": 25, + "doc_recall_at_k": 0.284, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 25.416, + "metric": "cosine", + "ndcg_at_k": 0.2147175964106842, + "p10_recall_at_k": 0.3, + "p90_depth": 36, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.556, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.814, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1062859.776, + "depth": 67, + "doc_recall_at_k": 0.3156666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 66.966, + "metric": "cosine", + "ndcg_at_k": 0.2276100391433336, + "p10_recall_at_k": 0.5, + "p90_depth": 106, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.7557999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.932, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1181583.3599999999, + "depth": 144, + "doc_recall_at_k": 0.33666666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 144.26, + "metric": "cosine", + "ndcg_at_k": 0.23926118760127632, + "p10_recall_at_k": 0.7, + "p90_depth": 244, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.8885999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.269, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.19358658187262145, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6163999999999995, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.484, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.269, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.21392037654343676, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6163999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.932, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.3196666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.23304121321430357, + "p10_recall_at_k": 0.7, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.830999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.33166666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.23710979770601678, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9075999999999977, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.33866666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.24153235363305478, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9637999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.33466666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.24007849790754515, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9913999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.33466666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.24007849790754515, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9985999999999998, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.33466666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.24007849790754515, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 2, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.33466666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.24007849790754515, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 6, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 988514.304, + "depth": 19, + "doc_recall_at_k": 0.315, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 18.564, + "metric": "cosine", + "ndcg_at_k": 0.23074526208270624, + "p10_recall_at_k": 0.7, + "p90_depth": 25, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8101999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1011855.36, + "depth": 34, + "doc_recall_at_k": 0.32966666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 33.76, + "metric": "cosine", + "ndcg_at_k": 0.2380963632675773, + "p10_recall_at_k": 0.8, + "p90_depth": 52, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9257999999999971, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1050983.424, + "depth": 59, + "doc_recall_at_k": 0.33366666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 59.234, + "metric": "cosine", + "ndcg_at_k": 0.2394305861132638, + "p10_recall_at_k": 0.9, + "p90_depth": 98, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9769999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 178 + ], + [ + 0.95, + 317 + ], + [ + 0.99, + 853 + ], + [ + 1.0, + 4123 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 190.0, + "metric": "cosine", + "neighbour_band": 0.016464248299598694, + "queries": 500, + "rank_noise": 0.0423662223916466, + "recall_at_k": 0.3667999999999999, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 29 + ], + [ + 0.95, + 44 + ], + [ + 0.99, + 93 + ], + [ + 1.0, + 358 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 33.0, + "metric": "cosine", + "neighbour_band": 0.016464248299598694, + "queries": 500, + "rank_noise": 0.017009062579887112, + "recall_at_k": 0.6163999999999995, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/kv-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/kv-bge-small-descriptive.json new file mode 100644 index 0000000..9b50788 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/kv-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.09511220015601164, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.11848966594854614, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.3025999999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.168, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.09511220015601164, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.13798941155327046, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3025999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.592, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.1268652017240255, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1644366651756854, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4264000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.688, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.13478381108088905, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.17047028526475563, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5034, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.736, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.1459836830376966, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.17916683073131873, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6017999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.772, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15490979997232135, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.18439285369233077, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7293999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.868, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.15909740616808687, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1889977902344367, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8328000000000004, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.92, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.16167762975817054, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.19001508397479483, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9231999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.964, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.16260603088920333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.19039712865416458, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9689999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1035420.672, + "depth": 49, + "doc_recall_at_k": 0.1460295081717094, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 49.102, + "metric": "cosine", + "ndcg_at_k": 0.17857258553556424, + "p10_recall_at_k": 0.3, + "p90_depth": 71, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5897999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.78, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1271568.384, + "depth": 203, + "doc_recall_at_k": 0.15849646085335553, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 202.844, + "metric": "cosine", + "ndcg_at_k": 0.18885776558529715, + "p10_recall_at_k": 0.6, + "p90_depth": 327, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8360000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.938, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1927772.16, + "depth": 630, + "doc_recall_at_k": 0.16255969519180377, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 630.06, + "metric": "cosine", + "ndcg_at_k": 0.19057481775782942, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.952399999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.1326244822305499, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.15580102729524237, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5553999999999992, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.42, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.1326244822305499, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.1644336031253538, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5553999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.15509963757397843, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.18172434331734966, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7536000000000004, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.946, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.15839829856499568, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.18646061282309606, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8411999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.976, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.16062056187268509, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.18867122546239373, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9203999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.16217004294716608, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.18976831833437918, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9711999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.16282837892001817, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.19076585480142486, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9925999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.1627244828161221, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.19070521998037948, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.1627244828161221, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.19070521998037948, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 997459.968, + "depth": 24, + "doc_recall_at_k": 0.15578646920801778, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 24.388, + "metric": "cosine", + "ndcg_at_k": 0.18342154665301866, + "p10_recall_at_k": 0.6, + "p90_depth": 33, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7906000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1048510.464, + "depth": 58, + "doc_recall_at_k": 0.16055235785448096, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 57.624, + "metric": "cosine", + "ndcg_at_k": 0.18861936377382288, + "p10_recall_at_k": 0.8, + "p90_depth": 84, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9373999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1153453.0559999999, + "depth": 126, + "doc_recall_at_k": 0.16289997699161626, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 125.946, + "metric": "cosine", + "ndcg_at_k": 0.19049551375118956, + "p10_recall_at_k": 0.9, + "p90_depth": 194, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9833999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 370 + ], + [ + 0.95, + 722 + ], + [ + 0.99, + 2039 + ], + [ + 1.0, + 9720 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 378.0, + "metric": "cosine", + "neighbour_band": 0.009874284267425537, + "queries": 500, + "rank_noise": 0.046493915656692544, + "recall_at_k": 0.3025999999999999, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 43 + ], + [ + 0.95, + 69 + ], + [ + 0.99, + 177 + ], + [ + 1.0, + 1163 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 50.0, + "metric": "cosine", + "neighbour_band": 0.009874284267425537, + "queries": 500, + "rank_noise": 0.011901817115159499, + "recall_at_k": 0.5553999999999992, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/kv-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/kv-bge-small-lookup.json new file mode 100644 index 0000000..eec244e --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/kv-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.3586666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.2590545766537737, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.25260000000000005, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.246, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.3586666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.32809415982515694, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.25260000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.566, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.39566666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3473363570145051, + "p10_recall_at_k": 0.1, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.35820000000000035, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.658, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.4136666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.35720613973525556, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4294, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.718, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.45733333333333337, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.38468396262845683, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5282, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.794, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.49633333333333335, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.40226590036354687, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6627999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.868, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.531, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.4220733597267482, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7853999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.926, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.544, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.43033964367935507, + "p10_recall_at_k": 0.7, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.902599999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.549, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.43197845684495695, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9529999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1038753.792, + "depth": 51, + "doc_recall_at_k": 0.45166666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 51.272, + "metric": "cosine", + "ndcg_at_k": 0.3801077698484994, + "p10_recall_at_k": 0.3, + "p90_depth": 73, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5261999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.788, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1310103.552, + "depth": 228, + "doc_recall_at_k": 0.5373333333333333, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 227.932, + "metric": "cosine", + "ndcg_at_k": 0.42519332416605343, + "p10_recall_at_k": 0.6, + "p90_depth": 350, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8009999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.954, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1994210.304, + "depth": 673, + "doc_recall_at_k": 0.545, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 673.314, + "metric": "cosine", + "ndcg_at_k": 0.43100144478979063, + "p10_recall_at_k": 0.8, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.933399999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.4713333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.35718302201402347, + "p10_recall_at_k": 0.3, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5289999999999997, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.476, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.4713333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3977408143097767, + "p10_recall_at_k": 0.3, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5289999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.5086666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.4158460776463377, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7169999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.96, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.534, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.4260434017876221, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8107999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.543, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.4293477652154254, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9037999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.555, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.4341907574343113, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9677999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.557, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.43468617900882073, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9927999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.559, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.43525936923790515, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.559, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.43525936923790515, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 997696.512, + "depth": 25, + "doc_recall_at_k": 0.524, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 24.542, + "metric": "cosine", + "ndcg_at_k": 0.4227412944321085, + "p10_recall_at_k": 0.6, + "p90_depth": 35, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7634, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1049416.704, + "depth": 58, + "doc_recall_at_k": 0.544, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 58.214, + "metric": "cosine", + "ndcg_at_k": 0.43026064274669124, + "p10_recall_at_k": 0.8, + "p90_depth": 94, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9159999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1155984.384, + "depth": 128, + "doc_recall_at_k": 0.557, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 127.594, + "metric": "cosine", + "ndcg_at_k": 0.4347697765993003, + "p10_recall_at_k": 0.9, + "p90_depth": 231, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9809999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 487 + ], + [ + 0.95, + 947 + ], + [ + 0.99, + 2152 + ], + [ + 1.0, + 6743 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 628.5, + "metric": "cosine", + "neighbour_band": 0.009523779153823853, + "queries": 500, + "rank_noise": 0.0471105978607601, + "recall_at_k": 0.25260000000000005, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 49 + ], + [ + 0.95, + 78 + ], + [ + 0.99, + 172 + ], + [ + 1.0, + 698 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 60.0, + "metric": "cosine", + "neighbour_band": 0.009523779153823853, + "queries": 500, + "rank_noise": 0.012737702651780227, + "recall_at_k": 0.5289999999999997, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/kv-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/kv-minilm-descriptive.json new file mode 100644 index 0000000..e51b1e3 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/kv-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.08998031292890955, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.10947132568262843, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.39699999999999974, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.282, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.08998031292890955, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.11438894330724907, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.39699999999999974, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.734, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.1054714960468702, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.12766969593630917, + "p10_recall_at_k": 0.3, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5464000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.818, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.11363398997092908, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.13435962763926043, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6342, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.862, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.11816041949344905, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.13643338515545464, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7366, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.902, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.1222826785402247, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.14023422761312215, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8451999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.936, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.12379608369182186, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.14212095998919164, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9211999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.12464527299909547, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.14213498657969995, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9727999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.12514773605328267, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.14237675304885292, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9867999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1015449.6, + "depth": 36, + "doc_recall_at_k": 0.1117552030306576, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 36.1, + "metric": "cosine", + "ndcg_at_k": 0.13266782615755301, + "p10_recall_at_k": 0.4, + "p90_depth": 55, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6663999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.888, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1142486.016, + "depth": 119, + "doc_recall_at_k": 0.12174535990470525, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 118.806, + "metric": "cosine", + "ndcg_at_k": 0.1402529553457006, + "p10_recall_at_k": 0.7, + "p90_depth": 213, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8779999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1455246.3360000001, + "depth": 322, + "doc_recall_at_k": 0.12455158190862417, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 322.426, + "metric": "cosine", + "ndcg_at_k": 0.14201043064017901, + "p10_recall_at_k": 0.9, + "p90_depth": 637, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9635999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.10755841087952485, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.12322216903858094, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6322000000000002, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.51, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.10755841087952485, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.12989980852881264, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6322000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.952, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.12076821533541073, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.13929959177427848, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8347999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.12428883958958956, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.14163280216885152, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9103999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.12667055404760863, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1431886257216553, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9615999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.12621329530023725, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.1431790122222418, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9899999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.1261704381573801, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.14328363464109778, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.1261704381573801, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.14328363464109778, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9995999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.1261704381573801, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.14328363464109778, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 991454.208, + "depth": 20, + "doc_recall_at_k": 0.12002949897930308, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.478, + "metric": "cosine", + "ndcg_at_k": 0.1389835419250213, + "p10_recall_at_k": 0.7, + "p90_depth": 29, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8417999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1023000.576, + "depth": 41, + "doc_recall_at_k": 0.12477503557972172, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 41.016, + "metric": "cosine", + "ndcg_at_k": 0.14230503458494753, + "p10_recall_at_k": 0.9, + "p90_depth": 68, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9539999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1082505.216, + "depth": 80, + "doc_recall_at_k": 0.12639534658228851, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 79.756, + "metric": "cosine", + "ndcg_at_k": 0.14306861840179608, + "p10_recall_at_k": 1.0, + "p90_depth": 150, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9919999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 155 + ], + [ + 0.95, + 312 + ], + [ + 0.99, + 1137 + ], + [ + 1.0, + 7996 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 149.0, + "metric": "cosine", + "neighbour_band": 0.019775748252868652, + "queries": 500, + "rank_noise": 0.04598879890536952, + "recall_at_k": 0.39699999999999974, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 29 + ], + [ + 0.95, + 44 + ], + [ + 0.99, + 99 + ], + [ + 1.0, + 1213 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 29.0, + "metric": "cosine", + "neighbour_band": 0.019775748252868652, + "queries": 500, + "rank_noise": 0.018839253033153817, + "recall_at_k": 0.6322000000000002, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/kv-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/kv-minilm-lookup.json new file mode 100644 index 0000000..51f9361 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/kv-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.267, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.18525895646799123, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.33500000000000024, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.272, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.267, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.21125443231245206, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.33500000000000024, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.612, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.3146666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.23187243478796904, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4704000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.728, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.352, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.2493254936250064, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5580000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.8, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.366, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.26086116924682556, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6654, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.858, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.375, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.2670766834970435, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7869999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.906, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.384, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.2729967509613047, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8735999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.964, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.389, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.27683818200612564, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9571999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.387, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.2760770006869296, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9825999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1023654.912, + "depth": 41, + "doc_recall_at_k": 0.35466666666666663, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 41.442, + "metric": "cosine", + "ndcg_at_k": 0.2582773683145546, + "p10_recall_at_k": 0.4, + "p90_depth": 58, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6202000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.844, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1197275.136, + "depth": 154, + "doc_recall_at_k": 0.386, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 154.476, + "metric": "cosine", + "ndcg_at_k": 0.27422654418109654, + "p10_recall_at_k": 0.7, + "p90_depth": 249, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8555999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.966, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1633870.848, + "depth": 439, + "doc_recall_at_k": 0.385, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 438.718, + "metric": "cosine", + "ndcg_at_k": 0.2753856031614746, + "p10_recall_at_k": 0.9, + "p90_depth": 778, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9583999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.343, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.24175039331467338, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.608799999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.55, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.343, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.2585731256019734, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.608799999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.926, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.379, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.2734040166731461, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8117999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.384, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.27514966597002144, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8907999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.383, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.2748968110429658, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9483999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.383, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.2747864566600778, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9855999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.385, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.2754988710342938, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9959999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.385, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.2754988710342938, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.385, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.2754988710342938, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 995281.92, + "depth": 23, + "doc_recall_at_k": 0.374, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 22.97, + "metric": "cosine", + "ndcg_at_k": 0.2717341518292762, + "p10_recall_at_k": 0.7, + "p90_depth": 32, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8361999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1038695.424, + "depth": 51, + "doc_recall_at_k": 0.381, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 51.234, + "metric": "cosine", + "ndcg_at_k": 0.27423480640183606, + "p10_recall_at_k": 0.9, + "p90_depth": 83, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9543999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1125789.696, + "depth": 108, + "doc_recall_at_k": 0.383, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 107.936, + "metric": "cosine", + "ndcg_at_k": 0.2748153264223212, + "p10_recall_at_k": 1.0, + "p90_depth": 188, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9921999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 250 + ], + [ + 0.95, + 441 + ], + [ + 0.99, + 1412 + ], + [ + 1.0, + 5258 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 264.0, + "metric": "cosine", + "neighbour_band": 0.016394585371017456, + "queries": 500, + "rank_noise": 0.04495338965963287, + "recall_at_k": 0.33500000000000024, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 32 + ], + [ + 0.95, + 51 + ], + [ + 0.99, + 124 + ], + [ + 1.0, + 1132 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 37.5, + "metric": "cosine", + "neighbour_band": 0.016394585371017456, + "queries": 500, + "rank_noise": 0.017943018844851544, + "recall_at_k": 0.608799999999999, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/paragraph-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/paragraph-bge-small-descriptive.json new file mode 100644 index 0000000..c0b6891 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/paragraph-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.08208201927830254, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.10343436813972624, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.28460000000000013, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.208, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.08208201927830254, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.12002606843091722, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.28460000000000013, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.584, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.10867877164527733, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1437802930500817, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.40800000000000025, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.72, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.11856390629739041, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.15134920501582536, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.48379999999999934, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.772, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.12941334858125908, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1604150364015217, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.585399999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.824, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.13960398966079246, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.16935894229913975, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7065999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.14958463268083644, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17566338502686696, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8203999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.948, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.14825119627468752, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.17314604467704053, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9189999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.14943493733982396, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.17317149730626566, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9665999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1046983.68, + "depth": 57, + "doc_recall_at_k": 0.1275287211301775, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 56.63, + "metric": "cosine", + "ndcg_at_k": 0.1600398323249309, + "p10_recall_at_k": 0.3, + "p90_depth": 81, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5999999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.844, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1321331.712, + "depth": 235, + "doc_recall_at_k": 0.14738836418504353, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 235.242, + "metric": "cosine", + "ndcg_at_k": 0.1737555826385768, + "p10_recall_at_k": 0.6, + "p90_depth": 359, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8419999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.96, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2093561.856, + "depth": 738, + "doc_recall_at_k": 0.14957830802143, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 737.996, + "metric": "cosine", + "ndcg_at_k": 0.17328208266781597, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9553999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.11319278636023211, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.14246509036752983, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5421999999999992, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.408, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.11319278636023211, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.14793579984799574, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5421999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.13186250697014654, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.16155989195597412, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7385999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.956, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.14272645194430736, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.16800278641387528, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8215999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.14587199708983214, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1694655988749826, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9045999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.1487695236154858, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.17233982625013988, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9669999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.14967307192476167, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17293744821820162, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9923999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.15061751636920612, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1735093954631112, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.15061751636920612, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1735093954631112, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1002682.368, + "depth": 28, + "doc_recall_at_k": 0.13924112602585387, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 27.788, + "metric": "cosine", + "ndcg_at_k": 0.16549394061110198, + "p10_recall_at_k": 0.6, + "p90_depth": 40, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8101999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.976, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1070932.992, + "depth": 72, + "doc_recall_at_k": 0.14763248200212836, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 72.222, + "metric": "cosine", + "ndcg_at_k": 0.1714566556922501, + "p10_recall_at_k": 0.9, + "p90_depth": 112, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9481999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1227297.792, + "depth": 174, + "doc_recall_at_k": 0.1494397385914283, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 174.022, + "metric": "cosine", + "ndcg_at_k": 0.17302248483652005, + "p10_recall_at_k": 1.0, + "p90_depth": 289, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9935999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 405 + ], + [ + 0.95, + 754 + ], + [ + 0.99, + 2334 + ], + [ + 1.0, + 11237 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 472.5, + "metric": "cosine", + "neighbour_band": 0.010970264673233032, + "queries": 500, + "rank_noise": 0.05022904805776396, + "recall_at_k": 0.28460000000000013, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 49 + ], + [ + 0.95, + 78 + ], + [ + 0.99, + 175 + ], + [ + 1.0, + 461 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 57.0, + "metric": "cosine", + "neighbour_band": 0.010970264673233032, + "queries": 500, + "rank_noise": 0.013679213780810105, + "recall_at_k": 0.5421999999999992, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/paragraph-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/paragraph-bge-small-lookup.json new file mode 100644 index 0000000..e4c77b5 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/paragraph-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.289, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.2091705478062366, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.24119999999999997, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.218, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.289, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.25826694255731036, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.24119999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.566, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.323, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.2795972599865897, + "p10_recall_at_k": 0.1, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3505999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.67, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.35866666666666663, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.29855571761250616, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.41960000000000014, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.732, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.3876666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.31781053517228547, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5065999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.78, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.4133333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.3352509457732701, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6386000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.852, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.45133333333333336, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.3581338382901026, + "p10_recall_at_k": 0.5, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7641999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.928, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.466, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.36536412288634534, + "p10_recall_at_k": 0.7, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8887999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.473, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.3694523966534786, + "p10_recall_at_k": 0.8, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9519999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1060890.624, + "depth": 66, + "doc_recall_at_k": 0.3916666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 65.684, + "metric": "cosine", + "ndcg_at_k": 0.32247293734112736, + "p10_recall_at_k": 0.3, + "p90_depth": 97, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5489999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.822, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1400352.768, + "depth": 287, + "doc_recall_at_k": 0.44733333333333336, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 286.688, + "metric": "cosine", + "ndcg_at_k": 0.3579113881651585, + "p10_recall_at_k": 0.6, + "p90_depth": 433, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8135999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.95, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2244025.344, + "depth": 836, + "doc_recall_at_k": 0.471, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 835.954, + "metric": "cosine", + "ndcg_at_k": 0.36851206390526603, + "p10_recall_at_k": 0.8, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9437999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.4103333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.3119751071250867, + "p10_recall_at_k": 0.3, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5148, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.498, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.4103333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.34183686315421924, + "p10_recall_at_k": 0.3, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5148, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.44233333333333336, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3546304512223516, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6969999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.954, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.449, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.35882370374941625, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7899999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.976, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.465, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.36651582723874365, + "p10_recall_at_k": 0.7, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8787999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.478, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.3706594221556972, + "p10_recall_at_k": 0.8, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9527999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.475, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.37054146614584493, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9889999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.473, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.36996333649320917, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.473, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.36996333649320917, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1005904.896, + "depth": 30, + "doc_recall_at_k": 0.44533333333333336, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 29.886, + "metric": "cosine", + "ndcg_at_k": 0.35707546590041855, + "p10_recall_at_k": 0.6, + "p90_depth": 44, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7856000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1085862.912, + "depth": 82, + "doc_recall_at_k": 0.473, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 81.942, + "metric": "cosine", + "ndcg_at_k": 0.3697683377926542, + "p10_recall_at_k": 0.8, + "p90_depth": 132, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9375999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1271092.224, + "depth": 203, + "doc_recall_at_k": 0.475, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 202.534, + "metric": "cosine", + "ndcg_at_k": 0.37054146614584493, + "p10_recall_at_k": 1.0, + "p90_depth": 337, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9911999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 550 + ], + [ + 0.95, + 966 + ], + [ + 0.99, + 2360 + ], + [ + 1.0, + 6874 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 625.0, + "metric": "cosine", + "neighbour_band": 0.009995579719543457, + "queries": 500, + "rank_noise": 0.049126464803224155, + "recall_at_k": 0.24119999999999997, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 59 + ], + [ + 0.95, + 98 + ], + [ + 0.99, + 207 + ], + [ + 1.0, + 569 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 68.0, + "metric": "cosine", + "neighbour_band": 0.009995579719543457, + "queries": 500, + "rank_noise": 0.014088010248940087, + "recall_at_k": 0.5148, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/paragraph-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/paragraph-minilm-descriptive.json new file mode 100644 index 0000000..8f0aba5 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/paragraph-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.08158369312854236, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.09966956222481498, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.4110000000000005, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.316, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.08158369312854236, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.10756890683389403, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4110000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.76, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.09431460822696484, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.11772401852892639, + "p10_recall_at_k": 0.3, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5691999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.852, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.10059299478418046, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.12254638579692716, + "p10_recall_at_k": 0.4, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6629999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.894, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.10295028066572165, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.12416242846886115, + "p10_recall_at_k": 0.5, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7610000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.938, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.1043618063468055, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.1263934025964298, + "p10_recall_at_k": 0.7, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8633999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.964, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.10501708702529078, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.12736031305937423, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9359999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.10590643441236099, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.12920825935051028, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9825999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.10567699718292375, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.12895130178276293, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9941999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1020374.016, + "depth": 39, + "doc_recall_at_k": 0.1007521150089329, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 39.306, + "metric": "cosine", + "ndcg_at_k": 0.12250981949174765, + "p10_recall_at_k": 0.5, + "p90_depth": 57, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7135999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.93, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1159317.504, + "depth": 130, + "doc_recall_at_k": 0.10390146302667957, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 129.764, + "metric": "cosine", + "ndcg_at_k": 0.12639006634102903, + "p10_recall_at_k": 0.8, + "p90_depth": 213, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9021999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1548103.68, + "depth": 383, + "doc_recall_at_k": 0.10590607663060785, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 382.88, + "metric": "cosine", + "ndcg_at_k": 0.12902283332530248, + "p10_recall_at_k": 0.9, + "p90_depth": 661, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9793999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.09317237174039228, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.11187079359034093, + "p10_recall_at_k": 0.5, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6515999999999998, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.51, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.09317237174039228, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.11896024038260575, + "p10_recall_at_k": 0.5, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6515999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.10144376680033018, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.12459287639386797, + "p10_recall_at_k": 0.7, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8493999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.10466969903079693, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.12743162909503025, + "p10_recall_at_k": 0.8, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9177999999999977, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.10617528607040554, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.12909492160458724, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9687999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.10645477496070153, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.1298464844255929, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9935999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.10634366384959042, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.129510184751575, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9987999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.10634366384959042, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.129510184751575, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.10634366384959042, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.129510184751575, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 993991.68, + "depth": 22, + "doc_recall_at_k": 0.10258206483862817, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 22.13, + "metric": "cosine", + "ndcg_at_k": 0.1256409239958563, + "p10_recall_at_k": 0.7, + "p90_depth": 31, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.872999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1031792.64, + "depth": 47, + "doc_recall_at_k": 0.10529028933739236, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 46.74, + "metric": "cosine", + "ndcg_at_k": 0.1290626917392832, + "p10_recall_at_k": 0.9, + "p90_depth": 73, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9727999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1105228.8, + "depth": 95, + "doc_recall_at_k": 0.10640248737900218, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 94.55, + "metric": "cosine", + "ndcg_at_k": 0.12965035820868281, + "p10_recall_at_k": 1.0, + "p90_depth": 163, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9959999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 138 + ], + [ + 0.95, + 246 + ], + [ + 0.99, + 702 + ], + [ + 1.0, + 2751 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 147.5, + "metric": "cosine", + "neighbour_band": 0.021275341510772705, + "queries": 500, + "rank_noise": 0.048614573070958664, + "recall_at_k": 0.4110000000000005, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 27 + ], + [ + 0.95, + 40 + ], + [ + 0.99, + 81 + ], + [ + 1.0, + 517 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 29.0, + "metric": "cosine", + "neighbour_band": 0.021275341510772705, + "queries": 500, + "rank_noise": 0.021389494657683458, + "recall_at_k": 0.6515999999999998, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/paragraph-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/paragraph-minilm-lookup.json new file mode 100644 index 0000000..519ede5 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/paragraph-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.18533333333333332, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.11722213376054388, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.33540000000000025, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.242, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.18533333333333332, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.14203487416129415, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.33540000000000025, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.666, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.2126666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1534334060061021, + "p10_recall_at_k": 0.3, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4806000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.776, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.21766666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.15339302455372075, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5700000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.83, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.2316666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.1639249878842186, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6701999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.904, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.2426666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.1698471735886078, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7965999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.2576666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17801932342445895, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8955999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.26166666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1787336907934579, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9689999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.26366666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.17925052920584064, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9911999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1035291.648, + "depth": 49, + "doc_recall_at_k": 0.22966666666666669, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 49.018, + "metric": "cosine", + "ndcg_at_k": 0.1638912423896498, + "p10_recall_at_k": 0.4, + "p90_depth": 71, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6591999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.91, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1235785.7280000001, + "depth": 180, + "doc_recall_at_k": 0.2566666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 179.548, + "metric": "cosine", + "ndcg_at_k": 0.17724807199834766, + "p10_recall_at_k": 0.7, + "p90_depth": 285, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8893999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1782558.72, + "depth": 536, + "doc_recall_at_k": 0.2576666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 535.52, + "metric": "cosine", + "ndcg_at_k": 0.1775774314881864, + "p10_recall_at_k": 0.9, + "p90_depth": 907, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9779999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.22666666666666668, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.15695914322927154, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.601, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.466, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.22666666666666668, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.16623995893406746, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.601, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.92, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.2586666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.175936786043338, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8059999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.2576666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.17554320929141667, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8867999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.26566666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.17994275019966516, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9461999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.26366666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.17925052920584064, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9863999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.26366666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17925052920584064, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.26366666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.17925052920584064, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.26366666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.17925052920584064, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 999705.6, + "depth": 26, + "doc_recall_at_k": 0.2586666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 25.85, + "metric": "cosine", + "ndcg_at_k": 0.17618671731612687, + "p10_recall_at_k": 0.7, + "p90_depth": 36, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8601999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1055769.6, + "depth": 62, + "doc_recall_at_k": 0.26566666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 62.35, + "metric": "cosine", + "ndcg_at_k": 0.17988145895941213, + "p10_recall_at_k": 0.9, + "p90_depth": 95, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9681999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1174278.1439999999, + "depth": 140, + "doc_recall_at_k": 0.26366666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 139.504, + "metric": "cosine", + "ndcg_at_k": 0.17927445954453283, + "p10_recall_at_k": 1.0, + "p90_depth": 231, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9971999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 212 + ], + [ + 0.95, + 367 + ], + [ + 0.99, + 952 + ], + [ + 1.0, + 6209 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 255.0, + "metric": "cosine", + "neighbour_band": 0.016923844814300537, + "queries": 500, + "rank_noise": 0.04740697834644542, + "recall_at_k": 0.33540000000000025, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 33 + ], + [ + 0.95, + 52 + ], + [ + 0.99, + 108 + ], + [ + 1.0, + 287 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 41.0, + "metric": "cosine", + "neighbour_band": 0.016923844814300537, + "queries": 500, + "rank_noise": 0.01977754710449133, + "recall_at_k": 0.601, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/table-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/table-bge-small-descriptive.json new file mode 100644 index 0000000..30f0c7c --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/table-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.10372338160060557, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.13447418626192612, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.3192, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.176, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.10372338160060557, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.14954182863291257, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.3192, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.586, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.13655605050475242, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.180182290949683, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4522000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.704, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.14523725305468277, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.18725198552228123, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.534, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.754, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.1529775018428986, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.19589970697005654, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6351999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.818, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.1640069482942459, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.20620795648444556, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7654000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.91, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.17052713503911282, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.210478446044394, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8641999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.944, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.1745327378205633, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.21252090888397365, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9535999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.17200662204444744, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.2108527563146365, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9815999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1023154.176, + "depth": 41, + "doc_recall_at_k": 0.15031799428343043, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 41.116, + "metric": "cosine", + "ndcg_at_k": 0.19411906489885095, + "p10_recall_at_k": 0.3, + "p90_depth": 59, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5895999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.806, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1169488.896, + "depth": 136, + "doc_recall_at_k": 0.1687500154616015, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 136.386, + "metric": "cosine", + "ndcg_at_k": 0.21066374189063589, + "p10_recall_at_k": 0.6, + "p90_depth": 216, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8147999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1549651.9679999999, + "depth": 384, + "doc_recall_at_k": 0.17359835018173414, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 383.888, + "metric": "cosine", + "ndcg_at_k": 0.21284787992898663, + "p10_recall_at_k": 0.8, + "p90_depth": 633, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9393999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.138005463346744, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.17126515983769813, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5727999999999996, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.426, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.138005463346744, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.18202213673798248, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5727999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.906, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.1577919219343615, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.2002537886264123, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7585999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.16575979248258102, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.2053799113281108, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.847999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.1697408917542597, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.2081492117546772, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9231999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.17092946852054694, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.21008321915245529, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.975199999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.17181199401016742, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.2108904963616776, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9943999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.172255472271037, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.21117126127645622, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9995999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.17234242879277611, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.21135857285197254, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 994759.68, + "depth": 23, + "doc_recall_at_k": 0.15896237765014234, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 22.63, + "metric": "cosine", + "ndcg_at_k": 0.20072684488282425, + "p10_recall_at_k": 0.6, + "p90_depth": 32, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7825999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036105.728, + "depth": 50, + "doc_recall_at_k": 0.16743784393621822, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 49.548, + "metric": "cosine", + "ndcg_at_k": 0.2075273403495445, + "p10_recall_at_k": 0.8, + "p90_depth": 74, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9197999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1114936.32, + "depth": 101, + "doc_recall_at_k": 0.17130368361704468, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.87, + "metric": "cosine", + "ndcg_at_k": 0.2107312842507962, + "p10_recall_at_k": 0.9, + "p90_depth": 164, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9803999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 271 + ], + [ + 0.95, + 476 + ], + [ + 0.99, + 1448 + ], + [ + 1.0, + 4213 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 294.5, + "metric": "cosine", + "neighbour_band": 0.009452968835830688, + "queries": 500, + "rank_noise": 0.04422727695990099, + "recall_at_k": 0.3192, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 42 + ], + [ + 0.95, + 68 + ], + [ + 0.99, + 156 + ], + [ + 1.0, + 546 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 45.0, + "metric": "cosine", + "neighbour_band": 0.009452968835830688, + "queries": 500, + "rank_noise": 0.01107405841104134, + "recall_at_k": 0.5727999999999996, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/table-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/table-bge-small-lookup.json new file mode 100644 index 0000000..0dee8a6 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/table-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.331, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.24162884599723902, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.27899999999999986, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.264, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.331, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3019937158465899, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.27899999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.598, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.379, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3310804512401088, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.39460000000000023, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.688, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.426, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.3613826121761931, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.46920000000000034, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.74, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.465, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.37822274482037904, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5644000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.806, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.491, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.3987935443687892, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6948, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.88, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.507, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.409274573924203, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8121999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.922, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.515, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.41468892431823134, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.927399999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.513, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.41557269804434427, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9685999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1026097.152, + "depth": 43, + "doc_recall_at_k": 0.447, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 43.032, + "metric": "cosine", + "ndcg_at_k": 0.3704917996050543, + "p10_recall_at_k": 0.3, + "p90_depth": 65, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5308, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.78, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1197047.808, + "depth": 154, + "doc_recall_at_k": 0.499, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 154.328, + "metric": "cosine", + "ndcg_at_k": 0.40492359698199976, + "p10_recall_at_k": 0.6, + "p90_depth": 242, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.7630000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.908, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1651958.784, + "depth": 450, + "doc_recall_at_k": 0.515, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 450.494, + "metric": "cosine", + "ndcg_at_k": 0.4144505061684112, + "p10_recall_at_k": 0.8, + "p90_depth": 743, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9187999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.433, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.32894921591160287, + "p10_recall_at_k": 0.3, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5395999999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.484, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.433, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3750447109170256, + "p10_recall_at_k": 0.3, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5395999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.91, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.4703333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.39499188341109176, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7355999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.407751258663722, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8285999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.976, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.511, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.41329985050786744, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9103999999999979, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.515, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.41503162514075675, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9745999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.513, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.41437201086747644, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9955999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.515, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.41537201086747644, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.515, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.41537201086747644, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 995500.032, + "depth": 23, + "doc_recall_at_k": 0.4753333333333334, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 23.112, + "metric": "cosine", + "ndcg_at_k": 0.39854663830856135, + "p10_recall_at_k": 0.6, + "p90_depth": 33, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.7619999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1038913.536, + "depth": 51, + "doc_recall_at_k": 0.501, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 51.376, + "metric": "cosine", + "ndcg_at_k": 0.410536459840365, + "p10_recall_at_k": 0.8, + "p90_depth": 85, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9133999999999978, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1125715.9679999999, + "depth": 108, + "doc_recall_at_k": 0.513, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 107.888, + "metric": "cosine", + "ndcg_at_k": 0.4143698978976414, + "p10_recall_at_k": 0.9, + "p90_depth": 209, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9795999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 381 + ], + [ + 0.95, + 712 + ], + [ + 0.99, + 1713 + ], + [ + 1.0, + 8717 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 462.0, + "metric": "cosine", + "neighbour_band": 0.009497731924057007, + "queries": 500, + "rank_noise": 0.04448227541575279, + "recall_at_k": 0.27899999999999986, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 47 + ], + [ + 0.95, + 71 + ], + [ + 0.99, + 145 + ], + [ + 1.0, + 628 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 52.0, + "metric": "cosine", + "neighbour_band": 0.009497731924057007, + "queries": 500, + "rank_noise": 0.011659371496484832, + "recall_at_k": 0.5395999999999999, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/table-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/table-minilm-descriptive.json new file mode 100644 index 0000000..105bb40 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/table-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.11848974553292915, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.1404309676201316, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.42720000000000014, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.282, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.11848974553292915, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.14630100194619203, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.42720000000000014, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.72, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.14110672456735263, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.16889178973278055, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5886000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.816, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.1467646935114131, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.17411602116488425, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6762, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.858, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.15052630280842028, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.17753254447440175, + "p10_recall_at_k": 0.5, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7673999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15178775747357703, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.17929465440722905, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8707999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.15452502398382068, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.18113806539291577, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9399999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.15416262451638899, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.18111675593981305, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9863999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.15396009671386116, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.18093204161667034, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9953999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1003425.792, + "depth": 28, + "doc_recall_at_k": 0.14334902372063166, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 28.272, + "metric": "cosine", + "ndcg_at_k": 0.1717467565822264, + "p10_recall_at_k": 0.4, + "p90_depth": 42, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6544000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.858, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1068183.552, + "depth": 70, + "doc_recall_at_k": 0.15270179097473585, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 70.432, + "metric": "cosine", + "ndcg_at_k": 0.17908054950326094, + "p10_recall_at_k": 0.6, + "p90_depth": 119, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8319999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.934, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1211145.216, + "depth": 164, + "doc_recall_at_k": 0.15342993018872692, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 163.506, + "metric": "cosine", + "ndcg_at_k": 0.18068937370466012, + "p10_recall_at_k": 0.8, + "p90_depth": 296, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9451999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.1370714488804315, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.1631618959707789, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6463999999999992, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.51, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.1370714488804315, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.17032363851781268, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6463999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.14790640248075188, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.17714812223045892, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8501999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.15202689766864766, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.17934119782935112, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9155999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.15341565226941672, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.18065505510447397, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9655999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15365136655513104, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.18077866692273448, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9887999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.15366247766624214, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.18076097829078805, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9983999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.15366247766624214, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.18076097829078805, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.15366247766624214, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.18076097829078805, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 988904.448, + "depth": 19, + "doc_recall_at_k": 0.1483020126463401, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 18.818, + "metric": "cosine", + "ndcg_at_k": 0.17698058872613007, + "p10_recall_at_k": 0.7, + "p90_depth": 26, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8327999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1013068.8, + "depth": 35, + "doc_recall_at_k": 0.15169689573489809, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 34.55, + "metric": "cosine", + "ndcg_at_k": 0.17915170487748783, + "p10_recall_at_k": 0.8, + "p90_depth": 54, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9449999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1053825.024, + "depth": 61, + "doc_recall_at_k": 0.15274343004719454, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 61.084, + "metric": "cosine", + "ndcg_at_k": 0.18034667811460292, + "p10_recall_at_k": 0.9, + "p90_depth": 110, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.987999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 127 + ], + [ + 0.95, + 228 + ], + [ + 0.99, + 624 + ], + [ + 1.0, + 6318 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 110.0, + "metric": "cosine", + "neighbour_band": 0.018120557069778442, + "queries": 500, + "rank_noise": 0.04086612698867343, + "recall_at_k": 0.42720000000000014, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 27 + ], + [ + 0.95, + 42 + ], + [ + 0.99, + 105 + ], + [ + 1.0, + 535 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 27.0, + "metric": "cosine", + "neighbour_band": 0.018120557069778442, + "queries": 500, + "rank_noise": 0.01687235458795014, + "recall_at_k": 0.6463999999999992, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/table-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/table-minilm-lookup.json new file mode 100644 index 0000000..7bd2b35 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/table-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.36066666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.24876125488605277, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.36780000000000007, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.322, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.36066666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.29575644596349265, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.36780000000000007, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.712, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.4213333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3289945720558713, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5115999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.804, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.43366666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.33213739519935404, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5938000000000004, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.856, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.44366666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.3367867430347391, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6939999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.902, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.4656666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.3485916872620389, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8139999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.954, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.4716666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.35031289032917484, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8953999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.4816666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.3535775003196395, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9637999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.4796666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.35331564081249656, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9885999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1009271.808, + "depth": 32, + "doc_recall_at_k": 0.43266666666666664, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 32.078, + "metric": "cosine", + "ndcg_at_k": 0.3348920029286285, + "p10_recall_at_k": 0.4, + "p90_depth": 46, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5999999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.868, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1093269.504, + "depth": 87, + "doc_recall_at_k": 0.4643333333333334, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 86.764, + "metric": "cosine", + "ndcg_at_k": 0.346767244152068, + "p10_recall_at_k": 0.6, + "p90_depth": 139, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.7962000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.954, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1290157.0559999999, + "depth": 215, + "doc_recall_at_k": 0.4796666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 214.946, + "metric": "cosine", + "ndcg_at_k": 0.3524474451205931, + "p10_recall_at_k": 0.8, + "p90_depth": 365, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9151999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.986, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.4206666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.30146762849091324, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6235999999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.498, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.4206666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.3291773893135548, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6235999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.45266666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.3426954470471714, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8207999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.4656666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.3471869512646732, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8999999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.4756666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.3520419747309507, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9607999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.4776666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.3526847110589251, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9911999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.4796666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.35331564081249656, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9989999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.4796666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.35331564081249656, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.4796666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.35331564081249656, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 991097.856, + "depth": 20, + "doc_recall_at_k": 0.44866666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.246, + "metric": "cosine", + "ndcg_at_k": 0.3417600537225101, + "p10_recall_at_k": 0.6, + "p90_depth": 28, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8228000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.984, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1020813.312, + "depth": 40, + "doc_recall_at_k": 0.46266666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 39.592, + "metric": "cosine", + "ndcg_at_k": 0.34775866058070176, + "p10_recall_at_k": 0.8, + "p90_depth": 60, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9417999999999975, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1072536.576, + "depth": 73, + "doc_recall_at_k": 0.4776666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 73.266, + "metric": "cosine", + "ndcg_at_k": 0.3525961740448943, + "p10_recall_at_k": 1.0, + "p90_depth": 122, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9895999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 208 + ], + [ + 0.95, + 398 + ], + [ + 0.99, + 1060 + ], + [ + 1.0, + 5206 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 192.5, + "metric": "cosine", + "neighbour_band": 0.0157146155834198, + "queries": 500, + "rank_noise": 0.04160117208060811, + "recall_at_k": 0.36780000000000007, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 30 + ], + [ + 0.95, + 45 + ], + [ + 0.99, + 96 + ], + [ + 1.0, + 363 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 34.0, + "metric": "cosine", + "neighbour_band": 0.0157146155834198, + "queries": 500, + "rank_noise": 0.016603972949732794, + "recall_at_k": 0.6235999999999999, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/wiki-bge-small-descriptive.json b/experiments/2026-11-text-format/results/rescore/wiki-bge-small-descriptive.json new file mode 100644 index 0000000..4bc268e --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/wiki-bge-small-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.06419716843598916, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.08442677380636439, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.2910000000000001, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.186, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.06419716843598916, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.09265051116126058, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.2910000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.522, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.08025012090244471, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.1066850667153572, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4130000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.634, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.0810597782623051, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.10676754402854534, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4995999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.71, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.08615190916587055, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.11110510672893395, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6045999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.806, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.08637306657203449, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.11256029880068048, + "p10_recall_at_k": 0.5, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7285999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.888, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.08977174069046054, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.11411453591146395, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8403999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.972, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.08943088294713453, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.11340820168326146, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9355999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.08946433936000266, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.11330525125479521, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9737999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1042430.976, + "depth": 54, + "doc_recall_at_k": 0.08758018254619958, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 53.666, + "metric": "cosine", + "ndcg_at_k": 0.11348705077941239, + "p10_recall_at_k": 0.4, + "p90_depth": 75, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6131999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.816, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1365559.296, + "depth": 264, + "doc_recall_at_k": 0.09001967722259253, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 264.036, + "metric": "cosine", + "ndcg_at_k": 0.11451017809199465, + "p10_recall_at_k": 0.7, + "p90_depth": 403, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8707999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2225366.016, + "depth": 824, + "doc_recall_at_k": 0.089007773703437, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 823.806, + "metric": "cosine", + "ndcg_at_k": 0.11312470974339751, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9667999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.08544062048005659, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.10929737880628104, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5485999999999996, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.416, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.08544062048005659, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.11062492567084943, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5485999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.92, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.0889746710776194, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.11297741604879966, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7487999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.962, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.08905000666188724, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.11264164582962856, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8397999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.08792798981863266, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.11210411162381813, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9193999999999981, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.08823252956034386, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.11276966177459814, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9743999999999985, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.08918809621591053, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1133754958639826, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9933999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.08904027986652643, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.11307960608351139, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9994000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.08904027986652643, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.11307960608351139, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1004814.336, + "depth": 29, + "doc_recall_at_k": 0.0874606231654604, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 29.176, + "metric": "cosine", + "ndcg_at_k": 0.11123294441069384, + "p10_recall_at_k": 0.7, + "p90_depth": 41, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8315999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1079393.28, + "depth": 78, + "doc_recall_at_k": 0.08733074985856416, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 77.73, + "metric": "cosine", + "ndcg_at_k": 0.11194927677820882, + "p10_recall_at_k": 0.9, + "p90_depth": 119, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9657999999999979, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1256343.552, + "depth": 193, + "doc_recall_at_k": 0.08875590998109271, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 192.932, + "metric": "cosine", + "ndcg_at_k": 0.11293337240662357, + "p10_recall_at_k": 1.0, + "p90_depth": 318, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9955999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 334 + ], + [ + 0.95, + 611 + ], + [ + 0.99, + 1823 + ], + [ + 1.0, + 8557 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 396.5, + "metric": "cosine", + "neighbour_band": 0.011578291654586792, + "queries": 500, + "rank_noise": 0.05171884692660382, + "recall_at_k": 0.2910000000000001, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 44 + ], + [ + 0.95, + 70 + ], + [ + 0.99, + 156 + ], + [ + 1.0, + 737 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 54.0, + "metric": "cosine", + "neighbour_band": 0.011578291654586792, + "queries": 500, + "rank_noise": 0.014603027522061725, + "recall_at_k": 0.5485999999999996, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/wiki-bge-small-lookup.json b/experiments/2026-11-text-format/results/rescore/wiki-bge-small-lookup.json new file mode 100644 index 0000000..3832c93 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/wiki-bge-small-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.137, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.09038446533807218, + "p10_recall_at_k": 0.1, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.25639999999999996, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.204, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.137, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.11205968317897601, + "p10_recall_at_k": 0.1, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.25639999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.518, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.168, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.13258080306237618, + "p10_recall_at_k": 0.1, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.36339999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.628, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.2056666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.14960073485052836, + "p10_recall_at_k": 0.2, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.43819999999999976, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.694, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.22766666666666668, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.15937792553777566, + "p10_recall_at_k": 0.3, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5387999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.768, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.2366666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.16433240190633314, + "p10_recall_at_k": 0.4, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6703999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.856, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.2566666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.17038535047440168, + "p10_recall_at_k": 0.6, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7933999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.92, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.27266666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1768576398784102, + "p10_recall_at_k": 0.8, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9119999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.27466666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.17747169168667573, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9631999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1066994.688, + "depth": 70, + "doc_recall_at_k": 0.2346666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 69.658, + "metric": "cosine", + "ndcg_at_k": 0.16201661817993618, + "p10_recall_at_k": 0.4, + "p90_depth": 98, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.5978, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.812, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1467396.096, + "depth": 330, + "doc_recall_at_k": 0.26266666666666666, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 330.336, + "metric": "cosine", + "ndcg_at_k": 0.1719035846288328, + "p10_recall_at_k": 0.7, + "p90_depth": 487, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8715999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.966, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2347613.184, + "depth": 903, + "doc_recall_at_k": 0.27466666666666667, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 903.394, + "metric": "cosine", + "ndcg_at_k": 0.17747169168667573, + "p10_recall_at_k": 0.9, + "p90_depth": 1000, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9609999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.23, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.15096341335277594, + "p10_recall_at_k": 0.3, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.5313999999999997, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.496, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.23, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.15973495784737032, + "p10_recall_at_k": 0.3, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5313999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.256, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.17174607160556105, + "p10_recall_at_k": 0.5, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7179999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.948, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.27366666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.1765560728415278, + "p10_recall_at_k": 0.6, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8129999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.976, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.27666666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.17773694075593938, + "p10_recall_at_k": 0.7, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8975999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.2806666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.17939494449180546, + "p10_recall_at_k": 0.9, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9657999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.2846666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.1810291137100681, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9925999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.2846666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.1810291137100681, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.2846666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.1810291137100681, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006442.496, + "depth": 30, + "doc_recall_at_k": 0.275, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.236, + "metric": "cosine", + "ndcg_at_k": 0.17705089814965036, + "p10_recall_at_k": 0.7, + "p90_depth": 41, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8121999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1088314.368, + "depth": 84, + "doc_recall_at_k": 0.2826666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 83.538, + "metric": "cosine", + "ndcg_at_k": 0.18025629760795223, + "p10_recall_at_k": 0.9, + "p90_depth": 124, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9569999999999975, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1279233.024, + "depth": 208, + "doc_recall_at_k": 0.2846666666666667, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 207.834, + "metric": "cosine", + "ndcg_at_k": 0.1810291137100681, + "p10_recall_at_k": 1.0, + "p90_depth": 327, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9955999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 445 + ], + [ + 0.95, + 804 + ], + [ + 0.99, + 1984 + ], + [ + 1.0, + 8081 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 554.0, + "metric": "cosine", + "neighbour_band": 0.010829746723175049, + "queries": 500, + "rank_noise": 0.04906278077779547, + "recall_at_k": 0.25639999999999996, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 51 + ], + [ + 0.95, + 81 + ], + [ + 0.99, + 180 + ], + [ + 1.0, + 708 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 61.5, + "metric": "cosine", + "neighbour_band": 0.010829746723175049, + "queries": 500, + "rank_noise": 0.014215374780827485, + "recall_at_k": 0.5313999999999997, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/wiki-minilm-descriptive.json b/experiments/2026-11-text-format/results/rescore/wiki-minilm-descriptive.json new file mode 100644 index 0000000..e99cf95 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/wiki-minilm-descriptive.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.050704437537062705, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.06298463521735145, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.38280000000000036, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.266, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.050704437537062705, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.06865690055240257, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.38280000000000036, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.686, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.062134437682019615, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.07944626927938281, + "p10_recall_at_k": 0.3, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5385999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.794, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.06558256489077305, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.08395401041939696, + "p10_recall_at_k": 0.4, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6262, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.848, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.0660562638870131, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.08540481542753695, + "p10_recall_at_k": 0.5, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.7267999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.06722427108225437, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.08829259122571413, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8471999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.948, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.06732059518299024, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.08843040134236213, + "p10_recall_at_k": 0.8, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9251999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.98, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.06901954454491914, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.09007286111108107, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9793999999999992, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.06899997932752784, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.09048656545394838, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9939999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1025101.824, + "depth": 42, + "doc_recall_at_k": 0.06642221314990619, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 42.384, + "metric": "cosine", + "ndcg_at_k": 0.08478699147117572, + "p10_recall_at_k": 0.5, + "p90_depth": 61, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6894, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.892, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1183002.624, + "depth": 145, + "doc_recall_at_k": 0.06701965828742418, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 145.184, + "metric": "cosine", + "ndcg_at_k": 0.08861562798556562, + "p10_recall_at_k": 0.8, + "p90_depth": 236, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9029999999999977, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1625115.648, + "depth": 433, + "doc_recall_at_k": 0.06911556700072422, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 433.018, + "metric": "cosine", + "ndcg_at_k": 0.09054140214189935, + "p10_recall_at_k": 0.9, + "p90_depth": 744, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9819999999999983, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.06029499219501658, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.07779229234579542, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6255999999999999, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.522, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.06029499219501658, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.08170433879114121, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6255999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.942, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.06850468513704776, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.0893842120890572, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8277999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.07011125509225613, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.09101278014976348, + "p10_recall_at_k": 0.8, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9057999999999979, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.06968288179521291, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.09088667750991714, + "p10_recall_at_k": 0.9, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.961999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.06967925860680713, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.09094496072439229, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9919999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.06956664599419451, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.09094107841990236, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9991999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.06956664599419451, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.09094107841990236, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.06956664599419451, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.09094107841990236, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 996523.008, + "depth": 24, + "doc_recall_at_k": 0.07033241397340817, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 23.778, + "metric": "cosine", + "ndcg_at_k": 0.09094308296994905, + "p10_recall_at_k": 0.7, + "p90_depth": 33, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8671999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1042689.024, + "depth": 54, + "doc_recall_at_k": 0.06842779340534191, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 53.834, + "metric": "cosine", + "ndcg_at_k": 0.09023852318315605, + "p10_recall_at_k": 0.9, + "p90_depth": 87, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9693999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1138627.584, + "depth": 116, + "doc_recall_at_k": 0.06956664599419451, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 116.294, + "metric": "cosine", + "ndcg_at_k": 0.09095131472135706, + "p10_recall_at_k": 1.0, + "p90_depth": 199, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9967999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 153 + ], + [ + 0.95, + 270 + ], + [ + 0.99, + 753 + ], + [ + 1.0, + 5470 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 168.0, + "metric": "cosine", + "neighbour_band": 0.021074384450912476, + "queries": 500, + "rank_noise": 0.04901172234731914, + "recall_at_k": 0.38280000000000036, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 30 + ], + [ + 0.95, + 45 + ], + [ + 0.99, + 94 + ], + [ + 1.0, + 259 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 34.0, + "metric": "cosine", + "neighbour_band": 0.021074384450912476, + "queries": 500, + "rank_noise": 0.02206189584306906, + "recall_at_k": 0.6255999999999999, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/rescore/wiki-minilm-lookup.json b/experiments/2026-11-text-format/results/rescore/wiki-minilm-lookup.json new file mode 100644 index 0000000..b98ab16 --- /dev/null +++ b/experiments/2026-11-text-format/results/rescore/wiki-minilm-lookup.json @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.096, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.05396964236887714, + "p10_recall_at_k": 0.2, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.342, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.22, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.096, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.06946914067075191, + "p10_recall_at_k": 0.2, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.342, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.6, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.116, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.07840323503177331, + "p10_recall_at_k": 0.2, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.4883999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.746, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.128, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.08486894818647818, + "p10_recall_at_k": 0.3, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.5786000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.792, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.136, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.08822417416071228, + "p10_recall_at_k": 0.4, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6845999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.832, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15866666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.09962646263534383, + "p10_recall_at_k": 0.6, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8064, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.936, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.16066666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.10036820899867463, + "p10_recall_at_k": 0.7, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9003999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.974, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.15466666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.09806310367710927, + "p10_recall_at_k": 0.9, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.970599999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.15466666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.09802736676401405, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9905999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1031516.16, + "depth": 47, + "doc_recall_at_k": 0.136, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 46.56, + "metric": "cosine", + "ndcg_at_k": 0.08711521127941214, + "p10_recall_at_k": 0.4, + "p90_depth": 67, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.6625999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.848, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1231346.688, + "depth": 177, + "doc_recall_at_k": 0.16066666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 176.658, + "metric": "cosine", + "ndcg_at_k": 0.09995189018920045, + "p10_recall_at_k": 0.8, + "p90_depth": 279, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.8909999999999986, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1797021.696, + "depth": 545, + "doc_recall_at_k": 0.15466666666666665, + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 544.936, + "metric": "cosine", + "ndcg_at_k": 0.09808865800426708, + "p10_recall_at_k": 0.9, + "p90_depth": 877, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9775999999999987, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 960000.0, + "depth": 0, + "doc_recall_at_k": 0.14066666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 0.0, + "metric": "cosine", + "ndcg_at_k": 0.08752228278292437, + "p10_recall_at_k": 0.4, + "p90_depth": 0, + "policy": { + "kind": "none" + }, + "queries": 500, + "recall_at_k": 0.6005999999999996, + "rescore_ms": 0, + "second": "none", + "second_bytes": 0, + "top1_agreement": 0.472, + "vectors": 20000 + }, + { + "bytes_read_per_query": 975360.0, + "depth": 10, + "doc_recall_at_k": 0.14066666666666666, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 10.0, + "metric": "cosine", + "ndcg_at_k": 0.09213805892317899, + "p10_recall_at_k": 0.4, + "p90_depth": 10, + "policy": { + "depth": 10, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.6005999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.926, + "vectors": 20000 + }, + { + "bytes_read_per_query": 990720.0, + "depth": 20, + "doc_recall_at_k": 0.15966666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 20.0, + "metric": "cosine", + "ndcg_at_k": 0.09872224728905075, + "p10_recall_at_k": 0.6, + "p90_depth": 20, + "policy": { + "depth": 20, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8035999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1006080.0, + "depth": 30, + "doc_recall_at_k": 0.15366666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 30.0, + "metric": "cosine", + "ndcg_at_k": 0.09753011232954834, + "p10_recall_at_k": 0.7, + "p90_depth": 30, + "policy": { + "depth": 30, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.8855999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1036800.0, + "depth": 50, + "doc_recall_at_k": 0.15466666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 50.0, + "metric": "cosine", + "ndcg_at_k": 0.0981259145724991, + "p10_recall_at_k": 0.8, + "p90_depth": 50, + "policy": { + "depth": 50, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9521999999999982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1113600.0, + "depth": 100, + "doc_recall_at_k": 0.15466666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 100.0, + "metric": "cosine", + "ndcg_at_k": 0.09805129710270624, + "p10_recall_at_k": 1.0, + "p90_depth": 100, + "policy": { + "depth": 100, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9899999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1267200.0, + "depth": 200, + "doc_recall_at_k": 0.15466666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 200.0, + "metric": "cosine", + "ndcg_at_k": 0.09802736676401405, + "p10_recall_at_k": 1.0, + "p90_depth": 200, + "policy": { + "depth": 200, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 0.9985999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1728000.0, + "depth": 500, + "doc_recall_at_k": 0.15466666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 500.0, + "metric": "cosine", + "ndcg_at_k": 0.09802736676401405, + "p10_recall_at_k": 1.0, + "p90_depth": 500, + "policy": { + "depth": 500, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 2496000.0, + "depth": 1000, + "doc_recall_at_k": 0.15466666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 1000.0, + "metric": "cosine", + "ndcg_at_k": 0.09802736676401405, + "p10_recall_at_k": 1.0, + "p90_depth": 1000, + "policy": { + "depth": 1000, + "kind": "fixed" + }, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 999518.208, + "depth": 26, + "doc_recall_at_k": 0.16166666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 25.728, + "metric": "cosine", + "ndcg_at_k": 0.10000116998403415, + "p10_recall_at_k": 0.7, + "p90_depth": 37, + "policy": { + "kind": "margin", + "sigmas": 1.0 + }, + "queries": 500, + "recall_at_k": 0.8573999999999984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1058918.4, + "depth": 64, + "doc_recall_at_k": 0.15266666666666664, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 64.4, + "metric": "cosine", + "ndcg_at_k": 0.09736775249073369, + "p10_recall_at_k": 0.9, + "p90_depth": 102, + "policy": { + "kind": "margin", + "sigmas": 2.0 + }, + "queries": 500, + "recall_at_k": 0.9709999999999988, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + }, + { + "bytes_read_per_query": 1187933.184, + "depth": 148, + "doc_recall_at_k": 0.15466666666666665, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "mean_depth": 148.394, + "metric": "cosine", + "ndcg_at_k": 0.09802736676401405, + "p10_recall_at_k": 1.0, + "p90_depth": 251, + "policy": { + "kind": "margin", + "sigmas": 3.0 + }, + "queries": 500, + "recall_at_k": 0.9979999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20000 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 199 + ], + [ + 0.95, + 365 + ], + [ + 0.99, + 948 + ], + [ + 1.0, + 4428 + ] + ], + "first": "binary-hamming", + "first_bytes": 48, + "k": 10, + "median_full_depth": 241.5, + "metric": "cosine", + "neighbour_band": 0.01845613121986389, + "queries": 500, + "rank_noise": 0.049314385762989105, + "recall_at_k": 0.342, + "vectors": 20000 + }, + { + "depth_for_recall": [ + [ + 0.9, + 33 + ], + [ + 0.95, + 49 + ], + [ + 0.99, + 100 + ], + [ + 1.0, + 333 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 38.0, + "metric": "cosine", + "neighbour_band": 0.01845613121986389, + "queries": 500, + "rank_noise": 0.021754727283730162, + "recall_at_k": 0.6005999999999996, + "vectors": 20000 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/bullets-bge-small.json b/experiments/2026-11-text-format/results/scan/bullets-bge-small.json new file mode 100644 index 0000000..c3c154e --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/bullets-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "bullets-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.21160916, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 11.345417, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.736708, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.19139166, + "gb_per_s": 7.329307898656266, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20117916000000002, + "gb_per_s": 4.771866032247077, + "recall_at_k": 0.348 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 4.072196679999999, + "gb_per_s": 0.2357449984463914, + "recall_at_k": 0.632 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 84.73231754260756 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/bullets-minilm.json b/experiments/2026-11-text-format/results/scan/bullets-minilm.json new file mode 100644 index 0000000..8c0c09b --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/bullets-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "bullets-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.06417916, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 11.145583, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 3.060833, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 3.9051850000000004, + "gb_per_s": 7.866464712939335, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.19941915999999998, + "gb_per_s": 4.813980762931706, + "recall_at_k": 0.414 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.91618418, + "gb_per_s": 0.24513658088471213, + "recall_at_k": 0.634 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 89.16270076811293 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/header-prose-bge-small.json b/experiments/2026-11-text-format/results/scan/header-prose-bge-small.json new file mode 100644 index 0000000..2998612 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/header-prose-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "header-prose-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 3.9012599999999997, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 17.833958, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.592042, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.11146834, + "gb_per_s": 7.471783182939455, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.18973916, + "gb_per_s": 5.059577580084154, + "recall_at_k": 0.368 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.65978084, + "gb_per_s": 0.26231078907992755, + "recall_at_k": 0.62 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 92.3080025350399 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/header-prose-minilm.json b/experiments/2026-11-text-format/results/scan/header-prose-minilm.json new file mode 100644 index 0000000..154c637 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/header-prose-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "header-prose-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 3.94850584, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 14.995625, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.620625, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 3.92853166, + "gb_per_s": 7.819715521905709, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20057918, + "gb_per_s": 4.786139817701916, + "recall_at_k": 0.428 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.9472625, + "gb_per_s": 0.24320652604178214, + "recall_at_k": 0.664 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 90.1862065582649 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/json-bge-small.json b/experiments/2026-11-text-format/results/scan/json-bge-small.json new file mode 100644 index 0000000..a1fab3a --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/json-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "json-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.1868799999999995, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 11.813707999999998, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.642042, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.19769834, + "gb_per_s": 7.31829624517516, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.21880334, + "gb_per_s": 4.3875015801861155, + "recall_at_k": 0.362 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 4.0988075, + "gb_per_s": 0.23421446359703402, + "recall_at_k": 0.614 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 79.60030032986468 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/json-minilm.json b/experiments/2026-11-text-format/results/scan/json-minilm.json new file mode 100644 index 0000000..c91f18f --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/json-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "json-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.21143668, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 12.831791, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.6809999999999996, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.16425584, + "gb_per_s": 7.377068359949758, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.21657918, + "gb_per_s": 4.4325590299122934, + "recall_at_k": 0.444 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 4.13865584, + "gb_per_s": 0.23195936968752637, + "recall_at_k": 0.674 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 82.77276050472382 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/kv-bge-small.json b/experiments/2026-11-text-format/results/scan/kv-bge-small.json new file mode 100644 index 0000000..9652cb9 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/kv-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "kv-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.209105, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 11.484708, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.699542, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.0628725, + "gb_per_s": 7.561152854292129, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20218166, + "gb_per_s": 4.748205153721658, + "recall_at_k": 0.346 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.99232584, + "gb_per_s": 0.24046133468905434, + "recall_at_k": 0.566 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 89.83281590352883 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/kv-minilm.json b/experiments/2026-11-text-format/results/scan/kv-minilm.json new file mode 100644 index 0000000..7d4bb00 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/kv-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "kv-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.03564166, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 10.900791, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.695208, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.05272334, + "gb_per_s": 7.580088109345259, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20207418, + "gb_per_s": 4.750730647527557, + "recall_at_k": 0.396 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.97777334, + "gb_per_s": 0.2413410513732288, + "recall_at_k": 0.63 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 89.54191085352124 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/paragraph-bge-small.json b/experiments/2026-11-text-format/results/scan/paragraph-bge-small.json new file mode 100644 index 0000000..5b86409 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/paragraph-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "paragraph-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.308935, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 14.478916, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.631709, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.18593834, + "gb_per_s": 7.338856310052575, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20931584, + "gb_per_s": 4.586370529817523, + "recall_at_k": 0.346 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 4.05231, + "gb_per_s": 0.23690191520392073, + "recall_at_k": 0.6 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 85.92912551773942 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/paragraph-minilm.json b/experiments/2026-11-text-format/results/scan/paragraph-minilm.json new file mode 100644 index 0000000..2b09379 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/paragraph-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "paragraph-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.249301679999999, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 14.368625, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.684916, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.24859, + "gb_per_s": 7.230634163334188, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20095584, + "gb_per_s": 4.7771689541344005, + "recall_at_k": 0.45 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 4.0619891599999995, + "gb_per_s": 0.23633741060992888, + "recall_at_k": 0.636 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 84.47343434820235 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/table-bge-small.json b/experiments/2026-11-text-format/results/scan/table-bge-small.json new file mode 100644 index 0000000..dfb637f --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/table-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "table-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 3.9332358199999997, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 11.884250000000002, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 5.896292, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 3.87095416, + "gb_per_s": 7.936027844876365, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.21519831999999997, + "gb_per_s": 4.4610013684121705, + "recall_at_k": 0.416 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.8941666600000002, + "gb_per_s": 0.24652257692535431, + "recall_at_k": 0.6 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 87.51428553730553 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/table-minilm.json b/experiments/2026-11-text-format/results/scan/table-minilm.json new file mode 100644 index 0000000..bfcac4f --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/table-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "table-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 3.9845399999999995, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 11.925042, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.599334, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 3.9864966600000002, + "gb_per_s": 7.706014232556764, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.2051525, + "gb_per_s": 4.679445778140652, + "recall_at_k": 0.466 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.9615424999999997, + "gb_per_s": 0.24232985005209462, + "recall_at_k": 0.69 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 87.54759396588443 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/wiki-bge-small.json b/experiments/2026-11-text-format/results/scan/wiki-bge-small.json new file mode 100644 index 0000000..642d579 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/wiki-bge-small.json @@ -0,0 +1,59 @@ +{ + "collection": "wiki-bge-small", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.089700000000001, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 12.255749999999999, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 3.171333, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.08832416, + "gb_per_s": 7.514081270894136, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20190416, + "gb_per_s": 4.7547311556136345, + "recall_at_k": 0.32 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.9898149999999997, + "gb_per_s": 0.24061265998548806, + "recall_at_k": 0.592 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 89.405232646052 + } + ] +} diff --git a/experiments/2026-11-text-format/results/scan/wiki-minilm.json b/experiments/2026-11-text-format/results/scan/wiki-minilm.json new file mode 100644 index 0000000..50cbab4 --- /dev/null +++ b/experiments/2026-11-text-format/results/scan/wiki-minilm.json @@ -0,0 +1,59 @@ +{ + "collection": "wiki-minilm", + "vectors": 20000, + "dimension": 384, + "queries": 50, + "k": 10, + "repeat": 3, + "paths": [ + { + "path": "cached buffer (Store::search)", + "ms_per_query": 4.033472499999999, + "note": "codes scanned in place in one allocation; text read for the hits only" + } + ], + "loads": [ + { + "source": "sqlite (borrowed blobs, one copy)", + "ms": 12.746458, + "bytes": 30720000 + }, + { + "source": "flat code file (one read)", + "ms": 2.7811250000000003, + "bytes": 30720000 + } + ], + "codecs": [ + { + "codec": "f32", + "bytes_per_vector": 1536, + "threads": 1, + "ms_per_query": 4.0176016599999995, + "gb_per_s": 7.64635287411744, + "recall_at_k": 1.0 + }, + { + "codec": "binary-hamming", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 0.20049583999999998, + "gb_per_s": 4.78812926991403, + "recall_at_k": 0.422 + }, + { + "codec": "pq48", + "bytes_per_vector": 48, + "threads": 1, + "ms_per_query": 3.9473649999999996, + "gb_per_s": 0.24320021077351603, + "recall_at_k": 0.652 + } + ], + "bandwidth": [ + { + "threads": 1, + "gb_per_s": 86.39816051584862 + } + ] +} diff --git a/experiments/2026-11-text-format/results/stats.json b/experiments/2026-11-text-format/results/stats.json new file mode 100644 index 0000000..37e9ea5 --- /dev/null +++ b/experiments/2026-11-text-format/results/stats.json @@ -0,0 +1,57 @@ +{ + "records": 20000, + "lookup_queries": 500, + "descriptive_queries": 500, + "seed": 2026, + "formats": { + "paragraph": { + "mean_bytes": 503.7568, + "mean_words": 79.5741, + "mean_tokens": 114.0377, + "p99_tokens": 231, + "over_400_tokens": 1 + }, + "bullets": { + "mean_bytes": 334.27645, + "mean_words": 60.1161, + "mean_tokens": 74.5787, + "p99_tokens": 155, + "over_400_tokens": 0 + }, + "kv": { + "mean_bytes": 322.8965, + "mean_words": 49.73615, + "mean_tokens": 73.5787, + "p99_tokens": 154, + "over_400_tokens": 0 + }, + "json": { + "mean_bytes": 366.6407, + "mean_words": 46.3079, + "mean_tokens": 113.895, + "p99_tokens": 216, + "over_400_tokens": 0 + }, + "table": { + "mean_bytes": 392.1762, + "mean_words": 73.49555, + "mean_tokens": 119.0985, + "p99_tokens": 221, + "over_400_tokens": 0 + }, + "wiki": { + "mean_bytes": 376.23385, + "mean_words": 71.19245, + "mean_tokens": 73.91285, + "p99_tokens": 158, + "over_400_tokens": 0 + }, + "header-prose": { + "mean_bytes": 701.13035, + "mean_words": 120.9286, + "mean_tokens": 147.49155, + "p99_tokens": 283, + "over_400_tokens": 0 + } + } +} diff --git a/experiments/2026-11-text-format/results/summary.txt b/experiments/2026-11-text-format/results/summary.txt new file mode 100644 index 0000000..60b08a2 --- /dev/null +++ b/experiments/2026-11-text-format/results/summary.txt @@ -0,0 +1,72 @@ +== bge-small, descriptive queries +format tokens f32 nDCG band binary-hamming n/b N@95% pq48 n/b N@95% +paragraph 114 0.1735 0.0110 4.579 754 1.247 78 +bullets 75 0.1797 0.0101 4.819 825 1.253 74 +kv 74 0.1907 0.0099 4.709 722 1.205 69 +json 114 0.2238 0.0093 4.782 505 1.079 62 +table 119 0.2114 0.0095 4.679 476 1.171 68 +wiki 74 0.1131 0.0116 4.467 611 1.261 70 +header-prose 147 0.1575 0.0108 4.759 606 1.277 69 +best nDCG@10 within a time budget (exact f32 scan ≈ 4.10 ms/query, one thread) + 10% (0.41 ms): json 0.2227 (binary-hamming 3.0σ, 0.28 ms); table 0.2128 (binary-hamming 3.0σ, 0.29 ms); kv 0.1906 (binary-hamming 3.0σ, 0.33 ms) + 25% (1.03 ms): json 0.2227 (binary-hamming 3.0σ, 0.28 ms); table 0.2128 (binary-hamming 3.0σ, 0.29 ms); kv 0.1906 (binary-hamming 3.0σ, 0.33 ms) + 50% (2.05 ms): json 0.2227 (binary-hamming 3.0σ, 0.28 ms); table 0.2128 (binary-hamming 3.0σ, 0.29 ms); kv 0.1906 (binary-hamming 3.0σ, 0.33 ms) + 100% (4.10 ms): json 0.2227 (binary-hamming 3.0σ, 0.28 ms); table 0.2128 (binary-hamming 3.0σ, 0.29 ms); kv 0.1908 (pq48 N=200, 4.03 ms) + +== bge-small, lookup queries +format tokens f32 nDCG band binary-hamming n/b N@95% pq48 n/b N@95% +paragraph 114 0.3700 0.0100 4.915 966 1.409 98 +bullets 75 0.4018 0.0097 4.985 860 1.377 85 +kv 74 0.4353 0.0095 4.947 947 1.337 78 +json 114 0.4407 0.0082 5.386 760 1.298 75 +table 119 0.4154 0.0095 4.683 712 1.228 71 +wiki 74 0.1810 0.0108 4.530 804 1.313 81 +header-prose 147 0.3702 0.0106 4.675 835 1.369 88 +best nDCG@10 within a time budget (exact f32 scan ≈ 4.10 ms/query, one thread) + 10% (0.41 ms): json 0.4395 (binary-hamming 3.0σ, 0.30 ms); kv 0.4320 (binary-hamming N=1000, 0.41 ms); table 0.4156 (binary-hamming N=1000, 0.41 ms) + 25% (1.03 ms): json 0.4411 (binary-hamming N=1000, 0.43 ms); kv 0.4320 (binary-hamming N=1000, 0.41 ms); table 0.4156 (binary-hamming N=1000, 0.41 ms) + 50% (2.05 ms): json 0.4411 (binary-hamming N=1000, 0.43 ms); kv 0.4320 (binary-hamming N=1000, 0.41 ms); table 0.4156 (binary-hamming N=1000, 0.41 ms) + 100% (4.10 ms): json 0.4411 (binary-hamming N=1000, 0.43 ms); kv 0.4353 (exact f32, 4.06 ms); table 0.4156 (binary-hamming N=1000, 0.41 ms) + +== minilm, descriptive queries +format tokens f32 nDCG band binary-hamming n/b N@95% pq48 n/b N@95% +paragraph 114 0.1295 0.0213 2.285 246 1.005 40 +bullets 75 0.1613 0.0195 2.316 315 0.981 42 +kv 74 0.1433 0.0198 2.326 312 0.953 44 +json 114 0.1422 0.0201 2.119 291 0.873 36 +table 119 0.1808 0.0181 2.255 228 0.931 42 +wiki 74 0.0909 0.0211 2.326 270 1.047 45 +header-prose 147 0.1359 0.0212 2.309 225 1.028 44 +best nDCG@10 within a time budget (exact f32 scan ≈ 4.04 ms/query, one thread) + 10% (0.40 ms): table 0.1811 (binary-hamming N=200, 0.25 ms); bullets 0.1604 (binary-hamming N=1000, 0.39 ms); kv 0.1421 (binary-hamming N=500, 0.30 ms) + 25% (1.01 ms): table 0.1811 (binary-hamming N=200, 0.25 ms); bullets 0.1604 (binary-hamming N=1000, 0.39 ms); kv 0.1424 (binary-hamming N=1000, 0.40 ms) + 50% (2.02 ms): table 0.1811 (binary-hamming N=200, 0.25 ms); bullets 0.1604 (binary-hamming N=1000, 0.39 ms); kv 0.1424 (binary-hamming N=1000, 0.40 ms) + 100% (4.04 ms): table 0.1811 (binary-hamming N=200, 0.25 ms); bullets 0.1613 (exact f32, 3.91 ms); kv 0.1433 (pq48 N=200, 4.02 ms) + +== minilm, lookup queries +format tokens f32 nDCG band binary-hamming n/b N@95% pq48 n/b N@95% +paragraph 114 0.1793 0.0169 2.801 367 1.169 52 +bullets 75 0.3359 0.0160 2.770 407 1.137 47 +kv 74 0.2755 0.0164 2.742 441 1.094 51 +json 114 0.2401 0.0165 2.573 317 1.033 44 +table 119 0.3533 0.0157 2.647 398 1.057 45 +wiki 74 0.0980 0.0185 2.672 365 1.179 49 +header-prose 147 0.2082 0.0185 2.606 330 1.122 46 +best nDCG@10 within a time budget (exact f32 scan ≈ 4.04 ms/query, one thread) + 10% (0.40 ms): table 0.3536 (binary-hamming N=500, 0.30 ms); bullets 0.3372 (binary-hamming 2.0σ, 0.23 ms); kv 0.2768 (binary-hamming N=500, 0.30 ms) + 25% (1.01 ms): table 0.3536 (binary-hamming N=500, 0.30 ms); bullets 0.3372 (binary-hamming 2.0σ, 0.23 ms); kv 0.2768 (binary-hamming N=500, 0.30 ms) + 50% (2.02 ms): table 0.3536 (binary-hamming N=500, 0.30 ms); bullets 0.3372 (binary-hamming 2.0σ, 0.23 ms); kv 0.2768 (binary-hamming N=500, 0.30 ms) + 100% (4.04 ms): table 0.3536 (binary-hamming N=500, 0.30 ms); bullets 0.3372 (binary-hamming 2.0σ, 0.23 ms); kv 0.2768 (binary-hamming N=500, 0.30 ms) + +Cross-format checks, leaving one format out at a time + depth for 90% recall: Spearman ρ(noise/band, depth) = 0.985, depth ≈ 3.08·k·r^1.70, held-out format within a factor of 1.14 on average (56 points) + depth for 95% recall: Spearman ρ(noise/band, depth) = 0.985, depth ≈ 4.74·k·r^1.83, held-out format within a factor of 1.16 on average (56 points) + depth for 99% recall: Spearman ρ(noise/band, depth) = 0.968, depth ≈ 10.59·k·r^1.98, held-out format within a factor of 1.24 on average (56 points) + hybrid nDCG at N=10: predicted from f32 nDCG and noise/band within 0.0067 on a held-out format (f32 nDCG times the mean retained share: 0.0181) + hybrid nDCG at N=20: predicted from f32 nDCG and noise/band within 0.0060 on a held-out format (f32 nDCG times the mean retained share: 0.0143) + hybrid nDCG at N=30: predicted from f32 nDCG and noise/band within 0.0047 on a held-out format (f32 nDCG times the mean retained share: 0.0117) + hybrid nDCG at N=50: predicted from f32 nDCG and noise/band within 0.0032 on a held-out format (f32 nDCG times the mean retained share: 0.0081) + hybrid nDCG at N=100: predicted from f32 nDCG and noise/band within 0.0026 on a held-out format (f32 nDCG times the mean retained share: 0.0048) + hybrid nDCG at N=200: predicted from f32 nDCG and noise/band within 0.0016 on a held-out format (f32 nDCG times the mean retained share: 0.0025) + hybrid nDCG at N=500: predicted from f32 nDCG and noise/band within 0.0008 on a held-out format (f32 nDCG times the mean retained share: 0.0010) + hybrid nDCG at N=1000: predicted from f32 nDCG and noise/band within 0.0005 on a held-out format (f32 nDCG times the mean retained share: 0.0006) diff --git a/experiments/2026-11-text-format/run.sh b/experiments/2026-11-text-format/run.sh new file mode 100755 index 0000000..0e2d9b1 --- /dev/null +++ b/experiments/2026-11-text-format/run.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Text format: the same WikiBio records written as prose, bullets, key-value, JSON, a table row +# and a key-value header above prose, plus each record's own Wikipedia paragraph. For each format, +# query type (lookup, descriptive) and embedding model: exact f32 nDCG@10, the neighbour band and +# each codec's rank noise, the re-score depth and nDCG behind a binary-hamming or pq48 scan, and +# the per-vector scan cost, so summarise.py can draw accuracy against time per query. +# Retrieval only, no chat model, no API key. One record per chunk (`ingest --whole-files`), so +# every format has the same number of vectors. +# Needs: a built `hvec` (cargo build --release), curl, and `uv` (to read WikiBio's parquet file). +# Downloads 41 MB on first run. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$HERE/../.." +HVEC="${HVEC:-$ROOT/target/release/hvec}" +WORK="${WORK:-$HERE/work}" +RESULTS="${RESULTS:-$HERE/results}" +K="${K:-10}" +RECORDS="${RECORDS:-20000}" +QUERIES="${QUERIES:-500}" +SEED="${SEED:-2026}" +FORMATS="${FORMATS:-paragraph bullets kv json table wiki header-prose}" +MODELS="${MODELS:-bge-small minilm}" +QTYPES="${QTYPES:-lookup descriptive}" +CODECS="${CODECS:-int8,binary,binary-hamming,pq48}" +FIRST="${FIRST:-binary-hamming,pq48}" +DEPTHS="${DEPTHS:-10,20,30,50,100,200,500,1000}" +mkdir -p "$WORK/dl" "$RESULTS/recall" "$RESULTS/rescore" "$RESULTS/scan" + +PARQUET="$WORK/dl/wikibio-test.parquet" +[ -f "$PARQUET" ] || curl -sfL -o "$PARQUET" \ + https://huggingface.co/datasets/thoughtworks/wiki_bio/resolve/main/data/test-00000-of-00001.parquet +[ -f "$WORK/beir/stats.json" ] || (cd "$WORK" && uv run -q --no-project --with pyarrow python -I \ + "$HERE/wikibio_to_formats.py" "$PARQUET" "$WORK/beir" "$RECORDS" "$QUERIES" "$SEED") +cp "$WORK/beir/stats.json" "$WORK/beir/examples.md" "$RESULTS/" + +export HVEC_CONFIG="$WORK/config.toml" +cat > "$HVEC_CONFIG" </dev/null || true + sysctl -n hw.memsize 2>/dev/null | awk '{printf "memory: %.0f GB\n", $1/1e9}' || true +} > "$RESULTS/machine.txt" + +for f in $FORMATS; do + for q in $QTYPES; do # the question sets are the same for every format; the corpus is not + [ -f "$WORK/questions-$q.jsonl" ] && [ -d "$WORK/corpus-$f" ] && continue + "$HVEC" questions import-beir "$WORK/beir/$f" --split "$q" --out "$WORK/questions-$q.jsonl" \ + --corpus-out "$WORK/corpus-$f" + done + for m in $MODELS; do + col="$f-$m" + if ! "$HVEC" collections list | grep -q "^$col "; then + "$HVEC" ingest "$WORK/corpus-$f" --collection "$col" --embedder "$m" --ext txt --whole-files --batch 64 + fi + for q in $QTYPES; do + "$HVEC" bench recall --collection "$col" -k "$K" --codecs "$CODECS" \ + --questions "$WORK/questions-$q.jsonl" --json > "$RESULTS/recall/$col-$q.json" + "$HVEC" bench rescore --collection "$col" -k "$K" --codecs "$FIRST" --rescore f32 \ + --depths "$DEPTHS" --questions "$WORK/questions-$q.jsonl" --json > "$RESULTS/rescore/$col-$q.json" + done + # Per-vector scan cost, one thread. It does not depend on the text, only on the codec and the + # vector count, which is the same for every format; measured per collection as a check. + "$HVEC" bench scan --collection "$col" --codecs "f32,${FIRST}" --threads 1 --skip-sqlite \ + --bandwidth-mib 256 --json > "$RESULTS/scan/$col.json" + echo "== $col done" + done +done +cp "$HVEC_CONFIG" "$RESULTS/config.toml" +python3 -I "$HERE/summarise.py" "$RESULTS" "$K" | tee "$RESULTS/summary.txt" diff --git a/experiments/2026-11-text-format/summarise.py b/experiments/2026-11-text-format/summarise.py new file mode 100644 index 0000000..4f70134 --- /dev/null +++ b/experiments/2026-11-text-format/summarise.py @@ -0,0 +1,198 @@ +"""Summarise the text-format run: accuracy against time per query, per format and context. + + python3 -I summarise.py + +Reads recall/--.json (bench recall), rescore/<...>.json (bench rescore), +scan/-.json (bench scan) and stats.json (record sizes). A context is an +(embedding model, query type) pair. For each context it prints: + +1. Per format: tokens per record, exact f32 nDCG@k, the neighbour band, each first stage's noise + over band and the depth an f32 re-score needs for 95% recall@k behind it. +2. The frontier: every operating point (exact f32; a first-stage scan alone; a scan plus an f32 + re-score of N candidates, fixed or adaptive), timed as scan + N x the f32 cost per vector on + one thread, and the best format at budgets of 10, 25, 50 and 100% of an exact f32 scan. + +Then the two cross-format checks: +- depth: does noise / band predict the re-score depth on a format it was not fitted on? +- accuracy: does (f32 nDCG, noise / band) predict a hybrid's nDCG on a held-out format? +""" + +import json +import math +import sys +from collections import defaultdict +from pathlib import Path + +QTYPES = ("lookup", "descriptive") +BUDGETS = (0.10, 0.25, 0.50, 1.00) + + +def ranks(xs): + order = sorted(range(len(xs)), key=lambda i: xs[i]) + r = [0.0] * len(xs) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and xs[order[j + 1]] == xs[order[i]]: + j += 1 + for t in range(i, j + 1): + r[order[t]] = (i + j) / 2 + 1 + i = j + 1 + return r + + +def pearson(a, b): + n = len(a) + ma, mb = sum(a) / n, sum(b) / n + cov = sum((x - ma) * (y - mb) for x, y in zip(a, b)) + va = sum((x - ma) ** 2 for x in a) ** 0.5 + vb = sum((y - mb) ** 2 for y in b) ** 0.5 + return cov / (va * vb) if va and vb else float("nan") + + +def fit(xs, ys): + """Least squares y = a + b x.""" + n = len(xs) + mx, my = sum(xs) / n, sum(ys) / n + sxx = sum((x - mx) ** 2 for x in xs) + b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sxx if sxx else 0.0 + return my - b * mx, b + + +def split_name(stem, formats): + """`-[-]` -> (format, model, qtype or None). Formats may contain '-'.""" + qtype = None + for q in QTYPES: + if stem.endswith("-" + q): + qtype, stem = q, stem[: -len(q) - 1] + fmt = max((f for f in formats if stem.startswith(f + "-")), key=len) + return fmt, stem[len(fmt) + 1:], qtype + + +def policy_name(p): + return {"none": "scan only", "fixed": f"N={p.get('depth')}", "margin": f"{p.get('sigmas')}σ"}[p["kind"]] + + +def main(results: Path, k: int) -> None: + stats = json.loads((results / "stats.json").read_text()) + formats = list(stats["formats"]) + + cost = {} # (format, model) -> {codec: ms per query, "f32_ns": ns per vector, "vectors": n} + for p in sorted((results / "scan").glob("*.json")): + r = json.loads(p.read_text()) + fmt, model, _ = split_name(p.stem, formats) + c = {x["codec"]: x["ms_per_query"] for x in r["codecs"] if x["threads"] == 1} + c["f32_ns"] = c["f32"] * 1e6 / r["vectors"] + c["vectors"] = r["vectors"] + cost[(fmt, model)] = c + + rows = defaultdict(dict) # (model, qtype) -> format -> row + profiles = [] # one per (format, model, qtype, first stage), for the cross-format checks + for p in sorted((results / "recall").glob("*.json")): + fmt, model, qtype = split_name(p.stem, formats) + if (fmt, model) not in cost: + continue + c = cost[(fmt, model)] + recall = {r["codec"]: r for r in json.loads(p.read_text())} + resc = json.loads((results / "rescore" / p.name).read_text()) + f32 = recall["f32"] + row = { + "tokens": stats["formats"][fmt]["mean_tokens"], + "ndcg": f32["ndcg_at_k"], + "band": f32["neighbour_band"], + "ratio": {n: r["rank_noise"] / f32["neighbour_band"] for n, r in recall.items() if n != "f32"}, + "depth95": {}, + "points": [("exact f32", c["f32"], f32["ndcg_at_k"])], + } + for pr in resc["profiles"]: + d = dict((t, n) for t, n in pr["depth_for_recall"]) + row["depth95"][pr["first"]] = d.get(0.95) + profiles.append({"format": fmt, "model": model, "qtype": qtype, "first": pr["first"], + "ratio": pr["rank_noise"] / pr["neighbour_band"], "depth": d, "f32_ndcg": f32["ndcg_at_k"], + "cells": {}}) + by_first = {pp["first"]: pp for pp in profiles[-len(resc["profiles"]):]} if resc["profiles"] else {} + for cell in resc["cells"]: + if cell["first"] not in c: + continue + ms = c[cell["first"]] + cell["mean_depth"] * c["f32_ns"] / 1e6 + row["points"].append((f"{cell['first']} {policy_name(cell['policy'])}", ms, cell["ndcg_at_k"])) + if cell["policy"]["kind"] == "fixed" and cell["first"] in by_first: + by_first[cell["first"]]["cells"][cell["policy"]["depth"]] = cell["ndcg_at_k"] + row["f32_ms"] = c["f32"] + rows[(model, qtype)][fmt] = row + + firsts = sorted({p["first"] for p in profiles}) + for (model, qtype), fr in sorted(rows.items()): + print(f"== {model}, {qtype} queries") + head = f"{'format':<13} {'tokens':>6} {'f32 nDCG':>8} {'band':>7}" + for f in firsts: + head += f" {f + ' n/b':>18} {'N@95%':>6}" + print(head) + for fmt in formats: + if fmt not in fr: + continue + r = fr[fmt] + line = f"{fmt:<13} {r['tokens']:>6.0f} {r['ndcg']:>8.4f} {r['band']:>7.4f}" + for f in firsts: + line += f" {r['ratio'].get(f, float('nan')):>18.3f} {r['depth95'].get(f) or '-':>6}" + print(line) + f32_ms = sum(r["f32_ms"] for r in fr.values()) / len(fr) + print(f"best nDCG@{k} within a time budget (exact f32 scan ≈ {f32_ms:.2f} ms/query, one thread)") + for b in BUDGETS: + limit = b * f32_ms + best = [] + for fmt, r in fr.items(): + pts = [p for p in r["points"] if p[1] <= limit] + if pts: + name, ms, nd = max(pts, key=lambda p: p[2]) + best.append((nd, fmt, name, ms)) + best.sort(reverse=True) + cells = "; ".join(f"{fmt} {nd:.4f} ({name}, {ms:.2f} ms)" for nd, fmt, name, ms in best[:3]) + print(f" {100 * b:>4.0f}% ({limit:.2f} ms): {cells or 'nothing fits'}") + print() + + if len({p["format"] for p in profiles}) < 3: + return + print("Cross-format checks, leaving one format out at a time") + for t in (0.9, 0.95, 0.99): + pts = [p for p in profiles if p["depth"].get(t)] + xs = [math.log(p["ratio"]) for p in pts] + ys = [math.log(max(p["depth"][t], k) / k) for p in pts] + errs = [] + for fmt in formats: + tr = [(x, y) for x, y, p in zip(xs, ys, pts) if p["format"] != fmt] + te = [(x, y) for x, y, p in zip(xs, ys, pts) if p["format"] == fmt] + if len(tr) < 3 or not te: + continue + a, b = fit(*zip(*tr)) + errs += [abs(a + b * x - y) for x, y in te] + a, b = fit(xs, ys) + print(f" depth for {t:.0%} recall: Spearman ρ(noise/band, depth) = {pearson(ranks(xs), ranks(ys)):.3f}, " + f"depth ≈ {math.exp(a):.2f}·k·r^{b:.2f}, held-out format within a factor of " + f"{math.exp(sum(errs) / len(errs)):.2f} on average ({len(pts)} points)") + depths = sorted({d for p in profiles for d in p["cells"]}) + for d in depths: + pts = [p for p in profiles if d in p["cells"] and p["f32_ndcg"]] + if len(pts) < 4: + continue + # The share of exact f32's nDCG a hybrid keeps, against log noise/band. + xs = [math.log(p["ratio"]) for p in pts] + ys = [p["cells"][d] / p["f32_ndcg"] for p in pts] + errs, naive = [], [] + for fmt in formats: + tr = [(x, y) for x, y, p in zip(xs, ys, pts) if p["format"] != fmt] + te = [(x, y, p) for x, y, p in zip(xs, ys, pts) if p["format"] == fmt] + if len(tr) < 3 or not te: + continue + a, b = fit(*zip(*tr)) + mean = sum(y for _, y in tr) / len(tr) + for x, y, p in te: + errs.append(abs((a + b * x - y) * p["f32_ndcg"])) + naive.append(abs((mean - y) * p["f32_ndcg"])) + if errs: + print(f" hybrid nDCG at N={d}: predicted from f32 nDCG and noise/band within {sum(errs) / len(errs):.4f} " + f"on a held-out format (f32 nDCG times the mean retained share: {sum(naive) / len(naive):.4f})") + + +if __name__ == "__main__": + main(Path(sys.argv[1]), int(sys.argv[2]) if len(sys.argv) > 2 else 10) diff --git a/experiments/2026-11-text-format/wikibio_to_formats.py b/experiments/2026-11-text-format/wikibio_to_formats.py new file mode 100644 index 0000000..69a121e --- /dev/null +++ b/experiments/2026-11-text-format/wikibio_to_formats.py @@ -0,0 +1,230 @@ +"""Write one WikiBio sample as several BEIR datasets, the same records in a different layout each. + + python3 -I wikibio_to_formats.py [records] [queries-per-type] [seed] + +Every format but `wiki` and `header-prose` holds exactly the same facts: the cleaned infobox +fields of each record. `wiki` is the record's own Wikipedia paragraph, which holds different +facts (it is the natural-prose reference), and `header-prose` is the infobox as a key-value +header above that paragraph. + +The queries and judgments are the same for every format, from the infobox: +- `lookup`: two of a record's attributes ("who was born in X and played as Y?"), relevant + to every record with both values, kept only when that is at most 3 records. +- `descriptive`: one attribute value shared by 5 to 50 records ("people who worked as X"), + relevant to all of them. + +Writes `//{corpus.jsonl,queries.jsonl,qrels/lookup.tsv,qrels/descriptive.tsv}`, +`/stats.json` (size per record per format) and `/examples.md` (one record in every +format). Needs pyarrow. +""" + +import json +import random +import re +import sys +from collections import defaultdict +from pathlib import Path + +import pyarrow.parquet as pq + +FORMATS = ["paragraph", "bullets", "kv", "json", "table", "wiki", "header-prose"] + +# Fields with no retrievable content (images, layout, update stamps). +DROP = { + "article_title", "image", "image_size", "imagesize", "caption", "alt", "website", "signature", + "pcupdate", "ntupdate", "image_caption", "logo", "landscape", "module", "footnotes", +} +MAX_FIELDS = 16 +MAX_VALUE_WORDS = 25 +MAX_WIKI_WORDS = 150 +JUNK_VALUES = {"", "unknown", "none", "n/a", "--", "-", "?"} + +# Query phrasing. Deliberately not the field labels the documents use, so no format gets the +# query's own words for free. +PHRASES = { + "birth_date": "was born on {}", + "birth_place": "was born in {}", + "death_date": "died on {}", + "death_place": "died in {}", + "position": "played as {}", + "occupation": "worked as {}", + "nationality": "had {} nationality", + "party": "belonged to the {} party", + "alma_mater": "studied at {}", + "genre": "worked in {}", + "religion": "followed {}", + "spouse": "was married to {}", + "clubs": "played for {}", + "residence": "lived in {}", +} +DESCRIPTIVE_FIELDS = ["occupation", "position", "nationality", "party", "genre", "religion", + "birth_place", "death_place", "alma_mater", "residence"] + +PTB = {"-lrb-": "(", "-rrb-": ")", "-lsb-": "[", "-rsb-": "]", "-lcb-": "{", "-rcb-": "}", + "``": '"', "''": '"'} +TOKEN = re.compile(r"\w+|[^\w\s]") + + +def clean(s: str) -> str: + for k, v in PTB.items(): + s = s.replace(k, v) + return " ".join(s.split()) + + +def record(row): + t = row["input_text"]["table"] + fields = {} + for h, v in zip(t["column_header"], t["content"]): + h = h.strip() + v = clean(v) + if h in DROP or h.endswith("image") or h.endswith("caption") or h in fields: + continue + if v.lower() in JUNK_VALUES or len(v.split()) > MAX_VALUE_WORDS: + continue + fields[h] = v + name = fields.pop("name", None) + if not name and "article_title" in t["column_header"]: + title = clean(t["content"][t["column_header"].index("article_title")]) + name = re.sub(r"\s*\(.*?\)\s*$", "", title) + if not name: + return None + fields = dict(list(fields.items())[: MAX_FIELDS - 1]) + if len(fields) < 3: + return None + words, wiki = 0, [] + for sent in row["target_text"].split("\n"): + sent = clean(sent) + if not sent: + continue + if wiki and words + len(sent.split()) > MAX_WIKI_WORDS: + break + wiki.append(sent) + words += len(sent.split()) + return {"name": name, "fields": fields, "wiki": " ".join(wiki)} + + +def label(h: str) -> str: + return h.replace("_", " ") + + +def render(r, fmt: str) -> str: + name, f = r["name"], r["fields"] + items = [("name", name)] + list(f.items()) + kv = "; ".join(f"{label(h)}: {v}" for h, v in items) + if fmt == "paragraph": + return " ".join([f"{name}."] + [f"{name}'s {label(h)} is {v}." for h, v in f.items()]) + if fmt == "bullets": + return "\n".join(f"- {label(h)}: {v}" for h, v in items) + if fmt == "kv": + return kv + if fmt == "json": + return json.dumps(dict(items), ensure_ascii=False) + if fmt == "table": + head = "| " + " | ".join(label(h) for h, _ in items) + " |" + rule = "|" + "---|" * len(items) + row = "| " + " | ".join(v.replace("|", "/") for _, v in items) + " |" + return "\n".join([head, rule, row]) + if fmt == "wiki": + return r["wiki"] + if fmt == "header-prose": + return kv + "\n\n" + r["wiki"] + raise ValueError(fmt) + + +def make_queries(recs, ids, n, rng): + by_value = defaultdict(set) + for i, r in zip(ids, recs): + for h, v in r["fields"].items(): + if h in PHRASES: + by_value[(h, v)].add(i) + + lookup, seen = [], set() + order = list(range(len(recs))) + rng.shuffle(order) + for j in order: + if len(lookup) >= n: + break + cand = [h for h in recs[j]["fields"] if h in PHRASES] + if len(cand) < 2: + continue + a, b = sorted(rng.sample(cand, 2), key=list(PHRASES).index) + va, vb = recs[j]["fields"][a], recs[j]["fields"][b] + rel = by_value[(a, va)] & by_value[(b, vb)] + if len(rel) > 3 or (a, va, b, vb) in seen: + continue + seen.add((a, va, b, vb)) + if (a, b) in (("birth_date", "birth_place"), ("death_date", "death_place")): + text = f"who {PHRASES[a].format(va)} in {vb}?" + else: + text = f"who {PHRASES[a].format(va)} and {PHRASES[b].format(vb)}?" + lookup.append((text, sorted(rel))) + + cats = [(h, v) for (h, v), s in by_value.items() if h in DESCRIPTIVE_FIELDS and 5 <= len(s) <= 50] + cats.sort() + rng.shuffle(cats) + descriptive = [(f"people who {PHRASES[h].format(v)}", sorted(by_value[(h, v)])) for h, v in cats[:n]] + return lookup, descriptive + + +def main(src: Path, out: Path, n_records: int, n_queries: int, seed: int) -> None: + rng = random.Random(seed) + rows = pq.read_table(src).to_pylist() + order = list(range(len(rows))) + rng.shuffle(order) + recs, ids = [], [] + for i in order: + r = record(rows[i]) + if r: + recs.append(r) + ids.append(f"wb{i:05d}") + if len(recs) >= n_records: + break + lookup, descriptive = make_queries(recs, ids, n_queries, rng) + + queries = [{"_id": f"L{q:04d}", "text": t} for q, (t, _) in enumerate(lookup)] + queries += [{"_id": f"D{q:04d}", "text": t} for q, (t, _) in enumerate(descriptive)] + qrels = { + "lookup": [(f"L{q:04d}", d) for q, (_, rel) in enumerate(lookup) for d in rel], + "descriptive": [(f"D{q:04d}", d) for q, (_, rel) in enumerate(descriptive) for d in rel], + } + + stats = {"records": len(recs), "lookup_queries": len(lookup), "descriptive_queries": len(descriptive), + "seed": seed, "formats": {}} + for fmt in FORMATS: + d = out / fmt + (d / "qrels").mkdir(parents=True, exist_ok=True) + texts = [render(r, fmt) for r in recs] + with open(d / "corpus.jsonl", "w") as fh: + for i, t in zip(ids, texts): + fh.write(json.dumps({"_id": i, "title": "", "text": t}, ensure_ascii=False) + "\n") + with open(d / "queries.jsonl", "w") as fh: + for q in queries: + fh.write(json.dumps(q, ensure_ascii=False) + "\n") + for split, rows_ in qrels.items(): + with open(d / "qrels" / f"{split}.tsv", "w") as fh: + fh.write("query-id\tcorpus-id\tscore\n") + fh.writelines(f"{q}\t{c}\t1\n" for q, c in rows_) + toks = sorted(len(TOKEN.findall(t)) for t in texts) + stats["formats"][fmt] = { + "mean_bytes": sum(len(t.encode()) for t in texts) / len(texts), + "mean_words": sum(len(t.split()) for t in texts) / len(texts), + "mean_tokens": sum(toks) / len(toks), + "p99_tokens": toks[int(0.99 * (len(toks) - 1))], + "over_400_tokens": sum(t > 400 for t in toks), + } + (out / "stats.json").write_text(json.dumps(stats, indent=2) + "\n") + + ex = recs[0] + md = [f"# One record ({ids[0]}) in every format\n"] + for fmt in FORMATS: + md += [f"## {fmt}\n", "```", render(ex, fmt), "```\n"] + md += ["## Queries\n"] + [f"- lookup: {t}" for t, _ in lookup[:3]] + [f"- descriptive: {t}" for t, _ in descriptive[:3]] + (out / "examples.md").write_text("\n".join(md) + "\n") + print(f"{len(recs)} records, {len(lookup)} lookup and {len(descriptive)} descriptive queries, " + f"{len(FORMATS)} formats in {out}") + + +if __name__ == "__main__": + a = sys.argv[1:] + main(Path(a[0]), Path(a[1]), int(a[2]) if len(a) > 2 else 20000, + int(a[3]) if len(a) > 3 else 500, int(a[4]) if len(a) > 4 else 2026)