diff --git a/ROADMAP.md b/ROADMAP.md index 88f26bc..35749fc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -127,6 +127,35 @@ rotation is a `d × d` matrix fitted on the corpus. Exit: Experiment 4 published, and a stated answer to whether rotation makes binary usable on this embedding model. +## Ideas from MoE compression + +Weight compression for Mixture-of-Experts models is working on a problem shaped like this one: +many similar objects (experts, vectors) sharing structure, where some matter more than others. +Three lines of work, and what each would mean as an hvec codec or experiment: + +- **MoE post-training quantization** (QuantMoE-Bench, UNITES-Lab; arXiv:2406.08155, + [code](https://github.com/UNITES-Lab/moe-quantization)). Fine-grained, structure-aware mixed + precision: blocks, experts and linear layers get different bit widths, chosen by importance + scorers (expert usage frequency, a linear-weight outlier score) tuned to MoE's sparse + activation. *In hvec:* `mixed` allocates a byte budget's bits per dimension by variance, + and `-w` weights that variance by how much energy queries put into the dimension, the analogue + of weighting an expert by how often it is routed to. The per-vector analogue of an outlier + scorer, giving more bits to vectors in crowded neighbourhoods, is not built yet. +- **MoE-SVD** (citation not yet verified; we have only the description so far). Training-free: + SVD decomposes each expert's weights into low-rank factors, the V matrices are shared across + experts, and the U matrices are trimmed to their top-k columns. *In hvec:* PCA is the SVD of + the centred corpus. `mixed-pca` with zero bits on the trailing components is a low-rank + codec, and its single rotation per collection is the shared V. Top-k U trimming corresponds to + truncating components, which is `mrl` applied after the rotation instead of on raw + dimensions. +- **Sub-MoE, subspace expert merging** (arXiv:2506.23266, + [code](https://github.com/siruihan2024/Sub-MoE)). Experts are clustered by cosine similarity, + each group gets a joint SVD, and the V matrices are merged by expert frequency to resolve + parameter conflicts. *In hvec:* one PCA per k-means cluster instead of one per collection + (closest to locally optimised PQ), with each cluster's bit allocation weighted by how often + queries land in it. This is the natural next codec if `mixed-pca` beats `pq` at equal + bytes. + ## Next - Run Experiments 1 and 3 and the judge re-scoring as soon as two chat models are reachable; @@ -134,6 +163,10 @@ this embedding model. - Codec follow-ups the write-ups flagged: a per-vector scale for `binary-centred`, and a rescoring stage (fetch 5-10× candidates by codec, re-rank with int8 or f32). - A second corpus with graded, denser judgments than SciFact's. +- Done 2026-11-04: neighbour spacing across five corpora (`experiments/2026-11-neighbour-spacing`). + Codec noise ÷ neighbour band predicts recall@10 loss (ρ = 0.98). Follow-ups: rescoring depth + as a function of that ratio, a deduplicated code corpus, and FiQA subsampled to separate + corpus size from spacing. ## Always diff --git a/crates/hvec-bench/src/lib.rs b/crates/hvec-bench/src/lib.rs index 1182bda..002089c 100644 --- a/crates/hvec-bench/src/lib.rs +++ b/crates/hvec-bench/src/lib.rs @@ -12,6 +12,7 @@ pub mod questions; pub mod recall; pub mod record; pub mod report; +pub mod rescore; pub use chunk::{ChunkOptions, chunk_text}; pub use judge::{JUDGE_PROMPT_VERSION, JUDGE_SYSTEM_PROMPT, Verdict, build_judge_prompt, parse_verdict}; diff --git a/crates/hvec-bench/src/recall.rs b/crates/hvec-bench/src/recall.rs index 64fcf7d..137064e 100644 --- a/crates/hvec-bench/src/recall.rs +++ b/crates/hvec-bench/src/recall.rs @@ -53,16 +53,64 @@ pub struct RecallReport { pub ndcg_at_k: Option, /// Mean fraction of relevant documents among the codec's top-k. pub doc_recall_at_k: Option, + /// Median over queries of the exact score gap between rank k and rank + /// k + 1: how far the last true neighbour sits above the first outsider. + /// A property of the corpus and embedding model; the same for every codec. + #[serde(default)] + pub boundary_gap: f64, + /// Median over queries of the exact score gap between rank k and rank + /// 2k: the width of the score window holding the next k competitors. + /// Codec-independent, like `boundary_gap`. + #[serde(default)] + pub neighbour_band: f64, + /// Mean over queries of the standard deviation of the codec's score error + /// over the exact top-2k. Per-query bias is removed, since a constant + /// shift does not change a ranking: this is the noise that reorders. + #[serde(default)] + pub rank_noise: f64, } -fn top_k(scores: &[(i64, f32)], k: usize, metric: Metric) -> Vec { - let mut v: Vec<(i64, f32)> = scores.to_vec(); +/// Indices into `scores`, best first, ties broken by id. +pub(crate) fn ranked(scores: &[(i64, f32)], metric: Metric) -> Vec { + let mut order: Vec = (0..scores.len()).collect(); + let by_id = |a: usize, b: usize| scores[a].0.cmp(&scores[b].0); if metric.higher_is_better() { - v.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + order.sort_by(|&a, &b| scores[b].1.total_cmp(&scores[a].1).then(by_id(a, b))); } else { - v.sort_by(|a, b| a.1.total_cmp(&b.1).then(a.0.cmp(&b.0))); + order.sort_by(|&a, &b| scores[a].1.total_cmp(&scores[b].1).then(by_id(a, b))); + } + order +} + +fn top_k(scores: &[(i64, f32)], k: usize, metric: Metric) -> Vec { + ranked(scores, metric) + .into_iter() + .take(k) + .map(|i| scores[i].0) + .collect() +} + +pub(crate) fn median(mut v: Vec) -> f64 { + if v.is_empty() { + return 0.0; + } + v.sort_by(f64::total_cmp); + let m = v.len() / 2; + if v.len() % 2 == 1 { + v[m] + } else { + (v[m - 1] + v[m]) / 2.0 + } +} + +/// Standard deviation of `approx − exact` over the given indices. +pub(crate) fn centred_noise(exact: &[(i64, f32)], approx: &[(i64, f32)], idx: &[usize]) -> f64 { + if idx.len() < 2 { + return 0.0; } - v.into_iter().take(k).map(|(id, _)| id).collect() + let errs: Vec = idx.iter().map(|&i| f64::from(approx[i].1 - exact[i].1)).collect(); + let mean = errs.iter().sum::() / errs.len() as f64; + (errs.iter().map(|e| (e - mean).powi(2)).sum::() / errs.len() as f64).sqrt() } /// Evaluate `codec` on `(id, f32 vector)` pairs with the given queries. @@ -101,6 +149,9 @@ pub fn evaluate( let mut labelled = 0usize; let mut ndcg_sum = 0f64; let mut doc_recall_sum = 0f64; + let mut boundary_gaps: Vec = Vec::with_capacity(queries.len()); + let mut bands: Vec = Vec::with_capacity(queries.len()); + let mut noise_sum = 0f64; for q in queries { let t = Instant::now(); @@ -113,11 +164,12 @@ pub fn evaluate( baseline_scan_ms += t.elapsed().as_millis() as u64; let t = Instant::now(); + let prepared = codec.prepare_query(&q.vector)?; let approx: Vec<(i64, f32)> = ids .iter() .zip(&encoded) .filter(|(id, _)| Some(**id) != q.exclude) - .map(|(id, e)| codec.score(&q.vector, e, metric).map(|s| (*id, s))) + .map(|(id, e)| codec.score_prepared(&prepared, e, metric).map(|s| (*id, s))) .collect::>()?; scan_ms += t.elapsed().as_millis() as u64; @@ -127,7 +179,17 @@ pub fn evaluate( err_max = err_max.max(e); pairs += 1; } - let truth = top_k(&exact, k, metric); + // `exact` and `approx` are index-aligned: both walk `ids` in order. + let order = ranked(&exact, metric); + let at = |rank: usize| order.get(rank - 1).map(|&i| f64::from(exact[i].1)); + if let (Some(sk), Some(sk1)) = (at(k), at(k + 1)) { + boundary_gaps.push((sk - sk1).abs()); + } + if let (Some(sk), Some(s2k)) = (at(k), at(2 * k)) { + bands.push((sk - s2k).abs()); + } + noise_sum += centred_noise(&exact, &approx, &order[..order.len().min(2 * k)]); + let truth: Vec = order.iter().take(k).map(|&i| exact[i].0).collect(); let got = top_k(&approx, k, metric); recall_sum += recall_at_k(&truth, &got, k); if !truth.is_empty() && truth.first() == got.first() { @@ -169,6 +231,9 @@ pub fn evaluate( labelled_queries: labelled, ndcg_at_k: labelled_mean(ndcg_sum), doc_recall_at_k: labelled_mean(doc_recall_sum), + boundary_gap: median(boundary_gaps), + neighbour_band: median(bands), + rank_noise: noise_sum / nq, }) } @@ -238,6 +303,35 @@ mod tests { assert_eq!(r.vectors, 200); assert_eq!(r.labelled_queries, 0); assert_eq!(r.ndcg_at_k, None); + assert_eq!(r.rank_noise, 0.0); + assert!(r.neighbour_band >= r.boundary_gap && r.boundary_gap > 0.0); + } + + #[test] + fn spacing_is_codec_independent_and_noise_is_not() { + let (ids, vecs) = clustered(); + let queries = self_queries(&ids, &vecs, 20); + let i8 = evaluate(&ids, &vecs, &queries, &Int8Codec::new(64), Metric::Cosine, 10, None).unwrap(); + let bin = evaluate(&ids, &vecs, &queries, &BinaryCodec::new(64), Metric::Cosine, 10, None).unwrap(); + assert_eq!(i8.boundary_gap, bin.boundary_gap); + assert_eq!(i8.neighbour_band, bin.neighbour_band); + assert!( + bin.rank_noise > 10.0 * i8.rank_noise, + "{} vs {}", + bin.rank_noise, + i8.rank_noise + ); + } + + #[test] + fn centred_noise_ignores_a_constant_shift() { + let exact = vec![(1, 0.9), (2, 0.8), (3, 0.7)]; + let shifted: Vec<(i64, f32)> = exact.iter().map(|(i, s)| (*i, s - 0.25)).collect(); + assert!(centred_noise(&exact, &shifted, &[0, 1, 2]) < 1e-6); + let noisy = vec![(1, 0.8), (2, 0.9), (3, 0.7)]; + assert!(centred_noise(&exact, &noisy, &[0, 1, 2]) > 0.05); + assert_eq!(median(vec![3.0, 1.0, 2.0]), 2.0); + assert_eq!(median(vec![4.0, 1.0, 2.0, 3.0]), 2.5); } #[test] diff --git a/crates/hvec-bench/src/rescore.rs b/crates/hvec-bench/src/rescore.rs new file mode 100644 index 0000000..eb1a594 --- /dev/null +++ b/crates/hvec-bench/src/rescore.rs @@ -0,0 +1,511 @@ +//! Two-stage retrieval: scan every vector with a small code, re-score the +//! best `depth` candidates with a larger one, keep the top k. +//! +//! The first stage is the codec held in memory for the scan (`binary`, +//! `pq48`); the second is the representation fetched only for candidates +//! (`int8`, `f32`). Ground truth is the exact f32 top-k, as in +//! [`crate::recall`]. +//! +//! Alongside the sweep over depths, [`evaluate`] reports where the true +//! neighbours land in the first stage's ranking. With an exact second stage, +//! recall at depth `N` is simply the fraction of true neighbours ranked within +//! `N` by the first stage, so the depth needed for a target recall is a +//! percentile of those ranks. That is what lets the depth be compared with +//! the first stage's noise-to-spacing ratio. + +use std::time::Instant; + +use hvec_core::{Codec, CodecError, Encoded, Metric, distance}; +use serde::{Deserialize, Serialize}; + +use crate::metrics::recall_at_k; +use crate::questions::relevance; +use crate::recall::{RecallQuery, centred_noise, median, ranked}; + +/// Recall targets for which the needed depth is reported. +pub const DEPTH_TARGETS: [f64; 4] = [0.90, 0.95, 0.99, 1.0]; + +/// Where the first stage puts the true neighbours, before any re-scoring. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FirstStageProfile { + pub first: String, + pub metric: Metric, + pub k: usize, + pub vectors: usize, + pub queries: usize, + pub first_bytes: usize, + /// As in [`crate::recall::RecallReport::rank_noise`]. + pub rank_noise: f64, + /// As in [`crate::recall::RecallReport::neighbour_band`]. + pub neighbour_band: f64, + /// Recall@k of the first stage alone (depth = k, no re-scoring). + pub recall_at_k: f64, + /// `(target, depth)`: the smallest candidate depth at which an exact + /// second stage reaches the target mean recall@k. + pub depth_for_recall: Vec<(f64, usize)>, + /// Median over queries of the depth at which that query's whole exact + /// top-k is among the candidates. + pub median_full_depth: f64, +} + +/// How many first-stage candidates a cell re-scores. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Policy { + /// No second stage: the first stage's top k is the answer. + None, + /// The same number of candidates for every query. + Fixed { depth: usize }, + /// Adaptive: every candidate whose first-stage score is within + /// `sigmas × noise` of the first stage's k-th score, so crowded queries + /// get more candidates and well-separated ones fewer. Bounded by k below + /// and by the plan's `max_depth` above. + Margin { sigmas: f64 }, +} + +/// What [`evaluate`] runs over one first stage. +#[derive(Debug, Clone)] +pub struct Plan<'a> { + /// Re-scoring representations, each already fitted if it needs it. + pub seconds: &'a [&'a dyn Codec], + /// Fixed depths; values below k are raised to k. + pub depths: &'a [usize], + /// Margins for the adaptive policy, in units of `noise`. + pub margins: &'a [f64], + /// The first stage's score noise, calibrated without labels (for example + /// `rank_noise` on self-queries); the adaptive policy's unit. + pub noise: f64, + /// Cap on any policy's per-query depth. + pub max_depth: usize, +} + +/// One (first stage, second stage, policy) cell. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RescoreReport { + pub first: String, + /// `none` for the no-re-score cell. + pub second: String, + pub policy: Policy, + pub metric: Metric, + pub k: usize, + /// The fixed depth, or the mean depth rounded for adaptive cells. + pub depth: usize, + /// Mean candidates re-scored per query. + pub mean_depth: f64, + /// 90th percentile of candidates re-scored per query. + pub p90_depth: usize, + pub vectors: usize, + pub queries: usize, + pub recall_at_k: f64, + /// 10th percentile of per-query recall@k: the worst queries' experience. + pub p10_recall_at_k: f64, + pub top1_agreement: f64, + pub labelled_queries: usize, + pub ndcg_at_k: Option, + pub doc_recall_at_k: Option, + pub first_bytes: usize, + pub second_bytes: usize, + /// Mean bytes touched per query: the whole first-stage scan plus the + /// re-scored candidates. Compare with `vectors × 4 × dim` for f32. + pub bytes_read_per_query: f64, + /// Time spent re-scoring candidates, over all queries. + pub rescore_ms: u64, +} + +struct Cell { + second: Option, + policy: Policy, + recalls: Vec, + depths: Vec, + top1: usize, + ndcg_sum: f64, + docrec_sum: f64, + ms: u64, +} + +fn percentile(v: &[T], p: f64) -> Option { + if v.is_empty() { + return None; + } + let mut s = v.to_vec(); + s.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let i = ((p * s.len() as f64).ceil() as usize).clamp(1, s.len()) - 1; + Some(s[i]) +} + +/// Run the first stage once and every (second stage, policy) cell of `plan` +/// over it, plus the no-re-score cell. +#[allow(clippy::too_many_arguments)] +pub fn evaluate( + ids: &[i64], + vectors: &[Vec], + queries: &[RecallQuery], + first: &dyn Codec, + metric: Metric, + k: usize, + plan: &Plan<'_>, + sources: Option<&[String]>, +) -> Result<(FirstStageProfile, Vec), CodecError> { + assert_eq!(ids.len(), vectors.len(), "ids and vectors must align"); + if let Some(s) = sources { + assert_eq!(s.len(), ids.len(), "ids and sources must align"); + } + let cap = plan.max_depth.max(k).min(vectors.len()); + let mut depths: Vec = plan.depths.iter().map(|d| (*d).max(k).min(cap)).collect(); + depths.sort_unstable(); + depths.dedup(); + + let enc1 = vectors.iter().map(|v| first.encode(v)).collect::, _>>()?; + let enc2: Vec> = plan + .seconds + .iter() + .map(|c| vectors.iter().map(|v| c.encode(v)).collect::, _>>()) + .collect::>()?; + // A score that ranks last, for a query's excluded self. + let worst = if metric.higher_is_better() { + f32::NEG_INFINITY + } else { + f32::INFINITY + }; + + let mut cells: Vec = Vec::new(); + let mut push = |second: Option, policy: Policy| { + cells.push(Cell { + second, + policy, + recalls: Vec::with_capacity(queries.len()), + depths: Vec::with_capacity(queries.len()), + top1: 0, + ndcg_sum: 0.0, + docrec_sum: 0.0, + ms: 0, + }); + }; + push(None, Policy::None); + for s_i in 0..plan.seconds.len() { + for &depth in &depths { + push(Some(s_i), Policy::Fixed { depth }); + } + for &sigmas in plan.margins { + push(Some(s_i), Policy::Margin { sigmas }); + } + } + let mut labelled = 0usize; + + let mut pooled_ranks: Vec = Vec::with_capacity(queries.len() * k); + let mut full_depths: Vec = Vec::with_capacity(queries.len()); + let mut bands: Vec = Vec::with_capacity(queries.len()); + let mut noise_sum = 0f64; + let mut first_recall_sum = 0f64; + + for q in queries { + // Scores indexed like `ids`; the excluded id scores worst everywhere. + let score_all = |f: &dyn Fn(usize) -> Result| -> Result, CodecError> { + (0..ids.len()) + .map(|i| Ok((ids[i], if Some(ids[i]) == q.exclude { worst } else { f(i)? }))) + .collect() + }; + let exact = score_all(&|i| Ok(distance::score(&q.vector, &vectors[i], metric)))?; + let prepared1 = first.prepare_query(&q.vector)?; + let approx = score_all(&|i| first.score_prepared(&prepared1, &enc1[i], metric))?; + let n_live = ids.len() - usize::from(q.exclude.is_some_and(|x| ids.contains(&x))); + + let exact_order = ranked(&exact, metric); + let approx_order = ranked(&approx, metric); + let truth: Vec = exact_order.iter().take(k).map(|&i| ids[i]).collect(); + + // Spacing and noise, exactly as `recall::evaluate` measures them. + if exact_order.len() >= 2 * k { + let s = |r: usize| f64::from(exact[exact_order[r - 1]].1); + bands.push((s(k) - s(2 * k)).abs()); + } + noise_sum += centred_noise(&exact, &approx, &exact_order[..exact_order.len().min(2 * k)]); + + // First-stage rank (1-based) of every true neighbour. + let mut rank_of = vec![0usize; ids.len()]; + for (r, &i) in approx_order.iter().enumerate() { + rank_of[i] = r + 1; + } + let ranks: Vec = exact_order.iter().take(k).map(|&i| rank_of[i]).collect(); + first_recall_sum += ranks.iter().filter(|&&r| r <= k).count() as f64 / k as f64; + full_depths.push(ranks.iter().copied().max().unwrap_or(0) as f64); + pooled_ranks.extend(ranks); + + // The adaptive policy's depth: candidates within the margin of the + // first stage's own k-th score. Uses first-stage scores only. + let kth = approx[approx_order[k.min(n_live).max(1) - 1]].1; + let within = |sigmas: f64| -> usize { + let margin = (sigmas * plan.noise) as f32; + let n = if metric.higher_is_better() { + approx_order[..cap.min(n_live)].partition_point(|&i| approx[i].1 >= kth - margin) + } else { + approx_order[..cap.min(n_live)].partition_point(|&i| approx[i].1 <= kth + margin) + }; + n.max(k).min(cap.min(n_live)) + }; + + let is_labelled = sources.is_some() && !q.relevant.is_empty(); + if is_labelled { + labelled += 1; + } + let prepared2: Vec> = plan + .seconds + .iter() + .map(|c| c.prepare_query(&q.vector)) + .collect::>()?; + for cell in &mut cells { + let t = Instant::now(); + let (kept, depth): (Vec, usize) = match (cell.second, cell.policy) { + (None, _) | (_, Policy::None) => (approx_order.iter().take(k).copied().collect(), 0), + (Some(s_i), policy) => { + let depth = match policy { + Policy::Fixed { depth } => depth.min(n_live), + Policy::Margin { sigmas } => within(sigmas), + Policy::None => unreachable!(), + }; + let second = plan.seconds[s_i]; + let cands: Vec<(i64, f32)> = approx_order[..depth] + .iter() + .map(|&i| { + second + .score_prepared(&prepared2[s_i], &enc2[s_i][i], metric) + .map(|s| (i as i64, s)) + }) + .collect::>()?; + let kept = ranked(&cands, metric) + .into_iter() + .take(k) + .map(|c| cands[c].0 as usize) + .collect(); + (kept, depth) + } + }; + cell.ms += t.elapsed().as_millis() as u64; + cell.depths.push(depth); + let got: Vec = kept.iter().map(|&i| ids[i]).collect(); + cell.recalls.push(recall_at_k(&truth, &got, k)); + if !truth.is_empty() && truth.first() == got.first() { + cell.top1 += 1; + } + if let (true, Some(src)) = (is_labelled, sources) { + let retrieved: Vec = kept.iter().map(|&i| src[i].clone()).collect(); + if let Some(r) = relevance(&q.relevant, &retrieved) { + cell.ndcg_sum += r.ndcg; + cell.docrec_sum += r.doc_recall; + } + } + } + } + + let nq = queries.len().max(1) as f64; + pooled_ranks.sort_unstable(); + let depth_for_recall = DEPTH_TARGETS + .iter() + .map(|&t| { + if pooled_ranks.is_empty() { + return (t, 0); + } + let need = ((t * pooled_ranks.len() as f64).ceil() as usize).clamp(1, pooled_ranks.len()); + (t, pooled_ranks[need - 1]) + }) + .collect(); + let profile = FirstStageProfile { + first: first.name().to_owned(), + metric, + k, + vectors: vectors.len(), + queries: queries.len(), + first_bytes: first.encoded_size(), + rank_noise: noise_sum / nq, + neighbour_band: median(bands), + recall_at_k: first_recall_sum / nq, + depth_for_recall, + median_full_depth: median(full_depths), + }; + let label_mean = |sum: f64| (labelled > 0).then(|| sum / labelled as f64); + let reports = cells + .into_iter() + .map(|c| { + let second_bytes = c.second.map_or(0, |s| plan.seconds[s].encoded_size()); + let mean_depth = c.depths.iter().sum::() as f64 / nq; + RescoreReport { + first: first.name().to_owned(), + second: c + .second + .map_or_else(|| "none".to_owned(), |s| plan.seconds[s].name().to_owned()), + policy: c.policy, + metric, + k, + depth: match c.policy { + Policy::Fixed { depth } => depth, + _ => mean_depth.round() as usize, + }, + mean_depth, + p90_depth: percentile(&c.depths, 0.9).unwrap_or(0), + vectors: vectors.len(), + queries: queries.len(), + recall_at_k: c.recalls.iter().sum::() / nq, + p10_recall_at_k: percentile(&c.recalls, 0.1).unwrap_or(0.0), + top1_agreement: c.top1 as f64 / nq, + labelled_queries: labelled, + ndcg_at_k: label_mean(c.ndcg_sum), + doc_recall_at_k: label_mean(c.docrec_sum), + first_bytes: first.encoded_size(), + second_bytes, + bytes_read_per_query: (vectors.len() * first.encoded_size()) as f64 + mean_depth * second_bytes as f64, + rescore_ms: c.ms, + } + }) + .collect(); + Ok((profile, reports)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::recall::{self, self_queries}; + use hvec_core::codecs::{BinaryCodec, F32Codec, Int8Codec}; + + struct XorShift(u64); + impl XorShift { + fn next_f32(&mut self) -> f32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + ((self.0 >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } + } + + fn clustered() -> (Vec, Vec>) { + let mut rng = XorShift(7); + let dim = 64; + let centres: Vec> = (0..8).map(|_| (0..dim).map(|_| rng.next_f32()).collect()).collect(); + let (mut ids, mut vecs) = (Vec::new(), Vec::new()); + for (c, centre) in centres.iter().enumerate() { + for m in 0..40 { + let mut v: Vec = centre.iter().map(|x| x + 0.3 * rng.next_f32()).collect(); + distance::normalize(&mut v); + ids.push((c * 100 + m) as i64); + vecs.push(v); + } + } + (ids, vecs) + } + + fn run( + seconds: &[&dyn Codec], + depths: &[usize], + margins: &[f64], + noise: f64, + max_depth: usize, + ) -> (FirstStageProfile, Vec) { + let (ids, vecs) = clustered(); + let queries = self_queries(&ids, &vecs, 30); + let plan = Plan { + seconds, + depths, + margins, + noise, + max_depth, + }; + evaluate( + &ids, + &vecs, + &queries, + &BinaryCodec::new(64), + Metric::Cosine, + 10, + &plan, + None, + ) + .unwrap() + } + + fn fixed(reports: &[RescoreReport]) -> Vec<&RescoreReport> { + reports + .iter() + .filter(|r| matches!(r.policy, Policy::Fixed { .. })) + .collect() + } + + #[test] + fn deeper_f32_rescoring_never_hurts_and_reaches_exact() { + let f32c = F32Codec::new(64); + let (profile, reports) = run(&[&f32c], &[10, 20, 40, 80, 320], &[], 0.0, 1000); + let none = &reports[0]; + assert_eq!(none.policy, Policy::None); + assert_eq!(none.second, "none"); + assert!((none.recall_at_k - profile.recall_at_k).abs() < 1e-12); + let f = fixed(&reports); + let recalls: Vec = f.iter().map(|r| r.recall_at_k).collect(); + assert!(recalls.windows(2).all(|w| w[1] >= w[0]), "{recalls:?}"); + // Every vector re-scored exactly (the excluded self drops out): exact. + assert_eq!(f.last().unwrap().recall_at_k, 1.0); + assert_eq!(f.last().unwrap().p10_recall_at_k, 1.0); + // Re-scoring the codec's own top k reorders but cannot change membership. + assert!((f[0].recall_at_k - profile.recall_at_k).abs() < 1e-12); + assert!(f[0].top1_agreement >= none.top1_agreement); + let d: Vec = profile.depth_for_recall.iter().map(|(_, d)| *d).collect(); + assert!(d.windows(2).all(|w| w[1] >= w[0]) && d[0] >= 10, "{d:?}"); + } + + #[test] + fn depth_for_recall_matches_the_sweep_for_an_exact_second_stage() { + let f32c = F32Codec::new(64); + let (profile, _) = run(&[&f32c], &[10], &[], 0.0, 1000); + let (t, d) = profile.depth_for_recall[1]; + let (_, at) = run(&[&f32c], &[d], &[], 0.0, 1000); + assert!(fixed(&at)[0].recall_at_k >= t - 1e-9, "at depth {d}"); + if d > 10 { + let (_, below) = run(&[&f32c], &[d - 1], &[], 0.0, 1000); + assert!(fixed(&below)[0].recall_at_k < t, "at depth {}", d - 1); + } + } + + #[test] + fn margin_policy_varies_depth_per_query_and_grows_with_the_margin() { + let f32c = F32Codec::new(64); + let (profile, reports) = run(&[&f32c], &[], &[0.0, 1.0, 3.0, 1e6], profile_noise(), 200); + let m: Vec<&RescoreReport> = reports + .iter() + .filter(|r| matches!(r.policy, Policy::Margin { .. })) + .collect(); + assert_eq!(m.len(), 4); + // A zero margin keeps only the first stage's top k (plus exact ties). + assert!(m[0].mean_depth >= 10.0 && m[0].mean_depth < 11.0, "{}", m[0].mean_depth); + let depths: Vec = m.iter().map(|r| r.mean_depth).collect(); + assert!(depths.windows(2).all(|w| w[1] >= w[0]), "{depths:?}"); + // An unbounded margin hits the cap for every query. + assert_eq!(m[3].mean_depth, 200.0); + assert!(m[2].p90_depth as f64 >= m[2].mean_depth); + // Depth differs between queries at a moderate margin. + assert!((m[1].p90_depth as f64) > m[1].mean_depth.floor(), "{:?}", m[1]); + assert!(m[2].recall_at_k >= profile.recall_at_k); + } + + fn profile_noise() -> f64 { + let (ids, vecs) = clustered(); + let q = self_queries(&ids, &vecs, 30); + recall::evaluate(&ids, &vecs, &q, &BinaryCodec::new(64), Metric::Cosine, 10, None) + .unwrap() + .rank_noise + } + + #[test] + fn profile_spacing_and_noise_match_recall_evaluate() { + let i8c = Int8Codec::new(64); + let (profile, reports) = run(&[&i8c], &[10, 50], &[], 0.0, 1000); + let (ids, vecs) = clustered(); + let queries = self_queries(&ids, &vecs, 30); + let r = recall::evaluate(&ids, &vecs, &queries, &BinaryCodec::new(64), Metric::Cosine, 10, None).unwrap(); + assert!((profile.rank_noise - r.rank_noise).abs() < 1e-9); + assert_eq!(profile.neighbour_band, r.neighbour_band); + assert!((profile.recall_at_k - r.recall_at_k).abs() < 1e-12); + let f = fixed(&reports); + assert_eq!(f[1].bytes_read_per_query, (320 * 8 + 50 * i8c.encoded_size()) as f64); + assert_eq!(reports[0].bytes_read_per_query, (320 * 8) as f64); + // Depth is capped at the collection size. + let (_, all) = run(&[&i8c], &[1000], &[], 0.0, 1000); + assert_eq!(fixed(&all)[0].depth, 320); + } +} diff --git a/crates/hvec-bench/tests/codec_accuracy.rs b/crates/hvec-bench/tests/codec_accuracy.rs index 5825aea..4e59a1e 100644 --- a/crates/hvec-bench/tests/codec_accuracy.rs +++ b/crates/hvec-bench/tests/codec_accuracy.rs @@ -64,16 +64,24 @@ fn corpus() -> Vec> { /// Every codec family, fitted on the corpus where the codec needs it, exactly /// as `hvec ingest --codec` and `hvec bench recall` do. fn codecs(vectors: &[Vec]) -> Vec> { - ["f32", "int8", "mrl32+int8", "pq8", "binary-centred", "binary"] - .iter() - .map(|name| { - let mut c = codec_by_name(name, DIM).expect(name); - if c.needs_fit() { - c.fit(vectors).expect(name); - } - c - }) - .collect() + [ + "f32", + "int8", + "mrl32+int8", + "pq8", + "mixed8-pca-w", + "binary-centred", + "binary", + ] + .iter() + .map(|name| { + let mut c = codec_by_name(name, DIM).expect(name); + if c.needs_fit() { + c.fit(vectors).expect(name); + } + c + }) + .collect() } #[test] diff --git a/crates/hvec-core/src/codecs/mixed.rs b/crates/hvec-core/src/codecs/mixed.rs new file mode 100644 index 0000000..fda7b35 --- /dev/null +++ b/crates/hvec-core/src/codecs/mixed.rs @@ -0,0 +1,643 @@ +//! Mixed-precision scalar quantization: a fixed byte budget per vector, with +//! bits allocated per dimension by variance. +//! +//! Uniform codecs spend the same bits on every dimension: one for `binary`, +//! eight for `int8`. `mixed` spends `8·B` bits in all, giving each +//! dimension between 0 and 8 by greedy reverse water-filling on its variance: +//! each extra bit roughly quarters a dimension's quantization error, so the +//! next bit always goes where it removes the most error. This is the classic +//! transform-coding allocation (Huang and Schultheiss, 1963), and the idea +//! mixed-precision model quantization applies to weights, where structures +//! that matter more get more bits. +//! +//! `mixed-pca` first rotates into the corpus's principal components, which +//! concentrates the variance in few dimensions and gives the allocation +//! something to work with; raw embedding dimensions have fairly even variance. +//! The rotation is orthogonal, so dot products and norms are preserved, and +//! the query is rotated once per search through [`Codec::prepare_query`]. +//! +//! `-w` changes what the allocation minimises from reconstruction error to +//! expected score error (see `fit`). +//! +//! Each dimension is quantized uniformly around its mean with the step that +//! minimises mean squared error for a Gaussian of its standard deviation +//! (Max, 1960). Error model: approximate for every metric; the error falls as +//! the budget grows and is zero only in the limit. + +use crate::codecs::{check_dim, check_payload}; +use crate::{Codec, CodecError, Encoded, Metric, Vector}; + +/// Optimal uniform quantizer step for a unit Gaussian, by bit depth 1..=8 +/// (Max 1960, Table II). Index 0 is unused: a 0-bit dimension decodes to its mean. +const GAUSSIAN_STEP: [f64; 9] = [0.0, 1.596, 0.9957, 0.5860, 0.3352, 0.1881, 0.1041, 0.0569, 0.0308]; +const MAX_BITS: u8 = 8; +/// Vectors used to fit; larger samples are thinned evenly. +const FIT_SAMPLE: usize = 20_000; + +/// Fixed byte budget, bits allocated per dimension by variance. +#[derive(Debug, Clone)] +pub struct MixedCodec { + name: String, + dimension: usize, + bytes: usize, + pca: bool, + weighted: bool, + fitted: Option, +} + +#[derive(Debug, Clone)] +struct Fitted { + /// Row-major `d × d`; row `j` is component `j`. `None` without PCA. + rotation: Option>, + /// Per (rotated) dimension. + mean: Vec, + step: Vec, + bits: Vec, +} + +impl MixedCodec { + /// `bytes` per vector, optionally after a PCA rotation, with bits + /// allocated by variance or (`weighted`) by variance × query energy. + pub fn new(dimension: usize, bytes: usize, pca: bool, weighted: bool) -> Result { + let name = format!( + "mixed{bytes}{}{}", + if pca { "-pca" } else { "" }, + if weighted { "-w" } else { "" } + ); + if dimension == 0 || bytes == 0 || bytes * 8 > dimension * usize::from(MAX_BITS) { + return Err(CodecError::BadParams { + codec: name, + reason: format!("{bytes} bytes cannot be spread over {dimension} dimensions at 0 to 8 bits each"), + }); + } + Ok(Self { + name, + dimension, + bytes, + pca, + weighted, + fitted: None, + }) + } + + /// Bits given to each (rotated) dimension, once fitted. + #[must_use] + pub fn bits(&self) -> Option<&[u8]> { + self.fitted.as_ref().map(|f| f.bits.as_slice()) + } + + fn state(&self) -> Result<&Fitted, CodecError> { + self.fitted + .as_ref() + .ok_or_else(|| CodecError::NotFitted(self.name.clone())) + } + + fn rotate(f: &Fitted, x: &[f32]) -> Vector { + match &f.rotation { + None => x.to_vec(), + Some(r) => { + let d = x.len(); + (0..d) + .map(|j| { + let row = &r[j * d..(j + 1) * d]; + row.iter() + .zip(x) + .map(|(&a, &b)| f64::from(a) * f64::from(b)) + .sum::() as f32 + }) + .collect() + } + } + } + + /// Calls `f(dimension, reconstructed value)` for every dimension. + fn for_each_value(fit: &Fitted, encoded: &Encoded, mut f: impl FnMut(usize, f64)) { + let mut reader = BitReader::new(&encoded.0); + for (i, &b) in fit.bits.iter().enumerate() { + let m = f64::from(fit.mean[i]); + if b == 0 { + f(i, m); + continue; + } + let code = reader.read(b); + let half = f64::from((1u32 << b) - 1) / 2.0; + f(i, m + (f64::from(code) - half) * f64::from(fit.step[i])); + } + } +} + +/// Greedy reverse water-filling: `budget` bits over dimensions with these +/// variances, at most [`MAX_BITS`] each. +fn allocate(variance: &[f64], budget: usize) -> Vec { + let mut bits = vec![0u8; variance.len()]; + for _ in 0..budget { + // Error left in a dimension at b bits is about var / 4^b. + let best = (0..variance.len()).filter(|&i| bits[i] < MAX_BITS).max_by(|&a, &b| { + let ea = variance[a] / 4f64.powi(i32::from(bits[a])); + let eb = variance[b] / 4f64.powi(i32::from(bits[b])); + ea.total_cmp(&eb).then(b.cmp(&a)) + }); + match best { + Some(i) => bits[i] += 1, + None => break, + } + } + bits +} + +/// Eigen-decomposition of a symmetric matrix by cyclic Jacobi rotations. +/// Returns `(eigenvalues, eigenvectors)`, eigenvectors as rows, sorted by +/// decreasing eigenvalue. +fn symmetric_eigen(mut a: Vec, n: usize) -> (Vec, Vec) { + let mut v = vec![0f64; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + let scale: f64 = (0..n).map(|i| a[i * n + i].abs()).sum::().max(f64::MIN_POSITIVE); + for _sweep in 0..60 { + let off: f64 = (0..n) + .flat_map(|p| ((p + 1)..n).map(move |q| (p, q))) + .map(|(p, q)| a[p * n + q] * a[p * n + q]) + .sum(); + if off.sqrt() < 1e-12 * scale { + break; + } + for p in 0..n { + for q in (p + 1)..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let akp = a[k * n + p]; + let akq = a[k * n + q]; + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let apk = a[p * n + k]; + let aqk = a[q * n + k]; + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + // Columns of V are eigenvectors. + for k in 0..n { + let vkp = v[k * n + p]; + let vkq = v[k * n + q]; + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[y * n + y].total_cmp(&a[x * n + x])); + let values = order.iter().map(|&j| a[j * n + j]).collect(); + let mut rows = vec![0f64; n * n]; + for (r, &j) in order.iter().enumerate() { + for k in 0..n { + rows[r * n + k] = v[k * n + j]; + } + } + (values, rows) +} + +struct BitWriter { + out: Vec, + acc: u64, + filled: u32, +} + +impl BitWriter { + fn new(bytes: usize) -> Self { + Self { + out: Vec::with_capacity(bytes), + acc: 0, + filled: 0, + } + } + fn write(&mut self, value: u32, bits: u8) { + self.acc |= u64::from(value) << self.filled; + self.filled += u32::from(bits); + while self.filled >= 8 { + self.out.push((self.acc & 0xff) as u8); + self.acc >>= 8; + self.filled -= 8; + } + } + fn finish(mut self, bytes: usize) -> Vec { + if self.filled > 0 { + self.out.push((self.acc & 0xff) as u8); + } + self.out.resize(bytes, 0); + self.out + } +} + +struct BitReader<'a> { + data: &'a [u8], + pos: usize, + acc: u64, + filled: u32, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { + data, + pos: 0, + acc: 0, + filled: 0, + } + } + fn read(&mut self, bits: u8) -> u32 { + while self.filled < u32::from(bits) { + let byte = self.data.get(self.pos).copied().unwrap_or(0); + self.acc |= u64::from(byte) << self.filled; + self.pos += 1; + self.filled += 8; + } + let v = (self.acc & ((1u64 << bits) - 1)) as u32; + self.acc >>= bits; + self.filled -= u32::from(bits); + v + } +} + +impl Codec for MixedCodec { + fn name(&self) -> &str { + &self.name + } + + fn dimension(&self) -> usize { + self.dimension + } + + fn encoded_size(&self) -> usize { + self.bytes + } + + fn supported_metrics(&self) -> &'static [Metric] { + &[Metric::Dot, Metric::Cosine, Metric::L2] + } + + fn needs_fit(&self) -> bool { + self.fitted.is_none() + } + + fn fit(&mut self, sample: &[Vector]) -> Result<(), CodecError> { + if sample.is_empty() { + return Err(CodecError::EmptySample { + codec: self.name.clone(), + }); + } + let d = self.dimension; + let step = (sample.len() as f64 / FIT_SAMPLE as f64).max(1.0); + let picked: Vec<&Vector> = (0..sample.len().min(FIT_SAMPLE)) + .map(|i| &sample[((i as f64 * step) as usize).min(sample.len() - 1)]) + .collect(); + for v in &picked { + check_dim(d, v.len())?; + } + let n = picked.len() as f64; + let mut mean = vec![0f64; d]; + for v in &picked { + for (m, &x) in mean.iter_mut().zip(v.iter()) { + *m += f64::from(x) / n; + } + } + let rotation = if self.pca { + let mut cov = vec![0f64; d * d]; + let mut c = vec![0f64; d]; + for v in &picked { + for (ci, (&x, &m)) in c.iter_mut().zip(v.iter().zip(&mean)) { + *ci = f64::from(x) - m; + } + for i in 0..d { + let ci = c[i]; + for j in i..d { + cov[i * d + j] += ci * c[j]; + } + } + } + for i in 0..d { + for j in i..d { + cov[i * d + j] /= n; + cov[j * d + i] = cov[i * d + j]; + } + } + let (_, rows) = symmetric_eigen(cov, d); + Some(rows.into_iter().map(|x| x as f32).collect::>()) + } else { + None + }; + let probe = Fitted { + rotation, + mean: vec![0.0; d], + step: vec![0.0; d], + bits: vec![0; d], + }; + // Statistics in the (possibly rotated) space. + let rotated: Vec = picked.iter().map(|v| Self::rotate(&probe, v)).collect(); + let mut rmean = vec![0f64; d]; + for y in &rotated { + for (m, &x) in rmean.iter_mut().zip(y) { + *m += f64::from(x) / n; + } + } + let mut var = vec![0f64; d]; + for y in &rotated { + for ((s, &x), &m) in var.iter_mut().zip(y).zip(&rmean) { + *s += (f64::from(x) - m).powi(2) / n; + } + } + // Plain allocation minimises reconstruction error, Σ Var(e_i). But a + // dimension's error reaches the score through the query: E[(q·e)²] = + // Σ E[q_i²]·Var(e_i). With `-w`, queries are assumed to look like the + // corpus and each dimension's variance is weighted by its mean + // energy, var + mean². + let importance: Vec = if self.weighted { + var.iter().zip(&rmean).map(|(&v, &m)| v * (v + m * m)).collect() + } else { + var.clone() + }; + let bits = allocate(&importance, self.bytes * 8); + let step = bits + .iter() + .zip(&var) + .map(|(&b, &v)| (GAUSSIAN_STEP[usize::from(b)] * v.sqrt()) as f32) + .collect(); + self.fitted = Some(Fitted { + rotation: probe.rotation, + mean: rmean.iter().map(|&m| m as f32).collect(), + step, + bits, + }); + Ok(()) + } + + fn params(&self) -> Vec { + let Some(f) = &self.fitted else { + return Vec::new(); + }; + let mut out = vec![u8::from(f.rotation.is_some())]; + out.extend_from_slice(&f.bits); + for x in f.mean.iter().chain(&f.step).chain(f.rotation.iter().flatten()) { + out.extend_from_slice(&x.to_le_bytes()); + } + out + } + + fn restore(&mut self, params: &[u8]) -> Result<(), CodecError> { + let d = self.dimension; + let bad = |reason: String| CodecError::BadParams { + codec: self.name.clone(), + reason, + }; + let floats = 2 * d + if self.pca { d * d } else { 0 }; + let expected = 1 + d + 4 * floats; + if params.len() != expected || params[0] != u8::from(self.pca) { + return Err(bad(format!( + "expected {expected} bytes of parameters, got {}", + params.len() + ))); + } + let bits = params[1..=d].to_vec(); + let used: usize = bits.iter().map(|&b| usize::from(b)).sum(); + if bits.iter().any(|&b| b > MAX_BITS) || used > self.bytes * 8 { + return Err(bad(format!("bit allocation uses {used} bits of {}", self.bytes * 8))); + } + let f: Vec = params[1 + d..] + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect(); + self.fitted = Some(Fitted { + mean: f[..d].to_vec(), + step: f[d..2 * d].to_vec(), + rotation: self.pca.then(|| f[2 * d..].to_vec()), + bits, + }); + Ok(()) + } + + fn encode(&self, vector: &[f32]) -> Result { + check_dim(self.dimension, vector.len())?; + let f = self.state()?; + let y = Self::rotate(f, vector); + let mut w = BitWriter::new(self.bytes); + for (i, &b) in f.bits.iter().enumerate() { + if b == 0 { + continue; + } + let levels = (1u32 << b) - 1; + let half = f64::from(levels) / 2.0; + let step = f64::from(f.step[i]); + let raw = if step > 0.0 { + ((f64::from(y[i]) - f64::from(f.mean[i])) / step + half).round() + } else { + half.round() + }; + w.write(raw.clamp(0.0, f64::from(levels)) as u32, b); + } + Ok(Encoded(w.finish(self.bytes))) + } + + fn decode(&self, encoded: &Encoded) -> Result { + check_payload(self.bytes, encoded.0.len())?; + let f = self.state()?; + let mut y = vec![0f32; self.dimension]; + Self::for_each_value(f, encoded, |i, v| y[i] = v as f32); + Ok(match &f.rotation { + None => y, + Some(r) => { + // x = Rᵀ y + let d = self.dimension; + let mut x = vec![0f64; d]; + for (j, &yj) in y.iter().enumerate() { + for (xk, &rjk) in x.iter_mut().zip(&r[j * d..(j + 1) * d]) { + *xk += f64::from(rjk) * f64::from(yj); + } + } + x.into_iter().map(|v| v as f32).collect() + } + }) + } + + fn score(&self, query: &[f32], encoded: &Encoded, metric: Metric) -> Result { + let prepared = self.prepare_query(query)?; + self.score_prepared(&prepared, encoded, metric) + } + + fn prepare_query(&self, query: &[f32]) -> Result { + check_dim(self.dimension, query.len())?; + Ok(Self::rotate(self.state()?, query)) + } + + fn score_prepared(&self, prepared: &[f32], encoded: &Encoded, metric: Metric) -> Result { + check_dim(self.dimension, prepared.len())?; + check_payload(self.bytes, encoded.0.len())?; + let f = self.state()?; + let (mut dot, mut stored_sq, mut query_sq) = (0f64, 0f64, 0f64); + Self::for_each_value(f, encoded, |i, v| { + let q = f64::from(prepared[i]); + dot += q * v; + stored_sq += v * v; + query_sq += q * q; + }); + let v = match metric { + Metric::Dot => dot, + Metric::Cosine => { + let denom = (query_sq * stored_sq).sqrt(); + if denom == 0.0 { 0.0 } else { dot / denom } + } + Metric::L2 => (query_sq - 2.0 * dot + stored_sq).max(0.0), + }; + Ok(v as f32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::distance; + + struct XorShift(u64); + impl XorShift { + fn next_f32(&mut self) -> f32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + ((self.0 >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } + } + + /// Correlated data: a few strong directions mixed into every coordinate, + /// so raw dimensions look alike but principal components do not. + fn correlated(n: usize, dim: usize) -> Vec> { + let mut rng = XorShift(99); + let dirs: Vec> = (0..4).map(|_| (0..dim).map(|_| rng.next_f32()).collect()).collect(); + (0..n) + .map(|_| { + let w: Vec = (0..4).map(|k| rng.next_f32() * (4 - k) as f32).collect(); + let mut v: Vec = (0..dim) + .map(|i| (0..4).map(|k| w[k] * dirs[k][i]).sum::() + 0.05 * rng.next_f32()) + .collect(); + distance::normalize(&mut v); + v + }) + .collect() + } + + fn fitted(dim: usize, bytes: usize, pca: bool, data: &[Vec]) -> MixedCodec { + let mut c = MixedCodec::new(dim, bytes, pca, false).unwrap(); + assert!(c.needs_fit()); + c.fit(data).unwrap(); + c + } + + #[test] + fn allocation_spends_the_budget_where_the_variance_is() { + let bits = allocate(&[16.0, 4.0, 1.0, 0.0], 6); + assert_eq!(bits.iter().map(|&b| usize::from(b)).sum::(), 6); + assert!(bits[0] >= bits[1] && bits[1] >= bits[2] && bits[3] == 0, "{bits:?}"); + assert_eq!(allocate(&[1.0, 1.0], 100), vec![8, 8]); + } + + #[test] + fn weighting_moves_bits_towards_high_energy_dimensions() { + let data = correlated(300, 16); + let mut plain = MixedCodec::new(16, 4, true, false).unwrap(); + let mut weighted = MixedCodec::new(16, 4, true, true).unwrap(); + plain.fit(&data).unwrap(); + weighted.fit(&data).unwrap(); + assert_eq!(weighted.name(), "mixed4-pca-w"); + let (p, w) = (plain.bits().unwrap(), weighted.bits().unwrap()); + assert_eq!(p.iter().map(|&b| u32::from(b)).sum::(), 32); + assert_eq!(w.iter().map(|&b| u32::from(b)).sum::(), 32); + assert!(w[0] >= p[0], "weighted {w:?} plain {p:?}"); + } + + #[test] + fn jacobi_recovers_a_known_spectrum() { + // [[2,1],[1,2]] has eigenvalues 3 and 1. + let (vals, rows) = symmetric_eigen(vec![2.0, 1.0, 1.0, 2.0], 2); + assert!((vals[0] - 3.0).abs() < 1e-9 && (vals[1] - 1.0).abs() < 1e-9, "{vals:?}"); + let r = std::f64::consts::FRAC_1_SQRT_2; + assert!((rows[0].abs() - r).abs() < 1e-9 && (rows[1].abs() - r).abs() < 1e-9); + } + + #[test] + fn score_is_consistent_with_own_decode() { + let data = correlated(300, 32); + for pca in [false, true] { + let c = fitted(32, 12, pca, &data); + for (q, v) in data.iter().zip(data.iter().skip(1)).take(20) { + let e = c.encode(v).unwrap(); + assert_eq!(e.0.len(), 12); + let d = c.decode(&e).unwrap(); + for m in [Metric::Dot, Metric::Cosine, Metric::L2] { + let a = c.score(q, &e, m).unwrap(); + let b = distance::score(q, &d, m); + assert!((a - b).abs() < 1e-4, "{m:?} pca={pca}: {a} vs {b}"); + } + let p = c.prepare_query(q).unwrap(); + assert_eq!( + c.score_prepared(&p, &e, Metric::Cosine).unwrap(), + c.score(q, &e, Metric::Cosine).unwrap() + ); + } + } + } + + #[test] + fn pca_beats_raw_allocation_on_correlated_data_and_more_bytes_help() { + let data = correlated(400, 32); + let err = |c: &MixedCodec| -> f64 { + data.iter() + .zip(data.iter().skip(7)) + .take(100) + .map(|(q, v)| { + let e = c.encode(v).unwrap(); + f64::from((c.score(q, &e, Metric::Cosine).unwrap() - distance::cosine(q, v)).abs()) + }) + .sum::() + / 100.0 + }; + // Budgets small enough that the 8-bit cap per dimension does not bind. + let raw = err(&fitted(32, 4, false, &data)); + let pca = err(&fitted(32, 4, true, &data)); + let pca_small = err(&fitted(32, 2, true, &data)); + assert!(pca < raw, "pca {pca} raw {raw}"); + assert!(pca < pca_small, "4 bytes {pca} vs 2 bytes {pca_small}"); + // PCA puts the bits on the leading components. + let c = fitted(32, 8, true, &data); + let bits = c.bits().unwrap(); + assert!(bits[0] >= bits[31], "{bits:?}"); + } + + #[test] + fn params_roundtrip_and_are_validated() { + let data = correlated(200, 16); + for pca in [false, true] { + let c = fitted(16, 6, pca, &data); + let mut r = MixedCodec::new(16, 6, pca, false).unwrap(); + r.restore(&c.params()).unwrap(); + let v = &data[3]; + assert_eq!(r.encode(v).unwrap(), c.encode(v).unwrap()); + assert!( + MixedCodec::new(16, 6, !pca, false) + .unwrap() + .restore(&c.params()) + .is_err() + ); + } + assert!(MixedCodec::new(16, 0, false, false).is_err()); + assert!(MixedCodec::new(16, 17, false, false).is_err()); + assert!(MixedCodec::new(16, 6, false, false).unwrap().encode(&data[0]).is_err()); + } +} diff --git a/crates/hvec-core/src/codecs/mod.rs b/crates/hvec-core/src/codecs/mod.rs index a501bd0..e8eec2f 100644 --- a/crates/hvec-core/src/codecs/mod.rs +++ b/crates/hvec-core/src/codecs/mod.rs @@ -5,9 +5,11 @@ //! approximate the reference distance on the original f32 vector within the //! codec's documented error model. +mod mixed; mod pq; mod truncate; +pub use mixed::MixedCodec; pub use pq::PqCodec; pub use truncate::TruncateCodec; diff --git a/crates/hvec-core/src/codecs/truncate.rs b/crates/hvec-core/src/codecs/truncate.rs index d220e30..6c5398c 100644 --- a/crates/hvec-core/src/codecs/truncate.rs +++ b/crates/hvec-core/src/codecs/truncate.rs @@ -119,6 +119,16 @@ impl Codec for TruncateCodec { fn score(&self, query: &[f32], encoded: &Encoded, metric: Metric) -> Result { self.inner.score(self.prefix(query)?, encoded, metric) } + + /// The prepared query lives in the inner codec's space: the prefix, + /// then whatever the inner codec does to it. + fn prepare_query(&self, query: &[f32]) -> Result { + self.inner.prepare_query(self.prefix(query)?) + } + + fn score_prepared(&self, prepared: &[f32], encoded: &Encoded, metric: Metric) -> Result { + self.inner.score_prepared(prepared, encoded, metric) + } } #[cfg(test)] diff --git a/crates/hvec-core/src/lib.rs b/crates/hvec-core/src/lib.rs index c76b475..b097b3a 100644 --- a/crates/hvec-core/src/lib.rs +++ b/crates/hvec-core/src/lib.rs @@ -116,6 +116,25 @@ pub trait Codec: Send + Sync + std::fmt::Debug { /// `Vector` unless the codec documents that it is a baseline. fn score(&self, query: &[f32], encoded: &Encoded, metric: Metric) -> Result; + /// Transform a query once per search, before it is scored against many + /// stored vectors with [`Codec::score_prepared`]. + /// + /// Codecs that work in a rotated or otherwise transformed space do that + /// work here, so the per-vector scan stays on the bytes. The default is + /// the identity. + fn prepare_query(&self, query: &[f32]) -> Result { + codecs::check_dim(self.dimension(), query.len())?; + Ok(query.to_vec()) + } + + /// Score a query returned by [`Codec::prepare_query`]. Must equal + /// `score(query, …)` for the original query. The default forwards to + /// [`Codec::score`], which is right whenever `prepare_query` is the + /// identity. + fn score_prepared(&self, prepared: &[f32], encoded: &Encoded, metric: Metric) -> Result { + self.score(prepared, encoded, metric) + } + /// Bytes per vector relative to uncompressed f32. fn compression_ratio(&self) -> f32 { (self.dimension() * 4) as f32 / self.encoded_size() as f32 @@ -168,8 +187,10 @@ pub trait Codec: Send + Sync + std::fmt::Debug { /// Names: `f32`, `int8`, `binary`, `binary-centred`, `pq` (product /// quantization with `m` subspaces; bare `pq` means one byte per 8 /// dimensions), `mrl` (keep the first `k` dimensions, Matryoshka style) -/// and `mrl+` (truncate, then encode with another codec, for -/// example `mrl256+int8` or `mrl128+pq16`). +/// `mrl+` (truncate, then encode with another codec, for +/// example `mrl256+int8` or `mrl128+pq16`), and `mixed[-pca][-w]` +/// (B bytes per vector, bits allocated per dimension by variance, optionally +/// after a PCA rotation, and with `-w` by variance × query energy). /// /// Trained codecs (see [`Codec::needs_fit`]) come back needing a [`Codec::fit`] /// call; use [`codec_with_params`] to restore one that was fitted earlier. @@ -184,6 +205,18 @@ pub fn codec_by_name(name: &str, dimension: usize) -> Result, Cod let m = parse_param(name, rest, Some(dimension / 8))?; return Ok(Box::new(codecs::PqCodec::new(dimension, m)?)); } + if let Some(rest) = name.strip_prefix("mixed") { + let (rest, weighted) = match rest.strip_suffix("-w") { + Some(r) => (r, true), + None => (rest, false), + }; + let (bytes, pca) = match rest.strip_suffix("-pca") { + Some(b) => (b, true), + None => (rest, false), + }; + let bytes = parse_param(name, bytes, Some(dimension / 8))?; + return Ok(Box::new(codecs::MixedCodec::new(dimension, bytes, pca, weighted)?)); + } if let Some(rest) = name.strip_prefix("mrl") { let (keep, inner) = match rest.split_once('+') { Some((k, inner)) => (k, inner), @@ -246,5 +279,6 @@ pub fn codec_names() -> &'static [&'static str] { "pq", "mrl", "mrl+", + "mixed[-pca][-w]", ] } diff --git a/crates/hvec-store/src/vectors.rs b/crates/hvec-store/src/vectors.rs index 4dce406..e36aca0 100644 --- a/crates/hvec-store/src/vectors.rs +++ b/crates/hvec-store/src/vectors.rs @@ -228,10 +228,11 @@ impl Store { )) })?; + let prepared = codec.prepare_query(query)?; let mut hits = Vec::new(); for row in rows { let (chunk_id, source, ordinal, text, blob) = row?; - let score = codec.score(query, &Encoded(blob), metric)?; + let score = codec.score_prepared(&prepared, &Encoded(blob), metric)?; hits.push(Hit { chunk_id, source, diff --git a/crates/hvec/src/cli.rs b/crates/hvec/src/cli.rs index 06277e2..60b001c 100644 --- a/crates/hvec/src/cli.rs +++ b/crates/hvec/src/cli.rs @@ -155,6 +155,8 @@ pub struct ImportBeirArgs { pub enum BenchCommand { /// Re-encode an f32 collection with other codecs and measure retrieval against the exact ranking. Recall(RecallArgs), + /// Two-stage retrieval: scan with a small code, re-score the top candidates with a larger one. + Rescore(RescoreArgs), /// Run a question set across collections and chat profiles, scoring every answer. Run(BenchRunArgs), /// Replay an agent memory script (writes and asks) once per codec, measuring every step. @@ -312,6 +314,50 @@ pub struct RecallArgs { pub json: bool, } +#[derive(Debug, Args)] +pub struct RescoreArgs { + /// An f32 collection to use as ground truth. + #[arg(short, long)] + pub collection: String, + /// First-stage codecs, scanned over every vector. Trained codecs are fitted on the collection first. + #[arg(long, value_delimiter = ',', default_value = "binary,pq48")] + pub codecs: Vec, + /// Second-stage representations, applied to the candidates only. + #[arg(long, value_delimiter = ',', default_value = "int8,f32")] + pub rescore: Vec, + /// Candidate depths to re-score. Values below k are raised to k. + #[arg(long, value_delimiter = ',', default_value = "10,20,30,50,100,200,500")] + pub depths: Vec, + /// Adaptive policy: re-score every candidate within this many noise units of the + /// first stage's k-th score. Noise is calibrated on self-queries, never on the test queries. + #[arg(long, value_delimiter = ',', default_value = "1,2,3")] + pub margins: Vec, + /// Cap on any query's candidate depth. + #[arg(long, default_value_t = 1000)] + pub max_depth: usize, + /// Self-queries used to calibrate the first stage's noise for the adaptive policy. + #[arg(long, default_value_t = 200)] + pub calibration_sample: usize, + /// Neighbours per query. + #[arg(short, long, default_value_t = 10)] + pub k: usize, + /// How many stored vectors to use as self-queries (each excludes itself). + #[arg(long, default_value_t = 100)] + pub sample: usize, + /// A text file with one query per line, instead of self-queries. + #[arg(long, conflicts_with = "questions")] + pub queries: Option, + /// A JSONL question set; questions with `relevant` judgments also yield nDCG@k and document recall@k. + #[arg(long)] + pub questions: Option, + /// Do not write the results to the run log. + #[arg(long)] + pub no_record: bool, + /// Print the results as JSON instead of tables. + #[arg(long)] + pub json: bool, +} + #[derive(Debug, Args)] pub struct IngestArgs { /// Files or directories to ingest. Directories are walked recursively. diff --git a/crates/hvec/src/commands/bench.rs b/crates/hvec/src/commands/bench.rs index 831abe3..99361e5 100644 --- a/crates/hvec/src/commands/bench.rs +++ b/crates/hvec/src/commands/bench.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use anyhow::{Context as _, Result, bail}; use hvec_bench::recall::{self, RecallQuery, RecallReport}; +use hvec_bench::rescore; use hvec_core::codecs::F32Codec; use hvec_core::{Codec, codec_by_name}; use hvec_store::RunRow; @@ -13,13 +14,14 @@ use hvec_bench::questions::{self, Question}; use hvec_bench::report::{self as rpt, Cell}; use hvec_connect::{ChatModel, ConnectError, Embedder}; -use crate::cli::{BenchCommand, BenchRunArgs, RecallArgs, ReportArgs}; +use crate::cli::{BenchCommand, BenchRunArgs, RecallArgs, ReportArgs, RescoreArgs}; use crate::commands::rag::Pipeline; use crate::context::Ctx; pub async fn run(config: &Option, cmd: BenchCommand) -> Result<()> { match cmd { BenchCommand::Recall(args) => recall_cmd(config, args).await, + BenchCommand::Rescore(args) => rescore_cmd(config, args).await, BenchCommand::Run(args) => run_cmd(config, args).await, BenchCommand::Memory(args) => crate::commands::memory::run(config, args).await, } @@ -309,62 +311,16 @@ async fn recall_cmd(config: &Option, args: RecallArgs) -> Result<()> { vectors.push(baseline.decode(&enc)?); } - let (queries, query_source) = match (&args.queries, &args.questions) { - (_, Some(path)) => { - let qs = questions::load(path)?; - let texts: Vec = qs.iter().map(|q| q.retrieval_text().to_owned()).collect(); - let embedder = hvec_connect::embedder(&ctx.config, &collection.embed_profile).await?; - let vectors = embedder.embed_queries(&texts).await?; - let labelled = qs.iter().filter(|q| !q.relevant.is_empty()).count(); - let queries: Vec = vectors - .into_iter() - .zip(qs) - .map(|(vector, q)| RecallQuery { - vector, - exclude: None, - relevant: q.relevant, - }) - .collect(); - let n = queries.len(); - ( - queries, - format!( - "questions:{} ({n} questions, {labelled} with judgments)", - path.display() - ), - ) - } - (Some(path), None) => { - let text = tokio::fs::read_to_string(path) - .await - .with_context(|| format!("reading {}", path.display()))?; - let lines: Vec = text - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .map(str::to_owned) - .collect(); - if lines.is_empty() { - bail!("{} has no queries", path.display()); - } - let embedder = hvec_connect::embedder(&ctx.config, &collection.embed_profile).await?; - let qs = embedder - .embed_queries(&lines) - .await? - .into_iter() - .map(|vector| RecallQuery { - vector, - exclude: None, - relevant: Vec::new(), - }) - .collect(); - (qs, format!("file:{}", path.display())) - } - (None, None) => ( - recall::self_queries(&ids, &vectors, args.sample), - format!("self-queries:{}", args.sample.min(vectors.len())), - ), - }; + let (queries, query_source) = load_queries( + &ctx, + &collection.embed_profile, + args.queries.as_ref(), + args.questions.as_ref(), + args.sample, + &ids, + &vectors, + ) + .await?; let mut names: Vec = vec!["f32".to_owned()]; names.extend(args.codecs.iter().filter(|c| c.as_str() != "f32").cloned()); @@ -423,12 +379,13 @@ async fn recall_cmd(config: &Option, args: RecallArgs) -> Result<()> { println!(); let labelled = reports.first().is_some_and(|r| r.labelled_queries > 0); println!( - "{:<15} {:>9} {:>7} {:>10} {:>10} {:>8} {:>7} {:>9} {:>8}{}", + "{:<15} {:>9} {:>7} {:>10} {:>10} {:>9} {:>8} {:>7} {:>9} {:>8}{}", "codec", "recall@k", "top1", "mean err", "max err", + "rank noise", "bytes", "ratio", "encode ms", @@ -437,12 +394,13 @@ async fn recall_cmd(config: &Option, args: RecallArgs) -> Result<()> { ); for r in &reports { println!( - "{:<15} {:>9.4} {:>7.3} {:>10.5} {:>10.5} {:>8} {:>6.1}x {:>9} {:>8}{}", + "{:<15} {:>9.4} {:>7.3} {:>10.5} {:>10.5} {:>9.5} {:>8} {:>6.1}x {:>9} {:>8}{}", r.codec, r.recall_at_k, r.top1_agreement, r.mean_abs_score_error, r.max_abs_score_error, + r.rank_noise, r.bytes_per_vector, r.compression_ratio, r.encode_ms, @@ -453,6 +411,12 @@ async fn recall_cmd(config: &Option, args: RecallArgs) -> Result<()> { } ); } + if let Some(r) = reports.first() { + println!( + "\nneighbour spacing (exact f32, median over queries): rank k to k+1 {:.5}, rank k to 2k {:.5}; rank noise is the codec's score error std over the exact top-2k, per-query bias removed", + r.boundary_gap, r.neighbour_band + ); + } if labelled { println!( "\nndcg@k and doc rec are against the question set's relevance judgments ({} labelled queries); the f32 row is the ceiling the codec is measured against", @@ -464,3 +428,251 @@ async fn recall_cmd(config: &Option, args: RecallArgs) -> Result<()> { } Ok(()) } + +/// The queries for `bench recall` and `bench rescore`: a question set, a +/// file of query lines, or self-queries sampled from the collection. +async fn load_queries( + ctx: &Ctx, + embed_profile: &str, + queries: Option<&PathBuf>, + questions: Option<&PathBuf>, + sample: usize, + ids: &[i64], + vectors: &[Vec], +) -> Result<(Vec, String)> { + Ok(match (queries, questions) { + (_, Some(path)) => { + let qs = questions::load(path)?; + let texts: Vec = qs.iter().map(|q| q.retrieval_text().to_owned()).collect(); + let embedder = hvec_connect::embedder(&ctx.config, embed_profile).await?; + let vectors = embedder.embed_queries(&texts).await?; + let labelled = qs.iter().filter(|q| !q.relevant.is_empty()).count(); + let queries: Vec = vectors + .into_iter() + .zip(qs) + .map(|(vector, q)| RecallQuery { + vector, + exclude: None, + relevant: q.relevant, + }) + .collect(); + let n = queries.len(); + ( + queries, + format!( + "questions:{} ({n} questions, {labelled} with judgments)", + path.display() + ), + ) + } + (Some(path), None) => { + let text = tokio::fs::read_to_string(path) + .await + .with_context(|| format!("reading {}", path.display()))?; + let lines: Vec = text + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .map(str::to_owned) + .collect(); + if lines.is_empty() { + bail!("{} has no queries", path.display()); + } + let embedder = hvec_connect::embedder(&ctx.config, embed_profile).await?; + let qs = embedder + .embed_queries(&lines) + .await? + .into_iter() + .map(|vector| RecallQuery { + vector, + exclude: None, + relevant: Vec::new(), + }) + .collect(); + (qs, format!("file:{}", path.display())) + } + (None, None) => ( + recall::self_queries(ids, vectors, sample), + format!("self-queries:{}", sample.min(vectors.len())), + ), + }) +} + +async fn rescore_cmd(config: &Option, args: RescoreArgs) -> Result<()> { + let ctx = Ctx::load(config.as_deref())?; + let store = ctx.open_store()?; + let collection = store.require_collection(&args.collection)?; + if collection.codec != "f32" { + bail!( + "`bench rescore` needs an f32 collection as ground truth; `{}` is stored as {}", + collection.name, + collection.codec + ); + } + if collection.chunk_count == 0 { + bail!("collection `{}` is empty", collection.name); + } + let baseline = F32Codec::new(collection.dimension); + let mut ids: Vec = Vec::new(); + let mut sources: Vec = Vec::new(); + let mut vectors: Vec> = Vec::new(); + for (id, source, enc) in store.vectors_with_sources(&collection.name)? { + ids.push(id); + sources.push(source); + vectors.push(baseline.decode(&enc)?); + } + let (queries, query_source) = load_queries( + &ctx, + &collection.embed_profile, + args.queries.as_ref(), + args.questions.as_ref(), + args.sample, + &ids, + &vectors, + ) + .await?; + + let fitted = |name: &str| -> Result> { + let mut codec = codec_by_name(name, collection.dimension)?; + if codec.needs_fit() { + codec.fit(&vectors)?; + } + Ok(codec) + }; + let seconds: Vec> = args.rescore.iter().map(|n| fitted(n)).collect::>()?; + let second_refs: Vec<&dyn Codec> = seconds.iter().map(AsRef::as_ref).collect(); + + // Calibration for the adaptive policy: self-queries only, so the test + // queries' exact scores never inform their own depth. + let calibration = recall::self_queries(&ids, &vectors, args.calibration_sample); + + let mut profiles = Vec::new(); + let mut cells = Vec::new(); + for name in &args.codecs { + let first = fitted(name)?; + let noise = recall::evaluate( + &ids, + &vectors, + &calibration, + first.as_ref(), + collection.metric, + args.k, + None, + )? + .rank_noise; + let plan = rescore::Plan { + seconds: &second_refs, + depths: &args.depths, + margins: &args.margins, + noise, + max_depth: args.max_depth, + }; + let (profile, reports) = rescore::evaluate( + &ids, + &vectors, + &queries, + first.as_ref(), + collection.metric, + args.k, + &plan, + Some(&sources), + )?; + if !args.no_record { + for r in &reports { + let mut row = RunRow::new("rescore"); + row.collection = Some(collection.name.clone()); + row.embed_profile = Some(collection.embed_profile.clone()); + row.embed_model = Some(collection.embed_model.clone()); + row.codec = Some(match r.policy { + rescore::Policy::None => r.first.clone(), + rescore::Policy::Fixed { depth } => format!("{}>{}@{depth}", r.first, r.second), + rescore::Policy::Margin { sigmas } => format!("{}>{}@{sigmas}σ", r.first, r.second), + }); + row.metric = Some(collection.metric.name().to_owned()); + row.metadata = serde_json::json!({ + "dimension": collection.dimension, + "queries": query_source, + "k": args.k, + "first_stage": serde_json::to_value(&profile)?, + "calibrated_noise": noise, + }); + row.metrics = serde_json::to_value(r)?; + store.insert_run(&row)?; + } + } + profiles.push(profile); + cells.extend(reports); + } + + if args.json { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ "profiles": profiles, "cells": cells }))? + ); + return Ok(()); + } + let f32_scan = vectors.len() * baseline.encoded_size(); + println!( + "collection `{}` · {} vectors · {} dims · {} · {} · k={}", + collection.name, + vectors.len(), + collection.dimension, + collection.embed_model, + collection.metric.name(), + args.k + ); + println!("queries: {query_source}"); + for p in &profiles { + println!(); + let depths: Vec = p + .depth_for_recall + .iter() + .map(|(t, d)| format!("{:.0}%: {d}", t * 100.0)) + .collect(); + println!( + "first stage {} ({} B): recall@k {:.4}, noise/band {:.3}; depth an exact re-score needs for recall {}; median query complete at {}", + p.first, + p.first_bytes, + p.recall_at_k, + if p.neighbour_band > 0.0 { + p.rank_noise / p.neighbour_band + } else { + f64::NAN + }, + depths.join(", "), + p.median_full_depth + ); + println!( + " {:<8} {:<8} {:>7} {:>6} {:>9} {:>7} {:>7} {:>8} {:>8} {:>10} {:>8}", + "rescore", "policy", "depth", "p90", "recall@k", "p10", "top1", "ndcg@k", "doc rec", "KB/query", "vs f32" + ); + for r in cells.iter().filter(|r| r.first == p.first) { + let policy = match r.policy { + rescore::Policy::None => "-".to_owned(), + rescore::Policy::Fixed { .. } => "fixed".to_owned(), + rescore::Policy::Margin { sigmas } => format!("{sigmas}σ"), + }; + println!( + " {:<8} {:<8} {:>7.1} {:>6} {:>9.4} {:>7.3} {:>7.3} {:>8} {:>8} {:>10.1} {:>7.3}x", + r.second, + policy, + r.mean_depth, + r.p90_depth, + r.recall_at_k, + r.p10_recall_at_k, + r.top1_agreement, + r.ndcg_at_k.map_or("-".into(), |v| format!("{v:.4}")), + r.doc_recall_at_k.map_or("-".into(), |v| format!("{v:.4}")), + r.bytes_read_per_query / 1024.0, + r.bytes_read_per_query / f32_scan as f64, + ); + } + } + println!( + "\ndepth is mean candidates re-scored per query, p90 its 90th percentile, p10 the 10th-percentile query's recall; Nσ re-scores every candidate within N calibrated noise units of the k-th first-stage score; KB/query is the first-stage scan plus the re-scored candidates; vs f32 compares it with an exact f32 scan of every vector" + ); + if !args.no_record { + eprintln!("\n{} runs recorded (kind: rescore)", cells.len()); + } + Ok(()) +} diff --git a/docs/_data/references.yml b/docs/_data/references.yml index 2a410c6..377c111 100644 --- a/docs/_data/references.yml +++ b/docs/_data/references.yml @@ -20,6 +20,38 @@ year: 2014 url: https://doi.org/10.1109/TPAMI.2013.240 note: Learns a rotation before splitting into subspaces, cutting PQ error substantially. The usual next step after plain PQ. +- key: max1960quantizing + topic: Quantization + authors: Joel Max + title: Quantizing for Minimum Distortion + venue: IRE Transactions on Information Theory 6(1) + year: 1960 + url: https://doi.org/10.1109/TIT.1960.1057548 + note: Optimal scalar quantizers for a Gaussian source. The step sizes `mixed` uses per dimension. +- key: huang1963block + topic: Quantization + authors: J. J. Y. Huang, P. M. Schultheiss + title: Block Quantization of Correlated Gaussian Random Variables + venue: IEEE Transactions on Communications Systems 11(3) + year: 1963 + url: https://doi.org/10.1109/TCOM.1963.1088759 + note: Transform coding. Decorrelate (here, PCA), then allocate bits across components by variance. The design of `mixed-pca`. +- key: li2024quantmoe + topic: Quantization + authors: Pingzhi Li, Xiaolong Jin, Yu Cheng, Tianlong Chen + title: "QuantMoE-Bench: Examining Post-Training Quantization for Mixture-of-Experts" + venue: arXiv:2406.08155 + year: 2024 + url: https://arxiv.org/abs/2406.08155 + note: "Structure-aware mixed-precision post-training quantization of MoE models: different blocks, experts and linear layers need different bit widths, chosen by importance scorers such as a linear-weight outlier score. Code: github.com/UNITES-Lab/moe-quantization. The model for hvec's adaptive bit allocation." +- key: li2025submoe + topic: Quantization + authors: Lujun Li, Zhu Qiyuan, Jiacheng Wang, Wei Li, Hao Gu, Sirui Han, Yike Guo + title: "Sub-MoE: Efficient Mixture-of-Expert LLMs Compression via Subspace Expert Merging" + venue: arXiv:2506.23266 + year: 2025 + url: https://arxiv.org/abs/2506.23266 + note: "Clusters experts by cosine similarity, takes a joint SVD of each group's weights and merges the V matrices weighted by expert frequency. Code: github.com/siruihan2024/Sub-MoE. Suggests per-cluster subspaces for embedding codecs." - key: babenko2014aq topic: Quantization authors: Artem Babenko, Victor Lempitsky @@ -210,6 +242,46 @@ year: 2021 url: https://arxiv.org/abs/2104.08663 note: Retrieval datasets with relevance judgments. A likely source for hvec's question sets. +- key: boteva2016nfcorpus + topic: Embeddings and RAG + authors: Vera Boteva, Demian Gholipour, Artem Sokolov, Stefan Riezler + title: A Full-Text Learning to Rank Dataset for Medical Information Retrieval + venue: European Conference on Information Retrieval (ECIR) + year: 2016 + url: https://doi.org/10.1007/978-3-319-30671-1_58 + note: NFCorpus, the medical corpus in BEIR. Used in the neighbour-spacing experiment. +- key: wachsmuth2018arguana + topic: Embeddings and RAG + authors: Henning Wachsmuth, Shahbaz Syed, Benno Stein + title: Retrieval of the Best Counterargument without Prior Topic Knowledge + venue: Annual Meeting of the Association for Computational Linguistics (ACL) + year: 2018 + url: https://aclanthology.org/P18-1023/ + note: ArguAna, argument-to-counterargument retrieval in BEIR. Used in the neighbour-spacing experiment. +- key: maia2018fiqa + topic: Embeddings and RAG + authors: Macedo Maia, Siegfried Handschuh, André Freitas, Brian Davis, Ross McDermott, Manel Zarrouk, Alexandra Balahur + title: "WWW'18 Open Challenge: Financial Opinion Mining and Question Answering" + venue: Companion Proceedings of The Web Conference + year: 2018 + url: https://doi.org/10.1145/3184558.3192301 + note: FiQA, financial question answering in BEIR. Used in the neighbour-spacing experiment. +- key: huang2021cosqa + topic: Embeddings and RAG + authors: Junjie Huang, Duyu Tang, Linjun Shou, Ming Gong, Ke Xu, Daxin Jiang, Ming Zhou, Nan Duan + title: "CoSQA: 20,000+ Web Queries for Code Search and Question Answering" + venue: Annual Meeting of the Association for Computational Linguistics (ACL) + year: 2021 + url: https://arxiv.org/abs/2105.13239 + note: Web queries paired with Python functions. Used, via CoIR, in the neighbour-spacing experiment. +- key: li2024coir + topic: Embeddings and RAG + authors: Xiangyang Li, Kuicai Dong, Yi Quan Lee, Wei Xia, Hao Zhang, Xinyi Dai, Yasheng Wang, Ruiming Tang + title: "CoIR: A Comprehensive Benchmark for Code Information Retrieval Models" + venue: arXiv:2407.02883 + year: 2024 + url: https://arxiv.org/abs/2407.02883 + note: Code retrieval datasets in BEIR's layout. The source of hvec's CoSQA copy. - key: morris2023vec2text topic: Embeddings and RAG diff --git a/docs/_experiments/neighbour-spacing.md b/docs/_experiments/neighbour-spacing.md new file mode 100644 index 0000000..95d4fef --- /dev/null +++ b/docs/_experiments/neighbour-spacing.md @@ -0,0 +1,70 @@ +--- +title: "Five corpora, two models: neighbour spacing predicts codec loss" +date: 2026-11-04 18:00:00 +0000 +result: "Codec noise divided by the corpus's neighbour spacing ranks recall@10 loss with ρ = 0.98 over 90 points, against 0.64 for noise alone. Content type matters only through that spacing, and changing the embedding model moves it as much as changing the corpus does." +references: [thakur2021beir, boteva2016nfcorpus, wachsmuth2018arguana, maia2018fiqa, huang2021cosqa, li2024coir, xiao2023bge, reimers2019sbert] +--- + +*Full write-up, raw results and a one-command reproduction are in the repository:* +[experiments/2026-11-neighbour-spacing](https://github.com/SwineCoder101/hvec/tree/main/experiments/2026-11-neighbour-spacing). +Follow-up to [SciFact: mean-centred binary]({{ "/experiments/scifact-centred-binary/" | relative_url }}), +which found that binary's loss is a resolution problem: top-10 neighbours sit closer together in +cosine than a sign code can tell apart. + +**Hypothesis.** A codec's loss of agreement with the exact f32 top-k is governed by the ratio of +its score noise to the spacing of the corpus's neighbours. The ratio will predict the loss across +corpora and models much better than the noise alone. + +**Setup.** Five BEIR-format sets [1] of different kinds: SciFact (scientific abstracts), +NFCorpus [2] (medical), ArguAna [3] (arguments), FiQA [4] (financial Q&A) and CoSQA [5] from +CoIR [6] (Python functions). They range from 6,260 to 71,274 chunks of 200 words. Two embedding +models, bge-small-en-v1.5 [7] and all-MiniLM-L6-v2 [8], both 384 dimensions. Nine codecs: +`int8`, `binary`, `binary-centred`, `pq96/48/24`, `mrl256/192/96`. k = 10, brute force, no +rescoring. `bench recall` gains three numbers: + +- **`neighbour_band`**: the median exact score gap between rank 10 and rank 20. It needs only f32. +- **`rank_noise`**: the spread of the codec's score error over the exact top 20, with each + query's bias removed. +- **`boundary_gap`**: the gap between rank 10 and rank 11. + +**Result.** Confirmed. + +| corpus | content | band, bge-small | band, MiniLM | binary recall@10, bge / MiniLM | pq48 recall@10, bge / MiniLM | +|---|---|---:|---:|---:|---:| +| FiQA | financial Q&A | **0.0108** | 0.0271 | 0.537 / 0.666 | 0.642 / 0.716 | +| NFCorpus | medical | 0.0121 | 0.0308 | 0.556 / 0.669 | 0.684 / 0.742 | +| SciFact | scientific abstracts | 0.0135 | 0.0307 | 0.611 / 0.705 | 0.711 / 0.764 | +| CoSQA | Python functions | 0.0148 | **0.0248** | 0.660 / 0.719 | 0.773 / 0.775 | +| ArguAna | arguments | **0.0233** | **0.0529** | 0.748 / 0.815 | 0.811 / 0.849 | + +| predictor of recall@10 loss, 90 points | Spearman ρ | held-out MAE (fit on four corpora, predict the fifth) | +|---|---:|---:| +| rank noise ÷ neighbour band | **0.980** | **0.024** | +| rank noise alone | 0.644 | 0.090 | + +The fitted curve is nearly proportional: **the odds of losing a neighbour ≈ 0.48 × noise ÷ +band**. At a ratio of 1, about two thirds of the top 10 survives. Predicting a new corpus from +its f32 band alone, with the codec's noise borrowed from the other four corpora, gives an MAE of +0.033 against 0.062 for averaging the other corpora's recall. It does better on four corpora and +worse on CoSQA. + +**Reading.** The same codec has about the same noise everywhere. Binary on bge-small has 0.0177 +on FiQA and 0.0183 on ArguAna, yet loses 46 points of the top 10 on FiQA and 25 on ArguAna. The +band explains the difference. Content type matters only through spacing, and code is not an +outlier: on bge-small, CoSQA sits between SciFact and ArguAna. The embedding model moves the +band as much as the whole range of corpora does. MiniLM's band is 1.7 to 2.5× wider than +bge-small's while its codec noise is only 1.3 to 1.5× larger, which is why MiniLM has tolerated +binary better in every experiment so far. Against the labels the ratio is a weaker guide +(ρ = 0.848 with nDCG loss), and binary costs only 0.7 to 5.4 nDCG points. + +**Caveats.** +- **CoSQA is mostly duplicates.** Only 6,267 of its 20,604 functions are distinct, which zeroes + its boundary gap and probably lowers its codec noise. +- **The most aggressive codecs (`pq24`, `mrl96`) miss the curve by up to 16 points.** They + miss in both directions. +- **Two models only.** +- **FiQA is both the largest and the tightest corpus**, so size and spacing are not separated. +- **The curve's form was chosen after seeing the data.** +- **No rescoring.** + +{% include references.html %} diff --git a/docs/_posts/2026-11-04-does-the-kind-of-text-matter.md b/docs/_posts/2026-11-04-does-the-kind-of-text-matter.md new file mode 100644 index 0000000..70cc9ea --- /dev/null +++ b/docs/_posts/2026-11-04-does-the-kind-of-text-matter.md @@ -0,0 +1,91 @@ +--- +title: "Does the kind of text change what a codec costs?" +date: 2026-11-04 18:30:00 +0000 +references: [thakur2021beir, li2024coir, xiao2023bge, reimers2019sbert, jegou2011pq] +--- + +Every hvec experiment so far has used one corpus, SciFact's scientific abstracts. So every +codec number on this site carries an unstated condition: *on this kind of text*. The obvious +question is whether it would come out differently on financial forum posts, medical articles, +arguments or source code. This post is about the experiment that answers it. The answer is yes, +and the reason it is yes is more useful than the answer itself. + +## A codec never sees the text + +A codec compresses an embedding vector, and the text is gone by then. So the kind of text can +only matter through the vectors the embedding model makes of it. The question is which +property of those vectors matters. + +The SciFact experiments had already pointed at one. When a 384-bit sign code (`binary`) drops a +neighbour from the top 10, it is almost always a near-tie. The 10th and 11th results on SciFact +differ in cosine by about a thousandth, and the sign code's score error is a couple of +hundredths. The codec is not getting the similarities badly wrong; it is getting them slightly +wrong, in a place where slightly wrong is enough to reorder the list. + +That suggests a rule. What a codec costs should depend on two numbers: + +- **its noise**: how much it jitters scores among the top candidates (the *rank noise*, with any + constant offset removed, because a constant offset reorders nothing); +- **the corpus's spacing**: how far apart the candidates it has to tell apart sit (the + *neighbour band*, the gap from the 10th to the 20th exact score). + +The noise is a property of the codec and the embedding model. The spacing is a property of the +corpus and the model, and it is the only way the kind of text enters. + +## The test + +`hvec bench recall` now reports both numbers. The experiment runs nine codecs (int8, two +sign-bit codecs, three product quantizers [5], three truncations) over five retrieval sets with +relevance judgments [1], each embedded with two small models [3, 4]: + +| corpus | what it is | band, bge-small | binary keeps of the f32 top 10 | +|---|---|---:|---:| +| FiQA | financial forum Q&A | 0.0108 | 0.537 | +| NFCorpus | medical articles | 0.0121 | 0.556 | +| SciFact | scientific abstracts | 0.0135 | 0.611 | +| CoSQA | Python functions [2] | 0.0148 | 0.660 | +| ArguAna | arguments and counterarguments | 0.0233 | 0.748 | + +The binary codec's noise is nearly the same on FiQA and ArguAna (0.0177 and 0.0183). It loses 46 +points of the top 10 on one and 25 on the other. The band explains the difference. + +Across all 90 combinations of corpus, model and codec, the ratio of noise to band ranks the +loss almost perfectly (Spearman ρ = 0.98). Noise alone, the number a codec's spec sheet would +give you, manages 0.64. The relationship is close to proportional: **the odds of losing a +neighbour are about half the noise-to-band ratio**. Fit the curve on four corpora and it +predicts the fifth's recall to within 0.024 on average. + +## So does the kind of text matter? + +It does, by up to 21 points of binary recall between the tightest corpus and the loosest. Two +things in the result are worth more than that headline, though. + +**Code is not special.** Source code was the content most unlike prose in the set, and it sits in +the middle of the range. "Text type" turns out to be a proxy, and the thing it stands in for can +be measured directly, in a few seconds, with nothing but the f32 vectors you already have. + +**The model matters as much as the corpus.** On the same corpus, swapping bge-small for MiniLM +widens the band 1.7 to 2.5×, as much as the whole spread across the five corpora. MiniLM's codec +noise grows by only 1.3 to 1.5×, so it tolerates binary better on every corpus. That had puzzled +the earlier experiments, and this is the explanation. + +## What to do with it + +Before choosing a codec for a new collection, measure its neighbour band in f32. You can then +estimate the codec's loss from a noise figure measured elsewhere. Done that way, without running +the codec on the new corpus, the estimate was off by 0.033 on average, half the error of assuming +the codec behaves as it did on other corpora. It worked on four of the five corpora. It failed +on CoSQA, whose corpus turned out to be 70% duplicated code; that lowers the codec noise and +fools the borrowed figure. + +None of this says how much the loss matters to an answer. Against the human relevance labels the +same codecs lose 0.7 to 5.4 nDCG points where they lose 19 to 46 points of agreement with f32, +and whether that reaches a chat model's answer is still the job of the RAG experiment that is +waiting for models to run on. The next retrieval-only question follows directly from this one: +if the ratio says how many true neighbours a codec scrambles, it should also say how deep a +cheap f32 rescoring pass has to go to recover them. + +The write-up, with the caveats, is +[on the experiments page]({{ "/experiments/neighbour-spacing/" | relative_url }}). + +{% include references.html %} diff --git a/experiments/2026-11-hybrid-adaptive-matrix/run.sh b/experiments/2026-11-hybrid-adaptive-matrix/run.sh new file mode 100755 index 0000000..09995e9 --- /dev/null +++ b/experiments/2026-11-hybrid-adaptive-matrix/run.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# The hybrid × adaptive matrix. Rows: what is held in memory and scanned, uniform codecs +# (binary, binary-centred, pq) against adaptive bit allocation (mixed, mixed-pca, +# mixed-pca-w) at 24, 48 and 96 bytes. Columns: the re-scoring policy, none, fixed depth +# (hybrid) or a per-query margin (hybrid + adaptive), with an int8 or f32 second stage. +# Retrieval only, no chat model. Reuses the ten collections from ../2026-11-neighbour-spacing +# (and builds them if missing). Needs: a built `hvec` (cargo build --release). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$HERE/../.." +HVEC="${HVEC:-$ROOT/target/release/hvec}" +SPACING="$HERE/../2026-11-neighbour-spacing" +SPACING_WORK="${SPACING_WORK:-$SPACING/work}" +K="${K:-10}" +CORPORA="${CORPORA:-scifact nfcorpus arguana cosqa fiqa}" +MODELS="${MODELS:-bge-small minilm}" +FIRST="${FIRST:-binary,binary-centred,pq48,mixed48,mixed48-pca,mixed48-pca-w,pq24,mixed24-pca,mixed24-pca-w,pq96,mixed96-pca-w}" +SECOND="${SECOND:-int8,f32}" +DEPTHS="${DEPTHS:-20,50,100}" +MARGINS="${MARGINS:-1,2,3}" +MAX_DEPTH="${MAX_DEPTH:-1000}" +mkdir -p "$HERE/results" + +export HVEC_CONFIG="$SPACING_WORK/config.toml" +if [ ! -f "$HVEC_CONFIG" ]; then + WORK="$SPACING_WORK" CORPORA="$CORPORA" MODELS="$MODELS" "$SPACING/run.sh" +fi + +for c in $CORPORA; do + for m in $MODELS; do + col="$c-$m" + "$HVEC" bench rescore --collection "$col" -k "$K" --codecs "$FIRST" --rescore "$SECOND" \ + --depths "$DEPTHS" --margins "$MARGINS" --max-depth "$MAX_DEPTH" \ + --questions "$SPACING_WORK/$c.jsonl" --json > "$HERE/results/$col-k$K.json" + echo "== $col done" + done +done +cp "$HVEC_CONFIG" "$HERE/results/config.toml" +python3 -I "$HERE/summarise.py" "$HERE/results" "$K" "$SPACING/results" | tee "$HERE/results/summary.txt" diff --git a/experiments/2026-11-hybrid-adaptive-matrix/summarise.py b/experiments/2026-11-hybrid-adaptive-matrix/summarise.py new file mode 100644 index 0000000..a39ca2c --- /dev/null +++ b/experiments/2026-11-hybrid-adaptive-matrix/summarise.py @@ -0,0 +1,169 @@ +"""Summarise the hybrid × adaptive matrix from `bench rescore` JSON. + + python3 -I summarise.py + +The spacing experiment's results give each collection's exact f32 nDCG, the ceiling that nDCG +loss is measured from. Prints: + 1. the matrix: every (scan codec, second stage, policy) averaged over collections; + 2. adaptive storage alone: mixed against uniform codecs at equal bytes, no re-score; + 3. adaptive depth against fixed depth at the same mean depth (fixed interpolated in log depth); + 4. benefits and drawbacks of each family of combination against the plain 48-byte binary scan. +""" + +import json +import math +import sys +from collections import defaultdict +from pathlib import Path + + +def policy_name(p): + if p["kind"] == "none": + return "none" + if p["kind"] == "fixed": + return f"fixed {p['depth']}" + return f"margin {p['sigmas']:g}σ" + + +def mean(xs): + xs = list(xs) + return sum(xs) / len(xs) if xs else float("nan") + + +def load(results: Path, k: int, spacing: Path): + ceiling = {} + for path in spacing.glob(f"*-k{k}.json"): + ceiling[path.name[: -len(f"-k{k}.json")]] = json.loads(path.read_text())[0]["ndcg_at_k"] + cells = [] + for path in sorted(results.glob(f"*-k{k}.json")): + col = path.name[: -len(f"-k{k}.json")] + data = json.loads(path.read_text()) + for c in data["cells"]: + c["collection"] = col + c["model"] = col.split("-", 1)[1] + c["policy_name"] = policy_name(c["policy"]) + c["ndcg_loss"] = ceiling[col] - c["ndcg_at_k"] if col in ceiling and c["ndcg_at_k"] is not None else None + c["f32_scan"] = c["vectors"] * 4 * 384 + cells.append(c) + return cells + + +def interp_fixed(fixed_pts, depth): + """Linear interpolation of (depth, value) in log depth; None outside the measured range.""" + pts = sorted(fixed_pts) + for (d0, v0), (d1, v1) in zip(pts, pts[1:]): + if d0 <= depth <= d1: + t = (math.log(depth) - math.log(d0)) / (math.log(d1) - math.log(d0)) if d1 > d0 else 0.0 + return v0 + t * (v1 - v0) + return None + + +def main(results: Path, k: int, spacing: Path) -> None: + cells = load(results, k, spacing) + cols = sorted({c["collection"] for c in cells}) + print(f"{len(cols)} collections: {', '.join(cols)}\n") + + groups = defaultdict(list) + for c in cells: + groups[(c["first"], c["second"], c["policy_name"])].append(c) + + def row(key): + g = groups[key] + return dict( + first=key[0], second=key[1], policy=key[2], ram=g[0]["first_bytes"], disk=g[0]["second_bytes"], + depth=mean(c["mean_depth"] for c in g), p90=mean(c["p90_depth"] for c in g), + recall=mean(c["recall_at_k"] for c in g), p10=mean(c["p10_recall_at_k"] for c in g), + top1=mean(c["top1_agreement"] for c in g), + ndcg_loss=mean(c["ndcg_loss"] for c in g if c["ndcg_loss"] is not None), + read=mean(c["bytes_read_per_query"] / c["f32_scan"] for c in g), n=len(g), + ) + + order = sorted(groups, key=lambda x: (x[0], x[1] != "none", x[1], x[2])) + rows = [row(key) for key in order] + print("1. The matrix (means over collections). ram = scanned bytes/vector, disk = second-stage bytes/vector,") + print(" read = bytes touched per query as a fraction of a full f32 scan") + hdr = f"{'scan codec':<15} {'second':<6} {'policy':<12} {'ram':>4} {'disk':>5} {'depth':>6} {'p90':>6} " \ + f"{'recall@10':>9} {'p10':>6} {'top1':>6} {'ndcg loss':>9} {'read':>7}" + print(hdr) + for r in rows: + print(f"{r['first']:<15} {r['second']:<6} {r['policy']:<12} {r['ram']:>4} {r['disk']:>5} {r['depth']:>6.1f} " + f"{r['p90']:>6.0f} {r['recall']:>9.4f} {r['p10']:>6.3f} {r['top1']:>6.3f} {r['ndcg_loss']:>9.4f} " + f"{r['read']:>7.4f}") + + print("\n2. Adaptive storage alone (no re-score), by byte budget; per model, then both") + budgets = defaultdict(list) + for r in rows: + if r["policy"] == "none": + budgets[r["ram"]].append(r) + for b in sorted(budgets): + for r in sorted(budgets[b], key=lambda r: -r["recall"]): + per_model = [] + for m in ("bge-small", "minilm"): + g = [c for c in groups[(r["first"], "none", "none")] if c["model"] == m] + per_model.append(mean(c["recall_at_k"] for c in g)) + print(f" {b:>3} B {r['first']:<15} recall@10 {r['recall']:.4f} (bge {per_model[0]:.4f}, " + f"MiniLM {per_model[1]:.4f}) p10 {r['p10']:.3f} ndcg loss {r['ndcg_loss']:.4f}") + + print("\n3. Adaptive depth against fixed depth at the same mean depth (per collection, then averaged)") + print(f" {'scan codec':<15} {'second':<6} {'margin':<6} {'mean depth':>10} {'Δrecall':>8} {'Δp10':>7} " + f"{'Δndcg':>8} {'p90/mean':>8} {'cells':>5}") + by_first_second = defaultdict(list) + for c in cells: + by_first_second[(c["first"], c["second"], c["collection"])].append(c) + deltas = defaultdict(list) + for (first, second, col), g in by_first_second.items(): + if second == "none": + continue + fixed = [c for c in g if c["policy"]["kind"] == "fixed"] + none = groups[(first, "none", "none")] + base = [c for c in none if c["collection"] == col][0] + # A fixed depth of k is the no-re-score membership with an exact reorder; anchor the curve there. + anchor_rec = base["recall_at_k"] + pts_rec = [(c["mean_depth"], c["recall_at_k"]) for c in fixed] + [(k, anchor_rec)] + pts_p10 = [(c["mean_depth"], c["p10_recall_at_k"]) for c in fixed] + [(k, base["p10_recall_at_k"])] + pts_ndcg = [(c["mean_depth"], c["ndcg_at_k"]) for c in fixed if c["ndcg_at_k"] is not None] + for c in g: + if c["policy"]["kind"] != "margin": + continue + fr, fp = interp_fixed(pts_rec, c["mean_depth"]), interp_fixed(pts_p10, c["mean_depth"]) + fn = interp_fixed(pts_ndcg, c["mean_depth"]) if pts_ndcg else None + if fr is None: + continue + deltas[(first, second, c["policy"]["sigmas"])].append(( + c["mean_depth"], c["recall_at_k"] - fr, c["p10_recall_at_k"] - fp, + (c["ndcg_at_k"] - fn) if fn is not None else None, + c["p90_depth"] / c["mean_depth"])) + for key in sorted(deltas): + d = deltas[key] + print(f" {key[0]:<15} {key[1]:<6} {key[2]:<6g} {mean(x[0] for x in d):>10.1f} " + f"{mean(x[1] for x in d):>+8.4f} {mean(x[2] for x in d):>+7.3f} " + f"{mean(x[3] for x in d if x[3] is not None):>+8.4f} {mean(x[4] for x in d):>8.2f} {len(d):>5}") + allr = [x[1] for d in deltas.values() for x in d] + allp = [x[2] for d in deltas.values() for x in d] + if allr: + wins = sum(1 for x in allr if x > 0) + print(f" overall: Δrecall {mean(allr):+.4f}, Δp10 {mean(allp):+.3f}; adaptive ahead in {wins} of {len(allr)} cells") + + print("\n4. Benefits and drawbacks against the plain 48-byte binary scan (binary, none)") + base = row(("binary", "none", "none")) + fam = [ + ("uniform scan, no re-score", lambda r: r["policy"] == "none" and not r["first"].startswith("mixed")), + ("adaptive bits, no re-score", lambda r: r["policy"] == "none" and r["first"].startswith("mixed")), + ("hybrid fixed depth, int8", lambda r: r["policy"].startswith("fixed") and r["second"] == "int8"), + ("hybrid fixed depth, f32", lambda r: r["policy"].startswith("fixed") and r["second"] == "f32"), + ("hybrid adaptive depth, int8", lambda r: r["policy"].startswith("margin") and r["second"] == "int8"), + ("hybrid adaptive depth, f32", lambda r: r["policy"].startswith("margin") and r["second"] == "f32"), + ] + print(f" baseline: recall {base['recall']:.4f}, p10 {base['p10']:.3f}, ndcg loss {base['ndcg_loss']:.4f}, " + f"48 B/vector, read {base['read']:.4f} of an f32 scan") + for name, pred in fam: + sel = [r for r in rows if pred(r)] + best = max(sel, key=lambda r: r["recall"]) + print(f" {name:<28} best: {best['first']} {best['second']} {best['policy']}: recall {best['recall']:.4f} " + f"({best['recall'] - base['recall']:+.4f}), p10 {best['p10']:.3f}, ndcg loss {best['ndcg_loss']:.4f}, " + f"ram {best['ram']} B + disk {best['disk']} B, read {best['read']:.4f}, mean depth {best['depth']:.0f} " + f"(p90 {best['p90']:.0f})") + + +if __name__ == "__main__": + main(Path(sys.argv[1]), int(sys.argv[2]), Path(sys.argv[3])) diff --git a/experiments/2026-11-neighbour-spacing/README.md b/experiments/2026-11-neighbour-spacing/README.md new file mode 100644 index 0000000..33f4654 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/README.md @@ -0,0 +1,182 @@ +# Five corpora, two models: neighbour spacing predicts codec loss + +The SciFact experiments kept running into the same explanation for binary's loss. Neighbours in +the top 10 differ in cosine by a few thousandths, and a 384-bit sign code estimates cosine with +a noise of a few hundredths, so it reorders them. This experiment checks whether that holds as +a quantitative rule across kinds of text, and answers the question behind it: does the *type* +of content change what a codec costs? + +**Hypothesis.** A codec's loss of agreement with the exact f32 top-k is governed by one number: +the ratio of the codec's score noise to the score spacing of the corpus's neighbours. Corpora +with tighter spacing lose more under the same codec. The ratio will predict the loss across +corpora and models much better than the noise alone, which is what a codec's error figure +usually reports. + +**Result.** Confirmed. Over 90 (corpus, model, codec) points, noise ÷ band ranks recall@10 +loss with Spearman ρ = 0.980; noise alone manages 0.644. A single two-parameter curve fitted +on four corpora predicts the fifth's recall@10 with a mean absolute error of 0.024, against 0.090 +for the same curve on noise alone. Content type matters, but only through spacing. Across the +five corpora the band varies about 2.2× within a model: FiQA's financial Q&A is the tightest, +ArguAna's arguments the loosest, and Python code sits in the middle. Changing the embedding +model on the *same* corpus moves the band by as much (MiniLM's is 1.7 to 2.5× bge-small's). +MiniLM spreads neighbours further apart than it amplifies codec noise, which is why it has +tolerated binary better in every experiment so far. + +## Setup + +| | | +|---|---| +| Corpora | Five BEIR-format retrieval sets [1]: **SciFact** (scientific abstracts, 300 claims), **NFCorpus** [2] (medical articles, 323 queries), **ArguAna** [3] (arguments; the query is itself an argument and the target its counterargument, 1,406 queries), **FiQA** [4] (financial forum Q&A, 648 queries), **CoSQA** [5] from CoIR [6] (Python functions, web-search queries, 500 queries) | +| Chunking | 200 words, no overlap: 8,036 / 6,260 / 11,232 / 71,274 / 20,648 chunks | +| Queries | Each set's test split with its relevance judgments, via `hvec questions import-beir`; the query text is embedded with the collection's model | +| Embedding models | `bge-small-en-v1.5` (Xenova ONNX, 384 dims) and `all-MiniLM-L6-v2` (Qdrant ONNX, 384 dims), fastembed on CPU, no query instruction | +| Metric | cosine | +| k | 10 | +| Codecs | `f32` baseline, `int8`, `binary`, `binary-centred`, `pq96`, `pq48`, `pq24`, `mrl256`, `mrl192`, `mrl96` | +| Fitting | Trained codecs (`binary-centred`, `pq`) fitted on every vector of the collection, as `ingest --codec` does | +| Search | brute force, no index, no rescoring | +| Hardware | Apple M4 Pro, 14 cores, release build | +| Code | hvec commit after `cc9a5eb` (the commit that adds this experiment) | +| Command | `./run.sh` in this directory. It downloads ~27 MB of data and needs `uv` to read CoSQA's parquet files. About 45 minutes, mostly embedding FiQA | + +### The three new numbers + +`bench recall` now reports, next to recall and nDCG: + +- **`boundary_gap`**: the median over queries of the exact score gap between rank k and rank + k + 1, the margin between the last true neighbour and the first outsider. +- **`neighbour_band`**: the median over queries of the exact score gap between rank k and rank + 2k, the width of the score window that holds the next k competitors. This is the spacing + measure used below. It needs only f32 and is the same for every codec on a collection. +- **`rank_noise`**: the mean over queries of the standard deviation of the codec's score error + over the exact top-2k, with each query's mean error subtracted first. A constant shift + changes no ranking, so only this spread can reorder neighbours. It is not the same as `mean + err`. Binary on bge-small has a mean error of 0.31, nearly all of it bias, but a rank noise of + 0.019. + +## Results + +### Spacing by corpus and model + +Exact f32, median over queries. Binary and pq48 are both 48 bytes, 32× smaller than f32. + +| corpus | content | chunks | band, bge-small | band, MiniLM | binary recall@10, bge / MiniLM | pq48 recall@10, bge / MiniLM | f32 nDCG@10, bge / MiniLM | +|---|---|---:|---:|---:|---:|---:|---:| +| FiQA | financial Q&A | 71,274 | **0.0108** | 0.0271 | 0.537 / 0.666 | 0.642 / 0.716 | 0.387 / 0.367 | +| NFCorpus | medical | 6,260 | 0.0121 | 0.0308 | 0.556 / 0.669 | 0.684 / 0.742 | 0.314 / 0.282 | +| SciFact | scientific abstracts | 8,036 | 0.0135 | 0.0307 | 0.611 / 0.705 | 0.711 / 0.764 | 0.698 / 0.627 | +| CoSQA | Python functions | 20,648 | 0.0148 | **0.0248** | 0.660 / 0.719 | 0.773 / 0.775 | 0.269 / 0.279 | +| ArguAna | arguments | 11,232 | **0.0233** | **0.0529** | 0.748 / 0.815 | 0.811 / 0.849 | 0.421 / 0.361 | + +On bge-small, binary's recall follows the band's order exactly. FiQA is tightest and loses +most; ArguAna is loosest and loses least. On MiniLM the order holds except for CoSQA, which +MiniLM packs tightest of all but which loses less than FiQA, NFCorpus or SciFact; its codec +noise is also the lowest (see the caveats). The two models agree on the order of the corpora +apart from CoSQA, and SciFact and NFCorpus, which tie on MiniLM. + +### Does the ratio predict the loss? + +All 90 lossy points (5 corpora × 2 models × 9 codecs). The fitted curve is +`ln(loss / recall) = −0.727 + 1.045 · ln(noise / band)`, so the odds of losing a neighbour are +very nearly proportional to the ratio: **odds ≈ 0.48 × noise ÷ band**. A ratio of 1 means about +0.67 of the top 10 survives. + +| predictor | Spearman ρ with 1 − recall@10 | Spearman ρ with nDCG@10 loss | held-out recall@10 MAE (fit on four corpora, predict the fifth) | +|---|---:|---:|---:| +| rank noise ÷ neighbour band | **0.980** | **0.848** | **0.024** | +| rank noise alone | 0.644 | 0.680 | 0.090 | + +Mean absolute error of the held-out prediction, by corpus: + +| held-out corpus | MAE, ratio | worst point | MAE, noise alone | +|---|---:|---|---:| +| SciFact | 0.016 | 0.048 | 0.079 | +| CoSQA | 0.015 | 0.031 | 0.046 | +| NFCorpus | 0.024 | 0.096 | 0.090 | +| ArguAna | 0.031 | 0.111 | 0.136 | +| FiQA | 0.035 | 0.166 (bge-small, pq24) | 0.101 | + +**Predicting before running the codec.** The useful version of this needs only f32 on the new +corpus. Measure its band, borrow the codec's noise for that model from the other four corpora, +and read the recall off the curve. Leave-one-corpus-out, that gives an MAE of 0.033, against +0.062 for simply averaging the codec's recall on the other four corpora. It is better on four +corpora and worse on one. On CoSQA it gives 0.062 against 0.039, because codec noise there is +lower than elsewhere (binary on bge-small 0.014, against 0.018 to 0.020 on the prose corpora). +Rank noise per (codec, model) varies by 1.4 to 1.75× across corpora, so borrowing it is an +approximation. + +The full per-codec table is in `results/summary.txt`; the raw reports, with score errors, +top-1 agreement, document recall and timings, are in `results/--k10.json`. + +## Discussion + +**The denominator does the work.** Noise alone is a property of the codec and the model, and it +gets the ordering within one collection right. It cannot say that the same codec will lose 46 +points on FiQA and 25 on ArguAna, because the codec's noise is about the same on both (binary +on bge-small: 0.0177 and 0.0183). The band can. A codec's error figure, quoted on its own as +it usually is, does not tell you what it will cost on your corpus. Its ratio to the corpus's +spacing does. + +**Content type matters through spacing, and less than the model.** The original question was +whether changing the kind of text changes the result. It does, by up to 21 points of binary +recall between FiQA and ArguAna on bge-small. But code, the content most unlike prose here, +is not an outlier: CoSQA sits between SciFact and ArguAna on bge-small. Changing the model on +one corpus shifts the band as much as the whole range of corpora does. ArguAna is loose because +its queries are full arguments whose best match is one specific counterargument, so the top of +the ranking is well separated. FiQA is tight because it is the largest corpus, full of short, +similar forum answers. Size alone does not explain the band, though. NFCorpus is the smallest +corpus and the second tightest. + +**Why MiniLM tolerates binary better.** The int8-and-binary experiment found that MiniLM keeps +0.705 of the top 10 under binary where bge-small keeps 0.611, and put it down to centring. The +centred-binary experiment showed centring was not the reason. This experiment gives the +answer. On every corpus MiniLM's band is 1.7 to 2.5× wider than bge-small's, while its codec +noise is only 1.3 to 1.5× larger. Its neighbours sit further apart in cosine than the extra +quantization noise can close. + +**What the curve misses.** The five worst predictions are all `pq24` or `mrl96`, the most +aggressive codecs. On FiQA and NFCorpus, `pq24` loses 9 to 16 points more than predicted. On +ArguAna, `pq24` and `mrl96` lose 8 to 10 points *less* than predicted. Rank noise is +measured over the exact top-2k only. A codec that occasionally scores a distant vector very +high, pulling it in from far outside the window, does damage that this noise does not see. +Measuring noise over a wider window, or using a tail statistic, is the obvious refinement. +ArguAna's better-than-predicted points suggest the opposite effect: when the top of the ranking +is well separated, the band understates how much noise it takes to displace the top 10. + +**Labels.** The ratio predicts nDCG loss less well (ρ = 0.848) than recall loss, as expected. +nDCG depends on whether the reordered neighbours are the judged ones, which spacing cannot +know. On these corpora binary costs 0.7 to 5.4 nDCG points where it costs 19 to 46 points of +agreement, the same gap the labelled SciFact experiment found. + +**What is wrong with this experiment.** +- **CoSQA's corpus is mostly duplicates.** Only 6,267 of its 20,604 functions are distinct, and + 86% sit in groups of identical code (the largest has 20 copies). Exact ties are why its + boundary gap is zero. A codec scores identical vectors identically, so duplicates move + together and do not count as losses. The effective corpus is about 6,300 functions, and that + probably explains CoSQA's lower codec noise. A deduplicated code set is the next code + corpus to try. +- **Only two models, both 384-dimensional English text models.** The model effect is two points, + not a trend. +- **Spacing and size are not separated.** FiQA is both the largest corpus and the tightest. + Subsampling FiQA to 8,000 chunks would show how much of its band comes from size. +- **The fit's form was chosen after seeing the results.** The curve is a logit-linear relation + between loss and log ratio. The held-out numbers test its parameters, not its form, and the + Spearman figures do not depend on it. +- **Brute force only, no rescoring.** With an index or an f32 rescoring pass the relationship + will change, and rescoring is exactly what the band would tell you how deep to do. + +**Next.** +- **Rescoring depth from the band.** If a codec's ratio is r, how many candidates must be rescored + in f32 to recover the exact top 10? A curve of that is directly useful. +- **A deduplicated code corpus.** +- **Identifier-heavy content.** The hashes-and-amounts corpus discussed as the next experiment + should show whether spacing can be measured at all where f32 itself does not retrieve. + +## References + +1. Thakur et al., *BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models*, NeurIPS Datasets and Benchmarks 2021. +2. Boteva et al., *A Full-Text Learning to Rank Dataset for Medical Information Retrieval*, ECIR 2016. +3. Wachsmuth, Syed and Stein, *Retrieval of the Best Counterargument without Prior Topic Knowledge*, ACL 2018. +4. Maia et al., *WWW'18 Open Challenge: Financial Opinion Mining and Question Answering*, WWW Companion 2018. +5. Huang et al., *CoSQA: 20,000+ Web Queries for Code Search and Question Answering*, ACL 2021. +6. Li et al., *CoIR: A Comprehensive Benchmark for Code Information Retrieval Models*, arXiv:2407.02883, 2024. diff --git a/experiments/2026-11-neighbour-spacing/coir_to_beir.py b/experiments/2026-11-neighbour-spacing/coir_to_beir.py new file mode 100644 index 0000000..60e3779 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/coir_to_beir.py @@ -0,0 +1,32 @@ +"""Convert a CoIR dataset (Hugging Face parquet) into BEIR's directory layout. + + uv run --no-project --with pyarrow python -I coir_to_beir.py + + holds corpus.parquet, queries.parquet and test.parquet as published under +huggingface.co/datasets/CoIR-Retrieval/. gets corpus.jsonl, queries.jsonl and +qrels/test.tsv, which `hvec questions import-beir` reads. +""" + +import json +import sys +from pathlib import Path + +import pyarrow.parquet as pq + + +def main(src: Path, out: Path) -> None: + (out / "qrels").mkdir(parents=True, exist_ok=True) + for name in ("corpus", "queries"): + rows = pq.read_table(src / f"{name}.parquet", columns=["_id", "title", "text"]).to_pylist() + with open(out / f"{name}.jsonl", "w") as f: + for r in rows: + f.write(json.dumps({"_id": r["_id"], "title": r["title"] or "", "text": r["text"] or ""}) + "\n") + qrels = pq.read_table(src / "test.parquet").to_pylist() + with open(out / "qrels" / "test.tsv", "w") as f: + f.write("query-id\tcorpus-id\tscore\n") + for r in qrels: + f.write(f"{r['query-id']}\t{r['corpus-id']}\t{r['score']}\n") + + +if __name__ == "__main__": + main(Path(sys.argv[1]), Path(sys.argv[2])) diff --git a/experiments/2026-11-neighbour-spacing/results/arguana-bge-small-k10.json b/experiments/2026-11-neighbour-spacing/results/arguana-bge-small-k10.json new file mode 100644 index 0000000..fae2a95 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/arguana-bge-small-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 7, + "scan_ms": 8481, + "baseline_scan_ms": 5624, + "labelled_queries": 1406, + "ndcg_at_k": 0.4213256309294448, + "doc_recall_at_k": 0.8186344238975818, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0005070832888020876, + "max_abs_score_error": 0.003560394048690796, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 1, + "scan_ms": 2939, + "baseline_scan_ms": 5624, + "labelled_queries": 1406, + "ndcg_at_k": 0.42094978413016126, + "doc_recall_at_k": 0.817923186344239, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.0004678626842599116 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.7481507823613063, + "top1_agreement": 0.9943100995732574, + "mean_abs_score_error": 0.29762755612312175, + "max_abs_score_error": 0.4604634642601013, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 15, + "scan_ms": 2820, + "baseline_scan_ms": 5602, + "labelled_queries": 1406, + "ndcg_at_k": 0.3929241125565956, + "doc_recall_at_k": 0.7802275960170697, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.018319500828550522 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.7726173541962991, + "top1_agreement": 0.9871977240398293, + "mean_abs_score_error": 0.053304979039357936, + "max_abs_score_error": 0.2943284809589386, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 16, + "scan_ms": 4218, + "baseline_scan_ms": 5599, + "labelled_queries": 1406, + "ndcg_at_k": 0.40185348712965036, + "doc_recall_at_k": 0.7930298719772404, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.01837565346242917 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.9030583214793728, + "top1_agreement": 0.9985775248933144, + "mean_abs_score_error": 0.011947056878045115, + "max_abs_score_error": 0.09793269634246826, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 287, + "scan_ms": 2812, + "baseline_scan_ms": 5565, + "labelled_queries": 1406, + "ndcg_at_k": 0.4216820791937366, + "doc_recall_at_k": 0.8229018492176386, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.006742915672471994 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.8109530583214749, + "top1_agreement": 0.9914651493598862, + "mean_abs_score_error": 0.036657516904853936, + "max_abs_score_error": 0.20435547828674316, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 209, + "scan_ms": 2812, + "baseline_scan_ms": 5534, + "labelled_queries": 1406, + "ndcg_at_k": 0.40950995673475227, + "doc_recall_at_k": 0.8015647226173542, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.015867100950084683 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.7120910384068273, + "top1_agreement": 0.9743954480796586, + "mean_abs_score_error": 0.06628232603735303, + "max_abs_score_error": 0.29073286056518555, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 171, + "scan_ms": 2812, + "baseline_scan_ms": 5479, + "labelled_queries": 1406, + "ndcg_at_k": 0.3772622749522497, + "doc_recall_at_k": 0.7638691322901849, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.027925833599693847 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.8470839260312912, + "top1_agreement": 0.9978662873399715, + "mean_abs_score_error": 0.029919522304035582, + "max_abs_score_error": 0.15840327739715576, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 0, + "scan_ms": 5617, + "baseline_scan_ms": 5383, + "labelled_queries": 1406, + "ndcg_at_k": 0.41361830689627715, + "doc_recall_at_k": 0.8044096728307255, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.009745048246812101 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.7800853485063998, + "top1_agreement": 0.9964438122332859, + "mean_abs_score_error": 0.02610146358819906, + "max_abs_score_error": 0.20414245128631592, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 1, + "scan_ms": 2818, + "baseline_scan_ms": 5181, + "labelled_queries": 1406, + "ndcg_at_k": 0.39457139037223404, + "doc_recall_at_k": 0.7731152204836416, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.014322082686674217 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.6362019914651502, + "top1_agreement": 0.9957325746799431, + "mean_abs_score_error": 0.17025921581168055, + "max_abs_score_error": 0.6403820514678955, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 0, + "scan_ms": 1406, + "baseline_scan_ms": 5414, + "labelled_queries": 1406, + "ndcg_at_k": 0.36042049508942947, + "doc_recall_at_k": 0.7162162162162162, + "boundary_gap": 0.002153545618057251, + "neighbour_band": 0.023316502571105957, + "rank_noise": 0.040702926858906266 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/arguana-minilm-k10.json b/experiments/2026-11-neighbour-spacing/results/arguana-minilm-k10.json new file mode 100644 index 0000000..a21e209 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/arguana-minilm-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 1, + "scan_ms": 8451, + "baseline_scan_ms": 5634, + "labelled_queries": 1406, + "ndcg_at_k": 0.36132092431655044, + "doc_recall_at_k": 0.7418207681365576, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.000272750532110221, + "max_abs_score_error": 0.0019255904480814934, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 1, + "scan_ms": 2842, + "baseline_scan_ms": 5635, + "labelled_queries": 1406, + "ndcg_at_k": 0.3611649488918465, + "doc_recall_at_k": 0.7411095305832148, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.00024864197759157493 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.814793741109527, + "top1_agreement": 0.9914651493598862, + "mean_abs_score_error": 0.03651406762163744, + "max_abs_score_error": 0.24589136242866516, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 15, + "scan_ms": 2812, + "baseline_scan_ms": 5629, + "labelled_queries": 1406, + "ndcg_at_k": 0.3394304159589659, + "doc_recall_at_k": 0.7041251778093883, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.027135932180800144 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.8239687055476486, + "top1_agreement": 0.9935988620199147, + "mean_abs_score_error": 0.03839855235688562, + "max_abs_score_error": 0.24003946781158447, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 16, + "scan_ms": 4218, + "baseline_scan_ms": 5616, + "labelled_queries": 1406, + "ndcg_at_k": 0.34389024522303585, + "doc_recall_at_k": 0.7162162162162162, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.027019165604858963 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.9241109530583209, + "top1_agreement": 0.9971550497866287, + "mean_abs_score_error": 0.011926905077324513, + "max_abs_score_error": 0.09263049066066742, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 294, + "scan_ms": 2813, + "baseline_scan_ms": 5586, + "labelled_queries": 1406, + "ndcg_at_k": 0.35495992093510664, + "doc_recall_at_k": 0.7304409672830725, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.010672786260429074 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.8486486486486458, + "top1_agreement": 0.9879089615931721, + "mean_abs_score_error": 0.02580232751380129, + "max_abs_score_error": 0.176820307970047, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 208, + "scan_ms": 2812, + "baseline_scan_ms": 5537, + "labelled_queries": 1406, + "ndcg_at_k": 0.3484476266305802, + "doc_recall_at_k": 0.7261735419630156, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.02566689404324917 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.7433854907539089, + "top1_agreement": 0.9566145092460882, + "mean_abs_score_error": 0.04128833279618005, + "max_abs_score_error": 0.30833709239959717, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 170, + "scan_ms": 2812, + "baseline_scan_ms": 5462, + "labelled_queries": 1406, + "ndcg_at_k": 0.3163081901975043, + "doc_recall_at_k": 0.6749644381223329, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.04914630085769442 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.8876955903271663, + "top1_agreement": 0.9992887624466572, + "mean_abs_score_error": 0.027926932851656962, + "max_abs_score_error": 0.1975320726633072, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 0, + "scan_ms": 5593, + "baseline_scan_ms": 4937, + "labelled_queries": 1406, + "ndcg_at_k": 0.3510238446592218, + "doc_recall_at_k": 0.7226173541963016, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.01549295996472911 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.847439544807962, + "top1_agreement": 0.9943100995732574, + "mean_abs_score_error": 0.04066228457676593, + "max_abs_score_error": 0.27043676376342773, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 1, + "scan_ms": 2924, + "baseline_scan_ms": 4937, + "labelled_queries": 1406, + "ndcg_at_k": 0.3443544322758491, + "doc_recall_at_k": 0.7119487908961594, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.021898287059816456 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 11232, + "queries": 1406, + "recall_at_k": 0.7453058321479342, + "top1_agreement": 0.9914651493598862, + "mean_abs_score_error": 0.0705306629356935, + "max_abs_score_error": 0.47160983085632324, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 0, + "scan_ms": 1406, + "baseline_scan_ms": 5131, + "labelled_queries": 1406, + "ndcg_at_k": 0.3158309102201568, + "doc_recall_at_k": 0.6678520625889047, + "boundary_gap": 0.00461888313293457, + "neighbour_band": 0.052856624126434326, + "rank_noise": 0.03721680054128291 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/config.toml b/experiments/2026-11-neighbour-spacing/results/config.toml new file mode 100644 index 0000000..c0cbbfe --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/config.toml @@ -0,0 +1,17 @@ +db_path = "/Users/liam/dev/hvec/experiments/2026-11-neighbour-spacing/work/hvec.db" +default_chat = "anthropic" +default_embedder = "bge-small" +metric = "cosine" + +[chat.anthropic] +provider = "anthropic" +model = "claude-opus-5-5" +api_key_env = "ANTHROPIC_API_KEY" + +[embed.bge-small] +provider = "local" +model = "BGESmallENV15" + +[embed.minilm] +provider = "local" +model = "AllMiniLML6V2" diff --git a/experiments/2026-11-neighbour-spacing/results/cosqa-bge-small-k10.json b/experiments/2026-11-neighbour-spacing/results/cosqa-bge-small-k10.json new file mode 100644 index 0000000..544e60e --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/cosqa-bge-small-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 6019, + "baseline_scan_ms": 3699, + "labelled_queries": 500, + "ndcg_at_k": 0.2692874106799324, + "doc_recall_at_k": 0.49, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "top1_agreement": 0.994, + "mean_abs_score_error": 0.00048643504310051444, + "max_abs_score_error": 0.003212153911590576, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2501, + "baseline_scan_ms": 3633, + "labelled_queries": 500, + "ndcg_at_k": 0.26910372035834845, + "doc_recall_at_k": 0.49, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.00034771140464969196 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.6598000000000006, + "top1_agreement": 0.636, + "mean_abs_score_error": 0.27137701679067594, + "max_abs_score_error": 0.41845062375068665, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2500, + "baseline_scan_ms": 3619, + "labelled_queries": 500, + "ndcg_at_k": 0.2619419566777617, + "doc_recall_at_k": 0.474, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.014175589010162073 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.7252, + "top1_agreement": 0.69, + "mean_abs_score_error": 0.053257550348530254, + "max_abs_score_error": 0.21956953406333923, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 30, + "scan_ms": 2514, + "baseline_scan_ms": 3511, + "labelled_queries": 500, + "ndcg_at_k": 0.252044032538105, + "doc_recall_at_k": 0.466, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.010298020049629607 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.8775999999999999, + "top1_agreement": 0.842, + "mean_abs_score_error": 0.010926779228652669, + "max_abs_score_error": 0.07641160488128662, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 555, + "scan_ms": 2028, + "baseline_scan_ms": 3989, + "labelled_queries": 500, + "ndcg_at_k": 0.2702221578083903, + "doc_recall_at_k": 0.492, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.004824377917958364 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.7727999999999999, + "top1_agreement": 0.76, + "mean_abs_score_error": 0.03171940466172124, + "max_abs_score_error": 0.15522488951683044, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 398, + "scan_ms": 2011, + "baseline_scan_ms": 3927, + "labelled_queries": 500, + "ndcg_at_k": 0.25522357072945623, + "doc_recall_at_k": 0.472, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.009104592511394003 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.6733999999999999, + "top1_agreement": 0.604, + "mean_abs_score_error": 0.05756296253417673, + "max_abs_score_error": 0.23461627960205078, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 328, + "scan_ms": 2001, + "baseline_scan_ms": 3969, + "labelled_queries": 500, + "ndcg_at_k": 0.24068518345340473, + "doc_recall_at_k": 0.462, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.014045970600847607 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.8011999999999995, + "top1_agreement": 0.794, + "mean_abs_score_error": 0.028956487218136706, + "max_abs_score_error": 0.1592731475830078, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 2, + "scan_ms": 3943, + "baseline_scan_ms": 3746, + "labelled_queries": 500, + "ndcg_at_k": 0.270182252074119, + "doc_recall_at_k": 0.494, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.00766111230023529 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.7266, + "top1_agreement": 0.73, + "mean_abs_score_error": 0.026370426615832193, + "max_abs_score_error": 0.19124138355255127, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 2, + "scan_ms": 2523, + "baseline_scan_ms": 3812, + "labelled_queries": 500, + "ndcg_at_k": 0.2643639077739527, + "doc_recall_at_k": 0.48, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.01132381092169114 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.5438, + "top1_agreement": 0.532, + "mean_abs_score_error": 0.11066608124492486, + "max_abs_score_error": 0.4681087136268616, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 1, + "scan_ms": 1000, + "baseline_scan_ms": 3777, + "labelled_queries": 500, + "ndcg_at_k": 0.23477534905947867, + "doc_recall_at_k": 0.442, + "boundary_gap": 0.0, + "neighbour_band": 0.014807552099227905, + "rank_noise": 0.0253390571447045 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/cosqa-minilm-k10.json b/experiments/2026-11-neighbour-spacing/results/cosqa-minilm-k10.json new file mode 100644 index 0000000..e253b8e --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/cosqa-minilm-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 2, + "scan_ms": 6005, + "baseline_scan_ms": 3521, + "labelled_queries": 500, + "ndcg_at_k": 0.2785507318131565, + "doc_recall_at_k": 0.496, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "top1_agreement": 0.998, + "mean_abs_score_error": 0.00026673176226905417, + "max_abs_score_error": 0.0018810927867889404, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 2, + "scan_ms": 2501, + "baseline_scan_ms": 3528, + "labelled_queries": 500, + "ndcg_at_k": 0.2785507318131565, + "doc_recall_at_k": 0.496, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.00020655862597241363 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.7190000000000004, + "top1_agreement": 0.718, + "mean_abs_score_error": 0.04212298762279244, + "max_abs_score_error": 0.22469326853752136, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 27, + "scan_ms": 2500, + "baseline_scan_ms": 3513, + "labelled_queries": 500, + "ndcg_at_k": 0.26045612373696797, + "doc_recall_at_k": 0.472, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.018597556555114886 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.7246000000000006, + "top1_agreement": 0.71, + "mean_abs_score_error": 0.04270127164316225, + "max_abs_score_error": 0.2396111786365509, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 30, + "scan_ms": 2538, + "baseline_scan_ms": 3527, + "labelled_queries": 500, + "ndcg_at_k": 0.2542238193998893, + "doc_recall_at_k": 0.468, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.01770042272474518 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.8969999999999995, + "top1_agreement": 0.892, + "mean_abs_score_error": 0.011120896451440956, + "max_abs_score_error": 0.08474792540073395, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 526, + "scan_ms": 2001, + "baseline_scan_ms": 3518, + "labelled_queries": 500, + "ndcg_at_k": 0.2713918004616955, + "doc_recall_at_k": 0.488, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.007792641547558008 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.775, + "top1_agreement": 0.758, + "mean_abs_score_error": 0.02571628411856676, + "max_abs_score_error": 0.18236444890499115, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 383, + "scan_ms": 2000, + "baseline_scan_ms": 3512, + "labelled_queries": 500, + "ndcg_at_k": 0.2586414286624784, + "doc_recall_at_k": 0.472, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.016073076858259247 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.6399999999999997, + "top1_agreement": 0.604, + "mean_abs_score_error": 0.044110078718467005, + "max_abs_score_error": 0.27030694484710693, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 314, + "scan_ms": 1993, + "baseline_scan_ms": 3512, + "labelled_queries": 500, + "ndcg_at_k": 0.24224725378424777, + "doc_recall_at_k": 0.436, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.02516490676753467 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.833, + "top1_agreement": 0.796, + "mean_abs_score_error": 0.027763805804227253, + "max_abs_score_error": 0.1750592738389969, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 2, + "scan_ms": 3528, + "baseline_scan_ms": 3511, + "labelled_queries": 500, + "ndcg_at_k": 0.2614298740810159, + "doc_recall_at_k": 0.478, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.01254640641921547 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.7662000000000007, + "top1_agreement": 0.746, + "mean_abs_score_error": 0.04010531444280398, + "max_abs_score_error": 0.2795931398868561, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 2, + "scan_ms": 2499, + "baseline_scan_ms": 3512, + "labelled_queries": 500, + "ndcg_at_k": 0.25747870044988713, + "doc_recall_at_k": 0.468, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.018201763699497647 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 20648, + "queries": 500, + "recall_at_k": 0.5994, + "top1_agreement": 0.6, + "mean_abs_score_error": 0.06647907316936946, + "max_abs_score_error": 0.41664114594459534, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 1, + "scan_ms": 1000, + "baseline_scan_ms": 3510, + "labelled_queries": 500, + "ndcg_at_k": 0.23139184114328024, + "doc_recall_at_k": 0.414, + "boundary_gap": 0.0, + "neighbour_band": 0.024774938821792603, + "rank_noise": 0.030621676357209204 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/fiqa-bge-small-k10.json b/experiments/2026-11-neighbour-spacing/results/fiqa-bge-small-k10.json new file mode 100644 index 0000000..6e99470 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/fiqa-bge-small-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 9, + "scan_ms": 27417, + "baseline_scan_ms": 17015, + "labelled_queries": 648, + "ndcg_at_k": 0.38702083481577326, + "doc_recall_at_k": 0.438503750193565, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "top1_agreement": 0.9814814814814815, + "mean_abs_score_error": 0.0005484225728834084, + "max_abs_score_error": 0.004337728023529053, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 9, + "scan_ms": 11704, + "baseline_scan_ms": 16307, + "labelled_queries": 648, + "ndcg_at_k": 0.3872220880286448, + "doc_recall_at_k": 0.4394958136856284, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.0005053578053552417 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.5367283950617282, + "top1_agreement": 0.49537037037037035, + "mean_abs_score_error": 0.2835777693232799, + "max_abs_score_error": 0.48717743158340454, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 95, + "scan_ms": 11048, + "baseline_scan_ms": 16206, + "labelled_queries": 648, + "ndcg_at_k": 0.3334523978137343, + "doc_recall_at_k": 0.3947765026468731, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.01769264290610194 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.6001543209876543, + "top1_agreement": 0.5432098765432098, + "mean_abs_score_error": 0.05047516723042159, + "max_abs_score_error": 0.37548398971557617, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 106, + "scan_ms": 13018, + "baseline_scan_ms": 16383, + "labelled_queries": 648, + "ndcg_at_k": 0.344364230170545, + "doc_recall_at_k": 0.40706942234720006, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.011478020154759652 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.8165123456790103, + "top1_agreement": 0.7916666666666666, + "mean_abs_score_error": 0.011983585897081088, + "max_abs_score_error": 0.14383669197559357, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 1827, + "scan_ms": 10397, + "baseline_scan_ms": 16259, + "labelled_queries": 648, + "ndcg_at_k": 0.37626699523918095, + "doc_recall_at_k": 0.4299113780595262, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.005679709089590111 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.6418209876543218, + "top1_agreement": 0.5925925925925926, + "mean_abs_score_error": 0.035776402318124303, + "max_abs_score_error": 0.24360127747058868, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 1324, + "scan_ms": 9133, + "baseline_scan_ms": 15826, + "labelled_queries": 648, + "ndcg_at_k": 0.3532630384905834, + "doc_recall_at_k": 0.4089782259921148, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.010514919675474296 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.4453703703703707, + "top1_agreement": 0.37808641975308643, + "mean_abs_score_error": 0.06284935984285886, + "max_abs_score_error": 0.31088903546333313, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 1085, + "scan_ms": 8886, + "baseline_scan_ms": 15685, + "labelled_queries": 648, + "ndcg_at_k": 0.2745208420960961, + "doc_recall_at_k": 0.3450280986160616, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.014464009981583719 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.7191358024691361, + "top1_agreement": 0.7006172839506173, + "mean_abs_score_error": 0.03184020888152214, + "max_abs_score_error": 0.14711269736289978, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 6, + "scan_ms": 16599, + "baseline_scan_ms": 15686, + "labelled_queries": 648, + "ndcg_at_k": 0.36396594125012394, + "doc_recall_at_k": 0.4213924921100849, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.008853871877378617 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.611574074074074, + "top1_agreement": 0.5787037037037037, + "mean_abs_score_error": 0.024797802730726654, + "max_abs_score_error": 0.21788272261619568, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 7, + "scan_ms": 11498, + "baseline_scan_ms": 15703, + "labelled_queries": 648, + "ndcg_at_k": 0.34959650702477446, + "doc_recall_at_k": 0.39497975875290686, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.013431924041213105 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.3912037037037035, + "top1_agreement": 0.39814814814814814, + "mean_abs_score_error": 0.15664787447068493, + "max_abs_score_error": 0.6938121914863586, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 5, + "scan_ms": 4549, + "baseline_scan_ms": 15662, + "labelled_queries": 648, + "ndcg_at_k": 0.2881940968941353, + "doc_recall_at_k": 0.3377247966368337, + "boundary_gap": 0.000988304615020752, + "neighbour_band": 0.010756641626358032, + "rank_noise": 0.033089520342411874 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/fiqa-minilm-k10.json b/experiments/2026-11-neighbour-spacing/results/fiqa-minilm-k10.json new file mode 100644 index 0000000..4448fb6 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/fiqa-minilm-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 9, + "scan_ms": 27692, + "baseline_scan_ms": 17267, + "labelled_queries": 648, + "ndcg_at_k": 0.36660237634408416, + "doc_recall_at_k": 0.4373899163251016, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "top1_agreement": 0.9953703703703703, + "mean_abs_score_error": 0.0002753687362949216, + "max_abs_score_error": 0.002191118896007538, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 9, + "scan_ms": 11723, + "baseline_scan_ms": 16659, + "labelled_queries": 648, + "ndcg_at_k": 0.3663844364427974, + "doc_recall_at_k": 0.43668261179835266, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.0002760758935835247 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.6655864197530874, + "top1_agreement": 0.6203703703703703, + "mean_abs_score_error": 0.03334946302009159, + "max_abs_score_error": 0.24864184856414795, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 97, + "scan_ms": 11046, + "baseline_scan_ms": 16350, + "labelled_queries": 648, + "ndcg_at_k": 0.3527147190089763, + "doc_recall_at_k": 0.4254477604014641, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.025144873356017765 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.6825617283950618, + "top1_agreement": 0.6388888888888888, + "mean_abs_score_error": 0.03403443928997409, + "max_abs_score_error": 0.2283090353012085, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 108, + "scan_ms": 12983, + "baseline_scan_ms": 16363, + "labelled_queries": 648, + "ndcg_at_k": 0.35197086253288135, + "doc_recall_at_k": 0.41028026757193414, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.023914843104149184 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.8567901234567868, + "top1_agreement": 0.7978395061728395, + "mean_abs_score_error": 0.012456979336543259, + "max_abs_score_error": 0.09894181787967682, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 1848, + "scan_ms": 10442, + "baseline_scan_ms": 16409, + "labelled_queries": 648, + "ndcg_at_k": 0.3611501871412196, + "doc_recall_at_k": 0.43741869841406883, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.011738154768883602 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.7162037037037043, + "top1_agreement": 0.6712962962962963, + "mean_abs_score_error": 0.025943000530134038, + "max_abs_score_error": 0.20088443160057068, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 1337, + "scan_ms": 9262, + "baseline_scan_ms": 16345, + "labelled_queries": 648, + "ndcg_at_k": 0.34514311125334995, + "doc_recall_at_k": 0.4148987629311702, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.0219992525810692 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.5069444444444439, + "top1_agreement": 0.43209876543209874, + "mean_abs_score_error": 0.040591166647062485, + "max_abs_score_error": 0.2780871093273163, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 1085, + "scan_ms": 9038, + "baseline_scan_ms": 16297, + "labelled_queries": 648, + "ndcg_at_k": 0.2827961021471452, + "doc_recall_at_k": 0.3483780879614214, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.03367902353177581 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.7637345679012335, + "top1_agreement": 0.7623456790123457, + "mean_abs_score_error": 0.028378887340793732, + "max_abs_score_error": 0.20322340726852417, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 7, + "scan_ms": 17021, + "baseline_scan_ms": 16207, + "labelled_queries": 648, + "ndcg_at_k": 0.3516388134181695, + "doc_recall_at_k": 0.42120024180209364, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.019558401648059515 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.6822530864197538, + "top1_agreement": 0.654320987654321, + "mean_abs_score_error": 0.04142851608382891, + "max_abs_score_error": 0.303480327129364, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 8, + "scan_ms": 11686, + "baseline_scan_ms": 16191, + "labelled_queries": 648, + "ndcg_at_k": 0.3369116766046438, + "doc_recall_at_k": 0.40855454987399437, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.027851927283667446 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 71274, + "queries": 648, + "recall_at_k": 0.4861111111111111, + "top1_agreement": 0.4567901234567901, + "mean_abs_score_error": 0.07305605469759935, + "max_abs_score_error": 0.48974698781967163, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 5, + "scan_ms": 4563, + "baseline_scan_ms": 16236, + "labelled_queries": 648, + "ndcg_at_k": 0.2874688093390195, + "doc_recall_at_k": 0.34846250286065095, + "boundary_gap": 0.0026566386222839355, + "neighbour_band": 0.027133822441101074, + "rank_noise": 0.048540993436266625 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/nfcorpus-bge-small-k10.json b/experiments/2026-11-neighbour-spacing/results/nfcorpus-bge-small-k10.json new file mode 100644 index 0000000..45573e1 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/nfcorpus-bge-small-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 0, + "scan_ms": 969, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.31359564285599023, + "doc_recall_at_k": 0.15054159523920038, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "top1_agreement": 0.9845201238390093, + "mean_abs_score_error": 0.0005034820935406243, + "max_abs_score_error": 0.003187239170074463, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 0, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.3143486384401113, + "doc_recall_at_k": 0.15094077665441813, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.0005408677863145403 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.5560371517027866, + "top1_agreement": 0.5356037151702786, + "mean_abs_score_error": 0.28630656971298996, + "max_abs_score_error": 0.4299890100955963, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 8, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.29321160644525546, + "doc_recall_at_k": 0.14004003836456763, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.02005583883466805 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.5826625386996898, + "top1_agreement": 0.5201238390092879, + "mean_abs_score_error": 0.043051372880571565, + "max_abs_score_error": 0.29372042417526245, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 9, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2875005723745878, + "doc_recall_at_k": 0.1417636243417844, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.013492521855551734 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.8349845201238411, + "top1_agreement": 0.7956656346749226, + "mean_abs_score_error": 0.009823754261657922, + "max_abs_score_error": 0.09399139881134033, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 163, + "scan_ms": 323, + "baseline_scan_ms": 647, + "labelled_queries": 323, + "ndcg_at_k": 0.31244343408405784, + "doc_recall_at_k": 0.14798313755819867, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.006044045887215805 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.6842105263157896, + "top1_agreement": 0.6408668730650154, + "mean_abs_score_error": 0.028181584920538835, + "max_abs_score_error": 0.18332970142364502, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 116, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.30503224240862437, + "doc_recall_at_k": 0.1450286165688322, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.010998554937267174 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.529411764705882, + "top1_agreement": 0.4613003095975232, + "mean_abs_score_error": 0.05001508060395308, + "max_abs_score_error": 0.24954304099082947, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 95, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2780692723416212, + "doc_recall_at_k": 0.1324171369609357, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.015120641347280507 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.7024767801857587, + "top1_agreement": 0.7492260061919505, + "mean_abs_score_error": 0.018536436225489755, + "max_abs_score_error": 0.13271552324295044, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 0, + "scan_ms": 647, + "baseline_scan_ms": 647, + "labelled_queries": 323, + "ndcg_at_k": 0.29341408928755514, + "doc_recall_at_k": 0.1368489577706274, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.010919478035133172 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.5845201238390091, + "top1_agreement": 0.631578947368421, + "mean_abs_score_error": 0.02748409267441801, + "max_abs_score_error": 0.22458510100841522, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 0, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.28761050224948126, + "doc_recall_at_k": 0.13678922672916272, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.017146109287093893 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.3826625386996904, + "top1_agreement": 0.38699690402476783, + "mean_abs_score_error": 0.12094133259915978, + "max_abs_score_error": 0.5094493627548218, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 0, + "scan_ms": 0, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.25005449614645925, + "doc_recall_at_k": 0.12046169197217445, + "boundary_gap": 0.0011455416679382324, + "neighbour_band": 0.012125968933105469, + "rank_noise": 0.037642149550326895 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/nfcorpus-minilm-k10.json b/experiments/2026-11-neighbour-spacing/results/nfcorpus-minilm-k10.json new file mode 100644 index 0000000..285e074 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/nfcorpus-minilm-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 0, + "scan_ms": 972, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.28194924865399074, + "doc_recall_at_k": 0.13680635104820643, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "top1_agreement": 0.9938080495356038, + "mean_abs_score_error": 0.000271466889607559, + "max_abs_score_error": 0.00203409674577415, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 0, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.28174030016550045, + "doc_recall_at_k": 0.13521942330574344, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.0003004581936229898 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.6690402476780186, + "top1_agreement": 0.6470588235294118, + "mean_abs_score_error": 0.03383667299692817, + "max_abs_score_error": 0.2205052375793457, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 8, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.27221531371670776, + "doc_recall_at_k": 0.1323602119067216, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.029061802535522818 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.6907120743034053, + "top1_agreement": 0.718266253869969, + "mean_abs_score_error": 0.03360379143128291, + "max_abs_score_error": 0.25990915298461914, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 9, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2776443014277795, + "doc_recall_at_k": 0.13227598658615683, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.026947140886363668 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.8743034055727568, + "top1_agreement": 0.9009287925696594, + "mean_abs_score_error": 0.010902563224942327, + "max_abs_score_error": 0.07609051465988159, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 163, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2834225478039319, + "doc_recall_at_k": 0.13579010339617162, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.011645806522831125 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.7424148606811146, + "top1_agreement": 0.7275541795665634, + "mean_abs_score_error": 0.02257439673325492, + "max_abs_score_error": 0.16411960124969482, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 116, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2786296514908264, + "doc_recall_at_k": 0.13757751316347785, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.02263408737491555 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.5944272445820431, + "top1_agreement": 0.5170278637770898, + "mean_abs_score_error": 0.034625802915938676, + "max_abs_score_error": 0.24361500144004822, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 95, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2544200174464856, + "doc_recall_at_k": 0.12050051484895005, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.03515438785727546 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.7510835913312697, + "top1_agreement": 0.8018575851393189, + "mean_abs_score_error": 0.02712823811017663, + "max_abs_score_error": 0.18324977159500122, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 0, + "scan_ms": 646, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.2696085021530307, + "doc_recall_at_k": 0.13066436568652956, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.023323394481303935 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.6724458204334361, + "top1_agreement": 0.6811145510835913, + "mean_abs_score_error": 0.04127554511561218, + "max_abs_score_error": 0.25699079036712646, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 0, + "scan_ms": 323, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.26130354304560105, + "doc_recall_at_k": 0.1260318245305586, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.03431767697873664 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 6260, + "queries": 323, + "recall_at_k": 0.4783281733746126, + "top1_agreement": 0.5139318885448917, + "mean_abs_score_error": 0.0673865409819313, + "max_abs_score_error": 0.4086548089981079, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 0, + "scan_ms": 0, + "baseline_scan_ms": 646, + "labelled_queries": 323, + "ndcg_at_k": 0.23492400790783888, + "doc_recall_at_k": 0.10993795281410705, + "boundary_gap": 0.0027038156986236572, + "neighbour_band": 0.030829548835754395, + "rank_noise": 0.055908736678008775 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/scifact-bge-small-k10.json b/experiments/2026-11-neighbour-spacing/results/scifact-bge-small-k10.json new file mode 100644 index 0000000..68bf825 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/scifact-bge-small-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 1, + "scan_ms": 1203, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.698128294524517, + "doc_recall_at_k": 0.8178888888888889, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "top1_agreement": 0.9966666666666667, + "mean_abs_score_error": 0.0004928094862294431, + "max_abs_score_error": 0.0033954381942749023, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 1, + "scan_ms": 600, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6978365964864137, + "doc_recall_at_k": 0.8178888888888889, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.0004970622783537161 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.6109999999999995, + "top1_agreement": 0.7, + "mean_abs_score_error": 0.30665168931459574, + "max_abs_score_error": 0.45795172452926636, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 10, + "scan_ms": 346, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6536746843702901, + "doc_recall_at_k": 0.7676111111111111, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.018715791052223443 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.6559999999999993, + "top1_agreement": 0.7266666666666667, + "mean_abs_score_error": 0.04820936800197808, + "max_abs_score_error": 0.2849487364292145, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 12, + "scan_ms": 600, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6576686434556663, + "doc_recall_at_k": 0.7695555555555555, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.013085613849637225 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.8446666666666696, + "top1_agreement": 0.8866666666666667, + "mean_abs_score_error": 0.010790545003816562, + "max_abs_score_error": 0.09855541586875916, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 210, + "scan_ms": 300, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6918877322341538, + "doc_recall_at_k": 0.8168888888888889, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.0059490096574797815 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.711333333333333, + "top1_agreement": 0.8033333333333333, + "mean_abs_score_error": 0.03272171884006245, + "max_abs_score_error": 0.1959289014339447, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 149, + "scan_ms": 300, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6775694097419889, + "doc_recall_at_k": 0.7968888888888889, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.011187321269100204 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.5766666666666663, + "top1_agreement": 0.6233333333333333, + "mean_abs_score_error": 0.059129319624044774, + "max_abs_score_error": 0.2571095824241638, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 122, + "scan_ms": 300, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6021298615038453, + "doc_recall_at_k": 0.7254444444444444, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.01640890269556495 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.7446666666666667, + "top1_agreement": 0.8433333333333334, + "mean_abs_score_error": 0.020080132697896963, + "max_abs_score_error": 0.1177527904510498, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 0, + "scan_ms": 900, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6836913510424513, + "doc_recall_at_k": 0.7962222222222223, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.010129348730168785 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.6486666666666661, + "top1_agreement": 0.75, + "mean_abs_score_error": 0.022870995990038216, + "max_abs_score_error": 0.15820246934890747, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 0, + "scan_ms": 600, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.65882372009652, + "doc_recall_at_k": 0.7674444444444444, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.01529782362373951 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.45599999999999985, + "top1_agreement": 0.5733333333333334, + "mean_abs_score_error": 0.11048025627193565, + "max_abs_score_error": 0.5446805357933044, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 0, + "scan_ms": 0, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.5929619619490947, + "doc_recall_at_k": 0.7116666666666666, + "boundary_gap": 0.001324295997619629, + "neighbour_band": 0.013464927673339844, + "rank_noise": 0.03383780103560711 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/scifact-minilm-k10.json b/experiments/2026-11-neighbour-spacing/results/scifact-minilm-k10.json new file mode 100644 index 0000000..01773c0 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/scifact-minilm-k10.json @@ -0,0 +1,222 @@ +[ + { + "codec": "f32", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 1.0, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.0, + "max_abs_score_error": 0.0, + "bytes_per_vector": 1536, + "compression_ratio": 1.0, + "encode_ms": 1, + "scan_ms": 1219, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6267937784557772, + "doc_recall_at_k": 0.777888888888889, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.0 + }, + { + "codec": "int8", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.995, + "top1_agreement": 1.0, + "mean_abs_score_error": 0.00027110359914932404, + "max_abs_score_error": 0.0019342228770256042, + "bytes_per_vector": 392, + "compression_ratio": 3.9183674, + "encode_ms": 1, + "scan_ms": 600, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6266916263886889, + "doc_recall_at_k": 0.777888888888889, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.00028841633469485104 + }, + { + "codec": "binary", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.7049999999999998, + "top1_agreement": 0.73, + "mean_abs_score_error": 0.034386584007278745, + "max_abs_score_error": 0.21312355995178223, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 11, + "scan_ms": 453, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.604971709997024, + "doc_recall_at_k": 0.7334444444444445, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.027713695127966537 + }, + { + "codec": "binary-centred", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.718666666666667, + "top1_agreement": 0.7166666666666667, + "mean_abs_score_error": 0.03625820329779845, + "max_abs_score_error": 0.2365032434463501, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 12, + "scan_ms": 601, + "baseline_scan_ms": 901, + "labelled_queries": 300, + "ndcg_at_k": 0.597754249570433, + "doc_recall_at_k": 0.7387777777777778, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.026426586664381465 + }, + { + "codec": "pq96", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.874666666666669, + "top1_agreement": 0.89, + "mean_abs_score_error": 0.01157137607152094, + "max_abs_score_error": 0.08639165759086609, + "bytes_per_vector": 96, + "compression_ratio": 16.0, + "encode_ms": 211, + "scan_ms": 303, + "baseline_scan_ms": 901, + "labelled_queries": 300, + "ndcg_at_k": 0.6303028117112374, + "doc_recall_at_k": 0.7863888888888888, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.011711888456250843 + }, + { + "codec": "pq48", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.7640000000000003, + "top1_agreement": 0.7633333333333333, + "mean_abs_score_error": 0.0243816094105812, + "max_abs_score_error": 0.1695815920829773, + "bytes_per_vector": 48, + "compression_ratio": 32.0, + "encode_ms": 157, + "scan_ms": 302, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6128831565370856, + "doc_recall_at_k": 0.7615, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.022971436105667608 + }, + { + "codec": "pq24", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.6363333333333332, + "top1_agreement": 0.5866666666666667, + "mean_abs_score_error": 0.03883274196373913, + "max_abs_score_error": 0.24105709791183472, + "bytes_per_vector": 24, + "compression_ratio": 64.0, + "encode_ms": 122, + "scan_ms": 300, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.5501427919742216, + "doc_recall_at_k": 0.7068333333333333, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.03588049972806333 + }, + { + "codec": "mrl256", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.7736666666666677, + "top1_agreement": 0.82, + "mean_abs_score_error": 0.027957764231806313, + "max_abs_score_error": 0.17353177070617676, + "bytes_per_vector": 1024, + "compression_ratio": 1.5, + "encode_ms": 0, + "scan_ms": 900, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.6156720278727258, + "doc_recall_at_k": 0.7657222222222222, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.02178886690430093 + }, + { + "codec": "mrl192", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.6733333333333337, + "top1_agreement": 0.74, + "mean_abs_score_error": 0.04183451366952325, + "max_abs_score_error": 0.2845783829689026, + "bytes_per_vector": 768, + "compression_ratio": 2.0, + "encode_ms": 0, + "scan_ms": 600, + "baseline_scan_ms": 900, + "labelled_queries": 300, + "ndcg_at_k": 0.5984043076326573, + "doc_recall_at_k": 0.7526666666666667, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.0320660825788346 + }, + { + "codec": "mrl96", + "metric": "cosine", + "k": 10, + "vectors": 8036, + "queries": 300, + "recall_at_k": 0.5103333333333332, + "top1_agreement": 0.5366666666666666, + "mean_abs_score_error": 0.06895892703074054, + "max_abs_score_error": 0.4210922420024872, + "bytes_per_vector": 384, + "compression_ratio": 4.0, + "encode_ms": 0, + "scan_ms": 0, + "baseline_scan_ms": 899, + "labelled_queries": 300, + "ndcg_at_k": 0.5403981855188065, + "doc_recall_at_k": 0.6877777777777777, + "boundary_gap": 0.00346451997756958, + "neighbour_band": 0.03073228895664215, + "rank_noise": 0.053958500341940836 + } +] diff --git a/experiments/2026-11-neighbour-spacing/results/summary.txt b/experiments/2026-11-neighbour-spacing/results/summary.txt new file mode 100644 index 0000000..ea59c6f --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/results/summary.txt @@ -0,0 +1,120 @@ +collection vectors queries gap k:k+1 band k:2k f32 ndcg +arguana-bge-small 11232 1406 0.00215 0.02332 0.4213 +arguana-minilm 11232 1406 0.00462 0.05286 0.3613 +cosqa-bge-small 20648 500 0.00000 0.01481 0.2693 +cosqa-minilm 20648 500 0.00000 0.02477 0.2786 +fiqa-bge-small 71274 648 0.00099 0.01076 0.3870 +fiqa-minilm 71274 648 0.00266 0.02713 0.3666 +nfcorpus-bge-small 6260 323 0.00115 0.01213 0.3136 +nfcorpus-minilm 6260 323 0.00270 0.03083 0.2819 +scifact-bge-small 8036 300 0.00132 0.01346 0.6981 +scifact-minilm 8036 300 0.00346 0.03073 0.6268 + +collection codec bytes noise noise/band recall@k ndcg loss +arguana-bge-small int8 392 0.00047 0.020 0.9911 0.0004 +arguana-bge-small binary 48 0.01832 0.786 0.7482 0.0284 +arguana-bge-small binary-centred 48 0.01838 0.788 0.7726 0.0195 +arguana-bge-small pq96 96 0.00674 0.289 0.9031 -0.0004 +arguana-bge-small pq48 48 0.01587 0.681 0.8110 0.0118 +arguana-bge-small pq24 24 0.02793 1.198 0.7121 0.0441 +arguana-bge-small mrl256 1024 0.00975 0.418 0.8471 0.0077 +arguana-bge-small mrl192 768 0.01432 0.614 0.7801 0.0268 +arguana-bge-small mrl96 384 0.04070 1.746 0.6362 0.0609 +arguana-minilm int8 392 0.00025 0.005 0.9982 0.0002 +arguana-minilm binary 48 0.02714 0.513 0.8148 0.0219 +arguana-minilm binary-centred 48 0.02702 0.511 0.8240 0.0174 +arguana-minilm pq96 96 0.01067 0.202 0.9241 0.0064 +arguana-minilm pq48 48 0.02567 0.486 0.8486 0.0129 +arguana-minilm pq24 24 0.04915 0.930 0.7434 0.0450 +arguana-minilm mrl256 1024 0.01549 0.293 0.8877 0.0103 +arguana-minilm mrl192 768 0.02190 0.414 0.8474 0.0170 +arguana-minilm mrl96 384 0.03722 0.704 0.7453 0.0455 +cosqa-bge-small int8 392 0.00035 0.023 0.9896 0.0002 +cosqa-bge-small binary 48 0.01418 0.957 0.6598 0.0073 +cosqa-bge-small binary-centred 48 0.01030 0.695 0.7252 0.0172 +cosqa-bge-small pq96 96 0.00482 0.326 0.8776 -0.0009 +cosqa-bge-small pq48 48 0.00910 0.615 0.7728 0.0141 +cosqa-bge-small pq24 24 0.01405 0.949 0.6734 0.0286 +cosqa-bge-small mrl256 1024 0.00766 0.517 0.8012 -0.0009 +cosqa-bge-small mrl192 768 0.01132 0.765 0.7266 0.0049 +cosqa-bge-small mrl96 384 0.02534 1.711 0.5438 0.0345 +cosqa-minilm int8 392 0.00021 0.008 0.9962 0.0000 +cosqa-minilm binary 48 0.01860 0.751 0.7190 0.0181 +cosqa-minilm binary-centred 48 0.01770 0.714 0.7246 0.0243 +cosqa-minilm pq96 96 0.00779 0.315 0.8970 0.0072 +cosqa-minilm pq48 48 0.01607 0.649 0.7750 0.0199 +cosqa-minilm pq24 24 0.02516 1.016 0.6400 0.0363 +cosqa-minilm mrl256 1024 0.01255 0.506 0.8330 0.0171 +cosqa-minilm mrl192 768 0.01820 0.735 0.7662 0.0211 +cosqa-minilm mrl96 384 0.03062 1.236 0.5994 0.0472 +fiqa-bge-small int8 392 0.00051 0.047 0.9787 -0.0002 +fiqa-bge-small binary 48 0.01769 1.645 0.5367 0.0536 +fiqa-bge-small binary-centred 48 0.01148 1.067 0.6002 0.0427 +fiqa-bge-small pq96 96 0.00568 0.528 0.8165 0.0108 +fiqa-bge-small pq48 48 0.01051 0.978 0.6418 0.0338 +fiqa-bge-small pq24 24 0.01446 1.345 0.4454 0.1125 +fiqa-bge-small mrl256 1024 0.00885 0.823 0.7191 0.0231 +fiqa-bge-small mrl192 768 0.01343 1.249 0.6116 0.0374 +fiqa-bge-small mrl96 384 0.03309 3.076 0.3912 0.0988 +fiqa-minilm int8 392 0.00028 0.010 0.9965 0.0002 +fiqa-minilm binary 48 0.02514 0.927 0.6656 0.0139 +fiqa-minilm binary-centred 48 0.02391 0.881 0.6826 0.0146 +fiqa-minilm pq96 96 0.01174 0.433 0.8568 0.0055 +fiqa-minilm pq48 48 0.02200 0.811 0.7162 0.0215 +fiqa-minilm pq24 24 0.03368 1.241 0.5069 0.0838 +fiqa-minilm mrl256 1024 0.01956 0.721 0.7637 0.0150 +fiqa-minilm mrl192 768 0.02785 1.026 0.6823 0.0297 +fiqa-minilm mrl96 384 0.04854 1.789 0.4861 0.0791 +nfcorpus-bge-small int8 392 0.00054 0.045 0.9768 -0.0008 +nfcorpus-bge-small binary 48 0.02006 1.654 0.5560 0.0204 +nfcorpus-bge-small binary-centred 48 0.01349 1.113 0.5827 0.0261 +nfcorpus-bge-small pq96 96 0.00604 0.498 0.8350 0.0012 +nfcorpus-bge-small pq48 48 0.01100 0.907 0.6842 0.0086 +nfcorpus-bge-small pq24 24 0.01512 1.247 0.5294 0.0355 +nfcorpus-bge-small mrl256 1024 0.01092 0.901 0.7025 0.0202 +nfcorpus-bge-small mrl192 768 0.01715 1.414 0.5845 0.0260 +nfcorpus-bge-small mrl96 384 0.03764 3.104 0.3827 0.0635 +nfcorpus-minilm int8 392 0.00030 0.010 0.9963 0.0002 +nfcorpus-minilm binary 48 0.02906 0.943 0.6690 0.0097 +nfcorpus-minilm binary-centred 48 0.02695 0.874 0.6907 0.0043 +nfcorpus-minilm pq96 96 0.01165 0.378 0.8743 -0.0015 +nfcorpus-minilm pq48 48 0.02263 0.734 0.7424 0.0033 +nfcorpus-minilm pq24 24 0.03515 1.140 0.5944 0.0275 +nfcorpus-minilm mrl256 1024 0.02332 0.757 0.7511 0.0123 +nfcorpus-minilm mrl192 768 0.03432 1.113 0.6724 0.0206 +nfcorpus-minilm mrl96 384 0.05591 1.813 0.4783 0.0470 +scifact-bge-small int8 392 0.00050 0.037 0.9870 0.0003 +scifact-bge-small binary 48 0.01872 1.390 0.6110 0.0445 +scifact-bge-small binary-centred 48 0.01309 0.972 0.6560 0.0405 +scifact-bge-small pq96 96 0.00595 0.442 0.8447 0.0062 +scifact-bge-small pq48 48 0.01119 0.831 0.7113 0.0206 +scifact-bge-small pq24 24 0.01641 1.219 0.5767 0.0960 +scifact-bge-small mrl256 1024 0.01013 0.752 0.7447 0.0144 +scifact-bge-small mrl192 768 0.01530 1.136 0.6487 0.0393 +scifact-bge-small mrl96 384 0.03384 2.513 0.4560 0.1052 +scifact-minilm int8 392 0.00029 0.009 0.9950 0.0001 +scifact-minilm binary 48 0.02771 0.902 0.7050 0.0218 +scifact-minilm binary-centred 48 0.02643 0.860 0.7187 0.0290 +scifact-minilm pq96 96 0.01171 0.381 0.8747 -0.0035 +scifact-minilm pq48 48 0.02297 0.747 0.7640 0.0139 +scifact-minilm pq24 24 0.03588 1.168 0.6363 0.0767 +scifact-minilm mrl256 1024 0.02179 0.709 0.7737 0.0111 +scifact-minilm mrl192 768 0.03207 1.043 0.6733 0.0284 +scifact-minilm mrl96 384 0.05396 1.756 0.5103 0.0864 + +90 (collection, codec) points +spearman(noise/band, 1 - recall@k) = 0.980 +spearman(noise alone, 1 - recall@k) = 0.644 +spearman(noise/band, ndcg loss) = 0.848 +spearman(noise alone, ndcg loss) = 0.680 + +within each codec, across collections: spearman(band, 1 - recall@k) + binary -0.794 (n=10) + binary-centred -0.612 (n=10) + int8 -0.927 (n=10) + mrl192 -0.600 (n=10) + mrl256 -0.612 (n=10) + mrl96 -0.612 (n=10) + pq24 -0.515 (n=10) + pq48 -0.648 (n=10) + pq96 -0.648 (n=10) diff --git a/experiments/2026-11-neighbour-spacing/run.sh b/experiments/2026-11-neighbour-spacing/run.sh new file mode 100755 index 0000000..7135e32 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/run.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Does neighbour spacing predict codec loss? The same codec grid on five corpora of different +# kinds (scientific abstracts, medical, argument, financial Q&A, Python code) and two embedding +# models, retrieval only, with each corpus's exact neighbour spacing recorded next to each +# codec's rank noise. Needs: a built `hvec` (cargo build --release), curl, unzip, and `uv` +# (only to read the code dataset's parquet files). Downloads ~27 MB of data on first run. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$HERE/../.." +if [ -f "$ROOT/.env" ]; then set -a; . "$ROOT/.env"; set +a; fi +HVEC="${HVEC:-$ROOT/target/release/hvec}" +WORK="${WORK:-$HERE/work}" +K="${K:-10}" +CORPORA="${CORPORA:-scifact nfcorpus arguana cosqa fiqa}" +MODELS="${MODELS:-bge-small minilm}" +CODECS="${CODECS:-int8,binary,binary-centred,pq96,pq48,pq24,mrl256,mrl192,mrl96}" +mkdir -p "$WORK/dl" "$HERE/results" + +fetch() { # fetch : leaves a BEIR directory at $WORK/dl/ + local c="$1" + [ -f "$WORK/dl/$c/corpus.jsonl" ] && return + if [ "$c" = cosqa ]; then + local base=https://huggingface.co/datasets/CoIR-Retrieval/cosqa/resolve/main + mkdir -p "$WORK/dl/cosqa-parquet" + curl -sL -o "$WORK/dl/cosqa-parquet/corpus.parquet" "$base/corpus/corpus-00000-of-00001.parquet" + curl -sL -o "$WORK/dl/cosqa-parquet/queries.parquet" "$base/queries/queries-00000-of-00001.parquet" + curl -sL -o "$WORK/dl/cosqa-parquet/test.parquet" "$base/data/test-00000-of-00001.parquet" + (cd "$WORK" && uv run -q --no-project --with pyarrow python -I "$HERE/coir_to_beir.py" \ + "$WORK/dl/cosqa-parquet" "$WORK/dl/cosqa") + else + curl -sL -o "$WORK/dl/$c.zip" "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/$c.zip" + (cd "$WORK/dl" && unzip -oq "$c.zip") + fi +} + +export HVEC_CONFIG="$WORK/config.toml" +cat > "$HVEC_CONFIG" < "$HERE/results/$col-k$K.json" + echo "== $col done" + done +done +cp "$HVEC_CONFIG" "$HERE/results/config.toml" +python3 -I "$HERE/summarise.py" "$HERE/results" "$K" | tee "$HERE/results/summary.txt" +echo "results in $HERE/results" diff --git a/experiments/2026-11-neighbour-spacing/summarise.py b/experiments/2026-11-neighbour-spacing/summarise.py new file mode 100644 index 0000000..3397e43 --- /dev/null +++ b/experiments/2026-11-neighbour-spacing/summarise.py @@ -0,0 +1,78 @@ +"""Summarise the bench recall JSON files: spacing per collection, then every codec's loss +against its noise-to-spacing ratio, and the rank correlation between the two. + + python3 -I summarise.py +""" + +import json +import sys +from pathlib import Path + + +def ranks(xs): + order = sorted(range(len(xs)), key=lambda i: xs[i]) + r = [0.0] * len(xs) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and xs[order[j + 1]] == xs[order[i]]: + j += 1 + for t in range(i, j + 1): + r[order[t]] = (i + j) / 2 + 1 + i = j + 1 + return r + + +def pearson(a, b): + n = len(a) + ma, mb = sum(a) / n, sum(b) / n + cov = sum((x - ma) * (y - mb) for x, y in zip(a, b)) + va = sum((x - ma) ** 2 for x in a) ** 0.5 + vb = sum((y - mb) ** 2 for y in b) ** 0.5 + return cov / (va * vb) + + +def spearman(a, b): + return pearson(ranks(a), ranks(b)) + + +def main(results: Path, k: str) -> None: + rows = [] + print(f"{'collection':<20} {'vectors':>8} {'queries':>8} {'gap k:k+1':>10} {'band k:2k':>10} {'f32 ndcg':>9}") + for path in sorted(results.glob(f"*-k{k}.json")): + col = path.name[: -len(f"-k{k}.json")] + reports = json.loads(path.read_text()) + f32 = reports[0] + print(f"{col:<20} {f32['vectors']:>8} {f32['queries']:>8} {f32['boundary_gap']:>10.5f} " + f"{f32['neighbour_band']:>10.5f} {f32['ndcg_at_k'] or 0:>9.4f}") + for r in reports[1:]: + rows.append((col, r, f32)) + print() + print(f"{'collection':<20} {'codec':<15} {'bytes':>6} {'noise':>8} {'noise/band':>10} " + f"{'recall@k':>9} {'ndcg loss':>9}") + for col, r, f32 in rows: + ndcg_loss = (f32["ndcg_at_k"] or 0) - (r["ndcg_at_k"] or 0) + print(f"{col:<20} {r['codec']:<15} {r['bytes_per_vector']:>6} {r['rank_noise']:>8.5f} " + f"{r['rank_noise'] / r['neighbour_band']:>10.3f} {r['recall_at_k']:>9.4f} {ndcg_loss:>9.4f}") + lossy = [(c, r, f) for c, r, f in rows if r["rank_noise"] > 0] + ratio = [r["rank_noise"] / r["neighbour_band"] for _, r, _ in lossy] + noise = [r["rank_noise"] for _, r, _ in lossy] + loss = [1 - r["recall_at_k"] for _, r, _ in lossy] + ndcg = [(f["ndcg_at_k"] or 0) - (r["ndcg_at_k"] or 0) for _, r, f in lossy] + print() + print(f"{len(lossy)} (collection, codec) points") + print(f"spearman(noise/band, 1 - recall@k) = {spearman(ratio, loss):.3f}") + print(f"spearman(noise alone, 1 - recall@k) = {spearman(noise, loss):.3f}") + print(f"spearman(noise/band, ndcg loss) = {spearman(ratio, ndcg):.3f}") + print(f"spearman(noise alone, ndcg loss) = {spearman(noise, ndcg):.3f}") + # Within each codec: does the corpus with the tighter band lose more? + print() + print("within each codec, across collections: spearman(band, 1 - recall@k)") + for codec in sorted({r["codec"] for _, r, _ in lossy}): + pts = [(r["neighbour_band"], 1 - r["recall_at_k"]) for _, r, _ in lossy if r["codec"] == codec] + if len(pts) >= 3: + print(f" {codec:<15} {spearman([p[0] for p in pts], [p[1] for p in pts]):>7.3f} (n={len(pts)})") + + +if __name__ == "__main__": + main(Path(sys.argv[1]), sys.argv[2]) diff --git a/experiments/2026-11-two-stage-rescoring/README.md b/experiments/2026-11-two-stage-rescoring/README.md new file mode 100644 index 0000000..44468b2 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/README.md @@ -0,0 +1,108 @@ +# Two-stage retrieval: how deep must the re-score go? + +The neighbour-spacing experiment found that a codec's loss of the exact top 10 is governed by its +noise ÷ the corpus's neighbour band. The standard way to win the loss back is *hybrid* +retrieval: scan every vector with a small code, then re-score the best N candidates with a +bigger representation and keep the top 10. This experiment measures how large N must be, and +whether the same ratio predicts it. + +**Hypothesis.** A codec with noise σ displaces a true neighbour past about (σ ÷ band) × k +competitors, so the candidate depth an exact re-score needs for a target recall grows with the +ratio, roughly in proportion to it. Re-scoring a few times k candidates will recover nearly all +of the loss at a small fraction of an f32 scan's bytes. + +**Result.** Confirmed, with one correction. Across 50 (collection, first stage) points, the +depth needed tracks the ratio with Spearman ρ ≈ 0.93 at every target. It is proportional at +90% recall (depth ≈ 3.1·k·r^1.06) but grows faster than proportional for higher targets +(r^1.29 at 95%, r^1.74 at 99%): the last neighbours are in the error distribution's tail. Fitted +on four corpora, the curve predicts the fifth's depth within a factor of 1.25 (90%) to 1.41 +(99%) on average. At 48 bytes per vector, re-scoring 50 candidates in f32 recovers 0.948 of the +top 10 for `binary` and 0.986 for `pq48`, and removes essentially all of the nDCG loss, for +3.2 to 3.8% of the bytes an f32 scan reads. An int8 second stage is just as good up to depth +about 30, then stops at int8's own recall of 0.990. + +## Setup + +| | | +|---|---| +| Collections | The ten from [neighbour spacing](../2026-11-neighbour-spacing/): SciFact, NFCorpus, ArguAna, CoSQA, FiQA × bge-small-en-v1.5, all-MiniLM-L6-v2 (384 dims, 200-word chunks) | +| Queries | Each set's judged test queries | +| First stage (scanned) | `binary`, `binary-centred` (48 B), `pq96`, `pq48`, `pq24` | +| Second stage (candidates only) | `int8` (392 B), `f32` (1,536 B) | +| Depths | 10, 20, 30, 50, 100, 200, 500, 1000 | +| k | 10 | +| Depth for a target recall | With an exact second stage, recall at depth N is the fraction of true top-10 neighbours the first stage ranks within N. The depth for target t is the t-quantile of those ranks, pooled over queries. It is exact, not interpolated from the sweep | +| Search | brute force, no index | +| Hardware | Apple M4 Pro, release build | +| Command | `./run.sh` (reuses the spacing experiment's collections; builds them if missing) | + +New command: `hvec bench rescore --collection --codecs --rescore --depths …`. + +## Results + +### Depth needed, as a multiple of k + +f32 second stage. The full table, all 50 points, is in `results/summary.txt`. + +| collection | first stage | noise/band | scan alone | 90% | 95% | 99% | +|---|---|---:|---:|---:|---:|---:| +| ArguAna, MiniLM | pq96 | 0.20 | 0.924 | 1.0 | 1.1 | 1.4 | +| SciFact, bge-small | pq48 | 0.83 | 0.711 | 2.1 | 3.1 | 5.9 | +| SciFact, bge-small | binary | 1.39 | 0.611 | 4.0 | 7.1 | 19.2 | +| FiQA, bge-small | binary | 1.65 | 0.537 | 6.0 | 10.5 | 32.7 | +| NFCorpus, bge-small | binary | 1.65 | 0.556 | 6.4 | 11.6 | 42.4 | +| FiQA, bge-small | pq24 | 1.35 | 0.445 | 9.8 | 16.5 | 45.3 | + +| target recall | fit, depth / k | Spearman ρ | held-out corpus error (geometric mean factor) | +|---|---|---:|---:| +| 90% | 3.07 × r^1.06 | 0.930 | 1.25 | +| 95% | 4.76 × r^1.29 | 0.934 | 1.31 | +| 99% | 11.22 × r^1.74 | 0.925 | 1.41 | + +### Recall@10 and nDCG loss by depth (means over the ten collections) + +| first stage | second | depth 10 | 20 | 30 | 50 | 100 | 1000 | nDCG loss at depth 10 → 50 | +|---|---|---:|---:|---:|---:|---:|---:|---| +| binary (48 B) | f32 | 0.669 | 0.841 | 0.900 | 0.948 | 0.981 | 1.000 | 0.0138 → 0.0003 | +| binary (48 B) | int8 | 0.669 | 0.839 | 0.898 | 0.944 | 0.974 | 0.991 | 0.0138 → 0.0004 | +| binary-centred (48 B) | f32 | 0.698 | 0.869 | 0.924 | 0.966 | 0.991 | 1.000 | 0.0125 → 0.0002 | +| pq48 (48 B) | f32 | 0.747 | 0.913 | 0.959 | 0.986 | 0.998 | 1.000 | 0.0078 → 0.0002 | +| pq48 (48 B) | int8 | 0.747 | 0.910 | 0.954 | 0.979 | 0.989 | 0.991 | 0.0078 → 0.0002 | +| pq24 (24 B) | f32 | 0.606 | 0.779 | 0.855 | 0.918 | 0.968 | 1.000 | 0.0309 → 0.0046 | +| pq96 (96 B) | f32 | 0.870 | 0.986 | 0.997 | 1.000 | 1.000 | 1.000 | 0.0005 → −0.0001 | + +Depth 10 re-scores the scan's own top 10. It cannot change which 10 are returned, but it fixes +their order, so nDCG already recovers part of the way: binary's scan alone loses 0.024 nDCG +on average, and that halves to 0.014. + +**Cost.** A 48-byte scan of every vector reads 3.1% of the bytes an f32 scan reads. Re-scoring +100 f32 candidates adds 150 KB per query. On FiQA's 71,274 chunks that brings the total to +3.3% of an f32 scan; on SciFact's 8,036 chunks, 4.4%. The scan dominates, so the re-score is +nearly free on large collections and only matters on small ones. + +## Discussion + +**The ratio predicts how deep to go, not just how much is lost.** The same number that +predicted the loss in the spacing experiment predicts the re-score depth. One f32 pass gives +the band, the codec's noise is roughly fixed per (codec, model), and the curve gives the depth +to within about a third. In practice: at 48 bytes on bge-small, plan for 50 to 100 candidates +with `binary` and 30 to 50 with `pq48`. MiniLM needs about half of that. + +**Why the exponent grows with the target.** At 90% the depth is proportional to the ratio, +which is the simple picture: noise σ moves a neighbour past σ ÷ band × k competitors. Reaching +99% means catching the neighbours the codec displaced furthest, which are in the tail of its +error. They also fall into the sparser region below rank 2k, where the band is a worse guide +to density. + +**int8 is the right second stage only up to a point.** An int8 re-score matches f32 up to a +depth of about 30. After that, int8's own ranking error (0.990 of the top 10) becomes the +ceiling. If 99% or more is the target, re-score in f32. The bytes are the same at these depths. + +**Caveats.** +- **Brute-force scan.** With an index (HNSW, IVF) the first stage is no longer a full scan, and + the cost side changes completely. The depth needed should not change. +- **f32 must still be stored somewhere.** The disk cost of the second stage is the full 1,536 + bytes per vector. Only the scan's 48 bytes need to be in memory. +- **The same queries and collections as the spacing experiment**, including CoSQA's duplicates. +- **Depth is fixed for every query.** Adaptive per-query depth is tested in the + [hybrid × adaptive matrix](../2026-11-hybrid-adaptive-matrix/). diff --git a/experiments/2026-11-two-stage-rescoring/results/arguana-bge-small-k10.json b/experiments/2026-11-two-stage-rescoring/results/arguana-bge-small-k10.json new file mode 100644 index 0000000..09fc1f1 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/arguana-bge-small-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 543056, + "depth": 10, + "doc_recall_at_k": 0.7802275960170697, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4091630405738321, + "queries": 1406, + "recall_at_k": 0.7481507823613063, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 546976, + "depth": 20, + "doc_recall_at_k": 0.817923186344239, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094676051288715, + "queries": 1406, + "recall_at_k": 0.9161450924608824, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 550896, + "depth": 30, + "doc_recall_at_k": 0.817923186344239, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4212636778463139, + "queries": 1406, + "recall_at_k": 0.9532716927453779, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 558736, + "depth": 50, + "doc_recall_at_k": 0.8172119487908962, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4208425207456014, + "queries": 1406, + "recall_at_k": 0.9766002844950221, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 578336, + "depth": 100, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4212198358866503, + "queries": 1406, + "recall_at_k": 0.9881223328591763, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 617536, + "depth": 200, + "doc_recall_at_k": 0.817923186344239, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.9903271692745387, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 735136, + "depth": 500, + "doc_recall_at_k": 0.817923186344239, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 931136, + "depth": 1000, + "doc_recall_at_k": 0.817923186344239, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 554496, + "depth": 10, + "doc_recall_at_k": 0.7802275960170697, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4091771423430781, + "queries": 1406, + "recall_at_k": 0.7481507823613063, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 569856, + "depth": 20, + "doc_recall_at_k": 0.8193456614509246, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42153573564741575, + "queries": 1406, + "recall_at_k": 0.9196301564722622, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 585216, + "depth": 30, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42162957434336046, + "queries": 1406, + "recall_at_k": 0.9581792318634432, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 615936, + "depth": 50, + "doc_recall_at_k": 0.817923186344239, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42116155692232027, + "queries": 1406, + "recall_at_k": 0.9834281650071123, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 692736, + "depth": 100, + "doc_recall_at_k": 0.8193456614509246, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42158630990448837, + "queries": 1406, + "recall_at_k": 0.9963726884779516, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 846336, + "depth": 200, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.99900426742532, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1307136, + "depth": 500, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.9998577524893316, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2075136, + "depth": 1000, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 543056, + "depth": 10, + "doc_recall_at_k": 0.7930298719772404, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4130608432254461, + "queries": 1406, + "recall_at_k": 0.7726173541962991, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 546976, + "depth": 20, + "doc_recall_at_k": 0.8172119487908962, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4210602304318324, + "queries": 1406, + "recall_at_k": 0.9354907539118054, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 550896, + "depth": 30, + "doc_recall_at_k": 0.8193456614509246, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42154382680236324, + "queries": 1406, + "recall_at_k": 0.9687766714082521, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 558736, + "depth": 50, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4212339848284331, + "queries": 1406, + "recall_at_k": 0.9852062588904712, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 578336, + "depth": 100, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42116388796776005, + "queries": 1406, + "recall_at_k": 0.9899004267425333, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 617536, + "depth": 200, + "doc_recall_at_k": 0.817923186344239, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.9907539118065445, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 735136, + "depth": 500, + "doc_recall_at_k": 0.817923186344239, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 931136, + "depth": 1000, + "doc_recall_at_k": 0.817923186344239, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 554496, + "depth": 10, + "doc_recall_at_k": 0.7930298719772404, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.41316111179695614, + "queries": 1406, + "recall_at_k": 0.7726173541962991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 569856, + "depth": 20, + "doc_recall_at_k": 0.8172119487908962, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42125593800881284, + "queries": 1406, + "recall_at_k": 0.9396870554765278, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 585216, + "depth": 30, + "doc_recall_at_k": 0.8200568990042674, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42194060721385185, + "queries": 1406, + "recall_at_k": 0.975248933143671, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 615936, + "depth": 50, + "doc_recall_at_k": 0.8193456614509246, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42160896892404215, + "queries": 1406, + "recall_at_k": 0.9931721194879096, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 692736, + "depth": 100, + "doc_recall_at_k": 0.8193456614509246, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4215397347670436, + "queries": 1406, + "recall_at_k": 0.9985064011379802, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 846336, + "depth": 200, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.99950213371266, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1307136, + "depth": 500, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2075136, + "depth": 1000, + "doc_recall_at_k": 0.8186344238975818, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1082192, + "depth": 10, + "doc_recall_at_k": 0.8229018492176386, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42247982731046296, + "queries": 1406, + "recall_at_k": 0.9030583214793728, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1086112, + "depth": 20, + "doc_recall_at_k": 0.8172119487908962, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4207604591141277, + "queries": 1406, + "recall_at_k": 0.989331436699859, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1090032, + "depth": 30, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.9910384068278817, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1097872, + "depth": 50, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1117472, + "depth": 100, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1156672, + "depth": 200, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1274272, + "depth": 500, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1470272, + "depth": 1000, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1093632, + "depth": 10, + "doc_recall_at_k": 0.8229018492176386, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4226542131137411, + "queries": 1406, + "recall_at_k": 0.9030583214793728, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1108992, + "depth": 20, + "doc_recall_at_k": 0.817923186344239, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4211327458869339, + "queries": 1406, + "recall_at_k": 0.9975817923186345, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1124352, + "depth": 30, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.9997866287339972, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1155072, + "depth": 50, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1231872, + "depth": 100, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1385472, + "depth": 200, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1846272, + "depth": 500, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2614272, + "depth": 1000, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 543056, + "depth": 10, + "doc_recall_at_k": 0.8015647226173542, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.41618718996770426, + "queries": 1406, + "recall_at_k": 0.8109530583214749, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 546976, + "depth": 20, + "doc_recall_at_k": 0.8165007112375533, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4206369794377869, + "queries": 1406, + "recall_at_k": 0.9633001422475114, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 550896, + "depth": 30, + "doc_recall_at_k": 0.8172119487908962, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42074950694857155, + "queries": 1406, + "recall_at_k": 0.9817211948790917, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 558736, + "depth": 50, + "doc_recall_at_k": 0.817923186344239, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.989331436699859, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 578336, + "depth": 100, + "doc_recall_at_k": 0.817923186344239, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.9910384068278817, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 617536, + "depth": 200, + "doc_recall_at_k": 0.817923186344239, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 735136, + "depth": 500, + "doc_recall_at_k": 0.817923186344239, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 931136, + "depth": 1000, + "doc_recall_at_k": 0.817923186344239, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 554496, + "depth": 10, + "doc_recall_at_k": 0.8015647226173542, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4162572791135644, + "queries": 1406, + "recall_at_k": 0.8109530583214749, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 569856, + "depth": 20, + "doc_recall_at_k": 0.8172119487908962, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4210836397781796, + "queries": 1406, + "recall_at_k": 0.9678520625889054, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 585216, + "depth": 30, + "doc_recall_at_k": 0.817923186344239, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4211473610575817, + "queries": 1406, + "recall_at_k": 0.988904694167853, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 615936, + "depth": 50, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.9978662873399715, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 692736, + "depth": 100, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.9998577524893316, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 846336, + "depth": 200, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1307136, + "depth": 500, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2075136, + "depth": 1000, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 273488, + "depth": 10, + "doc_recall_at_k": 0.7638691322901849, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.40333502059121823, + "queries": 1406, + "recall_at_k": 0.7120910384068273, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9992887624466572, + "vectors": 11232 + }, + { + "bytes_read_per_query": 277408, + "depth": 20, + "doc_recall_at_k": 0.8108108108108109, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.41917841142513224, + "queries": 1406, + "recall_at_k": 0.8990042674253194, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 281328, + "depth": 30, + "doc_recall_at_k": 0.8172119487908962, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4211665012647166, + "queries": 1406, + "recall_at_k": 0.9421052631578958, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 289168, + "depth": 50, + "doc_recall_at_k": 0.8157894736842105, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4205337735272126, + "queries": 1406, + "recall_at_k": 0.9735419630156483, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 308768, + "depth": 100, + "doc_recall_at_k": 0.817923186344239, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42099366194480314, + "queries": 1406, + "recall_at_k": 0.9871266002844967, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 347968, + "depth": 200, + "doc_recall_at_k": 0.817923186344239, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.9905405405405419, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 465568, + "depth": 500, + "doc_recall_at_k": 0.817923186344239, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.9910384068278817, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 661568, + "depth": 1000, + "doc_recall_at_k": 0.817923186344239, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42094978413016126, + "queries": 1406, + "recall_at_k": 0.991109530583216, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 284928, + "depth": 10, + "doc_recall_at_k": 0.7638691322901849, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4034627088742816, + "queries": 1406, + "recall_at_k": 0.7120910384068273, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9992887624466572, + "vectors": 11232 + }, + { + "bytes_read_per_query": 300288, + "depth": 20, + "doc_recall_at_k": 0.8108108108108109, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.419432075680669, + "queries": 1406, + "recall_at_k": 0.9010668563300135, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 315648, + "depth": 30, + "doc_recall_at_k": 0.8193456614509246, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4219780903146152, + "queries": 1406, + "recall_at_k": 0.9467283072546238, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 346368, + "depth": 50, + "doc_recall_at_k": 0.8172119487908962, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.42113643288141184, + "queries": 1406, + "recall_at_k": 0.9800853485064016, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 423168, + "depth": 100, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213822174614036, + "queries": 1406, + "recall_at_k": 0.9952347083926038, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 576768, + "depth": 200, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.9991465149359887, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1037568, + "depth": 500, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 0.9999288762446658, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1805568, + "depth": 1000, + "doc_recall_at_k": 0.8186344238975818, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.4213256309294448, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 18 + ], + [ + 0.95, + 28 + ], + [ + 0.99, + 62 + ], + [ + 1.0, + 606 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 19.0, + "metric": "cosine", + "neighbour_band": 0.023316502571105957, + "queries": 1406, + "rank_noise": 0.018319500828550522, + "recall_at_k": 0.7481507823613063, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 16 + ], + [ + 0.95, + 22 + ], + [ + 0.99, + 45 + ], + [ + 1.0, + 274 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 17.0, + "metric": "cosine", + "neighbour_band": 0.023316502571105957, + "queries": 1406, + "rank_noise": 0.01837565346242917, + "recall_at_k": 0.7726173541962991, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 10 + ], + [ + 0.95, + 12 + ], + [ + 0.99, + 16 + ], + [ + 1.0, + 48 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 11.0, + "metric": "cosine", + "neighbour_band": 0.023316502571105957, + "queries": 1406, + "rank_noise": 0.006742915672471994, + "recall_at_k": 0.9030583214793728, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 14 + ], + [ + 0.95, + 17 + ], + [ + 0.99, + 32 + ], + [ + 1.0, + 191 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 15.0, + "metric": "cosine", + "neighbour_band": 0.023316502571105957, + "queries": 1406, + "rank_noise": 0.015867100950084683, + "recall_at_k": 0.8109530583214749, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 20 + ], + [ + 0.95, + 32 + ], + [ + 0.99, + 72 + ], + [ + 1.0, + 568 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 22.0, + "metric": "cosine", + "neighbour_band": 0.023316502571105957, + "queries": 1406, + "rank_noise": 0.027925833599693847, + "recall_at_k": 0.7120910384068273, + "vectors": 11232 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/arguana-minilm-k10.json b/experiments/2026-11-two-stage-rescoring/results/arguana-minilm-k10.json new file mode 100644 index 0000000..e3285ca --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/arguana-minilm-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 543056, + "depth": 10, + "doc_recall_at_k": 0.7041251778093883, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3481569593852496, + "queries": 1406, + "recall_at_k": 0.814793741109527, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 546976, + "depth": 20, + "doc_recall_at_k": 0.732574679943101, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3581474655994647, + "queries": 1406, + "recall_at_k": 0.970768136557611, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 550896, + "depth": 30, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3614102218267703, + "queries": 1406, + "recall_at_k": 0.9904694167852075, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 558736, + "depth": 50, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9965149359886206, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 578336, + "depth": 100, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.998150782361309, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 617536, + "depth": 200, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.998150782361309, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 735136, + "depth": 500, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 931136, + "depth": 1000, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 554496, + "depth": 10, + "doc_recall_at_k": 0.7041251778093883, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.34811419755475026, + "queries": 1406, + "recall_at_k": 0.814793741109527, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 569856, + "depth": 20, + "doc_recall_at_k": 0.732574679943101, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3580681183914817, + "queries": 1406, + "recall_at_k": 0.9716927453769566, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 585216, + "depth": 30, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36136060349164634, + "queries": 1406, + "recall_at_k": 0.9921052631578958, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 615936, + "depth": 50, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9982930298719773, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 692736, + "depth": 100, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9999288762446658, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 846336, + "depth": 200, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9999288762446658, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1307136, + "depth": 500, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2075136, + "depth": 1000, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 543056, + "depth": 10, + "doc_recall_at_k": 0.7162162162162162, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3522540235317442, + "queries": 1406, + "recall_at_k": 0.8239687055476486, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 546976, + "depth": 20, + "doc_recall_at_k": 0.7375533428165008, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35989378316642145, + "queries": 1406, + "recall_at_k": 0.9743243243243254, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 550896, + "depth": 30, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3609492497444816, + "queries": 1406, + "recall_at_k": 0.9911806543385498, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 558736, + "depth": 50, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9970839260312947, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 578336, + "depth": 100, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 617536, + "depth": 200, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 735136, + "depth": 500, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 931136, + "depth": 1000, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 554496, + "depth": 10, + "doc_recall_at_k": 0.7162162162162162, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3522152962698992, + "queries": 1406, + "recall_at_k": 0.8239687055476486, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 569856, + "depth": 20, + "doc_recall_at_k": 0.7389758179231863, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3602468422731821, + "queries": 1406, + "recall_at_k": 0.9753200568990052, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 585216, + "depth": 30, + "doc_recall_at_k": 0.7411095305832148, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36089963140935766, + "queries": 1406, + "recall_at_k": 0.9926742532005697, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 615936, + "depth": 50, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9987908961593174, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 692736, + "depth": 100, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 846336, + "depth": 200, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1307136, + "depth": 500, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2075136, + "depth": 1000, + "doc_recall_at_k": 0.7418207681365576, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1082192, + "depth": 10, + "doc_recall_at_k": 0.7304409672830725, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35792687410466095, + "queries": 1406, + "recall_at_k": 0.9241109530583209, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1086112, + "depth": 20, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36137905272944537, + "queries": 1406, + "recall_at_k": 0.9972972972972978, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1090032, + "depth": 30, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1097872, + "depth": 50, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1117472, + "depth": 100, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1156672, + "depth": 200, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1274272, + "depth": 500, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1470272, + "depth": 1000, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1093632, + "depth": 10, + "doc_recall_at_k": 0.7304409672830725, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3578754992177762, + "queries": 1406, + "recall_at_k": 0.9241109530583209, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1108992, + "depth": 20, + "doc_recall_at_k": 0.7425320056899004, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36153502815414923, + "queries": 1406, + "recall_at_k": 0.9990753911806546, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1124352, + "depth": 30, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1155072, + "depth": 50, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1231872, + "depth": 100, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1385472, + "depth": 200, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1846272, + "depth": 500, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2614272, + "depth": 1000, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 543056, + "depth": 10, + "doc_recall_at_k": 0.7261735419630156, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35640126641161035, + "queries": 1406, + "recall_at_k": 0.8486486486486458, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9992887624466572, + "vectors": 11232 + }, + { + "bytes_read_per_query": 546976, + "depth": 20, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36119873128612784, + "queries": 1406, + "recall_at_k": 0.9835704125177823, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 550896, + "depth": 30, + "doc_recall_at_k": 0.7403982930298719, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36099615713487665, + "queries": 1406, + "recall_at_k": 0.9953769559032724, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 558736, + "depth": 50, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9980085348506403, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 578336, + "depth": 100, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 617536, + "depth": 200, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 735136, + "depth": 500, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 931136, + "depth": 1000, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 554496, + "depth": 10, + "doc_recall_at_k": 0.7261735419630156, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35638465179293977, + "queries": 1406, + "recall_at_k": 0.8486486486486458, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9992887624466572, + "vectors": 11232 + }, + { + "bytes_read_per_query": 569856, + "depth": 20, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611576230287749, + "queries": 1406, + "recall_at_k": 0.9849928876244678, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 585216, + "depth": 30, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611521325595805, + "queries": 1406, + "recall_at_k": 0.9970839260312948, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 615936, + "depth": 50, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9997866287339972, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 692736, + "depth": 100, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 846336, + "depth": 200, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1307136, + "depth": 500, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 2075136, + "depth": 1000, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 273488, + "depth": 10, + "doc_recall_at_k": 0.6749644381223329, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3368502077175326, + "queries": 1406, + "recall_at_k": 0.7433854907539089, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9957325746799431, + "vectors": 11232 + }, + { + "bytes_read_per_query": 277408, + "depth": 20, + "doc_recall_at_k": 0.7332859174964438, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35824525041819566, + "queries": 1406, + "recall_at_k": 0.9285206258890477, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9978662873399715, + "vectors": 11232 + }, + { + "bytes_read_per_query": 281328, + "depth": 30, + "doc_recall_at_k": 0.7354196301564723, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35893841861016257, + "queries": 1406, + "recall_at_k": 0.9699146514936003, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 289168, + "depth": 50, + "doc_recall_at_k": 0.7396870554765291, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36030920976583214, + "queries": 1406, + "recall_at_k": 0.9886913229018499, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 308768, + "depth": 100, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9970839260312945, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 347968, + "depth": 200, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.998150782361309, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 465568, + "depth": 500, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 661568, + "depth": 1000, + "doc_recall_at_k": 0.7411095305832148, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.3611649488918465, + "queries": 1406, + "recall_at_k": 0.9982219061166433, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 284928, + "depth": 10, + "doc_recall_at_k": 0.6749644381223329, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.33678447054018457, + "queries": 1406, + "recall_at_k": 0.7433854907539089, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9957325746799431, + "vectors": 11232 + }, + { + "bytes_read_per_query": 300288, + "depth": 20, + "doc_recall_at_k": 0.7347083926031295, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.35856513342429924, + "queries": 1406, + "recall_at_k": 0.9293029871977248, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9978662873399715, + "vectors": 11232 + }, + { + "bytes_read_per_query": 315648, + "depth": 30, + "doc_recall_at_k": 0.7354196301564723, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.358861626288644, + "queries": 1406, + "recall_at_k": 0.9709815078236144, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 346368, + "depth": 50, + "doc_recall_at_k": 0.7396870554765291, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36024932480117633, + "queries": 1406, + "recall_at_k": 0.9901849217638696, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 423168, + "depth": 100, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9988620199146513, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 576768, + "depth": 200, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 0.9999288762446658, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1037568, + "depth": 500, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + }, + { + "bytes_read_per_query": 1805568, + "depth": 1000, + "doc_recall_at_k": 0.7418207681365576, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 1406, + "metric": "cosine", + "ndcg_at_k": 0.36132092431655044, + "queries": 1406, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 11232 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 13 + ], + [ + 0.95, + 17 + ], + [ + 0.99, + 28 + ], + [ + 1.0, + 206 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 14.0, + "metric": "cosine", + "neighbour_band": 0.052856624126434326, + "queries": 1406, + "rank_noise": 0.027135932180800144, + "recall_at_k": 0.814793741109527, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 13 + ], + [ + 0.95, + 16 + ], + [ + 0.99, + 27 + ], + [ + 1.0, + 88 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 14.0, + "metric": "cosine", + "neighbour_band": 0.052856624126434326, + "queries": 1406, + "rank_noise": 0.027019165604858963, + "recall_at_k": 0.8239687055476486, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 10 + ], + [ + 0.95, + 11 + ], + [ + 0.99, + 14 + ], + [ + 1.0, + 27 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 11.0, + "metric": "cosine", + "neighbour_band": 0.052856624126434326, + "queries": 1406, + "rank_noise": 0.010672786260429074, + "recall_at_k": 0.9241109530583209, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 12 + ], + [ + 0.95, + 15 + ], + [ + 0.99, + 23 + ], + [ + 1.0, + 82 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 13.0, + "metric": "cosine", + "neighbour_band": 0.052856624126434326, + "queries": 1406, + "rank_noise": 0.02566689404324917, + "recall_at_k": 0.8486486486486458, + "vectors": 11232 + }, + { + "depth_for_recall": [ + [ + 0.9, + 17 + ], + [ + 0.95, + 24 + ], + [ + 0.99, + 49 + ], + [ + 1.0, + 283 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 18.0, + "metric": "cosine", + "neighbour_band": 0.052856624126434326, + "queries": 1406, + "rank_noise": 0.04914630085769442, + "recall_at_k": 0.7433854907539089, + "vectors": 11232 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/config.toml b/experiments/2026-11-two-stage-rescoring/results/config.toml new file mode 100644 index 0000000..c0cbbfe --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/config.toml @@ -0,0 +1,17 @@ +db_path = "/Users/liam/dev/hvec/experiments/2026-11-neighbour-spacing/work/hvec.db" +default_chat = "anthropic" +default_embedder = "bge-small" +metric = "cosine" + +[chat.anthropic] +provider = "anthropic" +model = "claude-opus-5-5" +api_key_env = "ANTHROPIC_API_KEY" + +[embed.bge-small] +provider = "local" +model = "BGESmallENV15" + +[embed.minilm] +provider = "local" +model = "AllMiniLML6V2" diff --git a/experiments/2026-11-two-stage-rescoring/results/cosqa-bge-small-k10.json b/experiments/2026-11-two-stage-rescoring/results/cosqa-bge-small-k10.json new file mode 100644 index 0000000..b168720 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/cosqa-bge-small-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 995024, + "depth": 10, + "doc_recall_at_k": 0.474, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26154368569291836, + "queries": 500, + "recall_at_k": 0.6598000000000006, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.86, + "vectors": 20648 + }, + { + "bytes_read_per_query": 998944, + "depth": 20, + "doc_recall_at_k": 0.488, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2702669986909615, + "queries": 500, + "recall_at_k": 0.831, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.938, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1002864, + "depth": 30, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2686692777730173, + "queries": 500, + "recall_at_k": 0.8963999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.962, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1010704, + "depth": 50, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27114672989209054, + "queries": 500, + "recall_at_k": 0.9491999999999996, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.986, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1030304, + "depth": 100, + "doc_recall_at_k": 0.492, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26968185001098427, + "queries": 500, + "recall_at_k": 0.9747999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1069504, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9833999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1187104, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9889999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1383104, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1006464, + "depth": 10, + "doc_recall_at_k": 0.474, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2614050388090651, + "queries": 500, + "recall_at_k": 0.6598000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.86, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1021824, + "depth": 20, + "doc_recall_at_k": 0.486, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2697005038212455, + "queries": 500, + "recall_at_k": 0.8334, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1037184, + "depth": 30, + "doc_recall_at_k": 0.488, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2681801385469015, + "queries": 500, + "recall_at_k": 0.8995999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.966, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1067904, + "depth": 50, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2713304202136745, + "queries": 500, + "recall_at_k": 0.9565999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.992, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1144704, + "depth": 100, + "doc_recall_at_k": 0.492, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26986554033256815, + "queries": 500, + "recall_at_k": 0.9833999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1298304, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 0.9938, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1759104, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 0.9994000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2527104, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 995024, + "depth": 10, + "doc_recall_at_k": 0.466, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.25794987042930617, + "queries": 500, + "recall_at_k": 0.7252, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.874, + "vectors": 20648 + }, + { + "bytes_read_per_query": 998944, + "depth": 20, + "doc_recall_at_k": 0.482, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2641119752169854, + "queries": 500, + "recall_at_k": 0.8637999999999995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.936, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1002864, + "depth": 30, + "doc_recall_at_k": 0.488, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2670284566923521, + "queries": 500, + "recall_at_k": 0.9103999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.964, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1010704, + "depth": 50, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2708119297851257, + "queries": 500, + "recall_at_k": 0.9541999999999995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.982, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1030304, + "depth": 100, + "doc_recall_at_k": 0.488, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26810372035834845, + "queries": 500, + "recall_at_k": 0.9827999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1069504, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9879999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1187104, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1383104, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1006464, + "depth": 10, + "doc_recall_at_k": 0.466, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2580627911945183, + "queries": 500, + "recall_at_k": 0.7252, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.876, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1021824, + "depth": 20, + "doc_recall_at_k": 0.482, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2644027044863718, + "queries": 500, + "recall_at_k": 0.8685999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.94, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1037184, + "depth": 30, + "doc_recall_at_k": 0.488, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2673191859617386, + "queries": 500, + "recall_at_k": 0.9171999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1067904, + "depth": 50, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2710602267820483, + "queries": 500, + "recall_at_k": 0.9627999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1144704, + "depth": 100, + "doc_recall_at_k": 0.488, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2682874106799324, + "queries": 500, + "recall_at_k": 0.9926, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1298304, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 0.9977999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1759104, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2527104, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1986128, + "depth": 10, + "doc_recall_at_k": 0.492, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2697157830455682, + "queries": 500, + "recall_at_k": 0.8775999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.958, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1990048, + "depth": 20, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26916145988283546, + "queries": 500, + "recall_at_k": 0.9691999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.99, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1993968, + "depth": 30, + "doc_recall_at_k": 0.492, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2697346501119199, + "queries": 500, + "recall_at_k": 0.9831999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2001808, + "depth": 50, + "doc_recall_at_k": 0.492, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2697057803496764, + "queries": 500, + "recall_at_k": 0.9885999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2021408, + "depth": 100, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9889999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2060608, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2178208, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2374208, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1997568, + "depth": 10, + "doc_recall_at_k": 0.492, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2699776425527111, + "queries": 500, + "recall_at_k": 0.8775999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.96, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2012928, + "depth": 20, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26931628044217587, + "queries": 500, + "recall_at_k": 0.9773999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2028288, + "depth": 30, + "doc_recall_at_k": 0.492, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2699183404335038, + "queries": 500, + "recall_at_k": 0.9934, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2059008, + "depth": 50, + "doc_recall_at_k": 0.492, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26988947067126035, + "queries": 500, + "recall_at_k": 0.999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2135808, + "depth": 100, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 0.9994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2289408, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2750208, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 3518208, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 995024, + "depth": 10, + "doc_recall_at_k": 0.472, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2558418274851606, + "queries": 500, + "recall_at_k": 0.7727999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.894, + "vectors": 20648 + }, + { + "bytes_read_per_query": 998944, + "depth": 20, + "doc_recall_at_k": 0.494, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2697765499060482, + "queries": 500, + "recall_at_k": 0.9017999999999995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.954, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1002864, + "depth": 30, + "doc_recall_at_k": 0.492, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2699532669000922, + "queries": 500, + "recall_at_k": 0.9509999999999996, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.984, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1010704, + "depth": 50, + "doc_recall_at_k": 0.494, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2702839100023122, + "queries": 500, + "recall_at_k": 0.9735999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.988, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1030304, + "depth": 100, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9877999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1069504, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1187104, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1383104, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1006464, + "depth": 10, + "doc_recall_at_k": 0.472, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2561036869923035, + "queries": 500, + "recall_at_k": 0.7727999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.896, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1021824, + "depth": 20, + "doc_recall_at_k": 0.494, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2699313704653887, + "queries": 500, + "recall_at_k": 0.9069999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.96, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1037184, + "depth": 30, + "doc_recall_at_k": 0.492, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2701369572216762, + "queries": 500, + "recall_at_k": 0.9571999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1067904, + "depth": 50, + "doc_recall_at_k": 0.494, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2704676003238962, + "queries": 500, + "recall_at_k": 0.9817999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1144704, + "depth": 100, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 0.9973999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1298304, + "depth": 200, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1759104, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2527104, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 499472, + "depth": 10, + "doc_recall_at_k": 0.462, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2552732755431611, + "queries": 500, + "recall_at_k": 0.6733999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.79, + "vectors": 20648 + }, + { + "bytes_read_per_query": 503392, + "depth": 20, + "doc_recall_at_k": 0.46, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2551116057297482, + "queries": 500, + "recall_at_k": 0.7922, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.87, + "vectors": 20648 + }, + { + "bytes_read_per_query": 507312, + "depth": 30, + "doc_recall_at_k": 0.464, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2564054037979404, + "queries": 500, + "recall_at_k": 0.8610000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.91, + "vectors": 20648 + }, + { + "bytes_read_per_query": 515152, + "depth": 50, + "doc_recall_at_k": 0.482, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26439293603700714, + "queries": 500, + "recall_at_k": 0.9195999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.95, + "vectors": 20648 + }, + { + "bytes_read_per_query": 534752, + "depth": 100, + "doc_recall_at_k": 0.486, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26786579118989773, + "queries": 500, + "recall_at_k": 0.9639999999999994, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.982, + "vectors": 20648 + }, + { + "bytes_read_per_query": 573952, + "depth": 200, + "doc_recall_at_k": 0.492, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2697057803496764, + "queries": 500, + "recall_at_k": 0.9821999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 691552, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9875999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 887552, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26910372035834845, + "queries": 500, + "recall_at_k": 0.9895999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 510912, + "depth": 10, + "doc_recall_at_k": 0.462, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2551856280414834, + "queries": 500, + "recall_at_k": 0.6733999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.792, + "vectors": 20648 + }, + { + "bytes_read_per_query": 526272, + "depth": 20, + "doc_recall_at_k": 0.458, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2544449390630815, + "queries": 500, + "recall_at_k": 0.7964, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.874, + "vectors": 20648 + }, + { + "bytes_read_per_query": 541632, + "depth": 30, + "doc_recall_at_k": 0.464, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2566315263919881, + "queries": 500, + "recall_at_k": 0.8662000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.914, + "vectors": 20648 + }, + { + "bytes_read_per_query": 572352, + "depth": 50, + "doc_recall_at_k": 0.482, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26468366530639353, + "queries": 500, + "recall_at_k": 0.9254, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.956, + "vectors": 20648 + }, + { + "bytes_read_per_query": 649152, + "depth": 100, + "doc_recall_at_k": 0.486, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2680494815114817, + "queries": 500, + "recall_at_k": 0.9707999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20648 + }, + { + "bytes_read_per_query": 802752, + "depth": 200, + "doc_recall_at_k": 0.492, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26988947067126035, + "queries": 500, + "recall_at_k": 0.9903999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1263552, + "depth": 500, + "doc_recall_at_k": 0.49, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 0.998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2031552, + "depth": 1000, + "doc_recall_at_k": 0.49, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2692874106799324, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 31 + ], + [ + 0.95, + 47 + ], + [ + 0.99, + 132 + ], + [ + 1.0, + 820 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 18.0, + "metric": "cosine", + "neighbour_band": 0.014807552099227905, + "queries": 500, + "rank_noise": 0.014175589010162073, + "recall_at_k": 0.6598000000000006, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 25 + ], + [ + 0.95, + 42 + ], + [ + 0.99, + 89 + ], + [ + 1.0, + 388 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 15.0, + "metric": "cosine", + "neighbour_band": 0.014807552099227905, + "queries": 500, + "rank_noise": 0.010298020049629607, + "recall_at_k": 0.7252, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 12 + ], + [ + 0.95, + 16 + ], + [ + 0.99, + 27 + ], + [ + 1.0, + 113 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 10.0, + "metric": "cosine", + "neighbour_band": 0.014807552099227905, + "queries": 500, + "rank_noise": 0.004824377917958364, + "recall_at_k": 0.8775999999999999, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 20 + ], + [ + 0.95, + 28 + ], + [ + 0.99, + 68 + ], + [ + 1.0, + 157 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 13.0, + "metric": "cosine", + "neighbour_band": 0.014807552099227905, + "queries": 500, + "rank_noise": 0.009104592511394003, + "recall_at_k": 0.7727999999999999, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 38 + ], + [ + 0.95, + 69 + ], + [ + 0.99, + 190 + ], + [ + 1.0, + 705 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 23.0, + "metric": "cosine", + "neighbour_band": 0.014807552099227905, + "queries": 500, + "rank_noise": 0.014045970600847607, + "recall_at_k": 0.6733999999999999, + "vectors": 20648 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/cosqa-minilm-k10.json b/experiments/2026-11-two-stage-rescoring/results/cosqa-minilm-k10.json new file mode 100644 index 0000000..f45a7d8 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/cosqa-minilm-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 995024, + "depth": 10, + "doc_recall_at_k": 0.472, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.265173614834576, + "queries": 500, + "recall_at_k": 0.7190000000000004, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.908, + "vectors": 20648 + }, + { + "bytes_read_per_query": 998944, + "depth": 20, + "doc_recall_at_k": 0.498, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2793901437116567, + "queries": 500, + "recall_at_k": 0.8735999999999995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.966, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1002864, + "depth": 30, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2762814278938833, + "queries": 500, + "recall_at_k": 0.9303999999999993, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.986, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1010704, + "depth": 50, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2766292580327331, + "queries": 500, + "recall_at_k": 0.9691999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.996, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1030304, + "depth": 100, + "doc_recall_at_k": 0.494, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27783831743894033, + "queries": 500, + "recall_at_k": 0.9910000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1069504, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9942000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1187104, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9956, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1383104, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1006464, + "depth": 10, + "doc_recall_at_k": 0.472, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.265173614834576, + "queries": 500, + "recall_at_k": 0.7190000000000004, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.908, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1021824, + "depth": 20, + "doc_recall_at_k": 0.498, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2793901437116567, + "queries": 500, + "recall_at_k": 0.8763999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.968, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1037184, + "depth": 30, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2762814278938833, + "queries": 500, + "recall_at_k": 0.9341999999999991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.988, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1067904, + "depth": 50, + "doc_recall_at_k": 0.49, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2766292580327331, + "queries": 500, + "recall_at_k": 0.9729999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1144704, + "depth": 100, + "doc_recall_at_k": 0.494, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27783831743894033, + "queries": 500, + "recall_at_k": 0.9948, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1298304, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1759104, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2527104, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 995024, + "depth": 10, + "doc_recall_at_k": 0.468, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2646193626391272, + "queries": 500, + "recall_at_k": 0.7246000000000006, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.906, + "vectors": 20648 + }, + { + "bytes_read_per_query": 998944, + "depth": 20, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2754095714284843, + "queries": 500, + "recall_at_k": 0.8913999999999994, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.958, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1002864, + "depth": 30, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27699094905805133, + "queries": 500, + "recall_at_k": 0.9371999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.976, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1010704, + "depth": 50, + "doc_recall_at_k": 0.498, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2778243978947023, + "queries": 500, + "recall_at_k": 0.9709999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.988, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1030304, + "depth": 100, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9899999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1069504, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9958, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1187104, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1383104, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1006464, + "depth": 10, + "doc_recall_at_k": 0.468, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2646193626391272, + "queries": 500, + "recall_at_k": 0.7246000000000006, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.906, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1021824, + "depth": 20, + "doc_recall_at_k": 0.49, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2754095714284843, + "queries": 500, + "recall_at_k": 0.8933999999999994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.958, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1037184, + "depth": 30, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27699094905805133, + "queries": 500, + "recall_at_k": 0.9403999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1067904, + "depth": 50, + "doc_recall_at_k": 0.498, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2778243978947023, + "queries": 500, + "recall_at_k": 0.9747999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1144704, + "depth": 100, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9937999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1298304, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1759104, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2527104, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1986128, + "depth": 10, + "doc_recall_at_k": 0.488, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27508585193205004, + "queries": 500, + "recall_at_k": 0.8969999999999995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.978, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1990048, + "depth": 20, + "doc_recall_at_k": 0.5, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2797922126973732, + "queries": 500, + "recall_at_k": 0.9735999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.996, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1993968, + "depth": 30, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9909999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2001808, + "depth": 50, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.996, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2021408, + "depth": 100, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2060608, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2178208, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2374208, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1997568, + "depth": 10, + "doc_recall_at_k": 0.488, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27508585193205004, + "queries": 500, + "recall_at_k": 0.8969999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.978, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2012928, + "depth": 20, + "doc_recall_at_k": 0.5, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2797922126973732, + "queries": 500, + "recall_at_k": 0.9763999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2028288, + "depth": 30, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9947999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2059008, + "depth": 50, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9997999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2135808, + "depth": 100, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2289408, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2750208, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 3518208, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 995024, + "depth": 10, + "doc_recall_at_k": 0.472, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26668662397669896, + "queries": 500, + "recall_at_k": 0.775, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9, + "vectors": 20648 + }, + { + "bytes_read_per_query": 998944, + "depth": 20, + "doc_recall_at_k": 0.48, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2721281668338113, + "queries": 500, + "recall_at_k": 0.8955999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.962, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1002864, + "depth": 30, + "doc_recall_at_k": 0.492, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2772580748164696, + "queries": 500, + "recall_at_k": 0.9571999999999996, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1010704, + "depth": 50, + "doc_recall_at_k": 0.494, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2778840651464898, + "queries": 500, + "recall_at_k": 0.9857999999999996, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1030304, + "depth": 100, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9956, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1069504, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1187104, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1383104, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1006464, + "depth": 10, + "doc_recall_at_k": 0.472, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.26668662397669896, + "queries": 500, + "recall_at_k": 0.775, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1021824, + "depth": 20, + "doc_recall_at_k": 0.48, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2721281668338113, + "queries": 500, + "recall_at_k": 0.8965999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.962, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1037184, + "depth": 30, + "doc_recall_at_k": 0.492, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2772580748164696, + "queries": 500, + "recall_at_k": 0.9603999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1067904, + "depth": 50, + "doc_recall_at_k": 0.494, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2778840651464898, + "queries": 500, + "recall_at_k": 0.9895999999999996, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.996, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1144704, + "depth": 100, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9994, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1298304, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1759104, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2527104, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 499472, + "depth": 10, + "doc_recall_at_k": 0.436, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.25251043525077344, + "queries": 500, + "recall_at_k": 0.6399999999999997, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.786, + "vectors": 20648 + }, + { + "bytes_read_per_query": 503392, + "depth": 20, + "doc_recall_at_k": 0.484, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27171809454861007, + "queries": 500, + "recall_at_k": 0.7968000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.886, + "vectors": 20648 + }, + { + "bytes_read_per_query": 507312, + "depth": 30, + "doc_recall_at_k": 0.498, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2772306438001571, + "queries": 500, + "recall_at_k": 0.8784, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.942, + "vectors": 20648 + }, + { + "bytes_read_per_query": 515152, + "depth": 50, + "doc_recall_at_k": 0.492, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2762354470011055, + "queries": 500, + "recall_at_k": 0.9396, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.98, + "vectors": 20648 + }, + { + "bytes_read_per_query": 534752, + "depth": 100, + "doc_recall_at_k": 0.492, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2759555389726801, + "queries": 500, + "recall_at_k": 0.9792, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.992, + "vectors": 20648 + }, + { + "bytes_read_per_query": 573952, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9928, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.996, + "vectors": 20648 + }, + { + "bytes_read_per_query": 691552, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 887552, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9962000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 510912, + "depth": 10, + "doc_recall_at_k": 0.436, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.25251043525077344, + "queries": 500, + "recall_at_k": 0.6399999999999997, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.786, + "vectors": 20648 + }, + { + "bytes_read_per_query": 526272, + "depth": 20, + "doc_recall_at_k": 0.484, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.27171809454861007, + "queries": 500, + "recall_at_k": 0.7982000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.886, + "vectors": 20648 + }, + { + "bytes_read_per_query": 541632, + "depth": 30, + "doc_recall_at_k": 0.498, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2772306438001571, + "queries": 500, + "recall_at_k": 0.8811999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.944, + "vectors": 20648 + }, + { + "bytes_read_per_query": 572352, + "depth": 50, + "doc_recall_at_k": 0.492, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2762354470011055, + "queries": 500, + "recall_at_k": 0.9431999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.982, + "vectors": 20648 + }, + { + "bytes_read_per_query": 649152, + "depth": 100, + "doc_recall_at_k": 0.492, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2759555389726801, + "queries": 500, + "recall_at_k": 0.9827999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.994, + "vectors": 20648 + }, + { + "bytes_read_per_query": 802752, + "depth": 200, + "doc_recall_at_k": 0.496, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 0.9965999999999999, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.998, + "vectors": 20648 + }, + { + "bytes_read_per_query": 1263552, + "depth": 500, + "doc_recall_at_k": 0.496, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + }, + { + "bytes_read_per_query": 2031552, + "depth": 1000, + "doc_recall_at_k": 0.496, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 500, + "metric": "cosine", + "ndcg_at_k": 0.2785507318131565, + "queries": 500, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 20648 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 24 + ], + [ + 0.95, + 36 + ], + [ + 0.99, + 80 + ], + [ + 1.0, + 713 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 15.0, + "metric": "cosine", + "neighbour_band": 0.024774938821792603, + "queries": 500, + "rank_noise": 0.018597556555114886, + "recall_at_k": 0.7190000000000004, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 22 + ], + [ + 0.95, + 34 + ], + [ + 0.99, + 79 + ], + [ + 1.0, + 395 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 15.0, + "metric": "cosine", + "neighbour_band": 0.024774938821792603, + "queries": 500, + "rank_noise": 0.01770042272474518, + "recall_at_k": 0.7246000000000006, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 11 + ], + [ + 0.95, + 16 + ], + [ + 0.99, + 26 + ], + [ + 1.0, + 53 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 10.0, + "metric": "cosine", + "neighbour_band": 0.024774938821792603, + "queries": 500, + "rank_noise": 0.007792641547558008, + "recall_at_k": 0.8969999999999995, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 21 + ], + [ + 0.95, + 28 + ], + [ + 0.99, + 51 + ], + [ + 1.0, + 123 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 14.0, + "metric": "cosine", + "neighbour_band": 0.024774938821792603, + "queries": 500, + "rank_noise": 0.016073076858259247, + "recall_at_k": 0.775, + "vectors": 20648 + }, + { + "depth_for_recall": [ + [ + 0.9, + 34 + ], + [ + 0.95, + 58 + ], + [ + 0.99, + 146 + ], + [ + 1.0, + 361 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 21.0, + "metric": "cosine", + "neighbour_band": 0.024774938821792603, + "queries": 500, + "rank_noise": 0.02516490676753467, + "recall_at_k": 0.6399999999999997, + "vectors": 20648 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/fiqa-bge-small-k10.json b/experiments/2026-11-two-stage-rescoring/results/fiqa-bge-small-k10.json new file mode 100644 index 0000000..091282b --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/fiqa-bge-small-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 3425072, + "depth": 10, + "doc_recall_at_k": 0.3947765026468731, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36246983850985903, + "queries": 648, + "recall_at_k": 0.5367283950617282, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9089506172839507, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3428992, + "depth": 20, + "doc_recall_at_k": 0.41408499764981255, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3754630596976833, + "queries": 648, + "recall_at_k": 0.7172839506172839, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9475308641975309, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3432912, + "depth": 30, + "doc_recall_at_k": 0.4228934474304846, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.37972037970667294, + "queries": 648, + "recall_at_k": 0.7970679012345672, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9598765432098766, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3440752, + "depth": 50, + "doc_recall_at_k": 0.43150970257451754, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38323410734130464, + "queries": 648, + "recall_at_k": 0.873765432098762, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9675925925925926, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3460352, + "depth": 100, + "doc_recall_at_k": 0.43791586071678656, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3869491884116496, + "queries": 648, + "recall_at_k": 0.9370370370370331, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9799382716049383, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3499552, + "depth": 200, + "doc_recall_at_k": 0.4381730623628771, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38688845689256757, + "queries": 648, + "recall_at_k": 0.9637345679012318, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3617152, + "depth": 500, + "doc_recall_at_k": 0.4394958136856284, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9753086419753059, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3813152, + "depth": 1000, + "doc_recall_at_k": 0.4394958136856284, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9782407407407381, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3436512, + "depth": 10, + "doc_recall_at_k": 0.3947765026468731, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.362888875146036, + "queries": 648, + "recall_at_k": 0.5367283950617282, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9182098765432098, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3451872, + "depth": 20, + "doc_recall_at_k": 0.41330726886282454, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.37522166756712916, + "queries": 648, + "recall_at_k": 0.7179012345679012, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9645061728395061, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3467232, + "depth": 30, + "doc_recall_at_k": 0.42426518954296755, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3802613535347553, + "queries": 648, + "recall_at_k": 0.7987654320987647, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9768518518518519, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3497952, + "depth": 50, + "doc_recall_at_k": 0.43210983974872874, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3834727426221282, + "queries": 648, + "recall_at_k": 0.8793209876543183, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9861111111111112, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3574752, + "depth": 100, + "doc_recall_at_k": 0.43637265084024335, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3866850426308114, + "queries": 648, + "recall_at_k": 0.9478395061728367, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3728352, + "depth": 200, + "doc_recall_at_k": 0.43666659557863263, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3864700858645261, + "queries": 648, + "recall_at_k": 0.9811728395061717, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4189152, + "depth": 500, + "doc_recall_at_k": 0.438503750193565, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9949074074074069, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4957152, + "depth": 1000, + "doc_recall_at_k": 0.438503750193565, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9993827160493828, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3425072, + "depth": 10, + "doc_recall_at_k": 0.40706942234720006, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36727580398727433, + "queries": 648, + "recall_at_k": 0.6001543209876543, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.933641975308642, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3428992, + "depth": 20, + "doc_recall_at_k": 0.42054929381318257, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.37795915453103246, + "queries": 648, + "recall_at_k": 0.7831790123456787, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9629629629629629, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3432912, + "depth": 30, + "doc_recall_at_k": 0.42596639471639464, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38184097188571026, + "queries": 648, + "recall_at_k": 0.8611111111111086, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9737654320987654, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3440752, + "depth": 50, + "doc_recall_at_k": 0.4348380143519032, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38515664675390215, + "queries": 648, + "recall_at_k": 0.9270061728395023, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9799382716049383, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3460352, + "depth": 100, + "doc_recall_at_k": 0.4379526038090853, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38676789222925323, + "queries": 648, + "recall_at_k": 0.9628086419753055, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3499552, + "depth": 200, + "doc_recall_at_k": 0.4394958136856284, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9740740740740713, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3617152, + "depth": 500, + "doc_recall_at_k": 0.4394958136856284, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9783950617283924, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3813152, + "depth": 1000, + "doc_recall_at_k": 0.4394958136856284, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9785493827160467, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3436512, + "depth": 10, + "doc_recall_at_k": 0.40706942234720006, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36734574989805635, + "queries": 648, + "recall_at_k": 0.6001543209876543, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9506172839506173, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3451872, + "depth": 20, + "doc_recall_at_k": 0.42003489052100157, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3777789702746299, + "queries": 648, + "recall_at_k": 0.7844135802469133, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3467232, + "depth": 30, + "doc_recall_at_k": 0.42606927537483075, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3821096116301374, + "queries": 648, + "recall_at_k": 0.8652777777777759, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9922839506172839, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3497952, + "depth": 50, + "doc_recall_at_k": 0.4323027409832965, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3845539233322665, + "queries": 648, + "recall_at_k": 0.9368827160493798, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3574752, + "depth": 100, + "doc_recall_at_k": 0.4367547790001494, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38650859474539234, + "queries": 648, + "recall_at_k": 0.9791666666666651, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3728352, + "depth": 200, + "doc_recall_at_k": 0.4379893469013839, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3868114953505997, + "queries": 648, + "recall_at_k": 0.9941358024691351, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4189152, + "depth": 500, + "doc_recall_at_k": 0.438503750193565, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9996913580246913, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4957152, + "depth": 1000, + "doc_recall_at_k": 0.438503750193565, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9998456790123457, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6846224, + "depth": 10, + "doc_recall_at_k": 0.4299113780595262, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3836335120132963, + "queries": 648, + "recall_at_k": 0.8165123456790103, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9783950617283951, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6850144, + "depth": 20, + "doc_recall_at_k": 0.43989018954296727, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38720727594391846, + "queries": 648, + "recall_at_k": 0.9629629629629596, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6854064, + "depth": 30, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9770061728395034, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6861904, + "depth": 50, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9785493827160467, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6881504, + "depth": 100, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6920704, + "depth": 200, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7038304, + "depth": 500, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7234304, + "depth": 1000, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6857664, + "depth": 10, + "doc_recall_at_k": 0.4299113780595262, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3837615791269325, + "queries": 648, + "recall_at_k": 0.8165123456790103, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969135802469136, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6873024, + "depth": 20, + "doc_recall_at_k": 0.43726673275284383, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38649561736777843, + "queries": 648, + "recall_at_k": 0.9777777777777762, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6888384, + "depth": 30, + "doc_recall_at_k": 0.438503750193565, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.996296296296296, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6919104, + "depth": 50, + "doc_recall_at_k": 0.438503750193565, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9993827160493826, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6995904, + "depth": 100, + "doc_recall_at_k": 0.438503750193565, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7149504, + "depth": 200, + "doc_recall_at_k": 0.438503750193565, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7610304, + "depth": 500, + "doc_recall_at_k": 0.438503750193565, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 8378304, + "depth": 1000, + "doc_recall_at_k": 0.438503750193565, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3425072, + "depth": 10, + "doc_recall_at_k": 0.4089782259921148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3705567184670649, + "queries": 648, + "recall_at_k": 0.6418209876543218, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.941358024691358, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3428992, + "depth": 20, + "doc_recall_at_k": 0.43074238433034717, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38113991606792236, + "queries": 648, + "recall_at_k": 0.8376543209876525, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9722222222222222, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3432912, + "depth": 30, + "doc_recall_at_k": 0.4329947358882543, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38329595204416195, + "queries": 648, + "recall_at_k": 0.9003086419753048, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9753086419753086, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3440752, + "depth": 50, + "doc_recall_at_k": 0.4387242087473568, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3869095924635434, + "queries": 648, + "recall_at_k": 0.949845679012342, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3460352, + "depth": 100, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872307841545922, + "queries": 648, + "recall_at_k": 0.9736111111111083, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3499552, + "depth": 200, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9779320987654295, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3617152, + "depth": 500, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3813152, + "depth": 1000, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9787037037037012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3436512, + "depth": 10, + "doc_recall_at_k": 0.4089782259921148, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3705509650909907, + "queries": 648, + "recall_at_k": 0.6418209876543218, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9583333333333334, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3451872, + "depth": 20, + "doc_recall_at_k": 0.4302279810381662, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3810573456364275, + "queries": 648, + "recall_at_k": 0.840277777777776, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9907407407407407, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3467232, + "depth": 30, + "doc_recall_at_k": 0.4312457646948387, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38296998334154314, + "queries": 648, + "recall_at_k": 0.9075617283950582, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938271604938271, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3497952, + "depth": 50, + "doc_recall_at_k": 0.4366518983417132, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38628352663861104, + "queries": 648, + "recall_at_k": 0.9646604938271585, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3574752, + "depth": 100, + "doc_recall_at_k": 0.438503750193565, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3870295309417207, + "queries": 648, + "recall_at_k": 0.9929012345679007, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3728352, + "depth": 200, + "doc_recall_at_k": 0.438503750193565, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9987654320987653, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4189152, + "depth": 500, + "doc_recall_at_k": 0.438503750193565, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4957152, + "depth": 1000, + "doc_recall_at_k": 0.438503750193565, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1714496, + "depth": 10, + "doc_recall_at_k": 0.3450280986160616, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3260610219315916, + "queries": 648, + "recall_at_k": 0.4453703703703707, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.8009259259259259, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1718416, + "depth": 20, + "doc_recall_at_k": 0.3940285922924811, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3592648687065969, + "queries": 648, + "recall_at_k": 0.6109567901234567, + "rescore_ms": 14, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.8672839506172839, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1722336, + "depth": 30, + "doc_recall_at_k": 0.39760868846517, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.363637017236129, + "queries": 648, + "recall_at_k": 0.7049382716049384, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.8981481481481481, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1730176, + "depth": 50, + "doc_recall_at_k": 0.40614472119101735, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36742712091667346, + "queries": 648, + "recall_at_k": 0.800308641975307, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9382716049382716, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1749776, + "depth": 100, + "doc_recall_at_k": 0.4213006343793381, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.376445895489762, + "queries": 648, + "recall_at_k": 0.8953703703703669, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9675925925925926, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1788976, + "depth": 200, + "doc_recall_at_k": 0.43161442038756853, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38326893697524306, + "queries": 648, + "recall_at_k": 0.9509259259259222, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9768518518518519, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1906576, + "depth": 500, + "doc_recall_at_k": 0.43911001121649257, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3869355787940895, + "queries": 648, + "recall_at_k": 0.9720679012345649, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9783950617283951, + "vectors": 71274 + }, + { + "bytes_read_per_query": 2102576, + "depth": 1000, + "doc_recall_at_k": 0.4394958136856284, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3872220880286448, + "queries": 648, + "recall_at_k": 0.9768518518518491, + "rescore_ms": 17, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1725936, + "depth": 10, + "doc_recall_at_k": 0.3450280986160616, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.32642444482798566, + "queries": 648, + "recall_at_k": 0.4453703703703707, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.808641975308642, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1741296, + "depth": 20, + "doc_recall_at_k": 0.3924853824159379, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3591083077698998, + "queries": 648, + "recall_at_k": 0.6109567901234567, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.8780864197530864, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1756656, + "depth": 30, + "doc_recall_at_k": 0.39709428517298895, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3637059668767849, + "queries": 648, + "recall_at_k": 0.7055555555555557, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9120370370370371, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1787376, + "depth": 50, + "doc_recall_at_k": 0.40730212859842474, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3683648980834326, + "queries": 648, + "recall_at_k": 0.8023148148148134, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9552469135802469, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1864176, + "depth": 100, + "doc_recall_at_k": 0.4211291666152777, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.37669196801223515, + "queries": 648, + "recall_at_k": 0.9023148148148125, + "rescore_ms": 3, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9845679012345679, + "vectors": 71274 + }, + { + "bytes_read_per_query": 2017776, + "depth": 200, + "doc_recall_at_k": 0.4313939618337767, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3833548728461407, + "queries": 648, + "recall_at_k": 0.9640432098765408, + "rescore_ms": 2, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 2478576, + "depth": 500, + "doc_recall_at_k": 0.43888955266270085, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3869857173287652, + "queries": 648, + "recall_at_k": 0.9910493827160484, + "rescore_ms": 34, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969135802469136, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3246576, + "depth": 1000, + "doc_recall_at_k": 0.438503750193565, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.38702083481577326, + "queries": 648, + "recall_at_k": 0.9976851851851848, + "rescore_ms": 105, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 60 + ], + [ + 0.95, + 105 + ], + [ + 0.99, + 327 + ], + [ + 1.0, + 1720 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 67.0, + "metric": "cosine", + "neighbour_band": 0.010756641626358032, + "queries": 648, + "rank_noise": 0.01769264290610194, + "recall_at_k": 0.5367283950617282, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 39 + ], + [ + 0.95, + 58 + ], + [ + 0.99, + 143 + ], + [ + 1.0, + 1243 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 42.0, + "metric": "cosine", + "neighbour_band": 0.010756641626358032, + "queries": 648, + "rank_noise": 0.011478020154759652, + "recall_at_k": 0.6001543209876543, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 13 + ], + [ + 0.95, + 16 + ], + [ + 0.99, + 25 + ], + [ + 1.0, + 65 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 15.0, + "metric": "cosine", + "neighbour_band": 0.010756641626358032, + "queries": 648, + "rank_noise": 0.005679709089590111, + "recall_at_k": 0.8165123456790103, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 29 + ], + [ + 0.95, + 44 + ], + [ + 0.99, + 88 + ], + [ + 1.0, + 438 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 34.0, + "metric": "cosine", + "neighbour_band": 0.010756641626358032, + "queries": 648, + "rank_noise": 0.010514919675474296, + "recall_at_k": 0.6418209876543218, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 98 + ], + [ + 0.95, + 165 + ], + [ + 0.99, + 453 + ], + [ + 1.0, + 3704 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 110.0, + "metric": "cosine", + "neighbour_band": 0.010756641626358032, + "queries": 648, + "rank_noise": 0.014464009981583719, + "recall_at_k": 0.4453703703703707, + "vectors": 71274 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/fiqa-minilm-k10.json b/experiments/2026-11-two-stage-rescoring/results/fiqa-minilm-k10.json new file mode 100644 index 0000000..6fd0d1d --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/fiqa-minilm-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 3425072, + "depth": 10, + "doc_recall_at_k": 0.4254477604014641, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3589523406539688, + "queries": 648, + "recall_at_k": 0.6655864197530874, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9783950617283951, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3428992, + "depth": 20, + "doc_recall_at_k": 0.43638438036586175, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3646050809255716, + "queries": 648, + "recall_at_k": 0.8563271604938248, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9922839506172839, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3432912, + "depth": 30, + "doc_recall_at_k": 0.4391535847554368, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36629984026178175, + "queries": 648, + "recall_at_k": 0.9228395061728364, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3440752, + "depth": 50, + "doc_recall_at_k": 0.4359238669423855, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3654117160275599, + "queries": 648, + "recall_at_k": 0.9679012345678984, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3460352, + "depth": 100, + "doc_recall_at_k": 0.4382258216748959, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36685271755991017, + "queries": 648, + "recall_at_k": 0.989969135802468, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3499552, + "depth": 200, + "doc_recall_at_k": 0.43668261179835266, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36640662986497513, + "queries": 648, + "recall_at_k": 0.9952160493827155, + "rescore_ms": 2, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3617152, + "depth": 500, + "doc_recall_at_k": 0.43668261179835266, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9962962962962958, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3813152, + "depth": 1000, + "doc_recall_at_k": 0.43668261179835266, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3436512, + "depth": 10, + "doc_recall_at_k": 0.4254477604014641, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.358825367739636, + "queries": 648, + "recall_at_k": 0.6655864197530874, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3451872, + "depth": 20, + "doc_recall_at_k": 0.43484117048931864, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3640474533471147, + "queries": 648, + "recall_at_k": 0.856790123456788, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969135802469136, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3467232, + "depth": 30, + "doc_recall_at_k": 0.4391535847554368, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3662002765988665, + "queries": 648, + "recall_at_k": 0.9237654320987625, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3497952, + "depth": 50, + "doc_recall_at_k": 0.4361167681769534, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36542031646367307, + "queries": 648, + "recall_at_k": 0.9694444444444417, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3574752, + "depth": 100, + "doc_recall_at_k": 0.4389331262016448, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3670706574611969, + "queries": 648, + "recall_at_k": 0.9932098765432092, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3728352, + "depth": 200, + "doc_recall_at_k": 0.4373899163251016, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3666245697662619, + "queries": 648, + "recall_at_k": 0.9987654320987653, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4189152, + "depth": 500, + "doc_recall_at_k": 0.4373899163251016, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 0.9998456790123457, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4957152, + "depth": 1000, + "doc_recall_at_k": 0.4373899163251016, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 9, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3425072, + "depth": 10, + "doc_recall_at_k": 0.41028026757193414, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.35551821433253844, + "queries": 648, + "recall_at_k": 0.6825617283950618, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9737654320987654, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3428992, + "depth": 20, + "doc_recall_at_k": 0.4343145194997046, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3641743663917034, + "queries": 648, + "recall_at_k": 0.8694444444444412, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9922839506172839, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3432912, + "depth": 30, + "doc_recall_at_k": 0.4344578175596694, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3648914060239702, + "queries": 648, + "recall_at_k": 0.9314814814814781, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938271604938271, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3440752, + "depth": 50, + "doc_recall_at_k": 0.438032920440328, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3665974130651472, + "queries": 648, + "recall_at_k": 0.9733024691358002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938271604938271, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3460352, + "depth": 100, + "doc_recall_at_k": 0.4362968093292169, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.366169843573182, + "queries": 648, + "recall_at_k": 0.9925925925925919, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3499552, + "depth": 200, + "doc_recall_at_k": 0.43668261179835266, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9959876543209871, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3617152, + "depth": 500, + "doc_recall_at_k": 0.43668261179835266, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3813152, + "depth": 1000, + "doc_recall_at_k": 0.43668261179835266, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3436512, + "depth": 10, + "doc_recall_at_k": 0.41028026757193414, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3554046891564703, + "queries": 648, + "recall_at_k": 0.6825617283950618, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9783950617283951, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3451872, + "depth": 20, + "doc_recall_at_k": 0.43302851126925196, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3637940209984614, + "queries": 648, + "recall_at_k": 0.8700617283950585, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969135802469136, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3467232, + "depth": 30, + "doc_recall_at_k": 0.4349722208518505, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3650021458626044, + "queries": 648, + "recall_at_k": 0.9333333333333302, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3497952, + "depth": 50, + "doc_recall_at_k": 0.43854732373250904, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36670815290378145, + "queries": 648, + "recall_at_k": 0.9757716049382698, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3574752, + "depth": 100, + "doc_recall_at_k": 0.4370041138559658, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36638778347446876, + "queries": 648, + "recall_at_k": 0.9959876543209875, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3728352, + "depth": 200, + "doc_recall_at_k": 0.4373899163251016, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 0.9995370370370369, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4189152, + "depth": 500, + "doc_recall_at_k": 0.4373899163251016, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4957152, + "depth": 1000, + "doc_recall_at_k": 0.4373899163251016, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 5, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6846224, + "depth": 10, + "doc_recall_at_k": 0.43741869841406883, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3668407882564445, + "queries": 648, + "recall_at_k": 0.8567901234567868, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938271604938271, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6850144, + "depth": 20, + "doc_recall_at_k": 0.4383972894389562, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36694619645549087, + "queries": 648, + "recall_at_k": 0.9844135802469122, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6854064, + "depth": 30, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9947530864197525, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6861904, + "depth": 50, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9962962962962958, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6881504, + "depth": 100, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6920704, + "depth": 200, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7038304, + "depth": 500, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7234304, + "depth": 1000, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6857664, + "depth": 10, + "doc_recall_at_k": 0.43741869841406883, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3667012549082697, + "queries": 648, + "recall_at_k": 0.8567901234567868, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6873024, + "depth": 20, + "doc_recall_at_k": 0.4391045939657051, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36716413635677764, + "queries": 648, + "recall_at_k": 0.9874999999999989, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6888384, + "depth": 30, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 0.9981481481481481, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6919104, + "depth": 50, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 0.9998456790123457, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 6995904, + "depth": 100, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7149504, + "depth": 200, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 7610304, + "depth": 500, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 8378304, + "depth": 1000, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3425072, + "depth": 10, + "doc_recall_at_k": 0.4148987629311702, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.355795679409617, + "queries": 648, + "recall_at_k": 0.7162037037037043, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9830246913580247, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3428992, + "depth": 20, + "doc_recall_at_k": 0.43350433431451957, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36460829357736985, + "queries": 648, + "recall_at_k": 0.9035493827160456, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3432912, + "depth": 30, + "doc_recall_at_k": 0.4349850809341551, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3649344935193805, + "queries": 648, + "recall_at_k": 0.9540123456790094, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938271604938271, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3440752, + "depth": 50, + "doc_recall_at_k": 0.4345882555373297, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3653889686962343, + "queries": 648, + "recall_at_k": 0.9842592592592576, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3460352, + "depth": 100, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9944444444444437, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3499552, + "depth": 200, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3617152, + "depth": 500, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3813152, + "depth": 1000, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9964506172839501, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3436512, + "depth": 10, + "doc_recall_at_k": 0.4148987629311702, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.35567333936271955, + "queries": 648, + "recall_at_k": 0.7162037037037043, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9861111111111112, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3451872, + "depth": 20, + "doc_recall_at_k": 0.43350433431451957, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3644617724185363, + "queries": 648, + "recall_at_k": 0.9041666666666629, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3467232, + "depth": 30, + "doc_recall_at_k": 0.43376337311522506, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3645238577945197, + "queries": 648, + "recall_at_k": 0.9558641975308614, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3497952, + "depth": 50, + "doc_recall_at_k": 0.4349097575949428, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3654327648608714, + "queries": 648, + "recall_at_k": 0.987654320987653, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3574752, + "depth": 100, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 0.9979938271604936, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3728352, + "depth": 200, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4189152, + "depth": 500, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 4957152, + "depth": 1000, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1714496, + "depth": 10, + "doc_recall_at_k": 0.3483780879614214, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3215096324401136, + "queries": 648, + "recall_at_k": 0.5069444444444439, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.875, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1718416, + "depth": 20, + "doc_recall_at_k": 0.39202557559039064, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3419938814094672, + "queries": 648, + "recall_at_k": 0.6929012345679012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9320987654320988, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1722336, + "depth": 30, + "doc_recall_at_k": 0.4127770377770378, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.35157849493497123, + "queries": 648, + "recall_at_k": 0.7890432098765428, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9645061728395061, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1730176, + "depth": 50, + "doc_recall_at_k": 0.4201561812672924, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.35669768061800894, + "queries": 648, + "recall_at_k": 0.8760802469135786, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814814814814815, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1749776, + "depth": 100, + "doc_recall_at_k": 0.4319243813456776, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3629208887706261, + "queries": 648, + "recall_at_k": 0.9495370370370331, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9907407407407407, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1788976, + "depth": 200, + "doc_recall_at_k": 0.43822582167489577, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36658112611500204, + "queries": 648, + "recall_at_k": 0.9811728395061711, + "rescore_ms": 3, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938271604938271, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1906576, + "depth": 500, + "doc_recall_at_k": 0.4382258216748959, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3668305241377324, + "queries": 648, + "recall_at_k": 0.9938271604938265, + "rescore_ms": 2, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 2102576, + "depth": 1000, + "doc_recall_at_k": 0.43668261179835266, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3663844364427974, + "queries": 648, + "recall_at_k": 0.9961419753086415, + "rescore_ms": 11, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1725936, + "depth": 10, + "doc_recall_at_k": 0.3483780879614214, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.321380551606386, + "queries": 648, + "recall_at_k": 0.5069444444444439, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.8765432098765432, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1741296, + "depth": 20, + "doc_recall_at_k": 0.3922827772364811, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.34204410298671056, + "queries": 648, + "recall_at_k": 0.6929012345679012, + "rescore_ms": 2, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.933641975308642, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1756656, + "depth": 30, + "doc_recall_at_k": 0.4127770377770378, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3514937292990951, + "queries": 648, + "recall_at_k": 0.7893518518518514, + "rescore_ms": 2, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9675925925925926, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1787376, + "depth": 50, + "doc_recall_at_k": 0.4201561812672924, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3565805061699114, + "queries": 648, + "recall_at_k": 0.8766975308641958, + "rescore_ms": 5, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9861111111111112, + "vectors": 71274 + }, + { + "bytes_read_per_query": 1864176, + "depth": 100, + "doc_recall_at_k": 0.4321172825802455, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36292948920673934, + "queries": 648, + "recall_at_k": 0.9515432098765395, + "rescore_ms": 19, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9953703703703703, + "vectors": 71274 + }, + { + "bytes_read_per_query": 2017776, + "depth": 200, + "doc_recall_at_k": 0.4389331262016447, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.3667990660162888, + "queries": 648, + "recall_at_k": 0.9845679012345663, + "rescore_ms": 31, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9984567901234568, + "vectors": 71274 + }, + { + "bytes_read_per_query": 2478576, + "depth": 500, + "doc_recall_at_k": 0.4389331262016448, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36704846403901914, + "queries": 648, + "recall_at_k": 0.9973765432098761, + "rescore_ms": 113, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + }, + { + "bytes_read_per_query": 3246576, + "depth": 1000, + "doc_recall_at_k": 0.4373899163251016, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 648, + "metric": "cosine", + "ndcg_at_k": 0.36660237634408416, + "queries": 648, + "recall_at_k": 0.9996913580246913, + "rescore_ms": 255, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 71274 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 26 + ], + [ + 0.95, + 38 + ], + [ + 0.99, + 86 + ], + [ + 1.0, + 539 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 29.5, + "metric": "cosine", + "neighbour_band": 0.027133822441101074, + "queries": 648, + "rank_noise": 0.025144873356017765, + "recall_at_k": 0.6655864197530874, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 24 + ], + [ + 0.95, + 36 + ], + [ + 0.99, + 73 + ], + [ + 1.0, + 334 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 28.0, + "metric": "cosine", + "neighbour_band": 0.027133822441101074, + "queries": 648, + "rank_noise": 0.023914843104149184, + "recall_at_k": 0.6825617283950618, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 12 + ], + [ + 0.95, + 14 + ], + [ + 0.99, + 22 + ], + [ + 1.0, + 63 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 13.0, + "metric": "cosine", + "neighbour_band": 0.027133822441101074, + "queries": 648, + "rank_noise": 0.011738154768883602, + "recall_at_k": 0.8567901234567868, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 20 + ], + [ + 0.95, + 29 + ], + [ + 0.99, + 56 + ], + [ + 1.0, + 159 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 23.0, + "metric": "cosine", + "neighbour_band": 0.027133822441101074, + "queries": 648, + "rank_noise": 0.0219992525810692, + "recall_at_k": 0.7162037037037043, + "vectors": 71274 + }, + { + "depth_for_recall": [ + [ + 0.9, + 60 + ], + [ + 0.95, + 99 + ], + [ + 0.99, + 250 + ], + [ + 1.0, + 1106 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 67.0, + "metric": "cosine", + "neighbour_band": 0.027133822441101074, + "queries": 648, + "rank_noise": 0.03367902353177581, + "recall_at_k": 0.5069444444444439, + "vectors": 71274 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/nfcorpus-bge-small-k10.json b/experiments/2026-11-two-stage-rescoring/results/nfcorpus-bge-small-k10.json new file mode 100644 index 0000000..ea3f80f --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/nfcorpus-bge-small-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 304400, + "depth": 10, + "doc_recall_at_k": 0.14004003836456763, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2960696749606821, + "queries": 323, + "recall_at_k": 0.5560371517027866, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.891640866873065, + "vectors": 6260 + }, + { + "bytes_read_per_query": 308320, + "depth": 20, + "doc_recall_at_k": 0.14555432331135698, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.30985187406128833, + "queries": 323, + "recall_at_k": 0.7207430340557278, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9318885448916409, + "vectors": 6260 + }, + { + "bytes_read_per_query": 312240, + "depth": 30, + "doc_recall_at_k": 0.1463569304872697, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3093775097654694, + "queries": 323, + "recall_at_k": 0.7894736842105272, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9411764705882353, + "vectors": 6260 + }, + { + "bytes_read_per_query": 320080, + "depth": 50, + "doc_recall_at_k": 0.14926856083000686, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31311085569410985, + "queries": 323, + "recall_at_k": 0.8640866873065022, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9597523219814241, + "vectors": 6260 + }, + { + "bytes_read_per_query": 339680, + "depth": 100, + "doc_recall_at_k": 0.15120477013099584, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3138913374417529, + "queries": 323, + "recall_at_k": 0.923839009287926, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9690402476780186, + "vectors": 6260 + }, + { + "bytes_read_per_query": 378880, + "depth": 200, + "doc_recall_at_k": 0.15096107592754282, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3144151694213623, + "queries": 323, + "recall_at_k": 0.9532507739938085, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.978328173374613, + "vectors": 6260 + }, + { + "bytes_read_per_query": 496480, + "depth": 500, + "doc_recall_at_k": 0.15094077665441813, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31438255792609027, + "queries": 323, + "recall_at_k": 0.9712074303405576, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 692480, + "depth": 1000, + "doc_recall_at_k": 0.15094077665441813, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9764705882352942, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 315840, + "depth": 10, + "doc_recall_at_k": 0.14004003836456763, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.29606735787989574, + "queries": 323, + "recall_at_k": 0.5560371517027866, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.8978328173374613, + "vectors": 6260 + }, + { + "bytes_read_per_query": 331200, + "depth": 20, + "doc_recall_at_k": 0.14566069345847105, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3097973045060363, + "queries": 323, + "recall_at_k": 0.721981424148607, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9380804953560371, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346560, + "depth": 30, + "doc_recall_at_k": 0.146236244594138, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.30871903614334645, + "queries": 323, + "recall_at_k": 0.7925696594427252, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9504643962848297, + "vectors": 6260 + }, + { + "bytes_read_per_query": 377280, + "depth": 50, + "doc_recall_at_k": 0.1486330024714243, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31222312359502585, + "queries": 323, + "recall_at_k": 0.8702786377708982, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9690402476780186, + "vectors": 6260 + }, + { + "bytes_read_per_query": 454080, + "depth": 100, + "doc_recall_at_k": 0.15080401726401932, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31301204658067466, + "queries": 323, + "recall_at_k": 0.9390092879256965, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 607680, + "depth": 200, + "doc_recall_at_k": 0.150508723473363, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31326823706820756, + "queries": 323, + "recall_at_k": 0.9712074303405578, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1068480, + "depth": 500, + "doc_recall_at_k": 0.15054159523920038, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3136377153922956, + "queries": 323, + "recall_at_k": 0.9928792569659446, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1836480, + "depth": 1000, + "doc_recall_at_k": 0.15054159523920038, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 0.9987616099071208, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 304400, + "depth": 10, + "doc_recall_at_k": 0.1417636243417844, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.299686736479972, + "queries": 323, + "recall_at_k": 0.5826625386996898, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9504643962848297, + "vectors": 6260 + }, + { + "bytes_read_per_query": 308320, + "depth": 20, + "doc_recall_at_k": 0.14680099183114234, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3091233032523487, + "queries": 323, + "recall_at_k": 0.7582043343653252, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9690402476780186, + "vectors": 6260 + }, + { + "bytes_read_per_query": 312240, + "depth": 30, + "doc_recall_at_k": 0.14665507249467855, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31093567602793837, + "queries": 323, + "recall_at_k": 0.8325077399380809, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.978328173374613, + "vectors": 6260 + }, + { + "bytes_read_per_query": 320080, + "depth": 50, + "doc_recall_at_k": 0.14959766585855885, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143036167043981, + "queries": 323, + "recall_at_k": 0.902786377708979, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 339680, + "depth": 100, + "doc_recall_at_k": 0.15094027742184882, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31468157000090535, + "queries": 323, + "recall_at_k": 0.9523219814241488, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 378880, + "depth": 200, + "doc_recall_at_k": 0.15094077665441813, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31438255792609027, + "queries": 323, + "recall_at_k": 0.9715170278637771, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 496480, + "depth": 500, + "doc_recall_at_k": 0.15094077665441813, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9764705882352942, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 692480, + "depth": 1000, + "doc_recall_at_k": 0.15094077665441813, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.977089783281734, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 315840, + "depth": 10, + "doc_recall_at_k": 0.1417636243417844, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.29933406112198657, + "queries": 323, + "recall_at_k": 0.5826625386996898, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9628482972136223, + "vectors": 6260 + }, + { + "bytes_read_per_query": 331200, + "depth": 20, + "doc_recall_at_k": 0.14668632608180168, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.30856801839852444, + "queries": 323, + "recall_at_k": 0.7597523219814243, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346560, + "depth": 30, + "doc_recall_at_k": 0.14633723337072485, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3098898150746725, + "queries": 323, + "recall_at_k": 0.836842105263158, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 377280, + "depth": 50, + "doc_recall_at_k": 0.14943298226205778, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3131855439477089, + "queries": 323, + "recall_at_k": 0.9120743034055732, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 454080, + "depth": 100, + "doc_recall_at_k": 0.15083268580558673, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3137917810022168, + "queries": 323, + "recall_at_k": 0.9693498452012385, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 607680, + "depth": 200, + "doc_recall_at_k": 0.15054159523920038, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3136377153922956, + "queries": 323, + "recall_at_k": 0.9925696594427246, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1068480, + "depth": 500, + "doc_recall_at_k": 0.15054159523920038, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 0.9990712074303405, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1836480, + "depth": 1000, + "doc_recall_at_k": 0.15054159523920038, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 604880, + "depth": 10, + "doc_recall_at_k": 0.14798313755819867, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31298150903117977, + "queries": 323, + "recall_at_k": 0.8349845201238411, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 608800, + "depth": 20, + "doc_recall_at_k": 0.15033572161362085, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3141140313730846, + "queries": 323, + "recall_at_k": 0.9625386996904027, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 612720, + "depth": 30, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9739938080495355, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 620560, + "depth": 50, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9764705882352942, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 640160, + "depth": 100, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 679360, + "depth": 200, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 796960, + "depth": 500, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 992960, + "depth": 1000, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 616320, + "depth": 10, + "doc_recall_at_k": 0.14798313755819867, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3128142787509656, + "queries": 323, + "recall_at_k": 0.8349845201238411, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 631680, + "depth": 20, + "doc_recall_at_k": 0.1499507205607601, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3133732113312625, + "queries": 323, + "recall_at_k": 0.9780185758513933, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 647040, + "depth": 30, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 0.9959752321981422, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 677760, + "depth": 50, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 0.9996904024767801, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 754560, + "depth": 100, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 908160, + "depth": 200, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1368960, + "depth": 500, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 2136960, + "depth": 1000, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 304400, + "depth": 10, + "doc_recall_at_k": 0.1450286165688322, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3100327464518468, + "queries": 323, + "recall_at_k": 0.6842105263157896, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9535603715170279, + "vectors": 6260 + }, + { + "bytes_read_per_query": 308320, + "depth": 20, + "doc_recall_at_k": 0.1493146320310786, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3124568936806491, + "queries": 323, + "recall_at_k": 0.8613003095975242, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814241486068112, + "vectors": 6260 + }, + { + "bytes_read_per_query": 312240, + "depth": 30, + "doc_recall_at_k": 0.15014131428701188, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3140960521815291, + "queries": 323, + "recall_at_k": 0.919504643962849, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814241486068112, + "vectors": 6260 + }, + { + "bytes_read_per_query": 320080, + "depth": 50, + "doc_recall_at_k": 0.15048929231030617, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31350941741767663, + "queries": 323, + "recall_at_k": 0.9563467492260063, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 339680, + "depth": 100, + "doc_recall_at_k": 0.15103459408569686, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31458234887825404, + "queries": 323, + "recall_at_k": 0.9724458204334364, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 378880, + "depth": 200, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9764705882352944, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 496480, + "depth": 500, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 692480, + "depth": 1000, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143486384401113, + "queries": 323, + "recall_at_k": 0.9767801857585141, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 315840, + "depth": 10, + "doc_recall_at_k": 0.1450286165688322, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3098427514406202, + "queries": 323, + "recall_at_k": 0.6842105263157896, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9659442724458205, + "vectors": 6260 + }, + { + "bytes_read_per_query": 331200, + "depth": 20, + "doc_recall_at_k": 0.1490577933145341, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31153948333094406, + "queries": 323, + "recall_at_k": 0.8656346749226017, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346560, + "depth": 30, + "doc_recall_at_k": 0.1500303090556048, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3136259110792146, + "queries": 323, + "recall_at_k": 0.9287925696594431, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 377280, + "depth": 50, + "doc_recall_at_k": 0.15014171048229172, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3130085789333469, + "queries": 323, + "recall_at_k": 0.9721362229102168, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969040247678018, + "vectors": 6260 + }, + { + "bytes_read_per_query": 454080, + "depth": 100, + "doc_recall_at_k": 0.1506354126704791, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3138340956735098, + "queries": 323, + "recall_at_k": 0.9931888544891639, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 607680, + "depth": 200, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 0.9990712074303407, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1068480, + "depth": 500, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1836480, + "depth": 1000, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31359564285599023, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 6, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 154160, + "depth": 10, + "doc_recall_at_k": 0.1324171369609357, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.29251760416068157, + "queries": 323, + "recall_at_k": 0.529411764705882, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.8761609907120743, + "vectors": 6260 + }, + { + "bytes_read_per_query": 158080, + "depth": 20, + "doc_recall_at_k": 0.13942914952903798, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.305174423921688, + "queries": 323, + "recall_at_k": 0.7024767801857589, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.934984520123839, + "vectors": 6260 + }, + { + "bytes_read_per_query": 162000, + "depth": 30, + "doc_recall_at_k": 0.14371723657399646, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.307925400574726, + "queries": 323, + "recall_at_k": 0.7956656346749231, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9535603715170279, + "vectors": 6260 + }, + { + "bytes_read_per_query": 169840, + "depth": 50, + "doc_recall_at_k": 0.14714524162142112, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.312078556591918, + "queries": 323, + "recall_at_k": 0.8687306501547996, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9721362229102167, + "vectors": 6260 + }, + { + "bytes_read_per_query": 189440, + "depth": 100, + "doc_recall_at_k": 0.1506877652343526, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31453641872607446, + "queries": 323, + "recall_at_k": 0.9315789473684215, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814241486068112, + "vectors": 6260 + }, + { + "bytes_read_per_query": 228640, + "depth": 200, + "doc_recall_at_k": 0.15102923308962377, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31463147496489163, + "queries": 323, + "recall_at_k": 0.9609907120743036, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9814241486068112, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346240, + "depth": 500, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31441220590972524, + "queries": 323, + "recall_at_k": 0.9736842105263159, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 542240, + "depth": 1000, + "doc_recall_at_k": 0.15094077665441813, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3143616412301621, + "queries": 323, + "recall_at_k": 0.9764705882352942, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 165600, + "depth": 10, + "doc_recall_at_k": 0.1324171369609357, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2926093630725904, + "queries": 323, + "recall_at_k": 0.529411764705882, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.8854489164086687, + "vectors": 6260 + }, + { + "bytes_read_per_query": 180960, + "depth": 20, + "doc_recall_at_k": 0.13955207673625142, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3049073625201839, + "queries": 323, + "recall_at_k": 0.7037151702786381, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9442724458204335, + "vectors": 6260 + }, + { + "bytes_read_per_query": 196320, + "depth": 30, + "doc_recall_at_k": 0.14377644270100753, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3074458874884881, + "queries": 323, + "recall_at_k": 0.7984520123839014, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9659442724458205, + "vectors": 6260 + }, + { + "bytes_read_per_query": 227040, + "depth": 50, + "doc_recall_at_k": 0.147006211853635, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31124137284888614, + "queries": 323, + "recall_at_k": 0.8764705882352943, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 303840, + "depth": 100, + "doc_recall_at_k": 0.15019931952206614, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3134601111901986, + "queries": 323, + "recall_at_k": 0.9464396284829723, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 457440, + "depth": 200, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31374011038247784, + "queries": 323, + "recall_at_k": 0.9792569659442725, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969040247678018, + "vectors": 6260 + }, + { + "bytes_read_per_query": 918240, + "depth": 500, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.31365921032560423, + "queries": 323, + "recall_at_k": 0.9956656346749226, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1686240, + "depth": 1000, + "doc_recall_at_k": 0.15054159523920038, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.3136086456460411, + "queries": 323, + "recall_at_k": 0.9993808049535604, + "rescore_ms": 2, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 64 + ], + [ + 0.95, + 116 + ], + [ + 0.99, + 424 + ], + [ + 1.0, + 1536 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 58.0, + "metric": "cosine", + "neighbour_band": 0.012125968933105469, + "queries": 323, + "rank_noise": 0.02005583883466805, + "recall_at_k": 0.5560371517027866, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 46 + ], + [ + 0.95, + 74 + ], + [ + 0.99, + 170 + ], + [ + 1.0, + 698 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 42.0, + "metric": "cosine", + "neighbour_band": 0.012125968933105469, + "queries": 323, + "rank_noise": 0.013492521855551734, + "recall_at_k": 0.5826625386996898, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 13 + ], + [ + 0.95, + 16 + ], + [ + 0.99, + 25 + ], + [ + 1.0, + 72 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 14.0, + "metric": "cosine", + "neighbour_band": 0.012125968933105469, + "queries": 323, + "rank_noise": 0.006044045887215805, + "recall_at_k": 0.8349845201238411, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 25 + ], + [ + 0.95, + 37 + ], + [ + 0.99, + 86 + ], + [ + 1.0, + 262 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 25.0, + "metric": "cosine", + "neighbour_band": 0.012125968933105469, + "queries": 323, + "rank_noise": 0.010998554937267174, + "recall_at_k": 0.6842105263157896, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 63 + ], + [ + 0.95, + 106 + ], + [ + 0.99, + 315 + ], + [ + 1.0, + 1091 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 62.0, + "metric": "cosine", + "neighbour_band": 0.012125968933105469, + "queries": 323, + "rank_noise": 0.015120641347280507, + "recall_at_k": 0.529411764705882, + "vectors": 6260 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/nfcorpus-minilm-k10.json b/experiments/2026-11-two-stage-rescoring/results/nfcorpus-minilm-k10.json new file mode 100644 index 0000000..49abd34 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/nfcorpus-minilm-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 304400, + "depth": 10, + "doc_recall_at_k": 0.1323602119067216, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.27497930241552654, + "queries": 323, + "recall_at_k": 0.6690402476780186, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9690402476780186, + "vectors": 6260 + }, + { + "bytes_read_per_query": 308320, + "depth": 20, + "doc_recall_at_k": 0.13394676509996806, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2796031413328297, + "queries": 323, + "recall_at_k": 0.8513931888544893, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 312240, + "depth": 30, + "doc_recall_at_k": 0.1341264706917567, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28062178175046637, + "queries": 323, + "recall_at_k": 0.909907120743034, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9876160990712074, + "vectors": 6260 + }, + { + "bytes_read_per_query": 320080, + "depth": 50, + "doc_recall_at_k": 0.13587406923511589, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2827704687548993, + "queries": 323, + "recall_at_k": 0.9594427244582038, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9876160990712074, + "vectors": 6260 + }, + { + "bytes_read_per_query": 339680, + "depth": 100, + "doc_recall_at_k": 0.13540153949587275, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2819581504344227, + "queries": 323, + "recall_at_k": 0.9866873065015479, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 378880, + "depth": 200, + "doc_recall_at_k": 0.13521942330574344, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9947368421052631, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 496480, + "depth": 500, + "doc_recall_at_k": 0.13521942330574344, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 692480, + "depth": 1000, + "doc_recall_at_k": 0.13521942330574344, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 3, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 315840, + "depth": 10, + "doc_recall_at_k": 0.1323602119067216, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2748642795172424, + "queries": 323, + "recall_at_k": 0.6690402476780186, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9721362229102167, + "vectors": 6260 + }, + { + "bytes_read_per_query": 331200, + "depth": 20, + "doc_recall_at_k": 0.1331082718079144, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.27922321371175185, + "queries": 323, + "recall_at_k": 0.8517027863777092, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9907120743034056, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346560, + "depth": 30, + "doc_recall_at_k": 0.13391674333731748, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28048284985375793, + "queries": 323, + "recall_at_k": 0.9108359133126934, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 377280, + "depth": 50, + "doc_recall_at_k": 0.13751437586089263, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28315264636433146, + "queries": 323, + "recall_at_k": 0.9619195046439623, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 454080, + "depth": 100, + "doc_recall_at_k": 0.13698846723833571, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28216709892291303, + "queries": 323, + "recall_at_k": 0.9904024767801856, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 607680, + "depth": 200, + "doc_recall_at_k": 0.13680635104820643, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 0.9984520123839009, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1068480, + "depth": 500, + "doc_recall_at_k": 0.13680635104820643, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1836480, + "depth": 1000, + "doc_recall_at_k": 0.13680635104820643, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 5, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 304400, + "depth": 10, + "doc_recall_at_k": 0.13227598658615683, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2789056659067166, + "queries": 323, + "recall_at_k": 0.6907120743034053, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9659442724458205, + "vectors": 6260 + }, + { + "bytes_read_per_query": 308320, + "depth": 20, + "doc_recall_at_k": 0.13500278125678464, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28305081519379827, + "queries": 323, + "recall_at_k": 0.8681114551083603, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9876160990712074, + "vectors": 6260 + }, + { + "bytes_read_per_query": 312240, + "depth": 30, + "doc_recall_at_k": 0.13340194628986174, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2814299059925465, + "queries": 323, + "recall_at_k": 0.9229102167182665, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9907120743034056, + "vectors": 6260 + }, + { + "bytes_read_per_query": 320080, + "depth": 50, + "doc_recall_at_k": 0.13612118155867187, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2812542357472304, + "queries": 323, + "recall_at_k": 0.962538699690403, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9907120743034056, + "vectors": 6260 + }, + { + "bytes_read_per_query": 339680, + "depth": 100, + "doc_recall_at_k": 0.13649672723750964, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2819610581130005, + "queries": 323, + "recall_at_k": 0.9876160990712073, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 378880, + "depth": 200, + "doc_recall_at_k": 0.13509558429645552, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2815253428220429, + "queries": 323, + "recall_at_k": 0.9947368421052634, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 496480, + "depth": 500, + "doc_recall_at_k": 0.13521942330574344, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 692480, + "depth": 1000, + "doc_recall_at_k": 0.13521942330574344, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 1, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 315840, + "depth": 10, + "doc_recall_at_k": 0.13227598658615683, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2789583670042322, + "queries": 323, + "recall_at_k": 0.6907120743034053, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9690402476780186, + "vectors": 6260 + }, + { + "bytes_read_per_query": 331200, + "depth": 20, + "doc_recall_at_k": 0.1350527163411749, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28296073137923217, + "queries": 323, + "recall_at_k": 0.8693498452012395, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346560, + "depth": 30, + "doc_recall_at_k": 0.1330545052429163, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2809203271647302, + "queries": 323, + "recall_at_k": 0.9247678018575852, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969040247678018, + "vectors": 6260 + }, + { + "bytes_read_per_query": 377280, + "depth": 50, + "doc_recall_at_k": 0.1362283782408485, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28134394718218203, + "queries": 323, + "recall_at_k": 0.9650154798761613, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9969040247678018, + "vectors": 6260 + }, + { + "bytes_read_per_query": 454080, + "depth": 100, + "doc_recall_at_k": 0.13659637276058328, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28184911293791765, + "queries": 323, + "recall_at_k": 0.9910216718266252, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 607680, + "depth": 200, + "doc_recall_at_k": 0.1366825120389185, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28173429131053324, + "queries": 323, + "recall_at_k": 0.9984520123839009, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1068480, + "depth": 500, + "doc_recall_at_k": 0.13680635104820643, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1836480, + "depth": 1000, + "doc_recall_at_k": 0.13680635104820643, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 5, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 604880, + "depth": 10, + "doc_recall_at_k": 0.13579010339617162, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28361180348677434, + "queries": 323, + "recall_at_k": 0.8743034055727568, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 608800, + "depth": 20, + "doc_recall_at_k": 0.13676741092184252, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2822890286075563, + "queries": 323, + "recall_at_k": 0.9885448916408671, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 612720, + "depth": 30, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9938080495356038, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 620560, + "depth": 50, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 640160, + "depth": 100, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 679360, + "depth": 200, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 796960, + "depth": 500, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 992960, + "depth": 1000, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 616320, + "depth": 10, + "doc_recall_at_k": 0.13579010339617162, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28350704954674805, + "queries": 323, + "recall_at_k": 0.8743034055727568, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 631680, + "depth": 20, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28197196199763014, + "queries": 323, + "recall_at_k": 0.9913312693498454, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 647040, + "depth": 30, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 0.9975232198142413, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 677760, + "depth": 50, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 754560, + "depth": 100, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 908160, + "depth": 200, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1368960, + "depth": 500, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 2136960, + "depth": 1000, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 304400, + "depth": 10, + "doc_recall_at_k": 0.13757751316347785, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28265014091821017, + "queries": 323, + "recall_at_k": 0.7424148606811146, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9752321981424149, + "vectors": 6260 + }, + { + "bytes_read_per_query": 308320, + "depth": 20, + "doc_recall_at_k": 0.13763616798998027, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28378486501937955, + "queries": 323, + "recall_at_k": 0.9216718266253878, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 312240, + "depth": 30, + "doc_recall_at_k": 0.13780905136920674, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2831509150408852, + "queries": 323, + "recall_at_k": 0.969969040247678, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9876160990712074, + "vectors": 6260 + }, + { + "bytes_read_per_query": 320080, + "depth": 50, + "doc_recall_at_k": 0.13729352435999384, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28274520343141396, + "queries": 323, + "recall_at_k": 0.9907120743034054, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 339680, + "depth": 100, + "doc_recall_at_k": 0.13676741092184252, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2823117419511956, + "queries": 323, + "recall_at_k": 0.9953560371517027, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 378880, + "depth": 200, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 496480, + "depth": 500, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 692480, + "depth": 1000, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9962848297213622, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 315840, + "depth": 10, + "doc_recall_at_k": 0.13757751316347785, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28239194618626445, + "queries": 323, + "recall_at_k": 0.7424148606811146, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9814241486068112, + "vectors": 6260 + }, + { + "bytes_read_per_query": 331200, + "depth": 20, + "doc_recall_at_k": 0.13768265927544712, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2836705802106119, + "queries": 323, + "recall_at_k": 0.922910216718267, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9907120743034056, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346560, + "depth": 30, + "doc_recall_at_k": 0.13940353027077265, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2835568319138924, + "queries": 323, + "recall_at_k": 0.9718266253869968, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 377280, + "depth": 50, + "doc_recall_at_k": 0.1388804521024568, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2829541519199043, + "queries": 323, + "recall_at_k": 0.9934984520123836, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 454080, + "depth": 100, + "doc_recall_at_k": 0.13835433866430552, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28252069043968603, + "queries": 323, + "recall_at_k": 0.9990712074303407, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 607680, + "depth": 200, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1068480, + "depth": 500, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1836480, + "depth": 1000, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 154160, + "depth": 10, + "doc_recall_at_k": 0.12050051484895005, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2686703540736248, + "queries": 323, + "recall_at_k": 0.5944272445820431, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9102167182662538, + "vectors": 6260 + }, + { + "bytes_read_per_query": 158080, + "depth": 20, + "doc_recall_at_k": 0.13158968993377898, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2812403179711474, + "queries": 323, + "recall_at_k": 0.7891640866873063, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9628482972136223, + "vectors": 6260 + }, + { + "bytes_read_per_query": 162000, + "depth": 30, + "doc_recall_at_k": 0.13254607910257685, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2788236964556737, + "queries": 323, + "recall_at_k": 0.8702786377708982, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9752321981424149, + "vectors": 6260 + }, + { + "bytes_read_per_query": 169840, + "depth": 50, + "doc_recall_at_k": 0.13419548878219, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2819054417018927, + "queries": 323, + "recall_at_k": 0.9359133126934994, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.978328173374613, + "vectors": 6260 + }, + { + "bytes_read_per_query": 189440, + "depth": 100, + "doc_recall_at_k": 0.13376127848737246, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28142400319823596, + "queries": 323, + "recall_at_k": 0.980185758513932, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9876160990712074, + "vectors": 6260 + }, + { + "bytes_read_per_query": 228640, + "depth": 200, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28172273450495766, + "queries": 323, + "recall_at_k": 0.9925696594427246, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 346240, + "depth": 500, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9959752321981425, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 542240, + "depth": 1000, + "doc_recall_at_k": 0.13521942330574344, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28174030016550045, + "queries": 323, + "recall_at_k": 0.9959752321981425, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 165600, + "depth": 10, + "doc_recall_at_k": 0.12050051484895005, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.26853390181509845, + "queries": 323, + "recall_at_k": 0.5944272445820431, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9102167182662538, + "vectors": 6260 + }, + { + "bytes_read_per_query": 180960, + "depth": 20, + "doc_recall_at_k": 0.13168956010255958, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28138691369431285, + "queries": 323, + "recall_at_k": 0.7894736842105261, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9690402476780186, + "vectors": 6260 + }, + { + "bytes_read_per_query": 196320, + "depth": 30, + "doc_recall_at_k": 0.13259257038804367, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.278781060013506, + "queries": 323, + "recall_at_k": 0.8712074303405577, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9814241486068112, + "vectors": 6260 + }, + { + "bytes_read_per_query": 227040, + "depth": 50, + "doc_recall_at_k": 0.13424198006765684, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28182097766599196, + "queries": 323, + "recall_at_k": 0.9374613003095984, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9845201238390093, + "vectors": 6260 + }, + { + "bytes_read_per_query": 303840, + "depth": 100, + "doc_recall_at_k": 0.13534820622983543, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28165175995522984, + "queries": 323, + "recall_at_k": 0.9832817337461301, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9938080495356038, + "vectors": 6260 + }, + { + "bytes_read_per_query": 457440, + "depth": 200, + "doc_recall_at_k": 0.13686705644491617, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.2821368044282913, + "queries": 323, + "recall_at_k": 0.9959752321981422, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 918240, + "depth": 500, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 0.9996904024767801, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + }, + { + "bytes_read_per_query": 1686240, + "depth": 1000, + "doc_recall_at_k": 0.13680635104820643, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 323, + "metric": "cosine", + "ndcg_at_k": 0.28194924865399074, + "queries": 323, + "recall_at_k": 0.9996904024767801, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 6260 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 28 + ], + [ + 0.95, + 43 + ], + [ + 0.99, + 100 + ], + [ + 1.0, + 291 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 31.0, + "metric": "cosine", + "neighbour_band": 0.030829548835754395, + "queries": 323, + "rank_noise": 0.029061802535522818, + "recall_at_k": 0.6690402476780186, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 25 + ], + [ + 0.95, + 40 + ], + [ + 0.99, + 91 + ], + [ + 1.0, + 263 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 27.0, + "metric": "cosine", + "neighbour_band": 0.030829548835754395, + "queries": 323, + "rank_noise": 0.026947140886363668, + "recall_at_k": 0.6907120743034053, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 11 + ], + [ + 0.95, + 13 + ], + [ + 0.99, + 20 + ], + [ + 1.0, + 46 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 12.0, + "metric": "cosine", + "neighbour_band": 0.030829548835754395, + "queries": 323, + "rank_noise": 0.011645806522831125, + "recall_at_k": 0.8743034055727568, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 18 + ], + [ + 0.95, + 25 + ], + [ + 0.99, + 46 + ], + [ + 1.0, + 152 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 19.0, + "metric": "cosine", + "neighbour_band": 0.030829548835754395, + "queries": 323, + "rank_noise": 0.02263408737491555, + "recall_at_k": 0.7424148606811146, + "vectors": 6260 + }, + { + "depth_for_recall": [ + [ + 0.9, + 36 + ], + [ + 0.95, + 59 + ], + [ + 0.99, + 136 + ], + [ + 1.0, + 1283 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 39.0, + "metric": "cosine", + "neighbour_band": 0.030829548835754395, + "queries": 323, + "rank_noise": 0.03515438785727546, + "recall_at_k": 0.5944272445820431, + "vectors": 6260 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/scifact-bge-small-k10.json b/experiments/2026-11-two-stage-rescoring/results/scifact-bge-small-k10.json new file mode 100644 index 0000000..fae2874 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/scifact-bge-small-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 389648, + "depth": 10, + "doc_recall_at_k": 0.7676111111111111, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6781137174619867, + "queries": 300, + "recall_at_k": 0.6109999999999995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9666666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 393568, + "depth": 20, + "doc_recall_at_k": 0.8002222222222222, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6908633912021365, + "queries": 300, + "recall_at_k": 0.7776666666666668, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9866666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 397488, + "depth": 30, + "doc_recall_at_k": 0.8128888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6956488701719227, + "queries": 300, + "recall_at_k": 0.8530000000000012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 405328, + "depth": 50, + "doc_recall_at_k": 0.8195555555555556, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6982262011446152, + "queries": 300, + "recall_at_k": 0.9143333333333347, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 424928, + "depth": 100, + "doc_recall_at_k": 0.8212222222222223, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6989925221637916, + "queries": 300, + "recall_at_k": 0.9613333333333336, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 464128, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6979128426656627, + "queries": 300, + "recall_at_k": 0.9796666666666671, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 581728, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9860000000000004, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 777728, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 401088, + "depth": 10, + "doc_recall_at_k": 0.7676111111111111, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6785501499738915, + "queries": 300, + "recall_at_k": 0.6109999999999995, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.97, + "vectors": 8036 + }, + { + "bytes_read_per_query": 416448, + "depth": 20, + "doc_recall_at_k": 0.7968888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6901997721917655, + "queries": 300, + "recall_at_k": 0.7790000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 431808, + "depth": 30, + "doc_recall_at_k": 0.8128888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6959886844804316, + "queries": 300, + "recall_at_k": 0.855000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 462528, + "depth": 50, + "doc_recall_at_k": 0.8195555555555556, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6985660154531242, + "queries": 300, + "recall_at_k": 0.9193333333333347, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 539328, + "depth": 100, + "doc_recall_at_k": 0.8212222222222223, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6993323364723006, + "queries": 300, + "recall_at_k": 0.9700000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 692928, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6982045407037659, + "queries": 300, + "recall_at_k": 0.991, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1153728, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 0.9986666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1921728, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 389648, + "depth": 10, + "doc_recall_at_k": 0.7695555555555555, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6767965729008821, + "queries": 300, + "recall_at_k": 0.6559999999999993, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.98, + "vectors": 8036 + }, + { + "bytes_read_per_query": 393568, + "depth": 20, + "doc_recall_at_k": 0.8168888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6965206494865323, + "queries": 300, + "recall_at_k": 0.8296666666666687, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 397488, + "depth": 30, + "doc_recall_at_k": 0.8212222222222223, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6988995913271191, + "queries": 300, + "recall_at_k": 0.8980000000000012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 405328, + "depth": 50, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.697876480384234, + "queries": 300, + "recall_at_k": 0.9526666666666671, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 424928, + "depth": 100, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.697876480384234, + "queries": 300, + "recall_at_k": 0.9806666666666672, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 464128, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9863333333333336, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 581728, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 777728, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 401088, + "depth": 10, + "doc_recall_at_k": 0.7695555555555555, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6768018204454359, + "queries": 300, + "recall_at_k": 0.6559999999999993, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 416448, + "depth": 20, + "doc_recall_at_k": 0.8168888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6967143846255784, + "queries": 300, + "recall_at_k": 0.8310000000000021, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9866666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 431808, + "depth": 30, + "doc_recall_at_k": 0.8212222222222223, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6992394056356281, + "queries": 300, + "recall_at_k": 0.9033333333333347, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 462528, + "depth": 50, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6981681784223372, + "queries": 300, + "recall_at_k": 0.9610000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 539328, + "depth": 100, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6981681784223372, + "queries": 300, + "recall_at_k": 0.9923333333333336, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 692928, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 0.9993333333333334, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1153728, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1921728, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 775376, + "depth": 10, + "doc_recall_at_k": 0.8168888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6976244150524953, + "queries": 300, + "recall_at_k": 0.8446666666666696, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 779296, + "depth": 20, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9753333333333338, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 783216, + "depth": 30, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9856666666666668, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 791056, + "depth": 50, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 810656, + "depth": 100, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 849856, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 967456, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1163456, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 786816, + "depth": 10, + "doc_recall_at_k": 0.8168888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978451063173536, + "queries": 300, + "recall_at_k": 0.8446666666666696, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 802176, + "depth": 20, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 0.985666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 817536, + "depth": 30, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 0.9983333333333333, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 848256, + "depth": 50, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 925056, + "depth": 100, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1078656, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1539456, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 2307456, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 389648, + "depth": 10, + "doc_recall_at_k": 0.7968888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.69251405938663, + "queries": 300, + "recall_at_k": 0.711333333333333, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 393568, + "depth": 20, + "doc_recall_at_k": 0.8045555555555556, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6941995572398264, + "queries": 300, + "recall_at_k": 0.8923333333333349, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 397488, + "depth": 30, + "doc_recall_at_k": 0.8112222222222223, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6960646966502353, + "queries": 300, + "recall_at_k": 0.9406666666666678, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 405328, + "depth": 50, + "doc_recall_at_k": 0.8145555555555556, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6968134855435288, + "queries": 300, + "recall_at_k": 0.9723333333333338, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 424928, + "depth": 100, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.697876480384234, + "queries": 300, + "recall_at_k": 0.9853333333333336, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 464128, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9866666666666668, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 581728, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 777728, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 401088, + "depth": 10, + "doc_recall_at_k": 0.7968888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6924171548640954, + "queries": 300, + "recall_at_k": 0.711333333333333, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 416448, + "depth": 20, + "doc_recall_at_k": 0.8045555555555556, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6945393715483354, + "queries": 300, + "recall_at_k": 0.8963333333333348, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 431808, + "depth": 30, + "doc_recall_at_k": 0.8112222222222223, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6963563946883384, + "queries": 300, + "recall_at_k": 0.9486666666666675, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 462528, + "depth": 50, + "doc_recall_at_k": 0.8145555555555556, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6971051835816321, + "queries": 300, + "recall_at_k": 0.9836666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 539328, + "depth": 100, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6981681784223372, + "queries": 300, + "recall_at_k": 0.9983333333333333, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 692928, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 0.9996666666666666, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1153728, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1921728, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 196784, + "depth": 10, + "doc_recall_at_k": 0.7254444444444444, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6462256170625684, + "queries": 300, + "recall_at_k": 0.5766666666666663, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9266666666666666, + "vectors": 8036 + }, + { + "bytes_read_per_query": 200704, + "depth": 20, + "doc_recall_at_k": 0.7884444444444444, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6787604975161192, + "queries": 300, + "recall_at_k": 0.7626666666666668, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9633333333333334, + "vectors": 8036 + }, + { + "bytes_read_per_query": 204624, + "depth": 30, + "doc_recall_at_k": 0.8084444444444444, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6916907818394616, + "queries": 300, + "recall_at_k": 0.8406666666666682, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.98, + "vectors": 8036 + }, + { + "bytes_read_per_query": 212464, + "depth": 50, + "doc_recall_at_k": 0.8107777777777777, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6936244935556362, + "queries": 300, + "recall_at_k": 0.9096666666666677, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 232064, + "depth": 100, + "doc_recall_at_k": 0.8145555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6968730470653541, + "queries": 300, + "recall_at_k": 0.9616666666666669, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 271264, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.697876480384234, + "queries": 300, + "recall_at_k": 0.9813333333333338, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 388864, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9866666666666668, + "rescore_ms": 2, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 584864, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6978365964864137, + "queries": 300, + "recall_at_k": 0.9870000000000002, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 208224, + "depth": 10, + "doc_recall_at_k": 0.7254444444444444, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6463520510583594, + "queries": 300, + "recall_at_k": 0.5766666666666663, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.93, + "vectors": 8036 + }, + { + "bytes_read_per_query": 223584, + "depth": 20, + "doc_recall_at_k": 0.7851111111111111, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6779906832341057, + "queries": 300, + "recall_at_k": 0.7630000000000002, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9666666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 238944, + "depth": 30, + "doc_recall_at_k": 0.8084444444444444, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6919710346261454, + "queries": 300, + "recall_at_k": 0.8430000000000014, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 269664, + "depth": 50, + "doc_recall_at_k": 0.8107777777777777, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6939161915937393, + "queries": 300, + "recall_at_k": 0.9136666666666677, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9866666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 346464, + "depth": 100, + "doc_recall_at_k": 0.8145555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6971647451034574, + "queries": 300, + "recall_at_k": 0.9703333333333334, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 500064, + "depth": 200, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6981681784223372, + "queries": 300, + "recall_at_k": 0.9923333333333335, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 960864, + "depth": 500, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 0.9996666666666666, + "rescore_ms": 1, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1728864, + "depth": 1000, + "doc_recall_at_k": 0.8178888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.698128294524517, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 7, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 40 + ], + [ + 0.95, + 71 + ], + [ + 0.99, + 192 + ], + [ + 1.0, + 633 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 45.0, + "metric": "cosine", + "neighbour_band": 0.013464927673339844, + "queries": 300, + "rank_noise": 0.018715791052223443, + "recall_at_k": 0.6109999999999995, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 30 + ], + [ + 0.95, + 45 + ], + [ + 0.99, + 90 + ], + [ + 1.0, + 441 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 33.0, + "metric": "cosine", + "neighbour_band": 0.013464927673339844, + "queries": 300, + "rank_noise": 0.013085613849637225, + "recall_at_k": 0.6559999999999993, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 12 + ], + [ + 0.95, + 15 + ], + [ + 0.99, + 22 + ], + [ + 1.0, + 37 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 13.0, + "metric": "cosine", + "neighbour_band": 0.013464927673339844, + "queries": 300, + "rank_noise": 0.0059490096574797815, + "recall_at_k": 0.8446666666666696, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 21 + ], + [ + 0.95, + 31 + ], + [ + 0.99, + 59 + ], + [ + 1.0, + 271 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 23.0, + "metric": "cosine", + "neighbour_band": 0.013464927673339844, + "queries": 300, + "rank_noise": 0.011187321269100204, + "recall_at_k": 0.711333333333333, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 45 + ], + [ + 0.95, + 71 + ], + [ + 0.99, + 183 + ], + [ + 1.0, + 601 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 50.0, + "metric": "cosine", + "neighbour_band": 0.013464927673339844, + "queries": 300, + "rank_noise": 0.01640890269556495, + "recall_at_k": 0.5766666666666663, + "vectors": 8036 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/scifact-minilm-k10.json b/experiments/2026-11-two-stage-rescoring/results/scifact-minilm-k10.json new file mode 100644 index 0000000..e004c19 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/scifact-minilm-k10.json @@ -0,0 +1,1596 @@ +{ + "cells": [ + { + "bytes_read_per_query": 389648, + "depth": 10, + "doc_recall_at_k": 0.7334444444444445, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6119280130166552, + "queries": 300, + "recall_at_k": 0.7049999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 393568, + "depth": 20, + "doc_recall_at_k": 0.762888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6226293323459402, + "queries": 300, + "recall_at_k": 0.8770000000000012, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 397488, + "depth": 30, + "doc_recall_at_k": 0.7795555555555556, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6284476930941422, + "queries": 300, + "recall_at_k": 0.9333333333333349, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 405328, + "depth": 50, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.628117302061305, + "queries": 300, + "recall_at_k": 0.9680000000000005, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 424928, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9883333333333333, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 464128, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9940000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 581728, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 777728, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 401088, + "depth": 10, + "doc_recall_at_k": 0.7334444444444445, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6114915805047504, + "queries": 300, + "recall_at_k": 0.7049999999999998, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 416448, + "depth": 20, + "doc_recall_at_k": 0.762888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6225443333756476, + "queries": 300, + "recall_at_k": 0.8770000000000012, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 431808, + "depth": 30, + "doc_recall_at_k": 0.7795555555555556, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6285937722636049, + "queries": 300, + "recall_at_k": 0.9343333333333347, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 462528, + "depth": 50, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6282633812307677, + "queries": 300, + "recall_at_k": 0.9706666666666672, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 539328, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6269398576252401, + "queries": 300, + "recall_at_k": 0.9926666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 692928, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9990000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1153728, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1921728, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "binary", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 389648, + "depth": 10, + "doc_recall_at_k": 0.7387777777777778, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6140355222756023, + "queries": 300, + "recall_at_k": 0.718666666666667, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9866666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 393568, + "depth": 20, + "doc_recall_at_k": 0.7806666666666667, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.628959126523926, + "queries": 300, + "recall_at_k": 0.8986666666666681, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 397488, + "depth": 30, + "doc_recall_at_k": 0.7745555555555556, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6268306157322476, + "queries": 300, + "recall_at_k": 0.9480000000000013, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 405328, + "depth": 50, + "doc_recall_at_k": 0.7745555555555556, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6265458335383647, + "queries": 300, + "recall_at_k": 0.9790000000000003, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 424928, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6279814220652761, + "queries": 300, + "recall_at_k": 0.9929999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 464128, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9946666666666666, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 581728, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 777728, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 401088, + "depth": 10, + "doc_recall_at_k": 0.7387777777777778, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6136430168660719, + "queries": 300, + "recall_at_k": 0.718666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9866666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 416448, + "depth": 20, + "doc_recall_at_k": 0.7773333333333334, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6279105781325739, + "queries": 300, + "recall_at_k": 0.8993333333333347, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 431808, + "depth": 30, + "doc_recall_at_k": 0.7745555555555556, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267456167619551, + "queries": 300, + "recall_at_k": 0.9496666666666678, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 462528, + "depth": 50, + "doc_recall_at_k": 0.7745555555555556, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6264169074656978, + "queries": 300, + "recall_at_k": 0.9820000000000004, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 539328, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6280835741323644, + "queries": 300, + "recall_at_k": 0.9973333333333331, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9966666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 692928, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9996666666666666, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1153728, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1921728, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 775376, + "depth": 10, + "doc_recall_at_k": 0.7863888888888888, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6293506897803763, + "queries": 300, + "recall_at_k": 0.874666666666669, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 779296, + "depth": 20, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9880000000000001, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 783216, + "depth": 30, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9936666666666667, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 791056, + "depth": 50, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9946666666666666, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 810656, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 849856, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 967456, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1163456, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 786816, + "depth": 10, + "doc_recall_at_k": 0.7863888888888888, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6294528418474648, + "queries": 300, + "recall_at_k": 0.874666666666669, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 802176, + "depth": 20, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9930000000000001, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 817536, + "depth": 30, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9986666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 848256, + "depth": 50, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9996666666666666, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 925056, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1078656, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1539456, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 2307456, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "pq96", + "first_bytes": 96, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 389648, + "depth": 10, + "doc_recall_at_k": 0.7615, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6203580387683242, + "queries": 300, + "recall_at_k": 0.7640000000000003, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 393568, + "depth": 20, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6256968192435285, + "queries": 300, + "recall_at_k": 0.937666666666668, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 397488, + "depth": 30, + "doc_recall_at_k": 0.7812222222222223, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6277573278768368, + "queries": 300, + "recall_at_k": 0.9696666666666668, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 405328, + "depth": 50, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9879999999999999, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 424928, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9939999999999998, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 464128, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.9943333333333332, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 581728, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 777728, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6266916263886889, + "queries": 300, + "recall_at_k": 0.995, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 401088, + "depth": 10, + "doc_recall_at_k": 0.7615, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6201269606285689, + "queries": 300, + "recall_at_k": 0.7640000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.99, + "vectors": 8036 + }, + { + "bytes_read_per_query": 416448, + "depth": 20, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6257989713106168, + "queries": 300, + "recall_at_k": 0.9390000000000011, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 431808, + "depth": 30, + "doc_recall_at_k": 0.7812222222222223, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6278594799439252, + "queries": 300, + "recall_at_k": 0.9720000000000003, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 462528, + "depth": 50, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9923333333333335, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 539328, + "depth": 100, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9986666666666666, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 692928, + "depth": 200, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 0.9993333333333332, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1153728, + "depth": 500, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1921728, + "depth": 1000, + "doc_recall_at_k": 0.777888888888889, + "first": "pq48", + "first_bytes": 48, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6267937784557772, + "queries": 300, + "recall_at_k": 1.0, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 196784, + "depth": 10, + "doc_recall_at_k": 0.7068333333333333, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.5905720033949968, + "queries": 300, + "recall_at_k": 0.6363333333333332, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9633333333333334, + "vectors": 8036 + }, + { + "bytes_read_per_query": 200704, + "depth": 20, + "doc_recall_at_k": 0.7556666666666667, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6150679548538306, + "queries": 300, + "recall_at_k": 0.8063333333333339, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9766666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 204624, + "depth": 30, + "doc_recall_at_k": 0.7667777777777778, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6209119745301801, + "queries": 300, + "recall_at_k": 0.8773333333333344, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 212464, + "depth": 50, + "doc_recall_at_k": 0.767888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6237653633002108, + "queries": 300, + "recall_at_k": 0.9340000000000004, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 232064, + "depth": 100, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6260214232766527, + "queries": 300, + "recall_at_k": 0.974666666666667, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 271264, + "depth": 200, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6256881930698088, + "queries": 300, + "recall_at_k": 0.9883333333333332, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 388864, + "depth": 500, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.625640076799403, + "queries": 300, + "recall_at_k": 0.9936666666666666, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 584864, + "depth": 1000, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.625640076799403, + "queries": 300, + "recall_at_k": 0.9946666666666666, + "rescore_ms": 0, + "second": "int8", + "second_bytes": 392, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 208224, + "depth": 10, + "doc_recall_at_k": 0.7068333333333333, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.591904389616847, + "queries": 300, + "recall_at_k": 0.6363333333333332, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9633333333333334, + "vectors": 8036 + }, + { + "bytes_read_per_query": 223584, + "depth": 20, + "doc_recall_at_k": 0.7556666666666667, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6161097014141002, + "queries": 300, + "recall_at_k": 0.8063333333333339, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9766666666666667, + "vectors": 8036 + }, + { + "bytes_read_per_query": 238944, + "depth": 30, + "doc_recall_at_k": 0.7667777777777778, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6212194809694179, + "queries": 300, + "recall_at_k": 0.8783333333333345, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9833333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 269664, + "depth": 50, + "doc_recall_at_k": 0.767888888888889, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6240728697394486, + "queries": 300, + "recall_at_k": 0.9370000000000005, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 0.9933333333333333, + "vectors": 8036 + }, + { + "bytes_read_per_query": 346464, + "depth": 100, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6261235753437411, + "queries": 300, + "recall_at_k": 0.9786666666666669, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 500064, + "depth": 200, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6257903451368972, + "queries": 300, + "recall_at_k": 0.9933333333333332, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 960864, + "depth": 500, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6257422288664916, + "queries": 300, + "recall_at_k": 0.9986666666666667, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + }, + { + "bytes_read_per_query": 1728864, + "depth": 1000, + "doc_recall_at_k": 0.7745555555555556, + "first": "pq24", + "first_bytes": 24, + "k": 10, + "labelled_queries": 300, + "metric": "cosine", + "ndcg_at_k": 0.6257422288664916, + "queries": 300, + "recall_at_k": 0.9996666666666666, + "rescore_ms": 0, + "second": "f32", + "second_bytes": 1536, + "top1_agreement": 1.0, + "vectors": 8036 + } + ], + "profiles": [ + { + "depth_for_recall": [ + [ + 0.9, + 24 + ], + [ + 0.95, + 36 + ], + [ + 0.99, + 85 + ], + [ + 1.0, + 432 + ] + ], + "first": "binary", + "first_bytes": 48, + "k": 10, + "median_full_depth": 26.0, + "metric": "cosine", + "neighbour_band": 0.03073228895664215, + "queries": 300, + "rank_noise": 0.027713695127966537, + "recall_at_k": 0.7049999999999998, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 21 + ], + [ + 0.95, + 31 + ], + [ + 0.99, + 60 + ], + [ + 1.0, + 372 + ] + ], + "first": "binary-centred", + "first_bytes": 48, + "k": 10, + "median_full_depth": 23.0, + "metric": "cosine", + "neighbour_band": 0.03073228895664215, + "queries": 300, + "rank_noise": 0.026426586664381465, + "recall_at_k": 0.718666666666667, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 11 + ], + [ + 0.95, + 13 + ], + [ + 0.99, + 19 + ], + [ + 1.0, + 60 + ] + ], + "first": "pq96", + "first_bytes": 96, + "k": 10, + "median_full_depth": 12.0, + "metric": "cosine", + "neighbour_band": 0.03073228895664215, + "queries": 300, + "rank_noise": 0.011711888456250843, + "recall_at_k": 0.874666666666669, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 17 + ], + [ + 0.95, + 22 + ], + [ + 0.99, + 45 + ], + [ + 1.0, + 261 + ] + ], + "first": "pq48", + "first_bytes": 48, + "k": 10, + "median_full_depth": 18.0, + "metric": "cosine", + "neighbour_band": 0.03073228895664215, + "queries": 300, + "rank_noise": 0.022971436105667608, + "recall_at_k": 0.7640000000000003, + "vectors": 8036 + }, + { + "depth_for_recall": [ + [ + 0.9, + 36 + ], + [ + 0.95, + 58 + ], + [ + 0.99, + 168 + ], + [ + 1.0, + 1083 + ] + ], + "first": "pq24", + "first_bytes": 24, + "k": 10, + "median_full_depth": 37.0, + "metric": "cosine", + "neighbour_band": 0.03073228895664215, + "queries": 300, + "rank_noise": 0.03588049972806333, + "recall_at_k": 0.6363333333333332, + "vectors": 8036 + } + ] +} diff --git a/experiments/2026-11-two-stage-rescoring/results/summary.txt b/experiments/2026-11-two-stage-rescoring/results/summary.txt new file mode 100644 index 0000000..95188f4 --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/results/summary.txt @@ -0,0 +1,140 @@ +Depth an exact (f32) re-score needs, as a multiple of k +collection first bytes noise/band recall@k 90% 95% 99% median full +arguana-bge-small binary 48 0.786 0.7482 1.8 2.8 6.2 1.9 +arguana-bge-small binary-centred 48 0.788 0.7726 1.6 2.2 4.5 1.7 +arguana-bge-small pq96 96 0.289 0.9031 1.0 1.2 1.6 1.1 +arguana-bge-small pq48 48 0.681 0.8110 1.4 1.7 3.2 1.5 +arguana-bge-small pq24 24 1.198 0.7121 2.0 3.2 7.2 2.2 +arguana-minilm binary 48 0.513 0.8148 1.3 1.7 2.8 1.4 +arguana-minilm binary-centred 48 0.511 0.8240 1.3 1.6 2.7 1.4 +arguana-minilm pq96 96 0.202 0.9241 1.0 1.1 1.4 1.1 +arguana-minilm pq48 48 0.486 0.8486 1.2 1.5 2.3 1.3 +arguana-minilm pq24 24 0.930 0.7434 1.7 2.4 4.9 1.8 +cosqa-bge-small binary 48 0.957 0.6598 3.1 4.7 13.2 1.8 +cosqa-bge-small binary-centred 48 0.695 0.7252 2.5 4.2 8.9 1.5 +cosqa-bge-small pq96 96 0.326 0.8776 1.2 1.6 2.7 1.0 +cosqa-bge-small pq48 48 0.615 0.7728 2.0 2.8 6.8 1.3 +cosqa-bge-small pq24 24 0.949 0.6734 3.8 6.9 19.0 2.3 +cosqa-minilm binary 48 0.751 0.7190 2.4 3.6 8.0 1.5 +cosqa-minilm binary-centred 48 0.714 0.7246 2.2 3.4 7.9 1.5 +cosqa-minilm pq96 96 0.315 0.8970 1.1 1.6 2.6 1.0 +cosqa-minilm pq48 48 0.649 0.7750 2.1 2.8 5.1 1.4 +cosqa-minilm pq24 24 1.016 0.6400 3.4 5.8 14.6 2.1 +fiqa-bge-small binary 48 1.645 0.5367 6.0 10.5 32.7 6.7 +fiqa-bge-small binary-centred 48 1.067 0.6002 3.9 5.8 14.3 4.2 +fiqa-bge-small pq96 96 0.528 0.8165 1.3 1.6 2.5 1.5 +fiqa-bge-small pq48 48 0.978 0.6418 2.9 4.4 8.8 3.4 +fiqa-bge-small pq24 24 1.345 0.4454 9.8 16.5 45.3 11.0 +fiqa-minilm binary 48 0.927 0.6656 2.6 3.8 8.6 3.0 +fiqa-minilm binary-centred 48 0.881 0.6826 2.4 3.6 7.3 2.8 +fiqa-minilm pq96 96 0.433 0.8568 1.2 1.4 2.2 1.3 +fiqa-minilm pq48 48 0.811 0.7162 2.0 2.9 5.6 2.3 +fiqa-minilm pq24 24 1.241 0.5069 6.0 9.9 25.0 6.7 +nfcorpus-bge-small binary 48 1.654 0.5560 6.4 11.6 42.4 5.8 +nfcorpus-bge-small binary-centred 48 1.113 0.5827 4.6 7.4 17.0 4.2 +nfcorpus-bge-small pq96 96 0.498 0.8350 1.3 1.6 2.5 1.4 +nfcorpus-bge-small pq48 48 0.907 0.6842 2.5 3.7 8.6 2.5 +nfcorpus-bge-small pq24 24 1.247 0.5294 6.3 10.6 31.5 6.2 +nfcorpus-minilm binary 48 0.943 0.6690 2.8 4.3 10.0 3.1 +nfcorpus-minilm binary-centred 48 0.874 0.6907 2.5 4.0 9.1 2.7 +nfcorpus-minilm pq96 96 0.378 0.8743 1.1 1.3 2.0 1.2 +nfcorpus-minilm pq48 48 0.734 0.7424 1.8 2.5 4.6 1.9 +nfcorpus-minilm pq24 24 1.140 0.5944 3.6 5.9 13.6 3.9 +scifact-bge-small binary 48 1.390 0.6110 4.0 7.1 19.2 4.5 +scifact-bge-small binary-centred 48 0.972 0.6560 3.0 4.5 9.0 3.3 +scifact-bge-small pq96 96 0.442 0.8447 1.2 1.5 2.2 1.3 +scifact-bge-small pq48 48 0.831 0.7113 2.1 3.1 5.9 2.3 +scifact-bge-small pq24 24 1.219 0.5767 4.5 7.1 18.3 5.0 +scifact-minilm binary 48 0.902 0.7050 2.4 3.6 8.5 2.6 +scifact-minilm binary-centred 48 0.860 0.7187 2.1 3.1 6.0 2.3 +scifact-minilm pq96 96 0.381 0.8747 1.1 1.3 1.9 1.2 +scifact-minilm pq48 48 0.747 0.7640 1.7 2.2 4.5 1.8 +scifact-minilm pq24 24 1.168 0.6363 3.6 5.8 16.8 3.7 + +50 (collection, first stage) points +recall 90%: depth/k = 3.07 × (noise/band)^1.06; spearman 0.930; held-out corpus error: a factor of 1.25 on average +recall 95%: depth/k = 4.76 × (noise/band)^1.29; spearman 0.934; held-out corpus error: a factor of 1.31 on average +recall 99%: depth/k = 11.22 × (noise/band)^1.74; spearman 0.925; held-out corpus error: a factor of 1.41 on average + +Recall@k and nDCG@k by cell (mean over collections), and the best first stage per byte +first second depth recall@k ndcg loss vs f32 +binary f32 10 0.6685 0.0138 +binary f32 20 0.8405 0.0035 +binary f32 30 0.8999 0.0017 +binary f32 50 0.9482 0.0003 +binary f32 100 0.9808 -0.0001 +binary f32 200 0.9930 0.0001 +binary f32 500 0.9985 -0.0000 +binary f32 1000 0.9998 0.0000 +binary int8 10 0.6685 0.0138 +binary int8 20 0.8392 0.0033 +binary int8 30 0.8976 0.0017 +binary int8 50 0.9439 0.0004 +binary int8 100 0.9739 -0.0001 +binary int8 200 0.9847 0.0001 +binary int8 500 0.9894 0.0001 +binary int8 1000 0.9905 0.0001 +binary-centred f32 10 0.6977 0.0125 +binary-centred f32 20 0.8691 0.0026 +binary-centred f32 30 0.9239 0.0014 +binary-centred f32 50 0.9662 0.0002 +binary-centred f32 100 0.9910 0.0000 +binary-centred f32 200 0.9981 0.0000 +binary-centred f32 500 0.9999 0.0000 +binary-centred f32 1000 1.0000 0.0000 +binary-centred int8 10 0.6977 0.0124 +binary-centred int8 20 0.8672 0.0024 +binary-centred int8 30 0.9202 0.0013 +binary-centred int8 50 0.9605 0.0002 +binary-centred int8 100 0.9830 0.0000 +binary-centred int8 200 0.9890 0.0001 +binary-centred int8 500 0.9905 0.0001 +binary-centred int8 1000 0.9906 0.0001 +pq24 f32 10 0.6058 0.0309 +pq24 f32 20 0.7791 0.0119 +pq24 f32 30 0.8551 0.0075 +pq24 f32 50 0.9182 0.0046 +pq24 f32 100 0.9680 0.0020 +pq24 f32 200 0.9896 0.0003 +pq24 f32 500 0.9980 0.0001 +pq24 f32 1000 0.9996 0.0001 +pq24 int8 10 0.6058 0.0311 +pq24 int8 20 0.7781 0.0119 +pq24 int8 30 0.8529 0.0076 +pq24 int8 50 0.9146 0.0048 +pq24 int8 100 0.9620 0.0020 +pq24 int8 200 0.9819 0.0004 +pq24 int8 500 0.9889 0.0001 +pq24 int8 1000 0.9902 0.0002 +pq48 f32 10 0.7467 0.0078 +pq48 f32 20 0.9125 0.0019 +pq48 f32 30 0.9588 0.0006 +pq48 f32 50 0.9863 0.0002 +pq48 f32 100 0.9977 -0.0001 +pq48 f32 200 0.9997 0.0000 +pq48 f32 500 1.0000 0.0000 +pq48 f32 1000 1.0000 0.0000 +pq48 int8 10 0.7467 0.0078 +pq48 int8 20 0.9098 0.0019 +pq48 int8 30 0.9539 0.0006 +pq48 int8 50 0.9788 0.0002 +pq48 int8 100 0.9888 -0.0000 +pq48 int8 200 0.9903 0.0001 +pq48 int8 500 0.9905 0.0001 +pq48 int8 1000 0.9905 0.0001 +pq96 f32 10 0.8704 0.0005 +pq96 f32 20 0.9864 -0.0001 +pq96 f32 30 0.9973 -0.0001 +pq96 f32 50 0.9997 -0.0001 +pq96 f32 100 0.9999 0.0000 +pq96 f32 200 1.0000 0.0000 +pq96 f32 500 1.0000 0.0000 +pq96 f32 1000 1.0000 0.0000 +pq96 int8 10 0.8704 0.0005 +pq96 int8 20 0.9791 -0.0002 +pq96 int8 30 0.9882 -0.0000 +pq96 int8 50 0.9903 -0.0000 +pq96 int8 100 0.9905 0.0001 +pq96 int8 200 0.9905 0.0001 +pq96 int8 500 0.9905 0.0001 +pq96 int8 1000 0.9905 0.0001 diff --git a/experiments/2026-11-two-stage-rescoring/run.sh b/experiments/2026-11-two-stage-rescoring/run.sh new file mode 100755 index 0000000..ce9900e --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/run.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Two-stage retrieval: scan with a 24-96 byte code, re-score the best N candidates with int8 or +# f32, keep the top 10. Measures how deep the re-score must go for a target recall, and whether +# the first stage's noise-to-spacing ratio predicts that depth. Retrieval only, no chat model. +# Reuses the ten collections built by ../2026-11-neighbour-spacing (and runs it if they are +# missing). Needs: a built `hvec` (cargo build --release). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$HERE/../.." +HVEC="${HVEC:-$ROOT/target/release/hvec}" +SPACING="$HERE/../2026-11-neighbour-spacing" +SPACING_WORK="${SPACING_WORK:-$SPACING/work}" +K="${K:-10}" +CORPORA="${CORPORA:-scifact nfcorpus arguana cosqa fiqa}" +MODELS="${MODELS:-bge-small minilm}" +FIRST="${FIRST:-binary,binary-centred,pq96,pq48,pq24}" +SECOND="${SECOND:-int8,f32}" +DEPTHS="${DEPTHS:-10,20,30,50,100,200,500,1000}" +mkdir -p "$HERE/results" + +export HVEC_CONFIG="$SPACING_WORK/config.toml" +if [ ! -f "$HVEC_CONFIG" ]; then + WORK="$SPACING_WORK" CORPORA="$CORPORA" MODELS="$MODELS" "$SPACING/run.sh" +fi + +for c in $CORPORA; do + for m in $MODELS; do + col="$c-$m" + "$HVEC" bench rescore --collection "$col" -k "$K" --codecs "$FIRST" --rescore "$SECOND" \ + --depths "$DEPTHS" --questions "$SPACING_WORK/$c.jsonl" --json > "$HERE/results/$col-k$K.json" + echo "== $col done" + done +done +cp "$HVEC_CONFIG" "$HERE/results/config.toml" +python3 -I "$HERE/summarise.py" "$HERE/results" "$K" | tee "$HERE/results/summary.txt" diff --git a/experiments/2026-11-two-stage-rescoring/summarise.py b/experiments/2026-11-two-stage-rescoring/summarise.py new file mode 100644 index 0000000..54f71fb --- /dev/null +++ b/experiments/2026-11-two-stage-rescoring/summarise.py @@ -0,0 +1,103 @@ +"""Summarise `bench rescore` JSON: the depth each first stage needs, against its noise-to-band +ratio, and the recall and nDCG of each (first, second, depth) cell. + + python3 -I summarise.py +""" + +import json +import math +import sys +from pathlib import Path + + +def ranks(xs): + order = sorted(range(len(xs)), key=lambda i: xs[i]) + r = [0.0] * len(xs) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and xs[order[j + 1]] == xs[order[i]]: + j += 1 + for t in range(i, j + 1): + r[order[t]] = (i + j) / 2 + 1 + i = j + 1 + return r + + +def pearson(a, b): + n = len(a) + ma, mb = sum(a) / n, sum(b) / n + cov = sum((x - ma) * (y - mb) for x, y in zip(a, b)) + return cov / ((sum((x - ma) ** 2 for x in a) * sum((y - mb) ** 2 for y in b)) ** 0.5) + + +def fit(xs, ys): + """Least squares y = a + b x.""" + n = len(xs) + mx, my = sum(xs) / n, sum(ys) / n + b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs) + return my - b * mx, b + + +def main(results: Path, k: int) -> None: + profiles, cells = [], [] + for path in sorted(results.glob(f"*-k{k}.json")): + col = path.name[: -len(f"-k{k}.json")] + data = json.loads(path.read_text()) + for p in data["profiles"]: + p["collection"] = col + p["ratio"] = p["rank_noise"] / p["neighbour_band"] + p["depth"] = {t: d for t, d in p["depth_for_recall"]} + profiles.append(p) + for c in data["cells"]: + c["collection"] = col + cells.append(c) + + print("Depth an exact (f32) re-score needs, as a multiple of k") + print(f"{'collection':<20} {'first':<15} {'bytes':>5} {'noise/band':>10} {'recall@k':>8} " + f"{'90%':>6} {'95%':>6} {'99%':>6} {'median full':>11}") + for p in profiles: + d = p["depth"] + print(f"{p['collection']:<20} {p['first']:<15} {p['first_bytes']:>5} {p['ratio']:>10.3f} " + f"{p['recall_at_k']:>8.4f} {d[0.9] / k:>6.1f} {d[0.95] / k:>6.1f} {d[0.99] / k:>6.1f} " + f"{p['median_full_depth'] / k:>11.1f}") + + print() + print(f"{len(profiles)} (collection, first stage) points") + for t in (0.9, 0.95, 0.99): + xs = [math.log(p["ratio"]) for p in profiles] + ys = [math.log(p["depth"][t] / k) for p in profiles] + a, b = fit(xs, ys) + rho = pearson(ranks(xs), ranks(ys)) + # Leave one corpus out. + errs = [] + for corpus in sorted({p["collection"].split("-")[0] for p in profiles}): + tr = [p for p in profiles if not p["collection"].startswith(corpus + "-")] + te = [p for p in profiles if p["collection"].startswith(corpus + "-")] + a2, b2 = fit([math.log(p["ratio"]) for p in tr], [math.log(p["depth"][t] / k) for p in tr]) + for p in te: + pred = math.exp(a2 + b2 * math.log(p["ratio"])) + errs.append(abs(math.log(pred / (p["depth"][t] / k)))) + geo = math.exp(sum(errs) / len(errs)) + print(f"recall {t:.0%}: depth/k = {math.exp(a):.2f} × (noise/band)^{b:.2f}; spearman {rho:.3f}; " + f"held-out corpus error: a factor of {geo:.2f} on average") + + print() + print("Recall@k and nDCG@k by cell (mean over collections), and the best first stage per byte") + keys = sorted({(c["first"], c["second"], c["depth"]) for c in cells}, key=lambda x: (x[0], x[1], x[2])) + print(f"{'first':<15} {'second':<6} {'depth':>6} {'recall@k':>9} {'ndcg loss vs f32':>16}") + # The exact f32 nDCG per collection comes from the spacing experiment's results. + f32_ndcg = {} + spacing = results.parent.parent / "2026-11-neighbour-spacing" / "results" + for path in spacing.glob(f"*-k{k}.json"): + f32_ndcg[path.name[: -len(f"-k{k}.json")]] = json.loads(path.read_text())[0]["ndcg_at_k"] + for key in keys: + sel = [c for c in cells if (c["first"], c["second"], c["depth"]) == key] + rec = sum(c["recall_at_k"] for c in sel) / len(sel) + losses = [f32_ndcg[c["collection"]] - c["ndcg_at_k"] for c in sel if c["collection"] in f32_ndcg] + loss = f"{sum(losses) / len(losses):>16.4f}" if losses else f"{'-':>16}" + print(f"{key[0]:<15} {key[1]:<6} {key[2]:>6} {rec:>9.4f} {loss}") + + +if __name__ == "__main__": + main(Path(sys.argv[1]), int(sys.argv[2]))