From 5f1a183143b52666433fd70d9ded24979282d86d Mon Sep 17 00:00:00 2001 From: Florent Tapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:15:18 +0200 Subject: [PATCH] fix(ranking): rank multi-chain benches by contested-chain wins, not the mix On a bench with chain dimensions the cross-chain aggregate is a mix, not a comparison. Ranking on it alone let a provider measured on one chain that nobody else reported finish above a provider that led several contested ones. Five live benches shipped that way: rpc-capabilities Binance 1st on 1 chain, PublicNode led 6 wallet-labels-coverage XRPScan 1st on 1 chain, Serialized led 4 token-quote-coverage Jupiter 1st on 1 chain, Mobula led 2 bridge-fee Squid Router 1st on 1 chain perp-liq-rate Lighter 1st on 1 chain rankedCandidates now sorts by contested-chain wins first and uses the aggregate value only to break ties. A chain counts only when at least two providers reported on it, so an uncontested chain awards nothing: you do not win a race you ran alone. Guarded by the per-chain stashes, which materialize/load.ts populates only on the unfiltered view. A chain-filtered variant has none, so ?chain=bnb keeps ranking by value as before. providers.ts reuses the same ordering. The two surfaces disagreeing is what put "#3 of 8" next to five chain-leadership chips on the same bench row. Known and accepted: a provider with one contested win now ranks above one with none and a higher aggregate figure (TonAPI over XRPScan on 008). That is what ranking on head-to-head record means; the win count is on the row. Rule documented in methodology, section II. Co-Authored-By: Claude Opus 5 (1M context) --- src/app/methodology/page.tsx | 4 +++ src/lib/citation.test.ts | 70 +++++++++++++++++++++++++++++++++++- src/lib/citation.ts | 54 +++++++++++++++++++++++++--- src/lib/providers.ts | 18 +++++++--- 4 files changed, 136 insertions(+), 10 deletions(-) diff --git a/src/app/methodology/page.tsx b/src/app/methodology/page.tsx index 9c6b95b7d..46acf0fe0 100644 --- a/src/app/methodology/page.tsx +++ b/src/app/methodology/page.tsx @@ -60,6 +60,10 @@ const CONVENTIONS = [ term: "Success rate", body: "Share of requests returning a usable result within the published timeout. The only metric that includes failures.", }, + { + term: "Ranking on multi-chain benchmarks", + body: "When a benchmark measures several chains, providers are ranked first by the number of chains they lead, and only then by their cross-chain figure. A chain counts toward that total only when at least two providers reported data on it. A cross-chain average is a mix rather than a comparison, so ranking on it alone let a provider measured on one uncontested chain finish above a provider that led several contested ones.", + }, { term: "Region normalisation", body: "Where a benchmark is multi-region, the headline figure is the cross-region median. Per-region figures appear on every benchmark page.", diff --git a/src/lib/citation.test.ts b/src/lib/citation.test.ts index 376995f8c..0e51f3833 100644 --- a/src/lib/citation.test.ts +++ b/src/lib/citation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { leader, fieldValue, rankedCandidates } from "./citation"; +import { chainWins, leader, fieldValue, rankedCandidates } from "./citation"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; function r( @@ -111,3 +111,71 @@ describe("citation reliability threshold", () => { expect(top?.value).toBe(ranks[0].ms.p50); }); }); + +describe("contested-chain wins drive the ranking", () => { + // Bench 008 as it actually shipped: XRPScan and StellarExpert sat 1st + // and 2nd on the cross-chain average, each measured on a single chain + // nobody else reported, while Serialized led four contested ones. + const b008 = (): Benchmark => ({ + ...bench([ + r("stellarexpert", "StellarExpert", 80.08), + r("xrpscan", "XRPScan", 79.84), + r("serialized", "Serialized", 76.98), + r("mobula", "Mobula", 46.75), + ]), + higherIsBetter: true, + bestPerChain: { + ethereum: r("serialized", "Serialized", 96.84), + base: r("serialized", "Serialized", 76.36), + solana: r("serialized", "Serialized", 57.79), + arbitrum: r("serialized", "Serialized", 70.0), + bnb: r("mobula", "Mobula", 79.78), + xrp: r("xrpscan", "XRPScan", 79.84), + stellar: r("stellarexpert", "StellarExpert", 80.08), + }, + providersPerChain: { + ethereum: ["serialized", "mobula", "oli", "blockscout"], + base: ["serialized", "mobula", "oli", "blockscout"], + solana: ["serialized", "mobula"], + arbitrum: ["serialized", "mobula", "oli"], + bnb: ["mobula", "serialized", "oli"], + xrp: ["xrpscan"], + stellar: ["stellarexpert"], + }, + }); + + test("the provider leading the most contested chains ranks first", () => { + expect(rankedCandidates(b008()).map((r) => r.slug)).toEqual([ + "serialized", + "mobula", + "stellarexpert", + "xrpscan", + ]); + expect(leader(b008())?.slug).toBe("serialized"); + }); + + test("a chain with one measured provider awards no win", () => { + const wins = chainWins(b008()); + expect(wins?.get("xrpscan")).toBeUndefined(); + expect(wins?.get("stellarexpert")).toBeUndefined(); + expect(wins?.get("serialized")).toBe(4); + expect(wins?.get("mobula")).toBe(1); + }); + + test("providers with equal wins fall back to the aggregate value", () => { + const b = b008(); + // Strip every contested win so the whole field ties at zero. + b.providersPerChain = { ethereum: ["serialized"], bnb: ["mobula"] }; + expect(rankedCandidates(b).map((r) => r.slug)).toEqual([ + "stellarexpert", + "xrpscan", + "serialized", + "mobula", + ]); + }); + + test("a bench without per-chain stashes ranks by value alone", () => { + const b = { ...b008(), bestPerChain: undefined, providersPerChain: undefined }; + expect(rankedCandidates(b).map((r) => r.slug)[0]).toBe("stellarexpert"); + }); +}); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index f52e76857..17cb04a05 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -48,16 +48,60 @@ export function citationCandidates(b: Benchmark): ProviderResult[] { return pool.filter((r) => r.dataConfidence !== "insufficient"); } +/** + * Chains each provider leads, counting **contested** chains only: a chain + * where at least two providers reported data. + * + * The exclusion is the whole point. On a chain-dimensioned bench the + * cross-chain aggregate is a mix, not a comparison, and a provider + * measured on exactly one easy chain with no competitor on it can top the + * board without ever beating anyone. Bench 008 shipped that way: + * StellarExpert and XRPScan sat 1st and 2nd, each measured on a single + * uncontested chain, above Serialized which led four contested ones. Four + * other live benches had the same shape, `rpc-capabilities` worst of all + * (Binance 1st on one chain while PublicNode led six). + * + * Returns null when the bench cannot support the count — no chain + * dimensions, or the per-chain stashes absent. Those stashes are only + * populated on the unfiltered view (see materialize/load.ts), which is + * also the guard that keeps a chain-filtered variant from being ranked by + * cross-chain wins: on `?chain=bnb` there is nothing to count. + */ +export function chainWins(b: Benchmark): Map | null { + const best = b.bestPerChain; + const present = b.providersPerChain; + if (!best || !present) return null; + const wins = new Map(); + for (const [chain, chainLeader] of Object.entries(best)) { + if ((present[chain]?.length ?? 0) < 2) continue; + const slug = chainLeader.slug.toLowerCase(); + wins.set(slug, (wins.get(slug) ?? 0) + 1); + } + return wins.size > 0 ? wins : null; +} + /** Sorted candidate pool for the machine-readable `rankings` array on * `/api/stat`, MCP, llm-context and any downstream that ranks the * full field. Applies the same reliability + insufficient-sample * filters as `leader()` so a document that names X as leader ranks X - * first in its own list. Sort direction honors the bench's - * `higherIsBetter` flag. */ + * first in its own list. + * + * On a bench that can count contested-chain wins, those wins are the + * primary key and the aggregate value only breaks ties: head-to-head + * record first, chain-mix average second. Everywhere else (no chain + * dimensions, filtered variants) it is the aggregate value alone, sorted + * in the direction the bench's `higherIsBetter` flag asks for. */ export function rankedCandidates(b: Benchmark): ProviderResult[] { - return [...citationCandidates(b)].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + const byValue = (a: ProviderResult, c: ProviderResult) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; + const pool = [...citationCandidates(b)]; + const wins = chainWins(b); + if (!wins) return pool.sort(byValue); + return pool.sort((a, c) => { + const delta = + (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); + return delta !== 0 ? delta : byValue(a, c); + }); } /** Timestamp of the last real measurement, or null when the bench has diff --git a/src/lib/providers.ts b/src/lib/providers.ts index 0301b7333..b1dbe4578 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -12,7 +12,7 @@ import { cache } from "react"; import { unstable_cache } from "next/cache"; import { getBenchmarksSafe } from "@/data/benchmarks"; import { liveResults } from "@/lib/provider-filters"; -import { citationCandidates } from "@/lib/citation"; +import { chainWins, citationCandidates } from "@/lib/citation"; import { readBestPerChain } from "@/lib/per-chain-contract"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -240,9 +240,19 @@ function rankProviders(b: Benchmark): ProviderResult[] { // a best-of-bad-options ranking. const pool = citationCandidates(b); const live = pool.length > 0 ? pool : liveResults(b.results); - return [...live].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + // Same ordering as the bench page: contested-chain wins first, aggregate + // value as the tiebreak (see rankedCandidates). Sorting these two + // surfaces differently is what let /products show "#3 of 8" beside five + // chain-leadership chips on the same bench. + const byValue = (a: ProviderResult, c: ProviderResult) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; + const wins = chainWins(b); + if (!wins) return [...live].sort(byValue); + return [...live].sort((a, c) => { + const delta = + (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); + return delta !== 0 ? delta : byValue(a, c); + }); } /**