From 721a4eb6e4f3c22be829b57723ed956417fff94e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 13:45:04 +0000 Subject: [PATCH 1/2] Admin classifier: per-day hover card and estimated cost The daily activity chart on /admin/classifier gets the same hover/focus card as the /admin/metrics charts: the day, each series, the total, and the day's model calls and estimated cost. Each OpenRouter Decisions call now writes its usage block to a new Analytics Engine dataset (uploads_classifier_calls). The overview reads it back per day and the page shows an "Est. cost" tile for the window. Calls without a reported cost are scaled from the priced ones. Absent binding or AE token is a no-op and the tile reads unavailable. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017RAV94xRKfKGB7yS6GwyNB --- apps/api/src/analytics-engine.test.ts | 42 +++++ apps/api/src/analytics-engine.ts | 68 ++++++++ apps/api/src/classifier-admin-wire.ts | 19 +++ apps/api/src/classifier-admin.ts | 28 ++-- .../api/src/classifier-call-analytics.test.ts | 36 ++++ apps/api/src/classifier-call-analytics.ts | 60 +++++++ apps/api/src/classifier.test.ts | 26 +++ apps/api/src/classifier.ts | 8 +- apps/api/src/routes/admin-ui.ts | 9 +- apps/api/wrangler.jsonc | 11 ++ .../src/components/admin/AdminClassifier.tsx | 154 +++++++++++++++--- apps/web/src/lib/admin-classifier.test.ts | 27 +++ apps/web/src/lib/admin-classifier.ts | 33 ++++ 13 files changed, 484 insertions(+), 37 deletions(-) create mode 100644 apps/api/src/classifier-call-analytics.test.ts create mode 100644 apps/api/src/classifier-call-analytics.ts diff --git a/apps/api/src/analytics-engine.test.ts b/apps/api/src/analytics-engine.test.ts index ae26390e8..4be215bd1 100644 --- a/apps/api/src/analytics-engine.test.ts +++ b/apps/api/src/analytics-engine.test.ts @@ -4,6 +4,7 @@ import { classifyMediaClass, fetchBreakdown, fetchSlowOps, + fetchClassifierCostSeries, fetchUploadClassSeries, slowOpsQuery, uploadClassSeriesQuery, @@ -321,3 +322,44 @@ describe("fetchSlowOps", () => { expect(result).toEqual({ available: false, reason: "query_failed" }); }); }); + +describe("fetchClassifierCostSeries", () => { + const now = new Date("2026-08-03T12:00:00Z"); + + it("reads calls, priced calls, and cost per day and fills quiet days", async () => { + let body = ""; + const impl = (async (_url: unknown, init?: RequestInit) => { + body = String(init?.body); + return new Response( + JSON.stringify({ + data: [ + { day: "2026-08-01", calls: 4, priced: 3, cost: 0.03 }, + { day: "2026-08-03", calls: "2", priced: "2", cost: "0.02" }, + ], + }), + ); + }) as unknown as typeof fetch; + const result = await fetchClassifierCostSeries(env(), 3, impl, now); + expect(body).toContain("FROM uploads_classifier_calls"); + expect(body).toContain("double1 >= 0"); + expect(result).toEqual({ + available: true, + days: [ + { day: "2026-08-01", calls: 4, priced: 3, cost: 0.03 }, + { day: "2026-08-02", calls: 0, priced: 0, cost: 0 }, + { day: "2026-08-03", calls: 2, priced: 2, cost: 0.02 }, + ], + }); + }); + + it("is unavailable without a token or when the query fails", async () => { + expect(await fetchClassifierCostSeries(env({ ANALYTICS_API_TOKEN: undefined }), 7)).toEqual({ + available: false, + reason: "not_configured", + }); + expect(await fetchClassifierCostSeries(env(), 7, jsonFetch({}, 500))).toEqual({ + available: false, + reason: "query_failed", + }); + }); +}); diff --git a/apps/api/src/analytics-engine.ts b/apps/api/src/analytics-engine.ts index c1261fff9..63e346656 100644 --- a/apps/api/src/analytics-engine.ts +++ b/apps/api/src/analytics-engine.ts @@ -21,10 +21,12 @@ import { BLOB_ORDER } from "./adoption"; import { fillDaySeries, normalizeUtcDay } from "./day-series"; import { windowStart } from "./adoption-queries"; import { SLOW_OP_BLOB_ORDER } from "./slow-op-analytics"; +import type { ClassifierCostDay, ClassifierCostSeries } from "./classifier-admin-wire"; const SQL_ENDPOINT = "https://api.cloudflare.com/client/v4/accounts"; const DATASET = "uploads_adoption"; const SLOW_OP_DATASET = "uploads_slow_ops"; +const CLASSIFIER_CALL_DATASET = "uploads_classifier_calls"; /** * Dimensions the breakdown panel lets an operator group by — a DELIBERATELY @@ -295,3 +297,69 @@ export async function fetchSlowOps( return { available: false, reason: "query_failed" }; } } + +/** + * Per-day Decisions calls and their summed `usage.cost` from + * `uploads_classifier_calls` (written by `classifier-call-analytics.ts`). + * `double1` is the cost, -1 when a response carried none, so priced calls + * are counted apart and only they add to the sum. + */ +export function classifierCostSeriesQuery(days: number): string { + const window = Number.isFinite(days) ? Math.max(1, Math.min(90, Math.floor(days))) : 30; + return `SELECT formatDateTime(toStartOfInterval(timestamp, INTERVAL '1' DAY), '%Y-%m-%d') AS day, + SUM(_sample_interval) AS calls, + SUM(if(double1 >= 0, _sample_interval, 0)) AS priced, + SUM(if(double1 >= 0, double1 * _sample_interval, 0)) AS cost + FROM ${CLASSIFIER_CALL_DATASET} + WHERE timestamp > NOW() - INTERVAL '${window}' DAY + GROUP BY day + ORDER BY day`; +} + +export async function fetchClassifierCostSeries( + env: Env, + days: number, + fetchImpl: typeof fetch = fetch, + now = new Date(), +): Promise { + const account = (env as { CLOUDFLARE_ACCOUNT_ID?: string }).CLOUDFLARE_ACCOUNT_ID; + const token = (env as { ANALYTICS_API_TOKEN?: string }).ANALYTICS_API_TOKEN; + if (!account || !token) return { available: false, reason: "not_configured" }; + + const windowDays = Number.isFinite(days) ? Math.max(1, Math.min(90, Math.floor(days))) : 30; + + try { + const res = await fetchImpl(`${SQL_ENDPOINT}/${account}/analytics_engine/sql`, { + method: "POST", + headers: { authorization: `Bearer ${token}` }, + body: classifierCostSeriesQuery(days), + }); + if (!res.ok) return { available: false, reason: "query_failed" }; + const payload = (await res.json()) as { data?: unknown }; + if (payload.data !== undefined && !Array.isArray(payload.data)) { + return { available: false, reason: "query_failed" }; + } + const points: ClassifierCostDay[] = []; + for (const row of (payload.data as Array> | undefined) ?? []) { + const day = normalizeUtcDay(row.day); + if (!day) continue; + points.push({ + day, + calls: Number(row.calls) || 0, + priced: Number(row.priced) || 0, + cost: Number(row.cost) || 0, + }); + } + return { + available: true, + days: fillDaySeries(windowStart(windowDays, now), windowDays, points, (day) => ({ + day, + calls: 0, + priced: 0, + cost: 0, + })), + }; + } catch { + return { available: false, reason: "query_failed" }; + } +} diff --git a/apps/api/src/classifier-admin-wire.ts b/apps/api/src/classifier-admin-wire.ts index 5d9efc004..022a2041b 100644 --- a/apps/api/src/classifier-admin-wire.ts +++ b/apps/api/src/classifier-admin-wire.ts @@ -92,8 +92,27 @@ export interface ClassifierOverview { daily: ClassifierDailyPoint[]; /** Platform uploads in the window from `daily_metrics`, or null when unavailable. */ windowUploads: number | null; + /** + * OpenRouter Decisions calls and their reported cost per UTC day, from + * Analytics Engine. Unavailable when AE is unconfigured or the query fails. + */ + costs: ClassifierCostSeries; +} + +export interface ClassifierCostDay { + day: string; + /** Decisions calls, including hold-area box calls. */ + calls: number; + /** Calls whose response carried a cost. */ + priced: number; + /** USD summed over the priced calls. */ + cost: number; } +export type ClassifierCostSeries = + | { available: true; days: ClassifierCostDay[] } + | { available: false; reason: string }; + export interface ClassifierDailyPoint { day: string; classified: number; diff --git a/apps/api/src/classifier-admin.ts b/apps/api/src/classifier-admin.ts index 88434e1ba..c6ab72746 100644 --- a/apps/api/src/classifier-admin.ts +++ b/apps/api/src/classifier-admin.ts @@ -15,6 +15,7 @@ import { CLASSIFIER_SAFETY_FIELDS, isConfidenceBucket, type ClassifierAdminWindow, + type ClassifierCostSeries, type ClassifierDailyPoint, type ClassifierFlagStats, type ClassifierFlaggedFile, @@ -208,18 +209,24 @@ export async function buildClassifierOverview( db: D1Queryable, days: ClassifierAdminWindow, now = new Date(), + costSeries: Promise = Promise.resolve({ + available: false, + reason: "not_configured", + }), ): Promise { const since = windowStart(days, now); - const [labels, confidence, flaggedFiles, workspaces, markers, windowUploads] = await Promise.all([ - labelCounts(db), - flaggedConfidence(db), - flaggedFileCount(db), - workspaceCount(db), - classifierMarkers(db, since, days), - platformSeries(db, "upload", since) - .then((rows) => rows.reduce((total, r) => total + Number(r.count), 0)) - .catch(() => null), - ]); + const [labels, confidence, flaggedFiles, workspaces, markers, windowUploads, costs] = + await Promise.all([ + labelCounts(db), + flaggedConfidence(db), + flaggedFileCount(db), + workspaceCount(db), + classifierMarkers(db, since, days), + platformSeries(db, "upload", since) + .then((rows) => rows.reduce((total, r) => total + Number(r.count), 0)) + .catch(() => null), + costSeries, + ]); const counts = (key: string) => labels.get(key) ?? []; const countOf = (value: string) => markers.versions.find((v) => v.value === value)?.count ?? 0; const skipped = countOf("skipped"); @@ -251,6 +258,7 @@ export async function buildClassifierOverview( flaggedFiles, daily: markers.daily, windowUploads, + costs, }; } diff --git a/apps/api/src/classifier-call-analytics.test.ts b/apps/api/src/classifier-call-analytics.test.ts new file mode 100644 index 000000000..31e08cf08 --- /dev/null +++ b/apps/api/src/classifier-call-analytics.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { decisionsUsage, writeClassifierCallPoint } from "./classifier-call-analytics"; + +describe("decisionsUsage", () => { + it("reads camelCase and snake_case usage", () => { + expect(decisionsUsage({ usage: { cost: 0.002, inputTokens: 10, outputTokens: 2 } })).toEqual({ + cost: 0.002, + inputTokens: 10, + outputTokens: 2, + }); + expect(decisionsUsage({ usage: { input_tokens: 7, output_tokens: 1 } })).toEqual({ + cost: -1, + inputTokens: 7, + outputTokens: 1, + }); + }); + + it("marks a missing usage block as unpriced", () => { + expect(decisionsUsage({ answers: {} })).toEqual({ cost: -1, inputTokens: 0, outputTokens: 0 }); + expect(decisionsUsage(null)).toEqual({ cost: -1, inputTokens: 0, outputTokens: 0 }); + }); +}); + +describe("writeClassifierCallPoint", () => { + it("is a no-op without the binding and never throws", () => { + expect(() => writeClassifierCallPoint({} as Env, "m", {})).not.toThrow(); + const env = { + CLASSIFIER_CALLS: { + writeDataPoint: () => { + throw new Error("ae down"); + }, + }, + } as unknown as Env; + expect(() => writeClassifierCallPoint(env, "m", {})).not.toThrow(); + }); +}); diff --git a/apps/api/src/classifier-call-analytics.ts b/apps/api/src/classifier-call-analytics.ts new file mode 100644 index 000000000..0a314e847 --- /dev/null +++ b/apps/api/src/classifier-call-analytics.ts @@ -0,0 +1,60 @@ +/** + * Analytics Engine write side for classifier spend (dataset + * `uploads_classifier_calls`). One point per OpenRouter Decisions call that + * came back ok, with the `usage` block the response carries. The operator + * classifier view (`/admin/classifier`) reads it back per day through + * `fetchClassifierCostSeries` in analytics-engine.ts. + * + * Only the model id and the usage numbers are written, never a workspace, + * key, or image. An absent binding is a silent no-op, and a write never + * throws, same contract as `writeSlowOpPoint`. + */ +import { reportHandledError } from "@uploads/observability"; + +/** + * Blob and double positions are a contract with the SQL read path. Append + * new fields at the end; never reorder or remove. + */ +export const CLASSIFIER_CALL_BLOB_ORDER = ["model"] as const; +export const CLASSIFIER_CALL_DOUBLE_ORDER = ["cost", "inputTokens", "outputTokens"] as const; + +export interface ClassifierCallUsage { + /** USD, or -1 when the response did not carry a cost. */ + cost: number; + inputTokens: number; + outputTokens: number; +} + +function usageNumber(usage: Record, ...names: string[]): number | undefined { + for (const name of names) { + const value = usage[name]; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value; + } + return undefined; +} + +/** The Decisions `usage` block, camelCase or snake_case. */ +export function decisionsUsage(result: unknown): ClassifierCallUsage { + const raw = (result as { usage?: unknown } | null)?.usage; + const usage = raw && typeof raw === "object" ? (raw as Record) : {}; + return { + cost: usageNumber(usage, "cost") ?? -1, + inputTokens: usageNumber(usage, "inputTokens", "input_tokens", "prompt_tokens") ?? 0, + outputTokens: usageNumber(usage, "outputTokens", "output_tokens", "completion_tokens") ?? 0, + }; +} + +export function writeClassifierCallPoint(env: Env, model: string, result: unknown): void { + const analytics = (env as { CLASSIFIER_CALLS?: AnalyticsEngineDataset }).CLASSIFIER_CALLS; + if (!analytics) return; + try { + const usage = decisionsUsage(result); + analytics.writeDataPoint({ + indexes: [model], + blobs: [model], + doubles: [usage.cost, usage.inputTokens, usage.outputTokens], + }); + } catch (err) { + reportHandledError(err); + } +} diff --git a/apps/api/src/classifier.test.ts b/apps/api/src/classifier.test.ts index dde2e16aa..ca0c5dd4c 100644 --- a/apps/api/src/classifier.test.ts +++ b/apps/api/src/classifier.test.ts @@ -564,6 +564,32 @@ describe("openRouterDecisionsRun", () => { expect(String(state[0])).toContain("filename: a.png"); }); + it("writes the response usage to the classifier calls dataset", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + answers: {}, + usage: { cost: 0.0021, inputTokens: 900, outputTokens: 40 }, + }), + { status: 200 }, + ), + ), + ); + const points: unknown[] = []; + const CLASSIFIER_CALLS = { writeDataPoint: (p: unknown) => points.push(p) }; + await openRouterDecisionsRun(env({ CLASSIFIER_CALLS }))(req); + expect(points).toEqual([ + { + indexes: [CLASSIFIER_JEV_MODEL], + blobs: [CLASSIFIER_JEV_MODEL], + doubles: [0.0021, 900, 40], + }, + ]); + }); + it("posts Luna to the same URL", async () => { let url = ""; let model = ""; diff --git a/apps/api/src/classifier.ts b/apps/api/src/classifier.ts index 3343b90db..f7c0f9544 100644 --- a/apps/api/src/classifier.ts +++ b/apps/api/src/classifier.ts @@ -88,6 +88,7 @@ import { import { imagePhash } from "./phash"; import { contentSha256Hex } from "./provenance"; import { CLASSIFIER_CLEAN_VALUES, type ClassifierConfidenceBucket } from "./classifier-admin-wire"; +import { writeClassifierCallPoint } from "./classifier-call-analytics"; import { dbFor } from "./db-session"; import { detectImageDimensions, uploadKind } from "./guards"; import type { WorkspaceRecord } from "./workspace"; @@ -1139,7 +1140,8 @@ export function workersAiClassifierRun(env: Env): ClassifierRun { } /** - * File-classifier runner. One direct OpenRouter Decisions call. Does not + * File-classifier runner. One direct OpenRouter Decisions call, with its + * `usage` written to Analytics Engine for the operator cost view. Does not * log image bytes, data URLs, or the API key. A missing key throws; callers * that classify an upload catch it and write nothing. */ @@ -1176,7 +1178,9 @@ export function openRouterDecisionsRun(env: Env): ClassifierRun { } throw new Error(`openrouter decisions failed: ${response.status}`); } - return response.json(); + const result: unknown = await response.json(); + writeClassifierCallPoint(env, req.model, result); + return result; }; } diff --git a/apps/api/src/routes/admin-ui.ts b/apps/api/src/routes/admin-ui.ts index e217b4f7c..49e8a17ea 100644 --- a/apps/api/src/routes/admin-ui.ts +++ b/apps/api/src/routes/admin-ui.ts @@ -27,6 +27,7 @@ import { import { runCliUpgradeCampaign } from "../cli-upgrade-campaign"; import { fetchBreakdown, + fetchClassifierCostSeries, fetchSlowOps, type BreakdownDimension, type SlowOpWindow, @@ -722,7 +723,13 @@ export const adminUi = new Hono() classifierCacheKey(days), CLASSIFIER_CACHE_TTL, c.req.query("fresh") === "1", - () => buildClassifierOverview(dbFor(c.env), days), + () => + buildClassifierOverview( + dbFor(c.env), + days, + new Date(), + fetchClassifierCostSeries(c.env, days), + ), ), ); }) diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index 43034d51b..0eb77d808 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -205,6 +205,13 @@ "binding": "SLOW_OPS", "dataset": "uploads_slow_ops", }, + // Classifier spend: one point per OpenRouter Decisions call with its + // usage block (src/classifier-call-analytics.ts), read per day by the + // /admin/classifier cost tile and chart. Absent binding is a no-op. + { + "binding": "CLASSIFIER_CALLS", + "dataset": "uploads_classifier_calls", + }, ], // Service binding to apps/auth (Phase 2, plan D1): forwards Cookie/ // Authorization to GET /api/auth/get-session over the binding — no public @@ -442,6 +449,10 @@ "binding": "SLOW_OPS", "dataset": "uploads_slow_ops_preview", }, + { + "binding": "CLASSIFIER_CALLS", + "dataset": "uploads_classifier_calls_preview", + }, ], // Everything below repeats the production bindings verbatim — stateless, // account-level, or per-colo, so sharing is safe. See the notes on the diff --git a/apps/web/src/components/admin/AdminClassifier.tsx b/apps/web/src/components/admin/AdminClassifier.tsx index 2d9152c60..7018634cd 100644 --- a/apps/web/src/components/admin/AdminClassifier.tsx +++ b/apps/web/src/components/admin/AdminClassifier.tsx @@ -20,6 +20,7 @@ import { TableRow, } from "@uploads/ui/components/ui/table"; import "@uploads/ui/styles.css"; +import "../../styles/admin-metrics.css"; import { IslandErrorBoundary } from "../IslandErrorBoundary"; import { ADMIN_METRIC_WINDOWS, @@ -36,10 +37,12 @@ import { makeAdminApi } from "../../lib/admin-api"; import { ADMIN_BTN, ADMIN_BTN_DANGER } from "../../lib/admin-ui"; import { coverageLabel, + estimatedCost, flaggedFileHref, flagLabel, flagTitle, flagVariant, + formatUsd, sharePercent, topFlaggedValues, withShares, @@ -94,6 +97,25 @@ function Tile({ label, value, note }: { label: string; value: string; note?: str ); } +/** Window spend from OpenRouter's reported usage, scaled to unpriced calls. */ +function CostTile({ costs, days }: { costs: ClassifierOverview["costs"]; days: number }) { + if (!costs.available) { + return ; + } + const estimate = estimatedCost(costs.days); + return ( + + ); +} + function Breakdown({ title, rows }: { title: string; rows: ClassifierValueCount[] }) { const shared = withShares(rows); return ( @@ -127,7 +149,34 @@ function Breakdown({ title, rows }: { title: string; rows: ClassifierValueCount[ ); } -function DailyChart({ daily }: { daily: ClassifierDailyPoint[] }) { +const DAY_LABEL = new Intl.DateTimeFormat("en-US", { + weekday: "short", + month: "short", + day: "numeric", + timeZone: "UTC", +}); + +const total = (d: ClassifierDailyPoint) => d.classified + d.skipped + d.failed; + +const dayLabel = (day: string) => DAY_LABEL.format(new Date(`${day}T00:00:00Z`)); + +/** + * Stacked bars with the same hover/focus card as the /admin/metrics charts + * (`.metrics-chart-tooltip`): the day, each series, the total, and the + * day's Decisions calls and cost when Analytics Engine has them. + */ +function DailyChart({ + daily, + costs, +}: { + daily: ClassifierDailyPoint[]; + costs: ClassifierOverview["costs"]; +}) { + const [hovered, setHovered] = useState(null); + const costByDay = useMemo( + () => new Map(costs.available ? costs.days.map((d) => [d.day, d]) : []), + [costs], + ); const first = daily[0]?.day; const last = daily.at(-1)?.day; if (!first || !last) { @@ -135,33 +184,89 @@ function DailyChart({ daily }: { daily: ClassifierDailyPoint[] }) { } const W = 960; const H = 180; - const max = Math.max(1, ...daily.map((d) => d.classified + d.skipped + d.failed)); + const max = Math.max(1, ...daily.map(total)); const slot = W / daily.length; const bar = Math.max(1, slot * 0.7); const scale = (n: number) => (n / max) * (H - 4); + const point = hovered === null ? undefined : daily[hovered]; + const pointCost = point ? costByDay.get(point.day) : undefined; + const pointEstimate = pointCost ? estimatedCost([pointCost]) : null; return (
- - {daily.map((d, i) => { - const x = i * slot + (slot - bar) / 2; - let top = H; - return ( - - {`${d.day}: ${d.classified} classified, ${d.skipped} skipped, ${d.failed} failed`} - {SERIES.map((s) => { - const h = scale(d[s.key]); - top -= h; - return ; - })} - - ); - })} - +
setHovered(null)}> + + {daily.map((d, i) => { + const x = i * slot + (slot - bar) / 2; + let top = H; + return ( + + {SERIES.map((s) => { + const h = scale(d[s.key]); + top -= h; + return ( + + ); + })} + setHovered(i)} + onFocus={() => setHovered(i)} + onBlur={() => setHovered(null)} + /> + + ); + })} + + {point && hovered !== null ? ( +
+ {dayLabel(point.day)} + {SERIES.map((s) => ( +
+
+ ))} + Total {total(point).toLocaleString()} + {pointCost && pointCost.calls > 0 ? ( +
+
+ Model calls + {pointCost.calls.toLocaleString()} +
+
+ Est. cost + {pointEstimate ? formatUsd(pointEstimate.cost) : "-"} +
+
+ ) : null} +
+ ) : null} +

{SERIES.map((s) => ( @@ -293,11 +398,12 @@ function OverviewSection({ apiOrigin }: AdminClassifierProps) { value={coverageLabel(data.daily, data.windowUploads) ?? "-"} note={`Last ${data.days} days. Only allowlisted workspaces are classified.`} /> +

Daily activity

- +
diff --git a/apps/web/src/lib/admin-classifier.test.ts b/apps/web/src/lib/admin-classifier.test.ts index 3e09c3629..57dd9bbd2 100644 --- a/apps/web/src/lib/admin-classifier.test.ts +++ b/apps/web/src/lib/admin-classifier.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import { coverageLabel, + estimatedCost, + formatUsd, flagLabel, flaggedFileHref, flagVariant, @@ -77,3 +79,28 @@ describe("flaggedFileHref", () => { ); }); }); + +describe("formatUsd", () => { + it("keeps fractions of a cent readable", () => { + expect(formatUsd(0)).toBe("$0"); + expect(formatUsd(0.00004)).toBe("<$0.0001"); + expect(formatUsd(0.01234)).toBe("$0.0123"); + expect(formatUsd(1234.5)).toBe("$1,234.50"); + }); +}); + +describe("estimatedCost", () => { + it("scales the reported cost up to unpriced calls", () => { + expect( + estimatedCost([ + { calls: 4, priced: 2, cost: 0.02 }, + { calls: 0, priced: 0, cost: 0 }, + ]), + ).toEqual({ cost: 0.04, calls: 4, perCall: 0.01 }); + }); + + it("is null when no call reported a cost", () => { + expect(estimatedCost([{ calls: 3, priced: 0, cost: 0 }])).toBeNull(); + expect(estimatedCost([])).toBeNull(); + }); +}); diff --git a/apps/web/src/lib/admin-classifier.ts b/apps/web/src/lib/admin-classifier.ts index 22f2b9a0d..ef791baa6 100644 --- a/apps/web/src/lib/admin-classifier.ts +++ b/apps/web/src/lib/admin-classifier.ts @@ -74,3 +74,36 @@ export function flagTitle(flag: { confidence: ClassifierConfidenceBucket | null export function flaggedFileHref(file: Pick): string { return filePath(file.workspace, file.key); } + +/** + * USD for the cost tile and tooltip. Classifier calls cost fractions of a + * cent, so small amounts keep four decimals instead of rounding to $0.00. + */ +export function formatUsd(amount: number): string { + if (!(amount > 0)) return "$0"; + if (amount >= 1) + return `$${amount.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; + if (amount < 0.0001) return "<$0.0001"; + return `$${amount.toFixed(4)}`; +} + +/** + * Estimated spend for some days of Decisions calls: the reported cost, + * scaled up to every call when some responses carried no cost. Null when no + * call in the range reported one. + */ +export function estimatedCost( + days: ReadonlyArray<{ calls: number; priced: number; cost: number }>, +): { cost: number; calls: number; perCall: number } | null { + let calls = 0; + let priced = 0; + let cost = 0; + for (const d of days) { + calls += d.calls; + priced += d.priced; + cost += d.cost; + } + if (!(priced > 0)) return null; + const perCall = cost / priced; + return { cost: perCall * calls, calls, perCall }; +} From 0cff918137c9f65a4775af196641afde903f5f53 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 13:49:09 +0000 Subject: [PATCH 2/2] Admin classifier: estimate cost from input tokens Jev bills input only, so the per-call AE point keeps cost and input tokens, and the estimate scales reported cost per input token to calls that carried no cost. The hover card shows the day's input tokens. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_017RAV94xRKfKGB7yS6GwyNB --- apps/api/src/analytics-engine.test.ts | 16 ++++++---- apps/api/src/analytics-engine.ts | 14 +++++---- apps/api/src/classifier-admin-wire.ts | 8 +++-- .../api/src/classifier-call-analytics.test.ts | 6 ++-- apps/api/src/classifier-call-analytics.ts | 9 +++--- apps/api/src/classifier.test.ts | 2 +- .../src/components/admin/AdminClassifier.tsx | 8 +++-- apps/web/src/lib/admin-classifier.test.ts | 24 ++++++++++----- apps/web/src/lib/admin-classifier.ts | 30 ++++++++++++------- 9 files changed, 74 insertions(+), 43 deletions(-) diff --git a/apps/api/src/analytics-engine.test.ts b/apps/api/src/analytics-engine.test.ts index 4be215bd1..6109e2a5d 100644 --- a/apps/api/src/analytics-engine.test.ts +++ b/apps/api/src/analytics-engine.test.ts @@ -333,8 +333,14 @@ describe("fetchClassifierCostSeries", () => { return new Response( JSON.stringify({ data: [ - { day: "2026-08-01", calls: 4, priced: 3, cost: 0.03 }, - { day: "2026-08-03", calls: "2", priced: "2", cost: "0.02" }, + { day: "2026-08-01", calls: 4, inputTokens: 4000, pricedInputTokens: 3000, cost: 0.03 }, + { + day: "2026-08-03", + calls: "2", + inputTokens: "900", + pricedInputTokens: "900", + cost: "0.02", + }, ], }), ); @@ -345,9 +351,9 @@ describe("fetchClassifierCostSeries", () => { expect(result).toEqual({ available: true, days: [ - { day: "2026-08-01", calls: 4, priced: 3, cost: 0.03 }, - { day: "2026-08-02", calls: 0, priced: 0, cost: 0 }, - { day: "2026-08-03", calls: 2, priced: 2, cost: 0.02 }, + { day: "2026-08-01", calls: 4, inputTokens: 4000, pricedInputTokens: 3000, cost: 0.03 }, + { day: "2026-08-02", calls: 0, inputTokens: 0, pricedInputTokens: 0, cost: 0 }, + { day: "2026-08-03", calls: 2, inputTokens: 900, pricedInputTokens: 900, cost: 0.02 }, ], }); }); diff --git a/apps/api/src/analytics-engine.ts b/apps/api/src/analytics-engine.ts index 63e346656..94a100e1d 100644 --- a/apps/api/src/analytics-engine.ts +++ b/apps/api/src/analytics-engine.ts @@ -301,14 +301,16 @@ export async function fetchSlowOps( /** * Per-day Decisions calls and their summed `usage.cost` from * `uploads_classifier_calls` (written by `classifier-call-analytics.ts`). - * `double1` is the cost, -1 when a response carried none, so priced calls - * are counted apart and only they add to the sum. + * `double1` is the cost, -1 when a response carried none, and `double2` + * the input tokens. Input tokens of priced calls are summed apart so the + * page can scale cost per input token to the calls without one. */ export function classifierCostSeriesQuery(days: number): string { const window = Number.isFinite(days) ? Math.max(1, Math.min(90, Math.floor(days))) : 30; return `SELECT formatDateTime(toStartOfInterval(timestamp, INTERVAL '1' DAY), '%Y-%m-%d') AS day, SUM(_sample_interval) AS calls, - SUM(if(double1 >= 0, _sample_interval, 0)) AS priced, + SUM(double2 * _sample_interval) AS inputTokens, + SUM(if(double1 >= 0, double2 * _sample_interval, 0)) AS pricedInputTokens, SUM(if(double1 >= 0, double1 * _sample_interval, 0)) AS cost FROM ${CLASSIFIER_CALL_DATASET} WHERE timestamp > NOW() - INTERVAL '${window}' DAY @@ -346,7 +348,8 @@ export async function fetchClassifierCostSeries( points.push({ day, calls: Number(row.calls) || 0, - priced: Number(row.priced) || 0, + inputTokens: Number(row.inputTokens) || 0, + pricedInputTokens: Number(row.pricedInputTokens) || 0, cost: Number(row.cost) || 0, }); } @@ -355,7 +358,8 @@ export async function fetchClassifierCostSeries( days: fillDaySeries(windowStart(windowDays, now), windowDays, points, (day) => ({ day, calls: 0, - priced: 0, + inputTokens: 0, + pricedInputTokens: 0, cost: 0, })), }; diff --git a/apps/api/src/classifier-admin-wire.ts b/apps/api/src/classifier-admin-wire.ts index 022a2041b..3b28dc8a0 100644 --- a/apps/api/src/classifier-admin-wire.ts +++ b/apps/api/src/classifier-admin-wire.ts @@ -103,9 +103,11 @@ export interface ClassifierCostDay { day: string; /** Decisions calls, including hold-area box calls. */ calls: number; - /** Calls whose response carried a cost. */ - priced: number; - /** USD summed over the priced calls. */ + /** Input tokens over every call. Jev bills input only. */ + inputTokens: number; + /** Input tokens over the calls whose response carried a cost. */ + pricedInputTokens: number; + /** USD summed over the calls that carried a cost. */ cost: number; } diff --git a/apps/api/src/classifier-call-analytics.test.ts b/apps/api/src/classifier-call-analytics.test.ts index 31e08cf08..9cedb210b 100644 --- a/apps/api/src/classifier-call-analytics.test.ts +++ b/apps/api/src/classifier-call-analytics.test.ts @@ -6,18 +6,16 @@ describe("decisionsUsage", () => { expect(decisionsUsage({ usage: { cost: 0.002, inputTokens: 10, outputTokens: 2 } })).toEqual({ cost: 0.002, inputTokens: 10, - outputTokens: 2, }); expect(decisionsUsage({ usage: { input_tokens: 7, output_tokens: 1 } })).toEqual({ cost: -1, inputTokens: 7, - outputTokens: 1, }); }); it("marks a missing usage block as unpriced", () => { - expect(decisionsUsage({ answers: {} })).toEqual({ cost: -1, inputTokens: 0, outputTokens: 0 }); - expect(decisionsUsage(null)).toEqual({ cost: -1, inputTokens: 0, outputTokens: 0 }); + expect(decisionsUsage({ answers: {} })).toEqual({ cost: -1, inputTokens: 0 }); + expect(decisionsUsage(null)).toEqual({ cost: -1, inputTokens: 0 }); }); }); diff --git a/apps/api/src/classifier-call-analytics.ts b/apps/api/src/classifier-call-analytics.ts index 0a314e847..eedc33e7c 100644 --- a/apps/api/src/classifier-call-analytics.ts +++ b/apps/api/src/classifier-call-analytics.ts @@ -16,13 +16,13 @@ import { reportHandledError } from "@uploads/observability"; * new fields at the end; never reorder or remove. */ export const CLASSIFIER_CALL_BLOB_ORDER = ["model"] as const; -export const CLASSIFIER_CALL_DOUBLE_ORDER = ["cost", "inputTokens", "outputTokens"] as const; +export const CLASSIFIER_CALL_DOUBLE_ORDER = ["cost", "inputTokens"] as const; export interface ClassifierCallUsage { /** USD, or -1 when the response did not carry a cost. */ cost: number; + /** Jev bills input only, so input tokens are what the estimate scales by. */ inputTokens: number; - outputTokens: number; } function usageNumber(usage: Record, ...names: string[]): number | undefined { @@ -33,14 +33,13 @@ function usageNumber(usage: Record, ...names: string[]): number return undefined; } -/** The Decisions `usage` block, camelCase or snake_case. */ +/** Cost and input tokens from the Decisions `usage` block, camelCase or snake_case. */ export function decisionsUsage(result: unknown): ClassifierCallUsage { const raw = (result as { usage?: unknown } | null)?.usage; const usage = raw && typeof raw === "object" ? (raw as Record) : {}; return { cost: usageNumber(usage, "cost") ?? -1, inputTokens: usageNumber(usage, "inputTokens", "input_tokens", "prompt_tokens") ?? 0, - outputTokens: usageNumber(usage, "outputTokens", "output_tokens", "completion_tokens") ?? 0, }; } @@ -52,7 +51,7 @@ export function writeClassifierCallPoint(env: Env, model: string, result: unknow analytics.writeDataPoint({ indexes: [model], blobs: [model], - doubles: [usage.cost, usage.inputTokens, usage.outputTokens], + doubles: [usage.cost, usage.inputTokens], }); } catch (err) { reportHandledError(err); diff --git a/apps/api/src/classifier.test.ts b/apps/api/src/classifier.test.ts index ca0c5dd4c..8e2e7437d 100644 --- a/apps/api/src/classifier.test.ts +++ b/apps/api/src/classifier.test.ts @@ -585,7 +585,7 @@ describe("openRouterDecisionsRun", () => { { indexes: [CLASSIFIER_JEV_MODEL], blobs: [CLASSIFIER_JEV_MODEL], - doubles: [0.0021, 900, 40], + doubles: [0.0021, 900], }, ]); }); diff --git a/apps/web/src/components/admin/AdminClassifier.tsx b/apps/web/src/components/admin/AdminClassifier.tsx index 7018634cd..06c094cb2 100644 --- a/apps/web/src/components/admin/AdminClassifier.tsx +++ b/apps/web/src/components/admin/AdminClassifier.tsx @@ -97,7 +97,7 @@ function Tile({ label, value, note }: { label: string; value: string; note?: str ); } -/** Window spend from OpenRouter's reported usage, scaled to unpriced calls. */ +/** Window spend from OpenRouter's reported usage, scaled by input tokens to unpriced calls. */ function CostTile({ costs, days }: { costs: ClassifierOverview["costs"]; days: number }) { if (!costs.available) { return ; @@ -109,7 +109,7 @@ function CostTile({ costs, days }: { costs: ClassifierOverview["costs"]; days: n value={estimate ? formatUsd(estimate.cost) : "-"} note={ estimate - ? `Last ${days} days. ${estimate.calls.toLocaleString()} model calls, about ${formatUsd(estimate.perCall)} each.` + ? `Last ${days} days. ${estimate.calls.toLocaleString()} model calls, ${estimate.inputTokens.toLocaleString()} input tokens.` : "No model calls recorded in this window yet." } /> @@ -258,6 +258,10 @@ function DailyChart({ Model calls {pointCost.calls.toLocaleString()}
+
+ Input tokens + {pointCost.inputTokens.toLocaleString()} +
Est. cost {pointEstimate ? formatUsd(pointEstimate.cost) : "-"} diff --git a/apps/web/src/lib/admin-classifier.test.ts b/apps/web/src/lib/admin-classifier.test.ts index 57dd9bbd2..7fd928948 100644 --- a/apps/web/src/lib/admin-classifier.test.ts +++ b/apps/web/src/lib/admin-classifier.test.ts @@ -90,17 +90,25 @@ describe("formatUsd", () => { }); describe("estimatedCost", () => { - it("scales the reported cost up to unpriced calls", () => { - expect( - estimatedCost([ - { calls: 4, priced: 2, cost: 0.02 }, - { calls: 0, priced: 0, cost: 0 }, - ]), - ).toEqual({ cost: 0.04, calls: 4, perCall: 0.01 }); + it("scales cost per input token to calls that carried no cost", () => { + const estimate = estimatedCost([ + { calls: 4, inputTokens: 4000, pricedInputTokens: 2000, cost: 0.02 }, + { calls: 0, inputTokens: 0, pricedInputTokens: 0, cost: 0 }, + ]); + expect(estimate?.cost).toBeCloseTo(0.04); + expect(estimate).toMatchObject({ calls: 4, inputTokens: 4000 }); + }); + + it("falls back to the reported cost without token counts", () => { + expect(estimatedCost([{ calls: 2, inputTokens: 0, pricedInputTokens: 0, cost: 0.01 }])).toEqual( + { cost: 0.01, calls: 2, inputTokens: 0 }, + ); }); it("is null when no call reported a cost", () => { - expect(estimatedCost([{ calls: 3, priced: 0, cost: 0 }])).toBeNull(); + expect( + estimatedCost([{ calls: 3, inputTokens: 900, pricedInputTokens: 0, cost: 0 }]), + ).toBeNull(); expect(estimatedCost([])).toBeNull(); }); }); diff --git a/apps/web/src/lib/admin-classifier.ts b/apps/web/src/lib/admin-classifier.ts index ef791baa6..da553ac43 100644 --- a/apps/web/src/lib/admin-classifier.ts +++ b/apps/web/src/lib/admin-classifier.ts @@ -88,22 +88,32 @@ export function formatUsd(amount: number): string { } /** - * Estimated spend for some days of Decisions calls: the reported cost, - * scaled up to every call when some responses carried no cost. Null when no - * call in the range reported one. + * Estimated spend for some days of Decisions calls. Jev bills input only, + * so the reported cost per input token is scaled to every call's input + * tokens, covering responses that carried no cost. Falls back to the + * reported sum when no priced call carried tokens, and null when nothing + * reported a cost. */ export function estimatedCost( - days: ReadonlyArray<{ calls: number; priced: number; cost: number }>, -): { cost: number; calls: number; perCall: number } | null { + days: ReadonlyArray<{ + calls: number; + inputTokens: number; + pricedInputTokens: number; + cost: number; + }>, +): { cost: number; calls: number; inputTokens: number } | null { let calls = 0; - let priced = 0; + let inputTokens = 0; + let pricedInputTokens = 0; let cost = 0; for (const d of days) { calls += d.calls; - priced += d.priced; + inputTokens += d.inputTokens; + pricedInputTokens += d.pricedInputTokens; cost += d.cost; } - if (!(priced > 0)) return null; - const perCall = cost / priced; - return { cost: perCall * calls, calls, perCall }; + if (pricedInputTokens > 0) { + return { cost: (cost / pricedInputTokens) * inputTokens, calls, inputTokens }; + } + return cost > 0 ? { cost, calls, inputTokens } : null; }