Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions apps/api/src/analytics-engine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
classifyMediaClass,
fetchBreakdown,
fetchSlowOps,
fetchClassifierCostSeries,
fetchUploadClassSeries,
slowOpsQuery,
uploadClassSeriesQuery,
Expand Down Expand Up @@ -321,3 +322,50 @@ describe("fetchSlowOps", () => {
expect(result).toEqual({ available: false, reason: "query_failed" });
});
});

describe("fetchClassifierCostSeries", () => {
const now = new Date("2026-08-03T12:00:00Z");

it("reads calls, priced calls, and cost per day and fills quiet days", async () => {
let body = "";
const impl = (async (_url: unknown, init?: RequestInit) => {
body = String(init?.body);
return new Response(
JSON.stringify({
data: [
{ day: "2026-08-01", calls: 4, inputTokens: 4000, pricedInputTokens: 3000, cost: 0.03 },
{
day: "2026-08-03",
calls: "2",
inputTokens: "900",
pricedInputTokens: "900",
cost: "0.02",
},
],
}),
);
}) as unknown as typeof fetch;
const result = await fetchClassifierCostSeries(env(), 3, impl, now);
expect(body).toContain("FROM uploads_classifier_calls");
expect(body).toContain("double1 >= 0");
expect(result).toEqual({
available: true,
days: [
{ day: "2026-08-01", calls: 4, inputTokens: 4000, pricedInputTokens: 3000, cost: 0.03 },
{ day: "2026-08-02", calls: 0, inputTokens: 0, pricedInputTokens: 0, cost: 0 },
{ day: "2026-08-03", calls: 2, inputTokens: 900, pricedInputTokens: 900, cost: 0.02 },
],
});
});

it("is unavailable without a token or when the query fails", async () => {
expect(await fetchClassifierCostSeries(env({ ANALYTICS_API_TOKEN: undefined }), 7)).toEqual({
available: false,
reason: "not_configured",
});
expect(await fetchClassifierCostSeries(env(), 7, jsonFetch({}, 500))).toEqual({
available: false,
reason: "query_failed",
});
});
});
72 changes: 72 additions & 0 deletions apps/api/src/analytics-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ import { BLOB_ORDER } from "./adoption";
import { fillDaySeries, normalizeUtcDay } from "./day-series";
import { windowStart } from "./adoption-queries";
import { SLOW_OP_BLOB_ORDER } from "./slow-op-analytics";
import type { ClassifierCostDay, ClassifierCostSeries } from "./classifier-admin-wire";

const SQL_ENDPOINT = "https://api.cloudflare.com/client/v4/accounts";
const DATASET = "uploads_adoption";
const SLOW_OP_DATASET = "uploads_slow_ops";
const CLASSIFIER_CALL_DATASET = "uploads_classifier_calls";

/**
* Dimensions the breakdown panel lets an operator group by — a DELIBERATELY
Expand Down Expand Up @@ -295,3 +297,73 @@ export async function fetchSlowOps(
return { available: false, reason: "query_failed" };
}
}

/**
* Per-day Decisions calls and their summed `usage.cost` from
* `uploads_classifier_calls` (written by `classifier-call-analytics.ts`).
* `double1` is the cost, -1 when a response carried none, and `double2`
* the input tokens. Input tokens of priced calls are summed apart so the
* page can scale cost per input token to the calls without one.
*/
export function classifierCostSeriesQuery(days: number): string {
const window = Number.isFinite(days) ? Math.max(1, Math.min(90, Math.floor(days))) : 30;
return `SELECT formatDateTime(toStartOfInterval(timestamp, INTERVAL '1' DAY), '%Y-%m-%d') AS day,
SUM(_sample_interval) AS calls,
SUM(double2 * _sample_interval) AS inputTokens,
SUM(if(double1 >= 0, double2 * _sample_interval, 0)) AS pricedInputTokens,
SUM(if(double1 >= 0, double1 * _sample_interval, 0)) AS cost
FROM ${CLASSIFIER_CALL_DATASET}
WHERE timestamp > NOW() - INTERVAL '${window}' DAY
GROUP BY day
ORDER BY day`;
}

export async function fetchClassifierCostSeries(
env: Env,
days: number,
fetchImpl: typeof fetch = fetch,
now = new Date(),
): Promise<ClassifierCostSeries> {
const account = (env as { CLOUDFLARE_ACCOUNT_ID?: string }).CLOUDFLARE_ACCOUNT_ID;
const token = (env as { ANALYTICS_API_TOKEN?: string }).ANALYTICS_API_TOKEN;
if (!account || !token) return { available: false, reason: "not_configured" };

const windowDays = Number.isFinite(days) ? Math.max(1, Math.min(90, Math.floor(days))) : 30;

try {
const res = await fetchImpl(`${SQL_ENDPOINT}/${account}/analytics_engine/sql`, {
method: "POST",
headers: { authorization: `Bearer ${token}` },
body: classifierCostSeriesQuery(days),
});
if (!res.ok) return { available: false, reason: "query_failed" };
const payload = (await res.json()) as { data?: unknown };
if (payload.data !== undefined && !Array.isArray(payload.data)) {
return { available: false, reason: "query_failed" };
}
const points: ClassifierCostDay[] = [];
for (const row of (payload.data as Array<Record<string, unknown>> | undefined) ?? []) {
const day = normalizeUtcDay(row.day);
if (!day) continue;
points.push({
day,
calls: Number(row.calls) || 0,
inputTokens: Number(row.inputTokens) || 0,
pricedInputTokens: Number(row.pricedInputTokens) || 0,
cost: Number(row.cost) || 0,
});
}
return {
available: true,
days: fillDaySeries(windowStart(windowDays, now), windowDays, points, (day) => ({
day,
calls: 0,
inputTokens: 0,
pricedInputTokens: 0,
cost: 0,
})),
};
} catch {
return { available: false, reason: "query_failed" };
}
}
21 changes: 21 additions & 0 deletions apps/api/src/classifier-admin-wire.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,29 @@ export interface ClassifierOverview {
daily: ClassifierDailyPoint[];
/** Platform uploads in the window from `daily_metrics`, or null when unavailable. */
windowUploads: number | null;
/**
* OpenRouter Decisions calls and their reported cost per UTC day, from
* Analytics Engine. Unavailable when AE is unconfigured or the query fails.
*/
costs: ClassifierCostSeries;
}

export interface ClassifierCostDay {
day: string;
/** Decisions calls, including hold-area box calls. */
calls: number;
/** Input tokens over every call. Jev bills input only. */
inputTokens: number;
/** Input tokens over the calls whose response carried a cost. */
pricedInputTokens: number;
/** USD summed over the calls that carried a cost. */
cost: number;
}

export type ClassifierCostSeries =
| { available: true; days: ClassifierCostDay[] }
| { available: false; reason: string };

export interface ClassifierDailyPoint {
day: string;
classified: number;
Expand Down
28 changes: 18 additions & 10 deletions apps/api/src/classifier-admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
CLASSIFIER_SAFETY_FIELDS,
isConfidenceBucket,
type ClassifierAdminWindow,
type ClassifierCostSeries,
type ClassifierDailyPoint,
type ClassifierFlagStats,
type ClassifierFlaggedFile,
Expand Down Expand Up @@ -208,18 +209,24 @@ export async function buildClassifierOverview(
db: D1Queryable,
days: ClassifierAdminWindow,
now = new Date(),
costSeries: Promise<ClassifierCostSeries> = Promise.resolve({
available: false,
reason: "not_configured",
}),
): Promise<ClassifierOverview> {
const since = windowStart(days, now);
const [labels, confidence, flaggedFiles, workspaces, markers, windowUploads] = await Promise.all([
labelCounts(db),
flaggedConfidence(db),
flaggedFileCount(db),
workspaceCount(db),
classifierMarkers(db, since, days),
platformSeries(db, "upload", since)
.then((rows) => rows.reduce((total, r) => total + Number(r.count), 0))
.catch(() => null),
]);
const [labels, confidence, flaggedFiles, workspaces, markers, windowUploads, costs] =
await Promise.all([
labelCounts(db),
flaggedConfidence(db),
flaggedFileCount(db),
workspaceCount(db),
classifierMarkers(db, since, days),
platformSeries(db, "upload", since)
.then((rows) => rows.reduce((total, r) => total + Number(r.count), 0))
.catch(() => null),
costSeries,
]);
const counts = (key: string) => labels.get(key) ?? [];
const countOf = (value: string) => markers.versions.find((v) => v.value === value)?.count ?? 0;
const skipped = countOf("skipped");
Expand Down Expand Up @@ -251,6 +258,7 @@ export async function buildClassifierOverview(
flaggedFiles,
daily: markers.daily,
windowUploads,
costs,
};
}

Expand Down
34 changes: 34 additions & 0 deletions apps/api/src/classifier-call-analytics.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import { decisionsUsage, writeClassifierCallPoint } from "./classifier-call-analytics";

describe("decisionsUsage", () => {
it("reads camelCase and snake_case usage", () => {
expect(decisionsUsage({ usage: { cost: 0.002, inputTokens: 10, outputTokens: 2 } })).toEqual({
cost: 0.002,
inputTokens: 10,
});
expect(decisionsUsage({ usage: { input_tokens: 7, output_tokens: 1 } })).toEqual({
cost: -1,
inputTokens: 7,
});
});

it("marks a missing usage block as unpriced", () => {
expect(decisionsUsage({ answers: {} })).toEqual({ cost: -1, inputTokens: 0 });
expect(decisionsUsage(null)).toEqual({ cost: -1, inputTokens: 0 });
});
});

describe("writeClassifierCallPoint", () => {
it("is a no-op without the binding and never throws", () => {
expect(() => writeClassifierCallPoint({} as Env, "m", {})).not.toThrow();
const env = {
CLASSIFIER_CALLS: {
writeDataPoint: () => {
throw new Error("ae down");
},
},
} as unknown as Env;
expect(() => writeClassifierCallPoint(env, "m", {})).not.toThrow();
});
});
59 changes: 59 additions & 0 deletions apps/api/src/classifier-call-analytics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/**
* Analytics Engine write side for classifier spend (dataset
* `uploads_classifier_calls`). One point per OpenRouter Decisions call that
* came back ok, with the `usage` block the response carries. The operator
* classifier view (`/admin/classifier`) reads it back per day through
* `fetchClassifierCostSeries` in analytics-engine.ts.
*
* Only the model id and the usage numbers are written, never a workspace,
* key, or image. An absent binding is a silent no-op, and a write never
* throws, same contract as `writeSlowOpPoint`.
*/
import { reportHandledError } from "@uploads/observability";

/**
* Blob and double positions are a contract with the SQL read path. Append
* new fields at the end; never reorder or remove.
*/
export const CLASSIFIER_CALL_BLOB_ORDER = ["model"] as const;
export const CLASSIFIER_CALL_DOUBLE_ORDER = ["cost", "inputTokens"] as const;

export interface ClassifierCallUsage {
/** USD, or -1 when the response did not carry a cost. */
cost: number;
/** Jev bills input only, so input tokens are what the estimate scales by. */
inputTokens: number;
}

function usageNumber(usage: Record<string, unknown>, ...names: string[]): number | undefined {
for (const name of names) {
const value = usage[name];
if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
}
return undefined;
}

/** Cost and input tokens from the Decisions `usage` block, camelCase or snake_case. */
export function decisionsUsage(result: unknown): ClassifierCallUsage {
const raw = (result as { usage?: unknown } | null)?.usage;
const usage = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
return {
cost: usageNumber(usage, "cost") ?? -1,
inputTokens: usageNumber(usage, "inputTokens", "input_tokens", "prompt_tokens") ?? 0,
};
}

export function writeClassifierCallPoint(env: Env, model: string, result: unknown): void {
const analytics = (env as { CLASSIFIER_CALLS?: AnalyticsEngineDataset }).CLASSIFIER_CALLS;
if (!analytics) return;
try {
const usage = decisionsUsage(result);
analytics.writeDataPoint({
indexes: [model],
blobs: [model],
doubles: [usage.cost, usage.inputTokens],
});
} catch (err) {
reportHandledError(err);
}
}
26 changes: 26 additions & 0 deletions apps/api/src/classifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,32 @@ describe("openRouterDecisionsRun", () => {
expect(String(state[0])).toContain("filename: a.png");
});

it("writes the response usage to the classifier calls dataset", async () => {
vi.stubGlobal(
"fetch",
vi.fn(
async () =>
new Response(
JSON.stringify({
answers: {},
usage: { cost: 0.0021, inputTokens: 900, outputTokens: 40 },
}),
{ status: 200 },
),
),
);
const points: unknown[] = [];
const CLASSIFIER_CALLS = { writeDataPoint: (p: unknown) => points.push(p) };
await openRouterDecisionsRun(env({ CLASSIFIER_CALLS }))(req);
expect(points).toEqual([
{
indexes: [CLASSIFIER_JEV_MODEL],
blobs: [CLASSIFIER_JEV_MODEL],
doubles: [0.0021, 900],
},
]);
});

it("posts Luna to the same URL", async () => {
let url = "";
let model = "";
Expand Down
8 changes: 6 additions & 2 deletions apps/api/src/classifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ import {
import { imagePhash } from "./phash";
import { contentSha256Hex } from "./provenance";
import { CLASSIFIER_CLEAN_VALUES, type ClassifierConfidenceBucket } from "./classifier-admin-wire";
import { writeClassifierCallPoint } from "./classifier-call-analytics";
import { dbFor } from "./db-session";
import { detectImageDimensions, uploadKind } from "./guards";
import type { WorkspaceRecord } from "./workspace";
Expand Down Expand Up @@ -1139,7 +1140,8 @@ export function workersAiClassifierRun(env: Env): ClassifierRun {
}

/**
* File-classifier runner. One direct OpenRouter Decisions call. Does not
* File-classifier runner. One direct OpenRouter Decisions call, with its
* `usage` written to Analytics Engine for the operator cost view. Does not
* log image bytes, data URLs, or the API key. A missing key throws; callers
* that classify an upload catch it and write nothing.
*/
Expand Down Expand Up @@ -1176,7 +1178,9 @@ export function openRouterDecisionsRun(env: Env): ClassifierRun {
}
throw new Error(`openrouter decisions failed: ${response.status}`);
}
return response.json();
const result: unknown = await response.json();
writeClassifierCallPoint(env, req.model, result);
return result;
};
}

Expand Down
Loading
Loading