From b321a01aad2f273ab85f8c14efe66d1f331ae83a Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:41 -0700 Subject: [PATCH 1/9] test(benchmarks): add queue latency and scheduling probes --- example/convex/_generated/api.d.ts | 6 + example/convex/schema.ts | 23 ++ example/convex/test/cleanup.test.ts | 51 +++ example/convex/test/cleanup.ts | 64 ++++ example/convex/test/latency.ts | 573 ++++++++++++++++++++++++++++ example/convex/test/scheduling.ts | 147 +++++++ package-lock.json | 13 +- package.json | 2 +- 8 files changed, 873 insertions(+), 6 deletions(-) create mode 100644 example/convex/test/cleanup.test.ts create mode 100644 example/convex/test/cleanup.ts create mode 100644 example/convex/test/latency.ts create mode 100644 example/convex/test/scheduling.ts diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index e794b2bd..bc515b6f 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -11,7 +11,9 @@ import type * as crons from "../crons.js"; import type * as example from "../example.js"; import type * as staticHosting from "../staticHosting.js"; +import type * as test_cleanup from "../test/cleanup.js"; import type * as test_dashboard from "../test/dashboard.js"; +import type * as test_latency from "../test/latency.js"; import type * as test_noisy from "../test/noisy.js"; import type * as test_nonRetryable from "../test/nonRetryable.js"; import type * as test_pool from "../test/pool.js"; @@ -25,6 +27,7 @@ import type * as test_scenarios_noisyNeighbor from "../test/scenarios/noisyNeigh import type * as test_scenarios_overhead from "../test/scenarios/overhead.js"; import type * as test_scenarios_sustained from "../test/scenarios/sustained.js"; import type * as test_scenarios_throughput from "../test/scenarios/throughput.js"; +import type * as test_scheduling from "../test/scheduling.js"; import type * as test_work from "../test/work.js"; import type { @@ -37,7 +40,9 @@ declare const fullApi: ApiFromModules<{ crons: typeof crons; example: typeof example; staticHosting: typeof staticHosting; + "test/cleanup": typeof test_cleanup; "test/dashboard": typeof test_dashboard; + "test/latency": typeof test_latency; "test/noisy": typeof test_noisy; "test/nonRetryable": typeof test_nonRetryable; "test/pool": typeof test_pool; @@ -51,6 +56,7 @@ declare const fullApi: ApiFromModules<{ "test/scenarios/overhead": typeof test_scenarios_overhead; "test/scenarios/sustained": typeof test_scenarios_sustained; "test/scenarios/throughput": typeof test_scenarios_throughput; + "test/scheduling": typeof test_scheduling; "test/work": typeof test_work; }>; diff --git a/example/convex/schema.ts b/example/convex/schema.ts index 522d959c..11d4d397 100644 --- a/example/convex/schema.ts +++ b/example/convex/schema.ts @@ -38,4 +38,27 @@ export default defineSchema({ name: v.string(), value: v.number(), }).index("name", ["name"]), + // When each probe in test/scheduling.ts ran, to check delayed and retried + // work against a real deployment. + schedulingProbes: defineTable({ + label: v.string(), + at: v.number(), + attempt: v.optional(v.number()), + }), + // One row per task in test/latency.ts, carrying when it was meant to run and + // when it actually did, so ordering and lateness can be measured directly + // rather than inferred from completion times. + latencyTasks: defineTable({ + cell: v.string(), // which experiment cell enqueued it + pool: v.union(v.literal("new"), v.literal("old")), + delayMs: v.number(), // 0 for immediate + seq: v.number(), // enqueue order within the cell + enqueuedAt: v.number(), // clock at the enqueuing mutation + runAt: v.number(), // when it was asked to run + startedAt: v.optional(v.number()), // clock inside the task itself + // Which enqueuing transaction produced it. Tasks sharing a pairId share a + // commit, so an immediate one is the control for a scheduled one. + pairId: v.optional(v.number()), + committedAt: v.optional(v.number()), // when its enqueue returned + }).index("cell", ["cell"]), }); diff --git a/example/convex/test/cleanup.test.ts b/example/convex/test/cleanup.test.ts new file mode 100644 index 00000000..a55824a7 --- /dev/null +++ b/example/convex/test/cleanup.test.ts @@ -0,0 +1,51 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { internal } from "../_generated/api"; +import { initConvexTest } from "../setup.test"; + +describe("benchmark cleanup", () => { + let t: ReturnType; + beforeEach(() => { + vi.useFakeTimers(); + t = initConvexTest(); + }); + afterEach(() => vi.useRealTimers()); + + it.each([0, -1, 1.5, Infinity, 1001])( + "rejects invalid limit %s before scheduling", + async (limit) => { + await expect( + t.mutation(internal.test.cleanup.start, { limit }), + ).rejects.toThrow("limit must be an integer between 1 and 1000"); + await expect( + t.mutation(internal.test.cleanup.step, { index: 0, after: 0, limit }), + ).rejects.toThrow("limit must be an integer between 1 and 1000"); + await t.run(async (ctx) => { + expect( + await ctx.db.system.query("_scheduled_functions").collect(), + ).toHaveLength(0); + }); + }, + ); + + it("clears multiple pages and advances past empty tables", async () => { + for (let i = 0; i < 3; i++) { + await t.run(async (ctx) => { + await ctx.db.insert("data", { data: i }); + await ctx.db.insert("schedulingProbes", { + label: "test", + at: Date.now(), + }); + }); + vi.advanceTimersByTime(1); + } + await t.mutation(internal.test.cleanup.start, { limit: 1 }); + await t.finishAllScheduledFunctions(vi.runAllTimers); + expect(await t.query(internal.test.cleanup.counts)).toEqual({ + tasks: 0, + latencyTasks: 0, + runs: 0, + schedulingProbes: 0, + data: 0, + }); + }); +}); diff --git a/example/convex/test/cleanup.ts b/example/convex/test/cleanup.ts new file mode 100644 index 00000000..1837f5f5 --- /dev/null +++ b/example/convex/test/cleanup.ts @@ -0,0 +1,64 @@ +import { v } from "convex/values"; +import { internalMutation, internalQuery } from "../_generated/server"; +import { internal } from "../_generated/api"; + +const TABLES = [ + "tasks", + "latencyTasks", + "runs", + "schedulingProbes", + "data", +] as const; + +/** Clear bookkeeping between runs; advance the cursor to skip tombstones. */ +export const step = internalMutation({ + args: { index: v.number(), limit: v.number(), after: v.number() }, + returns: v.null(), + handler: async (ctx, { index, limit, after }) => { + validateLimit(limit); + if (index >= TABLES.length) return null; + const table = TABLES[index]; + const docs = await ctx.db + .query(table) + .withIndex("by_creation_time", (q) => q.gt("_creationTime", after)) + .take(limit); + for (const doc of docs) await ctx.db.delete(table, doc._id); + const done = docs.length < limit; + await ctx.scheduler.runAfter(0, internal.test.cleanup.step, { + index: done ? index + 1 : index, + limit, + after: done ? 0 : docs[docs.length - 1]._creationTime, + }); + return null; + }, +}); + +export const start = internalMutation({ + args: { limit: v.optional(v.number()) }, + returns: v.null(), + handler: async (ctx, { limit = 1000 }) => { + validateLimit(limit); + await ctx.scheduler.runAfter(0, internal.test.cleanup.step, { + index: 0, + limit, + after: 0, + }); + return null; + }, +}); + +function validateLimit(limit: number) { + if (!Number.isInteger(limit) || limit < 1 || limit > 1000) { + throw new Error("limit must be an integer between 1 and 1000"); + } +} + +export const counts = internalQuery({ + args: {}, + returns: v.any(), + handler: async (ctx) => { + const out: Record = {}; + for (const t of TABLES) out[t] = await (ctx.db.query(t) as any).count(); + return out; + }, +}); diff --git a/example/convex/test/latency.ts b/example/convex/test/latency.ts new file mode 100644 index 00000000..8d8f644c --- /dev/null +++ b/example/convex/test/latency.ts @@ -0,0 +1,573 @@ +import { v } from "convex/values"; +import { + internalAction, + internalMutation, + internalQuery, +} from "../_generated/server"; +import { internal } from "../_generated/api"; +import type { Id } from "../_generated/dataModel"; +import { makePool, vPoolKind, enqueueFor, type PoolKind } from "./pool"; + +/** + * Scheduling latency and ordering harness. + * + * The throughput scenarios only enqueue ready-now work and only observe + * completions, which can't answer anything about *when* a task started + * relative to when it was asked to. Every task here records its own start + * clock, so start lateness and start ordering are measured directly. + * + * One run enqueues a set of `groups` — each a count of tasks scheduled the same + * distance out — waits for them all to start, and returns a row per task. The + * caller does the statistics, so the analysis can change without a redeploy. + * + * npx convex run test/latency:default '{ + * "cell": "demo", "pool": "new", + * "groups": [{"delayMs": 3000, "count": 600}, {"delayMs": 0, "count": 2000}] + * }' + * + * `cell` names the run and is cleared first, so results from different + * parameters never mix. `pool` picks this branch ("new") or the published + * baseline ("old"). See `.context/scheduling-experiments.md` for the full + * method and the questions this was built to answer. + * + * Two traps worth knowing before trusting a number: + * + * - Every entry in one enqueuing transaction shares a commit stamp *and* a + * start time, so a transaction is one sample, not `count` of them. Use + * `chunkSize` to get independent trials. + * - Lateness at large `count` is mostly enqueue backlog, not scheduler + * precision. Keep bursts small to measure the latter. + */ + +const MAX_PER_MUTATION = 200; + +/** + * The task itself: records when it actually began. + * + * Tolerates a missing row. `resetCell` clears rows without draining the queue, + * so a task enqueued by an earlier cell can still run after its row is gone — + * and throwing there would retry it, distorting a later cell's timings. + */ +export const probe = internalMutation({ + args: { taskId: v.id("latencyTasks") }, + returns: v.null(), + handler: async (ctx, { taskId }) => { + if (!(await ctx.db.get("latencyTasks", taskId))) return null; + await ctx.db.patch("latencyTasks", taskId, { startedAt: Date.now() }); + return null; + }, +}); + +// ~1KB, so a hold op costs commit bytes as well as a round of work. +const HOLD_BLOB = "x".repeat(1024); + +/** Trivial callee: nesting a mutation costs wall time without costing reads. */ +export const nestedNoop = internalMutation({ + args: {}, + returns: v.null(), + handler: async () => null, +}); + +/** + * Enqueue one chunk. `holdOps` extends this transaction *after* the enqueue so + * its commit lands later than the start time it wrote — the condition under + * which a scheduled entry can commit behind the loop's cursor. + * + * The hold is counted in operations, not milliseconds: `Date.now()` is frozen + * inside a mutation, so a wall-clock spin never terminates. The caller times + * the mutation from an action (where the clock is real) to learn what a given + * op count actually cost. + */ +export const enqueueChunk = internalMutation({ + args: { + cell: v.string(), + pool: vPoolKind, + delayMs: v.number(), + count: v.number(), + seqFrom: v.number(), + holdOps: v.optional(v.number()), + // Nested mutation calls: the cheapest way to stretch a transaction, since + // unlike read/write ops they don't count against the per-function limits. + nestedCalls: v.optional(v.number()), + maxParallelism: v.number(), + }, + returns: v.number(), + handler: async (ctx, args) => { + const pool = makePool(args.pool as PoolKind, { + maxParallelism: args.maxParallelism, + }); + const enqueuedAt = Date.now(); + const runAt = enqueuedAt + args.delayMs; + + for (let i = 0; i < args.count; i++) { + const taskId = await ctx.db.insert("latencyTasks", { + cell: args.cell, + pool: args.pool, + delayMs: args.delayMs, + seq: args.seqFrom + i, + enqueuedAt, + runAt, + }); + await pool.enqueueMutation( + ctx, + internal.test.latency.probe, + { taskId }, + args.delayMs > 0 ? { runAfter: args.delayMs } : {}, + ); + } + + // Extend the transaction so it commits after `runAt`. + let ops = 0; + for (let i = 0; i < (args.holdOps ?? 0); i++) { + const id = await ctx.db.insert("data", { misc: HOLD_BLOB }); + await ctx.db.delete("data", id); + ops++; + } + for (let i = 0; i < (args.nestedCalls ?? 0); i++) { + await ctx.runMutation(internal.test.latency.nestedNoop, {}); + ops++; + } + return ops; + }, +}); + +/** Clear a cell so it can be re-run. */ +export const resetCell = internalMutation({ + args: { cell: v.string() }, + returns: v.null(), + handler: async (ctx, { cell }) => { + for (const doc of await ctx.db + .query("latencyTasks") + .withIndex("cell", (q) => q.eq("cell", cell)) + .collect()) { + await ctx.db.delete("latencyTasks", doc._id); + } + return null; + }, +}); + +export const cellRows = internalQuery({ + args: { cell: v.string() }, + returns: v.any(), + handler: async (ctx, { cell }) => { + const rows = await ctx.db + .query("latencyTasks") + .withIndex("cell", (q) => q.eq("cell", cell)) + .collect(); + return rows.map((r) => ({ + pool: r.pool, + delayMs: r.delayMs, + seq: r.seq, + enqueuedAt: r.enqueuedAt, + runAt: r.runAt, + startedAt: r.startedAt, + })); + }, +}); + +export type Row = { + pool: PoolKind; + delayMs: number; + seq: number; + enqueuedAt: number; + runAt: number; + startedAt?: number; +}; + +/** + * Enqueue a cell and wait for it to finish. Returns the raw rows; the caller + * does the statistics so the analysis can change without a redeploy. + */ +export default internalAction({ + args: { + cell: v.string(), + pool: vPoolKind, + // Each group is `count` tasks all scheduled `delayMs` out. + groups: v.array(v.object({ delayMs: v.number(), count: v.number() })), + // Entries per enqueuing transaction. Every entry in one transaction shares + // a commit stamp and a start time, so it is one trial as far as + // out-of-order landing is concerned — cap this to get independent samples. + chunkSize: v.optional(v.number()), + holdOps: v.optional(v.number()), + nestedCalls: v.optional(v.number()), + maxParallelism: v.optional(v.number()), + // Gap between chunks, to spread enqueues out rather than one burst. + interChunkMs: v.optional(v.number()), + settleMs: v.optional(v.number()), + }, + returns: v.any(), + handler: async (ctx, args): Promise => { + const maxParallelism = args.maxParallelism ?? 50; + const chunkSize = Math.min( + args.chunkSize ?? MAX_PER_MUTATION, + MAX_PER_MUTATION, + ); + await ctx.runMutation(internal.test.latency.resetCell, { cell: args.cell }); + + const total = args.groups.reduce((n, g) => n + g.count, 0); + const startedEnqueue = Date.now(); + const chunkMs: number[] = []; + let seq = 0; + for (const group of args.groups) { + for (let sent = 0; sent < group.count; sent += chunkSize) { + const count = Math.min(chunkSize, group.count - sent); + const t0 = Date.now(); + await ctx.runMutation(internal.test.latency.enqueueChunk, { + cell: args.cell, + pool: args.pool, + delayMs: group.delayMs, + count, + seqFrom: seq, + holdOps: args.holdOps, + nestedCalls: args.nestedCalls, + maxParallelism, + }); + chunkMs.push(Date.now() - t0); + seq += count; + if (args.interChunkMs) { + await new Promise((r) => setTimeout(r, args.interChunkMs)); + } + } + } + const enqueueMs = Date.now() - startedEnqueue; + + // Wait for every task to have started, or give up. + const maxDelay = Math.max(...args.groups.map((g) => g.delayMs), 0); + const deadline = Date.now() + maxDelay + (args.settleMs ?? 120_000); + let rows: Row[] = []; + while (Date.now() < deadline) { + rows = await ctx.runQuery(internal.test.latency.cellRows, { + cell: args.cell, + }); + if ( + rows.length >= total && + rows.every((r) => r.startedAt !== undefined) + ) { + break; + } + await new Promise((r) => setTimeout(r, 250)); + } + const missing = rows.filter((r) => r.startedAt === undefined).length; + const sorted = [...chunkMs].sort((a, b) => a - b); + return { + cell: args.cell, + pool: args.pool, + total, + enqueueMs, + // What the hold actually cost, measured where the clock is real. + chunkMs: { + n: sorted.length, + min: sorted[0], + median: sorted[Math.floor(sorted.length / 2)], + max: sorted[sorted.length - 1], + }, + missing, + rows, + }; + }, +}); + +// ── Sibling comparison: does a "soon" scheduled entry start as promptly as a +// commitTs-ordered one enqueued in the same transaction? ──────────────────── + +/** + * Enqueue one control task ordered by `db.vars.commitTs` plus one "soon" + * scheduled task per entry in `soonDelays`, all in a single transaction. + * + * The control is the point: it shares the commit, so it becomes visible at the + * same instant. When the transaction takes longer to commit than a soon delay, + * that scheduled task is already overdue when it lands, so both are eligible + * together and any difference in start time is the ordering path, not the wait. + */ +export const enqueuePair = internalMutation({ + args: { + cell: v.string(), + pool: vPoolKind, + pairId: v.number(), + soonDelays: v.array(v.number()), + nestedCalls: v.optional(v.number()), + maxParallelism: v.number(), + }, + returns: v.array(v.id("latencyTasks")), + handler: async (ctx, args) => { + const pool = makePool(args.pool as PoolKind, { + maxParallelism: args.maxParallelism, + }); + const enqueuedAt = Date.now(); + const ids: Id<"latencyTasks">[] = []; + + for (const delayMs of [0, ...args.soonDelays]) { + const taskId = await ctx.db.insert("latencyTasks", { + cell: args.cell, + pool: args.pool, + delayMs, + seq: args.pairId, + pairId: args.pairId, + enqueuedAt, + runAt: enqueuedAt + delayMs, + }); + ids.push(taskId); + await pool.enqueueMutation( + ctx, + internal.test.latency.probe, + { taskId }, + delayMs > 0 ? { runAfter: delayMs } : {}, + ); + } + + // Stretch the transaction so the soon delays elapse before it commits. + for (let i = 0; i < (args.nestedCalls ?? 0); i++) { + await ctx.runMutation(internal.test.latency.nestedNoop, {}); + } + return ids; + }, +}); + +/** Record when each of a pair's enqueues actually returned (≈ commit). */ +export const markCommitted = internalMutation({ + args: { ids: v.array(v.id("latencyTasks")), committedAt: v.number() }, + returns: v.null(), + handler: async (ctx, { ids, committedAt }) => { + for (const id of ids) { + await ctx.db.patch("latencyTasks", id, { committedAt }); + } + return null; + }, +}); + +export const pairRows = internalQuery({ + args: { cell: v.string() }, + returns: v.any(), + handler: async (ctx, { cell }) => { + const rows = await ctx.db + .query("latencyTasks") + .withIndex("cell", (q) => q.eq("cell", cell)) + .collect(); + return rows.map((r) => ({ + pairId: r.pairId, + delayMs: r.delayMs, + enqueuedAt: r.enqueuedAt, + committedAt: r.committedAt, + runAt: r.runAt, + startedAt: r.startedAt, + })); + }, +}); + +export const pairs = internalAction({ + args: { + cell: v.string(), + pool: vPoolKind, + count: v.number(), + soonDelays: v.array(v.number()), + nestedCalls: v.optional(v.number()), + maxParallelism: v.optional(v.number()), + gapMs: v.optional(v.number()), + settleMs: v.optional(v.number()), + }, + returns: v.any(), + handler: async (ctx, args): Promise => { + const maxParallelism = args.maxParallelism ?? 200; + await ctx.runMutation(internal.test.latency.resetCell, { cell: args.cell }); + + for (let pairId = 0; pairId < args.count; pairId++) { + const t0 = Date.now(); + const ids = await ctx.runMutation(internal.test.latency.enqueuePair, { + cell: args.cell, + pool: args.pool, + pairId, + soonDelays: args.soonDelays, + nestedCalls: args.nestedCalls, + maxParallelism, + }); + const committedAt = Date.now(); + await ctx.runMutation(internal.test.latency.markCommitted, { + ids, + committedAt, + }); + void t0; + if (args.gapMs) await new Promise((r) => setTimeout(r, args.gapMs)); + } + + const expected = args.count * (1 + args.soonDelays.length); + const deadline = Date.now() + (args.settleMs ?? 120_000); + let rows: any[] = []; + while (Date.now() < deadline) { + rows = await ctx.runQuery(internal.test.latency.pairRows, { + cell: args.cell, + }); + if ( + rows.length >= expected && + rows.every((r: any) => r.startedAt !== undefined) + ) { + break; + } + await new Promise((r) => setTimeout(r, 250)); + } + return { + cell: args.cell, + pool: args.pool, + expected, + missing: rows.filter((r: any) => r.startedAt === undefined).length, + rows, + }; + }, +}); + +/** + * Enqueue `count` tasks all due `delayMs` out in a SINGLE transaction, the way + * `enqueueBatch` does. They therefore share one commit stamp, which is what + * makes them interesting: the sweep's cursor is inclusive, so a backlog sharing + * a stamp is re-read rather than passed. + */ +export const bulkScheduled = internalMutation({ + args: { + cell: v.string(), + pool: vPoolKind, + count: v.number(), + delayMs: v.number(), + maxParallelism: v.number(), + }, + returns: v.number(), + handler: async (ctx, args) => { + const pool = makePool(args.pool as PoolKind, { + maxParallelism: args.maxParallelism, + }); + const enqueuedAt = Date.now(); + for (let i = 0; i < args.count; i++) { + const taskId = await ctx.db.insert("latencyTasks", { + cell: args.cell, + pool: args.pool, + delayMs: args.delayMs, + seq: 10_000 + i, + enqueuedAt, + runAt: enqueuedAt + args.delayMs, + }); + await pool.enqueueMutation( + ctx, + internal.test.latency.probe, + { taskId }, + { runAfter: args.delayMs }, + ); + } + return args.count; + }, +}); + +/** + * The backlog case: a bulk of scheduled work sharing one commit stamp, then a + * single "soon" entry whose slow enqueue lands it behind the loop's cursor. If + * the backlog blocks the sweep, that entry can't be found by either path until + * the backlog retires. + */ +export const backlog = internalAction({ + args: { + cell: v.string(), + pool: vPoolKind, + bulkCount: v.number(), + bulkDelayMs: v.number(), + soonDelayMs: v.number(), + nestedCalls: v.number(), + maxParallelism: v.optional(v.number()), + settleMs: v.optional(v.number()), + }, + returns: v.any(), + handler: async (ctx, args): Promise => { + const maxParallelism = args.maxParallelism ?? 200; + await ctx.runMutation(internal.test.latency.resetCell, { cell: args.cell }); + + if (args.bulkCount > 0) { + await ctx.runMutation(internal.test.latency.bulkScheduledBatch, { + cell: args.cell, + pool: args.pool, + count: args.bulkCount, + delayMs: args.bulkDelayMs, + maxParallelism, + }); + } + // Let the loop see the backlog before the late entry lands. + await new Promise((r) => setTimeout(r, 3000)); + + const t0 = Date.now(); + const ids = await ctx.runMutation(internal.test.latency.enqueuePair, { + cell: args.cell, + pool: args.pool, + pairId: 0, + soonDelays: [args.soonDelayMs], + nestedCalls: args.nestedCalls, + maxParallelism, + }); + const committedAt = Date.now(); + await ctx.runMutation(internal.test.latency.markCommitted, { + ids, + committedAt, + }); + + const deadline = Date.now() + (args.settleMs ?? 150_000); + let soon: any = null; + let imm: any = null; + while (Date.now() < deadline) { + const rows = await ctx.runQuery(internal.test.latency.pairRows, { + cell: args.cell, + }); + soon = rows.find( + (r: any) => r.pairId === 0 && r.delayMs === args.soonDelayMs, + ); + imm = rows.find((r: any) => r.pairId === 0 && r.delayMs === 0); + if (soon?.startedAt && imm?.startedAt) break; + await new Promise((r) => setTimeout(r, 500)); + } + return { + commitMs: committedAt - t0, + soonLatenessMs: soon?.startedAt ? soon.startedAt - soon.runAt : null, + immLatenessMs: imm?.startedAt ? imm.startedAt - imm.runAt : null, + soonMinusImmMs: + soon?.startedAt && imm?.startedAt + ? soon.startedAt - imm.startedAt + : null, + soonStarted: !!soon?.startedAt, + }; + }, +}); + +/** + * The same bulk, but through the batch enqueue API — one component call + * carrying every item, which is how an app actually queues thousands at once. + * All of them land in one transaction under one commit stamp, packed 256 to a + * document, which is the configuration that can exceed SWEEP_BATCH. + */ +export const bulkScheduledBatch = internalMutation({ + args: { + cell: v.string(), + pool: vPoolKind, + count: v.number(), + delayMs: v.number(), + maxParallelism: v.number(), + }, + returns: v.number(), + handler: async (ctx, args) => { + const { component, enqueueMany } = enqueueFor(args.pool as PoolKind); + const enqueuedAt = Date.now(); + const argsArray: { taskId: Id<"latencyTasks"> }[] = []; + for (let i = 0; i < args.count; i++) { + const taskId = await ctx.db.insert("latencyTasks", { + cell: args.cell, + pool: args.pool, + delayMs: args.delayMs, + seq: 10_000 + i, + enqueuedAt, + runAt: enqueuedAt + args.delayMs, + }); + argsArray.push({ taskId }); + } + await enqueueMany( + component, + ctx, + "mutation", + internal.test.latency.probe, + argsArray, + { runAfter: args.delayMs, maxParallelism: args.maxParallelism }, + ); + return args.count; + }, +}); diff --git a/example/convex/test/scheduling.ts b/example/convex/test/scheduling.ts new file mode 100644 index 00000000..c0e14d2d --- /dev/null +++ b/example/convex/test/scheduling.ts @@ -0,0 +1,147 @@ +import { v } from "convex/values"; +import { + internalAction, + internalMutation, + internalQuery, +} from "../_generated/server"; +import { components, internal } from "../_generated/api"; +import { Workpool } from "@convex-dev/workpool"; + +/** + * End-to-end checks for the two paths the throughput scenarios never touch: + * work scheduled for later, and work that retries. Both put a wall-clock time + * into the queue's commit-timestamp-ordered field instead of the placeholder, + * so they're worth exercising against a real deployment and not just + * convex-test. + * + * Run: + * npx convex run test/scheduling:default + * npx convex run test/scheduling:default '{"delayMs":400000}' # past the + * # safe-future threshold + */ +const pool = new Workpool(components.testWorkpool, { maxParallelism: 10 }); + +const vProbe = v.object({ + label: v.string(), + at: v.number(), + attempt: v.optional(v.number()), +}); + +export const record = internalMutation({ + args: { label: v.string(), attempt: v.optional(v.number()) }, + returns: v.null(), + handler: async (ctx, { label, attempt }) => { + await ctx.db.insert("schedulingProbes", { label, at: Date.now(), attempt }); + return null; + }, +}); + +export const probes = internalQuery({ + args: {}, + returns: v.array(vProbe), + handler: async (ctx) => { + const docs = await ctx.db.query("schedulingProbes").collect(); + return docs + .map(({ label, at, attempt }) => ({ label, at, attempt })) + .sort((a, b) => a.at - b.at); + }, +}); + +export const reset = internalMutation({ + args: {}, + returns: v.null(), + handler: async (ctx) => { + for (const doc of await ctx.db.query("schedulingProbes").collect()) { + await ctx.db.delete("schedulingProbes", doc._id); + } + return null; + }, +}); + +/** Fails its first two attempts so the retry backoff path runs. */ +export const failTwice = internalAction({ + args: { label: v.string() }, + returns: v.null(), + handler: async (ctx, { label }) => { + const seen = ( + await ctx.runQuery(internal.test.scheduling.probes, {}) + ).filter((p) => p.label === label).length; + await ctx.runMutation(internal.test.scheduling.record, { + label, + attempt: seen + 1, + }); + if (seen < 2) throw new Error(`attempt ${seen + 1} fails on purpose`); + return null; + }, +}); + +export const enqueueBoth = internalMutation({ + args: { delayMs: v.number() }, + returns: v.null(), + handler: async (ctx, { delayMs }) => { + await pool.enqueueMutation( + ctx, + internal.test.scheduling.record, + { label: "delayed" }, + { runAfter: delayMs }, + ); + await pool.enqueueAction( + ctx, + internal.test.scheduling.failTwice, + { label: "retry" }, + { retry: { maxAttempts: 4, initialBackoffMs: 500, base: 2 } }, + ); + return null; + }, +}); + +export default internalAction({ + args: { delayMs: v.optional(v.number()) }, + handler: async (ctx, { delayMs = 8_000 }) => { + await ctx.runMutation(internal.test.scheduling.reset, {}); + + const enqueuedAt = Date.now(); + await ctx.runMutation(internal.test.scheduling.enqueueBoth, { delayMs }); + + const deadline = Date.now() + delayMs + 60_000; + // Keep the number of polls bounded — a long delay at a fixed short interval + // runs past an action's limit on how many functions it may call. + const pollMs = Math.max(250, Math.round(delayMs / 100)); + let probes: { label: string; at: number; attempt?: number }[]; + while (true) { + probes = await ctx.runQuery(internal.test.scheduling.probes, {}); + const delayed = probes.find((p) => p.label === "delayed"); + const retriesDone = probes.filter((p) => p.label === "retry").length >= 3; + // Read once at the deadline so the final result includes the last wait. + if ((delayed && retriesDone) || Date.now() >= deadline) break; + await new Promise((r) => setTimeout(r, pollMs)); + } + + const delayed = probes.find((p) => p.label === "delayed"); + const ranEarly = !!delayed && delayed.at < enqueuedAt + delayMs; + const retries = probes.filter((p) => p.label === "retry"); + const lateByMs = delayed ? delayed.at - (enqueuedAt + delayMs) : undefined; + + console.log("=== scheduling results ==="); + console.log( + delayed + ? `delayed (runAfter ${delayMs}ms): ran ${lateByMs}ms after its runAt` + + (ranEarly ? " — RAN EARLY" : "") + : `delayed (runAfter ${delayMs}ms): NEVER RAN`, + ); + console.log( + `retry: ${retries.length} attempts, gaps ${ + retries + .slice(1) + .map((p, i) => p.at - retries[i].at) + .join("/") || "n/a" + }ms`, + ); + return { + delayedRan: !!delayed, + delayedRanEarly: ranEarly, + delayedLateByMs: lateByMs, + retryAttempts: retries.length, + }; + }, +}); diff --git a/package-lock.json b/package-lock.json index be906265..246dcb81 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "devDependencies": { "@convex-dev/eslint-plugin": "^5.0.0", "@convex-dev/static-hosting": "^0.2.0", - "@convex-dev/workpool-old": "npm:@convex-dev/workpool@0.4.7", + "@convex-dev/workpool-old": "npm:@convex-dev/workpool@0.4.9", "@edge-runtime/vm": "5.0.0", "@eslint/eslintrc": "3.3.3", "@eslint/js": "10.0.1", @@ -610,13 +610,16 @@ }, "node_modules/@convex-dev/workpool-old": { "name": "@convex-dev/workpool", - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/@convex-dev/workpool/-/workpool-0.4.7.tgz", - "integrity": "sha512-4O3VKcJXqYZ9icDgKdVPxjDGUAFK3oG0hbUwLcyYMYgsvVKlZDhvZRmczqSBZHLyrCGPpf925byh0dBigCfAGA==", + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/@convex-dev/workpool/-/workpool-0.4.9.tgz", + "integrity": "sha512-2EauCO+fXqhBE0qN3aC2G/CA8Udc1nDiSha2uwdJi5SaElz6HmuM3Rv60AeQ3snMZ8PdHugKKlVeEFvAhSaokQ==", "dev": true, "license": "Apache-2.0", + "dependencies": { + "@convex-dev/batch-worker": "^0.2.0" + }, "peerDependencies": { - "convex": "^1.31.7", + "convex": "^1.36.1", "convex-helpers": "^0.1.94" } }, diff --git a/package.json b/package.json index c669df14..a4a48e75 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "devDependencies": { "@convex-dev/eslint-plugin": "^5.0.0", "@convex-dev/static-hosting": "^0.2.0", - "@convex-dev/workpool-old": "npm:@convex-dev/workpool@0.4.7", + "@convex-dev/workpool-old": "npm:@convex-dev/workpool@0.4.9", "@edge-runtime/vm": "5.0.0", "@eslint/eslintrc": "3.3.3", "@eslint/js": "10.0.1", From fc45ee7f5b4813ad6a1353a570420dd4908daca1 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:41 -0700 Subject: [PATCH 2/9] docs(benchmarks): record snapshot scheduling results --- example/BENCHMARKS.md | 221 ++ example/README.md | 11 +- example/benchmarks/2026-09-08.json | 5865 ++++++++++++++++++++++++++++ example/benchmarks/2026-09-08.md | 245 ++ 4 files changed, 6338 insertions(+), 4 deletions(-) create mode 100644 example/BENCHMARKS.md create mode 100644 example/benchmarks/2026-09-08.json create mode 100644 example/benchmarks/2026-09-08.md diff --git a/example/BENCHMARKS.md b/example/BENCHMARKS.md new file mode 100644 index 00000000..b7776d84 --- /dev/null +++ b/example/BENCHMARKS.md @@ -0,0 +1,221 @@ +# Benchmarking the workpool + +Recorded results: [2026-09-08, revision `8186121`](./benchmarks/2026-09-08.md). + +Harnesses in `example/convex/test/`, all driving a real deployment: + +| entry point | measures | pools | +| -------------------------------------- | ----------------------------------------------- | --------- | +| `test/scenarios/throughput:default` | end-to-end throughput and completion latency | new / old | +| `test/scenarios/burstyBatches:default` | concurrent enqueue waves, or a light trickle | new / old | +| `test/scenarios/sustained:default` | paced arrivals with variable-duration actions | new / old | +| `test/scenarios/noisyNeighbor:run` | slow/failing neighbors, per-class outcomes | new / old | +| `test/latency:default` | when tasks _start_, vs. when they were due | new / old | +| `test/latency:pairs` | scheduled tasks against an immediate sibling | new / old | +| `test/latency:backlog` | a late entry behind bulk scheduled work | new / old | +| `test/scheduling:default` | delayed work and retry attempts | new only | +| `test/cleanup:start` / `:counts` | clear app bookkeeping; does not drain workpools | shared | + +`"pool": "new"` is this checkout's `testWorkpool`; `"pool": "old"` is the +published package mounted as `oldWorkpool`. Check the installed baseline rather +than relying on dashboard labels or previous reports: + +```sh +node -p 'require("./node_modules/@convex-dev/workpool-old/package.json").version' +npm run build:codegen && npx convex dev --once +``` + +Run commands from the repo root against a dedicated dev deployment. The example +imports the component through its `dist` exports, so **build before deploying**. +A failed deploy means the previous design is still running. In particular, +pre-snapshot experimental deployments can retain the removed `lastCommitTs` +field in `internalState`, causing schema validation to fail. Resolve that state +compatibility issue before measuring; do not treat a failed push as a new build. + +Archive the commit, any source diff, dependency versions, deployment, exact +arguments, raw outputs, and run order. Paired runs on one deployment help +control time-varying noise, but different components still have different +storage histories. They compare complete designs, not an isolated source change. + +## Throughput + +```sh +npx convex run test/scenarios/throughput:default '{ + "taskCount": 5000, "batchSize": 100, "interBatchMs": 50, + "maxParallelism": 200, "taskType": "mutation", "pool": "new" +}' +``` + +Repeat with `"pool": "old"` for the baseline. For actions, use +`"taskType": "action", "taskDurationMs": 20`. **`taskDurationMs` is ignored for +mutations**, even though the current harness includes it in the returned +parameters and log message. The mutation workload above performs no artificial +delay or database workload beyond the completion recorder. + +Discard warmups, then measure at least three pairs, alternating `old,new`, +`new,old`, `old,new`. Clear bookkeeping and wait for both components to drain +between arms. Save JSON before cleanup. Report individual runs, per-pair ratios, +and the spread, not just the fastest run. + +`completedCount` currently counts terminal callbacks, including failures and +cancelations: `markTaskCompleted` does not save `result.kind`. Check runtime +errors as well as `timedOut`, `status`, and the exact expected count. A CLI exit +code of zero alone does not establish that a benchmark finished successfully. + +## Start latency and ordering + +```sh +npx convex run test/latency:default '{ + "cell": "demo", "pool": "new", + "groups": [{"delayMs": 3000, "count": 600}, {"delayMs": 0, "count": 2000}], + "chunkSize": 25, "maxParallelism": 200 +}' +``` + +Every task records its own start clock, supporting start lateness +(`startedAt - runAt`), start ordering, and per-delay-class breakdowns. `holdOps` +and `nestedCalls` stretch the enqueuing transaction, `interChunkMs` spreads +enqueues out, and `settleMs` bounds the additional wait. + +`pairs` enqueues an immediate control alongside scheduled tasks in one +transaction. On the new pool the control is ordered by `db.vars.commitTs`; the +old pool uses its own immediate ordering. Siblings become visible together. When +a scheduled task is already overdue at commit, their start-time difference is +useful for comparing scheduling paths, though execution noise and capacity still +contribute. + +```sh +npx convex run test/latency:pairs '{ + "cell": "pairs-demo", "pool": "new", "count": 40, + "soonDelays": [200], "nestedCalls": 800, "maxParallelism": 200 +}' +``` + +`backlog` puts bulk scheduled work sharing one commit stamp ahead of such a +pair: + +```sh +npx convex run test/latency:backlog '{ + "cell": "backlog-demo", "pool": "new", "bulkCount": 2000, "bulkDelayMs": 60000, + "soonDelayMs": 100, "nestedCalls": 800, "maxParallelism": 200 +}' +``` + +For a behind-cursor experiment, both need light concurrent ready-now work — see +the pool note below. The commands above do **not** generate that traffic. A +completed pair without evidence that the incoming cursor passed the scheduled +key is a latency sample, not proof the sweep ran. `committedAt` in the pair rows +is the action's clock after the enqueue RPC returned, not the database commit +timestamp; a worker can start before that observation. + +`backlog` returns when its two probes start, while the bulk may still be waiting +for `bulkDelayMs`. Wait for that work to finish before cleaning up or running +another cell. Use distinct `cell` names: `resetCell` deletes probe rows but does +not cancel their queued tasks. + +The delayed/retry smoke check is separate and only supports the new pool: + +```sh +npx convex run test/scheduling:default +npx convex run test/scheduling:default '{"delayMs":310000}' +``` + +The second crosses the five-minute threshold where enqueues omit `scanTs`. Its +lateness is approximate: the harness starts its timer before calling the +enqueuing mutation. Three recorded attempts do not independently prove the final +retry's terminal result. + +If the long-delay CLI call errors or times out, check the existing run before +retrying. The recorded 310-second run returned a generic CLI error even though +the server action completed successfully; the cause was not established. Read +persisted probes with `npx convex run test/scheduling:probes`, inspect the +server action's completion/error logs, and check `testWorkpool` queue and +running state using the drain checklist below. If work is still active, wait for +it to settle. Archive that evidence before cleanup or another run: rerunning the +command resets the probes and enqueues new delayed/retry work without canceling +the previous work. + +## Comparing a code change against itself + +Use the same component for both variants, with a build and successful deploy for +each source change, discarded warmups, cleanup, and balanced run order. Keep +edits in isolated checkouts so restoring a variant cannot overwrite other work. +Do not change constants during a comparison of the current design against the +published baseline. + +Historical `.context/bench-commitTs.sh`, `.context/exp-consts.py`, and +experiment reports are workspace artifacts, not tracked tooling available in a +fresh clone. They are not the authoritative procedure: their warmup counts/order +differ from their comments, and restoring with `git checkout` can erase +concurrent edits. + +## Getting numbers you can trust + +**Warm up, and discard it.** Earlier experiments showed substantial warmup +drift. Start with three discarded runs per workload and pool, and check whether +timings stabilize; three is a heuristic, not a guarantee. Balanced ordering does +not automatically cancel nonlinear drift. + +**Prefer paired runs.** Report the within-pair throughput ratio as well as +absolute measurements. Deployment-level variation can move both arms together. A +ratio does not remove differences in component storage history. + +**Drain, archive, then clear bookkeeping between runs.** On this dedicated +benchmark deployment: + +```sh +npx convex run test/cleanup:start +npx convex run test/cleanup:counts # repeat until every count is zero +``` + +Cleanup deletes **all** `tasks`, `latencyTasks`, `runs`, `schedulingProbes`, and +`data`, including dashboard history. It does not clear `counters` or component +tables, cancel tasks, or wait for workers. First verify each component has no +`work`, `pendingStart`, `pendingCompletion`, `pendingCancelation`, `payload`, or +running entries. Then wait for all cleanup counts to reach zero; a missing/error +response is not zero. Concurrent runs and delayed leftovers can repopulate +tables after cleanup. + +Bookkeeping writes are inside the measured path, so accumulated table state is a +confound. Cleanup bounds live rows; it does not reset storage history or prove +that the two components have identical costs. + +**Establish the noise floor before believing an effect.** Repeat identical +configurations and inspect the spread before interpreting a small effect. +Non-monotonic results are not proof of noise: batching, contention, and latency +tradeoffs can produce a real optimum. Investigate accumulated state, offered +load, failures, and resource limits before assigning a cause. + +**A transaction is one sample for commit ordering.** Entries in one enqueue +share a commit stamp and frozen enqueue clock. Their worker start times can +differ because workers run in separate transactions. Treat a chunk as one +correlated trial for out-of-order landing, and use `chunkSize` to obtain +multiple transactions. Those transactions still share deployment-level noise. + +**`Date.now()` is frozen inside a mutation.** A wall-clock spin loop cannot +measure duration there. To stretch a transaction, count operations (`holdOps`) +or call `nestedNoop` repeatedly (`nestedCalls`), and time it from an action, +where the clock advances. These remain subject to function execution limits; +nested calls that do database work consume budget. + +**Control pool load for cursor experiments.** A saturated pool's incoming cursor +can lag, and an idle pool's cursor may not advance. Neither reliably produces +the behind-cursor condition. Use light concurrent ready-now work at high +`maxParallelism`, and verify cursor advancement and available capacity during +the slow enqueue. High parallelism alone is insufficient. + +## Reading a slow loop + +```sh +npx convex logs --jsonl --success +``` + +Entries carry execution time and documents/bytes read and written. Execution +time rising while read/write volume falls is a reason to inspect sequential +round trips, not a diagnosis by itself. Check OCC and execution statistics too. +A past packing regression came from sequential per-entry point reads; +independent reads can be batched with `Promise.all`. + +Both current mounts use a nested `batchWorker`. Include +`testWorkpool/batchWorker` and `oldWorkpool/batchWorker` when comparing loop +executions; counting only the parent component misses loop scheduling. diff --git a/example/README.md b/example/README.md index 822f4d8f..6e383f3c 100644 --- a/example/README.md +++ b/example/README.md @@ -5,6 +5,9 @@ also doubles as a benchmark dashboard for the workpool component itself — it exercises the API and surfaces throughput and latency metrics for the scenarios in `convex/test/scenarios/`. +To run the benchmarks from the command line, and for what it takes to get +numbers worth trusting, see [BENCHMARKS.md](./BENCHMARKS.md). + ## Running the dashboard From the repo root, in two terminals: @@ -21,10 +24,10 @@ the vite config reads from the repo root (`envDir: "../"`). The dashboard is a single comparison workspace. Pick a preset (`burstyBatches`, `throughput`, `overhead`, `sustained`, `bigArgs`, `bigContext`, or -`bigReturnTypes`) and it runs the published 0.4.7 baseline, then this branch, -with identical parameters. The completed pair is selected automatically and -rendered as outcome cards, throughput and latency charts, and recent-run history -on the same page. +`bigReturnTypes`) and it runs the published baseline pinned by +`@convex-dev/workpool-old`, then this branch, with identical parameters. The +completed pair is selected automatically and rendered as outcome cards, +throughput and latency charts, and recent-run history on the same page. Scheduled-function instrumentation stays internal to the components. Backfill a checked run from its component system table with: diff --git a/example/benchmarks/2026-09-08.json b/example/benchmarks/2026-09-08.json new file mode 100644 index 00000000..1672925e --- /dev/null +++ b/example/benchmarks/2026-09-08.json @@ -0,0 +1,5865 @@ +{ + "metadata": { + "commit": "8186121dd366db20b38536215d27496f5c553b4a", + "deployment": "precious-raven-853", + "started": "2026-09-09T01:40:27.924006+00:00", + "loopSha256": "675db597be3cf2d42b6147beff9e6e6866fff7bfd99a85254c3f22e626273513", + "baselineVersion": "0.4.9", + "laterCommentOnlyCommit": "0d80a6c7f4535594b1cb37e09e980d91bd0681c8", + "distLoopSha256": "c67b4679b49d20974c3ed59af0aaf1f269c8c2b9099d00a7126df7bd2b369de1", + "versions": { + "convex": "1.43.0", + "batchWorker": "0.2.1", + "baseline": "0.4.9", + "source": "0.4.10" + }, + "timezone": "America/Los_Angeles", + "nodeVersion": "v22.22.2", + "preflight": { + "testsPassed": 148, + "testFilesPassed": 14, + "testRevision": "0d80a6c", + "builtAndDeployedRevision": "8186121" + }, + "finished": "2026-09-09T02:18:30.704824+00:00", + "finalState": { + "testWorkpool": { + "counts": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "work": 0 + }, + "maxParallelism": 200, + "running": 0 + }, + "oldWorkpool": { + "counts": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "work": 0 + }, + "maxParallelism": 200, + "running": 0 + }, + "bookkeeping": { + "data": 0, + "latencyTasks": 0, + "runs": 0, + "schedulingProbes": 0, + "tasks": 0 + } + }, + "laterBehavioralCommit": "e94b0f3a0c4451e57902a99bc37bef3aafc4164b", + "laterLocalBuildDetectedAt": "2026-09-09T02:13:46Z", + "laterDeploymentVerified": false, + "finalDistLoopSha256": "b778b4862326f9bc75f96eed6535f7066c052d6dbd963e181a9c97342ba42af0", + "revisionScope": "Throughput and diagnostics before 02:13:46 UTC cover 8186121. Far-delay completion and trickle pairs 4\u20136 occurred after a concurrent e94b0f3 build; their deployment revision was not verified." + }, + "runs": [ + { + "label": "mutation-warmup-1-old", + "started": "2026-09-09T01:40:34.554691+00:00", + "phase": "warmup", + "rep": 1, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 15384, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 34630, + "p50": 15511, + "p95": 32794, + "p99": 34595 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 49963 + }, + "tasksPerSec": 100, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-warmup-1-new", + "started": "2026-09-09T01:41:35.949496+00:00", + "phase": "warmup", + "rep": 1, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 12956, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 29953, + "p50": 15726, + "p95": 28606, + "p99": 29683 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 42812 + }, + "tasksPerSec": 117, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-warmup-2-new", + "started": "2026-09-09T01:42:29.111590+00:00", + "phase": "warmup", + "rep": 2, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 10378, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 28627, + "p50": 14598, + "p95": 27348, + "p99": 28344 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 38873 + }, + "tasksPerSec": 129, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-warmup-2-old", + "started": "2026-09-09T01:43:18.978031+00:00", + "phase": "warmup", + "rep": 2, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 10489, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 33490, + "p50": 15611, + "p95": 31903, + "p99": 33447 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 43884 + }, + "tasksPerSec": 114, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-warmup-3-old", + "started": "2026-09-09T01:44:13.688900+00:00", + "phase": "warmup", + "rep": 3, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 9015, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 35420, + "p50": 15892, + "p95": 32873, + "p99": 35403 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 44344 + }, + "tasksPerSec": 113, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-warmup-3-new", + "started": "2026-09-09T01:45:08.655404+00:00", + "phase": "warmup", + "rep": 3, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 9989, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 27568, + "p50": 14849, + "p95": 26159, + "p99": 27291 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 37445 + }, + "tasksPerSec": 134, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-measured-1-old", + "started": "2026-09-09T01:45:56.592261+00:00", + "phase": "measured", + "rep": 1, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 9056, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 39439, + "p50": 16931, + "p95": 37140, + "p99": 38889 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 48417 + }, + "tasksPerSec": 103, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-measured-1-new", + "started": "2026-09-09T01:46:55.814488+00:00", + "phase": "measured", + "rep": 1, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 8651, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 26853, + "p50": 12541, + "p95": 25441, + "p99": 26609 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 35491 + }, + "tasksPerSec": 141, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-measured-2-new", + "started": "2026-09-09T01:47:43.965902+00:00", + "phase": "measured", + "rep": 2, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 9065, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 28248, + "p50": 12848, + "p95": 27162, + "p99": 27975 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 37250 + }, + "tasksPerSec": 134, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-measured-2-old", + "started": "2026-09-09T01:48:31.878395+00:00", + "phase": "measured", + "rep": 2, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 8968, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 32239, + "p50": 14413, + "p95": 30669, + "p99": 32203 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 41108 + }, + "tasksPerSec": 122, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-measured-3-old", + "started": "2026-09-09T01:49:23.896317+00:00", + "phase": "measured", + "rep": 3, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 9292, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 35240, + "p50": 16448, + "p95": 33480, + "p99": 35181 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 44325 + }, + "tasksPerSec": 113, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mutation-measured-3-new", + "started": "2026-09-09T01:50:18.922472+00:00", + "phase": "measured", + "rep": 3, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 120000 + }, + "result": { + "enqueueTotal": 8167, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 25822, + "p50": 12935, + "p95": 24661, + "p99": 25543 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 33935 + }, + "tasksPerSec": 147, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-warmup-1-old", + "started": "2026-09-09T01:51:03.943269+00:00", + "phase": "warmup", + "rep": 1, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 9301, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 27573, + "p50": 12554, + "p95": 25946, + "p99": 27565 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 36785 + }, + "tasksPerSec": 136, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-warmup-1-new", + "started": "2026-09-09T01:51:51.200061+00:00", + "phase": "warmup", + "rep": 1, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 7769, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 19174, + "p50": 9329, + "p95": 18152, + "p99": 18872 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 26864 + }, + "tasksPerSec": 186, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-warmup-2-new", + "started": "2026-09-09T01:52:28.488326+00:00", + "phase": "warmup", + "rep": 2, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 7494, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 18085, + "p50": 9001, + "p95": 17015, + "p99": 17786 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 25511 + }, + "tasksPerSec": 196, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-warmup-2-old", + "started": "2026-09-09T01:53:04.072187+00:00", + "phase": "warmup", + "rep": 2, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 9425, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 29662, + "p50": 13487, + "p95": 28351, + "p99": 29651 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 38996 + }, + "tasksPerSec": 128, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-warmup-3-old", + "started": "2026-09-09T01:53:53.921210+00:00", + "phase": "warmup", + "rep": 3, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 9676, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 29676, + "p50": 13096, + "p95": 28055, + "p99": 29669 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 39249 + }, + "tasksPerSec": 127, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-warmup-3-new", + "started": "2026-09-09T01:54:44.357347+00:00", + "phase": "warmup", + "rep": 3, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 7357, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 19900, + "p50": 8956, + "p95": 18886, + "p99": 19585 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 27190 + }, + "tasksPerSec": 184, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-measured-1-old", + "started": "2026-09-09T01:55:22.052194+00:00", + "phase": "measured", + "rep": 1, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 9593, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 27906, + "p50": 12624, + "p95": 26525, + "p99": 27898 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 37408 + }, + "tasksPerSec": 134, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-measured-1-new", + "started": "2026-09-09T01:56:10.003042+00:00", + "phase": "measured", + "rep": 1, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 7942, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 18297, + "p50": 9250, + "p95": 17188, + "p99": 17970 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 26175 + }, + "tasksPerSec": 191, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-measured-2-new", + "started": "2026-09-09T01:56:46.440319+00:00", + "phase": "measured", + "rep": 2, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 7374, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 18192, + "p50": 9130, + "p95": 17160, + "p99": 17872 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 25483 + }, + "tasksPerSec": 196, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-measured-2-old", + "started": "2026-09-09T01:57:22.381364+00:00", + "phase": "measured", + "rep": 2, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 9080, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 27877, + "p50": 13228, + "p95": 25835, + "p99": 27869 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 36862 + }, + "tasksPerSec": 136, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-measured-3-old", + "started": "2026-09-09T01:58:09.506801+00:00", + "phase": "measured", + "rep": 3, + "pool": "old", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 8370, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 29481, + "p50": 13318, + "p95": 27858, + "p99": 28948 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 37767 + }, + "tasksPerSec": 132, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "action-measured-3-new", + "started": "2026-09-09T01:58:57.703056+00:00", + "phase": "measured", + "rep": 3, + "pool": "new", + "scenario": "throughput", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 120000, + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 7751, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 18307, + "p50": 8769, + "p95": 17485, + "p99": 18049 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 26064 + }, + "tasksPerSec": 192, + "timedOut": false + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-1-old", + "started": "2026-09-09T01:59:41.811445+00:00", + "phase": "measured", + "rep": 1, + "pool": "old", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10980, + "metrics": { + "completedCount": 20, + "latency": { + "max": 1145, + "p50": 290, + "p95": 1132, + "p99": 1145 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 11238, + "waves": [ + { + "count": 1, + "p50": 296, + "p99": 296, + "wave": 0 + }, + { + "count": 1, + "p50": 195, + "p99": 195, + "wave": 1 + }, + { + "count": 1, + "p50": 170, + "p99": 170, + "wave": 2 + }, + { + "count": 1, + "p50": 354, + "p99": 354, + "wave": 3 + }, + { + "count": 1, + "p50": 366, + "p99": 366, + "wave": 4 + }, + { + "count": 1, + "p50": 1145, + "p99": 1145, + "wave": 5 + }, + { + "count": 1, + "p50": 1132, + "p99": 1132, + "wave": 6 + }, + { + "count": 1, + "p50": 695, + "p99": 695, + "wave": 7 + }, + { + "count": 1, + "p50": 251, + "p99": 251, + "wave": 8 + }, + { + "count": 1, + "p50": 259, + "p99": 259, + "wave": 9 + }, + { + "count": 1, + "p50": 808, + "p99": 808, + "wave": 10 + }, + { + "count": 1, + "p50": 661, + "p99": 661, + "wave": 11 + }, + { + "count": 1, + "p50": 274, + "p99": 274, + "wave": 12 + }, + { + "count": 1, + "p50": 338, + "p99": 338, + "wave": 13 + }, + { + "count": 1, + "p50": 313, + "p99": 313, + "wave": 14 + }, + { + "count": 1, + "p50": 287, + "p99": 287, + "wave": 15 + }, + { + "count": 1, + "p50": 290, + "p99": 290, + "wave": 16 + }, + { + "count": 1, + "p50": 254, + "p99": 254, + "wave": 17 + }, + { + "count": 1, + "p50": 237, + "p99": 237, + "wave": 18 + }, + { + "count": 1, + "p50": 271, + "p99": 271, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 152, + "wave": 0 + }, + { + "enqueueMs": 27, + "wave": 1 + }, + { + "enqueueMs": 26, + "wave": 2 + }, + { + "enqueueMs": 27, + "wave": 3 + }, + { + "enqueueMs": 40, + "wave": 4 + }, + { + "enqueueMs": 297, + "wave": 5 + }, + { + "enqueueMs": 23, + "wave": 6 + }, + { + "enqueueMs": 330, + "wave": 7 + }, + { + "enqueueMs": 28, + "wave": 8 + }, + { + "enqueueMs": 45, + "wave": 9 + }, + { + "enqueueMs": 213, + "wave": 10 + }, + { + "enqueueMs": 23, + "wave": 11 + }, + { + "enqueueMs": 25, + "wave": 12 + }, + { + "enqueueMs": 39, + "wave": 13 + }, + { + "enqueueMs": 24, + "wave": 14 + }, + { + "enqueueMs": 27, + "wave": 15 + }, + { + "enqueueMs": 29, + "wave": 16 + }, + { + "enqueueMs": 21, + "wave": 17 + }, + { + "enqueueMs": 25, + "wave": 18 + }, + { + "enqueueMs": 37, + "wave": 19 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-1-new", + "started": "2026-09-09T02:00:01.725611+00:00", + "phase": "measured", + "rep": 1, + "pool": "new", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 11729, + "metrics": { + "completedCount": 20, + "latency": { + "max": 1923, + "p50": 446, + "p95": 1604, + "p99": 1923 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 12165, + "waves": [ + { + "count": 1, + "p50": 201, + "p99": 201, + "wave": 0 + }, + { + "count": 1, + "p50": 198, + "p99": 198, + "wave": 1 + }, + { + "count": 1, + "p50": 1604, + "p99": 1604, + "wave": 2 + }, + { + "count": 1, + "p50": 1305, + "p99": 1305, + "wave": 3 + }, + { + "count": 1, + "p50": 286, + "p99": 286, + "wave": 4 + }, + { + "count": 1, + "p50": 312, + "p99": 312, + "wave": 5 + }, + { + "count": 1, + "p50": 272, + "p99": 272, + "wave": 6 + }, + { + "count": 1, + "p50": 281, + "p99": 281, + "wave": 7 + }, + { + "count": 1, + "p50": 1923, + "p99": 1923, + "wave": 8 + }, + { + "count": 1, + "p50": 1381, + "p99": 1381, + "wave": 9 + }, + { + "count": 1, + "p50": 1144, + "p99": 1144, + "wave": 10 + }, + { + "count": 1, + "p50": 356, + "p99": 356, + "wave": 11 + }, + { + "count": 1, + "p50": 353, + "p99": 353, + "wave": 12 + }, + { + "count": 1, + "p50": 996, + "p99": 996, + "wave": 13 + }, + { + "count": 1, + "p50": 1294, + "p99": 1294, + "wave": 14 + }, + { + "count": 1, + "p50": 1109, + "p99": 1109, + "wave": 15 + }, + { + "count": 1, + "p50": 589, + "p99": 589, + "wave": 16 + }, + { + "count": 1, + "p50": 204, + "p99": 204, + "wave": 17 + }, + { + "count": 1, + "p50": 885, + "p99": 885, + "wave": 18 + }, + { + "count": 1, + "p50": 446, + "p99": 446, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 50, + "wave": 0 + }, + { + "enqueueMs": 28, + "wave": 1 + }, + { + "enqueueMs": 39, + "wave": 2 + }, + { + "enqueueMs": 655, + "wave": 3 + }, + { + "enqueueMs": 75, + "wave": 4 + }, + { + "enqueueMs": 42, + "wave": 5 + }, + { + "enqueueMs": 30, + "wave": 6 + }, + { + "enqueueMs": 34, + "wave": 7 + }, + { + "enqueueMs": 26, + "wave": 8 + }, + { + "enqueueMs": 26, + "wave": 9 + }, + { + "enqueueMs": 804, + "wave": 10 + }, + { + "enqueueMs": 31, + "wave": 11 + }, + { + "enqueueMs": 26, + "wave": 12 + }, + { + "enqueueMs": 26, + "wave": 13 + }, + { + "enqueueMs": 49, + "wave": 14 + }, + { + "enqueueMs": 75, + "wave": 15 + }, + { + "enqueueMs": 73, + "wave": 16 + }, + { + "enqueueMs": 31, + "wave": 17 + }, + { + "enqueueMs": 38, + "wave": 18 + }, + { + "enqueueMs": 34, + "wave": 19 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-2-new", + "started": "2026-09-09T02:00:24.765916+00:00", + "phase": "measured", + "rep": 2, + "pool": "new", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10337, + "metrics": { + "completedCount": 20, + "latency": { + "max": 488, + "p50": 244, + "p95": 443, + "p99": 488 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10581, + "waves": [ + { + "count": 1, + "p50": 160, + "p99": 160, + "wave": 0 + }, + { + "count": 1, + "p50": 488, + "p99": 488, + "wave": 1 + }, + { + "count": 1, + "p50": 400, + "p99": 400, + "wave": 2 + }, + { + "count": 1, + "p50": 314, + "p99": 314, + "wave": 3 + }, + { + "count": 1, + "p50": 287, + "p99": 287, + "wave": 4 + }, + { + "count": 1, + "p50": 254, + "p99": 254, + "wave": 5 + }, + { + "count": 1, + "p50": 244, + "p99": 244, + "wave": 6 + }, + { + "count": 1, + "p50": 226, + "p99": 226, + "wave": 7 + }, + { + "count": 1, + "p50": 197, + "p99": 197, + "wave": 8 + }, + { + "count": 1, + "p50": 181, + "p99": 181, + "wave": 9 + }, + { + "count": 1, + "p50": 193, + "p99": 193, + "wave": 10 + }, + { + "count": 1, + "p50": 171, + "p99": 171, + "wave": 11 + }, + { + "count": 1, + "p50": 184, + "p99": 184, + "wave": 12 + }, + { + "count": 1, + "p50": 182, + "p99": 182, + "wave": 13 + }, + { + "count": 1, + "p50": 148, + "p99": 148, + "wave": 14 + }, + { + "count": 1, + "p50": 443, + "p99": 443, + "wave": 15 + }, + { + "count": 1, + "p50": 324, + "p99": 324, + "wave": 16 + }, + { + "count": 1, + "p50": 293, + "p99": 293, + "wave": 17 + }, + { + "count": 1, + "p50": 275, + "p99": 275, + "wave": 18 + }, + { + "count": 1, + "p50": 249, + "p99": 249, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 38, + "wave": 0 + }, + { + "enqueueMs": 114, + "wave": 1 + }, + { + "enqueueMs": 67, + "wave": 2 + }, + { + "enqueueMs": 30, + "wave": 3 + }, + { + "enqueueMs": 32, + "wave": 4 + }, + { + "enqueueMs": 28, + "wave": 5 + }, + { + "enqueueMs": 28, + "wave": 6 + }, + { + "enqueueMs": 35, + "wave": 7 + }, + { + "enqueueMs": 26, + "wave": 8 + }, + { + "enqueueMs": 28, + "wave": 9 + }, + { + "enqueueMs": 30, + "wave": 10 + }, + { + "enqueueMs": 33, + "wave": 11 + }, + { + "enqueueMs": 39, + "wave": 12 + }, + { + "enqueueMs": 30, + "wave": 13 + }, + { + "enqueueMs": 25, + "wave": 14 + }, + { + "enqueueMs": 130, + "wave": 15 + }, + { + "enqueueMs": 31, + "wave": 16 + }, + { + "enqueueMs": 25, + "wave": 17 + }, + { + "enqueueMs": 28, + "wave": 18 + }, + { + "enqueueMs": 30, + "wave": 19 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-2-old", + "started": "2026-09-09T02:00:43.835416+00:00", + "phase": "measured", + "rep": 2, + "pool": "old", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10935, + "metrics": { + "completedCount": 20, + "latency": { + "max": 928, + "p50": 247, + "p95": 604, + "p99": 928 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 11265, + "waves": [ + { + "count": 1, + "p50": 928, + "p99": 928, + "wave": 0 + }, + { + "count": 1, + "p50": 377, + "p99": 377, + "wave": 1 + }, + { + "count": 1, + "p50": 351, + "p99": 351, + "wave": 2 + }, + { + "count": 1, + "p50": 604, + "p99": 604, + "wave": 3 + }, + { + "count": 1, + "p50": 421, + "p99": 421, + "wave": 4 + }, + { + "count": 1, + "p50": 233, + "p99": 233, + "wave": 5 + }, + { + "count": 1, + "p50": 201, + "p99": 201, + "wave": 6 + }, + { + "count": 1, + "p50": 195, + "p99": 195, + "wave": 7 + }, + { + "count": 1, + "p50": 164, + "p99": 164, + "wave": 8 + }, + { + "count": 1, + "p50": 341, + "p99": 341, + "wave": 9 + }, + { + "count": 1, + "p50": 310, + "p99": 310, + "wave": 10 + }, + { + "count": 1, + "p50": 285, + "p99": 285, + "wave": 11 + }, + { + "count": 1, + "p50": 273, + "p99": 273, + "wave": 12 + }, + { + "count": 1, + "p50": 247, + "p99": 247, + "wave": 13 + }, + { + "count": 1, + "p50": 229, + "p99": 229, + "wave": 14 + }, + { + "count": 1, + "p50": 216, + "p99": 216, + "wave": 15 + }, + { + "count": 1, + "p50": 194, + "p99": 194, + "wave": 16 + }, + { + "count": 1, + "p50": 166, + "p99": 166, + "wave": 17 + }, + { + "count": 1, + "p50": 168, + "p99": 168, + "wave": 18 + }, + { + "count": 1, + "p50": 326, + "p99": 326, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 646, + "wave": 0 + }, + { + "enqueueMs": 53, + "wave": 1 + }, + { + "enqueueMs": 29, + "wave": 2 + }, + { + "enqueueMs": 43, + "wave": 3 + }, + { + "enqueueMs": 204, + "wave": 4 + }, + { + "enqueueMs": 30, + "wave": 5 + }, + { + "enqueueMs": 30, + "wave": 6 + }, + { + "enqueueMs": 28, + "wave": 7 + }, + { + "enqueueMs": 28, + "wave": 8 + }, + { + "enqueueMs": 29, + "wave": 9 + }, + { + "enqueueMs": 24, + "wave": 10 + }, + { + "enqueueMs": 25, + "wave": 11 + }, + { + "enqueueMs": 31, + "wave": 12 + }, + { + "enqueueMs": 24, + "wave": 13 + }, + { + "enqueueMs": 31, + "wave": 14 + }, + { + "enqueueMs": 41, + "wave": 15 + }, + { + "enqueueMs": 32, + "wave": 16 + }, + { + "enqueueMs": 25, + "wave": 17 + }, + { + "enqueueMs": 29, + "wave": 18 + }, + { + "enqueueMs": 26, + "wave": 19 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-3-old", + "started": "2026-09-09T02:01:04.344878+00:00", + "phase": "measured", + "rep": 3, + "pool": "old", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10182, + "metrics": { + "completedCount": 20, + "latency": { + "max": 393, + "p50": 208, + "p95": 350, + "p99": 393 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10402, + "waves": [ + { + "count": 1, + "p50": 259, + "p99": 259, + "wave": 0 + }, + { + "count": 1, + "p50": 206, + "p99": 206, + "wave": 1 + }, + { + "count": 1, + "p50": 198, + "p99": 198, + "wave": 2 + }, + { + "count": 1, + "p50": 168, + "p99": 168, + "wave": 3 + }, + { + "count": 1, + "p50": 154, + "p99": 154, + "wave": 4 + }, + { + "count": 1, + "p50": 327, + "p99": 327, + "wave": 5 + }, + { + "count": 1, + "p50": 306, + "p99": 306, + "wave": 6 + }, + { + "count": 1, + "p50": 290, + "p99": 290, + "wave": 7 + }, + { + "count": 1, + "p50": 245, + "p99": 245, + "wave": 8 + }, + { + "count": 1, + "p50": 235, + "p99": 235, + "wave": 9 + }, + { + "count": 1, + "p50": 219, + "p99": 219, + "wave": 10 + }, + { + "count": 1, + "p50": 208, + "p99": 208, + "wave": 11 + }, + { + "count": 1, + "p50": 328, + "p99": 328, + "wave": 12 + }, + { + "count": 1, + "p50": 350, + "p99": 350, + "wave": 13 + }, + { + "count": 1, + "p50": 393, + "p99": 393, + "wave": 14 + }, + { + "count": 1, + "p50": 177, + "p99": 177, + "wave": 15 + }, + { + "count": 1, + "p50": 186, + "p99": 186, + "wave": 16 + }, + { + "count": 1, + "p50": 189, + "p99": 189, + "wave": 17 + }, + { + "count": 1, + "p50": 177, + "p99": 177, + "wave": 18 + }, + { + "count": 1, + "p50": 166, + "p99": 166, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 54, + "wave": 0 + }, + { + "enqueueMs": 30, + "wave": 1 + }, + { + "enqueueMs": 25, + "wave": 2 + }, + { + "enqueueMs": 21, + "wave": 3 + }, + { + "enqueueMs": 27, + "wave": 4 + }, + { + "enqueueMs": 27, + "wave": 5 + }, + { + "enqueueMs": 41, + "wave": 6 + }, + { + "enqueueMs": 37, + "wave": 7 + }, + { + "enqueueMs": 28, + "wave": 8 + }, + { + "enqueueMs": 27, + "wave": 9 + }, + { + "enqueueMs": 27, + "wave": 10 + }, + { + "enqueueMs": 30, + "wave": 11 + }, + { + "enqueueMs": 58, + "wave": 12 + }, + { + "enqueueMs": 53, + "wave": 13 + }, + { + "enqueueMs": 41, + "wave": 14 + }, + { + "enqueueMs": 25, + "wave": 15 + }, + { + "enqueueMs": 30, + "wave": 16 + }, + { + "enqueueMs": 27, + "wave": 17 + }, + { + "enqueueMs": 28, + "wave": 18 + }, + { + "enqueueMs": 28, + "wave": 19 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-3-new", + "started": "2026-09-09T02:01:23.089990+00:00", + "phase": "measured", + "rep": 3, + "pool": "new", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10625, + "metrics": { + "completedCount": 20, + "latency": { + "max": 1669, + "p50": 288, + "p95": 1430, + "p99": 1669 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 11168, + "waves": [ + { + "count": 1, + "p50": 1669, + "p99": 1669, + "wave": 0 + }, + { + "count": 1, + "p50": 1430, + "p99": 1430, + "wave": 1 + }, + { + "count": 1, + "p50": 802, + "p99": 802, + "wave": 2 + }, + { + "count": 1, + "p50": 400, + "p99": 400, + "wave": 3 + }, + { + "count": 1, + "p50": 193, + "p99": 193, + "wave": 4 + }, + { + "count": 1, + "p50": 164, + "p99": 164, + "wave": 5 + }, + { + "count": 1, + "p50": 1308, + "p99": 1308, + "wave": 6 + }, + { + "count": 1, + "p50": 1106, + "p99": 1106, + "wave": 7 + }, + { + "count": 1, + "p50": 851, + "p99": 851, + "wave": 8 + }, + { + "count": 1, + "p50": 352, + "p99": 352, + "wave": 9 + }, + { + "count": 1, + "p50": 288, + "p99": 288, + "wave": 10 + }, + { + "count": 1, + "p50": 269, + "p99": 269, + "wave": 11 + }, + { + "count": 1, + "p50": 265, + "p99": 265, + "wave": 12 + }, + { + "count": 1, + "p50": 270, + "p99": 270, + "wave": 13 + }, + { + "count": 1, + "p50": 228, + "p99": 228, + "wave": 14 + }, + { + "count": 1, + "p50": 214, + "p99": 214, + "wave": 15 + }, + { + "count": 1, + "p50": 196, + "p99": 196, + "wave": 16 + }, + { + "count": 1, + "p50": 180, + "p99": 180, + "wave": 17 + }, + { + "count": 1, + "p50": 372, + "p99": 372, + "wave": 18 + }, + { + "count": 1, + "p50": 347, + "p99": 347, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 38, + "wave": 0 + }, + { + "enqueueMs": 26, + "wave": 1 + }, + { + "enqueueMs": 30, + "wave": 2 + }, + { + "enqueueMs": 25, + "wave": 3 + }, + { + "enqueueMs": 27, + "wave": 4 + }, + { + "enqueueMs": 32, + "wave": 5 + }, + { + "enqueueMs": 410, + "wave": 6 + }, + { + "enqueueMs": 38, + "wave": 7 + }, + { + "enqueueMs": 44, + "wave": 8 + }, + { + "enqueueMs": 90, + "wave": 9 + }, + { + "enqueueMs": 31, + "wave": 10 + }, + { + "enqueueMs": 25, + "wave": 11 + }, + { + "enqueueMs": 32, + "wave": 12 + }, + { + "enqueueMs": 45, + "wave": 13 + }, + { + "enqueueMs": 26, + "wave": 14 + }, + { + "enqueueMs": 27, + "wave": 15 + }, + { + "enqueueMs": 40, + "wave": 16 + }, + { + "enqueueMs": 29, + "wave": 17 + }, + { + "enqueueMs": 38, + "wave": 18 + }, + { + "enqueueMs": 29, + "wave": 19 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "trickle-4-new", + "started": "2026-09-09T02:15:46.953635+00:00", + "phase": "measured", + "rep": 4, + "pool": "new", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10472, + "metrics": { + "completedCount": 20, + "latency": { + "max": 1496, + "p50": 261, + "p95": 1158, + "p99": 1496 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10750, + "waves": [ + { + "count": 1, + "p50": 294, + "p99": 294, + "wave": 0 + }, + { + "count": 1, + "p50": 291, + "p99": 291, + "wave": 1 + }, + { + "count": 1, + "p50": 400, + "p99": 400, + "wave": 2 + }, + { + "count": 1, + "p50": 350, + "p99": 350, + "wave": 3 + }, + { + "count": 1, + "p50": 1496, + "p99": 1496, + "wave": 4 + }, + { + "count": 1, + "p50": 1158, + "p99": 1158, + "wave": 5 + }, + { + "count": 1, + "p50": 592, + "p99": 592, + "wave": 6 + }, + { + "count": 1, + "p50": 322, + "p99": 322, + "wave": 7 + }, + { + "count": 1, + "p50": 233, + "p99": 233, + "wave": 8 + }, + { + "count": 1, + "p50": 217, + "p99": 217, + "wave": 9 + }, + { + "count": 1, + "p50": 317, + "p99": 317, + "wave": 10 + }, + { + "count": 1, + "p50": 270, + "p99": 270, + "wave": 11 + }, + { + "count": 1, + "p50": 261, + "p99": 261, + "wave": 12 + }, + { + "count": 1, + "p50": 228, + "p99": 228, + "wave": 13 + }, + { + "count": 1, + "p50": 238, + "p99": 238, + "wave": 14 + }, + { + "count": 1, + "p50": 236, + "p99": 236, + "wave": 15 + }, + { + "count": 1, + "p50": 217, + "p99": 217, + "wave": 16 + }, + { + "count": 1, + "p50": 207, + "p99": 207, + "wave": 17 + }, + { + "count": 1, + "p50": 193, + "p99": 193, + "wave": 18 + }, + { + "count": 1, + "p50": 163, + "p99": 163, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 38, + "wave": 0 + }, + { + "enqueueMs": 38, + "wave": 1 + }, + { + "enqueueMs": 34, + "wave": 2 + }, + { + "enqueueMs": 35, + "wave": 3 + }, + { + "enqueueMs": 312, + "wave": 4 + }, + { + "enqueueMs": 27, + "wave": 5 + }, + { + "enqueueMs": 33, + "wave": 6 + }, + { + "enqueueMs": 98, + "wave": 7 + }, + { + "enqueueMs": 28, + "wave": 8 + }, + { + "enqueueMs": 25, + "wave": 9 + }, + { + "enqueueMs": 27, + "wave": 10 + }, + { + "enqueueMs": 24, + "wave": 11 + }, + { + "enqueueMs": 30, + "wave": 12 + }, + { + "enqueueMs": 28, + "wave": 13 + }, + { + "enqueueMs": 26, + "wave": 14 + }, + { + "enqueueMs": 30, + "wave": 15 + }, + { + "enqueueMs": 26, + "wave": 16 + }, + { + "enqueueMs": 25, + "wave": 17 + }, + { + "enqueueMs": 37, + "wave": 18 + }, + { + "enqueueMs": 28, + "wave": 19 + } + ] + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + }, + { + "label": "trickle-4-old", + "started": "2026-09-09T02:16:06.095764+00:00", + "phase": "measured", + "rep": 4, + "pool": "old", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10046, + "metrics": { + "completedCount": 20, + "latency": { + "max": 381, + "p50": 244, + "p95": 361, + "p99": 381 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10492, + "waves": [ + { + "count": 1, + "p50": 278, + "p99": 278, + "wave": 0 + }, + { + "count": 1, + "p50": 244, + "p99": 244, + "wave": 1 + }, + { + "count": 1, + "p50": 215, + "p99": 215, + "wave": 2 + }, + { + "count": 1, + "p50": 194, + "p99": 194, + "wave": 3 + }, + { + "count": 1, + "p50": 176, + "p99": 176, + "wave": 4 + }, + { + "count": 1, + "p50": 361, + "p99": 361, + "wave": 5 + }, + { + "count": 1, + "p50": 328, + "p99": 328, + "wave": 6 + }, + { + "count": 1, + "p50": 315, + "p99": 315, + "wave": 7 + }, + { + "count": 1, + "p50": 290, + "p99": 290, + "wave": 8 + }, + { + "count": 1, + "p50": 301, + "p99": 301, + "wave": 9 + }, + { + "count": 1, + "p50": 276, + "p99": 276, + "wave": 10 + }, + { + "count": 1, + "p50": 252, + "p99": 252, + "wave": 11 + }, + { + "count": 1, + "p50": 234, + "p99": 234, + "wave": 12 + }, + { + "count": 1, + "p50": 216, + "p99": 216, + "wave": 13 + }, + { + "count": 1, + "p50": 223, + "p99": 223, + "wave": 14 + }, + { + "count": 1, + "p50": 182, + "p99": 182, + "wave": 15 + }, + { + "count": 1, + "p50": 168, + "p99": 168, + "wave": 16 + }, + { + "count": 1, + "p50": 161, + "p99": 161, + "wave": 17 + }, + { + "count": 1, + "p50": 337, + "p99": 337, + "wave": 18 + }, + { + "count": 1, + "p50": 381, + "p99": 381, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 28, + "wave": 0 + }, + { + "enqueueMs": 35, + "wave": 1 + }, + { + "enqueueMs": 25, + "wave": 2 + }, + { + "enqueueMs": 26, + "wave": 3 + }, + { + "enqueueMs": 34, + "wave": 4 + }, + { + "enqueueMs": 34, + "wave": 5 + }, + { + "enqueueMs": 23, + "wave": 6 + }, + { + "enqueueMs": 24, + "wave": 7 + }, + { + "enqueueMs": 22, + "wave": 8 + }, + { + "enqueueMs": 29, + "wave": 9 + }, + { + "enqueueMs": 24, + "wave": 10 + }, + { + "enqueueMs": 20, + "wave": 11 + }, + { + "enqueueMs": 25, + "wave": 12 + }, + { + "enqueueMs": 23, + "wave": 13 + }, + { + "enqueueMs": 31, + "wave": 14 + }, + { + "enqueueMs": 25, + "wave": 15 + }, + { + "enqueueMs": 21, + "wave": 16 + }, + { + "enqueueMs": 25, + "wave": 17 + }, + { + "enqueueMs": 22, + "wave": 18 + }, + { + "enqueueMs": 22, + "wave": 19 + } + ] + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + }, + { + "label": "trickle-5-old", + "started": "2026-09-09T02:16:25.327142+00:00", + "phase": "measured", + "rep": 5, + "pool": "old", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 11069, + "metrics": { + "completedCount": 20, + "latency": { + "max": 1672, + "p50": 265, + "p95": 1160, + "p99": 1672 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 11661, + "waves": [ + { + "count": 1, + "p50": 180, + "p99": 180, + "wave": 0 + }, + { + "count": 1, + "p50": 205, + "p99": 205, + "wave": 1 + }, + { + "count": 1, + "p50": 191, + "p99": 191, + "wave": 2 + }, + { + "count": 1, + "p50": 349, + "p99": 349, + "wave": 3 + }, + { + "count": 1, + "p50": 381, + "p99": 381, + "wave": 4 + }, + { + "count": 1, + "p50": 359, + "p99": 359, + "wave": 5 + }, + { + "count": 1, + "p50": 250, + "p99": 250, + "wave": 6 + }, + { + "count": 1, + "p50": 265, + "p99": 265, + "wave": 7 + }, + { + "count": 1, + "p50": 262, + "p99": 262, + "wave": 8 + }, + { + "count": 1, + "p50": 205, + "p99": 205, + "wave": 9 + }, + { + "count": 1, + "p50": 179, + "p99": 179, + "wave": 10 + }, + { + "count": 1, + "p50": 196, + "p99": 196, + "wave": 11 + }, + { + "count": 1, + "p50": 299, + "p99": 299, + "wave": 12 + }, + { + "count": 1, + "p50": 1672, + "p99": 1672, + "wave": 13 + }, + { + "count": 1, + "p50": 1160, + "p99": 1160, + "wave": 14 + }, + { + "count": 1, + "p50": 738, + "p99": 738, + "wave": 15 + }, + { + "count": 1, + "p50": 620, + "p99": 620, + "wave": 16 + }, + { + "count": 1, + "p50": 729, + "p99": 729, + "wave": 17 + }, + { + "count": 1, + "p50": 194, + "p99": 194, + "wave": 18 + }, + { + "count": 1, + "p50": 593, + "p99": 593, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 35, + "wave": 0 + }, + { + "enqueueMs": 31, + "wave": 1 + }, + { + "enqueueMs": 25, + "wave": 2 + }, + { + "enqueueMs": 31, + "wave": 3 + }, + { + "enqueueMs": 25, + "wave": 4 + }, + { + "enqueueMs": 119, + "wave": 5 + }, + { + "enqueueMs": 24, + "wave": 6 + }, + { + "enqueueMs": 24, + "wave": 7 + }, + { + "enqueueMs": 71, + "wave": 8 + }, + { + "enqueueMs": 25, + "wave": 9 + }, + { + "enqueueMs": 28, + "wave": 10 + }, + { + "enqueueMs": 24, + "wave": 11 + }, + { + "enqueueMs": 25, + "wave": 12 + }, + { + "enqueueMs": 660, + "wave": 13 + }, + { + "enqueueMs": 55, + "wave": 14 + }, + { + "enqueueMs": 25, + "wave": 15 + }, + { + "enqueueMs": 23, + "wave": 16 + }, + { + "enqueueMs": 246, + "wave": 17 + }, + { + "enqueueMs": 28, + "wave": 18 + }, + { + "enqueueMs": 24, + "wave": 19 + } + ] + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + }, + { + "label": "trickle-5-new", + "started": "2026-09-09T02:16:45.492342+00:00", + "phase": "measured", + "rep": 5, + "pool": "new", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10180, + "metrics": { + "completedCount": 20, + "latency": { + "max": 697, + "p50": 221, + "p95": 440, + "p99": 697 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10500, + "waves": [ + { + "count": 1, + "p50": 383, + "p99": 383, + "wave": 0 + }, + { + "count": 1, + "p50": 440, + "p99": 440, + "wave": 1 + }, + { + "count": 1, + "p50": 387, + "p99": 387, + "wave": 2 + }, + { + "count": 1, + "p50": 697, + "p99": 697, + "wave": 3 + }, + { + "count": 1, + "p50": 215, + "p99": 215, + "wave": 4 + }, + { + "count": 1, + "p50": 221, + "p99": 221, + "wave": 5 + }, + { + "count": 1, + "p50": 204, + "p99": 204, + "wave": 6 + }, + { + "count": 1, + "p50": 197, + "p99": 197, + "wave": 7 + }, + { + "count": 1, + "p50": 182, + "p99": 182, + "wave": 8 + }, + { + "count": 1, + "p50": 168, + "p99": 168, + "wave": 9 + }, + { + "count": 1, + "p50": 334, + "p99": 334, + "wave": 10 + }, + { + "count": 1, + "p50": 352, + "p99": 352, + "wave": 11 + }, + { + "count": 1, + "p50": 326, + "p99": 326, + "wave": 12 + }, + { + "count": 1, + "p50": 301, + "p99": 301, + "wave": 13 + }, + { + "count": 1, + "p50": 311, + "p99": 311, + "wave": 14 + }, + { + "count": 1, + "p50": 212, + "p99": 212, + "wave": 15 + }, + { + "count": 1, + "p50": 227, + "p99": 227, + "wave": 16 + }, + { + "count": 1, + "p50": 193, + "p99": 193, + "wave": 17 + }, + { + "count": 1, + "p50": 178, + "p99": 178, + "wave": 18 + }, + { + "count": 1, + "p50": 148, + "p99": 148, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 40, + "wave": 0 + }, + { + "enqueueMs": 53, + "wave": 1 + }, + { + "enqueueMs": 43, + "wave": 2 + }, + { + "enqueueMs": 40, + "wave": 3 + }, + { + "enqueueMs": 31, + "wave": 4 + }, + { + "enqueueMs": 25, + "wave": 5 + }, + { + "enqueueMs": 27, + "wave": 6 + }, + { + "enqueueMs": 28, + "wave": 7 + }, + { + "enqueueMs": 28, + "wave": 8 + }, + { + "enqueueMs": 37, + "wave": 9 + }, + { + "enqueueMs": 30, + "wave": 10 + }, + { + "enqueueMs": 33, + "wave": 11 + }, + { + "enqueueMs": 27, + "wave": 12 + }, + { + "enqueueMs": 30, + "wave": 13 + }, + { + "enqueueMs": 30, + "wave": 14 + }, + { + "enqueueMs": 36, + "wave": 15 + }, + { + "enqueueMs": 37, + "wave": 16 + }, + { + "enqueueMs": 29, + "wave": 17 + }, + { + "enqueueMs": 30, + "wave": 18 + }, + { + "enqueueMs": 25, + "wave": 19 + } + ] + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + }, + { + "label": "trickle-6-new", + "started": "2026-09-09T02:17:04.184752+00:00", + "phase": "measured", + "rep": 6, + "pool": "new", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10118, + "metrics": { + "completedCount": 20, + "latency": { + "max": 359, + "p50": 272, + "p95": 347, + "p99": 359 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10360, + "waves": [ + { + "count": 1, + "p50": 165, + "p99": 165, + "wave": 0 + }, + { + "count": 1, + "p50": 335, + "p99": 335, + "wave": 1 + }, + { + "count": 1, + "p50": 308, + "p99": 308, + "wave": 2 + }, + { + "count": 1, + "p50": 279, + "p99": 279, + "wave": 3 + }, + { + "count": 1, + "p50": 275, + "p99": 275, + "wave": 4 + }, + { + "count": 1, + "p50": 249, + "p99": 249, + "wave": 5 + }, + { + "count": 1, + "p50": 225, + "p99": 225, + "wave": 6 + }, + { + "count": 1, + "p50": 199, + "p99": 199, + "wave": 7 + }, + { + "count": 1, + "p50": 169, + "p99": 169, + "wave": 8 + }, + { + "count": 1, + "p50": 359, + "p99": 359, + "wave": 9 + }, + { + "count": 1, + "p50": 347, + "p99": 347, + "wave": 10 + }, + { + "count": 1, + "p50": 339, + "p99": 339, + "wave": 11 + }, + { + "count": 1, + "p50": 311, + "p99": 311, + "wave": 12 + }, + { + "count": 1, + "p50": 300, + "p99": 300, + "wave": 13 + }, + { + "count": 1, + "p50": 272, + "p99": 272, + "wave": 14 + }, + { + "count": 1, + "p50": 264, + "p99": 264, + "wave": 15 + }, + { + "count": 1, + "p50": 242, + "p99": 242, + "wave": 16 + }, + { + "count": 1, + "p50": 280, + "p99": 280, + "wave": 17 + }, + { + "count": 1, + "p50": 260, + "p99": 260, + "wave": 18 + }, + { + "count": 1, + "p50": 248, + "p99": 248, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 36, + "wave": 0 + }, + { + "enqueueMs": 34, + "wave": 1 + }, + { + "enqueueMs": 30, + "wave": 2 + }, + { + "enqueueMs": 31, + "wave": 3 + }, + { + "enqueueMs": 28, + "wave": 4 + }, + { + "enqueueMs": 31, + "wave": 5 + }, + { + "enqueueMs": 29, + "wave": 6 + }, + { + "enqueueMs": 27, + "wave": 7 + }, + { + "enqueueMs": 26, + "wave": 8 + }, + { + "enqueueMs": 35, + "wave": 9 + }, + { + "enqueueMs": 27, + "wave": 10 + }, + { + "enqueueMs": 33, + "wave": 11 + }, + { + "enqueueMs": 24, + "wave": 12 + }, + { + "enqueueMs": 26, + "wave": 13 + }, + { + "enqueueMs": 31, + "wave": 14 + }, + { + "enqueueMs": 28, + "wave": 15 + }, + { + "enqueueMs": 28, + "wave": 16 + }, + { + "enqueueMs": 27, + "wave": 17 + }, + { + "enqueueMs": 30, + "wave": 18 + }, + { + "enqueueMs": 28, + "wave": 19 + } + ] + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + }, + { + "label": "trickle-6-old", + "started": "2026-09-09T02:17:22.494517+00:00", + "phase": "measured", + "rep": 6, + "pool": "old", + "scenario": "burstyBatches", + "args": { + "waveCount": 20, + "tasksPerWave": 1, + "delayBetweenWavesMs": 500, + "taskType": "action", + "maxParallelism": 200, + "taskDurationMs": 50 + }, + "result": { + "enqueueTotal": 10066, + "metrics": { + "completedCount": 20, + "latency": { + "max": 348, + "p50": 253, + "p95": 346, + "p99": 348 + }, + "parameters": { + "delayBetweenWavesMs": 500, + "maxParallelism": 200, + "taskCount": 20, + "taskDurationMs": 50, + "taskType": "action", + "tasksPerWave": 1, + "waveCount": 20 + }, + "scenario": "burstyBatches", + "status": "completed", + "taskCount": 20, + "totalDurationMs": 10402, + "waves": [ + { + "count": 1, + "p50": 174, + "p99": 174, + "wave": 0 + }, + { + "count": 1, + "p50": 346, + "p99": 346, + "wave": 1 + }, + { + "count": 1, + "p50": 305, + "p99": 305, + "wave": 2 + }, + { + "count": 1, + "p50": 285, + "p99": 285, + "wave": 3 + }, + { + "count": 1, + "p50": 273, + "p99": 273, + "wave": 4 + }, + { + "count": 1, + "p50": 252, + "p99": 252, + "wave": 5 + }, + { + "count": 1, + "p50": 227, + "p99": 227, + "wave": 6 + }, + { + "count": 1, + "p50": 211, + "p99": 211, + "wave": 7 + }, + { + "count": 1, + "p50": 187, + "p99": 187, + "wave": 8 + }, + { + "count": 1, + "p50": 164, + "p99": 164, + "wave": 9 + }, + { + "count": 1, + "p50": 348, + "p99": 348, + "wave": 10 + }, + { + "count": 1, + "p50": 326, + "p99": 326, + "wave": 11 + }, + { + "count": 1, + "p50": 317, + "p99": 317, + "wave": 12 + }, + { + "count": 1, + "p50": 301, + "p99": 301, + "wave": 13 + }, + { + "count": 1, + "p50": 278, + "p99": 278, + "wave": 14 + }, + { + "count": 1, + "p50": 253, + "p99": 253, + "wave": 15 + }, + { + "count": 1, + "p50": 227, + "p99": 227, + "wave": 16 + }, + { + "count": 1, + "p50": 198, + "p99": 198, + "wave": 17 + }, + { + "count": 1, + "p50": 178, + "p99": 178, + "wave": 18 + }, + { + "count": 1, + "p50": 346, + "p99": 346, + "wave": 19 + } + ] + }, + "timedOut": false, + "waveTimings": [ + { + "enqueueMs": 37, + "wave": 0 + }, + { + "enqueueMs": 34, + "wave": 1 + }, + { + "enqueueMs": 22, + "wave": 2 + }, + { + "enqueueMs": 30, + "wave": 3 + }, + { + "enqueueMs": 29, + "wave": 4 + }, + { + "enqueueMs": 24, + "wave": 5 + }, + { + "enqueueMs": 24, + "wave": 6 + }, + { + "enqueueMs": 23, + "wave": 7 + }, + { + "enqueueMs": 24, + "wave": 8 + }, + { + "enqueueMs": 25, + "wave": 9 + }, + { + "enqueueMs": 29, + "wave": 10 + }, + { + "enqueueMs": 24, + "wave": 11 + }, + { + "enqueueMs": 28, + "wave": 12 + }, + { + "enqueueMs": 26, + "wave": 13 + }, + { + "enqueueMs": 23, + "wave": 14 + }, + { + "enqueueMs": 29, + "wave": 15 + }, + { + "enqueueMs": 24, + "wave": 16 + }, + { + "enqueueMs": 25, + "wave": 17 + }, + { + "enqueueMs": 35, + "wave": 18 + }, + { + "enqueueMs": 29, + "wave": 19 + } + ] + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + } + ], + "checks": [ + { + "label": "mixed-old", + "args": { + "cell": "mixed-old", + "pool": "old", + "groups": [ + { + "delayMs": 3000, + "count": 600 + }, + { + "delayMs": 0, + "count": 2000 + } + ], + "chunkSize": 25, + "maxParallelism": 200, + "settleMs": 120000 + }, + "result": { + "cell": "mixed-old", + "chunkMs": { + "max": 1621, + "median": 224, + "min": 185, + "n": 104 + }, + "enqueueMs": 27221, + "missing": 0, + "pool": "old", + "total": 2600, + "startLatenessByDelayMs": { + "0": { + "total": 2000, + "started": 2000, + "missing": 0, + "early": 0, + "min": 502, + "p50": 909, + "p95": 2144, + "max": 2558 + }, + "3000": { + "total": 600, + "started": 600, + "missing": 0, + "early": 0, + "min": 68, + "p50": 263, + "p95": 2192, + "max": 2415 + } + } + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "mixed-new", + "args": { + "cell": "mixed-new", + "pool": "new", + "groups": [ + { + "delayMs": 3000, + "count": 600 + }, + { + "delayMs": 0, + "count": 2000 + } + ], + "chunkSize": 25, + "maxParallelism": 200, + "settleMs": 120000 + }, + "result": { + "cell": "mixed-new", + "chunkMs": { + "max": 1176, + "median": 226, + "min": 188, + "n": 104 + }, + "enqueueMs": 25405, + "missing": 0, + "pool": "new", + "total": 2600, + "startLatenessByDelayMs": { + "0": { + "total": 2000, + "started": 2000, + "missing": 0, + "early": 0, + "min": 346, + "p50": 618, + "p95": 1077, + "max": 2624 + }, + "3000": { + "total": 600, + "started": 600, + "missing": 0, + "early": 0, + "min": 126, + "p50": 317, + "p95": 1901, + "max": 2079 + } + } + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "pairs-old", + "args": { + "cell": "pairs-old", + "pool": "old", + "count": 12, + "soonDelays": [200], + "nestedCalls": 800, + "maxParallelism": 200, + "settleMs": 60000 + }, + "result": { + "cell": "pairs-old", + "expected": 24, + "missing": 0, + "pool": "old", + "rows": [ + { + "committedAt": 1788919388906, + "delayMs": 0, + "enqueuedAt": 1788919386815, + "pairId": 0, + "runAt": 1788919386815, + "startedAt": 1788919388979 + }, + { + "committedAt": 1788919388906, + "delayMs": 200, + "enqueuedAt": 1788919386815, + "pairId": 0, + "runAt": 1788919387015, + "startedAt": 1788919388980 + }, + { + "committedAt": 1788919390864, + "delayMs": 0, + "enqueuedAt": 1788919388931, + "pairId": 1, + "runAt": 1788919388931, + "startedAt": 1788919390919 + }, + { + "committedAt": 1788919390864, + "delayMs": 200, + "enqueuedAt": 1788919388931, + "pairId": 1, + "runAt": 1788919389131, + "startedAt": 1788919390919 + }, + { + "committedAt": 1788919392664, + "delayMs": 0, + "enqueuedAt": 1788919390888, + "pairId": 2, + "runAt": 1788919390888, + "startedAt": 1788919392739 + }, + { + "committedAt": 1788919392664, + "delayMs": 200, + "enqueuedAt": 1788919390888, + "pairId": 2, + "runAt": 1788919391088, + "startedAt": 1788919392739 + }, + { + "committedAt": 1788919394462, + "delayMs": 0, + "enqueuedAt": 1788919392684, + "pairId": 3, + "runAt": 1788919392684, + "startedAt": 1788919394531 + }, + { + "committedAt": 1788919394462, + "delayMs": 200, + "enqueuedAt": 1788919392684, + "pairId": 3, + "runAt": 1788919392884, + "startedAt": 1788919394531 + }, + { + "committedAt": 1788919396365, + "delayMs": 0, + "enqueuedAt": 1788919394678, + "pairId": 4, + "runAt": 1788919394678, + "startedAt": 1788919396422 + }, + { + "committedAt": 1788919396365, + "delayMs": 200, + "enqueuedAt": 1788919394678, + "pairId": 4, + "runAt": 1788919394878, + "startedAt": 1788919396422 + }, + { + "committedAt": 1788919397916, + "delayMs": 0, + "enqueuedAt": 1788919396384, + "pairId": 5, + "runAt": 1788919396384, + "startedAt": 1788919398179 + }, + { + "committedAt": 1788919397916, + "delayMs": 200, + "enqueuedAt": 1788919396384, + "pairId": 5, + "runAt": 1788919396584, + "startedAt": 1788919398179 + }, + { + "committedAt": 1788919399467, + "delayMs": 0, + "enqueuedAt": 1788919397936, + "pairId": 6, + "runAt": 1788919397936, + "startedAt": 1788919399626 + }, + { + "committedAt": 1788919399467, + "delayMs": 200, + "enqueuedAt": 1788919397936, + "pairId": 6, + "runAt": 1788919398136, + "startedAt": 1788919399629 + }, + { + "committedAt": 1788919401085, + "delayMs": 0, + "enqueuedAt": 1788919399488, + "pairId": 7, + "runAt": 1788919399488, + "startedAt": 1788919401182 + }, + { + "committedAt": 1788919401085, + "delayMs": 200, + "enqueuedAt": 1788919399488, + "pairId": 7, + "runAt": 1788919399688, + "startedAt": 1788919401181 + }, + { + "committedAt": 1788919402702, + "delayMs": 0, + "enqueuedAt": 1788919401103, + "pairId": 8, + "runAt": 1788919401103, + "startedAt": 1788919402868 + }, + { + "committedAt": 1788919402702, + "delayMs": 200, + "enqueuedAt": 1788919401103, + "pairId": 8, + "runAt": 1788919401303, + "startedAt": 1788919402868 + }, + { + "committedAt": 1788919404192, + "delayMs": 0, + "enqueuedAt": 1788919402725, + "pairId": 9, + "runAt": 1788919402725, + "startedAt": 1788919404431 + }, + { + "committedAt": 1788919404192, + "delayMs": 200, + "enqueuedAt": 1788919402725, + "pairId": 9, + "runAt": 1788919402925, + "startedAt": 1788919404431 + }, + { + "committedAt": 1788919405891, + "delayMs": 0, + "enqueuedAt": 1788919404212, + "pairId": 10, + "runAt": 1788919404212, + "startedAt": 1788919405991 + }, + { + "committedAt": 1788919405891, + "delayMs": 200, + "enqueuedAt": 1788919404212, + "pairId": 10, + "runAt": 1788919404412, + "startedAt": 1788919405990 + }, + { + "committedAt": 1788919407491, + "delayMs": 0, + "enqueuedAt": 1788919405914, + "pairId": 11, + "runAt": 1788919405914, + "startedAt": 1788919407703 + }, + { + "committedAt": 1788919407491, + "delayMs": 200, + "enqueuedAt": 1788919405914, + "pairId": 11, + "runAt": 1788919406114, + "startedAt": 1788919407703 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "backlog-old", + "args": { + "cell": "backlog-old", + "pool": "old", + "bulkCount": 2000, + "bulkDelayMs": 60000, + "soonDelayMs": 100, + "nestedCalls": 800, + "maxParallelism": 200, + "settleMs": 90000 + }, + "result": { + "commitMs": 1622, + "immLatenessMs": 2963, + "soonLatenessMs": 2661, + "soonMinusImmMs": -202, + "soonStarted": true + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "cursor-bg-old", + "args": { + "cell": "cursor-bg-old", + "pool": "old", + "groups": [ + { + "delayMs": 0, + "count": 1000 + } + ], + "chunkSize": 1, + "interChunkMs": 100, + "maxParallelism": 200, + "settleMs": 60000 + }, + "result": { + "cell": "cursor-bg-old", + "chunkMs": { + "max": 1167, + "median": 29, + "min": 21, + "n": 1000 + }, + "enqueueMs": 142888, + "missing": 0, + "pool": "old", + "total": 1000, + "startLatenessByDelayMs": { + "0": { + "total": 1000, + "started": 1000, + "missing": 0, + "early": 0, + "min": 71, + "p50": 177, + "p95": 10112, + "max": 16882 + } + } + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "backlog-drained-old", + "args": { + "cell": "backlog-old", + "pool": "old", + "bulkCount": 2000, + "bulkDelayMs": 60000, + "soonDelayMs": 100, + "nestedCalls": 800, + "maxParallelism": 200, + "settleMs": 90000 + }, + "result": { + "total": 2002, + "startLatenessByDelayMs": { + "0": { + "total": 1, + "started": 1, + "missing": 0, + "early": 0, + "min": 2963, + "p50": 2963, + "p95": 2963, + "max": 2963 + }, + "100": { + "total": 1, + "started": 1, + "missing": 0, + "early": 0, + "min": 2661, + "p50": 2661, + "p95": 2661, + "max": 2661 + }, + "60000": { + "total": 2000, + "started": 2000, + "missing": 0, + "early": 0, + "min": 258, + "p50": 6809, + "p95": 16379, + "max": 16859 + } + } + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "pairs-new", + "args": { + "cell": "pairs-new", + "pool": "new", + "count": 12, + "soonDelays": [200], + "nestedCalls": 800, + "maxParallelism": 200, + "settleMs": 60000 + }, + "result": { + "cell": "pairs-new", + "expected": 24, + "missing": 0, + "pool": "new", + "rows": [ + { + "committedAt": 1788919542862, + "delayMs": 0, + "enqueuedAt": 1788919541122, + "pairId": 0, + "runAt": 1788919541122, + "startedAt": 1788919542949 + }, + { + "committedAt": 1788919542862, + "delayMs": 200, + "enqueuedAt": 1788919541122, + "pairId": 0, + "runAt": 1788919541322, + "startedAt": 1788919542950 + }, + { + "committedAt": 1788919544898, + "delayMs": 0, + "enqueuedAt": 1788919542880, + "pairId": 1, + "runAt": 1788919542880, + "startedAt": 1788919545003 + }, + { + "committedAt": 1788919544898, + "delayMs": 200, + "enqueuedAt": 1788919542880, + "pairId": 1, + "runAt": 1788919543080, + "startedAt": 1788919545004 + }, + { + "committedAt": 1788919546465, + "delayMs": 0, + "enqueuedAt": 1788919544918, + "pairId": 2, + "runAt": 1788919544918, + "startedAt": 1788919546589 + }, + { + "committedAt": 1788919546465, + "delayMs": 200, + "enqueuedAt": 1788919544918, + "pairId": 2, + "runAt": 1788919545118, + "startedAt": 1788919546589 + }, + { + "committedAt": 1788919548079, + "delayMs": 0, + "enqueuedAt": 1788919546482, + "pairId": 3, + "runAt": 1788919546482, + "startedAt": 1788919548313 + }, + { + "committedAt": 1788919548079, + "delayMs": 200, + "enqueuedAt": 1788919546482, + "pairId": 3, + "runAt": 1788919546682, + "startedAt": 1788919548314 + }, + { + "committedAt": 1788919549746, + "delayMs": 0, + "enqueuedAt": 1788919548095, + "pairId": 4, + "runAt": 1788919548095, + "startedAt": 1788919549866 + }, + { + "committedAt": 1788919549746, + "delayMs": 200, + "enqueuedAt": 1788919548095, + "pairId": 4, + "runAt": 1788919548295, + "startedAt": 1788919549866 + }, + { + "committedAt": 1788919551274, + "delayMs": 0, + "enqueuedAt": 1788919549761, + "pairId": 5, + "runAt": 1788919549761, + "startedAt": 1788919551472 + }, + { + "committedAt": 1788919551274, + "delayMs": 200, + "enqueuedAt": 1788919549761, + "pairId": 5, + "runAt": 1788919549961, + "startedAt": 1788919551472 + }, + { + "committedAt": 1788919552869, + "delayMs": 0, + "enqueuedAt": 1788919551294, + "pairId": 6, + "runAt": 1788919551294, + "startedAt": 1788919553028 + }, + { + "committedAt": 1788919552869, + "delayMs": 200, + "enqueuedAt": 1788919551294, + "pairId": 6, + "runAt": 1788919551494, + "startedAt": 1788919553027 + }, + { + "committedAt": 1788919554466, + "delayMs": 0, + "enqueuedAt": 1788919552885, + "pairId": 7, + "runAt": 1788919552885, + "startedAt": 1788919554597 + }, + { + "committedAt": 1788919554466, + "delayMs": 200, + "enqueuedAt": 1788919552885, + "pairId": 7, + "runAt": 1788919553085, + "startedAt": 1788919554598 + }, + { + "committedAt": 1788919555992, + "delayMs": 0, + "enqueuedAt": 1788919554483, + "pairId": 8, + "runAt": 1788919554483, + "startedAt": 1788919556128 + }, + { + "committedAt": 1788919555992, + "delayMs": 200, + "enqueuedAt": 1788919554483, + "pairId": 8, + "runAt": 1788919554683, + "startedAt": 1788919556126 + }, + { + "committedAt": 1788919557549, + "delayMs": 0, + "enqueuedAt": 1788919556031, + "pairId": 9, + "runAt": 1788919556031, + "startedAt": 1788919557667 + }, + { + "committedAt": 1788919557549, + "delayMs": 200, + "enqueuedAt": 1788919556031, + "pairId": 9, + "runAt": 1788919556231, + "startedAt": 1788919557669 + }, + { + "committedAt": 1788919559445, + "delayMs": 0, + "enqueuedAt": 1788919557577, + "pairId": 10, + "runAt": 1788919557577, + "startedAt": 1788919559644 + }, + { + "committedAt": 1788919559445, + "delayMs": 200, + "enqueuedAt": 1788919557577, + "pairId": 10, + "runAt": 1788919557777, + "startedAt": 1788919559649 + }, + { + "committedAt": 1788919561108, + "delayMs": 0, + "enqueuedAt": 1788919559468, + "pairId": 11, + "runAt": 1788919559468, + "startedAt": 1788919561171 + }, + { + "committedAt": 1788919561108, + "delayMs": 200, + "enqueuedAt": 1788919559468, + "pairId": 11, + "runAt": 1788919559668, + "startedAt": 1788919561171 + } + ] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "backlog-new", + "args": { + "cell": "backlog-new", + "pool": "new", + "bulkCount": 2000, + "bulkDelayMs": 60000, + "soonDelayMs": 100, + "nestedCalls": 800, + "maxParallelism": 200, + "settleMs": 90000 + }, + "result": { + "commitMs": 1577, + "immLatenessMs": 1722, + "soonLatenessMs": 1622, + "soonMinusImmMs": 0, + "soonStarted": true + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "cursor-bg-new", + "args": { + "cell": "cursor-bg-new", + "pool": "new", + "groups": [ + { + "delayMs": 0, + "count": 1000 + } + ], + "chunkSize": 1, + "interChunkMs": 100, + "maxParallelism": 200, + "settleMs": 60000 + }, + "result": { + "cell": "cursor-bg-new", + "chunkMs": { + "max": 1128, + "median": 31, + "min": 22, + "n": 1000 + }, + "enqueueMs": 136982, + "missing": 0, + "pool": "new", + "total": 1000, + "startLatenessByDelayMs": { + "0": { + "total": 1000, + "started": 1000, + "missing": 0, + "early": 0, + "min": 78, + "p50": 217, + "p95": 4344, + "max": 13644 + } + } + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "backlog-drained-new", + "args": { + "cell": "backlog-new", + "pool": "new", + "bulkCount": 2000, + "bulkDelayMs": 60000, + "soonDelayMs": 100, + "nestedCalls": 800, + "maxParallelism": 200, + "settleMs": 90000 + }, + "result": { + "total": 2002, + "startLatenessByDelayMs": { + "0": { + "total": 1, + "started": 1, + "missing": 0, + "early": 0, + "min": 1722, + "p50": 1722, + "p95": 1722, + "max": 1722 + }, + "100": { + "total": 1, + "started": 1, + "missing": 0, + "early": 0, + "min": 1622, + "p50": 1622, + "p95": 1622, + "max": 1622 + }, + "60000": { + "total": 2000, + "started": 2000, + "missing": 0, + "early": 0, + "min": 266, + "p50": 6574, + "p95": 12904, + "max": 13685 + } + } + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "slowNeighbor-old", + "args": { + "preset": "slowNeighbor", + "pool": "old" + }, + "result": { + "completedCount": 32, + "labels": [ + { + "count": 28, + "label": "fast", + "latencyMs": { + "max": 132, + "min": 108, + "p50": 122, + "p95": 131 + }, + "results": { + "success": 28 + } + }, + { + "count": 4, + "label": "slow", + "latencyMs": { + "max": 20171, + "min": 20058, + "p50": 20107, + "p95": 20171 + }, + "results": { + "success": 4 + } + } + ], + "pool": "old", + "scenario": "noisyNeighbor:slowNeighbor", + "status": "completed", + "taskCount": 32 + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "slowNeighbor-new", + "args": { + "preset": "slowNeighbor", + "pool": "new" + }, + "result": { + "completedCount": 32, + "labels": [ + { + "count": 28, + "label": "fast", + "latencyMs": { + "max": 215, + "min": 203, + "p50": 214, + "p95": 215 + }, + "results": { + "success": 28 + } + }, + { + "count": 4, + "label": "slow", + "latencyMs": { + "max": 20247, + "min": 20158, + "p50": 20213, + "p95": 20247 + }, + "results": { + "success": 4 + } + } + ], + "pool": "new", + "scenario": "noisyNeighbor:slowNeighbor", + "status": "completed", + "taskCount": 32 + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "failRetry-old", + "args": { + "preset": "failRetry", + "pool": "old" + }, + "result": { + "completedCount": 32, + "labels": [ + { + "count": 8, + "label": "fail", + "latencyMs": { + "max": 1894, + "min": 1103, + "p50": 1365, + "p95": 1894 + }, + "results": { + "failed": 8 + } + }, + { + "count": 24, + "label": "fast", + "latencyMs": { + "max": 118, + "min": 100, + "p50": 110, + "p95": 118 + }, + "results": { + "success": 24 + } + } + ], + "pool": "old", + "scenario": "noisyNeighbor:failRetry", + "status": "completed", + "taskCount": 32 + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "failRetry-new", + "args": { + "preset": "failRetry", + "pool": "new" + }, + "result": { + "completedCount": 32, + "labels": [ + { + "count": 8, + "label": "fail", + "latencyMs": { + "max": 1990, + "min": 1928, + "p50": 1989, + "p95": 1990 + }, + "results": { + "failed": 8 + } + }, + { + "count": 24, + "label": "fast", + "latencyMs": { + "max": 266, + "min": 259, + "p50": 265, + "p95": 266 + }, + "results": { + "success": 24 + } + } + ], + "pool": "new", + "scenario": "noisyNeighbor:failRetry", + "status": "completed", + "taskCount": 32 + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "scheduling-8000", + "args": { + "delayMs": 8000 + }, + "result": { + "delayedLateByMs": 120, + "delayedRan": true, + "delayedRanEarly": false, + "retryAttempts": 3, + "actualStoredDueMs": 1788919785042, + "actualStartLatenessMs": 118, + "hasScanTs": true, + "retryGapsMs": [928, 1768] + }, + "runtimeRevision": "8186121", + "revisionVerifiedForComparison": true + }, + { + "label": "scheduling-310000", + "args": { + "delayMs": 310000 + }, + "result": { + "delayedRan": true, + "delayedRanEarly": false, + "delayedLateByMs": null, + "retryAttempts": 3, + "actualStoredDueMs": 1788920103659, + "actualStartLatenessMs": 365, + "hasScanTs": false, + "retryGapsMs": [953, 1985], + "cliResultUnavailable": true, + "recoveredFromPersistedProbes": true, + "serverActionCompletedWithoutError": true + }, + "runtimeRevision": null, + "revisionVerifiedForComparison": false + } + ], + "validation": { + "expectedRuntimeErrors": { + "test/noisy:noisyAction": 48, + "test/scheduling:failTwice": 4 + }, + "unexpectedRuntimeErrors": 0, + "consoleErrorEntries": 52 + }, + "executionSummary": [ + { + "label": "mutation-measured-1-old", + "component": "oldWorkpool/batchWorker", + "function": "loop:loop", + "n": 84, + "readDocs": 30917, + "readBytes": 11317898, + "ioReadBytes": 11317898, + "writeDocs": 10167, + "errors": 0, + "occRetries": 0, + "meanMs": 492.5805772142857, + "p95Ms": 785.7500990000001 + }, + { + "label": "mutation-measured-1-old", + "component": "oldWorkpool", + "function": "worker:runMutationWrapper", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 52.0949992662, + "p95Ms": 128.18899 + }, + { + "label": "mutation-measured-1-old", + "component": "oldWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10030000, + "ioReadBytes": 10030000, + "writeDocs": 10000, + "errors": 0, + "occRetries": 0, + "meanMs": 63.312241217, + "p95Ms": 87.446283 + }, + { + "label": "mutation-measured-1-old", + "component": null, + "function": "test/run:metrics", + "n": 66, + "readDocs": 186503, + "readBytes": 50915257, + "ioReadBytes": 50915257, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 361.46932016666665, + "p95Ms": 1030.6275309999999 + }, + { + "label": "mutation-measured-1-new", + "component": "testWorkpool/batchWorker", + "function": "loop:loop", + "n": 86, + "readDocs": 16224, + "readBytes": 9068848, + "ioReadBytes": 9068848, + "writeDocs": 5292, + "errors": 0, + "occRetries": 0, + "meanMs": 311.3259808255814, + "p95Ms": 460.184284 + }, + { + "label": "mutation-measured-1-new", + "component": "testWorkpool", + "function": "worker:runMutationWrapper", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 13.5240312794, + "p95Ms": 19.687984 + }, + { + "label": "mutation-measured-1-new", + "component": "testWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10765000, + "ioReadBytes": 10765000, + "writeDocs": 10000, + "errors": 0, + "occRetries": 0, + "meanMs": 38.6628605854, + "p95Ms": 52.706527 + }, + { + "label": "mutation-measured-1-new", + "component": null, + "function": "test/run:metrics", + "n": 51, + "readDocs": 151432, + "readBytes": 41340885, + "ioReadBytes": 41340885, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 299.81155007843137, + "p95Ms": 496.513541 + }, + { + "label": "mutation-measured-2-new", + "component": "testWorkpool/batchWorker", + "function": "loop:loop", + "n": 84, + "readDocs": 16251, + "readBytes": 9933308, + "ioReadBytes": 9933308, + "writeDocs": 5294, + "errors": 0, + "occRetries": 0, + "meanMs": 354.94637652380953, + "p95Ms": 754.849021 + }, + { + "label": "mutation-measured-2-new", + "component": "testWorkpool", + "function": "worker:runMutationWrapper", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 35.435624986, + "p95Ms": 62.83594600000001 + }, + { + "label": "mutation-measured-2-new", + "component": "testWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10765000, + "ioReadBytes": 10765000, + "writeDocs": 10000, + "errors": 0, + "occRetries": 0, + "meanMs": 58.0929243804, + "p95Ms": 131.408557 + }, + { + "label": "mutation-measured-2-new", + "component": null, + "function": "test/run:metrics", + "n": 50, + "readDocs": 139785, + "readBytes": 38161256, + "ioReadBytes": 38161256, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 330.6231965, + "p95Ms": 646.996096 + }, + { + "label": "mutation-measured-2-old", + "component": "oldWorkpool/batchWorker", + "function": "loop:loop", + "n": 80, + "readDocs": 30823, + "readBytes": 10902163, + "ioReadBytes": 10902163, + "writeDocs": 10152, + "errors": 0, + "occRetries": 0, + "meanMs": 434.2514817875, + "p95Ms": 580.109451 + }, + { + "label": "mutation-measured-2-old", + "component": "oldWorkpool", + "function": "worker:runMutationWrapper", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 15.393052099, + "p95Ms": 23.827752 + }, + { + "label": "mutation-measured-2-old", + "component": "oldWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10030000, + "ioReadBytes": 10030000, + "writeDocs": 10000, + "errors": 0, + "occRetries": 0, + "meanMs": 39.9427033388, + "p95Ms": 58.093384 + }, + { + "label": "mutation-measured-2-old", + "component": null, + "function": "test/run:metrics", + "n": 61, + "readDocs": 182368, + "readBytes": 49786403, + "ioReadBytes": 49786403, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 293.0244032622951, + "p95Ms": 473.055543 + }, + { + "label": "mutation-measured-3-old", + "component": "oldWorkpool/batchWorker", + "function": "loop:loop", + "n": 80, + "readDocs": 30988, + "readBytes": 10930708, + "ioReadBytes": 10930708, + "writeDocs": 10152, + "errors": 0, + "occRetries": 0, + "meanMs": 470.738386325, + "p95Ms": 582.73357 + }, + { + "label": "mutation-measured-3-old", + "component": "oldWorkpool", + "function": "worker:runMutationWrapper", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 13.8183584096, + "p95Ms": 19.873255999999998 + }, + { + "label": "mutation-measured-3-old", + "component": "oldWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10030000, + "ioReadBytes": 10030000, + "writeDocs": 10000, + "errors": 0, + "occRetries": 0, + "meanMs": 40.0013815148, + "p95Ms": 55.161521 + }, + { + "label": "mutation-measured-3-old", + "component": null, + "function": "test/run:metrics", + "n": 68, + "readDocs": 194946, + "readBytes": 53220192, + "ioReadBytes": 53220192, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 276.3560780588235, + "p95Ms": 473.561405 + }, + { + "label": "mutation-measured-3-new", + "component": "testWorkpool/batchWorker", + "function": "loop:loop", + "n": 85, + "readDocs": 16217, + "readBytes": 9067225, + "ioReadBytes": 9067225, + "writeDocs": 5291, + "errors": 0, + "occRetries": 0, + "meanMs": 315.1600237764706, + "p95Ms": 399.92236599999995 + }, + { + "label": "mutation-measured-3-new", + "component": "testWorkpool", + "function": "worker:runMutationWrapper", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 13.6587506456, + "p95Ms": 18.787502 + }, + { + "label": "mutation-measured-3-new", + "component": "testWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10765000, + "ioReadBytes": 10765000, + "writeDocs": 10000, + "errors": 0, + "occRetries": 0, + "meanMs": 38.7771624772, + "p95Ms": 48.291743999999994 + }, + { + "label": "mutation-measured-3-new", + "component": null, + "function": "test/run:metrics", + "n": 49, + "readDocs": 144574, + "readBytes": 39468653, + "ioReadBytes": 39468653, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 298.6429910612245, + "p95Ms": 471.961611 + }, + { + "label": "action-measured-1-old", + "component": "oldWorkpool/batchWorker", + "function": "loop:loop", + "n": 80, + "readDocs": 30823, + "readBytes": 10847163, + "ioReadBytes": 10847163, + "writeDocs": 10152, + "errors": 0, + "occRetries": 0, + "meanMs": 425.7483263375, + "p95Ms": 572.547425 + }, + { + "label": "action-measured-1-old", + "component": null, + "function": "test/work:configurableAction", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 27.0957376824, + "p95Ms": 29.75369 + }, + { + "label": "action-measured-1-old", + "component": "oldWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 9865000, + "ioReadBytes": 9865000, + "writeDocs": 15000, + "errors": 0, + "occRetries": 0, + "meanMs": 41.5032498868, + "p95Ms": 52.425485 + }, + { + "label": "action-measured-1-old", + "component": "oldWorkpool", + "function": "worker:runBatch", + "n": 226, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 116.49344889380531, + "p95Ms": 137.44403799999998 + }, + { + "label": "action-measured-1-old", + "component": null, + "function": "test/run:metrics", + "n": 55, + "readDocs": 168000, + "readBytes": 45527945, + "ioReadBytes": 45527945, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 271.07999245454545, + "p95Ms": 454.133053 + }, + { + "label": "action-measured-1-new", + "component": "testWorkpool/batchWorker", + "function": "loop:loop", + "n": 83, + "readDocs": 16203, + "readBytes": 9884155, + "ioReadBytes": 9884155, + "writeDocs": 5289, + "errors": 0, + "occRetries": 0, + "meanMs": 284.0297452891566, + "p95Ms": 344.87104800000003 + }, + { + "label": "action-measured-1-new", + "component": null, + "function": "test/work:configurableAction", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 27.3641645278, + "p95Ms": 30.679698 + }, + { + "label": "action-measured-1-new", + "component": "testWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10600000, + "ioReadBytes": 10600000, + "writeDocs": 15000, + "errors": 0, + "occRetries": 0, + "meanMs": 42.8861220626, + "p95Ms": 54.650470999999996 + }, + { + "label": "action-measured-1-new", + "component": "testWorkpool", + "function": "worker:runBatch", + "n": 158, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 124.88785169620253, + "p95Ms": 156.45608099999998 + }, + { + "label": "action-measured-1-new", + "component": null, + "function": "test/run:metrics", + "n": 36, + "readDocs": 111070, + "readBytes": 30099934, + "ioReadBytes": 30099934, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 278.65986683333335, + "p95Ms": 462.189642 + }, + { + "label": "action-measured-2-new", + "component": "testWorkpool/batchWorker", + "function": "loop:loop", + "n": 84, + "readDocs": 16210, + "readBytes": 9860656, + "ioReadBytes": 9860656, + "writeDocs": 5290, + "errors": 0, + "occRetries": 0, + "meanMs": 275.29697486904763, + "p95Ms": 315.632143 + }, + { + "label": "action-measured-2-new", + "component": null, + "function": "test/work:configurableAction", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 27.1931548568, + "p95Ms": 29.737452 + }, + { + "label": "action-measured-2-new", + "component": "testWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10600000, + "ioReadBytes": 10600000, + "writeDocs": 15000, + "errors": 0, + "occRetries": 0, + "meanMs": 41.9372139256, + "p95Ms": 51.582461 + }, + { + "label": "action-measured-2-new", + "component": "testWorkpool", + "function": "worker:runBatch", + "n": 158, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 116.88096496202532, + "p95Ms": 131.409673 + }, + { + "label": "action-measured-2-new", + "component": null, + "function": "test/run:metrics", + "n": 37, + "readDocs": 113697, + "readBytes": 30811850, + "ioReadBytes": 30811850, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 264.3334618918919, + "p95Ms": 417.99442799999997 + }, + { + "label": "action-measured-2-old", + "component": "oldWorkpool/batchWorker", + "function": "loop:loop", + "n": 79, + "readDocs": 30685, + "readBytes": 10797209, + "ioReadBytes": 10797209, + "writeDocs": 10086, + "errors": 0, + "occRetries": 0, + "meanMs": 426.46241750632913, + "p95Ms": 692.601037 + }, + { + "label": "action-measured-2-old", + "component": null, + "function": "test/work:configurableAction", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 27.5963701226, + "p95Ms": 30.628121 + }, + { + "label": "action-measured-2-old", + "component": "oldWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 9865000, + "ioReadBytes": 9865000, + "writeDocs": 15000, + "errors": 0, + "occRetries": 0, + "meanMs": 42.8743175576, + "p95Ms": 57.330701999999995 + }, + { + "label": "action-measured-2-old", + "component": "oldWorkpool", + "function": "worker:runBatch", + "n": 226, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 120.58574453539823, + "p95Ms": 168.77518899999998 + }, + { + "label": "action-measured-2-old", + "component": null, + "function": "test/run:metrics", + "n": 54, + "readDocs": 162977, + "readBytes": 44166714, + "ioReadBytes": 44166714, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 278.5757092037037, + "p95Ms": 585.6387159999999 + }, + { + "label": "action-measured-3-old", + "component": "oldWorkpool/batchWorker", + "function": "loop:loop", + "n": 80, + "readDocs": 30931, + "readBytes": 10819253, + "ioReadBytes": 10819253, + "writeDocs": 10124, + "errors": 0, + "occRetries": 0, + "meanMs": 421.9712155, + "p95Ms": 572.882687 + }, + { + "label": "action-measured-3-old", + "component": null, + "function": "test/work:configurableAction", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 27.8108826152, + "p95Ms": 31.397975000000002 + }, + { + "label": "action-measured-3-old", + "component": "oldWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 9865000, + "ioReadBytes": 9865000, + "writeDocs": 15000, + "errors": 0, + "occRetries": 0, + "meanMs": 44.42629257, + "p95Ms": 58.588311999999995 + }, + { + "label": "action-measured-3-old", + "component": "oldWorkpool", + "function": "worker:runBatch", + "n": 228, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 120.34940114912281, + "p95Ms": 146.879652 + }, + { + "label": "action-measured-3-old", + "component": null, + "function": "test/run:metrics", + "n": 55, + "readDocs": 169097, + "readBytes": 45825232, + "ioReadBytes": 45825232, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 296.0566932909091, + "p95Ms": 444.30523300000004 + }, + { + "label": "action-measured-3-new", + "component": "testWorkpool/batchWorker", + "function": "loop:loop", + "n": 82, + "readDocs": 16196, + "readBytes": 9909244, + "ioReadBytes": 9909244, + "writeDocs": 5288, + "errors": 0, + "occRetries": 0, + "meanMs": 284.16673641463416, + "p95Ms": 401.961709 + }, + { + "label": "action-measured-3-new", + "component": null, + "function": "test/work:configurableAction", + "n": 5000, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 38.1816069136, + "p95Ms": 47.444826 + }, + { + "label": "action-measured-3-new", + "component": "testWorkpool", + "function": "complete:complete", + "n": 5000, + "readDocs": 25000, + "readBytes": 10600000, + "ioReadBytes": 10600000, + "writeDocs": 15000, + "errors": 0, + "occRetries": 0, + "meanMs": 50.6595804904, + "p95Ms": 67.941904 + }, + { + "label": "action-measured-3-new", + "component": "testWorkpool", + "function": "worker:runBatch", + "n": 158, + "readDocs": 0, + "readBytes": 0, + "ioReadBytes": 0, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 172.80440097468355, + "p95Ms": 696.792319 + }, + { + "label": "action-measured-3-new", + "component": null, + "function": "test/run:metrics", + "n": 32, + "readDocs": 97773, + "readBytes": 26496451, + "ioReadBytes": 26496451, + "writeDocs": 0, + "errors": 0, + "occRetries": 0, + "meanMs": 337.736538625, + "p95Ms": 1415.7383559999998 + } + ] +} diff --git a/example/benchmarks/2026-09-08.md b/example/benchmarks/2026-09-08.md new file mode 100644 index 00000000..e1809948 --- /dev/null +++ b/example/benchmarks/2026-09-08.md @@ -0,0 +1,245 @@ +# Design at 8186121 — benchmarks, 2026-09-08 + +Bulk throughput improved consistently against 0.4.9: median **141 vs 113 +mutations/s** and **192 vs 134 actions/s**. All twelve measured throughput runs +completed without timeouts or runtime execution errors. Light-load tail latency +was variable and is assessed separately below. Keep the design for its +bulk-throughput gain; investigate light-load tails before claiming an +across-the-board latency improvement. + +Deployment: `dev:precious-raven-853`. Measured source: `8186121` on +`ian/snapshot-ts`; the subsequent `0d80a6c` changes comments and test setup, +with identical emitted runtime code in all changed implementation files. +Baseline: installed `@convex-dev/workpool-old@0.4.9`. Both use +`@convex-dev/batch-worker@0.2.1`; Convex is pinned at `1.43.0`. A further +commit, `e94b0f3`, changed eligibility during this run. A concurrent local build +updated `dist` at 19:13:46 PDT, after throughput and the earlier diagnostic +checks. Whether that build was deployed could not be verified. The far-delay +completion and light-load follow-up pairs 4–6 are therefore labeled with an +unverified runtime revision and are not combined with the pinned comparison. +Times below are milliseconds unless stated otherwise. + +The component was built and successfully deployed before measurement. The first +push failed because the dev deployment retained `internalState.lastCommitTs`. +After verifying all four current-code mounts had no work or pending entries, I +backed up the state and reset the single obsolete, idle `testWorkpool` state +row. No queued work was removed. The benchmark implementation was unchanged. + +## Throughput and completion latency + +Each arm enqueues 5,000 tasks in batches of 100 with a 50 ms inter-batch gap and +`maxParallelism=200`. Mutations have no simulated work; actions sleep 20 ms. +Each workload/pool gets three discarded warmups followed by three measured pairs +in `old,new / new,old / old,new` order. All bookkeeping is archived before +cleanup; both pools must drain and all cleanup counts must reach zero before the +next arm. + +| Workload | Pool | Runs | Median tasks/s | Tasks/s range | Median total ms | Median p50 ms | Median p95 ms | +| -------- | ---- | ---- | -------------- | ------------- | --------------- | ------------- | ------------- | +| mutation | old | 3 | 112.8 | 103.3–121.6 | 44,325 | 16,448 | 33,480 | +| mutation | new | 3 | 140.9 | 134.2–147.3 | 35,491 | 12,848 | 25,441 | +| action | old | 3 | 133.7 | 132.4–135.6 | 37,408 | 13,228 | 26,525 | +| action | new | 3 | 191.8 | 191.0–196.2 | 26,064 | 9,130 | 17,188 | + +Mutation throughput ratios (new/old) by pair: **1.364×, 1.104×, 1.306×**; median +**1.306×** (+30.6%). + +Action throughput ratios (new/old) by pair: **1.429×, 1.447×, 1.449×**; median +**1.447×** (+44.7%). + +Individual measured runs: + +| Workload | Pair | Pool | Total ms | Tasks/s | p95 ms | Enqueue ms | +| -------- | ---- | ---- | -------- | ------- | ------ | ---------- | +| mutation | 1 | old | 48417 | 103.3 | 37140 | 9056 | +| mutation | 1 | new | 35491 | 140.9 | 25441 | 8651 | +| mutation | 2 | new | 37250 | 134.2 | 27162 | 9065 | +| mutation | 2 | old | 41108 | 121.6 | 30669 | 8968 | +| mutation | 3 | old | 44325 | 112.8 | 33480 | 9292 | +| mutation | 3 | new | 33935 | 147.3 | 24661 | 8167 | +| action | 1 | old | 37408 | 133.7 | 26525 | 9593 | +| action | 1 | new | 26175 | 191.0 | 17188 | 7942 | +| action | 2 | new | 25483 | 196.2 | 17160 | 7374 | +| action | 2 | old | 36862 | 135.6 | 25835 | 9080 | +| action | 3 | old | 37767 | 132.4 | 27858 | 8370 | +| action | 3 | new | 26064 | 191.8 | 17485 | 7751 | + +Completion latency is enqueue → terminal callback, including work and recorder +overhead. These are medians of run-level percentiles, not pooled task +percentiles. Three pairs give descriptive evidence, not a precise confidence +interval. The comparison includes all differences between this design and 0.4.9; +it does not isolate the snapshot change alone. Separate component storage +histories remain a confound. + +## Light-load completion latency + +3 pairs of 20 waves × one 50 ms action, 500 ms between waves, at +parallelism 200. These are paced latency checks, not saturation throughput. +These initial pairs ran before the concurrent build. No separate light-load +warmup was discarded; this is a latency diagnostic, not a stable p95 estimate. + +| Pool | Runs | Median p50 ms | Median p95 ms | Individual p95 ms | +| ---- | ---- | ------------- | ------------- | ----------------- | +| old | 3 | 247.0 | 604.0 | 1132, 604, 350 | +| new | 3 | 288.0 | 1430.0 | 1604, 443, 1430 | + +The new pool’s median run-level p95 was higher, but individual results varied +widely (443–1,604 ms new, 350–1,132 ms old). Some slow runs also stretched the +identical 50 ms action itself. Repeat with more samples and use the same +component for a source comparison before attributing this to the design. + +Follow-up pairs 4–6 were added to investigate the variation. They ran after the +concurrent local build, so the runtime revision is unverified. They are retained +as observations and excluded from the pinned summary above. + +| Pair | Pool | p50 ms | p95 ms | +| ---- | ---- | ------ | ------ | +| 4 | new | 261 | 1158 | +| 4 | old | 244 | 361 | +| 5 | old | 265 | 1160 | +| 5 | new | 221 | 440 | +| 6 | new | 272 | 347 | +| 6 | old | 253 | 346 | + +## Start latency and scheduling checks + +These are paired smoke/diagnostic runs, not repeated performance estimates. +Every cell uses parallelism 200 unless specified otherwise. + +Mixed cell: 600 tasks delayed 3 seconds, then 2,000 immediate tasks; chunk +size 25. Values are start lateness relative to each task’s due time. + +| Pool | Delay ms | Started | p50 ms | p95 ms | Min ms | Max ms | Early starts | +| ---- | -------- | ------- | ------ | ------ | ------ | ------ | ------------ | +| old | 0 | 2000 | 909 | 2144 | 502 | 2558 | 0 | +| old | 3000 | 600 | 263 | 2192 | 68 | 2415 | 0 | +| new | 0 | 2000 | 618 | 1077 | 346 | 2624 | 0 | +| new | 3000 | 600 | 317 | 1901 | 126 | 2079 | 0 | + +Sibling checks: 12 enqueue transactions, a 200 ms delayed sibling, and 800 +nested no-op calls per enqueue. Concurrent traffic supplies 1,000 immediate +probes in single-task chunks at 100 ms gaps. The same traffic continues through +the backlog case. + +| Pool | Pairs | Overdue at RPC return | Median scheduled − immediate ms | p95 difference ms | Min ms | Max ms | Missing | +| ---- | ----- | --------------------- | ------------------------------- | ----------------- | ------ | ------ | ------- | +| old | 12 | 12 | 0 | 3 | -1 | 3 | 0 | +| new | 12 | 12 | 0 | 5 | -2 | 5 | 0 | + +RPC-return time is only an upper bound on commit time. Concurrent traffic +creates opportunities for cursor advancement; no direct sweep instrumentation +was added, so these results do not independently prove which path rescued each +task. + +Backlog checks: 2,000 tasks delayed 60 seconds in one batch, then one 100 ms +delayed/immediate pair with 800 nested calls. After the pair returned, the +driver waited for all bulk/background work and verified every bulk probe started +before cleanup. + +| Pool | Pair enqueue RPC ms | Scheduled lateness ms | Immediate lateness ms | Scheduled − immediate ms | Scheduled started | +| ---- | ------------------- | --------------------- | --------------------- | ------------------------ | ----------------- | +| old | 1622 | 2661 | 2963 | -202 | True | +| new | 1577 | 1622 | 1722 | 0 | True | + +Delayed/retry checks use only the new pool at parallelism 10. The 310-second +case crosses the five-minute scan-index cutoff. The harness records its origin +outside the enqueue transaction. The driver also read the pending queue key +during the wait, verifying scan-index presence/absence and measuring start +lateness against the actual stored due time. + +| Delay ms | Ran | Early | Harness lateness ms | Lateness from stored key ms | Has scanTs | Retry attempts | +| -------- | ---- | ----- | ------------------- | --------------------------- | ---------- | -------------- | +| 8000 | True | False | 120 | 118 | True | 3 | +| 310000 | True | False | — | 365 | False | 3 | + +The 310-second CLI call returned a generic error instead of JSON. The persisted +delayed/retry probes, an empty workpool, and the server action’s completion log +(313.46 seconds, no error) confirmed completion. The exact stored due time had +already been captured, so lateness could be recovered without enqueueing another +run. The CLI failure’s cause was not established. This completion occurred after +the concurrent local build, so the worker’s runtime revision is unverified; it +is not a certified result for `8186121`. + +## Mixed-action checks + +One run per pool/preset at parallelism 40: `slowNeighbor` mixes 28 × 50 ms +actions with four × 20-second actions; `failRetry` mixes 24 successes with eight +failures allowed three attempts. Outcomes were explicitly checked. + +| Preset | Pool | Class | Count | Terminal outcomes | p50 ms | p95 ms | +| ------------ | ---- | ----- | ----- | ----------------- | ------ | ------ | +| slowNeighbor | old | fast | 28 | {"success": 28} | 122 | 131 | +| slowNeighbor | old | slow | 4 | {"success": 4} | 20107 | 20171 | +| slowNeighbor | new | fast | 28 | {"success": 28} | 214 | 215 | +| slowNeighbor | new | slow | 4 | {"success": 4} | 20213 | 20247 | +| failRetry | old | fail | 8 | {"failed": 8} | 1365 | 1894 | +| failRetry | old | fast | 24 | {"success": 24} | 110 | 118 | +| failRetry | new | fail | 8 | {"failed": 8} | 1989 | 1990 | +| failRetry | new | fast | 24 | {"success": 24} | 265 | 266 | + +## Runtime evidence + +Median across measured runs, from each pool’s nested `batchWorker` loop: + +| Workload | Pool | Mean loop execution ms | Loop document reads/run | Loop document writes/run | +| -------- | ---- | ---------------------- | ----------------------- | ------------------------ | +| mutation | old | 470.7 | 30,917 | 10,152 | +| mutation | new | 315.2 | 16,224 | 5,292 | +| action | old | 425.7 | 30,823 | 10,124 | +| action | new | 284.0 | 16,203 | 5,289 | + +The reduction in loop work is consistent with the throughput improvement. This +is supporting evidence, not an isolated causal test of a single source change. +Progress polling itself read roughly 98,000–195,000 task rows per measured +5,000-task run (about 26–53 MB), exceeding the loop’s reads. These measurements +include that harness load. + +## Evidence and limits + +Raw outputs, exact arguments, run metadata, state backup, runtime logs, and the +one-off drivers are in `.context/benchmark-2026-09-08/`. `runs.jsonl` includes +discarded warmups; `checks.jsonl` contains the diagnostic results. These raw +artifacts are local and gitignored. This report and +[compact JSON results](./2026-09-08.json) live under `example/benchmarks/` for +version control; the JSON preserves metadata, all throughput outputs including +warmups, diagnostic summaries, and selected execution statistics. + +Preflight: 148 tests in 14 files passed, with no type errors, for the measured +implementation (including the equivalent `0d80a6c` test/comment refinement). +Final checks found zero work, pending entries, payloads, and running jobs in +both pools; app bookkeeping counts were also zero. Both pools were left at +parallelism 200. Runtime logs contained only the expected 48 noisy-action errors +and four deliberate retry errors. The long-delay CLI error is recorded above +separately from server function results. + +The ordinary throughput recorder does not retain outcome kind. Exact callback +counts and no timeout are necessary checks, but success/error evidence must also +come from runtime logs. Intentional worker errors in the retry scenarios are +expected. + +## What belongs in code, and what belongs in the guide + +Keep `BENCHMARKS.md`, but make its eventual job metric definitions and +experiment interpretation. Most operational instructions are compensating for +missing harness behavior and should become executable checks. I corrected the +guide now; the backend and dashboard improvements below are recommendations, not +changes made during this measurement. + +| Priority | Finding | Recommendation | +| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | The deploy command omitted the build, although component exports point at `dist`. The actual first push also failed on the removed `lastCommitTs` field in persisted experimental state. | A checked-in `npm run bench` should build, require a successful deploy, and record the deployed component fingerprint/versions before each run, and reject runs across a fingerprint change. A concurrent build during this session made the late follow-up revision uncertain. Use an isolated checkout/deployment for a comparison. Provide an explicit compatibility migration or guarded reset for obsolete experimental state; avoid permanent instructions to manually edit data. | +| 1 | `completedCount` counts callbacks, even failed/canceled tasks. Timeouts can return JSON normally, giving CLI exit code zero. The ordinary recorder discards `result.kind`, although the schema already supports it. | Persist terminal outcomes and work IDs, require exactly the expected unique successful tasks, return a run handle early with structured failure/timeout status, and make the runner fail on invalid results. Resume that handle after transport errors instead of repeating enqueues; the 310-second CLI call failed even though the server completed. Expected failures in noisy-neighbor tests should have explicit assertions. | +| 1 | Cleanup deletes all run history and several shared app tables, but does not drain either component or clear `counters`. `backlog` can return with 2,000 delayed tasks still queued; resetting a cell can delete rows while its old workers remain active. | A runner should own a run/cell, wait for all its work and callbacks to settle, archive results, and clean up only that run. Make `run.start` check active runs per pool instead of only the latest global run. Have `backlog` wait for its bulk or return an explicit outstanding-work handle. Expose drain/cleanup status instead of requiring manual polling. | +| 1 | Main instructions depended on gitignored `.context` scripts. The shell driver has one discarded pair rather than three per workload, filters away raw JSON, can mask errors through a pipeline, and continues after cleanup failure. | Check in a runner with explicit warmup/repetition counts, balanced serial order, bounded waits, error propagation, raw JSON output, metadata, and per-pair summaries. No ad hoc regex parsing of human logs. | +| 2 | The mutation example claimed 20 ms of work, but `taskDurationMs` is action-only. Both stored parameters and printed messages preserve the misleading value. | Validate arguments by workload type and report effective parameters. Make mutation work explicit in operations/bytes; do not simulate it with a frozen wall-clock spin loop. | +| 2 | `pairs`/`backlog` examples do not supply their necessary concurrent traffic. `committedAt` is RPC-return time, and the scheduling smoke test measures its due time from outside the enqueuing mutation. The latter only records attempts, not the final retry outcome. | Make cursor experiments self-contained with bounded ready traffic and proof that the intended cursor condition occurred. Record actual `runAt` inside the enqueue transaction, distinguish commit timestamp from RPC-return time, and assert delayed-start and terminal retry outcomes. Keep clock/metric definitions in the guide. | +| 2 | The documented 2,000-task backlog now packs into eight pending-start documents; the sweep budget is 256 documents. Its old `SWEEP_BATCH` comment describes a different implementation. | Add a fixture with enough distinct scheduled keys to exceed the document budget, and assert the actual document count/commit grouping. Keep the existing packed backlog as a smoke check. | +| 2 | Repeated metrics polling collects every task in the run, so measurement overhead grows with completed count. Larger tests also approach query limits. `sustained` ignores rejected enqueue promises from `Promise.allSettled`. | Check every enqueue outcome. Use bounded progress queries or a completion signal while waiting; calculate full quantiles once from archived/paginated rows. Avoid introducing a single contended per-task counter in the measured path. | +| 3 | The constant-editing script is not generally mirrored as described and `git checkout` can overwrite concurrent edits. Three warmups are only a heuristic. | Use isolated checkouts and an explicit experiment matrix. Keep warmup drift, paired comparisons, storage-history confounds, and noise-floor interpretation in documentation. These cannot be removed by changing pool functions. | +| 3 | Claims that non-monotonic results must be noise, that all workers from one enqueue share a start time, and that higher execution time with fewer reads proves round trips were too strong or wrong. | Keep corrected methodology in the guide: transactions share an enqueue clock/commit, worker starts differ, nonlinear optima can be real, and round trips are one hypothesis to investigate. | +| 3 | The example README and four dashboard labels still said 0.4.7; the installed baseline is 0.4.9. Both mounts now have nested `batchWorker` components, so earlier 0.4.7 scheduler-count comparisons are not current evidence. | Report the mounted package version in benchmark metadata and drive UI labels from it. Count nested loop components consistently. Date/version historical reports instead of treating them as current results. | + +The guide was updated to remove workspace scripts from the supported procedure, +correct the build step and metric definitions, inventory current harnesses, and +explain the limits found above. The example README's baseline description was +also corrected. No workpool implementation or benchmark function was changed. From 49e21555c778931bab7f15032bdc2618480f6771 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:48:42 -0700 Subject: [PATCH 3/9] fix(benchmarks): remove unused probes and record run versions --- example/convex/schema.ts | 1 + example/convex/test/dashboard.ts | 2 ++ example/convex/test/latency.ts | 47 +------------------------------- example/convex/test/run.ts | 4 +++ example/convex/test/work.ts | 1 + example/src/App.tsx | 22 +++++++++++---- 6 files changed, 26 insertions(+), 51 deletions(-) diff --git a/example/convex/schema.ts b/example/convex/schema.ts index 11d4d397..092a246a 100644 --- a/example/convex/schema.ts +++ b/example/convex/schema.ts @@ -14,6 +14,7 @@ export default defineSchema({ taskCount: v.optional(v.number()), endTime: v.optional(v.number()), pool: v.optional(v.string()), + poolVersion: v.optional(v.string()), scheduledFunctions: v.optional(v.number()), }), tasks: defineTable({ diff --git a/example/convex/test/dashboard.ts b/example/convex/test/dashboard.ts index e07abb4c..5cac5662 100644 --- a/example/convex/test/dashboard.ts +++ b/example/convex/test/dashboard.ts @@ -20,6 +20,7 @@ export const listRuns = query({ _id: run._id, scenario: run.scenario, pool: run.pool, + poolVersion: run.poolVersion, startTime: run.startTime, taskCount: run.taskCount, scheduledFunctions: run.scheduledFunctions, @@ -53,6 +54,7 @@ export const getRun = query({ scenario: run.scenario, parameters: run.parameters, pool: run.pool, + poolVersion: run.poolVersion, startTime: run.startTime, taskCount: run.taskCount, scheduledFunctions: run.scheduledFunctions, diff --git a/example/convex/test/latency.ts b/example/convex/test/latency.ts index 8d8f644c..989edf53 100644 --- a/example/convex/test/latency.ts +++ b/example/convex/test/latency.ts @@ -371,7 +371,6 @@ export const pairs = internalAction({ await ctx.runMutation(internal.test.latency.resetCell, { cell: args.cell }); for (let pairId = 0; pairId < args.count; pairId++) { - const t0 = Date.now(); const ids = await ctx.runMutation(internal.test.latency.enqueuePair, { cell: args.cell, pool: args.pool, @@ -385,7 +384,6 @@ export const pairs = internalAction({ ids, committedAt, }); - void t0; if (args.gapMs) await new Promise((r) => setTimeout(r, args.gapMs)); } @@ -414,46 +412,6 @@ export const pairs = internalAction({ }, }); -/** - * Enqueue `count` tasks all due `delayMs` out in a SINGLE transaction, the way - * `enqueueBatch` does. They therefore share one commit stamp, which is what - * makes them interesting: the sweep's cursor is inclusive, so a backlog sharing - * a stamp is re-read rather than passed. - */ -export const bulkScheduled = internalMutation({ - args: { - cell: v.string(), - pool: vPoolKind, - count: v.number(), - delayMs: v.number(), - maxParallelism: v.number(), - }, - returns: v.number(), - handler: async (ctx, args) => { - const pool = makePool(args.pool as PoolKind, { - maxParallelism: args.maxParallelism, - }); - const enqueuedAt = Date.now(); - for (let i = 0; i < args.count; i++) { - const taskId = await ctx.db.insert("latencyTasks", { - cell: args.cell, - pool: args.pool, - delayMs: args.delayMs, - seq: 10_000 + i, - enqueuedAt, - runAt: enqueuedAt + args.delayMs, - }); - await pool.enqueueMutation( - ctx, - internal.test.latency.probe, - { taskId }, - { runAfter: args.delayMs }, - ); - } - return args.count; - }, -}); - /** * The backlog case: a bulk of scheduled work sharing one commit stamp, then a * single "soon" entry whose slow enqueue lands it behind the loop's cursor. If @@ -531,10 +489,7 @@ export const backlog = internalAction({ }); /** - * The same bulk, but through the batch enqueue API — one component call - * carrying every item, which is how an app actually queues thousands at once. - * All of them land in one transaction under one commit stamp, packed 256 to a - * document, which is the configuration that can exceed SWEEP_BATCH. + * Enqueue scheduled work through the batch API in one transaction. */ export const bulkScheduledBatch = internalMutation({ args: { diff --git a/example/convex/test/run.ts b/example/convex/test/run.ts index d5912d93..034f1e77 100644 --- a/example/convex/test/run.ts +++ b/example/convex/test/run.ts @@ -7,6 +7,8 @@ import { v } from "convex/values"; import type { Doc, Id } from "../_generated/dataModel"; import { assert } from "convex-helpers"; import { getComponent } from "./pool"; +import currentPackage from "@convex-dev/workpool/package.json"; +import baselinePackage from "@convex-dev/workpool-old/package.json"; export async function runStatus( ctx: QueryCtx, @@ -68,6 +70,8 @@ export const start = internalMutation({ parameters: args.parameters, taskCount: args.parameters.taskCount, pool: args.pool ?? "new", + poolVersion: + args.pool === "old" ? baselinePackage.version : currentPackage.version, }); return runId; diff --git a/example/convex/test/work.ts b/example/convex/test/work.ts index cc831b74..60978884 100644 --- a/example/convex/test/work.ts +++ b/example/convex/test/work.ts @@ -87,6 +87,7 @@ export const markTaskCompleted = internalMutation({ endTime: Date.now(), enqueuedAt: args.context.enqueuedAt, wave: args.context.wave, + resultKind: args.result.kind, }); }, }); diff --git a/example/src/App.tsx b/example/src/App.tsx index 6414f506..3c6a66c9 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -3,6 +3,7 @@ import { useEffect, useMemo, useState } from "react"; import { useAction, useQuery } from "convex/react"; import { api } from "../convex/_generated/api"; import type { Id } from "../convex/_generated/dataModel"; +import baselinePackage from "@convex-dev/workpool-old/package.json"; import { Area, AreaChart, @@ -129,7 +130,9 @@ function App() {
Current branch - Workpool 0.4.7 + + Workpool {baselinePackage.version} +
@@ -359,6 +362,7 @@ type RunSummary = { _id: RunId; scenario: string; pool?: string; + poolVersion?: string; startTime: number; taskCount?: number; scheduledFunctions?: number; @@ -590,7 +594,13 @@ function Comparison({ ids }: { ids: CompareIds }) { )} - +
Run details and parameters @@ -747,6 +757,7 @@ function RunCharts({ throughputData, cdfData, showBaseline = true, + baselineLabel = "Baseline", }: { throughputData: Array<{ tMs: number; @@ -755,6 +766,7 @@ function RunCharts({ }>; cdfData: Array<{ ms: number; baseline: number; current: number }>; showBaseline?: boolean; + baselineLabel?: string; }) { return (
@@ -800,7 +812,7 @@ function RunCharts({ - {isOld ? "0.4.7" : "Current"} + {isOld ? (run.poolVersion ?? "Baseline") : "Current"} {run.scenario} From 4ff190091bb55c1433ebb375f659f9f17720f594 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:17:44 -0700 Subject: [PATCH 4/9] fix(benchmarks): preserve timestamp ties during cleanup --- example/convex/test/cleanup.test.ts | 19 ++++++++++++++++++- example/convex/test/cleanup.ts | 22 ++++++++++++---------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/example/convex/test/cleanup.test.ts b/example/convex/test/cleanup.test.ts index a55824a7..e00ac425 100644 --- a/example/convex/test/cleanup.test.ts +++ b/example/convex/test/cleanup.test.ts @@ -17,7 +17,11 @@ describe("benchmark cleanup", () => { t.mutation(internal.test.cleanup.start, { limit }), ).rejects.toThrow("limit must be an integer between 1 and 1000"); await expect( - t.mutation(internal.test.cleanup.step, { index: 0, after: 0, limit }), + t.mutation(internal.test.cleanup.step, { + index: 0, + cursor: null, + limit, + }), ).rejects.toThrow("limit must be an integer between 1 and 1000"); await t.run(async (ctx) => { expect( @@ -48,4 +52,17 @@ describe("benchmark cleanup", () => { data: 0, }); }); + + it("clears rows sharing a creation time across page boundaries", async () => { + // At this clock, convex-test's 0.001 ms creation-time increment rounds away. + vi.setSystemTime(2 ** 44); + await t.run(async (ctx) => { + for (let i = 0; i < 3; i++) await ctx.db.insert("data", { data: i }); + const rows = await ctx.db.query("data").collect(); + expect(new Set(rows.map((row) => row._creationTime)).size).toBe(1); + }); + await t.mutation(internal.test.cleanup.start, { limit: 1 }); + await t.finishAllScheduledFunctions(vi.runAllTimers); + expect((await t.query(internal.test.cleanup.counts)).data).toBe(0); + }); }); diff --git a/example/convex/test/cleanup.ts b/example/convex/test/cleanup.ts index 1837f5f5..86965442 100644 --- a/example/convex/test/cleanup.ts +++ b/example/convex/test/cleanup.ts @@ -12,22 +12,24 @@ const TABLES = [ /** Clear bookkeeping between runs; advance the cursor to skip tombstones. */ export const step = internalMutation({ - args: { index: v.number(), limit: v.number(), after: v.number() }, + args: { + index: v.number(), + limit: v.number(), + cursor: v.union(v.string(), v.null()), + }, returns: v.null(), - handler: async (ctx, { index, limit, after }) => { + handler: async (ctx, { index, limit, cursor }) => { validateLimit(limit); if (index >= TABLES.length) return null; const table = TABLES[index]; - const docs = await ctx.db + const result = await ctx.db .query(table) - .withIndex("by_creation_time", (q) => q.gt("_creationTime", after)) - .take(limit); - for (const doc of docs) await ctx.db.delete(table, doc._id); - const done = docs.length < limit; + .paginate({ cursor, numItems: limit }); + for (const doc of result.page) await ctx.db.delete(table, doc._id); await ctx.scheduler.runAfter(0, internal.test.cleanup.step, { - index: done ? index + 1 : index, + index: result.isDone ? index + 1 : index, limit, - after: done ? 0 : docs[docs.length - 1]._creationTime, + cursor: result.isDone ? null : result.continueCursor, }); return null; }, @@ -41,7 +43,7 @@ export const start = internalMutation({ await ctx.scheduler.runAfter(0, internal.test.cleanup.step, { index: 0, limit, - after: 0, + cursor: null, }); return null; }, From 46486612cd1da496e43be0cf22b3085bcf1d1af4 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:36:06 -0700 Subject: [PATCH 5/9] test(benchmarks): measure ordering and packed starts separately --- example/BENCHMARKS.md | 65 +- example/benchmarks/2026-09-09.md | 138 + example/benchmarks/2026-09-09/analyze.py | 185 + example/benchmarks/2026-09-09/metadata.json | 115 + example/benchmarks/2026-09-09/mounts.patch | 108 + example/benchmarks/2026-09-09/prepare.py | 66 + example/benchmarks/2026-09-09/results.json | 7332 +++++++++++++++++++ example/benchmarks/2026-09-09/run.py | 258 + 8 files changed, 8246 insertions(+), 21 deletions(-) create mode 100644 example/benchmarks/2026-09-09.md create mode 100644 example/benchmarks/2026-09-09/analyze.py create mode 100644 example/benchmarks/2026-09-09/metadata.json create mode 100644 example/benchmarks/2026-09-09/mounts.patch create mode 100644 example/benchmarks/2026-09-09/prepare.py create mode 100644 example/benchmarks/2026-09-09/results.json create mode 100644 example/benchmarks/2026-09-09/run.py diff --git a/example/BENCHMARKS.md b/example/BENCHMARKS.md index b7776d84..6d263b9d 100644 --- a/example/BENCHMARKS.md +++ b/example/BENCHMARKS.md @@ -1,6 +1,8 @@ # Benchmarking the workpool -Recorded results: [2026-09-08, revision `8186121`](./benchmarks/2026-09-08.md). +Recorded results: +[2026-09-09, ordering vs. packed starts](./benchmarks/2026-09-09.md) and +[2026-09-08, revision `8186121` vs. 0.4.9](./benchmarks/2026-09-08.md). Harnesses in `example/convex/test/`, all driving a real deployment: @@ -17,8 +19,9 @@ Harnesses in `example/convex/test/`, all driving a real deployment: | `test/cleanup:start` / `:counts` | clear app bookkeeping; does not drain workpools | shared | `"pool": "new"` is this checkout's `testWorkpool`; `"pool": "old"` is the -published package mounted as `oldWorkpool`. Check the installed baseline rather -than relying on dashboard labels or previous reports: +published package mounted as `oldWorkpool`. New runs record the installed +package version for dashboard labels; historical runs without it display +“Baseline”. Check the installed baseline with: ```sh node -p 'require("./node_modules/@convex-dev/workpool-old/package.json").version' @@ -27,10 +30,8 @@ npm run build:codegen && npx convex dev --once Run commands from the repo root against a dedicated dev deployment. The example imports the component through its `dist` exports, so **build before deploying**. -A failed deploy means the previous design is still running. In particular, -pre-snapshot experimental deployments can retain the removed `lastCommitTs` -field in `internalState`, causing schema validation to fail. Resolve that state -compatibility issue before measuring; do not treat a failed push as a new build. +A failed deploy leaves the previous design running; resolve schema compatibility +errors before measuring a new build. Archive the commit, any source diff, dependency versions, deployment, exact arguments, raw outputs, and run order. Paired runs on one deployment help @@ -57,10 +58,11 @@ Discard warmups, then measure at least three pairs, alternating `old,new`, between arms. Save JSON before cleanup. Report individual runs, per-pair ratios, and the spread, not just the fastest run. -`completedCount` currently counts terminal callbacks, including failures and -cancelations: `markTaskCompleted` does not save `result.kind`. Check runtime -errors as well as `timedOut`, `status`, and the exact expected count. A CLI exit -code of zero alone does not establish that a benchmark finished successfully. +`completedCount` counts terminal callbacks, including failures and cancelations. +The recorder stores each task's `resultKind`: require the expected number of +unique work IDs with successful outcomes, a completed status, and no timeout. +Check runtime errors too; a CLI exit code of zero alone does not establish +success. ## Start latency and ordering @@ -137,17 +139,38 @@ the previous work. ## Comparing a code change against itself +The [September 9 experiment](./benchmarks/2026-09-09.md) compares pinned +baseline, timestamp-ordering, and packed-start revisions using three fresh +component mounts and one harness. Its checked-in driver verifies outcomes, +drains pools, archives every run, and balances warmup and measured order. With +Node 24+ and Python 3.12+ installed, prepare and run it from the repository +root: + +```sh +python3 example/benchmarks/2026-09-09/prepare.py \ + --checkout .context/stack-benchmark --env-file .env.local +cd .context/stack-benchmark +npm run build && npx convex dev --once --typecheck enable +cd ../.. +python3 example/benchmarks/2026-09-09/run.py \ + --checkout .context/stack-benchmark --output .context/stack-results +python3 example/benchmarks/2026-09-09/analyze.py .context/stack-results +``` + +This dated experiment targets `dev:precious-raven-853` and clears shared example +bookkeeping between runs. Use it only on that dedicated deployment without +concurrent traffic. Preparation rejects a different target or dependency +versions. After archiving results and confirming the pools drained, build and +deploy the normal checkout to remove the temporary mounts, then remove the +isolated checkout so its tests are not discovered by the repository test runner. +Keep the output directory. A new experiment should record its own revisions, +target, dependencies, and arguments. + Use the same component for both variants, with a build and successful deploy for -each source change, discarded warmups, cleanup, and balanced run order. Keep -edits in isolated checkouts so restoring a variant cannot overwrite other work. -Do not change constants during a comparison of the current design against the -published baseline. - -Historical `.context/bench-commitTs.sh`, `.context/exp-consts.py`, and -experiment reports are workspace artifacts, not tracked tooling available in a -fresh clone. They are not the authoritative procedure: their warmup counts/order -differ from their comments, and restoring with `git checkout` can erase -concurrent edits. +each source change, to further control component storage differences. Discard +warmups, clean up, and balance run order. Keep edits in isolated checkouts so +restoring a variant cannot overwrite other work. Do not change constants during +a comparison of the current design against the published baseline. ## Getting numbers you can trust diff --git a/example/benchmarks/2026-09-09.md b/example/benchmarks/2026-09-09.md new file mode 100644 index 00000000..84cea20a --- /dev/null +++ b/example/benchmarks/2026-09-09.md @@ -0,0 +1,138 @@ +# Ordering and packed-start benchmarks — 2026-09-09 (PDT) + +Mutation throughput: **+14.3% from the ordering PR**, then **+4.7% from packing +pending starts** (median within-round changes). + +Action throughput: **+13.6% from the ordering PR**, then **+7.3% from packing +pending starts** (median within-round changes). + +All 183,000 tasks succeeded: 3,000 smoke-check tasks, 90,000 warmup tasks, and +90,000 measured tasks. No throughput run timed out, and captured runtime logs +contained no execution errors. During mutation round 3, a cleanup cursor skipped +one completed row with a tied creation time. The driver stopped before the next +arm; another cleanup pass cleared it and the driver resumed from the next +unmeasured run. This lengthened the gap within that round; its timings remain +included. The cleanup helper was subsequently fixed with pagination cursors and +a regression test (175 tests pass). + +Keep the PRs separate: ordering improved throughput in every measured round. +Packing consistently reduced document work and enqueue time, but its mutation +throughput change ranged from -4.9% to +10.3%; treat that timing gain as +tentative. + +## Throughput and completion latency + +| Workload | Variant | Median tasks/s | Tasks/s range | Median enqueue ms | Median p95 ms | +| -------- | -------- | -------------: | ------------- | ----------------: | ------------: | +| mutation | baseline | 93.2 | 79.6–95.5 | 12,353 | 39,415 | +| mutation | ordering | 102.4 | 98.9–106.5 | 13,200 | 33,995 | +| mutation | packed | 107.1 | 101.2–109.1 | 10,764 | 34,561 | +| action | baseline | 112.7 | 107.8–115.2 | 11,925 | 30,460 | +| action | ordering | 124.4 | 122.5–132.6 | 12,252 | 26,596 | +| action | packed | 136.3 | 126.4–142.3 | 10,899 | 24,368 | + +Individual measured throughput, in tasks/s: + +| Workload | Round | Order | Baseline | Ordering | Packed | +| -------- | ----- | ----- | -------: | -------: | -----: | +| mutation | 1 | P/O/B | 79.6 | 98.9 | 109.1 | +| mutation | 2 | O/B/P | 95.5 | 102.4 | 107.1 | +| mutation | 3 | B/P/O | 93.2 | 106.5 | 101.2 | +| action | 1 | P/O/B | 112.7 | 132.6 | 142.3 | +| action | 2 | O/B/P | 115.2 | 124.4 | 126.4 | +| action | 3 | B/P/O | 107.8 | 122.5 | 136.3 | + +Incremental ratios compare variants within the same round: + +| Workload | Change | Throughput ratios, rounds 1–3 | Median throughput change | Median enqueue-time change | +| -------- | ------------------- | ----------------------------- | -----------------------: | -------------------------: | +| mutation | baseline → ordering | 1.242×, 1.071×, 1.143× | +14.3% | +9.5% | +| mutation | ordering → packed | 1.103×, 1.047×, 0.951× | +4.7% | -19.7% | +| action | baseline → ordering | 1.177×, 1.080×, 1.136× | +13.6% | +2.8% | +| action | ordering → packed | 1.073×, 1.016×, 1.113× | +7.3% | -12.1% | + +## Runtime work + +Median across measured runs; document counts are normalized per 1,000 tasks. +Enqueue counters cover the parent component’s `lib:enqueueBatch`; loop counters +cover its nested `batchWorker`’s `loop:loop`. Nested calls are not added again. +The window runs from benchmark start through the successful outcome check and +excludes later idle cooldown work. All 50 enqueue calls were captured in every +full run. Loop execution time is the median of each run's median. + +| Workload | Variant | Enqueue reads | Enqueue writes | Loop reads | Loop writes | Median loop execution ms | +| -------- | -------- | ------------: | -------------: | ---------: | ----------: | -----------------------: | +| mutation | baseline | 21.2 | 2,000.4 | 6,214.0 | 2,033.2 | 576.7 | +| mutation | ordering | 1,021.2 | 2,000.4 | 6,194.6 | 2,033.8 | 505.2 | +| mutation | packed | 1,021.2 | 1,010.4 | 3,247.6 | 1,058.8 | 487.7 | +| action | baseline | 21.2 | 2,000.4 | 6,174.0 | 2,033.0 | 517.5 | +| action | ordering | 1,021.2 | 2,000.4 | 6,193.2 | 2,033.6 | 447.6 | +| action | packed | 1,021.2 | 1,010.4 | 3,245.6 | 1,058.4 | 425.5 | + +Packing reduces pending-start document writes and loop reads. The ordering layer +adds roughly one enqueue document read per task while tracking the pending-start +pointer. Packing reduced enqueue writes by 49.5% and loop reads by about 47.6%. +For mutations, packed loop writes used 13.2% more bytes, while enqueue plus loop +write bytes fell 20.9% overall. + +## Warmup variation + +Discarded warmup throughput in run order within each variant, tasks/s: + +| Workload | Variant | Warmup 1 | Warmup 2 | Warmup 3 | +| -------- | -------- | -------: | -------: | -------: | +| mutation | baseline | 97.6 | 94.0 | 83.1 | +| mutation | ordering | 103.6 | 106.3 | 102.1 | +| mutation | packed | 110.3 | 105.6 | 97.6 | +| action | baseline | 113.9 | 90.0 | 99.5 | +| action | ordering | 136.6 | 129.3 | 127.4 | +| action | packed | 148.7 | 130.5 | 136.4 | + +## Method and scope + +This compares the stack's three component revisions with a common client and +harness, on `dev:precious-raven-853`: + +| Variant | Revision | Change | +| -------- | --------- | ---------------------------------------------------------------------------------------------------- | +| Baseline | `9a64c83` | `main`, version 0.4.11 | +| Ordering | `0b0b60e` | [#220](https://github.com/get-convex/workpool/pull/220), commit/snapshot ordering and cursor changes | +| Packed | `387a3ae` | [#234](https://github.com/get-convex/workpool/pull/234), multiple pending starts per document | + +Harness: `95d074c`, with the archived mount-selection patch. Dependencies: +Convex 1.43.0, batch-worker 0.2.2, convex-helpers 0.1.111; Node 24.20.0. The +three components were copied into an isolated checkout, mounted fresh, built, +and successfully deployed before any measurements. Their source was unchanged +during the experiment. The baseline here is **0.4.11**, not the published 0.4.9 +package used in the September 8 report. + +Every full run enqueues 5,000 tasks in batches of 100, with a 50 ms inter-batch +gap and parallelism 200. Mutations have no artificial workload; actions sleep 20 +ms. All variants first passed a 1,000-mutation smoke check. Each workload then +gets three discarded warmups per variant followed by three measured rounds. +Warmup order is B/O/P, O/P/B, P/B/O; measured order is P/O/B, O/B/P, B/P/O. Each +variant occupies every position once in each phase. Runs are serial, with all +three components drained and app bookkeeping cleared between arms. + +Throughput is tasks divided by time from run start through the final completion +callback. Enqueue time includes batch RPCs and configured gaps. Latency is +per-task enqueue to completion callback; table p95s are medians of run-level +percentiles. Polling and completion bookkeeping are part of the measured load. + +These are descriptive results from three rounds, with separate component storage +histories and time-varying deployment load. They isolate the two PR layers, not +individual lines within the ordering PR. Immediate bulk throughput does not +establish scheduled-work correctness or light-load tail latency. Do not compare +absolute rates directly with measurements from another day. + +## Reproduction and evidence + +[Instructions](../BENCHMARKS.md#comparing-a-code-change-against-itself), +[pinned metadata](./2026-09-09/metadata.json), and +[JSON results](./2026-09-09/results.json) are checked in. The JSON includes +every smoke, warmup, and measured output, outcome checks, drain counts, paired +ratios, and selected execution statistics. The dated directory also contains the +preparation, run, and analysis scripts and exact harness patch. Full CLI outputs +and runtime logs remain in `.context/benchmark-2026-09-09/` locally. All +components drained before the normal dev deployment was restored and the +temporary mounts removed. App bookkeeping counts were zero afterward. diff --git a/example/benchmarks/2026-09-09/analyze.py b/example/benchmarks/2026-09-09/analyze.py new file mode 100644 index 00000000..f2852748 --- /dev/null +++ b/example/benchmarks/2026-09-09/analyze.py @@ -0,0 +1,185 @@ +"""Summarize per-run timing and runtime resource counters without double counting nested calls.""" + +import argparse +import collections +import json +import statistics +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("output", type=Path, help="Directory produced by run.py") +OUT = parser.parse_args().output +metadata = json.loads((OUT / "metadata.json").read_text()) +runs = [json.loads(line) for line in (OUT / "runs.jsonl").read_text().splitlines()] +windows = collections.defaultdict(list) +for run in runs: + run["runtime"] = {} + component = metadata["variants"][run["variant"]]["component"] + windows[component].append(run) + +errors = [] +all_errors = [] +seen = set() +for line in (OUT / "runtime.jsonl").open(): + event = json.loads(line) + if event.get("kind") != "Completion": + continue + key = (event.get("executionId"), event.get("timestamp")) + if key in seen: + continue + seen.add(key) + if event.get("error"): + all_errors.append( + { + key: event.get(key) + for key in [ + "componentPath", + "identifier", + "executionTimestamp", + "error", + "occInfo", + "willRetry", + ] + } + ) + component = event.get("componentPath") or "" + root_component = component.split("/")[0] + for run in windows.get(root_component, []): + start = run["evidence"]["run"]["startTime"] + end = run["evidence"]["observedAt"] + if not start <= event.get("executionTimestamp", 0) * 1000 <= end: + continue + if event.get("error"): + errors.append( + { + "label": run["label"], + "component": component, + "identifier": event["identifier"], + "error": event["error"], + "occInfo": event.get("occInfo"), + "willRetry": event.get("willRetry"), + } + ) + group = None + if ( + component == root_component + "/batchWorker" + and event["identifier"] == "loop:loop" + ): + group = "loop" + elif component == root_component and event["identifier"] == "lib:enqueueBatch": + group = "enqueue" + if group is None: + break + stats = run["runtime"].setdefault( + group, + { + "executions": 0, + "executionMs": [], + "userExecutionMs": [], + "readDocuments": 0, + "writeDocuments": 0, + "readBytes": 0, + "writeBytes": 0, + "ioReadBytes": 0, + "ioWriteBytes": 0, + "writeIndexRows": 0, + "occExecutions": 0, + }, + ) + stats["executions"] += 1 + stats["executionMs"].append(event["executionTime"] * 1000) + stats["userExecutionMs"].append(event.get("userExecutionTime", 0) * 1000) + if event.get("occInfo"): + stats["occExecutions"] += 1 + usage = event["usageStats"] + for target, source in [ + ("readDocuments", "databaseReadDocuments"), + ("writeDocuments", "databaseWriteDocuments"), + ("readBytes", "databaseReadBytes"), + ("writeBytes", "databaseWriteBytes"), + ("ioReadBytes", "databaseIoReadBytes"), + ("ioWriteBytes", "databaseIoWriteBytes"), + ("writeIndexRows", "databaseWriteIndexRows"), + ]: + stats[target] += usage[source] + break + +summary = {} +for workload in ["mutation", "action"]: + summary[workload] = {} + for variant in metadata["variants"]: + selected = [ + r + for r in runs + if r["phase"] == "measured" + and r["workload"] == workload + and r["variant"] == variant + ] + if not selected: + continue + rates = [ + r["args"]["taskCount"] * 1000 / r["result"]["metrics"]["totalDurationMs"] + for r in selected + ] + summary[workload][variant] = { + "n": len(selected), + "throughputMedian": statistics.median(rates), + "throughputRange": [min(rates), max(rates)], + "enqueueMsMedian": statistics.median( + r["result"]["enqueueTotal"] for r in selected + ), + "p95MsMedian": statistics.median( + r["result"]["metrics"]["latency"]["p95"] for r in selected + ), + } + +comparisons = [] +for workload in ["mutation", "action"]: + for rep in [1, 2, 3]: + group = { + r["variant"]: r + for r in runs + if r["phase"] == "measured" + and r["workload"] == workload + and r["rep"] == rep + } + if len(group) != 3: + continue + for before, after in [("baseline", "ordering"), ("ordering", "packed")]: + a, b = group[before], group[after] + comparisons.append( + { + "workload": workload, + "rep": rep, + "before": before, + "after": after, + "throughputRatio": a["result"]["metrics"]["totalDurationMs"] + / b["result"]["metrics"]["totalDurationMs"], + "enqueueRatio": b["result"]["enqueueTotal"] + / a["result"]["enqueueTotal"], + } + ) + +result = { + "metadata": metadata, + "summary": summary, + "comparisons": comparisons, + "executionErrors": errors, + "allExecutionErrors": all_errors, + "runs": runs, +} +incident = OUT / "cleanup-incident.json" +if incident.exists(): + result["cleanupIncident"] = json.loads(incident.read_text()) +(OUT / "analysis.json").write_text(json.dumps(result, indent=2) + "\n") +print( + json.dumps( + { + "summary": summary, + "comparisons": comparisons, + "executionErrors": errors, + "allExecutionErrors": all_errors, + }, + indent=2, + ) +) diff --git a/example/benchmarks/2026-09-09/metadata.json b/example/benchmarks/2026-09-09/metadata.json new file mode 100644 index 00000000..43021243 --- /dev/null +++ b/example/benchmarks/2026-09-09/metadata.json @@ -0,0 +1,115 @@ +{ + "harnessRevision": "95d074c5019c262d99e5e32682d53752aa40c74b", + "deployment": "precious-raven-853", + "variants": { + "baseline": { + "revision": "9a64c831b2a4bf3ce565f1a364e59007df4354bb", + "ref": "main", + "component": "benchBaseline0909", + "sourceHashes": { + "README.md": "f9c721a8c664976d120cbb40fa45f9d1969aba1af8500a737eb009ff9cd5943b", + "complete.ts": "f5564e078420eac14a6aa9ac74ae9d2dcb2c189521217a9181b54082969a6364", + "config.ts": "ee66976f43f4c1d2b9e2ee421a3acc467d3bcc8515db3f9db46f8fb36ceac7e9", + "convex.config.ts": "2ea59a7cfc1dfa20c1ce5745cac1ff5af8c3248efcf9728a89d329ce2c143b9c", + "danger.ts": "ff5b3fbd55da13609f4c726589f09d879cb1884cff9ff941979abaca16270e40", + "errors.ts": "c677629a6c52dabdde240051bd3e04618c456bd9f3fcd3f1d2e5e66fb1142f4a", + "future.ts": "e829cfdea8d13ab691fe3e07926d5a8c61b4bb97bc59bd37e7c1b298ff29ff9c", + "kick.ts": "d93a625ad33d08dbe3f30db8375f0a0bbfbad343723ad5a003df6874c53dd3a1", + "lib.ts": "a9acbc26710c061c9da57e4a98b91444d17926ec303bbf73892b9aa3b84b16a7", + "logging.ts": "47b5efd66076a0f1f76551e552bbb811bc243a51c8d60a1cb425b68b1b47e274", + "loop.ts": "7fdd28ac32dd614bc36d48aca8042b523c3366066bf28546be2e4b7814b17ae4", + "recovery.ts": "92e9ef5fbb1c7fce1b2feab215d31c99773b9c9658f4f95f4e0a70ef2fd1aecc", + "schema.ts": "e27b79cce087238dbb315ec57539158f0835026ad4cc152593c3045b6cf360a9", + "shared.ts": "fc295186c876a04bcebab2d7833390d5e43737e4272a42d82838c7de9801c620", + "stats.ts": "9b9ca4a977ddde02c078943e322a0a82b799567184f9e50ea6456cef7ca3a82f", + "worker.ts": "fe53abaed84bf19a9af3ec413fb97cc4c9808a6504cd9230abcd94fa64202040" + } + }, + "ordering": { + "revision": "0b0b60e63a580b3897112097a6e5d3746cc3427f", + "ref": "ian/use-commit-ts", + "component": "benchOrdering0909", + "sourceHashes": { + "README.md": "a87d03fc62b64fd862b45fd1deb77c6a16b04a413b6df4a69fd7278cb0207ff5", + "complete.ts": "964843a6e025ad24ea3fbbb6da8251281ba06252db94c56c1f8a73f10b3531ff", + "config.ts": "ee66976f43f4c1d2b9e2ee421a3acc467d3bcc8515db3f9db46f8fb36ceac7e9", + "convex.config.ts": "2ea59a7cfc1dfa20c1ce5745cac1ff5af8c3248efcf9728a89d329ce2c143b9c", + "danger.ts": "4caf8bd4d33aa4a45e88cb0ee308ace2c1e9d7d7b46954afa24afeb57dccf5c6", + "errors.ts": "c677629a6c52dabdde240051bd3e04618c456bd9f3fcd3f1d2e5e66fb1142f4a", + "future.ts": "e829cfdea8d13ab691fe3e07926d5a8c61b4bb97bc59bd37e7c1b298ff29ff9c", + "kick.ts": "d93a625ad33d08dbe3f30db8375f0a0bbfbad343723ad5a003df6874c53dd3a1", + "lib.ts": "5e1eafde3527aca80cb8a8f07b11b18c2c018697e04cb344e9e6992e1eb791f3", + "logging.ts": "47b5efd66076a0f1f76551e552bbb811bc243a51c8d60a1cb425b68b1b47e274", + "loop.ts": "e7ce0fd5bed9853dda1aef7b8b58e055e1e0c1b9b5b478dd4656d246792880f1", + "pendingStart.ts": "ff93ba12ce6705e00911ecf25a4234b53340f395954bde117a834ebc8de13861", + "recovery.ts": "92e9ef5fbb1c7fce1b2feab215d31c99773b9c9658f4f95f4e0a70ef2fd1aecc", + "schema.ts": "e6e7a4628a270bbff5225bea546ad370cb40981f0099217b996f7dc7c6c905d1", + "shared.ts": "6ce8d96801fb716fb4a504a145ed53428cad4728ab58e28dfc4a6f352587f411", + "stats.ts": "7da861b6ffca4d3757ff27509a62dfd5c5137245eacc33a41b25f5ef410d9962", + "worker.ts": "fe53abaed84bf19a9af3ec413fb97cc4c9808a6504cd9230abcd94fa64202040" + } + }, + "packed": { + "revision": "387a3ae8990f883da48751ea68c104d6e46e5418", + "ref": "ian/shared-start-documents", + "component": "benchPacked0909", + "sourceHashes": { + "README.md": "1ae5af85b1dff525b3498af5de10943772a1b811275e72f3ca41bee021be8c7d", + "complete.ts": "964843a6e025ad24ea3fbbb6da8251281ba06252db94c56c1f8a73f10b3531ff", + "config.ts": "ee66976f43f4c1d2b9e2ee421a3acc467d3bcc8515db3f9db46f8fb36ceac7e9", + "convex.config.ts": "2ea59a7cfc1dfa20c1ce5745cac1ff5af8c3248efcf9728a89d329ce2c143b9c", + "danger.ts": "d8d97d6b8fc0fdd8ed69b29b0a5d201466058024cbcd0603cba4e59d6d457662", + "errors.ts": "c677629a6c52dabdde240051bd3e04618c456bd9f3fcd3f1d2e5e66fb1142f4a", + "future.ts": "e829cfdea8d13ab691fe3e07926d5a8c61b4bb97bc59bd37e7c1b298ff29ff9c", + "kick.ts": "d93a625ad33d08dbe3f30db8375f0a0bbfbad343723ad5a003df6874c53dd3a1", + "lib.ts": "b85a2a316f766c2abb89695f95a6bc102558a9cd24fdadcd8a1e6dd1839130a6", + "logging.ts": "47b5efd66076a0f1f76551e552bbb811bc243a51c8d60a1cb425b68b1b47e274", + "loop.ts": "ad836b9b65a75014676de4618d92be2c72eda18837f5c39218e1cc09f6ab8290", + "pendingStart.ts": "ee89a014b166664d11ee164da297e19261cfa6b8d740075b2a668df58b28f2b9", + "recovery.ts": "92e9ef5fbb1c7fce1b2feab215d31c99773b9c9658f4f95f4e0a70ef2fd1aecc", + "schema.ts": "6e658b94c49a7341b8f6571f79a3b2f7128a2d608a1789064225035359c7f4ba", + "shared.ts": "6ce8d96801fb716fb4a504a145ed53428cad4728ab58e28dfc4a6f352587f411", + "stats.ts": "7da861b6ffca4d3757ff27509a62dfd5c5137245eacc33a41b25f5ef410d9962", + "worker.ts": "fe53abaed84bf19a9af3ec413fb97cc4c9808a6504cd9230abcd94fa64202040" + } + } + }, + "nodeVersion": "v24.20.0", + "dependencies": { + "convex": "1.43.0", + "@convex-dev/batch-worker": "0.2.2", + "convex-helpers": "0.1.111" + }, + "warmupOrders": [ + ["baseline", "ordering", "packed"], + ["ordering", "packed", "baseline"], + ["packed", "baseline", "ordering"] + ], + "measuredOrders": [ + ["packed", "ordering", "baseline"], + ["ordering", "baseline", "packed"], + ["baseline", "packed", "ordering"] + ], + "validation": { + "tasksPerVariant": 1000, + "variants": 3, + "allSuccessful": true, + "executionErrors": 0 + }, + "workloads": { + "mutation": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "artificialWork": null + }, + "action": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskDurationMs": 20 + } + } +} diff --git a/example/benchmarks/2026-09-09/mounts.patch b/example/benchmarks/2026-09-09/mounts.patch new file mode 100644 index 00000000..197072f6 --- /dev/null +++ b/example/benchmarks/2026-09-09/mounts.patch @@ -0,0 +1,108 @@ +--- a/example/convex/convex.config.ts ++++ b/example/convex/convex.config.ts +@@ -2,6 +2,12 @@ + import workpool from "@convex-dev/workpool/convex.config"; + import workpoolOld from "@convex-dev/workpool-old/convex.config"; + import staticHosting from "@convex-dev/static-hosting/convex.config"; ++ ++import baseline from "../../benchmark-components/baseline/src/component/convex.config"; ++ ++import ordering from "../../benchmark-components/ordering/src/component/convex.config"; ++ ++import packed from "../../benchmark-components/packed/src/component/convex.config"; + + const app = defineApp({ httpPrefix: "/api" }); + app.use(workpool, { name: "smallPool" }); +@@ -11,4 +17,10 @@ + app.use(workpoolOld, { name: "oldWorkpool" }); + app.use(staticHosting, { httpPrefix: "/" }); + ++app.use(baseline, { name: "benchBaseline0909" }); ++ ++app.use(ordering, { name: "benchOrdering0909" }); ++ ++app.use(packed, { name: "benchPacked0909" }); ++ + export default app; +--- a/example/convex/test/pool.ts ++++ b/example/convex/test/pool.ts +@@ -9,9 +9,9 @@ + } from "@convex-dev/workpool-old"; + import { ActionCtx, MutationCtx } from "../_generated/server"; + +-export const POOL_KINDS = ["new", "old"] as const; ++export const POOL_KINDS = ["new", "old", "baseline", "ordering", "packed"] as const; + export type PoolKind = (typeof POOL_KINDS)[number]; +-export const vPoolKind = v.union(v.literal("new"), v.literal("old")); ++export const vPoolKind = v.union(...POOL_KINDS.map((kind) => v.literal(kind))); + + type Ctx = ActionCtx | MutationCtx; + +@@ -20,8 +20,7 @@ + kind: PoolKind, + maxParallelism: number, + ): Promise { +- const component = +- kind === "new" ? components.testWorkpool : components.oldWorkpool; ++ const component = getComponent(kind); + await ctx.runMutation(component.config.update, { maxParallelism }); + } + +@@ -34,12 +33,12 @@ + kind: PoolKind, + opts: { maxParallelism: number }, + ): Workpool { +- if (kind === "new") return new Workpool(components.testWorkpool, opts); ++ if (kind !== "old") return new Workpool(getComponent(kind), opts); + return new OldWorkpool(components.oldWorkpool, opts) as Workpool; + } + + export function getComponent(kind: PoolKind) { +- return kind === "new" ? components.testWorkpool : components.oldWorkpool; ++ return { new: components.testWorkpool, old: components.oldWorkpool, baseline: components.benchBaseline0909, ordering: components.benchOrdering0909, packed: components.benchPacked0909 }[kind]; + } + + export function enqueueFor(kind: PoolKind): { +@@ -47,9 +46,9 @@ + enqueueOne: typeof enqueue; + enqueueMany: typeof enqueueBatch; + } { +- if (kind === "new") { ++ if (kind !== "old") { + return { +- component: components.testWorkpool, ++ component: getComponent(kind), + enqueueOne: enqueue, + enqueueMany: enqueueBatch, + }; +--- a/example/convex/schema.ts ++++ b/example/convex/schema.ts +@@ -51,7 +51,7 @@ + // rather than inferred from completion times. + latencyTasks: defineTable({ + cell: v.string(), // which experiment cell enqueued it +- pool: v.union(v.literal("new"), v.literal("old")), ++ pool: v.union(v.literal("new"), v.literal("old"), v.literal("baseline"), v.literal("ordering"), v.literal("packed")), + delayMs: v.number(), // 0 for immediate + seq: v.number(), // enqueue order within the cell + enqueuedAt: v.number(), // clock at the enqueuing mutation +--- a/example/convex/test/run.ts ++++ b/example/convex/test/run.ts +@@ -33,7 +33,7 @@ + args: { + scenario: v.string(), + parameters: v.any(), +- pool: v.optional(v.union(v.literal("new"), v.literal("old"))), ++ pool: v.optional(v.union(v.literal("new"), v.literal("old"), v.literal("baseline"), v.literal("ordering"), v.literal("packed"))), + }, + handler: async (ctx, args) => { + // Check for in-flight tasks from the latest run +@@ -127,7 +127,7 @@ + + export const latestRunForPoolSince = internalQuery({ + args: { +- pool: v.union(v.literal("new"), v.literal("old")), ++ pool: v.union(v.literal("new"), v.literal("old"), v.literal("baseline"), v.literal("ordering"), v.literal("packed")), + since: v.number(), + }, + returns: v.union(v.id("runs"), v.null()), diff --git a/example/benchmarks/2026-09-09/prepare.py b/example/benchmarks/2026-09-09/prepare.py new file mode 100644 index 00000000..e6f7ca61 --- /dev/null +++ b/example/benchmarks/2026-09-09/prepare.py @@ -0,0 +1,66 @@ +"""Prepare the pinned three-component experiment in an isolated checkout.""" + +import argparse +import io +import json +import os +import shutil +import subprocess +import tarfile +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--checkout", type=Path, required=True) +parser.add_argument("--env-file", type=Path, required=True) +args = parser.parse_args() +root = Path( + subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() +) +record = Path(__file__).resolve().parent +metadata = json.loads((record / "metadata.json").read_text()) +stage = args.checkout.resolve() +if stage.exists(): + raise RuntimeError(f"Checkout already exists: {stage}") +env = args.env_file.read_text() +deployment = next( + ( + line.split("=", 1)[1].split("#", 1)[0].strip().strip("\"'") + for line in env.splitlines() + if line.startswith("CONVEX_DEPLOYMENT=") + ), + None, +) +if ( + deployment != f"dev:{metadata['deployment']}" + or "CONVEX_DEPLOY_KEY=" in env + or os.environ.get("CONVEX_DEPLOY_KEY") +): + raise RuntimeError("Use the recorded dev deployment without a deploy-key override") +for name, version in metadata["dependencies"].items(): + installed = json.loads((root / "node_modules" / name / "package.json").read_text())[ + "version" + ] + if installed != version: + raise RuntimeError(f"{name}: expected {version}, installed {installed}") +stage.mkdir(parents=True) + + +def extract(revision, destination, *paths): + contents = subprocess.check_output(["git", "archive", revision, *paths], cwd=root) + with tarfile.open(fileobj=io.BytesIO(contents)) as archive: + archive.extractall(destination, filter="data") + + +extract(metadata["harnessRevision"], stage) +(stage / "node_modules").symlink_to(root / "node_modules", target_is_directory=True) +shutil.copy2(args.env_file, stage / ".env.local") +for name, variant in metadata["variants"].items(): + destination = stage / "benchmark-components" / name + destination.mkdir(parents=True) + extract(variant["revision"], destination, "src/component") +subprocess.run( + ["patch", "-p1", "-i", str(record / "mounts.patch")], cwd=stage, check=True +) +print( + f"Prepared {stage}; target dev:{metadata['deployment']}. Build and deploy before running." +) diff --git a/example/benchmarks/2026-09-09/results.json b/example/benchmarks/2026-09-09/results.json new file mode 100644 index 00000000..4307aa97 --- /dev/null +++ b/example/benchmarks/2026-09-09/results.json @@ -0,0 +1,7332 @@ +{ + "metadata": { + "harnessRevision": "95d074c5019c262d99e5e32682d53752aa40c74b", + "deployment": "precious-raven-853", + "variants": { + "baseline": { + "revision": "9a64c831b2a4bf3ce565f1a364e59007df4354bb", + "ref": "main", + "component": "benchBaseline0909", + "sourceHashes": { + "README.md": "f9c721a8c664976d120cbb40fa45f9d1969aba1af8500a737eb009ff9cd5943b", + "complete.ts": "f5564e078420eac14a6aa9ac74ae9d2dcb2c189521217a9181b54082969a6364", + "config.ts": "ee66976f43f4c1d2b9e2ee421a3acc467d3bcc8515db3f9db46f8fb36ceac7e9", + "convex.config.ts": "2ea59a7cfc1dfa20c1ce5745cac1ff5af8c3248efcf9728a89d329ce2c143b9c", + "danger.ts": "ff5b3fbd55da13609f4c726589f09d879cb1884cff9ff941979abaca16270e40", + "errors.ts": "c677629a6c52dabdde240051bd3e04618c456bd9f3fcd3f1d2e5e66fb1142f4a", + "future.ts": "e829cfdea8d13ab691fe3e07926d5a8c61b4bb97bc59bd37e7c1b298ff29ff9c", + "kick.ts": "d93a625ad33d08dbe3f30db8375f0a0bbfbad343723ad5a003df6874c53dd3a1", + "lib.ts": "a9acbc26710c061c9da57e4a98b91444d17926ec303bbf73892b9aa3b84b16a7", + "logging.ts": "47b5efd66076a0f1f76551e552bbb811bc243a51c8d60a1cb425b68b1b47e274", + "loop.ts": "7fdd28ac32dd614bc36d48aca8042b523c3366066bf28546be2e4b7814b17ae4", + "recovery.ts": "92e9ef5fbb1c7fce1b2feab215d31c99773b9c9658f4f95f4e0a70ef2fd1aecc", + "schema.ts": "e27b79cce087238dbb315ec57539158f0835026ad4cc152593c3045b6cf360a9", + "shared.ts": "fc295186c876a04bcebab2d7833390d5e43737e4272a42d82838c7de9801c620", + "stats.ts": "9b9ca4a977ddde02c078943e322a0a82b799567184f9e50ea6456cef7ca3a82f", + "worker.ts": "fe53abaed84bf19a9af3ec413fb97cc4c9808a6504cd9230abcd94fa64202040" + } + }, + "ordering": { + "revision": "0b0b60e63a580b3897112097a6e5d3746cc3427f", + "ref": "ian/use-commit-ts", + "component": "benchOrdering0909", + "sourceHashes": { + "README.md": "a87d03fc62b64fd862b45fd1deb77c6a16b04a413b6df4a69fd7278cb0207ff5", + "complete.ts": "964843a6e025ad24ea3fbbb6da8251281ba06252db94c56c1f8a73f10b3531ff", + "config.ts": "ee66976f43f4c1d2b9e2ee421a3acc467d3bcc8515db3f9db46f8fb36ceac7e9", + "convex.config.ts": "2ea59a7cfc1dfa20c1ce5745cac1ff5af8c3248efcf9728a89d329ce2c143b9c", + "danger.ts": "4caf8bd4d33aa4a45e88cb0ee308ace2c1e9d7d7b46954afa24afeb57dccf5c6", + "errors.ts": "c677629a6c52dabdde240051bd3e04618c456bd9f3fcd3f1d2e5e66fb1142f4a", + "future.ts": "e829cfdea8d13ab691fe3e07926d5a8c61b4bb97bc59bd37e7c1b298ff29ff9c", + "kick.ts": "d93a625ad33d08dbe3f30db8375f0a0bbfbad343723ad5a003df6874c53dd3a1", + "lib.ts": "5e1eafde3527aca80cb8a8f07b11b18c2c018697e04cb344e9e6992e1eb791f3", + "logging.ts": "47b5efd66076a0f1f76551e552bbb811bc243a51c8d60a1cb425b68b1b47e274", + "loop.ts": "e7ce0fd5bed9853dda1aef7b8b58e055e1e0c1b9b5b478dd4656d246792880f1", + "pendingStart.ts": "ff93ba12ce6705e00911ecf25a4234b53340f395954bde117a834ebc8de13861", + "recovery.ts": "92e9ef5fbb1c7fce1b2feab215d31c99773b9c9658f4f95f4e0a70ef2fd1aecc", + "schema.ts": "e6e7a4628a270bbff5225bea546ad370cb40981f0099217b996f7dc7c6c905d1", + "shared.ts": "6ce8d96801fb716fb4a504a145ed53428cad4728ab58e28dfc4a6f352587f411", + "stats.ts": "7da861b6ffca4d3757ff27509a62dfd5c5137245eacc33a41b25f5ef410d9962", + "worker.ts": "fe53abaed84bf19a9af3ec413fb97cc4c9808a6504cd9230abcd94fa64202040" + } + }, + "packed": { + "revision": "387a3ae8990f883da48751ea68c104d6e46e5418", + "ref": "ian/shared-start-documents", + "component": "benchPacked0909", + "sourceHashes": { + "README.md": "1ae5af85b1dff525b3498af5de10943772a1b811275e72f3ca41bee021be8c7d", + "complete.ts": "964843a6e025ad24ea3fbbb6da8251281ba06252db94c56c1f8a73f10b3531ff", + "config.ts": "ee66976f43f4c1d2b9e2ee421a3acc467d3bcc8515db3f9db46f8fb36ceac7e9", + "convex.config.ts": "2ea59a7cfc1dfa20c1ce5745cac1ff5af8c3248efcf9728a89d329ce2c143b9c", + "danger.ts": "d8d97d6b8fc0fdd8ed69b29b0a5d201466058024cbcd0603cba4e59d6d457662", + "errors.ts": "c677629a6c52dabdde240051bd3e04618c456bd9f3fcd3f1d2e5e66fb1142f4a", + "future.ts": "e829cfdea8d13ab691fe3e07926d5a8c61b4bb97bc59bd37e7c1b298ff29ff9c", + "kick.ts": "d93a625ad33d08dbe3f30db8375f0a0bbfbad343723ad5a003df6874c53dd3a1", + "lib.ts": "b85a2a316f766c2abb89695f95a6bc102558a9cd24fdadcd8a1e6dd1839130a6", + "logging.ts": "47b5efd66076a0f1f76551e552bbb811bc243a51c8d60a1cb425b68b1b47e274", + "loop.ts": "ad836b9b65a75014676de4618d92be2c72eda18837f5c39218e1cc09f6ab8290", + "pendingStart.ts": "ee89a014b166664d11ee164da297e19261cfa6b8d740075b2a668df58b28f2b9", + "recovery.ts": "92e9ef5fbb1c7fce1b2feab215d31c99773b9c9658f4f95f4e0a70ef2fd1aecc", + "schema.ts": "6e658b94c49a7341b8f6571f79a3b2f7128a2d608a1789064225035359c7f4ba", + "shared.ts": "6ce8d96801fb716fb4a504a145ed53428cad4728ab58e28dfc4a6f352587f411", + "stats.ts": "7da861b6ffca4d3757ff27509a62dfd5c5137245eacc33a41b25f5ef410d9962", + "worker.ts": "fe53abaed84bf19a9af3ec413fb97cc4c9808a6504cd9230abcd94fa64202040" + } + } + }, + "nodeVersion": "v24.20.0", + "dependencies": { + "convex": "1.43.0", + "@convex-dev/batch-worker": "0.2.2", + "convex-helpers": "0.1.111" + }, + "warmupOrders": [ + ["baseline", "ordering", "packed"], + ["ordering", "packed", "baseline"], + ["packed", "baseline", "ordering"] + ], + "measuredOrders": [ + ["packed", "ordering", "baseline"], + ["ordering", "baseline", "packed"], + ["baseline", "packed", "ordering"] + ], + "validation": { + "tasksPerVariant": 1000, + "variants": 3, + "allSuccessful": true, + "executionErrors": 0 + }, + "workloads": { + "mutation": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "artificialWork": null + }, + "action": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskDurationMs": 20 + } + } + }, + "summary": { + "mutation": { + "baseline": { + "n": 3, + "throughputMedian": 93.17059536010436, + "throughputRange": [79.61149590000797, 95.54018420147514], + "enqueueMsMedian": 12353, + "p95MsMedian": 39415 + }, + "ordering": { + "n": 3, + "throughputMedian": 102.36671853247073, + "throughputRange": [98.89240506329114, 106.49627263045794], + "enqueueMsMedian": 13200, + "p95MsMedian": 33995 + }, + "packed": { + "n": 3, + "throughputMedian": 107.1397960058284, + "throughputRange": [101.23916740908723, 109.07504363001745], + "enqueueMsMedian": 10764, + "p95MsMedian": 34561 + } + }, + "action": { + "baseline": { + "n": 3, + "throughputMedian": 112.67605633802818, + "throughputRange": [107.75397612171889, 115.20206442099442], + "enqueueMsMedian": 11925, + "p95MsMedian": 30460 + }, + "ordering": { + "n": 3, + "throughputMedian": 124.37810945273633, + "throughputRange": [122.45297805642633, 132.6365493275327], + "enqueueMsMedian": 12252, + "p95MsMedian": 26596 + }, + "packed": { + "n": 3, + "throughputMedian": 136.3252174387218, + "throughputRange": [126.3807092485403, 142.34470193019416], + "enqueueMsMedian": 10899, + "p95MsMedian": 24368 + } + } + }, + "comparisons": [ + { + "workload": "mutation", + "rep": 1, + "before": "baseline", + "after": "ordering", + "throughputRatio": 1.2421875, + "enqueueRatio": 1.1054804500930948 + }, + { + "workload": "mutation", + "rep": 1, + "before": "ordering", + "after": "packed", + "throughputRatio": 1.1029668411867364, + "enqueueRatio": 0.7985500878734623 + }, + { + "workload": "mutation", + "rep": 2, + "before": "baseline", + "after": "ordering", + "throughputRatio": 1.0714519695356646, + "enqueueRatio": 1.0947089069497429 + }, + { + "workload": "mutation", + "rep": 2, + "before": "ordering", + "after": "packed", + "throughputRatio": 1.0466272392217366, + "enqueueRatio": 0.8033333333333333 + }, + { + "workload": "mutation", + "rep": 3, + "before": "baseline", + "after": "ordering", + "throughputRatio": 1.143024494142705, + "enqueueRatio": 0.9901037895244992 + }, + { + "workload": "mutation", + "rep": 3, + "before": "ordering", + "after": "packed", + "throughputRatio": 0.950635781971329, + "enqueueRatio": 0.8746952705997074 + }, + { + "workload": "action", + "rep": 1, + "before": "baseline", + "after": "ordering", + "throughputRatio": 1.1771493752818527, + "enqueueRatio": 1.028369984891724 + }, + { + "workload": "action", + "rep": 1, + "before": "ordering", + "after": "packed", + "throughputRatio": 1.0731936457325058, + "enqueueRatio": 0.8788769180541952 + }, + { + "workload": "action", + "rep": 2, + "before": "baseline", + "after": "ordering", + "throughputRatio": 1.0796517412935323, + "enqueueRatio": 1.019874213836478 + }, + { + "workload": "action", + "rep": 2, + "before": "ordering", + "after": "packed", + "throughputRatio": 1.0161009023582641, + "enqueueRatio": 0.8961519486926492 + }, + { + "workload": "action", + "rep": 3, + "before": "baseline", + "after": "ordering", + "throughputRatio": 1.136412617554859, + "enqueueRatio": 1.0768586036375827 + }, + { + "workload": "action", + "rep": 3, + "before": "ordering", + "after": "packed", + "throughputRatio": 1.113286255691578, + "enqueueRatio": 0.8504047322540473 + } + ], + "executionErrors": [], + "allExecutionErrors": [], + "runs": [ + { + "label": "mutation-smoke-1-baseline", + "phase": "smoke", + "rep": 1, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T00:52:18.122471+00:00", + "args": { + "taskCount": 1000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 2503, + "metrics": { + "completedCount": 1000, + "latency": { + "max": 8157, + "p50": 4664, + "p95": 7725, + "p99": 8144 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 1000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 1000, + "totalDurationMs": 10522 + }, + "tasksPerSec": 95, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001550328, + "outcomes": { + "success": 1000 + }, + "run": { + "_creationTime": 1789001538674.0854, + "_id": "jd745ff6bxfx8dwr0vkc61ehxn8e5jyy", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 1000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001538674, + "taskCount": 1000 + }, + "uniqueWorkIds": 1000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 10, + "executionMs": [ + 112.931007, 88.935841, 116.363016, 122.140647, 124.21663500000001, + 139.716578, 103.451374, 104.468143, 151.768297, 113.779475 + ], + "userExecutionMs": [ + 20.136505999999997, 14.585228, 19.576256, 16.012898, + 19.735526999999998, 16.36732, 16.567561, 16.06565, 29.75868, + 22.939042 + ], + "readDocuments": 26, + "writeDocuments": 2002, + "readBytes": 5513, + "writeBytes": 831604, + "ioReadBytes": 5513, + "ioWriteBytes": 831604, + "writeIndexRows": 6005, + "occExecutions": 0 + }, + "loop": { + "executions": 21, + "executionMs": [ + 212.93077799999998, 484.032307, 564.161922, 721.564271, 619.593435, + 659.999475, 491.127066, 644.387265, 547.025302, 575.6323199999999, + 482.343421, 560.677689, 509.30251000000004, 512.986847, 539.844195, + 496.262288, 378.544295, 265.053866, 121.545342, 91.341925, 65.357503 + ], + "userExecutionMs": [ + 8.042172, 7.233368, 9.583649, 18.528956, 7.697131000000001, + 7.796374, 6.324650999999999, 8.861654, 7.322104, 9.061878, 6.242524, + 11.568048, 7.607189, 6.871857, 6.480981, 5.743071, 9.221767, + 3.103017, 3.571784, 2.288658, 2.223147 + ], + "readDocuments": 6236, + "writeDocuments": 2039, + "readBytes": 2043264, + "writeBytes": 234812, + "ioReadBytes": 2043264, + "ioWriteBytes": 234812, + "writeIndexRows": 8078, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-smoke-1-ordering", + "phase": "smoke", + "rep": 1, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T00:52:39.437576+00:00", + "args": { + "taskCount": 1000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 2545, + "metrics": { + "completedCount": 1000, + "latency": { + "max": 7489, + "p50": 3888, + "p95": 6928, + "p99": 7479 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 1000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 1000, + "totalDurationMs": 9837 + }, + "tasksPerSec": 102, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001571041, + "outcomes": { + "success": 1000 + }, + "run": { + "_creationTime": 1789001560197.2761, + "_id": "jd78v4vp3y6mfewyy27kf06zvx8e5tab", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 1000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001560197, + "taskCount": 1000 + }, + "uniqueWorkIds": 1000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 10, + "executionMs": [ + 125.48362199999998, 99.94191199999999, 149.701738, 111.801198, + 172.52272799999997, 109.976765, 150.287893, 110.906939, 95.681792, + 187.88918099999998 + ], + "userExecutionMs": [ + 25.52899, 16.826699, 24.702624999999998, 18.194984, 17.906892, + 21.627268, 17.901629, 17.567731, 16.961426000000003, 18.882688 + ], + "readDocuments": 1026, + "writeDocuments": 2002, + "readBytes": 520513, + "writeBytes": 855604, + "ioReadBytes": 520513, + "ioWriteBytes": 855604, + "writeIndexRows": 6005, + "occExecutions": 0 + }, + "loop": { + "executions": 21, + "executionMs": [ + 220.10406, 329.270576, 570.785257, 573.4124969999999, 503.428137, + 509.26433299999997, 502.97211200000004, 529.153186, 515.893008, + 510.081212, 510.603475, 494.60643, 567.3057249999999, 514.14886, + 624.2304330000001, 604.963796, 408.060889, 232.303397, + 90.88317599999999, 81.982702, 68.80117299999999 + ], + "userExecutionMs": [ + 3.422454, 6.937033, 9.184821, 6.4437169999999995, + 6.5592690000000005, 6.203192, 9.296346, 6.901564, 6.253458, + 6.255657, 6.790735, 6.107516, 6.044107, 8.688826, 6.10487, + 7.097087999999999, 8.443045, 4.87135, 2.4392519999999998, 2.273704, + 2.2705300000000004 + ], + "readDocuments": 6252, + "writeDocuments": 2039, + "readBytes": 2056592, + "writeBytes": 222002, + "ioReadBytes": 2056592, + "ioWriteBytes": 222002, + "writeIndexRows": 8078, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-smoke-1-packed", + "phase": "smoke", + "rep": 1, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T00:52:59.457759+00:00", + "args": { + "taskCount": 1000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 2173, + "metrics": { + "completedCount": 1000, + "latency": { + "max": 7108, + "p50": 3523, + "p95": 6398, + "p99": 7105 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 1000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 1000, + "totalDurationMs": 9125 + }, + "tasksPerSec": 110, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001590097, + "outcomes": { + "success": 1000 + }, + "run": { + "_creationTime": 1789001580003.0884, + "_id": "jd75cs59dzckahcq402abk0rsn8e4jam", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 1000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001580003, + "taskCount": 1000 + }, + "uniqueWorkIds": 1000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 10, + "executionMs": [ + 102.256468, 90.22437699999999, 126.935154, 106.444737, 163.324722, + 100.560558, 166.375372, 88.157077, 102.012157, 152.350219 + ], + "userExecutionMs": [ + 15.070493, 12.594615, 14.485723, 14.093097, 14.323967999999999, + 18.29839, 19.104835, 13.783182, 14.703303, 16.267684000000003 + ], + "readDocuments": 1026, + "writeDocuments": 1012, + "readBytes": 520513, + "writeBytes": 601204, + "ioReadBytes": 520513, + "ioWriteBytes": 601204, + "writeIndexRows": 2045, + "occExecutions": 0 + }, + "loop": { + "executions": 21, + "executionMs": [ + 169.41288500000002, 283.329299, 482.946567, 477.57336, 488.682302, + 474.298386, 473.838522, 471.999394, 486.710244, 489.98225099999996, + 511.32451599999996, 476.27618900000004, 494.446724, 529.446847, + 518.448467, 709.907095, 372.189023, 216.886928, 101.706722, + 83.66124699999999, 73.824938 + ], + "userExecutionMs": [ + 3.155515, 4.7468520000000005, 9.354273999999998, 5.197178, 9.926999, + 6.490569, 4.978501, 9.632746, 6.402023, 6.826886, 4.466902, 6.4225, + 7.474095, 6.2977240000000005, 6.2664349999999995, 5.950658, + 5.676851, 4.012886, 2.355003, 2.407838, 2.257145 + ], + "readDocuments": 3277, + "writeDocuments": 1064, + "readBytes": 1817625, + "writeBytes": 250382, + "ioReadBytes": 1817625, + "ioWriteBytes": 250382, + "writeIndexRows": 4178, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-1-baseline", + "phase": "warmup", + "rep": 1, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T00:54:09.674533+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 11920, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 39499, + "p50": 18678, + "p95": 37508, + "p99": 39457 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 51253 + }, + "tasksPerSec": 98, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001703348, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789001650248.183, + "_id": "jd73zh9mfbtj383b0vvn5rcpe58e47b2", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001650248, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 108.937438, 81.26673, 125.970114, 121.06743, 129.281131, 82.683671, + 122.839564, 138.872464, 83.604654, 98.875017, 93.86013899999999, + 89.708177, 167.445974, 92.449696, 110.28455600000001, 107.014062, + 86.499467, 147.68759, 97.974932, 121.002061, 90.306771, 88.067987, + 143.979427, 83.46333200000001, 109.575186, 91.677197, 107.820783, + 145.23382800000002, 92.50916500000001, 116.120891, 98.440678, + 78.468425, 101.149271, 89.638484, 100.347252, 129.878995, 87.926, + 87.685045, 161.77462400000002, 83.195678, 131.510997, 88.704168, + 94.518621, 114.30066699999999, 80.250577, 77.11198599999999, + 151.371704, 92.25774100000001, 131.864065, 117.669423 + ], + "userExecutionMs": [ + 19.394339, 13.795031, 19.649265999999997, 20.422331, + 17.540718000000002, 14.140902, 16.187202000000003, 17.751511, + 13.559839, 17.007023, 14.124716000000001, 13.047486000000001, + 16.680325, 14.605343, 15.393331, 15.015285, 13.764868, 17.965683, + 16.458335, 21.022264999999997, 14.488802999999999, 14.272803, + 16.354351, 14.059627, 15.632423, 14.776451, 17.604709, + 18.190475000000003, 18.866746, 13.992573, 15.327185, 14.070454, + 13.530655000000001, 14.306930000000001, 15.470630000000002, + 15.464702, 15.166262, 14.420346, 16.64993, 16.533005, 15.593335, + 16.109497, 14.010368, 14.434729, 14.709709, 13.522291000000001, + 13.318486, 18.176182999999998, 16.270153999999998, 14.970212 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4155604, + "ioReadBytes": 23953, + "ioWriteBytes": 4155604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 85, + "executionMs": [ + 228.10124100000002, 317.297765, 506.477936, 511.06241500000004, + 499.13577399999997, 519.7267330000001, 550.5378119999999, + 513.399748, 501.419932, 514.558528, 573.907753, 510.05264500000004, + 517.827831, 625.041496, 528.860015, 532.310558, 563.998002, + 505.41834600000004, 517.4720149999999, 631.975329, 525.387039, + 531.877237, 534.734135, 540.4845799999999, 555.434203, 521.246232, + 547.1296150000001, 540.892467, 548.638584, 624.2038180000001, + 562.244839, 545.340702, 551.58192, 552.169585, 558.9102969999999, + 617.748335, 577.640212, 661.942394, 564.537914, 594.742481, + 547.6572610000001, 565.043669, 551.595064, 568.8849660000001, + 573.0390600000001, 604.4622710000001, 587.2537629999999, 571.366846, + 578.885529, 564.181722, 597.1057050000001, 579.147583, + 585.3147680000001, 572.700535, 604.815884, 619.0798910000001, + 576.196279, 608.962521, 605.863954, 622.611672, 691.940144, + 624.320111, 603.3898519999999, 811.95619, 590.596177, 612.546789, + 605.2427889999999, 589.113054, 598.051419, 595.973198, 609.560704, + 705.157927, 597.084053, 626.94132, 614.259725, 614.582098, + 753.581362, 588.0397459999999, 323.37907, 320.029292, + 157.45346800000002, 103.77350100000001, 59.835279, 100.96329, + 68.624396 + ], + "userExecutionMs": [ + 7.214928, 10.42742, 9.065306, 6.684639, 6.37697, 7.257033000000001, + 6.3846, 6.204071, 6.129919, 7.091914, 6.187643, 5.910895, + 4.4213890000000005, 10.380096, 6.972003, 6.729569000000001, + 9.158049, 6.1519390000000005, 6.5895980000000005, 6.426113, 4.53364, + 6.201307, 6.82849, 6.093515, 6.218192, 6.3678349999999995, 5.056927, + 8.392932, 6.680028999999999, 6.905934, 6.344997, 6.98798, 6.259388, + 6.313353, 6.559051, 6.477479000000001, 6.781631, 6.632969, + 6.7755860000000006, 6.16359, 7.607547, 6.411884, 6.650991, 7.175428, + 4.58231, 5.62175, 4.810641, 6.367006, 4.300161999999999, + 4.277076999999999, 6.565732000000001, 6.628393, 5.89926, 4.137719, + 6.721418, 6.76226, 5.018289, 4.560114, 6.3574139999999995, 4.428597, + 8.651086000000001, 6.155543000000001, 7.951645999999999, 4.719585, + 4.194039, 6.439235, 6.31985, 4.2025, 5.305852, 5.507669, 4.835287, + 6.631306, 6.091813, 6.24978, 4.857982000000001, 6.207564, 5.151515, + 4.3423430000000005, 4.162154, 3.630443, 2.724231, 2.406089, + 2.668133, 2.0423839999999998, 2.418425 + ], + "readDocuments": 30877, + "writeDocuments": 10166, + "readBytes": 10916661, + "writeBytes": 1420957, + "ioReadBytes": 10916661, + "ioWriteBytes": 1420957, + "writeIndexRows": 40332, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-1-ordering", + "phase": "warmup", + "rep": 1, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T00:55:14.226706+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 12797, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 35653, + "p50": 18137, + "p95": 34015, + "p99": 35279 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 48281 + }, + "tasksPerSec": 104, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001764358, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789001714787.097, + "_id": "jd7afy9ad0sdmr2hseb25x282x8e5p2q", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001714787, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 113.305194, 99.378019, 193.12479399999998, 122.18485199999999, + 157.67314900000002, 105.055194, 143.612117, 99.06509700000001, + 144.33628000000002, 116.105305, 103.40384499999999, 168.69418, + 105.351877, 132.01487500000002, 101.050151, 155.609836, 102.047949, + 102.049385, 183.152447, 106.946498, 147.054366, 95.60094199999999, + 131.669667, 152.882192, 100.1489, 177.460929, 112.045517, + 149.955069, 114.890247, 92.94609299999999, 169.97820099999998, + 106.657256, 143.993665, 111.841625, 105.50641900000001, 185.522043, + 102.21161400000001, 138.08798099999998, 123.89354499999999, + 98.86664400000001, 153.107214, 109.84405, 145.885814, 97.86668, + 149.74684299999998, 119.243821, 102.17514299999999, 155.453269, + 95.792569, 124.884813 + ], + "userExecutionMs": [ + 21.211414, 16.395375, 21.540148, 16.460128, 21.500455000000002, + 22.487678, 18.787132000000003, 17.78068, 16.757122, + 16.661203999999998, 18.642250999999998, 18.005994, + 16.149859999999997, 17.597499, 17.531905000000002, + 19.682152000000002, 16.77326, 17.246000000000002, + 20.857988000000002, 20.557163, 16.096415999999998, + 17.030988999999998, 17.705037, 21.609221, 16.349657, 17.496369, + 16.456069, 18.05772, 17.770888, 15.570609, 16.819217000000002, + 17.819218000000003, 19.123364, 16.420830000000002, 17.31005, + 17.485899999999997, 19.755324, 20.601952999999998, 24.024611, + 16.781535, 16.665684, 18.438207000000002, 21.171298, 17.05541, + 17.592037, 16.876385, 18.259001, 15.898942, 17.465283000000003, + 17.078758 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2598953, + "writeBytes": 4275604, + "ioReadBytes": 2598953, + "ioWriteBytes": 4275604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 86, + "executionMs": [ + 202.320367, 350.836489, 518.133995, 511.28871799999996, 524.600866, + 503.660655, 511.576279, 514.6874799999999, 507.52583699999997, + 514.9040590000001, 528.875812, 571.23971, 529.926885, 532.985977, + 522.324397, 595.241228, 539.587025, 536.893062, 526.985862, + 504.94989499999997, 506.70853999999997, 522.737386, 526.548841, + 521.7965300000001, 510.783383, 494.827373, 499.871962, + 499.92319699999996, 495.691795, 508.778995, 508.448802, 536.311876, + 493.715127, 524.436346, 546.776546, 548.990903, 500.457858, + 522.759109, 650.535761, 545.857655, 507.95220500000005, 510.850488, + 508.86243900000005, 521.884106, 513.939093, 500.141991, + 503.09358199999997, 570.038083, 541.773901, 528.622103, 492.511081, + 564.103156, 470.481824, 495.16995099999997, 496.25162, 487.710663, + 513.434151, 522.9411940000001, 490.705536, 509.099243, 500.147907, + 539.5049389999999, 488.159476, 507.42855499999996, 567.745729, + 476.379803, 501.06939100000005, 557.134838, 501.08054999999996, + 497.03148699999997, 519.05459, 812.3628389999999, 515.7557889999999, + 531.40628, 498.025551, 500.970036, 494.622409, 514.908527, + 458.525417, 291.950345, 121.854805, 88.66273299999999, 71.21491, + 80.53110500000001, 102.76195100000001, 75.286332 + ], + "userExecutionMs": [ + 3.182903, 6.937189, 6.093156, 6.011431, 6.7706420000000005, + 6.633379, 6.458988, 6.330479, 5.733578, 6.281172, + 6.1775459999999995, 6.395806, 6.387663, 6.3692899999999995, + 6.696898, 6.167049, 7.319102, 4.826268, 4.513883, 6.124557, + 4.259619, 6.216029, 6.309368999999999, 7.274888, 6.230925, 6.504692, + 6.8491539999999995, 4.303972, 6.3439380000000005, 6.046595999999999, + 4.348683, 6.262897, 4.830087000000001, 6.258025, 9.405992, 4.507243, + 6.151596, 4.747049, 6.422256, 9.155206000000002, 8.401851, + 6.7350389999999996, 4.669713, 4.585185, 4.302638, 6.895906999999999, + 5.068941000000001, 6.770919, 6.380580999999999, 6.40815, 5.608218, + 6.709297, 6.917161, 9.160216, 4.828753, 6.609042, 7.748408, + 6.801325, 6.148463, 6.2462860000000004, 6.151559, 6.074583, 6.3706, + 4.917596, 5.432185, 4.649628, 4.757999, 6.443288, 6.084751, + 5.651959, 6.050034, 6.158363, 4.3960989999999995, 4.819058999999999, + 4.285514, 6.29447, 6.001684, 5.025571, 4.347219, 5.247055, 2.596945, + 3.728109, 2.128155, 2.324296, 2.398168, 5.547218 + ], + "readDocuments": 30959, + "writeDocuments": 10167, + "readBytes": 10222296, + "writeBytes": 1103190, + "ioReadBytes": 10222296, + "ioWriteBytes": 1103190, + "writeIndexRows": 40334, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-1-packed", + "phase": "warmup", + "rep": 1, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T00:56:14.489893+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 10984, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 34514, + "p50": 17525, + "p95": 32895, + "p99": 34174 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 45313 + }, + "tasksPerSec": 110, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001821915, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789001775069.0696, + "_id": "jd7bbxmwyvrkzqn694w5w317rh8e5pkh", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001775069, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 133.406264, 84.322418, 133.744867, 152.413425, 160.273065, + 89.643514, 109.22874, 155.42710200000002, 81.913398, 135.188794, + 106.7341, 82.619574, 134.171558, 80.41448, 87.603856, 163.693152, + 79.93922400000001, 143.441924, 109.126556, 88.03666, + 126.57454400000002, 95.453233, 115.59049, 168.13563200000002, + 89.777574, 148.934167, 83.480356, 95.204203, 160.605527, 82.14549, + 157.489484, 111.13133499999999, 83.94992699999999, + 143.23967100000002, 98.454037, 81.963536, 167.046661, 86.492041, + 78.799642, 168.553897, 85.83888400000001, 120.39480499999999, + 92.172368, 93.90259099999999, 159.722582, 92.34129100000001, + 124.94200900000001, 112.34490699999999, 89.736073, 133.742892 + ], + "userExecutionMs": [ + 19.39544, 12.547857, 14.364264, 19.669296, 19.333928, 21.462023, + 14.237213, 16.851251, 12.762088, 14.243598, 16.340028, + 13.081328999999998, 18.580053, 13.940824, 17.150886, 18.220106, + 13.352722, 14.830315, 13.666305, 13.894696999999999, 14.163709, + 13.710817, 18.082261, 13.696747, 14.466226, 14.549266, 14.40265, + 12.480712, 14.471929999999999, 14.04818, 15.994385, 13.326494, + 13.214988, 14.767067, 13.452688, 11.790905, 14.131924, 13.843305, + 13.630155, 14.991621, 13.726857, 12.524363000000001, + 14.412652999999999, 14.028096999999999, 17.225598, 13.865237, + 13.197415000000001, 14.1281, 12.67307, 12.301934000000001 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2598953, + "writeBytes": 3003604, + "ioReadBytes": 2598953, + "ioWriteBytes": 3003604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 162.789563, 342.652219, 523.5336040000001, 503.129195, + 508.60031100000003, 527.393584, 498.19523399999997, 499.426358, + 475.466582, 492.384208, 465.940888, 465.237845, 511.301715, + 490.602728, 533.65887, 483.380441, 465.003519, 478.368648, + 528.7016719999999, 558.420236, 496.069127, 484.160931, + 474.32597599999997, 471.90408099999996, 483.796811, + 485.06936099999996, 482.961889, 509.70680000000004, + 506.71227200000004, 477.38506800000005, 514.936411, + 477.77254800000003, 539.64269, 464.860391, 497.62863000000004, + 540.797471, 556.191251, 468.224642, 468.885725, 445.85021800000004, + 487.783046, 455.230445, 515.740757, 490.11339, 493.796142, + 519.859005, 497.77168900000004, 495.47935, 493.730404, 561.347863, + 602.618468, 514.0571219999999, 530.4160380000001, 473.220797, + 453.363117, 451.96202800000003, 475.12787099999997, + 490.07320999999996, 485.31489300000004, 528.5595069999999, + 659.791227, 484.267742, 508.42106099999995, 481.72413400000005, + 491.263341, 475.023347, 486.488498, 498.60008700000003, 462.327507, + 461.44136299999997, 492.883593, 469.03409600000003, + 448.19007100000005, 488.752308, 479.008634, 460.747074, 471.611071, + 491.850505, 473.096935, 266.661678, 126.171484, 73.908096, + 61.717461, 72.188423, 64.929419, 97.837723, 72.398553 + ], + "userExecutionMs": [ + 3.013975, 3.659872, 4.575349, 6.479053, 6.1739120000000005, + 6.113121, 4.9944500000000005, 5.038079, 8.015234, 6.042311, + 6.047623000000001, 5.98339, 4.7321990000000005, 6.2136190000000004, + 5.229467, 6.2126, 5.182024, 6.329487, 4.86435, 6.375571, 4.554939, + 4.991878, 6.14643, 6.17708, 4.438468, 6.533715, 6.6252, 6.489408, + 4.93893, 6.137669, 4.908286, 6.22329, 6.726165999999999, 4.986076, + 4.91133, 5.898593, 6.0670790000000006, 4.448031, 6.433859, 5.483361, + 7.948693, 6.577252, 6.1311279999999995, 9.992282999999999, 4.379957, + 6.697623, 6.260198, 7.018675, 4.895791, 4.874535, 5.213397, + 6.113508, 4.309386, 4.7745299999999995, 6.678929999999999, + 6.109357999999999, 5.614882, 5.245444, 6.924958, 8.948268, + 6.079613999999999, 4.309977000000001, 6.592784, 4.907951, + 5.7109250000000005, 4.8729059999999995, 6.736559, 4.854288, + 4.3500179999999995, 6.403881999999999, 4.865854000000001, 4.29655, + 4.451626999999999, 5.469836, 9.284495999999999, 6.141701, 5.922655, + 6.029294999999999, 4.701262000000001, 5.957596, 3.018513, 2.419425, + 3.49083, 2.1563820000000002, 2.776174, 2.144537, 2.533291 + ], + "readDocuments": 16231, + "writeDocuments": 5293, + "readBytes": 9070471, + "writeBytes": 1249248, + "ioReadBytes": 9070471, + "ioWriteBytes": 1249248, + "writeIndexRows": 20836, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-2-ordering", + "phase": "warmup", + "rep": 2, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T00:57:12.637636+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 12549, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 34679, + "p50": 17839, + "p95": 33011, + "p99": 34362 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 47055 + }, + "tasksPerSec": 106, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001881939, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789001833356.4326, + "_id": "jd736kvfp51qcy9kmmtjk772518e5gsy", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001833356, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 111.251988, 84.568354, 140.401709, 118.935542, 154.78655, 92.103603, + 146.37894, 108.523892, 89.879301, 142.08303899999999, 95.696282, + 127.07015599999998, 97.019103, 90.080094, 164.605267, 96.664066, + 164.415806, 103.44344, 147.915657, 107.921418, 99.409327, + 156.949881, 90.53235099999999, 130.327538, 95.43407499999999, + 148.149852, 106.570421, 96.36447, 170.127492, 96.508557, + 171.87012099999998, 94.155248, 142.62945699999997, + 125.17389700000001, 135.40376500000002, 133.732798, 92.850683, + 174.63279300000002, 99.531252, 135.817083, 97.56139399999999, + 126.37902700000001, 140.49426100000002, 101.570008, 177.623514, + 99.767326, 125.359667, 95.40508799999999, 105.877955, 158.36181 + ], + "userExecutionMs": [ + 18.026151000000002, 15.829218, 17.690656999999998, + 17.603089999999998, 16.885759, 16.608377, 18.877541, + 21.509912999999997, 16.476405, 17.681754, 16.139836000000003, + 17.206455, 16.400178, 15.624805, 19.711347, 21.700115, 17.154511, + 17.882254, 18.201563, 20.574665, 16.435084999999997, 16.733109, + 16.340062999999997, 17.82845, 15.278551, 17.709188, 17.274681, + 15.970939999999999, 16.912076000000003, 16.446375999999997, + 18.696979, 15.458150999999999, 17.574032, 16.692591999999998, + 16.595036, 19.739047, 15.870988999999998, 21.808073, 15.429264, + 19.194840000000003, 17.670293, 16.161443000000002, + 18.380945999999998, 18.256674, 20.68536, 15.980355000000001, + 15.982878, 16.49269, 17.694566, 18.728351 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2598953, + "writeBytes": 4275604, + "ioReadBytes": 2598953, + "ioWriteBytes": 4275604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 228.142384, 325.380068, 498.879146, 514.72901, 530.42, 482.679521, + 502.383326, 544.256027, 515.69997, 506.40834, 517.43574, + 508.01372499999997, 520.268768, 508.248288, 502.962687, + 501.71932100000004, 499.93474100000003, 493.430528, + 500.95669799999996, 515.882214, 485.63344600000005, 496.361446, + 635.818796, 497.134387, 534.43418, 514.428867, 546.898656, + 525.000941, 510.655081, 510.84537900000004, 522.783489, 493.185318, + 521.748678, 515.96067, 551.216013, 499.748672, 499.37832399999996, + 561.252031, 501.03188, 515.130589, 534.9178380000001, 496.278657, + 485.770518, 487.850896, 561.0363619999999, 475.109132, 491.968682, + 507.270954, 497.735276, 509.815685, 511.64078199999994, 537.540735, + 531.106831, 499.012749, 501.21989499999995, 489.528795, + 506.12277500000005, 497.48719800000003, 509.882194, + 508.55043700000004, 506.714247, 531.215334, 488.37822800000004, + 489.53580200000005, 507.17101799999995, 484.04542, 517.670411, + 502.971278, 572.897263, 510.96141100000006, 481.400564, 521.602685, + 517.615356, 511.98084700000004, 526.414819, 523.592515, + 511.74081000000007, 509.41893400000004, 450.39500300000003, + 266.489944, 120.702639, 79.526465, 79.7618, 76.593171, + 80.88319800000001, 67.855689, 71.91990700000001 + ], + "userExecutionMs": [ + 3.07772, 3.869158, 4.768581, 9.305993, 5.950277000000001, 4.397093, + 5.377323, 4.743711, 5.568765, 4.229644, 5.576573, 5.179584, + 6.909337, 10.610691000000001, 7.263412000000001, 4.40913, 6.533238, + 5.2532309999999995, 5.988201999999999, 4.396073, 8.863667000000001, + 4.461668, 5.5549550000000005, 4.3399600000000005, 4.880362, + 4.674618000000001, 6.268226, 5.0136199999999995, 6.196567, 5.78382, + 5.325435, 6.852572, 4.849409, 4.86196, 4.50435, 4.365671, 6.135992, + 4.279426, 6.09271, 5.305191, 4.890246, 6.307428000000001, 6.542035, + 5.107083, 6.604533, 6.224604, 6.339335, 4.835315, 6.012817, + 7.8037920000000005, 5.705334000000001, 6.7482310000000005, 6.725337, + 5.748844, 12.96125, 4.970726, 4.617051, 6.073559, 4.804881000000001, + 7.384413, 5.898599, 4.996874, 4.908202, 5.303867, 5.547516, + 5.999354, 6.911851, 5.719309, 5.996763, 6.843042, 4.373513, + 4.846462, 4.630107, 4.778294, 4.771346, 4.7261120000000005, + 5.237269, 6.1955480000000005, 5.087915, 3.570613, 2.5978, 2.459167, + 2.093325, 2.7865189999999997, 2.638713, 2.236778, 2.295816 + ], + "readDocuments": 30966, + "writeDocuments": 10168, + "readBytes": 10223919, + "writeBytes": 1103404, + "ioReadBytes": 10223919, + "ioWriteBytes": 1103404, + "writeIndexRows": 40336, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-2-packed", + "phase": "warmup", + "rep": 2, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T00:58:12.320674+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 10765, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 36735, + "p50": 17797, + "p95": 33581, + "p99": 35761 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 47357 + }, + "tasksPerSec": 106, + "timedOut": false + }, + "evidence": { + "observedAt": 1789001941751, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789001892923.9758, + "_id": "jd7fx2x7bbxkz39bwydvs94by18e5v2a", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001892924, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 94.232023, 81.77361900000001, 112.068006, 90.08948, 155.383668, + 92.693951, 130.163324, 93.29458, 82.073657, 156.496048, 86.839146, + 126.670951, 110.44763, 84.24820500000001, 145.357906, 87.799054, + 141.326336, 109.448093, 92.87659400000001, 149.449413, + 84.92330100000001, 138.227654, 117.78567299999999, 90.885485, + 128.070417, 86.608119, 100.04892099999999, 153.74956500000002, + 81.340117, 101.853104, 105.26131, 92.452143, 169.544004, + 86.73679800000001, 136.628152, 99.829191, 91.826874, 185.150483, + 78.67702899999999, 127.685874, 90.350316, 92.99524600000001, + 175.717531, 87.70091500000001, 116.173535, 125.121344, + 92.42446799999999, 158.972128, 88.272382, 128.020514 + ], + "userExecutionMs": [ + 13.462693, 12.029866, 13.872116, 13.853847, 14.600603999999999, + 13.093091, 17.047687999999997, 12.007164000000001, 11.781279, + 14.401667999999999, 14.219387, 13.881713, 14.713298, + 12.241691000000001, 17.628906, 13.069872, 16.902466, 13.969199, + 13.538507, 14.939896, 13.132067000000001, 13.928576999999999, + 14.391585000000001, 15.341805, 15.201525, 14.644703999999999, + 12.289791, 14.141852, 13.537656, 14.331894, 16.453965, 14.455354, + 14.863159000000001, 13.626328, 13.154997, 17.860602, 15.653816, + 14.785434, 14.386885, 17.139805000000003, 13.861781, 14.718843, + 14.243694, 14.735608000000001, 13.428321, 19.305014999999997, + 11.91993, 13.976506, 15.062713, 14.297611999999999 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2598953, + "writeBytes": 3003604, + "ioReadBytes": 2598953, + "ioWriteBytes": 3003604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 147.384189, 257.336707, 467.854174, 480.45144600000003, 495.603789, + 468.271298, 477.887566, 465.558332, 458.861311, 494.632927, + 484.347601, 478.409483, 473.06400199999996, 462.97105799999997, + 483.889519, 480.670791, 469.655657, 459.94932300000005, 481.931985, + 457.11827999999997, 459.949668, 467.33725999999996, + 468.46500000000003, 498.428621, 503.725866, 484.624913, 486.83434, + 493.699415, 645.093081, 514.426146, 506.102316, 506.166676, + 505.459219, 472.16930299999996, 525.2895279999999, 527.960418, + 510.12880000000007, 519.280305, 519.9542200000001, + 633.7276880000001, 799.999026, 613.994248, 472.140224, 534.627557, + 543.252327, 551.415356, 543.302886, 456.141768, 483.143329, + 471.68305100000003, 475.689129, 523.269734, 463.842914, 464.607618, + 498.944715, 457.92175699999996, 479.74168099999997, 449.264142, + 461.238322, 514.4863389999999, 482.89879, 453.493018, 483.520792, + 504.86231699999996, 483.339233, 473.634893, 471.374751, 478.33827, + 478.230711, 475.94966300000004, 546.505829, 479.042133, 477.999446, + 496.76199199999996, 500.467283, 488.413154, 517.5281269999999, + 822.290684, 907.5035879999999, 470.336453, 118.960576, + 78.27359100000001, 63.842063, 68.27841699999999, 67.188354, + 82.97353, 107.354249 + ], + "userExecutionMs": [ + 3.301596, 3.334606, 4.325401, 5.305692, 5.30981, 6.737109, 4.403673, + 4.898522, 4.300462, 4.7586770000000005, 4.360862, 5.038601, + 5.075824, 6.160317999999999, 4.912928000000001, 5.383382, 6.984013, + 6.143181, 4.894259, 5.898689, 4.3339240000000006, 5.036526, + 6.557124999999999, 5.962276, 5.50856, 6.068754, 5.32028, 8.123495, + 4.589262000000001, 4.875964, 5.51222, 5.274807, 5.957956, + 6.282916999999999, 4.671004, 4.840304, 5.643046, 4.458575000000001, + 5.418399, 5.147225, 6.603046, 6.065386, 6.770365, 5.442695, + 6.927498, 6.19773, 4.835800000000001, 7.509531, 8.368034, 4.847077, + 6.125442, 5.960466, 5.112585999999999, 5.230824, 5.178414999999999, + 5.575074, 5.732387999999999, 4.809535, 4.488585, 6.12101, 4.88366, + 4.790102, 8.861964, 7.989872, 6.648039, 5.228625, 6.533557, + 7.869866, 4.7711120000000005, 5.184317, 6.486687, 4.314385, + 6.366069, 4.817318, 4.751754, 7.119172, 5.265586, 5.479323, + 5.542044000000001, 8.026438, 2.5461970000000003, 2.348737, 2.218304, + 2.286671, 2.154126, 2.240561, 2.3585979999999998 + ], + "readDocuments": 16231, + "writeDocuments": 5293, + "readBytes": 9070471, + "writeBytes": 1249248, + "ioReadBytes": 9070471, + "ioWriteBytes": 1249248, + "writeIndexRows": 20836, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-2-baseline", + "phase": "warmup", + "rep": 2, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T00:59:14.510984+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 12057, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 41255, + "p50": 18561, + "p95": 38980, + "p99": 41214 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 53217 + }, + "tasksPerSec": 94, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002010038, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789001955083.2478, + "_id": "jd75ndrvcxb705ezftxtgyvc198e4cp7", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789001955083, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 104.045447, 99.479906, 113.672742, 113.23938799999999, 94.200843, + 79.279191, 74.43332699999999, 128.043044, 80.151388, + 151.07489199999998, 107.51746299999999, 125.58812499999999, + 81.307575, 142.69807500000002, 85.939027, 132.56262999999998, + 79.293378, 94.537863, 154.04960400000002, 95.309491, 89.373021, + 83.254074, 75.537987, 135.433418, 86.550068, 142.770951, 92.058419, + 97.10423, 100.063517, 86.865082, 77.603179, 106.842645, + 76.07394400000001, 82.467849, 198.529682, 86.901988, 120.958609, + 90.219283, 79.738259, 157.51762399999998, 74.117091, + 128.69554499999998, 95.17378, 85.905199, 126.45829499999999, + 90.67417499999999, 110.355726, 120.58083599999999, + 77.31763799999999, 122.852497 + ], + "userExecutionMs": [ + 16.01095, 13.397485000000001, 19.383926, 19.21178, + 18.873579000000003, 15.588906, 15.286443, 15.556319, 15.176209, + 16.709637, 17.880083000000003, 14.552446, 16.476418, 15.800849, + 17.595104, 18.126407, 17.280888, 15.289575999999999, 15.491707, + 15.171591999999999, 14.076852, 14.975912, 13.462793000000001, + 14.336336, 18.413351000000002, 14.266156, 17.837815, + 15.556092000000001, 15.872786999999999, 13.590005, 13.640045, + 14.95499, 14.250786, 13.489424, 15.765643999999998, + 13.523914999999999, 15.566992, 15.093575, 15.053063, + 14.577364000000001, 15.797449000000002, 14.682212999999999, + 19.745485, 13.037952, 16.383367, 16.660616, 15.705796000000001, + 16.377613999999998, 13.479919, 15.538825 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4155604, + "ioReadBytes": 23953, + "ioWriteBytes": 4155604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 84, + "executionMs": [ + 242.06398900000002, 294.482968, 648.283449, 502.857672, 806.861007, + 510.08031099999994, 529.0865620000001, 534.82895, 501.074895, + 492.090558, 614.132011, 557.109132, 507.49751799999996, 555.931333, + 479.318445, 554.767613, 514.079619, 515.587907, 531.553386, + 551.915573, 528.474093, 575.2730190000001, 580.8669560000001, + 548.096357, 508.39978500000007, 502.267848, 510.046974, 562.548347, + 524.378631, 556.450758, 560.3270070000001, 540.980284, + 530.9277989999999, 553.2752370000001, 517.084149, 530.989344, + 561.5696829999999, 567.1338030000001, 518.865605, 586.565284, + 562.668772, 554.330823, 574.7393920000001, 614.517529, 601.35527, + 814.630246, 645.534562, 738.5805389999999, 577.114051, 621.496153, + 786.6107850000001, 607.2883850000001, 633.278198, 845.64615, + 565.403957, 623.8843830000001, 593.977935, 591.067967, 641.62281, + 599.896418, 599.362242, 601.0354249999999, 570.193626, 609.946106, + 618.7704550000001, 577.3621129999999, 676.310206, 663.373993, + 682.759035, 679.027856, 636.127539, 668.0418420000001, 623.3746, + 606.077848, 729.171114, 709.4883050000001, 688.063713, + 594.8791530000001, 352.90777, 372.481115, 153.71832999999998, + 100.13167800000001, 52.879919, 104.52724500000001 + ], + "userExecutionMs": [ + 3.143656, 6.953585, 157.368582, 6.713661999999999, 4.803662, + 7.8550960000000005, 6.016812, 6.526222, 8.271341999999999, + 4.8001629999999995, 6.146049, 6.629375, 9.091477000000001, 5.499171, + 6.1818029999999995, 8.291311, 5.927314, 5.357197, 5.997828999999999, + 6.639977, 6.435689, 6.244596, 6.164134, 6.1098419999999996, + 6.915812000000001, 6.033467, 4.949489, 4.382022, 4.872935999999999, + 6.3568549999999995, 5.995844999999999, 6.110413, 5.979122, 8.213564, + 6.034196000000001, 5.0789040000000005, 8.516235, 6.512241, + 5.0758540000000005, 5.952668, 4.600787, 5.990885, 8.537697, + 8.740749000000001, 9.712017, 7.338259, 5.035501, 4.923046, 5.439686, + 5.8488180000000005, 4.687581, 5.116825, 5.906593, 4.409445, + 5.565507, 5.441256, 6.202903, 4.838737, 6.13788, 5.943035999999999, + 4.2158880000000005, 4.961839, 6.610993, 4.3809249999999995, + 6.1001780000000005, 4.804777, 5.016332, 4.804712, 5.886945, + 4.277973, 7.484566, 4.744751, 5.984101000000001, 4.2499340000000005, + 8.438936, 6.707467, 5.805699, 9.263656000000001, 5.7697579999999995, + 6.110366, 2.576997, 2.5085110000000004, 2.458835, 2.170398 + ], + "readDocuments": 30870, + "writeDocuments": 10165, + "readBytes": 10915033, + "writeBytes": 1420743, + "ioReadBytes": 10915033, + "ioWriteBytes": 1420743, + "writeIndexRows": 40330, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-3-packed", + "phase": "warmup", + "rep": 3, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T01:00:22.200240+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 11514, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 39812, + "p50": 18390, + "p95": 36730, + "p99": 39197 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 51244 + }, + "tasksPerSec": 98, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002075812, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002022754.1309, + "_id": "jd73agt6frfnhv8sg681ery5258e4bcm", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002022754, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 93.434977, 83.042462, 154.65127, 100.15652999999999, 170.620737, + 87.187334, 166.781771, 101.342682, 82.741405, 166.172593, + 87.71807299999999, 111.632251, 132.07954, 99.656069, 131.476122, + 94.292495, 94.683297, 126.49663199999999, 85.468901, + 105.41672200000001, 198.899533, 92.702446, 131.249599, 152.931089, + 87.473597, 137.173466, 123.58930699999999, 96.34461599999999, + 146.238794, 133.78341, 92.909367, 122.610979, 106.78120000000001, + 90.34647799999999, 113.617189, 85.256917, 79.145461, + 127.64988899999999, 93.35230299999999, 86.374246, 188.140436, + 80.11926, 132.928533, 101.469498, 86.249563, 186.20028, 100.226059, + 105.849993, 159.871839, 86.117044 + ], + "userExecutionMs": [ + 16.794213999999997, 11.871107, 19.134460999999998, 14.378874, + 14.18466, 14.702934, 13.906127000000001, 16.741598, + 13.437743000000001, 16.367836, 14.045776, 14.152645, + 13.270629000000001, 15.461431, 15.83762, 14.03553, 12.600343, + 13.274517, 13.434085, 12.221466, 14.643116, 13.445132000000001, + 14.103945, 14.677054, 17.005979, 15.596122000000001, 15.352506, + 14.302641, 12.242202, 17.545536000000002, 12.668565, 14.643262, + 16.798372, 18.362422, 16.978786, 13.987273, 12.579421, 12.798645, + 12.652555, 12.264811, 12.020314, 12.885556, 13.489021000000001, + 12.334138999999999, 13.538071, 14.424743, 13.37997, 15.738664, + 15.8502, 13.822187 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2598953, + "writeBytes": 3003604, + "ioReadBytes": 2598953, + "ioWriteBytes": 3003604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 153.516318, 286.64486300000004, 495.689188, 533.426379, 522.122842, + 476.318415, 546.0920500000001, 545.095598, 604.814893, + 567.8117179999999, 539.052118, 628.91698, 604.469127, 548.307458, + 495.645373, 508.85252199999996, 483.364458, 546.755348, 520.76491, + 492.590523, 546.5974930000001, 483.267903, 459.069849, 442.908754, + 534.201422, 455.92166699999996, 497.49959, 496.251532, + 493.07462400000003, 510.155201, 482.033981, 525.2715730000001, + 501.71714999999995, 448.525861, 555.087095, 493.126234, + 521.9017249999999, 517.464124, 524.693361, 524.4847729999999, + 478.364382, 651.979203, 709.554672, 701.74609, 617.0529449999999, + 520.359167, 580.261179, 619.053128, 580.818854, 522.122317, + 490.385917, 491.285525, 574.4681579999999, 792.035834, 483.389906, + 560.420221, 530.7245499999999, 531.685164, 560.860697, + 507.73289099999994, 497.01590899999997, 479.895375, 548.459941, + 521.785537, 498.943586, 492.85062400000004, 495.16810100000004, + 467.843913, 533.3463670000001, 533.282959, 553.573122, 494.677797, + 713.879328, 670.582843, 779.4185729999999, 768.7506149999999, + 771.7284629999999, 887.875087, 713.935154, 465.667007, 222.054584, + 94.129255, 113.453842, 66.581211, 78.159041, 72.90557, 81.35025 + ], + "userExecutionMs": [ + 3.141222, 6.981631, 8.649367999999999, 6.436834, 6.334879, + 10.739288, 9.219268, 6.970587999999999, 6.068917, 6.227782, + 7.3811160000000005, 5.959804, 6.171486000000001, 5.929683, 6.759765, + 4.5120000000000005, 6.117264, 7.175675, 6.817633, 7.061742000000001, + 4.888408, 6.134329, 6.5417380000000005, 8.87258, 5.891686999999999, + 7.022600000000001, 6.403411999999999, 5.932325, 5.868361, 8.266139, + 4.8907359999999995, 6.1471219999999995, 4.333495, 5.978566, + 6.868647999999999, 6.475207999999999, 4.310594, 9.245938, 5.530582, + 4.300139, 6.325551, 4.408437, 6.528243000000001, 6.101818, 6.171023, + 4.5677829999999995, 4.360126, 5.082659, 5.704501, 8.059550999999999, + 6.035559, 8.532547999999998, 6.791987, 6.026315, 5.255906, 4.688148, + 6.076556, 7.832972000000001, 6.800981999999999, 4.880161, 6.736703, + 5.180115, 4.866437, 5.903094, 8.918947, 4.8471649999999995, + 5.910615999999999, 4.855901, 4.459573, 5.9978430000000005, 4.353658, + 8.364179, 4.24652, 6.060656, 6.042946000000001, 5.0568599999999995, + 5.949573, 4.88361, 6.258291, 5.346916, 4.20219, 2.3301089999999998, + 2.26092, 2.312106, 2.155354, 2.284389, 2.3312150000000003 + ], + "readDocuments": 16231, + "writeDocuments": 5293, + "readBytes": 9070471, + "writeBytes": 1249248, + "ioReadBytes": 9070471, + "ioWriteBytes": 1249248, + "writeIndexRows": 20836, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-3-baseline", + "phase": "warmup", + "rep": 3, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T01:01:28.972435+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 15336, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 44967, + "p50": 20957, + "p95": 42182, + "p99": 44237 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 60196 + }, + "tasksPerSec": 83, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002151604, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002089570.4312, + "_id": "jd70ach2y75fs7jd1dspaxrgy98e4tpe", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002089570, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 120.153389, 76.14921700000001, 362.85287, 388.56986, 218.078026, + 83.55198, 134.361807, 248.726475, 110.905296, 91.23498400000001, + 168.81408100000002, 109.087493, 152.607105, 155.045285, + 162.33562400000002, 120.072694, 122.415796, 89.35118, 107.671593, + 136.792408, 91.79032099999999, 177.35242000000002, 82.29031, + 83.77723900000001, 241.710354, 77.013307, 94.921034, 145.017571, + 90.393555, 130.23915399999998, 90.41152500000001, 97.955266, + 132.903719, 79.246093, 128.915454, 92.974028, 94.89038099999999, + 100.463829, 79.046244, 99.5703, 144.76914299999999, + 83.29402499999999, 284.285587, 85.99749700000001, 122.391954, + 96.941215, 91.456585, 216.654795, 104.72852, 81.171982 + ], + "userExecutionMs": [ + 14.288513, 13.674257, 32.629115, 33.697958, 21.823843999999998, + 16.679626, 16.600193, 17.223021000000003, 14.980201000000001, + 15.004577999999999, 25.494276, 17.673726000000002, 19.251961, + 19.006016, 16.181853999999998, 15.018606, 17.80942, 13.804596, + 17.242392, 14.498801, 14.3006, 15.390872, 15.295378, + 15.418757999999999, 17.207612, 13.590105, 14.201134000000001, + 18.072385, 19.277144, 15.747720999999999, 14.574445, 16.062278, + 15.343426000000001, 13.971336, 15.770800000000001, 14.304307, + 19.567629999999998, 18.102646, 15.450569000000002, 21.268397, + 16.802296000000002, 15.378188, 27.890143, 15.805364999999998, + 12.644499, 18.940434, 13.827570000000001, 24.7818, 21.026272, + 18.879282 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4155604, + "ioReadBytes": 23953, + "ioWriteBytes": 4155604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 82, + "executionMs": [ + 277.294258, 279.804438, 532.095111, 516.010798, 1005.9304689999999, + 1021.6860819999999, 793.342857, 835.705092, 709.39854, 602.228984, + 810.573428, 528.522626, 588.25289, 696.576351, 527.685825, + 542.858031, 534.3656119999999, 497.205174, 569.707012, + 690.7256739999999, 520.0205100000001, 629.961017, 582.773831, + 556.4109860000001, 668.355393, 1152.135454, 809.854757, + 732.6541050000001, 544.517142, 698.26647, 539.2181740000001, + 658.877388, 564.3282670000001, 527.160271, 554.597704, + 600.1756409999999, 660.64871, 644.320103, 635.572208, 551.557455, + 551.6700519999999, 548.8604320000001, 533.8589780000001, + 537.4537230000001, 579.168597, 606.384226, 583.924437, 572.938381, + 605.619736, 569.9705280000001, 621.359161, 755.111396, + 694.3052349999999, 766.466445, 639.086857, 599.551594, 626.762997, + 619.684962, 850.512426, 765.8003490000001, 695.769775, 936.754836, + 621.81056, 967.599458, 593.444292, 647.062444, 590.645149, + 641.801268, 730.177362, 634.104734, 596.102243, 851.998029, + 594.1932909999999, 822.111543, 640.1127829999999, 667.107977, + 889.9251409999999, 646.307857, 877.80831, 850.374006, + 250.36788700000002, 97.211291 + ], + "userExecutionMs": [ + 3.651312, 11.295976, 8.600996, 6.7243010000000005, 6.719639, + 14.745526, 4.2452380000000005, 6.5526990000000005, 10.335963, + 6.786702, 6.071499, 6.560537, 8.06911, 5.984332, 7.382129, 8.490232, + 4.3230070000000005, 6.041651, 4.370133, 6.76406, 6.077613, 5.981223, + 7.260250999999999, 6.433433, 6.825634, 5.921175, 4.403492999999999, + 6.379925, 5.977442, 5.978161, 7.85344, 4.8594479999999995, + 6.1239550000000005, 6.088030000000001, 6.1485639999999995, 6.280316, + 6.10797, 7.5701789999999995, 5.086067, 6.240218, 6.373443, 6.809677, + 5.01492, 5.4175830000000005, 6.594456999999999, 6.643111, 6.700606, + 7.160947, 6.209119, 5.972472000000001, 6.616095, 4.3514859999999995, + 6.171503, 6.164405, 5.843315, 8.460402, 6.066120000000001, 4.876418, + 6.286741999999999, 6.1891560000000005, 6.555112, 4.9461829999999996, + 6.686615, 4.277877999999999, 7.205112, 4.832937, 5.5565750000000005, + 6.096705, 5.889509, 4.371624, 4.772309, 5.377256, 5.138272, 6.84679, + 4.170706, 4.9783800000000005, 7.4255320000000005, 5.843129, + 8.512680999999999, 3.559066, 3.0948070000000003, 2.304745 + ], + "readDocuments": 30889, + "writeDocuments": 10163, + "readBytes": 11492430, + "writeBytes": 1611963, + "ioReadBytes": 11492430, + "ioWriteBytes": 1611963, + "writeIndexRows": 40326, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-warmup-3-ordering", + "phase": "warmup", + "rep": 3, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T01:02:41.825263+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 13603, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 35554, + "p50": 18545, + "p95": 33977, + "p99": 35217 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 48966 + }, + "tasksPerSec": 102, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002212920, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002162395.2715, + "_id": "jd72cztj84rhqtbxec2dcnbsjx8e5bwy", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002162395, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 124.02413, 95.594428, 162.747625, 104.645504, 153.77960800000002, + 94.56430999999999, 145.321266, 99.73393399999999, 144.927009, + 108.601786, 91.670244, 163.62907700000002, 107.87796800000001, + 150.90348600000002, 97.70418000000001, 147.156415, + 114.84949399999999, 94.084525, 145.746817, 99.031318, 172.567606, + 124.89081300000001, 129.02308200000002, 143.257305, 108.966382, + 168.566131, 135.52181399999998, 88.16978999999999, 174.906794, + 88.86364300000001, 105.13425, 102.219212, 100.475221, 158.967037, + 105.853875, 181.1835, 85.773601, 131.04080000000002, 99.585448, + 97.06114000000001, 159.623574, 102.997577, 120.979404, 93.93082, + 104.547134, 141.990991, 92.301468, 186.802575, 105.845334, + 142.76777199999998 + ], + "userExecutionMs": [ + 21.03816, 15.730728, 22.446934000000002, 19.912592, 19.049916, + 20.711530999999997, 21.467993, 20.626431, 18.329359999999998, + 17.764128, 17.678453, 17.871178, 16.916311999999998, 16.701711, + 16.952748, 15.901087, 16.237093, 16.143521, 17.524504999999998, + 16.545587, 16.060868999999997, 17.122204, 16.884376, 17.294258, + 16.715917, 16.082214, 17.076138, 15.726694, 17.213435999999998, + 16.635058, 16.094894, 20.872268, 17.355106, 17.51658, 18.719477, + 17.499834, 16.801347, 17.843178, 17.035138, 20.328168, 18.339116, + 17.610977, 15.247076999999999, 20.977238, 16.841638, + 16.916490999999997, 16.854446, 20.718203000000003, 20.412332, + 19.077349 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2598953, + "writeBytes": 4275604, + "ioReadBytes": 2598953, + "ioWriteBytes": 4275604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 186.49660500000002, 307.232305, 496.512725, 496.69837900000005, + 545.3288689999999, 525.2228650000001, 546.7865889999999, 567.8318, + 527.986856, 684.421871, 712.990894, 631.890368, 594.2647880000001, + 536.457562, 514.3577740000001, 509.32613299999997, + 496.49642900000003, 513.021455, 494.293205, 489.092122, 529.392113, + 501.4878739999999, 523.06123, 499.86694, 509.401569, 510.432031, + 505.91949400000004, 518.2965379999999, 504.99196300000006, + 501.38737899999995, 557.057465, 576.138503, 518.859875, 625.395949, + 564.348348, 495.153197, 527.328115, 533.906476, 563.256889, + 513.278291, 509.57407499999994, 523.0310820000001, 498.970552, + 599.078366, 498.567327, 503.10742600000003, 582.378534, 615.509175, + 515.057687, 539.2280310000001, 513.9639980000001, 526.124495, + 493.06181599999996, 495.62984900000004, 469.989601, 501.659879, + 496.329273, 502.242859, 507.355549, 508.748976, 470.38852799999995, + 505.73886, 528.6276300000001, 535.381495, 485.01196200000004, + 499.611642, 534.293138, 504.896735, 533.492144, 482.58581699999996, + 500.43822000000006, 509.08814599999994, 495.232657, 557.317166, + 533.2349760000001, 498.142362, 504.77367699999996, + 517.8139699999999, 437.735719, 290.876212, 136.67548, + 78.63687499999999, 84.18778400000001, 105.72581000000001, + 170.81328200000002, 74.975611, 79.571511 + ], + "userExecutionMs": [ + 3.2108420000000004, 4.698131, 4.792645, 6.285399, 8.960602999999999, + 6.402163, 8.828509, 6.554538, 8.756085, 4.481091, 4.42881, 6.226999, + 5.4514439999999995, 4.7818130000000005, 4.420119, 5.063503, + 5.490736, 5.232845, 5.883743, 6.023975, 6.033138, 5.853586, + 6.2291229999999995, 8.074423000000001, 4.888135, 6.080645, 4.390614, + 6.040093, 6.213591, 6.58849, 8.667387999999999, 4.282296, 4.376652, + 6.559644, 4.291103, 4.856205, 7.730378000000001, 4.77837, + 10.033558999999999, 4.8273459999999995, 6.0482000000000005, + 6.3927130000000005, 4.803678, 5.941642, 5.997623, 6.431671, + 5.9292620000000005, 6.750257, 6.541666, 4.363674, 8.728384, + 6.711627, 5.0599, 9.745061, 4.33679, 6.171023, 5.577558, 6.666154, + 5.44123, 8.919162, 4.244847, 4.819526, 6.576182, 6.4684349999999995, + 6.039219, 5.371418, 5.964459, 6.186387, 6.4530009999999995, + 4.754919, 6.033041, 6.237131, 5.039646, 6.766828, 4.709785, + 6.695589, 5.236222, 6.771821, 4.297479, 4.14321, 2.623013, + 2.3172610000000002, 2.600885, 2.3423710000000004, 2.509496, + 2.372463, 2.3490439999999997 + ], + "readDocuments": 30966, + "writeDocuments": 10168, + "readBytes": 10223919, + "writeBytes": 1103404, + "ioReadBytes": 10223919, + "ioWriteBytes": 1103404, + "writeIndexRows": 40336, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-1-packed", + "phase": "measured", + "rep": 1, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T01:03:43.722520+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 10905, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 35037, + "p50": 17823, + "p95": 33303, + "p99": 34700 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 45840 + }, + "tasksPerSec": 109, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002271750, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002224295.517, + "_id": "jd73kyr0q4mhx1bbc8bf6grgbx8e4n73", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002224295, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 90.375645, 76.909548, 165.513493, 107.848977, 165.677674, 84.062609, + 121.831734, 98.68139, 101.073886, 173.776596, 89.557913, + 148.87716700000001, 134.787556, 89.20902099999999, 173.182697, + 77.392894, 141.055941, 100.634247, 74.566273, 149.768055, 94.522143, + 118.188292, 124.29517, 101.53221599999999, 127.405658, 92.210662, + 101.889936, 150.67665699999998, 89.574618, 120.311886, 89.067436, + 81.467557, 166.96917, 85.69170600000001, 111.83163, 116.721686, + 93.846734, 159.948218, 83.182315, 117.220532, 122.150039, 90.680419, + 144.847895, 97.358167, 136.326121, 111.590093, 81.967741, + 129.67557299999999, 80.85261100000001, 89.25443600000001 + ], + "userExecutionMs": [ + 14.532194, 12.871843, 26.741403000000002, 15.929751999999999, + 18.175597000000003, 13.654589, 14.918336, 17.460884999999998, + 19.021507, 14.943117, 13.459352, 14.992444, 20.128678, 12.669803, + 17.093978, 12.196876000000001, 13.363272, 13.754014, 12.348827, + 18.342850000000002, 18.059964, 12.869447, 17.880992, 14.094086, + 13.452499999999999, 13.211307, 13.296591999999999, 12.436603, + 17.904574, 13.659997, 13.859121, 13.355779, 14.494556000000001, + 12.915177, 12.862563, 17.420145, 12.449833, 14.136130999999999, + 13.385162000000001, 13.76484, 13.823402, 12.585687, 17.689014, + 16.78784, 14.826426, 14.382281, 13.845722, 17.68468, 12.495837, + 13.402587 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2598953, + "writeBytes": 3003604, + "ioReadBytes": 2598953, + "ioWriteBytes": 3003604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 88, + "executionMs": [ + 169.6125, 271.296311, 443.332322, 520.64612, 482.39501, 512.360797, + 495.033858, 446.352309, 485.182952, 478.584473, 487.11674100000005, + 468.461105, 474.20374799999996, 497.50782100000004, 496.338624, + 450.112643, 546.039468, 460.140604, 483.317621, 482.631284, + 485.50284300000004, 459.23822800000005, 468.737847, 483.764185, + 467.655307, 478.970709, 492.21864500000004, 520.755545, 513.441813, + 512.4268669999999, 484.459212, 517.158985, 525.523988, 500.759339, + 697.826152, 663.369956, 602.47511, 623.559138, 494.234065, + 503.719021, 481.15407700000003, 471.784, 488.31534500000004, + 625.473608, 514.6249809999999, 495.83451299999996, 528.399865, + 476.806116, 515.050573, 523.081845, 495.937788, 462.902684, + 470.203756, 484.62346099999996, 474.41100200000005, 475.41443, + 464.253921, 478.294794, 468.807499, 488.64376500000003, 456.578465, + 455.48181999999997, 468.877922, 476.169402, 492.03506899999996, + 467.54332700000003, 476.261864, 459.103857, 502.84173, 486.506722, + 480.599912, 583.587446, 581.310295, 565.5791399999999, 527.18281, + 482.44061500000004, 508.641247, 510.517758, 451.209719, 301.36501, + 129.705457, 71.204706, 62.624898, 74.78739900000001, 58.402743, + 80.353996, 71.514337, 73.837173 + ], + "userExecutionMs": [ + 3.013322, 3.260504, 4.863435999999999, 5.880567, 4.80952, + 7.281796999999999, 6.524839, 4.939552, 6.993157, 6.039059, + 5.892205000000001, 4.3519630000000005, 5.990851999999999, 6.197721, + 5.927028, 7.073374, 6.038581000000001, 5.50196, 6.865225, 5.961818, + 6.822284, 5.382618, 4.839882, 5.977218000000001, 6.658989, + 8.866857000000001, 9.807697000000001, 6.647329, 6.676917, 4.35475, + 7.014295000000001, 4.74847, 8.370531, 6.694407, 4.325540999999999, + 4.462756, 5.5116000000000005, 8.076768999999999, 6.08082, 6.000502, + 4.966235999999999, 6.467049, 4.291119, 6.593202000000001, + 5.957897999999999, 6.248348, 6.099608, 9.130823999999999, 6.571581, + 6.066796, 8.575203, 5.31432, 4.816219, 6.563968, 5.159096, 6.022339, + 6.5686800000000005, 4.776281, 9.554144, 5.978775000000001, + 4.8996200000000005, 6.235026, 4.799086, 4.74229, 5.051723, + 6.5201210000000005, 5.5709599999999995, 6.0499350000000005, 5.42915, + 6.330261, 4.75054, 5.348909, 5.920568, 4.803339, 6.462321, + 5.089627999999999, 4.683237, 8.925828000000001, 5.413147, 4.204206, + 2.7012009999999997, 3.8552690000000003, 2.413682, 2.194526, + 2.264459, 2.305986, 2.220303, 2.31189 + ], + "readDocuments": 16238, + "writeDocuments": 5294, + "readBytes": 9072094, + "writeBytes": 1249462, + "ioReadBytes": 9072094, + "ioWriteBytes": 1249462, + "writeIndexRows": 20838, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-1-ordering", + "phase": "measured", + "rep": 1, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T01:04:44.134117+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 13656, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 37113, + "p50": 18394, + "p95": 35370, + "p99": 36715 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 50560 + }, + "tasksPerSec": 99, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002337140, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002284711.309, + "_id": "jd7fg0w9bzwb79nrk10h8bb67s8e5fbv", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002284711, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 119.649327, 95.673817, 150.726517, 170.04465599999997, + 145.92485299999998, 107.258424, 167.45511, 108.034852, + 116.55781599999999, 110.381926, 151.35585600000002, 169.808722, + 105.486487, 178.74227000000002, 106.024241, 121.204505, 115.027096, + 103.34244600000001, 157.57594699999999, 117.36796700000001, + 119.799083, 105.890574, 100.08306999999999, 157.958118, + 100.56224800000001, 152.36017700000002, 107.770201, 141.108215, + 115.96039599999999, 96.091803, 184.183274, 84.240025, 161.63675, + 102.68096899999999, 111.09388000000001, 158.337154, + 103.21254400000001, 123.183167, 88.817152, 144.125971, 112.944609, + 100.787762, 154.070084, 102.044559, 158.471756, 95.581142, + 132.08011199999999, 100.605177, 98.620058, 162.202811 + ], + "userExecutionMs": [ + 20.63286, 15.811305, 21.094399, 20.975405, 17.173142, 21.554128, + 17.754622, 16.122076999999997, 24.251372, 18.812879000000002, + 17.495093, 17.712517000000002, 16.706951, 19.993091, 17.07467, + 19.027289999999997, 20.646241, 17.671896, 20.450608, 20.797052, + 18.187160000000002, 16.587829, 17.079323, 17.216057, 17.216792, + 20.014629, 19.686381, 16.258824, 19.999647999999997, 15.954988, + 20.577818, 16.603815, 16.223984, 16.411973, 20.034234, + 17.305153999999998, 15.813002, 16.788318, 17.068136000000003, + 16.949996, 18.777951, 17.657071, 16.209525, 19.084985, 16.87328, + 21.137367, 15.944631000000001, 17.594034, 16.918604000000002, + 16.262549 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2598953, + "writeBytes": 4275604, + "ioReadBytes": 2598953, + "ioWriteBytes": 4275604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 89, + "executionMs": [ + 192.40422900000002, 293.622236, 513.836428, 494.770946, + 505.28079199999996, 655.60662, 548.152032, 518.415803, + 656.0444130000001, 525.456645, 506.78513300000003, + 616.2838840000001, 551.937116, 534.106819, 605.005174, + 590.7152970000001, 507.71978500000006, 498.998245, + 511.42909599999996, 515.946424, 538.588667, 521.266799, 507.423584, + 510.831913, 484.886931, 504.17795899999993, 511.11084900000003, + 571.865513, 523.386143, 514.7985080000001, 541.911688, 517.383353, + 514.6836460000001, 503.544872, 516.120977, 488.595858, + 506.29661500000003, 615.4757320000001, 586.269783, 587.306694, + 572.126397, 528.9627999999999, 521.057252, 548.431137, 535.048806, + 521.238576, 517.349428, 546.837989, 534.5060139999999, 555.202145, + 507.28378100000003, 479.626283, 492.052212, 489.278111, 505.646315, + 547.269176, 508.331753, 514.753128, 488.776492, 534.489999, + 815.1380740000001, 560.486877, 579.225153, 742.889417, 558.730135, + 936.333733, 532.976952, 612.88211, 519.514585, 502.95813899999996, + 504.96046400000006, 491.41927400000003, 482.29141500000003, + 505.351718, 553.0884219999999, 480.682601, 523.047667, 534.252195, + 496.67816, 292.30345400000004, 126.85868, 77.45634999999999, + 64.525291, 109.490823, 68.96281, 76.206651, 78.29503799999999, + 69.780591, 87.95990400000001 + ], + "userExecutionMs": [ + 3.3671640000000003, 3.5670849999999996, 4.484296, 6.501439, + 6.757142, 6.09459, 7.283277, 4.915778, 6.03296, 6.51306, 7.430532, + 6.58552, 6.149366, 6.2179, 6.884525, 6.516242, 4.893105, 8.944347, + 6.663575, 4.3501959999999995, 4.935623, 4.782357, 8.498926, + 6.347602, 5.353870000000001, 4.989658, 4.974271, 6.282431, 5.895062, + 6.1244510000000005, 6.020694, 5.960882, 6.989377, 6.110778, + 6.3167409999999995, 4.886984, 4.9003689999999995, 6.518941, + 5.327953, 4.97429, 5.918349, 7.0852889999999995, 6.045551, 6.220421, + 6.481109, 6.21042, 6.644431999999999, 10.023875, 5.298667, 4.81498, + 6.782964, 5.575486, 7.263160999999999, 4.685323, 5.417983, 7.338531, + 4.392064, 4.376950999999999, 6.521216, 4.969888, 6.067178999999999, + 4.850793, 4.831255, 12.257011, 4.688718, 4.7800899999999995, + 6.35763, 5.791241, 4.963291, 7.265535, 4.888299, 4.805486999999999, + 4.824524, 4.816958, 4.364014, 5.256899, 7.692633, 6.127876, + 3.942612, 4.134006, 2.7263100000000002, 2.612266, 2.296551, + 2.3166789999999997, 3.5014450000000004, 2.3650339999999996, + 2.271693, 2.16125, 2.191138 + ], + "readDocuments": 30980, + "writeDocuments": 10170, + "readBytes": 10227165, + "writeBytes": 1103832, + "ioReadBytes": 10227165, + "ioWriteBytes": 1103832, + "writeIndexRows": 40340, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-1-baseline", + "phase": "measured", + "rep": 1, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T01:05:47.421799+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 12353, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 50595, + "p50": 27351, + "p95": 48567, + "p99": 50564 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 62805 + }, + "tasksPerSec": 80, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002412742, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002348000.6948, + "_id": "jd70gv28g66sted8njqf81890d8e5eqm", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002348000, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 101.15445100000001, 76.791905, 112.198376, 105.038291, 155.743351, + 70.855452, 159.164824, 100.448028, 104.281153, 115.480685, + 87.048119, 111.190609, 125.906792, 94.56242900000001, + 148.68703399999998, 82.501179, 153.46973599999998, + 102.97685800000001, 84.338083, 185.097621, 94.05347499999999, + 78.2517, 154.661664, 80.945459, 140.588357, 88.252207, + 99.85620300000001, 94.837247, 88.688351, 77.402683, + 139.96972699999998, 71.93691700000001, 138.10408099999998, + 97.232688, 97.35276, 150.306154, 97.703391, 76.84805999999999, + 183.08442499999998, 81.210684, 128.198965, 94.851712, 89.524035, + 120.842839, 87.682634, 76.992542, 156.469564, 79.411187, 118.175493, + 112.123752 + ], + "userExecutionMs": [ + 18.771622, 13.419068, 17.563748, 16.738440999999998, 19.95912, + 14.985547, 16.094075, 14.897400999999999, 15.534379, 14.435098, + 14.59426, 17.918438, 26.248831, 15.800910000000002, 15.692477, + 13.792423999999999, 18.438646, 16.212573000000003, + 16.236760999999998, 17.337193, 14.492129, 14.209658, 19.365639, + 17.242151, 17.836176, 14.002172, 14.275972, 13.254827, + 17.743166000000002, 18.887808, 14.92959, 14.638634, 16.426307, + 19.468523, 16.542715, 23.249828, 17.076752, 14.421766000000002, + 20.143729, 13.99057, 18.282117, 17.190752, 13.743793, 14.87695, + 14.72632, 18.468777, 15.107881, 14.596724, 15.196898000000001, + 15.179599999999999 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4155604, + "ioReadBytes": 23953, + "ioWriteBytes": 4155604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 85, + "executionMs": [ + 173.937177, 293.39619, 543.113485, 487.542688, 513.749352, + 604.544231, 538.234472, 575.709876, 661.396077, 526.262549, + 553.447383, 551.3931699999999, 493.014026, 545.704746, + 659.6686109999999, 547.1301050000001, 558.5197870000001, 573.137511, + 582.0211979999999, 572.108779, 569.4480050000001, 757.81252, + 599.1478559999999, 1082.9781679999999, 1804.648033, 1587.680918, + 1272.724568, 1123.9407159999998, 1019.2650379999999, 748.30399, + 1130.437357, 769.271755, 1020.8759190000001, 751.935323, 760.781048, + 548.896919, 692.831419, 671.609285, 561.8477039999999, 745.097358, + 696.5328870000001, 525.835034, 601.248872, 595.050869, 594.966284, + 782.179916, 517.906988, 883.681315, 708.22201, 579.622667, + 589.114236, 681.103249, 623.312205, 702.659449, 572.431715, + 1149.385096, 612.76013, 621.53263, 576.14098, 571.855786, + 681.216217, 541.611292, 644.7301610000001, 599.2684889999999, + 557.1768950000001, 573.276567, 607.1443509999999, 651.919727, + 791.15542, 781.697082, 628.1801839999999, 743.355906, 585.210378, + 739.8070740000001, 736.181192, 596.696288, 638.685542, 502.265441, + 336.035422, 334.604786, 134.687696, 109.29955199999999, 54.268036, + 64.163807, 41.573105999999996 + ], + "userExecutionMs": [ + 3.239607, 3.659585, 4.695588, 4.82924, 7.54415, 8.331461000000001, + 9.436057, 7.068403, 6.157609, 9.628777999999999, 8.74995, 7.259042, + 4.370998, 6.112507, 15.063175, 4.242962, 6.135144, 6.309999, + 6.075868, 6.216000999999999, 6.780332, 6.11233, 5.947458, 5.656129, + 6.615272, 4.297725, 4.952553, 4.788927999999999, 6.031461, 4.933221, + 4.42048, 5.383565, 6.085614, 4.775144999999999, 5.268868, 4.827385, + 8.163853, 7.186875, 4.453739, 6.634849, 6.049364, + 6.7562050000000005, 6.5374099999999995, 7.6710270000000005, + 6.126628, 8.452304000000002, 9.687927, 6.0957930000000005, 6.775936, + 6.1282369999999995, 4.476521, 6.173807, 6.6330729999999996, + 8.655429, 6.399945, 6.351481, 5.9329860000000005, 5.86304, 6.561085, + 6.463414, 6.4168769999999995, 6.505329, 6.857571, 4.758469000000001, + 6.25351, 4.38899, 6.070696000000001, 5.899343, 6.569654, + 6.6457690000000005, 4.427859, 6.724467, 6.610805, 6.242873, + 6.130223, 4.887607, 7.004991, 6.348596, 5.435161, 6.95533, 2.679526, + 2.293555, 2.3120179999999997, 2.212071, 2.585287 + ], + "readDocuments": 30941, + "writeDocuments": 10166, + "readBytes": 10927733, + "writeBytes": 1420957, + "ioReadBytes": 10927733, + "ioWriteBytes": 1420957, + "writeIndexRows": 40332, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-2-ordering", + "phase": "measured", + "rep": 2, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T01:07:03.830367+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 13200, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 35719, + "p50": 17655, + "p95": 33995, + "p99": 35370 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 48844 + }, + "tasksPerSec": 102, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002474781, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002424355.1624, + "_id": "jd70905bn93kqbmfskrpp7p7q58e4we8", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002424355, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 129.836063, 74.927195, 158.02586, 100.305615, 164.861923, 86.90185, + 107.385448, 93.650653, 106.725349, 144.594222, 240.01943, + 438.388316, 215.066642, 140.319465, 97.714625, 104.866816, + 195.74975899999998, 87.471087, 103.528462, 111.411586, + 87.85324999999999, 150.028778, 115.321082, 150.48338999999999, + 95.039122, 159.819343, 112.958272, 84.516255, 155.71935200000001, + 133.06474400000002, 189.51768199999998, 92.157406, 95.216695, + 92.79039399999999, 86.690728, 150.449216, 106.53924400000001, + 141.67554199999998, 87.69069999999999, 120.20679, + 110.93204700000001, 83.235344, 171.805765, 113.76444400000001, + 156.674887, 87.700809, 131.598409, 107.620413, 88.315699, 161.579523 + ], + "userExecutionMs": [ + 27.047825, 14.895227, 24.18476, 16.739911, 17.417427, 17.459498, + 16.529135, 16.84229, 18.022876, 19.593389, 26.718244, + 47.984852000000004, 20.111071000000003, 20.44305, + 18.784889999999997, 17.418671, 18.164782000000002, 16.553717, + 15.881498, 17.126963, 16.436018999999998, 17.603839999999998, + 22.330741, 20.702849999999998, 21.804406, 20.904897, + 16.916130000000003, 16.507704, 20.158812, 20.36841, + 21.251872000000002, 18.921879, 18.624322, 15.988393, 17.146794, + 16.378176, 17.495388000000002, 20.168399, 14.753666, 17.485494, + 17.289554, 16.244153, 17.594275, 15.499251000000001, + 19.953438000000002, 20.656119, 16.970955, 20.344608, + 14.726329999999999, 16.447302 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2598953, + "writeBytes": 4275604, + "ioReadBytes": 2598953, + "ioWriteBytes": 4275604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 88, + "executionMs": [ + 202.395188, 300.210767, 478.077217, 475.182207, 553.700947, + 465.325467, 473.766759, 465.544647, 570.102152, 545.235106, + 470.75793, 508.98325300000005, 473.031716, 472.414981, 546.024966, + 469.361444, 537.5359870000001, 498.72866999999997, 484.898977, + 566.5480859999999, 544.846904, 498.988022, 475.056523, 689.126401, + 505.11940799999996, 508.51801399999994, 502.48515299999997, + 462.92827, 615.908572, 517.6453349999999, 537.808888, 517.966264, + 523.278047, 475.33945, 459.604128, 567.029852, 474.12966, + 519.892372, 486.305673, 574.9148299999999, 611.497967, 577.513418, + 586.526604, 511.308856, 518.9735469999999, 587.350047, 559.217767, + 493.89547, 764.229453, 538.0271730000001, 508.45835900000003, + 659.761613, 466.648736, 515.3213, 479.998626, 465.298352, 479.25747, + 532.918526, 530.97398, 547.3463370000001, 515.793561, 496.277232, + 559.3823249999999, 584.118607, 530.363498, 516.677564, 475.483561, + 498.244168, 527.358058, 486.148957, 464.575043, 505.198161, + 507.67957399999995, 504.415652, 516.196096, 500.51752000000005, + 510.261449, 514.309781, 449.29741, 258.810486, 97.658912, 80.735099, + 78.84030899999999, 59.566477, 74.29189099999999, 54.988292, + 77.706727, 51.087033 + ], + "userExecutionMs": [ + 2.8469860000000002, 3.620248, 5.053255, 4.9261230000000005, 6.61023, + 6.2407010000000005, 6.5363620000000004, 6.152525, 8.188084, + 6.8817569999999995, 6.822787, 6.274024, 6.204612, 6.300631999999999, + 7.060374, 4.857482, 7.1118809999999995, 6.079168, 7.020764, + 6.324132, 9.563686, 8.046924, 10.208293, 6.099214000000001, + 8.186343, 6.458372000000001, 5.904188, 4.706323, 4.4157399999999996, + 7.488137, 6.235290999999999, 6.0190269999999995, 4.5655030000000005, + 8.279693, 6.199421999999999, 5.986517999999999, 4.40548, 4.854825, + 6.973094, 6.853743, 6.734014, 6.479579, 6.082964, + 5.6102620000000005, 5.9754439999999995, 6.840628, 6.525443, + 4.605035, 5.827738, 6.191936999999999, 6.2091780000000005, 8.886835, + 6.143097999999999, 6.191439, 4.845404, 6.2695549999999995, 4.830959, + 4.810036, 5.278109, 6.065692, 5.105404, 4.963924, 6.902406, + 4.967762, 5.9854, 5.1369739999999995, 5.069789, 6.067312, 6.201425, + 6.306567, 5.109046, 6.108492, 6.178874, 6.6116220000000006, 8.58305, + 4.425542, 7.481603, 5.4589929999999995, 6.086822000000001, 5.396305, + 5.986361, 3.430509, 2.146901, 2.686008, 2.308551, + 2.0873559999999998, 2.207169, 2.435227 + ], + "readDocuments": 30973, + "writeDocuments": 10169, + "readBytes": 10225542, + "writeBytes": 1103618, + "ioReadBytes": 10225542, + "ioWriteBytes": 1103618, + "writeIndexRows": 40338, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-2-baseline", + "phase": "measured", + "rep": 2, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T01:08:06.404994+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 12058, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 40379, + "p50": 19289, + "p95": 38324, + "p99": 40335 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 52334 + }, + "tasksPerSec": 96, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002541098, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002486936.151, + "_id": "jd76kcts309c1fz3kfv6ctzk2n8e4zpa", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002486936, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 96.841803, 83.65268900000001, 103.959124, 77.774377, + 109.52372000000001, 83.000009, 73.53889600000001, 146.901213, + 68.42779, 119.73603800000001, 114.215126, 93.530786, 157.378339, + 85.87552199999999, 150.696464, 81.61428699999999, 105.219235, + 76.54646199999999, 69.370459, 131.09167699999998, 65.892552, + 116.829065, 96.313633, 102.408843, 82.062934, 89.766411, 76.674894, + 88.23336900000001, 97.047658, 78.549081, 100.56725399999999, + 86.912622, 77.555493, 133.27757200000002, 70.103385, 143.56782, + 68.107139, 76.832856, 137.057127, 85.443162, 96.472826, + 68.52989199999999, 64.76187399999999, 146.679654, 73.90898999999999, + 112.4717, 75.71394400000001, 86.24513999999999, 92.852751, 73.005056 + ], + "userExecutionMs": [ + 17.644961000000002, 13.328493, 25.430455000000002, + 19.137636999999998, 13.837099, 14.808414, 18.294047, + 18.302646999999997, 14.51455, 14.822065, 17.597295, + 17.199817999999997, 15.085135, 13.880953, 14.709985999999999, + 13.49051, 15.702613000000001, 16.403162, 13.092259, 14.59358, + 13.848513, 13.350552, 15.858902, 15.537886, 14.357877, 16.002328, + 14.567321, 15.410673999999998, 13.716667, 13.237611999999999, + 11.968160000000001, 14.232087, 13.95818, 14.611766999999999, + 13.699804, 17.501002000000003, 14.146021999999999, 13.429849, + 13.354769000000001, 15.144236, 14.381544, 14.709829000000001, + 13.577738, 14.446788999999999, 16.36672, 13.88467, + 14.528861000000001, 14.546519, 16.778067, 14.787355999999999 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4155604, + "ioReadBytes": 23953, + "ioWriteBytes": 4155604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 84, + "executionMs": [ + 172.19383399999998, 309.991599, 527.5259759999999, + 511.00316499999997, 534.130125, 467.88104200000004, 517.955557, + 464.82478, 575.843092, 559.6998189999999, 538.474173, 627.511514, + 518.0621560000001, 476.92031099999997, 558.1379450000001, + 495.036331, 509.429869, 599.212657, 524.436196, 490.30168499999996, + 525.5278020000001, 560.308726, 503.873168, 508.08344999999997, + 562.5673009999999, 520.986252, 501.024746, 745.68818, + 549.7542520000001, 588.449522, 523.2849160000001, 547.8157689999999, + 576.080224, 574.93092, 585.5312190000001, 570.363114, 623.937486, + 591.26779, 698.6893749999999, 822.368995, 599.125552, 738.968123, + 598.025955, 589.539438, 514.563758, 515.995731, 533.373156, + 521.394546, 568.823845, 587.1731970000001, 557.4522870000001, + 532.138652, 531.3225620000001, 696.013184, 598.7529470000001, + 592.840428, 581.110108, 599.658907, 587.216838, 649.2363009999999, + 544.070281, 684.508021, 582.121428, 603.1729740000001, 884.515596, + 606.8439599999999, 560.762615, 676.378872, 552.5202529999999, + 618.701941, 682.739653, 563.9725639999999, 579.1788789999999, + 607.735201, 618.477346, 724.150859, 659.648327, 510.014734, + 351.84139, 470.753707, 133.37911, 108.81852, 50.303008000000005, + 63.99358600000001 + ], + "userExecutionMs": [ + 3.384626, 7.453691, 8.494282, 8.256671, 6.192488, 6.388983, + 8.747401, 4.801065, 4.1942200000000005, 6.71039, 6.90299, 5.028269, + 4.36818, 6.663097, 4.806896, 5.027088, 4.902367, 4.852786, 5.94001, + 5.095394, 7.159625, 5.9437739999999994, 8.007729999999999, 4.376033, + 6.286204, 8.670416, 5.915097, 8.800795, 6.0630679999999995, + 5.942847, 6.06563, 6.099342, 4.809947, 4.470962, 5.290113, + 6.643012000000001, 5.886438, 6.611154, 6.4609000000000005, 6.809193, + 6.738922, 5.596769, 5.957822, 6.318848999999999, 5.13217, 4.953208, + 6.4812769999999995, 6.093648, 6.235689, 5.919127, + 5.6622959999999996, 5.335153, 7.700608, 6.155946, 4.848194, + 6.4112409999999995, 6.7368999999999994, 6.15435, 6.917874, 4.343877, + 5.01179, 6.177365, 4.749176, 4.424634, 4.330604, 4.291673, 4.969854, + 4.7312259999999995, 4.718756999999999, 5.864592, 6.92634, 5.0065, + 6.2659780000000005, 4.602646999999999, 6.77323, 6.102061, 6.629371, + 4.142468, 6.7980469999999995, 3.6834580000000003, 2.630173, + 2.1994100000000003, 2.37655, 2.1904160000000004 + ], + "readDocuments": 31070, + "writeDocuments": 10165, + "readBytes": 10949633, + "writeBytes": 1420743, + "ioReadBytes": 10949633, + "ioWriteBytes": 1420743, + "writeIndexRows": 40330, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-2-packed", + "phase": "measured", + "rep": 2, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T01:09:11.832632+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 10604, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 36150, + "p50": 18063, + "p95": 34561, + "p99": 35794 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 46668 + }, + "tasksPerSec": 107, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002601075, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002552415.331, + "_id": "jd757dxrsr4ads8e95p0rx79js8e5y4n", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002552415, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 135.80806, 76.694661, 135.056001, 114.33333599999999, 172.739771, + 80.722268, 92.570264, 79.724626, 73.46936, 105.73790600000001, + 88.20897599999999, 71.015236, 154.466142, 82.479289, 138.660904, + 103.244052, 99.408942, 124.183755, 96.298002, 95.069973, + 169.22352700000002, 78.351285, 133.627283, 89.109627, 126.26772, + 95.419283, 90.42088, 125.63597800000001, 63.421485, 89.97448, + 174.69082799999998, 71.963074, 94.128837, 102.17359900000001, + 72.399315, 135.23269299999998, 90.799966, 100.454256, + 177.53686299999998, 79.87340499999999, 125.39883800000001, + 94.155421, 79.15657, 133.596507, 120.438029, 111.593234, 142.997099, + 90.925223, 162.191629, 95.61053199999999 + ], + "userExecutionMs": [ + 18.390889, 14.044683000000001, 20.307712, 16.720138, 14.068988, + 15.464955000000002, 13.729393, 14.389566, 11.552793000000001, + 14.3097, 13.766471000000001, 13.210285, 17.465162, 14.642617, + 16.880486, 19.500996, 14.365399, 14.485239, 17.290153999999998, + 12.725072, 11.788665, 13.618659, 14.147222, 16.524845, 13.726098, + 11.8728, 16.988011, 12.893668, 11.544804000000001, + 12.973082999999999, 19.44366, 13.029057, 12.845775999999999, + 14.687637, 13.116925, 14.291045, 13.243234, 14.133168999999999, + 14.205772, 13.904345000000001, 12.786449, 16.318738, 13.468568, + 13.758543999999999, 14.112101000000001, 14.087261, 14.013313, + 13.267793, 14.497418999999999, 15.919470000000002 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2598953, + "writeBytes": 3003604, + "ioReadBytes": 2598953, + "ioWriteBytes": 3003604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 90, + "executionMs": [ + 164.478768, 267.56084699999997, 447.028653, 502.053984, + 434.46169699999996, 479.877742, 472.484268, 493.207409, + 425.14378800000003, 443.468026, 488.041755, 487.279128, 442.173855, + 468.13760199999996, 488.923519, 464.058197, 519.668754, 471.659873, + 487.23132499999997, 480.767998, 538.992381, 507.44584100000003, + 522.482419, 515.327454, 470.682981, 526.469633, 494.712176, + 449.191412, 512.983064, 557.123391, 583.660386, 723.369752, + 628.127738, 507.96618400000006, 531.331071, 496.912815, 448.308845, + 520.115457, 456.011779, 489.202497, 483.01557099999997, 499.630137, + 619.274786, 477.58927, 462.314026, 435.617091, 529.5160330000001, + 445.26106799999997, 469.717987, 490.569978, 455.128349, 528.814434, + 510.57801400000005, 456.471274, 497.896854, 490.96028800000005, + 550.6020139999999, 500.42994200000004, 453.547604, 454.19409, + 515.1172349999999, 475.010989, 486.250575, 505.17123, 636.282075, + 738.7423749999999, 566.3379699999999, 550.6563249999999, 534.004582, + 558.196956, 613.705095, 590.307184, 567.356929, 525.843901, + 529.5780500000001, 515.0296599999999, 449.28490700000003, + 441.257369, 483.721253, 249.988882, 131.399372, 54.606069, + 70.508501, 76.797218, 51.545484, 79.823637, 57.131075, 73.93447, + 56.614867, 74.358084 + ], + "userExecutionMs": [ + 4.685894, 6.609179, 8.979759, 6.104603999999999, 4.956909, + 9.078864000000001, 6.681067, 8.670458, 4.944838, 6.2209639999999995, + 5.450018, 6.690910000000001, 5.517459, 8.597244, 5.930752, 6.724654, + 6.3959719999999995, 5.097708, 4.884697, 4.800903, 8.548677, + 4.820861, 4.953639, 5.642669000000001, 5.598318, 5.893338, 6.852363, + 5.051914, 5.593097, 5.707415, 4.390702, 5.060282999999999, + 6.111845000000001, 8.333091000000001, 6.106274, 4.543183999999999, + 6.429133, 4.343381, 5.973992, 5.091076, 4.402947, 6.663613, + 7.059588000000001, 5.48418, 4.4001470000000005, 4.866304, 6.161608, + 6.829270999999999, 7.821982, 6.06106, 4.974224, 4.486504, 6.497215, + 5.448342, 6.342344, 5.455531, 6.789427, 6.890835, 5.095535, + 5.481274, 6.2391749999999995, 4.995937, 7.6138129999999995, + 6.704173, 5.228743, 4.895213, 11.042833, 5.8471779999999995, + 6.007751000000001, 5.011569, 6.204053999999999, 5.092218, 6.165106, + 5.479185, 5.998531, 6.099902, 5.231623, 5.521105, 4.320570999999999, + 4.123793, 2.536138, 2.559861, 2.357602, 2.250108, 2.262259, + 2.493761, 2.927193, 2.273285, 2.236614, 2.230531 + ], + "readDocuments": 16252, + "writeDocuments": 5296, + "readBytes": 9075340, + "writeBytes": 1249890, + "ioReadBytes": 9075340, + "ioWriteBytes": 1249890, + "writeIndexRows": 20842, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-3-baseline", + "phase": "measured", + "rep": 3, + "variant": "baseline", + "workload": "mutation", + "began": "2026-09-10T01:10:12.071859+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "baseline" + }, + "result": { + "enqueueTotal": 12429, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 41414, + "p50": 19545, + "p95": 39415, + "p99": 41368 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 53665 + }, + "tasksPerSec": 93, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002668004, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002612623.421, + "_id": "jd7b4vxvqqb1wee9rs9nnkceax8e5hz6", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002612623, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 70.77208399999999, 78.078191, 213.176575, 78.932616, 136.779655, + 93.766425, 113.49764800000001, 132.681518, 89.045801, + 95.41442599999999, 122.838597, 103.231605, 147.225765, 85.868212, + 95.494785, 141.016251, 94.895002, 159.562751, 84.772069, 98.064156, + 89.398484, 72.188762, 142.47805699999998, 69.472482, 128.606159, + 88.06141000000001, 112.993675, 129.527298, 86.223049, + 87.26787900000001, 131.26094899999998, 75.84742, 77.229055, + 92.335105, 67.70456, 104.006086, 91.222207, 94.50673900000001, + 73.47971100000001, 90.76131699999999, 74.24968999999999, + 144.64611299999999, 73.282251, 125.718589, 94.22306900000001, + 87.911091, 66.647924, 132.722025, 77.583288, 118.78638199999999 + ], + "userExecutionMs": [ + 14.653165, 16.95862, 26.510461999999997, 13.417513000000001, + 22.64813, 14.791244, 17.575706, 17.974985999999998, 20.087459, + 19.196476, 14.891207, 17.82369, 17.031526000000003, + 19.892674999999997, 15.995967, 15.517032, 18.350071999999997, + 14.685592000000002, 13.212349, 13.529799, 14.638336, 14.304698, + 14.905337, 15.228679, 15.334052000000002, 15.778489, 14.953469, + 14.732775, 14.724816, 13.03599, 15.430836, 14.223855, 15.159639, + 14.210567000000001, 14.954581, 14.160512, 13.422589, + 15.195134000000001, 13.406231, 14.760900000000001, 13.088263, + 19.433965, 15.307497999999999, 16.048482, 15.000961, 15.456694, + 13.250001000000001, 13.053412, 12.646355999999999, 13.832345 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4155604, + "ioReadBytes": 23953, + "ioWriteBytes": 4155604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 85, + "executionMs": [ + 206.83948999999998, 549.115187, 706.377814, 566.839556, 543.048488, + 676.478735, 497.462876, 494.420193, 518.5258, 502.028515, + 527.496958, 577.417185, 479.719618, 575.232032, 544.092074, + 500.505846, 477.855637, 724.513725, 557.148144, 585.537886, + 502.74306300000006, 666.140662, 475.301798, 694.1836380000001, + 504.057434, 583.256904, 562.179588, 525.7632669999999, + 673.5190319999999, 576.660026, 578.9208729999999, 620.66588, + 492.60252199999996, 548.150761, 565.67091, 601.547664, 578.889706, + 655.576091, 572.289241, 571.190021, 617.351567, 534.977758, + 572.216135, 652.45419, 556.589833, 523.9313719999999, 595.693315, + 838.210442, 661.472578, 733.786666, 587.609302, 591.795805, + 675.712708, 653.971709, 609.187006, 578.246206, 603.457803, + 617.178393, 603.0140309999999, 581.448494, 554.350141, 570.699654, + 608.8097260000001, 825.69441, 552.511071, 559.232034, 916.948368, + 555.009132, 558.531033, 622.100623, 576.700072, 595.497424, + 710.363014, 648.749729, 681.251449, 605.7970660000001, + 589.2071559999999, 589.821101, 268.15983, 259.881399, + 157.39456600000003, 90.253181, 62.103344, 44.913832, + 119.81335299999999 + ], + "userExecutionMs": [ + 4.880721, 7.388238, 5.3352, 7.045707, 5.249009, 11.406553, 6.997276, + 6.60265, 6.244694, 6.048855, 4.6266490000000005, 8.424356, 5.02445, + 6.1669860000000005, 6.61896, 5.030387, 4.846828, 6.299096, 6.561554, + 5.430088, 5.627372, 10.778727, 4.454407, 6.524478, 5.593428, + 8.576112, 5.949982, 7.4201299999999994, 5.772826, 6.359244, + 6.164033, 6.061654, 4.9711620000000005, 6.1478019999999995, + 6.127195, 6.177428000000001, 6.1176, 9.41516, 4.21851, 5.350533, + 5.988681, 4.972117, 6.842097, 5.998657, 5.4995829999999994, + 5.019107, 6.039697, 6.533373, 6.845765999999999, 5.373861, 6.35121, + 6.766144, 6.126546, 5.908427, 4.8217989999999995, 8.517057, + 5.2743269999999995, 4.787465999999999, 6.175885, 5.8905520000000005, + 4.8021970000000005, 8.140883, 5.878016, 9.283182, 5.771871, + 5.412748000000001, 6.652305, 5.047013000000001, 4.8641049999999995, + 5.994168, 4.526122, 5.035207, 8.911249, 5.9516860000000005, + 6.0772070000000005, 5.114948, 4.965497, 5.196588, 4.185169, + 3.694744, 2.714591, 2.3285620000000002, 2.25127, 2.28214, 2.545532 + ], + "readDocuments": 31077, + "writeDocuments": 10166, + "readBytes": 10951261, + "writeBytes": 1420957, + "ioReadBytes": 10951261, + "ioWriteBytes": 1420957, + "writeIndexRows": 40332, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-3-packed", + "phase": "measured", + "rep": 3, + "variant": "packed", + "workload": "mutation", + "began": "2026-09-10T01:15:12.684662+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "packed" + }, + "result": { + "enqueueTotal": 10764, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 38789, + "p50": 19634, + "p95": 37096, + "p99": 38431 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 49388 + }, + "tasksPerSec": 101, + "timedOut": false + }, + "evidence": { + "observedAt": 1789002964323, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002913241.434, + "_id": "jd7bwxhg8psyef9gv5r144rcsx8e434m", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002913241, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 93.994917, 70.77461100000001, 134.165837, 80.126246, 153.556052, + 96.17593000000001, 85.51122799999999, 169.514299, 72.288355, + 126.11688099999999, 168.900486, 82.241841, 111.141466, 92.082386, + 70.377252, 121.63319, 87.085303, 83.956705, 173.337711, 88.297063, + 143.95525700000002, 99.327387, 114.32273900000001, 156.822477, + 79.912695, 157.42246600000001, 92.557593, 145.571481, + 121.65090500000001, 89.040794, 144.258906, 71.62579500000001, + 125.565187, 125.87069000000001, 83.122522, 98.800151, 82.008483, + 74.905815, 86.599503, 128.29919, 73.586349, 119.867735, + 76.46953500000001, 73.525763, 160.306834, 81.761621, 123.485595, + 129.249145, 84.17145400000001, 156.976583 + ], + "userExecutionMs": [ + 18.383819, 12.119326999999998, 15.483639, 13.818166999999999, + 18.532279, 13.83394, 12.705084, 15.967647000000001, + 14.281980999999998, 14.138636, 17.37387, 16.337087, 12.617589, + 16.240240999999997, 12.913717, 13.8585, 17.030467, 18.457752, + 14.525705, 13.442811, 14.325834, 16.615204000000002, 11.867651, + 12.852165000000001, 14.080153999999999, 14.013019, 13.397006, + 15.145335, 14.022217, 12.457093, 13.204423, 14.211440000000001, + 17.974875, 14.187114000000001, 16.974452, 16.776775, + 13.018101000000001, 11.758151, 13.582772, 13.899609, + 12.119981000000001, 13.492766999999999, 13.352234, + 13.072659999999999, 12.552456, 12.967108999999999, + 15.054364000000001, 15.738410000000002, 11.675928, 14.199012 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2598953, + "writeBytes": 3003604, + "ioReadBytes": 2598953, + "ioWriteBytes": 3003604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 88, + "executionMs": [ + 179.833753, 300.87817900000005, 494.266983, 552.404802, 558.516064, + 492.79136800000003, 447.515486, 499.89348700000005, + 445.82621300000005, 446.16612, 502.194555, 469.563145, 468.908724, + 485.75237100000004, 489.828659, 488.628024, 489.709645, + 477.51924499999996, 481.99879200000004, 500.59114000000005, + 492.860347, 509.62836899999996, 621.906117, 481.604239, + 509.66154900000004, 509.40023700000006, 490.64878899999997, + 528.090142, 533.512394, 539.496917, 541.228523, 486.22617099999997, + 605.976369, 633.804161, 544.220842, 712.122594, 669.856356, + 614.594101, 775.447684, 561.2117840000001, 644.8879870000001, + 526.005843, 529.726263, 544.3235149999999, 501.615268, 539.748275, + 685.79768, 673.494497, 580.85241, 708.4780579999999, 581.413437, + 654.642186, 613.767938, 456.22251200000005, 567.059659, 492.861049, + 482.397807, 506.47949800000004, 546.948879, 533.826102, 566.196012, + 497.406144, 492.479781, 476.90006600000004, 503.11696900000004, + 436.215542, 488.994219, 551.506306, 443.777096, 488.411748, + 498.98365700000005, 452.115833, 495.56259400000005, + 442.54627600000003, 504.050891, 465.462716, 462.437995, 541.563916, + 481.86903199999995, 317.99299399999995, 101.534666, 90.421312, + 81.126958, 54.503482999999996, 70.350097, 63.757287999999996, + 81.117645, 57.831345999999996 + ], + "userExecutionMs": [ + 7.018401, 6.892335, 6.617774, 8.841840999999999, 6.602282, 6.635046, + 6.436261, 6.323462999999999, 6.291031, 9.262979, 6.513211, 6.843703, + 6.162479, 4.385005, 5.971685, 6.05085, 4.411082, 4.533378, 6.539156, + 5.992643, 7.2508099999999995, 5.965082, 6.18147, 5.893259, 7.058249, + 6.145981, 6.284321, 6.5696579999999996, 5.005507000000001, + 6.1891359999999995, 6.050622000000001, 4.978763, 6.907115, + 6.275402000000001, 10.681745, 6.994434, 5.717763, 10.064445, + 6.136426, 6.761269, 6.302998, 6.512034, 9.491359, 6.497882, + 8.500294, 4.464575, 6.253305, 6.132776000000001, 9.542959, + 5.344701000000001, 7.011767000000001, 4.871898000000001, + 5.4618660000000006, 4.8040210000000005, 8.422949000000001, + 4.819007999999999, 6.0117069999999995, 4.383365, 5.940615, 6.232594, + 5.9843910000000005, 6.105464, 6.723898, 6.115017, 6.704791, 6.0886, + 6.690385, 6.348444, 6.800888, 5.995218, 6.788263, 4.992094, + 7.159024, 6.289309, 7.281711, 6.368075, 7.25544, 6.954669, 3.829615, + 5.208782, 3.1411830000000003, 2.18031, 2.241855, 2.593171, 2.093739, + 6.454538, 2.204297, 2.2522789999999997 + ], + "readDocuments": 16238, + "writeDocuments": 5294, + "readBytes": 9072094, + "writeBytes": 1249462, + "ioReadBytes": 9072094, + "ioWriteBytes": 1249462, + "writeIndexRows": 20838, + "occExecutions": 0 + } + } + }, + { + "label": "mutation-measured-3-ordering", + "phase": "measured", + "rep": 3, + "variant": "ordering", + "workload": "mutation", + "began": "2026-09-10T01:16:14.589085+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "mutation", + "pollTimeoutMs": 180000, + "pool": "ordering" + }, + "result": { + "enqueueTotal": 12306, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 34776, + "p50": 18052, + "p95": 33262, + "p99": 34437 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 46950 + }, + "tasksPerSec": 106, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003023754, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789002975153.9414, + "_id": "jd78fstsfdxa2fj48f6sd5nwrn8e5bzk", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "mutation" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789002975153, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 93.432996, 74.31876799999999, 148.602698, 103.832018, + 141.52016700000001, 85.902202, 146.576976, 105.385033, 106.550861, + 146.67719200000002, 99.830412, 159.269194, 86.03869399999999, + 108.027395, 104.445333, 88.145757, 145.084048, 103.593345, 154.9983, + 79.462235, 112.71365899999999, 91.520865, 80.229548, + 123.81964099999999, 105.898426, 96.98062, 140.28489199999999, + 109.367013, 115.282374, 100.076241, 130.666102, 108.537515, + 100.92266699999999, 178.49473799999998, 79.904418, 116.089882, + 92.35838, 87.573608, 146.670111, 105.527676, 139.392241, 91.589959, + 105.023751, 153.031682, 85.21517800000001, 110.446271, + 93.14235699999999, 135.61968299999998, 98.68987999999999, 97.240758 + ], + "userExecutionMs": [ + 17.539167000000003, 15.013648, 16.962096, 17.876614, 16.571023, + 17.305031, 19.657882, 16.613125999999998, 16.975412, 16.841207, + 17.231863999999998, 17.185254, 20.298634999999997, 16.497007, + 16.819882, 17.049076, 17.71176, 15.922373, 17.426968000000002, + 15.742529000000001, 17.257501, 21.127643000000003, + 15.729801000000002, 16.770455, 16.479079000000002, + 17.392637999999998, 17.17307, 19.455797, 16.979304, 16.509648, + 16.319802000000003, 17.126707, 20.153072, 16.795142, + 15.625452000000001, 18.057131000000002, 16.691144, + 17.023994000000002, 18.389112, 22.272391, 16.665783, + 15.479728999999999, 16.189077, 16.611905999999998, 20.723677, + 16.029242999999997, 15.725550000000002, 16.887800000000002, + 18.264157, 16.255692 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2598953, + "writeBytes": 4275604, + "ioReadBytes": 2598953, + "ioWriteBytes": 4275604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 88, + "executionMs": [ + 200.036496, 319.43583800000005, 499.528945, 471.251742, 489.98132, + 549.153046, 471.518428, 513.704437, 459.06912700000004, 521.090301, + 560.45023, 520.6156100000001, 476.595289, 469.641975, 536.255737, + 505.966268, 536.7028750000001, 499.835864, 546.745953, + 453.05734199999995, 517.2663560000001, 542.4119039999999, + 483.426603, 477.736076, 512.631587, 528.3319829999999, 492.984169, + 570.034801, 480.920908, 495.48456200000004, 621.7229199999999, + 529.9510720000001, 488.417003, 584.388755, 678.5741039999999, + 471.023948, 502.164877, 456.061172, 513.07826, 466.15849, + 485.162714, 469.234188, 501.44968199999994, 513.470598, + 492.65061199999997, 466.32744599999995, 508.303131, 508.808853, + 510.70490700000005, 476.867364, 480.11695199999997, + 510.33756200000005, 460.434939, 521.408424, 523.629042, 509.684955, + 503.527812, 602.1994619999999, 528.147143, 466.54235, + 505.71576699999997, 493.725545, 474.977003, 476.038513, 583.678271, + 530.907213, 479.587631, 535.790804, 459.027103, 459.938947, + 520.581732, 521.6492069999999, 463.267064, 477.067146, 527.228991, + 467.192462, 465.807931, 475.017315, 435.86377, 328.983923, + 107.472812, 75.08669499999999, 73.949191, 64.371708, 75.674426, + 64.591166, 82.15196399999999, 52.167631 + ], + "userExecutionMs": [ + 3.168073, 4.376039, 5.540283, 6.259073000000001, 5.723543, 5.340038, + 6.09045, 6.04132, 4.985783, 6.59323, 4.27405, 5.646412, 6.37065, + 4.488229, 6.176597, 6.372097999999999, 4.902438, 4.406155999999999, + 5.350522, 6.802773, 9.130719000000001, 6.264554, 6.514985, 4.939611, + 4.497576, 6.124308999999999, 6.851149, 7.117057000000001, 4.880693, + 5.122426, 5.916365, 5.486056, 4.853312, 4.8581579999999995, + 7.120612, 6.70831, 4.952297, 6.436878, 6.1570540000000005, 6.800056, + 6.760396999999999, 4.368938, 5.637073, 4.7984789999999995, 5.672292, + 9.351693, 6.896814, 5.485576, 6.983249000000001, 5.433316, 4.946598, + 6.503641, 6.319104, 5.507714, 5.178081000000001, 4.855808, 6.632834, + 5.007098, 4.416963, 6.054486000000001, 4.997383, 6.646179, 6.207708, + 5.257073, 5.363187, 6.62774, 6.183432, 6.951798, 5.0760700000000005, + 5.088063, 6.006755, 5.016922, 5.970465, 5.51044, 5.534434, 4.443343, + 6.15824, 5.063895, 4.054911, 4.674434, 2.6859539999999997, 2.390763, + 2.2353020000000003, 2.919693, 2.164034, 2.248485, 2.277081, 2.372486 + ], + "readDocuments": 30973, + "writeDocuments": 10169, + "readBytes": 10225542, + "writeBytes": 1103618, + "ioReadBytes": 10225542, + "ioWriteBytes": 1103618, + "writeIndexRows": 40338, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-1-baseline", + "phase": "warmup", + "rep": 1, + "variant": "baseline", + "workload": "action", + "began": "2026-09-10T01:17:14.344710+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "baseline", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 11457, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 32549, + "p50": 14235, + "p95": 30074, + "p99": 31975 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 43892 + }, + "tasksPerSec": 114, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003080316, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003034988.7893, + "_id": "jd78m8zws2qfk3kn44jh0x0f9x8e4nqz", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003034988, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 252.90711100000001, 81.19988000000001, 148.533022, 100.323369, + 85.994742, 112.942041, 83.518752, 97.48174900000001, 91.041004, + 93.418353, 85.076752, 106.23061200000001, 82.995759, 130.827942, + 95.451323, 118.681787, 79.492762, 106.895173, 80.778054, 106.623417, + 97.274623, 153.53972, 92.63253, 107.87849800000001, + 90.11631200000001, 74.333944, 135.685218, 74.841848, 117.570596, + 68.140782, 81.145152, 99.815128, 73.99465099999999, + 125.38275400000002, 66.25859000000001, 121.96877, 72.45949, + 86.235221, 91.542657, 80.57183599999999, 129.864842, + 83.50413800000001, 148.661203, 80.323801, 148.72606299999998, + 81.392679, 139.269053, 85.168935, 132.09575700000002, 86.822294 + ], + "userExecutionMs": [ + 21.00341, 13.258726, 18.168132, 18.291595, 15.944442, 17.985888, + 14.738968999999999, 14.449171, 15.131967, 12.968275, 14.646372, + 15.060636, 14.659229, 15.4979, 18.475959, 18.596425, 13.402958, + 19.272119, 15.234857, 14.389012, 14.925492, 17.134611, 15.781725, + 14.740796, 18.428455, 12.68261, 14.693954999999999, + 15.319522000000001, 14.911595, 13.76325, 13.099222000000001, + 15.695431999999998, 16.388627, 17.465975, 12.77485, + 15.066170999999999, 16.147095, 12.558652, 14.617012, 13.932018, + 14.792459000000001, 14.564363, 14.892327, 12.736912, 18.865867, + 13.727245, 14.746331999999999, 15.172166, 18.760994, 16.082301 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4100604, + "ioReadBytes": 23953, + "ioWriteBytes": 4100604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 84, + "executionMs": [ + 145.70792699999998, 216.73429099999998, 315.91794, 405.652754, + 405.37614, 404.405737, 426.41333199999997, 404.46169599999996, + 408.720822, 407.075332, 452.930018, 554.352369, 470.352322, + 465.444412, 478.289195, 422.136204, 473.413441, 498.613937, + 431.66326599999996, 436.59185299999996, 425.540855, 419.787273, + 433.97267, 461.582148, 479.473115, 439.322008, 527.332884, + 467.224749, 545.378063, 519.092326, 471.30157, 500.01382900000004, + 470.78794600000003, 515.1407300000001, 525.906662, + 460.20043100000004, 533.678175, 634.2057599999999, 543.638839, + 480.645179, 558.3911489999999, 475.41917700000005, 522.041259, + 528.117984, 577.937494, 566.122162, 475.458274, 537.252752, + 502.22538000000003, 589.3638209999999, 509.73805000000004, + 569.482344, 498.91652, 548.2231290000001, 517.982735, + 554.8882090000001, 531.906844, 570.046362, 499.917888, 575.678608, + 506.12914200000006, 595.946006, 506.890531, 606.49399, + 521.5139989999999, 568.8637709999999, 521.293671, 589.316134, + 521.902307, 518.797324, 582.060348, 826.047817, 533.495155, + 605.841712, 911.044936, 722.3999960000001, 540.0562279999999, + 540.8634490000001, 452.586943, 276.64105700000005, 213.697914, + 90.02415599999999, 107.707673, 40.443743999999995 + ], + "userExecutionMs": [ + 3.4177, 3.098739, 4.537288, 4.404557, 4.433719, 4.723647, 8.244287, + 4.385626, 4.33693, 4.975048, 4.366183, 4.666187, 6.40777, 4.48009, + 4.2987269999999995, 4.852887, 4.824302, 8.026772, 4.845606, + 4.5467260000000005, 4.267391, 4.751832, 4.504074, 4.686509999999999, + 5.070384, 4.316387, 4.447777, 6.416698, 4.76572, 4.66196, 4.688072, + 4.405788, 6.524032, 4.9208430000000005, 4.790665000000001, 6.615161, + 4.107549000000001, 4.457096, 6.191438, 5.662791, 4.677729, 4.385953, + 4.703897, 4.91906, 8.087045, 4.230625, 4.882724, 5.923591, 4.621765, + 4.909431, 4.594923, 4.8557749999999995, 5.673366, 4.331992, + 6.297648000000001, 5.301323999999999, 4.608605, 6.340858000000001, + 5.037742, 4.9372039999999995, 4.416029, 4.838596, 4.724706, + 7.040549, 4.522111, 4.755077, 6.667689, 6.476102, 4.573503, + 5.1848149999999995, 8.955629, 4.382803, 8.475603, 4.915705, + 6.142867, 5.000137, 4.552577, 3.9864389999999994, + 4.4215729999999995, 5.736809, 4.02581, 2.109981, 2.3580010000000002, + 2.1006169999999997 + ], + "readDocuments": 30869, + "writeDocuments": 10165, + "readBytes": 10831240, + "writeBytes": 1411203, + "ioReadBytes": 10831240, + "ioWriteBytes": 1411203, + "writeIndexRows": 40330, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-1-ordering", + "phase": "warmup", + "rep": 1, + "variant": "ordering", + "workload": "action", + "began": "2026-09-10T01:18:10.397949+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "ordering", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 11963, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 24725, + "p50": 12761, + "p95": 23389, + "p99": 24252 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 36612 + }, + "tasksPerSec": 137, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003129471, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003090975.4277, + "_id": "jd7a163n0x779wr6vra84bj77s8e5aab", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003090975, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 95.681848, 89.580963, 160.625768, 135.836803, 95.631584, 145.0449, + 90.070094, 124.129671, 88.591625, 124.178088, 87.295996, + 107.38139100000001, 82.038163, 111.098913, 86.685652, + 123.11316599999999, 80.55546, 130.933778, 80.473534, + 125.16805000000001, 101.568287, 108.868254, 83.218718, 125.189949, + 82.72955499999999, 110.748068, 94.96292100000001, + 138.53732799999997, 82.779195, 114.187738, 83.36864700000001, + 132.393236, 87.999924, 106.658295, 101.89037300000001, + 124.75154099999999, 101.821611, 170.20854799999998, 89.389501, + 157.665239, 86.781871, 152.46875100000003, 100.389349, + 183.78536400000002, 87.82200599999999, 134.793969, + 74.58345700000001, 127.085488, 118.04319100000001, 137.420488 + ], + "userExecutionMs": [ + 18.336973999999998, 19.989381, 22.624744, 16.645881000000003, + 20.470734, 18.14428, 15.940861, 16.256471, 15.948160999999999, + 17.604089, 14.757394999999999, 16.31689, 15.564691999999999, + 18.543575999999998, 15.034067, 15.583962, 15.293743000000001, + 19.716268, 14.882503, 16.227330000000002, 19.806196, + 18.170005999999997, 15.388327, 16.021598, 15.116476, 18.964896, + 20.380546, 16.901707, 15.304554999999999, 19.019714999999998, + 16.516148, 15.318564, 16.660924, 16.539505, 15.100094, + 17.792067000000003, 15.050569999999999, 24.685195, 17.16755, + 16.752536, 15.227948, 17.386255, 16.869218, 17.585673999999997, + 17.122772, 16.765004, 16.829862000000002, 17.019162, + 17.848967000000002, 15.797063999999999 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2543953, + "writeBytes": 4220604, + "ioReadBytes": 2543953, + "ioWriteBytes": 4220604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 89, + "executionMs": [ + 162.552098, 253.68987900000002, 388.586714, 410.389367, 432.011725, + 410.04236199999997, 406.761418, 415.52876, 406.996548, 411.150032, + 402.352808, 424.078836, 425.772914, 438.362721, 420.226071, + 439.660041, 413.51015, 413.206099, 525.219277, 424.250831, + 430.40686500000004, 456.087581, 453.100825, 406.978225, + 429.83392799999996, 409.630087, 418.67153099999996, + 412.07392999999996, 398.364686, 407.30086, 415.145445, 412.915957, + 392.576167, 390.39768200000003, 412.507438, 433.436288, 445.356766, + 415.95707999999996, 404.81337499999995, 407.525306, + 412.76189200000005, 404.025369, 419.019405, 423.339536, 415.151237, + 400.309082, 398.02723899999995, 444.708303, 454.552217, 427.622991, + 396.981327, 418.968846, 409.904354, 403.735274, 412.961658, + 403.03042500000004, 394.689858, 418.39682799999997, 394.455893, + 402.589573, 403.120508, 402.774792, 406.228478, 397.125077, + 410.00586599999997, 397.44527600000004, 402.057307, 401.050876, + 398.93982, 415.731695, 401.34564900000004, 397.177123, 404.639617, + 401.80029499999995, 409.61288, 400.2819, 399.163365, 408.391192, + 422.383367, 306.779112, 98.528898, 62.233521, 59.079211, 61.497642, + 48.481744, 60.881137, 53.805063, 55.060252, 53.945918 + ], + "userExecutionMs": [ + 3.043253, 3.1504369999999997, 8.619988000000001, 4.567015, + 6.695373999999999, 6.289784, 5.932624, 5.033152, 6.319263, + 4.8399339999999995, 9.611464999999999, 4.937368, 5.881946, + 5.646738999999999, 4.387551, 6.406333, 6.254662000000001, 4.978839, + 6.533379, 6.854984, 4.821934, 6.110926, 5.963218, 8.385379, + 6.5349580000000005, 4.285616, 6.861368, 4.512867, 4.649336, + 4.914875, 4.45888, 4.528614, 4.442591, 4.6078600000000005, 4.357881, + 4.365913, 4.738244, 4.385066, 4.5240849999999995, 4.9428, + 4.2500990000000005, 4.404715, 5.87391, 7.014879, 4.375431, 4.305957, + 4.529062000000001, 4.519641, 4.392669, 4.922263, 4.473496000000001, + 4.237712, 6.541999, 4.569991, 4.599617, 5.0191539999999994, + 4.286427, 4.7510829999999995, 6.027803, 4.378758, 4.730478, + 4.462728, 4.878126, 4.639359, 4.7928, 4.47591, 4.225383, 4.288149, + 4.65003, 4.553523, 6.445257000000001, 4.304411999999999, 4.733975, + 4.331383000000001, 4.868565, 5.068893, 4.804424, 4.438776, 3.795069, + 3.7096080000000002, 2.63185, 2.184985, 2.24026, 2.293803, 2.17841, + 2.236561, 2.284208, 2.167296, 2.278022 + ], + "readDocuments": 30980, + "writeDocuments": 10170, + "readBytes": 10172165, + "writeBytes": 1103832, + "ioReadBytes": 10172165, + "ioWriteBytes": 1103832, + "writeIndexRows": 40340, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-1-packed", + "phase": "warmup", + "rep": 1, + "variant": "packed", + "workload": "action", + "began": "2026-09-10T01:18:59.984667+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "packed", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 10218, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 23411, + "p50": 11399, + "p95": 22132, + "p99": 22938 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 33634 + }, + "tasksPerSec": 149, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003175681, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003140528.9973, + "_id": "jd7fnd9rmmjpj3jmy9kg6mc7cs8e4ae7", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003140529, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 82.46185, 76.411949, 164.686174, 79.43051600000001, 129.338819, + 75.5047, 153.646904, 78.560608, 177.620439, 71.73903700000001, + 153.338906, 78.659437, 152.199761, 77.102405, 128.394493, 88.666276, + 126.09385200000001, 66.609077, 144.194458, 72.099014, + 141.22742699999998, 66.40916299999999, 102.112448, 136.293565, + 75.510829, 126.27553499999999, 74.982675, 138.110441, 80.960291, + 127.67399800000001, 75.907778, 143.513684, 68.977285, 139.248117, + 79.706769, 149.309642, 65.079621, 125.475805, 70.010874, 135.562133, + 70.07579199999999, 95.06589799999999, 77.107327, 64.047699, + 164.849674, 73.868268, 151.060173, 79.523207, 141.149865, + 74.24660800000001 + ], + "userExecutionMs": [ + 13.228475999999999, 11.879212, 17.248501, 12.188941999999999, + 13.280334, 16.628281, 14.359316, 14.364557, 14.69424, 14.631281, + 16.929693999999998, 13.658134, 12.945245, 15.775386, 14.562118, + 15.628192999999998, 13.776605, 13.961523, 17.842182, 12.708965, + 14.085454, 14.492702999999999, 12.557982, 14.373522, 12.507062, + 14.056733999999999, 13.435155, 11.440847, 16.561058, 13.982339, + 12.982743999999999, 12.569926, 12.331121, 14.52908, 14.971789, + 13.042817, 13.284387, 17.144543000000002, 13.830244, 18.303623, + 12.018674, 12.13865, 11.915790000000001, 11.555459, 13.64254, + 12.022423000000002, 12.017113, 12.541142, 14.307398000000001, + 12.146494 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2543953, + "writeBytes": 2948604, + "ioReadBytes": 2543953, + "ioWriteBytes": 2948604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 115.954414, 178.583825, 193.418871, 425.108559, 389.60398899999996, + 397.10511199999996, 391.591741, 407.095286, 393.042921, 373.866578, + 388.588082, 381.988432, 432.703423, 392.980014, 390.69475, + 405.243425, 380.66792300000003, 397.011608, 380.175675, 389.228496, + 397.519684, 399.727527, 428.300177, 381.370072, 399.750887, + 384.29873799999996, 409.029452, 373.277797, 378.51940299999995, + 382.81107, 384.62716600000005, 379.677452, 367.435843, 379.940686, + 379.516044, 387.280113, 399.4946, 382.81635400000005, 381.442365, + 381.48362000000003, 390.425834, 425.855869, 381.499949, 380.739971, + 376.16968699999995, 435.773965, 414.61385099999995, 415.288111, + 391.733271, 392.06625299999996, 427.33914699999997, 393.216377, + 387.510032, 374.671221, 395.11866699999996, 370.839582, 395.337551, + 388.69788300000005, 376.352206, 426.27336199999996, 524.414454, + 375.39615599999996, 375.86735, 372.108407, 412.64647199999996, + 400.640851, 394.121607, 390.3566, 382.470596, 421.86852400000004, + 386.52729000000005, 373.91348600000003, 389.69242599999995, + 377.62795900000003, 385.58250599999997, 382.231178, 369.960191, + 385.427719, 432.926292, 292.084361, 224.276161, 99.036558, + 58.331063, 54.252758, 57.286364999999996, 55.876308, 69.401342 + ], + "userExecutionMs": [ + 3.826685, 2.772227, 5.120737, 8.601786, 6.57568, 4.528351, 6.270077, + 6.0166059999999995, 5.191434, 6.217793, 4.507362, 5.052848, + 4.6149260000000005, 8.731536, 4.45061, 6.268073, 6.076117, + 4.557417999999999, 4.222357000000001, 4.356787, 5.312632000000001, + 4.843384, 4.537665, 6.567258000000001, 5.306984, 4.751854, 4.15632, + 4.369509, 4.224178, 4.368814, 5.280527, 4.371031, + 4.2728269999999995, 4.696645999999999, 4.225458, 4.348307, 4.65304, + 4.261055, 4.691647000000001, 4.5227059999999994, 5.948836, 4.157457, + 4.63677, 4.194243, 4.463113, 4.968867, 4.575509, 5.155261, 4.440897, + 4.517035, 5.01042, 4.576689999999999, 8.317066, 4.253697, 4.517348, + 4.2946800000000005, 6.0900490000000005, 4.938568, 4.495301, + 4.653599, 4.471977, 6.302537, 4.688302999999999, 4.163265, + 4.472936000000001, 4.650106, 6.208723999999999, 4.251624, 4.613861, + 5.119836, 5.179325, 4.765358, 6.221883999999999, 4.642119, 4.298929, + 4.47343, 4.379682, 4.872795, 4.0814889999999995, 3.70442, 4.276224, + 2.967753, 3.619459, 2.25334, 2.046275, 2.575659, 2.086827 + ], + "readDocuments": 16231, + "writeDocuments": 5293, + "readBytes": 9915769, + "writeBytes": 1549262, + "ioReadBytes": 9915769, + "ioWriteBytes": 1549262, + "writeIndexRows": 20834, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-2-ordering", + "phase": "warmup", + "rep": 2, + "variant": "ordering", + "workload": "action", + "began": "2026-09-10T01:19:46.026346+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "ordering", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 12413, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 26449, + "p50": 12628, + "p95": 24957, + "p99": 25996 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 38681 + }, + "tasksPerSec": 129, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003226892, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003186612.6265, + "_id": "jd7698c96m5rs3ftq9wc2krk9s8e55rq", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003186612, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 105.435624, 77.523171, 156.37579300000002, 152.131106, 102.804992, + 134.799499, 82.343895, 142.724378, 90.67521500000001, 157.329948, + 91.002822, 147.32267299999998, 119.396873, 133.899455, 120.965564, + 117.10285300000001, 84.95480699999999, 137.89197000000001, + 89.00946499999999, 116.333538, 79.969064, 128.08704500000002, + 73.95399900000001, 133.484898, 78.644585, 111.33131999999999, + 80.802845, 121.701436, 109.63793700000001, 110.973501, + 82.39356000000001, 141.61030599999998, 85.23793500000001, + 99.94587200000001, 86.54334700000001, 99.554643, 73.895236, + 96.894049, 73.215659, 118.70826199999999, 79.472491, 107.10727, + 79.108716, 138.568519, 80.86679500000001, 138.839849, 100.28276, + 163.853294, 74.670187, 143.628319 + ], + "userExecutionMs": [ + 20.199631, 15.272763999999999, 21.548489, 18.056215, 15.486867, + 17.203333, 17.814047, 18.015102000000002, 16.31063, 17.362354, + 16.276138999999997, 21.135523, 17.450057, 17.888367, 21.435097, + 17.132464, 15.170928, 16.228644, 15.364101999999999, 17.547796, + 15.411237, 16.237821999999998, 15.045501, 15.968870999999998, + 15.395458000000001, 15.486208, 15.404671, 15.658102999999999, + 15.745213000000001, 17.276436, 16.776574999999998, 15.441318, + 16.226436, 18.902387, 14.569676, 15.231465, 15.553401, + 17.051004000000002, 15.138495, 16.118131, 15.686192000000002, + 15.501106, 15.373745, 15.563675, 15.619717999999999, 19.009849, + 16.058823999999998, 15.658568, 16.916385, 19.595866 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2543953, + "writeBytes": 4220604, + "ioReadBytes": 2543953, + "ioWriteBytes": 4220604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 170.563527, 269.22220200000004, 455.34710299999995, 454.021844, + 423.167607, 431.613337, 427.39561199999997, 491.271044, 471.154177, + 431.699703, 460.06766999999996, 451.66636900000003, 402.443129, + 402.12883, 410.483361, 463.108891, 425.57657, 394.83829399999996, + 393.462674, 515.207169, 400.134767, 420.421337, 398.67964900000004, + 528.369321, 444.684078, 407.86838, 395.031258, 420.24699799999996, + 404.47021700000005, 408.39152099999995, 427.528435, 410.65254, + 426.50891, 400.73215600000003, 393.049423, 396.326426, 428.835279, + 425.644766, 424.52496, 393.53758100000005, 452.38624599999997, + 416.040322, 399.013324, 414.102374, 410.02624199999997, + 413.44140600000003, 415.145751, 652.3187750000001, 561.824035, + 549.816223, 422.662506, 422.722353, 540.532256, 451.79229599999996, + 413.161813, 424.745814, 473.434008, 408.15788, 414.497731, + 403.597951, 409.12861, 446.70794600000005, 417.097065, 425.898202, + 517.9577039999999, 441.317436, 431.185461, 417.167096, + 426.09310500000004, 428.702229, 415.243195, 418.482837, 437.581903, + 488.302888, 424.32033, 419.25933100000003, 459.18820999999997, + 460.746606, 429.26636099999996, 288.144271, 122.965293, 91.465923, + 55.922067, 78.33417800000001, 56.897253, 67.283303, + 46.268795000000004 + ], + "userExecutionMs": [ + 3.294852, 3.312084, 4.442873, 8.887222999999999, 4.468272, 4.517643, + 5.2933140000000005, 6.267475, 8.251719, 6.519567, 4.453793999999999, + 5.990426, 4.810952, 6.3381289999999995, 5.923852, 4.996366, + 7.014757, 4.826304, 6.101401, 4.9685559999999995, 6.126916, 4.92188, + 6.370426, 5.121896, 6.232911, 7.2257, 6.2388449999999995, 4.494382, + 4.298977, 4.296441, 4.579865, 4.369695, 4.204775, 4.656433, + 4.216518, 4.129360999999999, 4.553693, 4.194579, 4.1999059999999995, + 4.161347, 4.832164, 4.330143, 4.324875, 4.753226, 4.298515999999999, + 4.304693, 4.842264, 4.407973, 4.7397089999999995, 5.208894000000001, + 4.404243, 5.169993000000001, 4.404334, 7.986897999999999, 4.477705, + 4.715247, 4.977405, 6.451197, 4.437516, 4.286453, 4.569539, + 4.931981, 4.599942, 5.079108, 4.803980999999999, 4.513124, 4.946258, + 6.003868, 4.300334, 4.574911, 4.264691999999999, 4.829218, 4.632745, + 4.517333, 4.886993, 6.2102829999999996, 7.508079, 8.921322, + 4.133905, 4.441999, 2.829615, 2.531631, 3.427611, 3.848404, + 2.269355, 2.545579, 2.275747 + ], + "readDocuments": 30966, + "writeDocuments": 10168, + "readBytes": 10168919, + "writeBytes": 1103404, + "ioReadBytes": 10168919, + "ioWriteBytes": 1103404, + "writeIndexRows": 40336, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-2-packed", + "phase": "warmup", + "rep": 2, + "variant": "packed", + "workload": "action", + "began": "2026-09-10T01:20:38.911882+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "packed", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 11763, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 26755, + "p50": 13793, + "p95": 25500, + "p99": 26288 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 38306 + }, + "tasksPerSec": 131, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003279747, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003239485.237, + "_id": "jd768hv5k4q5tg80w7hkbe1ynh8e48vt", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003239485, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 174.431838, 95.045915, 195.183745, 150.14988200000002, 82.08869, + 117.46609099999999, 71.31438800000001, 81.521317, 130.687073, + 68.303829, 118.618667, 95.86468099999999, 155.221916, 62.648938, + 126.87594599999998, 89.53686800000001, 98.13616999999999, + 127.222377, 85.851178, 236.88283099999998, 69.831668, 116.092061, + 98.139888, 110.917703, 82.87301099999999, 100.342192, 118.321993, + 95.86501799999999, 132.69446100000002, 76.781246, 144.704751, + 72.10373100000001, 138.595845, 88.003457, 128.55989300000002, + 120.010854, 101.016205, 124.72210799999999, 74.563459, 136.269777, + 97.577896, 144.638544, 69.99059100000001, 140.65334099999998, + 233.689768, 95.46025, 125.95657800000001, 124.70280100000001, + 90.669359, 178.123829 + ], + "userExecutionMs": [ + 21.037886, 13.760179, 24.796136999999998, 18.336288999999997, + 14.773741, 13.117992000000001, 13.749528, 14.920421999999999, + 16.102531, 14.139574, 16.506563, 14.644959, 14.704735, 11.539657, + 14.469334, 14.572803, 12.228373000000001, 12.950384, 14.375608, + 23.638348, 11.641957, 14.030289, 17.312697, 13.653293, 13.660822, + 12.266205000000001, 14.471269999999999, 19.210213, + 14.088605000000001, 12.363301, 15.365343, 12.115804, 16.246056, + 15.039629, 14.399333, 16.472624000000003, 12.620619, 13.892413, + 13.724333999999999, 14.760692, 15.587777, 14.950339, 13.988426, + 14.713356000000001, 35.515992000000004, 12.123258, 16.095666, + 15.161100000000001, 12.092996999999999, 19.046192 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2543953, + "writeBytes": 2948604, + "ioReadBytes": 2543953, + "ioWriteBytes": 2948604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 89, + "executionMs": [ + 133.231615, 199.873798, 167.26269499999998, 374.059971, 363.356774, + 381.53461, 495.687344, 406.116254, 383.308385, 587.543552, + 393.928269, 541.766932, 387.086741, 453.160139, 472.984582, + 425.907417, 433.473032, 463.369151, 534.818905, 483.482908, + 394.761344, 892.885146, 822.568458, 541.604039, 434.813945, + 374.07154799999995, 377.342854, 380.800494, 384.278049, + 375.15950300000003, 388.187635, 392.690564, 458.782784, 391.868451, + 406.634454, 424.951118, 436.52086699999995, 427.644539, 409.370796, + 429.051668, 422.436713, 511.680244, 400.179669, 408.58307, + 405.36884499999996, 434.40877500000005, 417.257653, 422.114093, + 415.604507, 437.602323, 430.673995, 402.454875, 383.55184199999997, + 391.524663, 508.956949, 463.18891099999996, 420.994738, + 424.22970399999997, 633.114406, 806.6198909999999, 555.484878, + 407.54330799999997, 395.360887, 399.808606, 414.48611, 441.738948, + 499.0266, 459.425207, 439.742309, 384.542583, 374.756382, + 424.141852, 438.716104, 433.84126000000003, 368.567988, 391.164823, + 410.880164, 399.810673, 439.778657, 329.77814700000005, 221.831047, + 117.31881800000001, 50.743556000000005, 63.274345, 67.211279, + 56.008646, 63.873283, 58.527646000000004, 65.340537 + ], + "userExecutionMs": [ + 7.047137, 2.788525, 4.924478000000001, 6.311221, 4.354621000000001, + 4.642393, 7.230378, 7.1665030000000005, 4.231185999999999, 7.160184, + 6.126394, 6.937518, 5.841006, 4.678605, 6.3460410000000005, + 5.944037, 4.417461, 6.055301, 5.059326, 6.835266000000001, 6.392577, + 6.338623, 6.526182, 4.791311, 4.683929, 4.700996, 4.420056000000001, + 4.877529, 4.410945, 4.39833, 6.737911, 6.895341999999999, 6.8299, + 4.3563279999999995, 6.033863, 4.686337, 4.941824, 4.469391, + 4.532588, 4.331482, 4.439327, 4.882841, 4.388309, 4.326135, + 5.106325, 4.611433, 4.576391, 4.947045, 4.6355260000000005, + 4.499053, 5.593375, 9.089526000000001, 4.534161999999999, 4.327641, + 4.406681, 4.93811, 4.608686, 5.046047, 5.611231, 4.548112, 4.929304, + 5.967122, 4.621377000000001, 4.186344, 4.576583, 4.738376, 4.963079, + 4.639359, 6.104651, 4.35671, 4.636227, 4.557224000000001, 4.648328, + 5.276617, 4.2988100000000005, 4.263541, 6.648511999999999, 4.645342, + 4.211439, 7.750682, 3.5862979999999998, 7.359051999999999, 2.292819, + 2.508563, 2.318905, 2.148609, 2.2895630000000002, 2.06235, 2.371639 + ], + "readDocuments": 16245, + "writeDocuments": 5295, + "readBytes": 9868771, + "writeBytes": 1532942, + "ioReadBytes": 9868771, + "ioWriteBytes": 1532942, + "writeIndexRows": 20838, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-2-baseline", + "phase": "warmup", + "rep": 2, + "variant": "baseline", + "workload": "action", + "began": "2026-09-10T01:21:30.131324+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "baseline", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 16909, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 38698, + "p50": 19836, + "p95": 36306, + "p99": 37838 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 55556 + }, + "tasksPerSec": 90, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003348082, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003290673.4326, + "_id": "jd7d7m35c7d5tk6atshr8n20hx8e5afg", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003290673, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 162.311881, 116.670002, 122.92254399999999, 93.54794199999999, + 642.270719, 83.590438, 795.016571, 72.35368, 161.58218, 157.410091, + 65.88757100000001, 73.702323, 557.243928, 65.133477, 812.636665, + 241.810523, 121.642831, 81.696746, 65.564295, 95.591545, + 223.67376099999998, 66.564301, 81.928447, 268.868411, 65.311566, + 275.471287, 150.82878499999998, 201.533203, 78.15159200000001, + 150.063588, 95.438203, 66.188415, 118.78905200000001, + 918.4456560000001, 97.50763699999999, 555.247222, 82.65263300000001, + 277.02585899999997, 220.986707, 77.403609, 72.104912, 168.888797, + 70.54572200000001, 505.51775200000003, 77.19481, 79.416994, + 68.29579600000001, 288.444119, 72.43915, 84.718949 + ], + "userExecutionMs": [ + 28.85716, 20.061612999999998, 25.901407000000003, + 14.915980000000001, 33.639636, 14.682353, 30.217613, 14.191886, + 34.596275000000006, 27.344440000000002, 14.422014, + 16.663556999999997, 34.917131, 12.835835, 34.648856, + 26.284095999999998, 18.294303000000003, 13.43619, + 14.565862000000001, 17.35358, 19.376368000000003, + 14.157625999999999, 14.727032, 24.194874000000002, 13.352904, + 21.622735000000002, 20.508202999999998, 16.924979999999998, + 16.705674000000002, 17.187867, 16.445886, 13.176326, 14.806844, + 20.322625000000002, 14.599249, 28.502035, 15.081917, 22.269501, + 24.830612, 12.364348000000001, 15.676314, 16.884059999999998, + 15.163711999999999, 28.891002, 14.212162000000001, 13.330467, + 13.498326, 22.685072, 14.830653, 14.485398 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4100604, + "ioReadBytes": 23953, + "ioWriteBytes": 4100604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 180.160598, 136.96329300000002, 637.398648, 339.792694, 294.440904, + 191.710285, 225.360827, 849.387448, 343.776429, 1122.725844, + 1213.4984220000001, 398.066142, 566.756657, 409.32176300000003, + 811.543213, 724.440327, 397.031142, 608.334833, 928.415894, + 389.660328, 1320.282309, 466.378084, 1006.040308, 359.896958, + 1134.222026, 816.08834, 434.13349, 414.489738, 447.07378, + 778.1928660000001, 1154.5573419999998, 410.327224, 1569.450616, + 447.84051000000005, 702.4253120000001, 446.2671, 717.285779, + 442.20801900000004, 461.12714900000003, 466.315095, 447.096163, + 492.050587, 491.865499, 522.170587, 458.60425100000003, 465.354818, + 492.426605, 650.799311, 904.297487, 652.118687, 891.923028, + 473.935208, 599.753483, 456.093062, 463.725122, 520.841085, + 473.35195699999997, 461.878494, 588.49764, 556.352145, 578.382559, + 691.731305, 502.52377400000006, 680.525577, 802.960825, 730.014805, + 520.033427, 525.280121, 1005.252996, 524.678163, 1216.838134, + 491.915622, 1147.528828, 455.94568200000003, 1075.697616, + 469.095003, 480.772598, 715.450496, 790.318302, 485.145076, + 841.1054839999999, 726.9736399999999, 359.52774900000003, + 229.807026, 79.670422, 74.248863, 37.108723000000005 + ], + "userExecutionMs": [ + 3.304792, 2.807009, 8.895965, 8.801222, 4.068801000000001, 5.249557, + 5.007646, 10.80967, 6.528285, 9.032943, 13.463341999999999, + 5.952828, 14.917559, 5.87285, 6.282476, 8.021867, 5.936035, + 4.530087, 6.260869, 6.519387, 10.643358000000001, 4.2129, 6.799965, + 6.044669, 13.069765, 4.659044000000001, 4.33075, 5.4314849999999995, + 4.626129, 9.874389, 7.201979, 6.162459999999999, 6.683583, 6.227377, + 10.197958, 6.750675, 4.614957, 4.427887999999999, + 4.6459079999999995, 4.44689, 4.328881, 4.776861, 4.274784, + 4.456472000000001, 4.20611, 4.174444, 5.243, 4.34199, 7.513431, + 6.048105, 9.08139, 6.237749, 4.583509, 4.371510000000001, 4.565017, + 4.7611989999999995, 4.313400000000001, 4.740054, 7.858020000000001, + 4.4025419999999995, 4.466005, 8.236830000000001, 6.313913, + 11.413775, 7.404064, 9.591446, 4.908528, 4.851208, 4.696089, + 5.069498, 7.764725, 4.443939, 8.310561, 5.923839999999999, + 8.461915000000001, 4.699417, 4.483094, 8.73752, 5.18248, 8.83853, + 12.444344, 5.255229, 6.516924, 2.928148, 2.1934620000000002, + 5.883710000000001, 2.176743 + ], + "readDocuments": 30971, + "writeDocuments": 10171, + "readBytes": 11824156, + "writeBytes": 1737244, + "ioReadBytes": 11824156, + "ioWriteBytes": 1737244, + "writeIndexRows": 40342, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-3-packed", + "phase": "warmup", + "rep": 3, + "variant": "packed", + "workload": "action", + "began": "2026-09-10T01:22:38.509609+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "packed", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 10546, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 26295, + "p50": 12891, + "p95": 24957, + "p99": 25878 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 36654 + }, + "tasksPerSec": 136, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003397441, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003359077.2053, + "_id": "jd72tp5gq869jttnszyp767bhx8e5a34", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003359077, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 221.350549, 90.030052, 141.946706, 100.804402, 159.39214900000002, + 71.212072, 142.747979, 82.76794699999999, 140.07561099999998, + 77.796431, 119.32715, 85.31780499999999, 65.75868799999999, + 128.08293700000002, 93.988064, 160.440015, 74.714397, 149.223737, + 83.709631, 144.282596, 64.94923399999999, 124.687179, 86.396029, + 96.00781599999999, 78.120642, 116.599587, 87.972417, 75.261854, + 105.922308, 86.99254300000001, 148.568159, 75.07511699999999, + 151.598542, 84.45169399999999, 153.465452, 71.34631800000001, + 132.28268400000002, 82.03838900000001, 151.921457, 77.342692, + 108.40592000000001, 105.42517699999999, 74.16067199999999, + 114.507588, 83.81424899999999, 161.574803, 75.596026, 166.02116, + 95.302235, 160.006834 + ], + "userExecutionMs": [ + 22.792603000000003, 12.771417999999999, 17.919435, 13.902913, + 14.370562, 13.923763000000001, 14.773105999999999, 14.202835, + 14.129658999999998, 13.961819, 13.562486999999999, 15.6827, + 11.846964, 17.333049, 13.080377, 13.138778, 13.103264, 14.104947, + 13.152657999999999, 15.609283999999999, 13.265551, + 14.889617000000001, 13.285625000000001, 11.955121, 16.398014, + 13.526387000000001, 12.406479999999998, 15.327615, 13.391736, + 11.577676, 12.18129, 15.694304999999998, 14.026373, + 13.656187000000001, 15.453957, 12.392167, 15.350607, 13.325782, + 11.803053, 12.539946, 13.320587, 14.688173, 13.316405, 17.601999, + 11.414971, 12.982321, 11.384366, 20.954792, 13.408725, 13.854283 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2543953, + "writeBytes": 2948604, + "ioReadBytes": 2543953, + "ioWriteBytes": 2948604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 116.88403699999999, 204.57690300000002, 219.256353, 376.237118, + 431.408955, 422.796082, 414.664438, 431.28488799999997, 381.708891, + 426.342689, 367.403523, 410.49833, 378.44585600000005, 374.521663, + 417.893347, 390.962027, 424.875153, 392.925592, 421.825907, + 488.780416, 382.58533300000005, 446.286521, 383.617215, 491.98554, + 414.324938, 434.808077, 436.848505, 421.396074, 483.01221300000003, + 433.72292, 428.703857, 438.41709899999995, 559.157857, 416.599233, + 434.998924, 418.63203999999996, 535.639076, 428.033913, 420.067943, + 426.99206899999996, 469.787029, 418.249614, 431.548504, 429.77833, + 516.202269, 495.103741, 538.9894370000001, 503.060053, + 630.6165259999999, 431.489486, 420.846431, 442.785951, 422.267925, + 412.234336, 417.346115, 376.234857, 392.04167, 377.196127, + 373.49566, 412.17156, 418.679596, 436.716184, 428.125537, + 431.945435, 376.764504, 448.108918, 392.292086, 430.559238, + 414.12582, 495.288697, 432.851248, 385.352288, 414.645802, + 415.210254, 440.443202, 423.371626, 434.418226, 391.570048, + 393.998892, 282.331693, 239.295794, 75.297734, 53.345044, 75.001699, + 52.531997, 76.510211, 59.33061 + ], + "userExecutionMs": [ + 3.309253, 2.773175, 8.500472, 4.599991999999999, 8.965555, 10.11319, + 5.0217480000000005, 7.522296000000001, 5.8724110000000005, 6.379595, + 5.923750999999999, 4.386776, 4.635345, 4.553441, 6.2824040000000005, + 6.535286, 7.133223999999999, 4.582057, 4.252671, 5.000778, + 4.320926999999999, 5.864362, 6.1651240000000005, 8.182784999999999, + 4.232919, 4.226339, 4.6094420000000005, 4.354634, 4.891025, + 4.780907, 4.258202, 4.647751, 4.240314000000001, 6.116083, 4.446586, + 4.243962000000001, 4.287955, 4.722262, 4.496485, 4.377693, + 5.2062100000000004, 4.4603, 4.382352, 4.693032, 4.563872, 5.128227, + 4.901527, 9.721967999999999, 4.379671, 4.811701, 4.77427, 4.442271, + 4.557611, 4.853597000000001, 4.290333, 7.352525, 5.214434000000001, + 4.147372, 4.8598490000000005, 4.45745, 4.45962, 4.691717, 4.21364, + 4.227972, 5.90979, 4.2206090000000005, 5.850655, 4.712502000000001, + 4.335408, 8.183739000000001, 4.804577, 4.283695, 4.577554, 4.279339, + 6.699592, 4.45903, 4.550351999999999, 6.589203, 4.438321999999999, + 3.524535, 5.290083, 2.3794210000000002, 3.3662449999999997, + 2.266222, 2.56, 2.1306049999999996, 2.2549710000000003 + ], + "readDocuments": 16228, + "writeDocuments": 5292, + "readBytes": 9342017, + "writeBytes": 1357944, + "ioReadBytes": 9342017, + "ioWriteBytes": 1357944, + "writeIndexRows": 20832, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-3-baseline", + "phase": "warmup", + "rep": 3, + "variant": "baseline", + "workload": "action", + "began": "2026-09-10T01:23:27.783894+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "baseline", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 16687, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 34764, + "p50": 21556, + "p95": 33873, + "p99": 34713 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 50245 + }, + "tasksPerSec": 100, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003460638, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003408363.6118, + "_id": "jd78vgp48x7xcr23vq08edyknd8e43ht", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003408363, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 103.934214, 81.130776, 143.65304500000002, 103.236339, 79.486311, + 95.293705, 119.35359500000001, 74.509298, 127.250642, + 68.38490999999999, 145.086683, 92.75167400000001, 141.613002, + 100.992501, 160.817033, 91.53221, 136.754691, 73.742445, 78.460944, + 110.356245, 77.340333, 152.951272, 67.693658, 155.936462, 70.916267, + 127.89482299999999, 61.127596, 104.32683, 64.543154, 82.370588, + 105.99901100000001, 79.449675, 123.396668, 80.494029, + 123.12625700000001, 87.022339, 122.409537, 79.813271, 115.392235, + 2027.3533160000002, 297.422828, 73.586502, 402.410248, 81.41036, + 116.91916300000001, 74.217956, 468.86735899999996, 385.806823, + 492.4419, 1145.651177 + ], + "userExecutionMs": [ + 18.617913, 13.042443, 18.444991, 16.721375000000002, 15.197134, + 14.17694, 20.713777999999998, 13.992016, 14.756338000000001, + 12.873703, 17.093708, 12.99017, 14.669934000000001, + 13.337527999999999, 16.888075999999998, 13.212541, 15.147142, + 12.810971, 16.892403, 15.730903000000001, 13.93835, + 14.808174000000001, 13.67117, 20.995366, 14.469541000000001, + 21.575084, 12.777818, 13.598568, 15.045805000000001, + 12.684857000000001, 14.36455, 15.595064, 18.647016999999998, + 12.562695, 13.792864999999999, 13.349038, 13.540021, 13.747753, + 16.361901000000003, 50.870047, 19.746967, 16.145853000000002, + 23.03941, 15.003234, 15.198659999999999, 14.478646, 27.853191, + 25.108845000000002, 36.944297, 30.229003000000002 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4100604, + "ioReadBytes": 23953, + "ioWriteBytes": 4100604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 187.019495, 268.744048, 437.133581, 426.40995399999997, 409.206393, + 417.153146, 425.30755, 582.0166009999999, 605.9830929999999, + 472.36198, 457.444759, 463.15480399999996, 464.705953, 407.436712, + 423.456706, 415.079635, 407.93733, 412.371256, 435.408362, + 350.641005, 257.739597, 201.167221, 164.539957, 92.524996, + 78.502485, 651.303254, 990.8028089999999, 410.218428, 605.695115, + 640.199162, 485.382654, 599.849081, 444.379145, 1346.891225, + 1519.2490599999999, 398.31323499999996, 1126.424516, 486.792774, + 2206.635229, 415.079017, 1168.073782, 407.637986, + 1045.2651799999999, 459.309664, 435.999799, 431.574595, 490.487251, + 821.511388, 455.534729, 467.35698399999995, 522.097273, 674.887045, + 457.253982, 444.809477, 742.040315, 441.964419, 443.25086000000005, + 483.687947, 492.903099, 476.559295, 456.532614, 548.741061, + 491.970446, 458.52813999999995, 474.620033, 548.065546, + 858.5715710000001, 499.480407, 496.88507599999997, 468.595328, + 490.445487, 501.56569299999995, 474.31321599999995, 536.032915, + 498.69721, 527.10214, 487.570654, 508.21962099999996, 500.684856, + 520.047965, 541.193842, 560.6921030000001, 687.313527, + 1178.4010110000002, 452.69116399999996, 304.12757, 173.827119 + ], + "userExecutionMs": [ + 3.098128, 6.497261, 8.808432, 4.6799479999999996, 4.20842, 4.622714, + 4.609055, 4.788912, 4.266272, 6.501323, 4.585451, 4.379765, 4.92932, + 5.867402, 4.542517999999999, 4.305763, 6.661201999999999, 4.528671, + 4.369548, 4.881393, 3.549352, 2.8980910000000004, 2.72819, 2.226884, + 2.227204, 3.624162, 10.776862999999999, 6.19999, 6.347143, 6.089072, + 4.961914, 6.0606469999999995, 7.359717, 5.606096, 15.787645, + 6.179987, 9.328235000000001, 6.043273, 5.756069, 5.893261000000001, + 7.681291, 4.417815, 6.070784, 9.548094, 4.573184, 4.707323000000001, + 4.274193, 6.148017, 4.20657, 4.636278, 4.219519, 4.520563, + 4.870997999999999, 4.459338, 4.503369, 5.248716, 4.211229, 4.529526, + 4.396914, 4.234285, 5.114403, 4.702875, 4.611988, + 4.1409769999999995, 4.683706, 4.876015, 4.300674, 6.572218, + 6.737436000000001, 4.160848, 4.348944, 4.384109, 4.341201, 9.474394, + 4.564885, 4.4817219999999995, 6.712249, 4.191189, 4.209526, + 5.0935950000000005, 4.127994, 4.366544, 4.845484, + 3.7597780000000003, 3.77448, 5.318294000000001, 2.349949 + ], + "readDocuments": 30954, + "writeDocuments": 10171, + "readBytes": 11543283, + "writeBytes": 1623400, + "ioReadBytes": 11543283, + "ioWriteBytes": 1623400, + "writeIndexRows": 40342, + "occExecutions": 0 + } + } + }, + { + "label": "action-warmup-3-ordering", + "phase": "warmup", + "rep": 3, + "variant": "ordering", + "workload": "action", + "began": "2026-09-10T01:24:32.481208+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "ordering", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 12575, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 26719, + "p50": 14293, + "p95": 25355, + "p99": 26348 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 39244 + }, + "tasksPerSec": 127, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003513975, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003473035.659, + "_id": "jd7fnhf9gq4wdqqmpjzf0eec9s8e4nm7", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003473035, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 142.584243, 80.39952100000001, 118.994631, 115.570032, 129.870451, + 82.48786600000001, 123.15899800000001, 91.39364, 102.63083300000001, + 77.275699, 122.53532200000001, 83.418409, 126.002792, 76.513039, + 93.564922, 88.712879, 175.775148, 88.288494, 116.561758, 91.730384, + 115.979406, 93.83290099999999, 83.23114199999999, 145.807277, + 84.687016, 113.87795, 90.096651, 141.598241, 84.658331, 150.113368, + 106.28616000000001, 86.125232, 95.036832, 95.997954, 207.460787, + 107.542424, 192.022255, 115.382454, 159.198028, 121.302949, + 137.4023, 92.258966, 166.392615, 91.540751, 145.182443, 95.472795, + 181.13524999999998, 81.165015, 99.769386, 89.277624 + ], + "userExecutionMs": [ + 25.163493, 15.72204, 21.076398, 20.706259000000003, + 17.462721000000002, 17.494166, 18.147969, 16.690407, + 17.153862999999998, 15.9228, 17.268906, 15.337319, 17.842263, + 15.61492, 17.429232, 17.236898, 21.469444, 16.634201, + 19.170537000000003, 16.707584, 19.037128, 20.470660000000002, + 15.199804, 18.202195, 19.561689, 17.657669000000002, 19.8958, + 20.259182, 15.677731999999999, 18.698242, 17.175092, 15.192804, + 19.647923000000002, 16.380869, 16.513747, 20.380848999999998, + 17.602726999999998, 15.295128, 17.407337, 17.0173, + 18.279887000000002, 16.076619, 15.393584, 16.889611, 17.61583, + 17.742262, 19.543218, 16.314301999999998, 18.55294, 15.784247 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2543953, + "writeBytes": 4220604, + "ioReadBytes": 2543953, + "ioWriteBytes": 4220604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 88, + "executionMs": [ + 210.432078, 329.780884, 397.114874, 404.083004, 414.589716, + 408.666112, 395.146256, 397.027196, 439.69202, 591.644036, + 462.64516, 422.146897, 463.81235, 436.241931, 705.008706, 475.76262, + 526.918378, 554.670087, 414.07898900000004, 468.05543500000005, + 462.883908, 440.46676, 393.005069, 424.835337, 387.078824, + 395.960697, 393.694287, 405.773129, 405.35733400000004, + 393.15112600000003, 394.423635, 411.236024, 395.259615, 404.719291, + 501.06510299999997, 430.659358, 543.502094, 504.065341, 440.346549, + 387.683965, 458.503922, 533.968763, 396.212783, 363.324634, + 442.705956, 396.795951, 383.450582, 421.83323099999996, 407.500017, + 441.971709, 399.58912000000004, 403.85549199999997, 430.163517, + 406.21682500000003, 407.29018299999996, 418.057246, + 382.21796600000005, 402.056872, 418.192858, 416.20170099999996, + 411.402071, 444.704178, 453.604562, 516.677716, 470.91001900000003, + 416.047972, 401.630123, 485.161258, 689.830977, 477.417028, + 433.893464, 438.930275, 493.695699, 412.966825, 417.72671, + 391.322645, 466.986853, 461.61913599999997, 355.578255, 239.009575, + 95.88037000000001, 56.637898, 76.673551, 63.985744, 53.651575, + 64.321894, 61.526961, 64.894617 + ], + "userExecutionMs": [ + 3.433375, 9.527258, 5.247681, 8.551304, 6.156995, 6.946912, + 6.447185999999999, 6.344111000000001, 5.962188, 13.271371, + 9.451953000000001, 6.245308, 7.029896, 6.433328, 6.056208000000001, + 5.234726, 6.197907, 6.351052, 7.688314, 4.185989999999999, + 6.0791710000000005, 4.570982000000001, 6.371531, 6.286038, 9.574895, + 4.413168000000001, 4.437825999999999, 4.719511, 4.4313649999999996, + 4.623048, 4.881608, 5.189153999999999, 4.401422, 4.783082, 4.467982, + 5.281272, 4.454552, 6.894474000000001, 4.559769, 4.78812, + 5.0291559999999995, 4.594653, 5.219937, 5.078539, 9.539346, + 4.937404, 4.880169, 4.9096720000000005, 4.5769649999999995, + 6.104971, 5.078403000000001, 4.518679000000001, 4.612591, 4.843902, + 4.289057, 6.8458570000000005, 4.4823390000000005, 6.210642, + 4.648014, 6.135238999999999, 4.964495, 4.459149, 5.530824, 9.402443, + 4.673772, 9.417026, 6.490079000000001, 4.908531, 4.530659, 7.085738, + 4.491099999999999, 4.489077, 4.372778, 4.779104, 4.795043, 4.389836, + 4.992787, 7.817576, 3.849418, 3.7283910000000002, 2.648967, + 2.276547, 2.2695369999999997, 4.15321, 2.366988, 2.45564, 2.46755, + 2.406187 + ], + "readDocuments": 30973, + "writeDocuments": 10169, + "readBytes": 10170542, + "writeBytes": 1103618, + "ioReadBytes": 10170542, + "ioWriteBytes": 1103618, + "writeIndexRows": 40338, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-1-packed", + "phase": "measured", + "rep": 1, + "variant": "packed", + "workload": "action", + "began": "2026-09-10T01:25:24.212776+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "packed", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 10768, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 24438, + "p50": 12016, + "p95": 23178, + "p99": 24041 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 35126 + }, + "tasksPerSec": 142, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003561415, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003524790.6082, + "_id": "jd7c3scvs4bk1r03vbkd9cwm598e4k4e", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003524790, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 157.699972, 102.045905, 110.256996, 103.719062, 108.912128, + 127.31051400000001, 73.497083, 130.72681699999998, 71.657398, + 132.272511, 76.673023, 133.06373299999998, 72.94835900000001, + 113.05074800000001, 75.93606, 70.401093, 107.544023, + 73.21045099999999, 135.22979600000002, 72.68521, 147.190011, + 69.256204, 196.817552, 83.589838, 165.867357, 100.148308, + 166.74916100000002, 68.822326, 137.40788, 88.048691, + 144.23101899999997, 76.44755, 140.489872, 75.965615, 153.187812, + 83.07231800000001, 162.166193, 79.603609, 136.06365, 75.012468, + 141.739833, 93.777845, 112.51989499999999, 94.99291099999999, + 160.78185, 65.50101, 97.551902, 94.83005899999999, 72.313688, + 128.43009 + ], + "userExecutionMs": [ + 19.923287000000002, 18.411830000000002, 17.58409, 19.10079, + 16.309589, 14.662859000000001, 14.048622, 17.961929, + 14.795501000000002, 14.869546000000001, 13.435151, 14.699482, + 14.091357, 16.336903, 13.883736, 13.839341999999998, + 15.874474999999999, 15.699435000000001, 16.336416999999997, + 11.891707, 14.091933, 12.243508, 13.642414, 14.767822, 14.423323, + 18.377571999999997, 17.933287, 14.725952000000001, + 14.435111000000001, 15.53707, 13.523442, 14.398378999999998, + 14.598244999999999, 12.820143000000002, 15.687153999999998, + 12.430195999999999, 14.050474, 16.147529000000002, 11.865357, + 14.381329, 13.125536, 12.257202999999999, 14.390316, 12.525976, + 13.884946000000001, 11.913638, 12.7101, 14.007159, 14.424426, + 13.849886 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2543953, + "writeBytes": 2948604, + "ioReadBytes": 2543953, + "ioWriteBytes": 2948604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 127.81858099999998, 337.76818199999997, 448.710442, 321.913789, + 375.524188, 407.041887, 387.856039, 387.239648, 383.709599, + 374.816566, 393.8889, 419.406656, 396.836199, 373.53508899999997, + 372.490473, 401.77103, 382.860794, 397.72322199999996, 392.799238, + 371.75240399999996, 481.307752, 489.760825, 412.237546, 409.216927, + 388.629028, 363.161314, 379.60768700000006, 363.64215, + 384.12634499999996, 499.137926, 383.188443, 400.35353899999996, + 370.35681999999997, 366.56609000000003, 400.448824, 367.590333, + 373.85528, 374.977703, 467.406183, 369.81187900000003, 363.704106, + 373.730802, 391.66769899999997, 400.326611, 371.852775, 669.119502, + 398.805327, 703.9731390000001, 434.614218, 966.909067, + 470.03216799999996, 368.517478, 370.70726199999996, 389.819193, + 379.487024, 378.459229, 386.171828, 366.04628, 378.28022100000004, + 417.650054, 368.072843, 360.45927700000004, 370.140487, 374.307998, + 457.282808, 378.140285, 382.05555000000004, 365.950159, + 356.79253300000005, 359.33189799999997, 360.635265, 367.877769, + 375.09508400000004, 369.20227300000005, 445.27368299999995, + 360.543222, 368.47303999999997, 376.284517, 366.99211599999995, + 274.79670699999997, 149.64172100000002, 49.864106, 51.972456, + 47.500689, 53.129505, 48.418406999999995, 50.533058 + ], + "userExecutionMs": [ + 3.318835, 9.434132, 8.70956, 8.739462, 4.766342, 4.659699, 4.501246, + 4.9365060000000005, 4.6047199999999995, 6.191543, 6.375805, + 6.124279, 6.797519, 6.855511, 4.450152, 4.426143, 4.268079, + 6.694618, 6.747785, 6.12988, 6.445153, 6.651974, 6.292792, + 4.9111199999999995, 6.011341, 6.174247, 4.597295000000001, + 4.4900139999999995, 4.9073210000000005, 4.442463, + 4.5957300000000005, 7.410006999999999, 4.4125939999999995, 5.916281, + 4.433923, 4.892106, 4.594056, 4.454981999999999, 4.9248199999999995, + 4.5060459999999996, 4.554261, 4.9541200000000005, 4.668030999999999, + 5.0563400000000005, 4.559159999999999, 4.522149, 6.578046, 5.425333, + 5.633458, 4.765403, 5.093669, 6.186112, 4.764888, 4.31711, 6.716589, + 6.138088, 4.886385, 4.251607999999999, 4.35462, 6.339849, + 4.3253889999999995, 4.73547, 4.641077, 7.316048, 4.289658, 4.505419, + 4.467607, 4.771402, 4.885861, 4.318026, 4.373848, 4.977762, + 4.342577, 5.471826, 4.309437, 4.945641, 4.421162, 4.552007000000001, + 3.9489750000000003, 6.990213000000001, 3.044995, 2.244825, 3.3223, + 2.281518, 2.122359, 2.323751, 2.2657279999999997 + ], + "readDocuments": 16228, + "writeDocuments": 5292, + "readBytes": 9010661, + "writeBytes": 1247492, + "ioReadBytes": 9010661, + "ioWriteBytes": 1247492, + "writeIndexRows": 20832, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-1-ordering", + "phase": "measured", + "rep": 1, + "variant": "ordering", + "workload": "action", + "began": "2026-09-10T01:26:11.824078+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "ordering", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 12252, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 25638, + "p50": 13074, + "p95": 23723, + "p99": 25234 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 37697 + }, + "tasksPerSec": 133, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003611481, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003572403.6948, + "_id": "jd79txzhp5tfpxgtb5cmab9v8s8e4v9d", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003572403, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 116.149389, 79.18865, 158.149694, 126.513679, 84.07202, 147.741098, + 103.014583, 129.263403, 79.422697, 148.283061, 91.57897, 168.011481, + 78.649945, 126.85410499999999, 80.45046, 109.497808, 84.459648, + 102.008253, 74.45539099999999, 121.350645, 79.593878, 112.232263, + 84.095274, 135.76802600000002, 76.56329000000001, 107.994558, + 91.386999, 107.61483299999999, 86.9418, 107.120992, 86.99075, + 97.37604599999999, 73.37498500000001, 100.473743, 103.103468, + 146.24139499999998, 93.060304, 99.62444500000001, 82.742257, + 114.124436, 95.46359, 135.553499, 78.075687, 129.211935, 86.109955, + 149.121668, 85.016642, 198.27798099999998, 83.37047, 154.679026 + ], + "userExecutionMs": [ + 20.522991, 16.752515, 20.135285999999997, 16.844064, + 17.781055000000002, 17.668521, 18.124738, 17.897636000000002, + 17.032854999999998, 17.130533, 16.783693, 21.186867000000003, + 15.25114, 17.206184999999998, 15.652796, 16.688705000000002, + 16.222242, 18.410277, 15.709282000000002, 18.135275999999998, + 15.776898000000001, 17.274021, 16.415042, 16.254105000000003, + 15.145318, 16.153164, 17.935566, 16.595264, 15.643418999999998, + 16.935719, 15.504153, 16.000853, 14.994399, 16.7999, + 15.717732000000002, 15.891929, 15.388748000000001, 15.562314, + 16.123314, 16.261233, 20.262997000000002, 16.177266999999997, + 16.120205, 17.505321000000002, 19.842258, 16.330191, 16.518858, + 21.081243999999998, 16.321321, 16.871099 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2543953, + "writeBytes": 4220604, + "ioReadBytes": 2543953, + "ioWriteBytes": 4220604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 148.25751, 271.720253, 394.691558, 487.610834, 418.072765, + 396.74734099999995, 390.679791, 390.829638, 399.98589400000003, + 388.53853100000003, 392.874014, 404.983616, 436.357574, 421.356629, + 423.460425, 431.57167899999996, 451.32521399999996, 525.859094, + 605.606643, 407.648846, 536.398651, 405.428499, 392.556678, + 393.33818599999995, 437.03601, 391.64915, 477.130465, 397.227227, + 443.029668, 394.048227, 397.169751, 383.156234, 395.70262, + 409.311665, 396.11109700000003, 390.539289, 381.694313, 512.207055, + 464.179287, 440.043047, 391.572129, 412.60956999999996, 382.308412, + 385.673082, 391.650946, 394.74796200000003, 387.721076, + 405.71283200000005, 490.155866, 405.552138, 380.59022500000003, + 390.58140299999997, 411.784822, 389.429088, 389.74209199999996, + 397.727958, 386.377389, 392.335249, 384.849517, 400.689601, + 392.420803, 388.594922, 403.67479000000003, 389.719659, 389.60264, + 395.054401, 402.999685, 419.20531700000004, 414.381754, 445.294945, + 440.00383, 568.838649, 575.77386, 407.790758, 492.21354699999995, + 591.1015659999999, 741.071315, 413.942288, 387.42469800000003, + 223.369071, 87.14077, 51.19753, 53.299274000000004, 53.843223, + 52.93394, 67.188839, 77.25944799999999 + ], + "userExecutionMs": [ + 7.7636, 3.291579, 8.681227999999999, 6.238352, 4.825297, 7.147405, + 4.8212079999999995, 6.4284040000000005, 5.964591, 4.335502, + 6.2877849999999995, 6.226716000000001, 8.626668, 5.072063, 6.376764, + 6.1894350000000005, 6.172811, 6.083028, 6.730647, 6.139944, + 7.090441, 4.325372, 4.444895, 4.736477, 4.650881, 6.238639, + 7.950944999999999, 7.6262989999999995, 4.312003, 6.293416, + 4.908545999999999, 4.323984, 6.207756, 4.711044, 9.173079000000001, + 4.522112, 4.387192, 4.734882, 4.6923140000000005, 4.432321, + 4.729736, 4.2352240000000005, 4.434488, 4.697388, 4.372507000000001, + 4.917991, 5.080241, 5.240675, 7.52055, 4.411808, 4.460562, 4.980745, + 4.3158069999999995, 4.315478000000001, 8.938022, 4.525173000000001, + 4.281761, 4.599061, 4.205661, 6.205310000000001, 4.263651, 5.014119, + 4.665973999999999, 6.389735, 4.265917, 4.790653, 4.3509150000000005, + 4.634598, 4.459725000000001, 4.395199, 6.426692, 4.378464999999999, + 4.574807, 4.376222, 7.842131, 4.324413, 4.472208, 4.458438999999999, + 4.247954, 4.036055999999999, 2.7205329999999996, 2.1829240000000003, + 2.765177, 5.807910000000001, 2.555907, 2.347086, 2.148406 + ], + "readDocuments": 30966, + "writeDocuments": 10168, + "readBytes": 10168919, + "writeBytes": 1103404, + "ioReadBytes": 10168919, + "ioWriteBytes": 1103404, + "writeIndexRows": 40336, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-1-baseline", + "phase": "measured", + "rep": 1, + "variant": "baseline", + "workload": "action", + "began": "2026-09-10T01:27:03.534211+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "baseline", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 11914, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 32503, + "p50": 14558, + "p95": 30460, + "p99": 32149 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 44375 + }, + "tasksPerSec": 113, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003670379, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003624154.206, + "_id": "jd74wg5zjeye4w7qjzqfkbgp1x8e4j8z", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003624154, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 176.70176600000002, 83.538659, 151.00387600000002, 150.916017, + 108.323861, 181.416261, 118.539576, 105.688784, 83.803384, + 84.74817900000001, 98.727703, 96.976983, 84.812251, 90.498346, + 90.090189, 80.183023, 102.80577000000001, 90.332285, 95.629907, + 93.421006, 82.155891, 147.215296, 65.160235, 88.027706, + 69.49260100000001, 84.63958600000001, 155.01926699999999, 76.343317, + 145.116973, 83.767422, 149.81194, 81.155992, 157.574984, 88.956566, + 106.86600899999999, 86.171503, 147.37108300000003, 82.110797, + 161.201565, 83.60783500000001, 126.86201799999999, 76.999214, + 118.15203100000001, 76.906284, 128.239067, 83.03452899999999, + 156.548099, 80.660019, 148.08695999999998, 74.63910299999999 + ], + "userExecutionMs": [ + 22.731521, 12.821254000000001, 20.311524000000002, 22.948045, + 15.201849, 15.853501, 13.249108000000001, 14.583412000000001, + 14.943796, 13.042525, 16.249176000000002, 14.575958, 16.545048, + 15.150732, 16.99542, 14.880813999999999, 15.974151999999998, + 16.595702, 14.549273999999999, 14.379993, 16.225164, 25.901746, + 13.737397999999999, 13.298959, 13.703536, 12.841826000000001, + 17.857933, 12.802266, 14.605576000000001, 13.187190999999999, + 14.169686, 14.158403999999999, 14.757111, 12.752279, 14.045004, + 14.23138, 13.908213000000002, 12.988752, 13.848023, 12.645247, + 16.452351, 12.631546, 13.158677, 14.532477, 17.67531, 14.54362, + 14.157737000000001, 12.613736000000001, 14.686975, + 23.069218999999997 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4100604, + "ioReadBytes": 23953, + "ioWriteBytes": 4100604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 85, + "executionMs": [ + 158.754645, 222.25851899999998, 449.947785, 809.817303, 430.832449, + 388.562912, 403.15531500000003, 389.456458, 539.42398, 475.794054, + 526.045585, 472.234147, 413.035176, 423.340308, 393.825785, + 420.317326, 421.738436, 417.18315, 421.779649, 436.33423500000004, + 423.918406, 446.694998, 458.068197, 439.539262, 445.73270199999996, + 439.42227499999996, 680.7686600000001, 531.542301, 460.21278, + 530.1241279999999, 565.0751349999999, 506.3241550000001, 508.865498, + 533.7310669999999, 527.254407, 431.869639, 501.715308, 474.811492, + 479.621501, 455.274149, 590.406418, 685.5149, 526.49374, 585.813152, + 554.343967, 531.596971, 549.1977009999999, 490.879387, 529.156525, + 455.718186, 569.0402770000001, 541.9564529999999, 561.1591559999999, + 457.003659, 528.9184740000001, 552.141564, 602.7385979999999, + 618.011972, 499.244831, 584.202455, 601.257793, 517.451188, + 558.1828989999999, 574.577219, 553.277876, 560.973779, + 553.3358440000001, 626.997389, 579.586421, 566.202093, 565.201669, + 555.230955, 575.048365, 586.81715, 557.240853, 568.743534, + 623.110759, 586.7496060000001, 486.51962599999996, + 327.34573700000004, 303.31827300000003, 97.276757, 59.203317, + 36.568057, 60.804649000000005 + ], + "userExecutionMs": [ + 3.5802210000000003, 3.180132, 8.735779, 4.825111, 11.339375, + 4.442357, 4.667889, 4.2508729999999995, 4.2015910000000005, + 6.108671, 4.884027, 4.521001, 6.429782, 4.346183, 4.701411, + 4.252828, 4.218869, 4.7929569999999995, 4.2994900000000005, + 4.275612, 4.8486199999999995, 4.42335, 4.609177, 7.1564499999999995, + 4.942968, 6.316405, 4.441545, 4.759921, 6.954865, 5.955228, + 4.291995, 4.9756089999999995, 4.4873639999999995, 4.874759, + 4.471455, 6.0130289999999995, 4.187543, 4.770134, 4.359656, 4.62668, + 5.002429, 4.49031, 5.969524000000001, 4.258919, 4.733463, 4.386533, + 4.593401, 6.279807, 4.47097, 6.476415, 7.675578, 4.335903, + 4.405018999999999, 6.190112, 4.3493960000000005, 4.894542, + 4.3741509999999995, 4.328811999999999, 4.242396, 4.755304, + 4.2137329999999995, 6.404227000000001, 4.810728, 4.554779, 4.843102, + 4.614965000000001, 4.28234, 4.767937, 8.77907, 4.709978, 4.934166, + 4.45711, 4.483836, 5.048798000000001, 4.448751, 4.455397, 4.961273, + 4.453519, 3.769598, 3.9552759999999996, 3.611758, 3.555803, + 2.347518, 2.2136679999999997, 2.374966 + ], + "readDocuments": 30928, + "writeDocuments": 10166, + "readBytes": 11625734, + "writeBytes": 1672707, + "ioReadBytes": 11625734, + "ioWriteBytes": 1672707, + "writeIndexRows": 40332, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-2-ordering", + "phase": "measured", + "rep": 2, + "variant": "ordering", + "workload": "action", + "began": "2026-09-10T01:28:00.724394+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "ordering", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 12162, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 28218, + "p50": 13466, + "p95": 26784, + "p99": 27788 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 40200 + }, + "tasksPerSec": 124, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003723270, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003681247.4138, + "_id": "jd745m5dmr3zfet1mb4md31x198e5xp3", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003681247, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 88.78430800000001, 75.365306, 150.83962400000001, 135.077292, + 96.219071, 162.584516, 90.36266499999999, 134.879477, 101.039951, + 172.89905199999998, 86.81773199999999, 123.408821, + 110.31386599999999, 145.36136199999999, 80.58658100000001, + 179.033433, 105.622632, 141.154254, 89.173142, 91.37666899999999, + 109.21632000000001, 134.702012, 87.523089, 103.56427599999999, + 95.634786, 146.750808, 85.901081, 135.47125300000002, 105.340514, + 162.278414, 91.366143, 177.342242, 108.26964699999999, 160.512302, + 89.166882, 118.265822, 97.89582300000001, 175.034388, 86.582759, + 125.972646, 105.452421, 156.671227, 75.74601, 148.609399, + 102.28914999999999, 167.76481700000002, 85.06158900000001, + 135.42653700000002, 96.01864, 185.18704 + ], + "userExecutionMs": [ + 21.588301, 15.491425, 21.13352, 23.079671, 16.967612, 20.37434, + 17.961566, 17.738151000000002, 16.346975999999998, 17.762889, + 15.735750999999999, 19.495906, 15.766494999999999, + 15.150122999999999, 15.460053, 19.871534, 16.68709, 15.76816, + 15.621274, 20.322561999999998, 15.462091000000001, 16.383748, + 16.000617000000002, 16.669328, 16.997416, 17.85254, 16.554321, + 16.877348, 16.336447, 17.874169, 15.754699, 16.809374, + 16.630266000000002, 15.863497, 18.171692, 20.890666, + 16.212870000000002, 20.774825, 16.487522000000002, + 16.113846000000002, 15.549691, 16.034749, 15.804753000000002, + 16.872884000000003, 14.885067, 17.621199, 16.262023, + 18.956979999999998, 15.652690999999999, 17.498713 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2543953, + "writeBytes": 4220604, + "ioReadBytes": 2543953, + "ioWriteBytes": 4220604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 88, + "executionMs": [ + 188.594834, 280.439736, 386.54686399999997, 392.666723, 440.916484, + 387.6477, 455.748013, 444.653882, 454.196363, 398.34054299999997, + 464.34957199999997, 394.412863, 461.597469, 407.64042700000005, + 527.268913, 439.398963, 489.507094, 396.153346, 493.461907, + 397.765106, 443.963593, 398.198247, 504.49299, 417.340774, + 436.28206600000004, 442.147718, 455.031969, 435.659144, 447.527316, + 451.245473, 457.774546, 454.782842, 462.514607, 452.012195, + 478.57214, 455.432043, 458.227995, 392.55518700000005, 476.158876, + 550.769457, 445.32377299999996, 446.424625, 450.876233, + 453.93589099999997, 478.30028400000003, 461.385726, 506.099893, + 440.879177, 463.225794, 451.915437, 720.3198669999999, 520.540377, + 409.477503, 502.575337, 425.619202, 465.045592, 450.97098, + 478.544485, 447.70550799999995, 467.30024399999996, 472.894722, + 466.10825, 461.943037, 470.652115, 414.07217299999996, 415.950069, + 398.079205, 380.74683500000003, 1388.906066, 372.778782, 448.282703, + 490.247154, 521.321072, 392.95153300000004, 397.67742499999997, + 388.5138, 487.367832, 462.76943, 412.82745, 236.507903, 227.401591, + 62.706979999999994, 56.888383, 78.7007, 59.030815, 73.774018, + 46.537737, 179.293652 + ], + "userExecutionMs": [ + 3.60113, 3.270474, 8.853772, 4.413843, 6.2962549999999995, 8.375442, + 6.456591, 6.265696999999999, 4.819483, 6.47181, 6.2087, + 6.020835000000001, 6.656712000000001, 6.1517610000000005, 6.204414, + 6.304703, 6.386896, 4.686245, 6.572757999999999, 6.582932, 6.406643, + 5.121549000000001, 6.4799430000000005, 6.880737, 4.9359399999999996, + 6.364128999999999, 6.671781, 4.237886, 4.3825069999999995, 4.999999, + 4.324981999999999, 4.645334, 4.829428, 4.363818, 4.372821, + 7.1970979999999996, 4.423574, 6.999347, 6.896297, 4.471617, + 4.613561000000001, 4.788511, 7.113403, 4.442295, 4.3580950000000005, + 4.764233, 4.263894, 4.454943, 4.7053199999999995, 4.339123000000001, + 5.146484, 6.424952, 6.878443, 4.57201, 4.807863, 6.548271, 5.170668, + 4.465159, 4.627006000000001, 4.352924, 4.546769, 5.235905, + 4.412326999999999, 4.3615260000000005, 6.148685, 4.320062, 4.443143, + 5.088243, 6.482530000000001, 4.843965000000001, 5.5781659999999995, + 9.169798, 4.672606, 4.24945, 4.930298, 4.300325, 4.476865, 4.313002, + 3.9057150000000003, 5.192047, 3.455277, 3.144505, + 2.2046069999999998, 3.5018100000000003, 2.178572, 2.179636, + 2.212384, 2.4247180000000004 + ], + "readDocuments": 30973, + "writeDocuments": 10169, + "readBytes": 10292654, + "writeBytes": 1144322, + "ioReadBytes": 10292654, + "ioWriteBytes": 1144322, + "writeIndexRows": 40338, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-2-baseline", + "phase": "measured", + "rep": 2, + "variant": "baseline", + "workload": "action", + "began": "2026-09-10T01:28:55.609883+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "baseline", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 11925, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 31615, + "p50": 14142, + "p95": 30080, + "p99": 31574 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 43402 + }, + "tasksPerSec": 115, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003781592, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003736205.2993, + "_id": "jd73ytqk4ha1x4prh9mtv5skb98e4wyt", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003736205, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 104.15710899999999, 81.170594, 141.028815, 107.991629, 96.565311, + 89.27100899999999, 157.450443, 83.202802, 149.672523, 86.830113, + 149.686538, 83.464203, 138.434575, 81.132605, 152.05601099999998, + 74.17492, 131.81285200000002, 78.69982300000001, 127.13349099999999, + 79.65974700000001, 143.31820000000002, 79.765509, + 149.20287900000002, 86.316523, 91.13940500000001, 81.05382999999999, + 100.698508, 77.542249, 114.852282, 83.759689, 144.11921, 72.821838, + 127.77421799999999, 89.358663, 111.437848, 90.50036899999999, + 146.494771, 81.465062, 157.42558100000002, 82.960796, 143.446419, + 86.3925, 145.904612, 75.488692, 126.518184, 87.345106, 119.395629, + 144.701279, 71.44400200000001, 96.986424 + ], + "userExecutionMs": [ + 18.68472, 13.480511, 22.328312999999998, 16.341857, + 15.582118000000001, 13.543348, 15.089923, 14.814012, 14.898489, + 13.400802, 19.711366, 13.540006, 15.142802, 14.064713, 16.202795, + 13.017153, 14.728354000000001, 12.894544000000002, 14.168905, + 12.590157, 15.295009, 12.849089, 24.270811, 14.160708, 14.292526, + 16.744081, 13.704479000000001, 13.113723, 14.974569, 12.37872, + 15.548283, 12.353345, 14.918482000000001, 12.746713999999999, + 13.964177, 13.133954999999998, 14.648893, 14.233547, 14.987753, + 14.060063, 17.983843999999998, 13.282666, 15.432803, + 14.580888999999999, 14.172789, 15.112563000000002, + 13.519817999999999, 14.561528000000001, 15.067869, + 13.321482000000001 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4100604, + "ioReadBytes": 23953, + "ioWriteBytes": 4100604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 83, + "executionMs": [ + 158.853853, 317.829765, 537.279951, 432.913057, 416.637175, + 400.70579499999997, 474.78964099999996, 417.602777, 418.717865, + 419.43523500000003, 404.069062, 404.56926999999996, + 393.35189199999996, 409.406331, 434.38714600000003, 467.820543, + 403.998738, 419.04056699999995, 439.749744, 424.77925500000003, + 436.854709, 486.329514, 640.0609059999999, 507.23436300000003, + 423.44611199999997, 505.96164, 514.03987, 430.349791, 496.146771, + 475.49332300000003, 588.965168, 435.397638, 514.831478, 512.885185, + 436.732464, 531.948755, 455.279638, 641.685057, 511.70599100000004, + 516.26642, 449.28954100000004, 531.741457, 445.336542, 526.094163, + 457.947616, 553.086994, 456.80922499999997, 568.889753, + 670.6430439999999, 561.622265, 449.22798800000004, 569.000036, + 478.292852, 583.219121, 576.072951, 550.005133, 481.340884, + 539.756573, 750.65311, 640.063983, 514.561031, 884.6051, + 640.4682809999999, 553.341296, 509.234536, 497.849662, 573.279705, + 494.101557, 597.939312, 534.025891, 618.755071, 515.60632, + 586.043326, 537.6564510000001, 539.9580109999999, 511.740934, + 505.52756200000005, 810.257154, 250.249228, 363.13572999999997, + 129.50618, 118.193723, 38.874269000000005 + ], + "userExecutionMs": [ + 3.6099080000000003, 3.431459, 4.412814, 5.182374, + 4.3568940000000005, 4.422594, 4.589056, 6.215858, 4.207546000000001, + 4.694992999999999, 4.350482, 4.206002000000001, 4.745859, + 4.2739020000000005, 4.208016, 4.752229, 4.113589, 5.046194000000001, + 4.366661, 4.385109, 4.316243, 4.3143080000000005, 6.528603, + 6.148619, 4.528722, 6.257403, 4.883124, 6.507429, 4.626, 6.442044, + 8.512185, 4.730073, 4.191324, 6.528932999999999, 4.420718, + 6.596239000000001, 6.435995, 5.988296, 6.177704, 4.732093, + 6.555731000000001, 4.406231, 4.431375, 6.21816, 4.829244, 4.868227, + 4.374452, 6.497901000000001, 6.366315, 6.269923, 4.155847, 4.489212, + 4.3896370000000005, 6.059938, 6.332207, 4.440042, 4.708842000000001, + 4.528676, 4.918832, 4.510918, 6.316594, 6.382771, 8.639419, + 6.234667999999999, 6.256047, 6.614931, 4.716171, 4.620086, + 6.150161000000001, 4.239981, 4.74458, 5.924836, 6.1260259999999995, + 8.694886, 4.428906, 4.71615, 6.223634, 3.647857, 3.818392, + 6.495538000000001, 2.946984, 2.452689, 2.3642950000000003 + ], + "readDocuments": 30863, + "writeDocuments": 10164, + "readBytes": 10858405, + "writeBytes": 1420529, + "ioReadBytes": 10858405, + "ioWriteBytes": 1420529, + "writeIndexRows": 40328, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-2-packed", + "phase": "measured", + "rep": 2, + "variant": "packed", + "workload": "action", + "began": "2026-09-10T01:29:51.963496+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "packed", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 10899, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 28756, + "p50": 13845, + "p95": 27265, + "p99": 28296 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 39563 + }, + "tasksPerSec": 126, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003833736, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003792550.2625, + "_id": "jd79q4kcb8qjh44tttkrxbx7pd8e56v9", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003792550, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 144.841403, 72.473258, 125.741115, 125.25140100000002, 99.612162, + 104.695257, 80.13195900000001, 100.421885, 143.560984, 75.986472, + 109.775118, 88.115306, 145.372059, 72.90548, 160.340643, 80.25313, + 126.093314, 64.239011, 162.277336, 151.163365, 150.24216900000002, + 87.432423, 157.137004, 90.78370299999999, 152.79789499999998, + 86.835323, 133.32815300000001, 83.651326, 81.082726, 125.050257, + 92.413858, 145.374693, 85.758658, 222.965417, 103.486565, 87.502633, + 133.743374, 83.99019100000001, 152.756796, 89.96462500000001, + 156.863753, 91.751504, 123.991922, 85.055639, 80.025401, 163.677211, + 88.321989, 143.49339999999998, 93.949517, 164.007666 + ], + "userExecutionMs": [ + 19.055289, 11.424421, 19.247777, 19.468255, 14.325474, + 12.875518000000001, 14.342522, 14.999639, 13.757202, 12.165115, + 14.423783, 12.122825, 16.820734, 14.182224999999999, 17.47601, + 16.139628, 14.328293, 13.25354, 14.008491, 17.643144, 19.330472, + 13.043846, 13.501455, 16.070415, 15.570626, 13.566021, + 14.302375000000001, 16.837872, 11.858149000000001, 11.86774, + 14.091447, 14.509688, 13.812069999999999, 16.480216, 12.097322, + 11.462067, 13.990262999999999, 14.256351, 21.275817, 13.219564, + 12.912685999999999, 14.331745999999999, 13.04614, 14.539692, + 14.065245, 15.804379, 12.781839999999999, 13.607409, 18.620752, + 16.396166 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2543953, + "writeBytes": 2948604, + "ioReadBytes": 2543953, + "ioWriteBytes": 2948604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 87, + "executionMs": [ + 148.58611, 264.576297, 392.102585, 428.302238, 366.014459, + 438.204506, 372.61716299999995, 428.767501, 374.378172, 430.646964, + 426.358267, 427.000626, 456.188023, 423.938859, 412.217983, + 433.535278, 705.883924, 468.088326, 433.073823, 448.077802, + 430.743874, 439.445336, 432.615468, 463.325199, 456.194703, + 428.710468, 423.995498, 422.990503, 498.079851, 427.430337, + 423.860805, 427.498479, 477.916757, 463.993471, 566.9321000000001, + 469.466506, 560.29948, 444.274905, 418.925309, 425.090615, + 467.66506400000003, 441.594726, 422.016664, 421.784143, 551.049105, + 432.018114, 504.336437, 441.4457, 553.599226, 433.877262, + 422.248085, 443.622747, 566.123628, 433.86881700000004, 419.732988, + 447.937906, 625.7564329999999, 464.422326, 429.69708099999997, + 469.565366, 497.488975, 460.724943, 443.421618, 447.97797, + 462.955588, 456.85884300000004, 564.625039, 622.927457, 595.356915, + 549.946008, 551.30133, 535.879381, 437.476698, 429.28568299999995, + 421.23924800000003, 456.630671, 456.79678, 450.036786, 438.32149, + 323.65283700000003, 130.900137, 79.62782, 108.095667, 76.321102, + 86.258529, 79.145434, 78.04623 + ], + "userExecutionMs": [ + 3.856216, 5.874193, 6.102376, 8.610018, 6.7208939999999995, + 5.932252, 6.244156, 4.426605, 8.941667, 4.259417, 7.104556, + 8.633036, 6.285566, 4.912599, 4.878158, 4.956148, 8.295554000000001, + 4.652583, 6.649221000000001, 5.146661, 4.367755, 4.776594, + 6.811903999999999, 5.938022, 6.484221, 4.385857, 4.3607890000000005, + 4.79947, 8.495108, 6.641237, 4.370061, 4.407705, 5.014848, + 4.3008630000000005, 4.582911999999999, 4.660471, 4.326969, 4.373449, + 5.1033859999999995, 4.4627989999999995, 4.507395, 5.131478, + 4.345834, 4.334192, 5.285459, 4.66426, 5.380147, 4.621895, 4.653659, + 4.465139, 4.552609, 4.443961, 5.40732, 4.866956, 6.931665, + 5.0065789999999994, 6.682988, 4.4885589999999995, 4.313819, + 4.421689000000001, 5.016457, 4.90323, 4.352647999999999, 6.287972, + 4.402824, 4.69088, 6.510769, 4.586881, 4.735081, 11.486005, + 5.0835799999999995, 6.701591, 6.211581, 4.417725, 4.745289, + 5.0335220000000005, 7.295453999999999, 4.711558999999999, 4.267883, + 3.88925, 2.626045, 2.327697, 3.5768229999999996, 2.487537, + 2.4491959999999997, 2.237812, 2.230222 + ], + "readDocuments": 16231, + "writeDocuments": 5293, + "readBytes": 9015471, + "writeBytes": 1249248, + "ioReadBytes": 9015471, + "ioWriteBytes": 1249248, + "writeIndexRows": 20836, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-3-baseline", + "phase": "measured", + "rep": 3, + "variant": "baseline", + "workload": "action", + "began": "2026-09-10T01:30:44.418155+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "baseline", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 11931, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 34574, + "p50": 15890, + "p95": 32567, + "p99": 34553 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 46402 + }, + "tasksPerSec": 108, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003893506, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003845003.6274, + "_id": "jd7be6tdvjrm5t85nt65pm2mc18e521w", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "baseline", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003845003, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 222.07054200000002, 89.282315, 107.129659, 111.785021, 137.548899, + 78.18007, 144.76044499999998, 100.679134, 168.982807, 87.523792, + 150.860832, 93.98948999999999, 106.150799, 91.529229, 84.924802, + 129.3234, 89.87907, 147.43488799999997, 90.09196299999999, 108.6755, + 103.97749, 87.45415600000001, 103.09196800000001, 89.21810099999999, + 157.17166699999999, 85.41248, 126.338341, 83.84338600000001, + 82.43168800000001, 104.43395699999999, 87.734634, 135.921008, + 82.62939499999999, 146.474011, 86.436863, 96.619805, 121.486076, + 86.238991, 127.028059, 83.536037, 109.50902599999999, + 87.62823999999999, 77.9698, 146.251204, 81.87330700000001, + 116.493844, 86.550961, 88.03131, 131.85802099999998, 89.99887 + ], + "userExecutionMs": [ + 28.080251, 13.56782, 18.236494999999998, 15.814293, + 15.168918999999999, 14.022919, 20.361682, 13.57918, 20.492846, + 13.487862999999999, 16.368294, 13.135099, 13.84273, + 14.843606000000001, 16.148557, 19.033029, 13.094658, 17.956507, + 14.802555, 13.207794999999999, 14.089261, 15.202149, 13.986069, + 14.810007, 17.556505, 13.193251, 14.650738, 14.410108, 12.513411, + 15.2822, 13.859184, 13.148204999999999, 14.247463, 18.338831, + 13.939701000000001, 16.967008, 15.180769, 13.908838, 14.817769, + 13.704522, 12.897314, 15.093156, 12.54648, 14.599301, 12.979381, + 14.980252, 16.403658, 15.28157, 14.417378, 14.814112000000002 + ], + "readDocuments": 106, + "writeDocuments": 10002, + "readBytes": 23953, + "writeBytes": 4100604, + "ioReadBytes": 23953, + "ioWriteBytes": 4100604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 84, + "executionMs": [ + 299.220806, 305.0097, 476.79030300000005, 443.86409199999997, + 462.056218, 518.818254, 492.98168, 491.135948, 472.815649, + 529.523925, 482.537479, 473.34923299999997, 489.075709, 465.985534, + 480.165939, 509.99644700000005, 481.484166, 469.43835900000005, + 495.504024, 523.038388, 476.14095000000003, 470.975084, 449.664781, + 546.3620500000001, 456.860606, 520.955514, 486.826611, 457.339309, + 523.267708, 481.22315, 553.331237, 503.424429, 846.376414, + 618.267664, 479.681591, 527.4629749999999, 634.8586720000001, + 493.288232, 739.604702, 511.93525100000005, 567.132276, 514.749967, + 844.9958760000001, 492.946436, 549.8615580000001, 535.968917, + 543.332312, 502.884298, 545.703639, 513.702561, 563.754313, + 551.295704, 522.5822519999999, 565.100758, 509.54121200000003, + 524.22437, 696.114194, 648.33553, 514.9661920000001, 544.702345, + 503.85666100000003, 565.7106160000001, 536.39816, 563.151241, + 530.486766, 567.619909, 526.577506, 571.859559, 537.992181, + 561.966621, 602.530954, 586.098863, 912.476195, 629.4424290000001, + 734.060673, 689.3133019999999, 640.472085, 685.337911, 355.468124, + 334.268644, 164.233927, 115.003867, 54.462632, 64.074026 + ], + "userExecutionMs": [ + 2.989162, 7.452854, 9.153295, 4.452821, 4.8202490000000004, + 4.326345, 4.622535, 6.3197, 4.634514, 6.657687, 4.938525, 4.441224, + 4.269475, 6.253966, 5.259589, 6.026392, 6.451682, 7.199993, + 4.381034, 4.857979, 5.954242, 4.580766, 4.23151, 9.081539, 8.909077, + 4.4247890000000005, 6.327425, 4.177697, 6.147048, 6.108789, + 5.9969280000000005, 6.144198, 13.771546, 4.627681, 6.005203, + 5.001169, 6.053890999999999, 4.2257229999999995, 10.997899, + 6.289890000000001, 5.029799, 4.096019, 5.039601, 4.2596739999999995, + 6.25319, 5.913311, 6.711964, 5.775722, 6.078906, 5.956315, 6.079252, + 4.986781, 4.248565999999999, 9.396109999999998, 4.266786, 4.225987, + 4.745541, 6.789403, 6.147936, 4.409122, 5.864051, 4.429892000000001, + 4.550168, 6.218714, 4.14923, 6.252687, 4.941752, 4.305782, + 4.220586999999999, 6.418169, 4.508666, 4.2320709999999995, + 6.641125000000001, 12.208877, 4.631352, 4.615865, 4.763211, 4.18989, + 4.258067, 4.230981, 4.335674, 2.668639, 2.169906, 2.39667 + ], + "readDocuments": 30870, + "writeDocuments": 10165, + "readBytes": 10860033, + "writeBytes": 1420743, + "ioReadBytes": 10860033, + "ioWriteBytes": 1420743, + "writeIndexRows": 40330, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-3-packed", + "phase": "measured", + "rep": 3, + "variant": "packed", + "workload": "action", + "began": "2026-09-10T01:31:46.182825+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "packed", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 10926, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 25806, + "p50": 12666, + "p95": 24368, + "p99": 25371 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 36677 + }, + "tasksPerSec": 136, + "timedOut": false + }, + "evidence": { + "observedAt": 1789003945032, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003906752.9941, + "_id": "jd7f6axpt2p0k6hxrt1ge56r898e5gam", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "packed", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003906753, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 161.98333300000002, 91.726868, 147.176267, 118.77877600000001, + 155.559589, 107.44928900000001, 122.683005, 99.496442, + 126.13175099999998, 163.678086, 90.93802600000001, 149.782379, + 120.736885, 165.72773700000002, 93.363218, 143.280022, 86.323124, + 140.017694, 90.90132600000001, 89.275904, 111.41430700000001, + 98.157185, 141.85777299999998, 80.49975400000001, 167.964386, + 99.942143, 164.210884, 94.21555099999999, 159.196589, 85.715334, + 164.465459, 89.636094, 150.302148, 92.358083, 101.969965, 92.051649, + 82.510695, 146.895627, 75.965993, 157.527299, 103.847501, + 156.609901, 82.567712, 123.27173499999999, 91.499872, 93.937118, + 107.12883199999999, 98.49789200000001, 166.30595300000002, 95.080711 + ], + "userExecutionMs": [ + 21.801195, 12.651952, 18.516270000000002, 15.383505000000001, + 12.846426, 14.872748, 15.143618, 14.049151, 27.159008, + 16.070652000000003, 14.027669, 17.536758, 13.99567, 17.201099, + 12.736823, 15.012733, 13.896108, 16.628183, 14.088652999999999, + 12.853963, 13.909057, 13.52746, 12.612224999999999, + 12.038193000000001, 14.234847, 12.924669, 14.160314999999999, + 15.248247, 14.873536, 13.760576, 14.42764, 13.597529, 16.806281, + 14.078921, 14.149391000000001, 13.8923, 12.173778, 12.591723, + 14.217773, 14.131776, 12.672704, 16.929539000000002, + 13.843240999999999, 12.527256000000001, 13.19821, + 14.769934000000001, 13.842488999999999, 16.310247, 14.492302, + 17.897634 + ], + "readDocuments": 5106, + "writeDocuments": 5052, + "readBytes": 2543953, + "writeBytes": 2948604, + "ioReadBytes": 2543953, + "ioWriteBytes": 2948604, + "writeIndexRows": 10205, + "occExecutions": 0 + }, + "loop": { + "executions": 86, + "executionMs": [ + 140.17367, 225.82196900000002, 240.054662, 476.438314, + 457.17147800000004, 437.795991, 476.656058, 439.923852, 415.993376, + 436.930614, 441.43901400000004, 399.583257, 437.193435, 422.054109, + 446.75977300000005, 443.159851, 439.311594, 428.747054, 419.188607, + 511.48791500000004, 425.163222, 414.75777600000004, 441.966337, + 439.717912, 413.90093900000005, 423.805004, 418.651773, 396.994244, + 422.673505, 409.07939500000003, 433.164569, 417.394763, 418.725257, + 429.254039, 398.354112, 405.457532, 398.012793, 404.36672200000004, + 399.267997, 432.536528, 434.273233, 491.664567, 450.45355, + 419.269361, 397.994766, 459.051827, 426.568708, 432.521128, + 443.710558, 471.28099899999995, 447.653595, 433.237294, 451.730684, + 436.516231, 440.33964100000003, 407.098054, 413.723346, 418.075008, + 515.560132, 435.879878, 430.73719, 431.267437, 451.71987, + 434.472348, 399.88703300000003, 399.172978, 397.540386, 408.355637, + 435.79497200000003, 431.432586, 425.80355, 406.289379, 407.968191, + 409.413365, 449.058989, 440.03914199999997, 439.169447, 424.753986, + 406.62230500000004, 345.00732800000003, 282.773238, 86.261553, + 65.268888, 74.53001200000001, 72.13032799999999, 80.599803 + ], + "userExecutionMs": [ + 6.119267000000001, 2.878811, 8.229211, 4.813986, 7.695406, 6.782497, + 9.214182, 6.103133000000001, 5.938657, 5.044902, 4.273372999999999, + 5.962001, 6.321332, 6.636581, 5.157529, 6.547225, 4.439361, + 4.5010069999999995, 6.150934, 6.449466, 6.283751, 5.286369, + 6.6450759999999995, 5.873555, 4.747225, 4.317797, 4.355824, + 5.0199620000000005, 4.396294999999999, 4.1992, 7.025774999999999, + 4.070262, 7.406259, 4.270829, 4.46546, 4.280728, 4.419808, 4.899854, + 4.227304999999999, 4.674411, 5.618749, 4.274558000000001, 4.157683, + 6.079702999999999, 4.615081, 4.688262, 4.5712399999999995, 4.902789, + 4.567439, 5.200744, 4.5516380000000005, 7.121014, 4.914898, + 4.524029, 4.789334, 6.377795000000001, 4.369783, 4.356288, 4.776419, + 6.499831, 5.047712000000001, 4.335566, 5.428237, 4.7081290000000005, + 6.025318, 4.2873909999999995, 4.460871, 4.21468, 4.359351, + 4.743047000000001, 4.273530999999999, 4.289193999999999, + 4.448180000000001, 7.898656000000001, 8.397571, 4.567769, 4.396959, + 4.948383, 4.53646, 3.760269, 3.470733, 2.418881, 5.488334999999999, + 2.27412, 2.184411, 2.336687 + ], + "readDocuments": 16221, + "writeDocuments": 5291, + "readBytes": 9390002, + "writeBytes": 1374266, + "ioReadBytes": 9390002, + "ioWriteBytes": 1374266, + "writeIndexRows": 20830, + "occExecutions": 0 + } + } + }, + { + "label": "action-measured-3-ordering", + "phase": "measured", + "rep": 3, + "variant": "ordering", + "workload": "action", + "began": "2026-09-10T01:32:37.252221+00:00", + "args": { + "taskCount": 5000, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": "action", + "pollTimeoutMs": 180000, + "pool": "ordering", + "taskDurationMs": 20 + }, + "result": { + "enqueueTotal": 12848, + "metrics": { + "completedCount": 5000, + "latency": { + "max": 28122, + "p50": 14169, + "p95": 26596, + "p99": 27676 + }, + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "scenario": "throughput", + "status": "completed", + "taskCount": 5000, + "totalDurationMs": 40832 + }, + "tasksPerSec": 122, + "timedOut": false + }, + "evidence": { + "observedAt": 1789004000126, + "outcomes": { + "success": 5000 + }, + "run": { + "_creationTime": 1789003957834.9783, + "_id": "jd76exsqxa5qk3ewxtk8b57pch8e51j8", + "parameters": { + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskCount": 5000, + "taskDurationMs": 20, + "taskType": "action" + }, + "pool": "ordering", + "poolVersion": "0.4.11", + "scenario": "throughput", + "startTime": 1789003957835, + "taskCount": 5000 + }, + "uniqueWorkIds": 5000 + }, + "drained": { + "baseline": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "ordering": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + }, + "packed": { + "payload": 0, + "pendingCancelation": 0, + "pendingCompletion": 0, + "pendingStart": 0, + "running": 0, + "work": 0 + } + }, + "runtime": { + "enqueue": { + "executions": 50, + "executionMs": [ + 182.918385, 95.844938, 164.717733, 154.67233, 104.251653, + 164.220274, 94.937083, 140.803353, 106.812637, 172.22794, 94.101292, + 157.822506, 103.233227, 168.01753399999998, 96.054108, + 158.61171700000003, 101.33047, 169.533534, 88.084693, 152.203267, + 100.77515500000001, 160.33951299999998, 91.627088, + 175.31513700000002, 93.129886, 152.07241399999998, 103.826814, + 155.416572, 97.48836700000001, 144.526155, 91.92054800000001, + 140.493255, 107.159435, 155.39893099999998, 94.403946, 144.368734, + 102.674056, 155.891804, 97.324194, 159.01001300000001, 103.66749, + 155.62684900000002, 96.509686, 152.28151, 104.195077, 164.628629, + 97.920219, 157.736929, 99.716985, 112.40434 + ], + "userExecutionMs": [ + 22.570456999999998, 17.799102, 23.46363, 17.29805, 17.692441, + 17.373291, 15.733865999999999, 16.618895000000002, 16.851813, + 17.963917, 16.556923, 16.48206, 16.77958, 17.864805, 16.528729, + 16.187617000000003, 17.159729, 17.421184, 15.260084, 18.507934, + 16.91773, 20.895429, 14.976339, 21.755999000000003, + 15.846734000000001, 17.434828, 17.296254, 16.326994000000003, + 20.683647999999998, 21.345128000000003, 16.080769, 17.7349, + 16.039978, 20.715517, 15.239942, 16.621062, 17.405482, 17.637328, + 18.170519, 16.065798, 15.302825, 17.232115999999998, 15.039678, + 17.018195, 17.256311999999998, 16.425156, 15.678505, + 17.892526999999998, 17.97027, 14.777821 + ], + "readDocuments": 5106, + "writeDocuments": 10002, + "readBytes": 2543953, + "writeBytes": 4220604, + "ioReadBytes": 2543953, + "ioWriteBytes": 4220604, + "writeIndexRows": 30005, + "occExecutions": 0 + }, + "loop": { + "executions": 86, + "executionMs": [ + 200.274392, 277.86208899999997, 469.364646, 448.480911, + 481.11219500000004, 466.90871699999997, 457.73326199999997, + 445.97161800000003, 450.146741, 446.318064, 467.83919099999997, + 436.455339, 462.850871, 446.626722, 468.294616, 487.13282699999996, + 479.023264, 459.274537, 466.810041, 451.066744, 468.603024, + 472.880628, 516.416066, 457.90061599999996, 502.85881099999995, + 470.472805, 460.959273, 480.086135, 530.769638, 467.666025, + 600.880748, 447.990101, 453.74922000000004, 470.117695, 446.816848, + 468.452569, 459.917488, 454.180022, 475.818327, 471.452315, + 474.826207, 534.524696, 504.95665999999994, 462.57619900000003, + 464.53247, 474.963353, 468.26933399999996, 467.399254, 465.071365, + 456.022011, 463.785251, 466.06106400000004, 479.467037, 463.084736, + 453.03128999999996, 453.373675, 485.77593, 475.49291700000003, + 476.00714700000003, 469.270086, 485.161328, 463.928716, + 446.82109399999996, 452.056958, 448.97436600000003, 432.374723, + 442.248505, 473.034314, 548.7658279999999, 479.18778899999995, + 455.534882, 448.36930600000005, 458.061545, 442.309002, 469.407108, + 450.072496, 524.297945, 489.710604, 411.934889, 307.966872, + 138.72180500000002, 80.235607, 89.607324, 83.912853, 86.631636, + 69.96682299999999 + ], + "userExecutionMs": [ + 3.1364460000000003, 3.108446, 4.737164, 9.196857, 4.553648000000001, + 4.877343, 5.147444999999999, 4.634494999999999, 4.420325, 4.388187, + 4.529541, 4.294105, 4.449043, 4.334375, 4.61712, 5.276879, 5.002153, + 4.47458, 4.490737, 4.429989, 4.497326, 7.207809, 8.438904, 4.297186, + 4.388293, 6.4693190000000005, 4.617859, 5.069503, 6.830583000000001, + 6.245241, 9.307555, 6.201545, 4.84301, 4.569135, 4.854501, + 4.414318000000001, 4.4450970000000005, 5.126715, 4.681193, 4.402323, + 5.341981, 4.672197000000001, 6.436902000000001, 4.372382, 4.891612, + 4.498251, 4.477538, 4.7782659999999995, 4.317551, 4.369610000000001, + 4.7279230000000005, 4.322037, 4.887565, 4.776603, 4.383742000000001, + 4.617473, 4.737701, 5.046054000000001, 4.950666, 4.958551, 4.367475, + 6.172263, 6.018745, 6.360777, 4.201305, 4.333277, 4.814318, + 6.490169, 7.180141, 4.59675, 4.79747, 6.110396, 4.363365, 4.655342, + 4.8763499999999995, 4.418233, 4.609898, 5.140009, + 3.9372720000000005, 5.352262, 2.634557, 2.3805110000000003, + 2.404665, 5.750019, 2.5066260000000002, 2.127847 + ], + "readDocuments": 30959, + "writeDocuments": 10167, + "readBytes": 10167296, + "writeBytes": 1103190, + "ioReadBytes": 10167296, + "ioWriteBytes": 1103190, + "writeIndexRows": 40334, + "occExecutions": 0 + } + } + } + ], + "cleanupIncident": { + "afterRun": "mutation-measured-3-baseline", + "beforeRun": "mutation-measured-3-packed", + "failure": "Bookkeeping cleanup left one completed task; driver stopped after its 180-second cleanup deadline.", + "remainingTask": { + "_creationTime": 1789002623195.6785, + "_id": "j57dgd1knrsmf514q2fb4bfyp98e45dr", + "resultKind": "success", + "runId": "jd7b4vxvqqb1wee9rs9nnkceax8e5hz6", + "workId": "jx720g183x637dy246vnrrx8ph8e5ab6" + }, + "scheduledCleanupBoundary": { + "after": 1789002623195.6785, + "index": 0, + "limit": 1000, + "state": "success" + }, + "recovery": "Repeated cleanup:start after all pools drained, then resumed the same driver. Completed run files were skipped. No component, measured harness source, or timed arguments changed.", + "fix": "After the measurements, replace creation-time-only cleanup bounds with pagination cursors and add a tied-timestamp regression test." + } +} diff --git a/example/benchmarks/2026-09-09/run.py b/example/benchmarks/2026-09-09/run.py new file mode 100644 index 00000000..50e858c2 --- /dev/null +++ b/example/benchmarks/2026-09-09/run.py @@ -0,0 +1,258 @@ +"""Serial, balanced measurements of three pinned workpool component mounts.""" + +import argparse +import datetime +import hashlib +import json +import os +import shutil +import subprocess +import time +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--checkout", type=Path, required=True) +parser.add_argument("--output", type=Path, required=True) +parser.add_argument( + "--metadata", type=Path, default=Path(__file__).with_name("metadata.json") +) +parser.add_argument("--smoke", action="store_true") +ARGS = parser.parse_args() +OUT = ARGS.output.resolve() +OUT.mkdir(parents=True, exist_ok=True) +STAGE = ARGS.checkout.resolve() +META = json.loads(ARGS.metadata.read_text()) +NODE = shutil.which("node") +if ( + not NODE + or int( + subprocess.check_output([NODE, "--version"], text=True) + .strip() + .lstrip("v") + .split(".")[0] + ) + < 24 +): + raise RuntimeError("Run with Node 24 or later on PATH") +CLI = STAGE / "node_modules/convex/dist/cli.bundle.cjs" +DEPLOYMENT = META["deployment"] +VARIANTS = META["variants"] +if ( + os.environ.get("CONVEX_DEPLOY_KEY") + or "CONVEX_DEPLOY_KEY=" in (STAGE / ".env.local").read_text() +): + raise RuntimeError( + "Remove the deploy-key override before using the recorded dev target" + ) +for name, info in VARIANTS.items(): + source = STAGE / "benchmark-components" / name / "src/component" + for path, expected in info["sourceHashes"].items(): + if hashlib.sha256((source / path).read_bytes()).hexdigest() != expected: + raise RuntimeError( + f"Component source differs from the recorded revision: {name}/{path}" + ) +for name, expected in META["dependencies"].items(): + if ( + json.loads((STAGE / "node_modules" / name / "package.json").read_text())[ + "version" + ] + != expected + ): + raise RuntimeError(f"Dependency differs from the recorded version: {name}") +if (OUT / "metadata.json").exists() and json.loads( + (OUT / "metadata.json").read_text() +) != META: + raise RuntimeError( + "Output directory belongs to a different benchmark configuration" + ) +(OUT / "metadata.json").write_text(json.dumps(META, indent=2) + "\n") +COUNT_QUERY = """const counts = {}; +for (const table of ["work", "pendingStart", "pendingCompletion", "pendingCancelation", "payload"]) + counts[table] = await ctx.db.query(table).count(); +const state = await ctx.db.query("internalState").first(); +counts.running = state?.running.length ?? 0; +return counts;""" + + +def call( + function=None, + args=None, + *, + component=None, + query=None, + label=None, + allow_empty=False, +): + cmd = [NODE, str(CLI), "run", "--deployment", DEPLOYMENT] + if component: + cmd += ["--component", component] + cmd += ["--inline-query", query] if query else [function, json.dumps(args or {})] + result = subprocess.run( + cmd, cwd=STAGE, text=True, capture_output=True, timeout=660, check=False + ) + if label: + (OUT / f"{label}.stdout").write_text(result.stdout) + (OUT / f"{label}.stderr").write_text(result.stderr) + if result.returncode: + raise RuntimeError(f"{function or component}: {result.stderr[-3000:]}") + if not result.stdout.strip(): + if allow_empty: + return None + raise RuntimeError(f"Missing response from {function or component}") + return json.loads(result.stdout) + + +def drain(): + deadline = time.monotonic() + 180 + while True: + counts = { + name: call(component=info["component"], query=COUNT_QUERY) + for name, info in VARIANTS.items() + } + if all(not any(c.values()) for c in counts.values()): + return counts + if time.monotonic() >= deadline: + raise RuntimeError(f"Components did not drain: {counts}") + time.sleep(1) + + +def cleanup(): + drain() + call("test/cleanup:start", {"limit": 1000}, allow_empty=True) + deadline = time.monotonic() + 180 + while True: + counts = call("test/cleanup:counts") + if not any(counts.values()): + return + if time.monotonic() >= deadline: + raise RuntimeError(f"Bookkeeping did not clear: {counts}") + time.sleep(1) + + +def measure(kind, variant, phase, rep, task_count): + label = f"{kind}-{phase}-{rep}-{variant}" + if (OUT / f"{label}.json").exists(): + print(f"ALREADY RECORDED {label}", flush=True) + return + cleanup() + args = { + "taskCount": task_count, + "batchSize": 100, + "interBatchMs": 50, + "maxParallelism": 200, + "taskType": kind, + "pollTimeoutMs": 180_000, + "pool": variant, + } + if kind == "action": + args["taskDurationMs"] = 20 + print(f"START {label} target=dev:{DEPLOYMENT}", flush=True) + began = datetime.datetime.now(datetime.timezone.utc).isoformat() + result = call("test/scenarios/throughput:default", args, label=label) + metrics = result.get("metrics") or {} + if ( + result.get("timedOut") + or metrics.get("status") != "completed" + or metrics.get("completedCount") != task_count + ): + raise RuntimeError(f"Invalid completion: {label}: {result}") + evidence = call( + query="""const run = await ctx.db.query("runs").order("desc").first(); +const tasks = await ctx.db.query("tasks").withIndex("runId", q => q.eq("runId", run._id)).take(6001); +const outcomes = {}; for (const t of tasks) outcomes[t.resultKind ?? "unknown"] = (outcomes[t.resultKind ?? "unknown"] ?? 0) + 1; +return {run, observedAt: Date.now(), outcomes, uniqueWorkIds: new Set(tasks.map(t => t.workId)).size};""", + label=f"{label}-evidence", + ) + if ( + evidence["run"]["pool"] != variant + or evidence["uniqueWorkIds"] != task_count + or evidence["outcomes"] != {"success": task_count} + ): + raise RuntimeError(f"Invalid outcomes: {label}: {evidence}") + drained = drain() + record = { + "label": label, + "phase": phase, + "rep": rep, + "variant": variant, + "workload": kind, + "began": began, + "args": args, + "result": result, + "evidence": evidence, + "drained": drained, + } + (OUT / f"{label}.json").write_text(json.dumps(record, indent=2) + "\n") + with (OUT / "runs.jsonl").open("a") as f: + f.write(json.dumps(record) + "\n") + print( + json.dumps( + { + "label": label, + "tps": round(task_count * 1000 / metrics["totalDurationMs"], 2), + "enqueueMs": result["enqueueTotal"], + "latency": metrics.get("latency"), + } + ), + flush=True, + ) + + +def main(): + logs = (OUT / "runtime.jsonl").open("a") + errors = (OUT / "runtime.stderr").open("a") + logger = subprocess.Popen( + [ + NODE, + str(CLI), + "logs", + "--deployment", + DEPLOYMENT, + "--success", + "--jsonl", + "--history", + "0", + ], + cwd=STAGE, + stdout=logs, + stderr=errors, + start_new_session=True, + ) + try: + for component in ["testWorkpool", "oldWorkpool"]: + counts = call(component=component, query=COUNT_QUERY) + if any(counts.values()): + raise RuntimeError(f"Unrelated pool is active: {component}: {counts}") + for variant, info in VARIANTS.items(): + call( + "config:update", + {"maxParallelism": 200, "logLevel": "REPORT"}, + component=info["component"], + allow_empty=True, + ) + for variant in VARIANTS: + measure("mutation", variant, "smoke", 1, 1000) + if not ARGS.smoke: + for kind in ["mutation", "action"]: + for rep, order in enumerate(META["warmupOrders"], 1): + for variant in order: + measure(kind, variant, "warmup", rep, 5000) + for rep, order in enumerate(META["measuredOrders"], 1): + for variant in order: + measure(kind, variant, "measured", rep, 5000) + cleanup() + print("COMPLETE", flush=True) + finally: + time.sleep(2) + logger.terminate() + try: + logger.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.kill() + logger.wait() + logs.close() + errors.close() + + +if __name__ == "__main__": + main() From 97cd2e110e1e85e767fcbb6dad96b435ebfda97f Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:11:49 -0700 Subject: [PATCH 6/9] fix(benchmarks): validate inputs and require live logging --- example/benchmarks/2026-09-09/environment.py | 24 ++ example/benchmarks/2026-09-09/prepare.py | 19 +- example/benchmarks/2026-09-09/run.py | 87 +++++-- example/benchmarks/2026-09-09/test_runner.py | 255 +++++++++++++++++++ 4 files changed, 342 insertions(+), 43 deletions(-) create mode 100644 example/benchmarks/2026-09-09/environment.py create mode 100644 example/benchmarks/2026-09-09/test_runner.py diff --git a/example/benchmarks/2026-09-09/environment.py b/example/benchmarks/2026-09-09/environment.py new file mode 100644 index 00000000..f9046575 --- /dev/null +++ b/example/benchmarks/2026-09-09/environment.py @@ -0,0 +1,24 @@ +"""Validate the environment used by the pinned benchmark checkout.""" + +import os +import re +import shlex + + +def check_environment(text, deployment): + assignments = {} + for line in text.lstrip("\ufeff").splitlines(): + assignment = re.fullmatch( + r"\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*|:\s+)(.*)", line + ) + if assignment: + assignments[assignment[1]] = assignment[2] + target = shlex.split(assignments.get("CONVEX_DEPLOYMENT", ""), comments=True) + if ( + target != [f"dev:{deployment}"] + or "CONVEX_DEPLOY_KEY" in assignments + or os.environ.get("CONVEX_DEPLOY_KEY") + ): + raise RuntimeError( + "Use the recorded dev deployment without a deploy-key override" + ) diff --git a/example/benchmarks/2026-09-09/prepare.py b/example/benchmarks/2026-09-09/prepare.py index e6f7ca61..74f3b651 100644 --- a/example/benchmarks/2026-09-09/prepare.py +++ b/example/benchmarks/2026-09-09/prepare.py @@ -3,12 +3,13 @@ import argparse import io import json -import os import shutil import subprocess import tarfile from pathlib import Path +from environment import check_environment + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--checkout", type=Path, required=True) parser.add_argument("--env-file", type=Path, required=True) @@ -21,21 +22,7 @@ stage = args.checkout.resolve() if stage.exists(): raise RuntimeError(f"Checkout already exists: {stage}") -env = args.env_file.read_text() -deployment = next( - ( - line.split("=", 1)[1].split("#", 1)[0].strip().strip("\"'") - for line in env.splitlines() - if line.startswith("CONVEX_DEPLOYMENT=") - ), - None, -) -if ( - deployment != f"dev:{metadata['deployment']}" - or "CONVEX_DEPLOY_KEY=" in env - or os.environ.get("CONVEX_DEPLOY_KEY") -): - raise RuntimeError("Use the recorded dev deployment without a deploy-key override") +check_environment(args.env_file.read_text(), metadata["deployment"]) for name, version in metadata["dependencies"].items(): installed = json.loads((root / "node_modules" / name / "package.json").read_text())[ "version" diff --git a/example/benchmarks/2026-09-09/run.py b/example/benchmarks/2026-09-09/run.py index 50e858c2..e5879903 100644 --- a/example/benchmarks/2026-09-09/run.py +++ b/example/benchmarks/2026-09-09/run.py @@ -4,12 +4,13 @@ import datetime import hashlib import json -import os import shutil import subprocess import time from pathlib import Path +from environment import check_environment + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--checkout", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) @@ -37,13 +38,33 @@ CLI = STAGE / "node_modules/convex/dist/cli.bundle.cjs" DEPLOYMENT = META["deployment"] VARIANTS = META["variants"] +check_environment((STAGE / ".env.local").read_text(), DEPLOYMENT) +WORKLOADS = {kind: settings.copy() for kind, settings in META["workloads"].items()} +if set(WORKLOADS) != {"mutation", "action"}: + raise RuntimeError("Expected mutation and action workloads") +if WORKLOADS["mutation"].pop("artificialWork", None) is not None: + raise RuntimeError("The mutation workload does not support artificial work") +for kind, settings in WORKLOADS.items(): + fields = {"taskCount", "batchSize", "interBatchMs", "maxParallelism"} + if kind == "action": + fields.add("taskDurationMs") + if set(settings) != fields: + raise RuntimeError(f"Unsupported or missing settings for {kind}") + for name, value in settings.items(): + minimum = 0 if name in {"interBatchMs", "taskDurationMs"} else 1 + if type(value) is not int or value < minimum: + raise RuntimeError(f"{kind}.{name} must be an integer >= {minimum}") if ( - os.environ.get("CONVEX_DEPLOY_KEY") - or "CONVEX_DEPLOY_KEY=" in (STAGE / ".env.local").read_text() + type(META["validation"]["tasksPerVariant"]) is not int + or META["validation"]["tasksPerVariant"] < 1 ): - raise RuntimeError( - "Remove the deploy-key override before using the recorded dev target" - ) + raise RuntimeError("validation.tasksPerVariant must be a positive integer") +if ( + META["validation"]["variants"] != len(VARIANTS) + or META["validation"]["allSuccessful"] is not True + or META["validation"]["executionErrors"] != 0 +): + raise RuntimeError("Validation must cover every variant with no failed executions") for name, info in VARIANTS.items(): source = STAGE / "benchmark-components" / name / "src/component" for path, expected in info["sourceHashes"].items(): @@ -72,6 +93,12 @@ const state = await ctx.db.query("internalState").first(); counts.running = state?.running.length ?? 0; return counts;""" +LOGGER = None + + +def check_logger(): + if LOGGER is None or LOGGER.poll() is not None: + raise RuntimeError("Runtime logging stopped; see runtime.stderr") def call( @@ -83,6 +110,7 @@ def call( label=None, allow_empty=False, ): + check_logger() cmd = [NODE, str(CLI), "run", "--deployment", DEPLOYMENT] if component: cmd += ["--component", component] @@ -93,6 +121,7 @@ def call( if label: (OUT / f"{label}.stdout").write_text(result.stdout) (OUT / f"{label}.stderr").write_text(result.stderr) + check_logger() if result.returncode: raise RuntimeError(f"{function or component}: {result.stderr[-3000:]}") if not result.stdout.strip(): @@ -129,23 +158,21 @@ def cleanup(): time.sleep(1) -def measure(kind, variant, phase, rep, task_count): +def measure(kind, variant, phase, rep): label = f"{kind}-{phase}-{rep}-{variant}" if (OUT / f"{label}.json").exists(): print(f"ALREADY RECORDED {label}", flush=True) return cleanup() args = { - "taskCount": task_count, - "batchSize": 100, - "interBatchMs": 50, - "maxParallelism": 200, + **WORKLOADS[kind], "taskType": kind, "pollTimeoutMs": 180_000, "pool": variant, } - if kind == "action": - args["taskDurationMs"] = 20 + if phase == "smoke": + args["taskCount"] = META["validation"]["tasksPerVariant"] + task_count = args["taskCount"] print(f"START {label} target=dev:{DEPLOYMENT}", flush=True) began = datetime.datetime.now(datetime.timezone.utc).isoformat() result = call("test/scenarios/throughput:default", args, label=label) @@ -157,10 +184,10 @@ def measure(kind, variant, phase, rep, task_count): ): raise RuntimeError(f"Invalid completion: {label}: {result}") evidence = call( - query="""const run = await ctx.db.query("runs").order("desc").first(); -const tasks = await ctx.db.query("tasks").withIndex("runId", q => q.eq("runId", run._id)).take(6001); -const outcomes = {}; for (const t of tasks) outcomes[t.resultKind ?? "unknown"] = (outcomes[t.resultKind ?? "unknown"] ?? 0) + 1; -return {run, observedAt: Date.now(), outcomes, uniqueWorkIds: new Set(tasks.map(t => t.workId)).size};""", + query=f"""const run = await ctx.db.query("runs").order("desc").first(); +const tasks = await ctx.db.query("tasks").withIndex("runId", q => q.eq("runId", run._id)).take({task_count + 1}); +const outcomes = {{}}; for (const t of tasks) outcomes[t.resultKind ?? "unknown"] = (outcomes[t.resultKind ?? "unknown"] ?? 0) + 1; +return {{run, observedAt: Date.now(), outcomes, uniqueWorkIds: new Set(tasks.map(t => t.workId)).size}};""", label=f"{label}-evidence", ) if ( @@ -182,6 +209,7 @@ def measure(kind, variant, phase, rep, task_count): "evidence": evidence, "drained": drained, } + check_logger() (OUT / f"{label}.json").write_text(json.dumps(record, indent=2) + "\n") with (OUT / "runs.jsonl").open("a") as f: f.write(json.dumps(record) + "\n") @@ -199,9 +227,10 @@ def measure(kind, variant, phase, rep, task_count): def main(): + global LOGGER logs = (OUT / "runtime.jsonl").open("a") errors = (OUT / "runtime.stderr").open("a") - logger = subprocess.Popen( + LOGGER = subprocess.Popen( [ NODE, str(CLI), @@ -226,30 +255,34 @@ def main(): for variant, info in VARIANTS.items(): call( "config:update", - {"maxParallelism": 200, "logLevel": "REPORT"}, + { + "maxParallelism": WORKLOADS["mutation"]["maxParallelism"], + "logLevel": "REPORT", + }, component=info["component"], allow_empty=True, ) for variant in VARIANTS: - measure("mutation", variant, "smoke", 1, 1000) + measure("mutation", variant, "smoke", 1) if not ARGS.smoke: for kind in ["mutation", "action"]: for rep, order in enumerate(META["warmupOrders"], 1): for variant in order: - measure(kind, variant, "warmup", rep, 5000) + measure(kind, variant, "warmup", rep) for rep, order in enumerate(META["measuredOrders"], 1): for variant in order: - measure(kind, variant, "measured", rep, 5000) + measure(kind, variant, "measured", rep) cleanup() + time.sleep(2) + check_logger() print("COMPLETE", flush=True) finally: - time.sleep(2) - logger.terminate() + LOGGER.terminate() try: - logger.wait(timeout=10) + LOGGER.wait(timeout=10) except subprocess.TimeoutExpired: - logger.kill() - logger.wait() + LOGGER.kill() + LOGGER.wait() logs.close() errors.close() diff --git a/example/benchmarks/2026-09-09/test_runner.py b/example/benchmarks/2026-09-09/test_runner.py new file mode 100644 index 00000000..705fe28c --- /dev/null +++ b/example/benchmarks/2026-09-09/test_runner.py @@ -0,0 +1,255 @@ +import contextlib +import io +import json +import os +import runpy +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from environment import check_environment + +HERE = Path(__file__).resolve().parent + + +class EnvironmentTests(unittest.TestCase): + def test_rejects_deploy_key_assignments(self): + with patch.dict(os.environ, {}, clear=True): + for assignment in [ + "CONVEX_DEPLOY_KEY=test-key", + " CONVEX_DEPLOY_KEY = 'test-key'", + "export CONVEX_DEPLOY_KEY = test-key", + "\texport\tCONVEX_DEPLOY_KEY\t=\ttest-key", + "UNRELATED=\nCONVEX_DEPLOY_KEY=", + "CONVEX_DEPLOY_KEY: test-key", + ]: + with ( + self.subTest(assignment=assignment), + self.assertRaises(RuntimeError), + ): + check_environment( + "CONVEX_DEPLOYMENT=dev:target\n" + assignment, "target" + ) + with self.assertRaises(RuntimeError): + check_environment( + "\ufeffCONVEX_DEPLOY_KEY = test-key\nCONVEX_DEPLOYMENT=dev:target", + "target", + ) + + def test_accepts_exported_target_and_ignores_comments(self): + with patch.dict(os.environ, {}, clear=True): + check_environment( + '# CONVEX_DEPLOY_KEY = ignored\nexport CONVEX_DEPLOYMENT = "dev:target" # comment', + "target", + ) + + def test_rejects_process_override(self): + with ( + patch.dict(os.environ, {"CONVEX_DEPLOY_KEY": "test-key"}, clear=True), + self.assertRaises(RuntimeError), + ): + check_environment("CONVEX_DEPLOYMENT=dev:target", "target") + + def test_checks_last_target_assignment_and_preserves_quoted_hash(self): + with patch.dict(os.environ, {}, clear=True): + for text in [ + "CONVEX_DEPLOYMENT=dev:target\nCONVEX_DEPLOYMENT=prod:other", + 'CONVEX_DEPLOYMENT="dev:target#other"', + ]: + with self.subTest(text=text), self.assertRaises(RuntimeError): + check_environment(text, "target") + + +class RunnerTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.output = self.root / "output" + self.metadata = json.loads((HERE / "metadata.json").read_text()) + self.metadata["dependencies"] = {} + for variant in self.metadata["variants"].values(): + variant["sourceHashes"] = {} + self.metadata["validation"]["tasksPerVariant"] = 7 + self.metadata["workloads"]["mutation"].update( + taskCount=23, batchSize=5, interBatchMs=2, maxParallelism=16 + ) + self.metadata["workloads"]["action"].update( + taskCount=31, + batchSize=8, + interBatchMs=3, + maxParallelism=12, + taskDurationMs=45, + ) + (self.root / ".env.local").write_text( + f"CONVEX_DEPLOYMENT=dev:{self.metadata['deployment']}\n" + ) + + def load_runner(self): + metadata_file = self.root / "metadata.json" + metadata_file.write_text(json.dumps(self.metadata)) + argv = [ + str(HERE / "run.py"), + "--checkout", + str(self.root), + "--output", + str(self.output), + "--metadata", + str(metadata_file), + ] + with ( + patch.object(sys, "argv", argv), + patch.dict(os.environ, {}, clear=True), + patch("shutil.which", return_value="unused-node"), + patch("subprocess.check_output", return_value="v24.0.0"), + ): + module = runpy.run_path(str(HERE / "run.py")) + namespace = module["measure"].__globals__ + namespace["LOGGER"] = Mock(poll=Mock(return_value=None)) + namespace["cleanup"] = Mock() + namespace["drain"] = Mock(return_value={"baseline": {"work": 0}}) + return namespace + + def test_metadata_controls_measured_and_smoke_arguments(self): + runner = self.load_runner() + for kind, phase, count in [ + ("mutation", "measured", 23), + ("action", "measured", 31), + ("mutation", "smoke", 7), + ]: + captured = [] + + def invoke(cmd, *, calls=captured, task_count=count, **kwargs): + calls.append(cmd) + if "--inline-query" in cmd: + result = { + "run": {"pool": "baseline"}, + "uniqueWorkIds": task_count, + "outcomes": {"success": task_count}, + } + else: + result = { + "metrics": { + "status": "completed", + "completedCount": task_count, + "totalDurationMs": 100, + }, + "enqueueTotal": 10, + } + return subprocess.CompletedProcess(cmd, 0, json.dumps(result), "") + + with ( + self.subTest(kind=kind, phase=phase), + patch("subprocess.run", side_effect=invoke), + contextlib.redirect_stdout(io.StringIO()), + ): + runner["measure"](kind, "baseline", phase, 1) + sent = json.loads(captured[0][-1]) + expected = self.metadata["workloads"][kind].copy() + expected.pop("artificialWork", None) + expected.update( + taskCount=count, + taskType=kind, + pool="baseline", + pollTimeoutMs=180_000, + ) + self.assertEqual(sent, expected) + self.assertIn(f".take({count + 1})", captured[1][-1]) + record = json.loads( + (self.output / f"{kind}-{phase}-1-baseline.json").read_text() + ) + self.assertEqual(record["args"], expected) + + def test_rejects_inconsistent_validation_metadata(self): + for field, value in [ + ("variants", 4), + ("allSuccessful", False), + ("executionErrors", 1), + ]: + with self.subTest(field=field): + original = self.metadata["validation"][field] + self.metadata["validation"][field] = value + with self.assertRaisesRegex(RuntimeError, "Validation must cover"): + self.load_runner() + self.metadata["validation"][field] = original + self.assertFalse((self.output / "metadata.json").exists()) + + def test_rejects_unsupported_mutation_work(self): + self.metadata["workloads"]["mutation"]["artificialWork"] = 20 + with self.assertRaisesRegex(RuntimeError, "does not support artificial work"): + self.load_runner() + + def test_rejects_invalid_counts_and_ignored_settings(self): + for field, value in [ + ("taskCount", 0), + ("batchSize", 0), + ("taskCount", 1.5), + ("taskDurationMs", 20), + ]: + with self.subTest(field=field, value=value): + original = self.metadata["workloads"]["mutation"].copy() + self.metadata["workloads"]["mutation"][field] = value + with self.assertRaises(RuntimeError): + self.load_runner() + self.metadata["workloads"]["mutation"] = original + self.metadata["validation"]["tasksPerVariant"] = 0 + with self.assertRaisesRegex(RuntimeError, "must be a positive integer"): + self.load_runner() + + def test_stopped_logger_prevents_calls_even_after_a_clean_exit(self): + runner = self.load_runner() + for exit_code in [0, 1]: + with self.subTest(exit_code=exit_code), patch("subprocess.run") as invoke: + runner["LOGGER"].poll.return_value = exit_code + with self.assertRaisesRegex(RuntimeError, "Runtime logging stopped"): + runner["call"]("test/cleanup:start") + invoke.assert_not_called() + + def test_logger_exit_during_a_call_prevents_recording_the_run(self): + runner = self.load_runner() + + def invoke(cmd, **kwargs): + runner["LOGGER"].poll.return_value = 0 + result = { + "metrics": { + "status": "completed", + "completedCount": 7, + "totalDurationMs": 100, + }, + "enqueueTotal": 10, + } + return subprocess.CompletedProcess(cmd, 0, json.dumps(result), "") + + with ( + patch("subprocess.run", side_effect=invoke), + contextlib.redirect_stdout(io.StringIO()), + self.assertRaisesRegex(RuntimeError, "Runtime logging stopped"), + ): + runner["measure"]("mutation", "baseline", "smoke", 1) + self.assertTrue((self.output / "mutation-smoke-1-baseline.stdout").exists()) + self.assertFalse((self.output / "mutation-smoke-1-baseline.json").exists()) + self.assertFalse((self.output / "runs.jsonl").exists()) + + def test_final_log_failure_prevents_complete(self): + runner = self.load_runner() + logger = runner["LOGGER"] + runner["call"] = Mock(return_value={"work": 0}) + runner["measure"] = Mock() + runner["cleanup"].side_effect = lambda: setattr(logger.poll, "return_value", 0) + output = io.StringIO() + with ( + patch("subprocess.Popen", return_value=logger), + patch("time.sleep"), + contextlib.redirect_stdout(output), + self.assertRaisesRegex(RuntimeError, "Runtime logging stopped"), + ): + runner["main"]() + self.assertNotIn("COMPLETE", output.getvalue()) + logger.terminate.assert_called_once() + + +if __name__ == "__main__": + unittest.main() From 767c2212495918b38373de0534ac9847d1b21cc1 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:11:50 -0700 Subject: [PATCH 7/9] fix(benchmarks): reject incomplete enqueue logs --- example/benchmarks/2026-09-09/analyze.py | 18 +++- example/benchmarks/2026-09-09/test_analyze.py | 92 +++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 example/benchmarks/2026-09-09/test_analyze.py diff --git a/example/benchmarks/2026-09-09/analyze.py b/example/benchmarks/2026-09-09/analyze.py index f2852748..a244962d 100644 --- a/example/benchmarks/2026-09-09/analyze.py +++ b/example/benchmarks/2026-09-09/analyze.py @@ -9,8 +9,12 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("output", type=Path, help="Directory produced by run.py") OUT = parser.parse_args().output +analysis_file = OUT / "analysis.json" +analysis_file.unlink(missing_ok=True) metadata = json.loads((OUT / "metadata.json").read_text()) runs = [json.loads(line) for line in (OUT / "runs.jsonl").read_text().splitlines()] +if not runs: + raise RuntimeError("No completed benchmark runs to analyze") windows = collections.defaultdict(list) for run in runs: run["runtime"] = {} @@ -104,6 +108,16 @@ stats[target] += usage[source] break +for run in runs: + count = run["args"]["taskCount"] + batch_size = run["args"]["batchSize"] + expected = (count + batch_size - 1) // batch_size + actual = run["runtime"].get("enqueue", {}).get("executions", 0) + if actual != expected: + raise RuntimeError( + f"{run['label']}: expected {expected} lib:enqueueBatch completions, found {actual}" + ) + summary = {} for workload in ["mutation", "action"]: summary[workload] = {} @@ -135,7 +149,7 @@ comparisons = [] for workload in ["mutation", "action"]: - for rep in [1, 2, 3]: + for rep in range(1, len(metadata["measuredOrders"]) + 1): group = { r["variant"]: r for r in runs @@ -171,7 +185,7 @@ incident = OUT / "cleanup-incident.json" if incident.exists(): result["cleanupIncident"] = json.loads(incident.read_text()) -(OUT / "analysis.json").write_text(json.dumps(result, indent=2) + "\n") +analysis_file.write_text(json.dumps(result, indent=2) + "\n") print( json.dumps( { diff --git a/example/benchmarks/2026-09-09/test_analyze.py b/example/benchmarks/2026-09-09/test_analyze.py new file mode 100644 index 00000000..87e88821 --- /dev/null +++ b/example/benchmarks/2026-09-09/test_analyze.py @@ -0,0 +1,92 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +class AnalysisTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.output = Path(self.directory.name) + metadata = { + "variants": {"baseline": {"component": "pool"}}, + "measuredOrders": [], + } + run = { + "label": "smoke", + "variant": "baseline", + "phase": "smoke", + "workload": "mutation", + "args": {"taskCount": 3, "batchSize": 2}, + "evidence": {"run": {"startTime": 1000}, "observedAt": 2000}, + } + (self.output / "metadata.json").write_text(json.dumps(metadata)) + (self.output / "runs.jsonl").write_text(json.dumps(run) + "\n") + self.events = [ + { + "kind": "Completion", + "executionId": str(i), + "timestamp": 1.5, + "executionTimestamp": 1.5, + "componentPath": "pool", + "identifier": "lib:enqueueBatch", + "executionTime": 0.01, + "usageStats": { + key: 1 + for key in [ + "databaseReadDocuments", + "databaseWriteDocuments", + "databaseReadBytes", + "databaseWriteBytes", + "databaseIoReadBytes", + "databaseIoWriteBytes", + "databaseWriteIndexRows", + ] + }, + } + for i in range(2) + ] + + def analyze(self, events): + (self.output / "runtime.jsonl").write_text( + "".join(json.dumps(e) + "\n" for e in events) + ) + return subprocess.run( + [sys.executable, str(HERE / "analyze.py"), str(self.output)], + text=True, + capture_output=True, + check=False, + ) + + def test_accepts_partial_final_batch_and_deduplicates_logs(self): + result = self.analyze(self.events + self.events) + self.assertEqual(result.returncode, 0, result.stderr) + analysis = json.loads((self.output / "analysis.json").read_text()) + self.assertEqual(analysis["runs"][0]["runtime"]["enqueue"]["executions"], 2) + + def test_rejects_missing_logs_and_removes_stale_analysis(self): + for count in [0, 1]: + with self.subTest(count=count): + (self.output / "analysis.json").write_text("{}") + result = self.analyze(self.events[:count]) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + f"expected 2 lib:enqueueBatch completions, found {count}", + result.stderr, + ) + self.assertFalse((self.output / "analysis.json").exists()) + + def test_rejects_empty_run_list(self): + (self.output / "runs.jsonl").write_text("") + result = self.analyze([]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("No completed benchmark runs", result.stderr) + + +if __name__ == "__main__": + unittest.main() From 6f136b4a1f1bff611a598eac0d0a63e259d5591d Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:11:50 -0700 Subject: [PATCH 8/9] ci: test benchmark tooling --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 942a715c..349a1904 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,3 +43,5 @@ jobs: npm run typecheck npm run lint npm run format:check + - name: Test benchmark tooling + run: python3 -B -m unittest discover -s example/benchmarks/2026-09-09 From 5352d16f81efddcb7a95d4aca3d33c6b53d09b33 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:50:49 -0700 Subject: [PATCH 9/9] fix(benchmarks): flush runtime logs after failures --- example/benchmarks/2026-09-09/run.py | 56 +++++++++++--------- example/benchmarks/2026-09-09/test_runner.py | 37 ++++++++++++- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/example/benchmarks/2026-09-09/run.py b/example/benchmarks/2026-09-09/run.py index e5879903..fa738a27 100644 --- a/example/benchmarks/2026-09-09/run.py +++ b/example/benchmarks/2026-09-09/run.py @@ -248,32 +248,36 @@ def main(): start_new_session=True, ) try: - for component in ["testWorkpool", "oldWorkpool"]: - counts = call(component=component, query=COUNT_QUERY) - if any(counts.values()): - raise RuntimeError(f"Unrelated pool is active: {component}: {counts}") - for variant, info in VARIANTS.items(): - call( - "config:update", - { - "maxParallelism": WORKLOADS["mutation"]["maxParallelism"], - "logLevel": "REPORT", - }, - component=info["component"], - allow_empty=True, - ) - for variant in VARIANTS: - measure("mutation", variant, "smoke", 1) - if not ARGS.smoke: - for kind in ["mutation", "action"]: - for rep, order in enumerate(META["warmupOrders"], 1): - for variant in order: - measure(kind, variant, "warmup", rep) - for rep, order in enumerate(META["measuredOrders"], 1): - for variant in order: - measure(kind, variant, "measured", rep) - cleanup() - time.sleep(2) + try: + for component in ["testWorkpool", "oldWorkpool"]: + counts = call(component=component, query=COUNT_QUERY) + if any(counts.values()): + raise RuntimeError( + f"Unrelated pool is active: {component}: {counts}" + ) + for variant, info in VARIANTS.items(): + call( + "config:update", + { + "maxParallelism": WORKLOADS["mutation"]["maxParallelism"], + "logLevel": "REPORT", + }, + component=info["component"], + allow_empty=True, + ) + for variant in VARIANTS: + measure("mutation", variant, "smoke", 1) + if not ARGS.smoke: + for kind in ["mutation", "action"]: + for rep, order in enumerate(META["warmupOrders"], 1): + for variant in order: + measure(kind, variant, "warmup", rep) + for rep, order in enumerate(META["measuredOrders"], 1): + for variant in order: + measure(kind, variant, "measured", rep) + cleanup() + finally: + time.sleep(2) check_logger() print("COMPLETE", flush=True) finally: diff --git a/example/benchmarks/2026-09-09/test_runner.py b/example/benchmarks/2026-09-09/test_runner.py index 705fe28c..5d40ca81 100644 --- a/example/benchmarks/2026-09-09/test_runner.py +++ b/example/benchmarks/2026-09-09/test_runner.py @@ -238,11 +238,13 @@ def test_final_log_failure_prevents_complete(self): logger = runner["LOGGER"] runner["call"] = Mock(return_value={"work": 0}) runner["measure"] = Mock() - runner["cleanup"].side_effect = lambda: setattr(logger.poll, "return_value", 0) output = io.StringIO() with ( patch("subprocess.Popen", return_value=logger), - patch("time.sleep"), + patch( + "time.sleep", + side_effect=lambda _: setattr(logger.poll, "return_value", 0), + ), contextlib.redirect_stdout(output), self.assertRaisesRegex(RuntimeError, "Runtime logging stopped"), ): @@ -250,6 +252,37 @@ def test_final_log_failure_prevents_complete(self): self.assertNotIn("COMPLETE", output.getvalue()) logger.terminate.assert_called_once() + def test_log_flush_precedes_shutdown_on_success_and_failure(self): + for failure in [None, RuntimeError("Benchmark failed"), KeyboardInterrupt()]: + with self.subTest(failure=failure): + runner = self.load_runner() + logger = runner["LOGGER"] + runner["call"] = Mock(return_value={"work": 0}) + runner["measure"] = Mock(side_effect=failure) + output = io.StringIO() + + def flush(seconds, logger=logger, output=output): + self.assertEqual(seconds, 2) + logger.terminate.assert_not_called() + self.assertNotIn("COMPLETE", output.getvalue()) + + with ( + patch("subprocess.Popen", return_value=logger), + patch("time.sleep", side_effect=flush) as sleep, + contextlib.redirect_stdout(output), + ): + if failure is None: + runner["main"]() + else: + with self.assertRaises(type(failure)) as raised: + runner["main"]() + self.assertIs(raised.exception, failure) + + sleep.assert_called_once_with(2) + logger.terminate.assert_called_once() + logger.wait.assert_called_once_with(timeout=10) + self.assertEqual("COMPLETE" in output.getvalue(), failure is None) + if __name__ == "__main__": unittest.main()