From 08a41d36f43ce9590661935355874c74a92c7d9a Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 10:05:32 +1100 Subject: [PATCH 1/4] fix(web): start the webhook cleanup job, and delete old calls in small batches The nightly webhook cleanup job was never imported, so it never ran: webhook call history has grown since webhooks shipped in January. - Start it at server startup with the other Redis-backed jobs. - Delete in batches of 1000 ids (each its own short transaction) with a short pause between batches, stopping after 10 minutes; the next run carries on. A single DELETE of ~10 months of rows would write a burst of WAL and hold a long transaction on a database whose disk is ~78% full. Supersedes #507. Co-authored-by: Polylane --- apps/web/src/instrumentation.ts | 3 + apps/web/src/instrumentation.unit.test.ts | 63 +++++++++++ .../src/server/jobs/webhook-cleanup-job.ts | 81 +++++++++++--- .../jobs/webhook-cleanup-job.unit.test.ts | 101 ++++++++++++++++++ 4 files changed, 236 insertions(+), 12 deletions(-) create mode 100644 apps/web/src/instrumentation.unit.test.ts create mode 100644 apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts diff --git a/apps/web/src/instrumentation.ts b/apps/web/src/instrumentation.ts index c8043fe25..24fe2794f 100644 --- a/apps/web/src/instrumentation.ts +++ b/apps/web/src/instrumentation.ts @@ -30,6 +30,9 @@ export async function register() { ); await initDomainVerificationJob(); + + // Deletes webhook calls older than 30 days, in small batches, nightly. + await import("~/server/jobs/webhook-cleanup-job"); } if (isEmailCleanupEnabled()) { diff --git a/apps/web/src/instrumentation.unit.test.ts b/apps/web/src/instrumentation.unit.test.ts new file mode 100644 index 000000000..99cb069cd --- /dev/null +++ b/apps/web/src/instrumentation.unit.test.ts @@ -0,0 +1,63 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const flags = vi.hoisted(() => ({ webhookCleanupImported: false })); + +const mocks = vi.hoisted(() => ({ + init: vi.fn().mockResolvedValue(undefined), + initDomain: vi.fn().mockResolvedValue(undefined), + start: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("~/utils/common", () => ({ + isCloud: () => true, + isEmailCleanupEnabled: () => false, +})); + +vi.mock("~/server/service/email-queue-service", () => ({ + EmailQueueService: { init: mocks.init }, +})); + +vi.mock("~/server/jobs/usage-job", () => ({})); + +vi.mock("~/server/jobs/domain-verification-job", () => ({ + initDomainVerificationJob: mocks.initDomain, +})); + +vi.mock("~/server/jobs/cleanup-email-bodies", () => ({})); + +vi.mock("~/server/jobs/campaign-scheduler-job", () => ({ + CampaignSchedulerService: { start: mocks.start }, +})); + +vi.mock("~/server/jobs/webhook-cleanup-job", () => { + flags.webhookCleanupImported = true; + return {}; +}); + +describe("instrumentation register", () => { + beforeEach(() => { + vi.resetModules(); + flags.webhookCleanupImported = false; + vi.stubEnv("NEXT_RUNTIME", "nodejs"); + vi.stubEnv("REDIS_URL", "redis://127.0.0.1:6380/15"); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("starts the webhook cleanup job when Redis is configured", async () => { + const { register } = await import("~/instrumentation"); + await register(); + expect(mocks.init).toHaveBeenCalledTimes(1); + expect(mocks.initDomain).toHaveBeenCalledTimes(1); + expect(flags.webhookCleanupImported).toBe(true); + }); + + it("does not start the webhook cleanup job without a Redis URL", async () => { + vi.stubEnv("REDIS_URL", ""); + const { register } = await import("~/instrumentation"); + await register(); + expect(flags.webhookCleanupImported).toBe(false); + }); +}); diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.ts b/apps/web/src/server/jobs/webhook-cleanup-job.ts index 6eb64261b..4afcf4e1e 100644 --- a/apps/web/src/server/jobs/webhook-cleanup-job.ts +++ b/apps/web/src/server/jobs/webhook-cleanup-job.ts @@ -2,11 +2,72 @@ import { Queue, Worker } from "bullmq"; import { subDays } from "date-fns"; import { db } from "~/server/db"; import { getRedis, BULL_PREFIX } from "~/server/redis"; -import { DEFAULT_QUEUE_OPTIONS, WEBHOOK_CLEANUP_QUEUE } from "../queue/queue-constants"; +import { + DEFAULT_QUEUE_OPTIONS, + WEBHOOK_CLEANUP_QUEUE, +} from "../queue/queue-constants"; import { logger } from "../logger/log"; const WEBHOOK_RETENTION_DAYS = 30; +// Delete in small batches, each its own short transaction, with a pause in +// between. One big DELETE of months of rows writes a lot of WAL at once and +// holds a long transaction; small batches keep both bounded. A run stops after +// MAX_RUN_MS and the next night's run carries on. +const DELETE_BATCH_SIZE = 1000; +const PAUSE_BETWEEN_BATCHES_MS = 250; +const MAX_RUN_MS = 10 * 60 * 1000; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export async function deleteOldWebhookCalls({ + cutoff, + batchSize = DELETE_BATCH_SIZE, + pauseMs = PAUSE_BETWEEN_BATCHES_MS, + maxRunMs = MAX_RUN_MS, + now = () => Date.now(), +}: { + cutoff: Date; + batchSize?: number; + pauseMs?: number; + maxRunMs?: number; + now?: () => number; +}) { + const startedAt = now(); + let deleted = 0; + let batches = 0; + + for (;;) { + // Uses the createdAt index; ids keep each DELETE small and exact. + const rows = await db.webhookCall.findMany({ + where: { createdAt: { lt: cutoff } }, + select: { id: true }, + orderBy: { createdAt: "asc" }, + take: batchSize, + }); + + if (rows.length === 0) { + return { deleted, batches, finished: true }; + } + + const result = await db.webhookCall.deleteMany({ + where: { id: { in: rows.map((row) => row.id) } }, + }); + deleted += result.count; + batches += 1; + + if (rows.length < batchSize) { + return { deleted, batches, finished: true }; + } + + if (now() - startedAt >= maxRunMs) { + return { deleted, batches, finished: false }; + } + + await sleep(pauseMs); + } +} + const webhookCleanupQueue = new Queue(WEBHOOK_CLEANUP_QUEUE, { connection: getRedis(), prefix: BULL_PREFIX, @@ -17,24 +78,20 @@ const worker = new Worker( WEBHOOK_CLEANUP_QUEUE, async () => { const cutoff = subDays(new Date(), WEBHOOK_RETENTION_DAYS); - const result = await db.webhookCall.deleteMany({ - where: { - createdAt: { - lt: cutoff, - }, - }, - }); + const result = await deleteOldWebhookCalls({ cutoff }); logger.info( - { deleted: result.count, cutoff: cutoff.toISOString() }, - "[WebhookCleanupJob]: Deleted old webhook calls", + { ...result, cutoff: cutoff.toISOString() }, + result.finished + ? "[WebhookCleanupJob]: Deleted old webhook calls" + : "[WebhookCleanupJob]: Deleted old webhook calls; more remain for the next run", ); }, { connection: getRedis(), prefix: BULL_PREFIX, skipVersionCheck: true, - } + }, ); await webhookCleanupQueue.upsertJobScheduler( @@ -47,7 +104,7 @@ await webhookCleanupQueue.upsertJobScheduler( opts: { ...DEFAULT_QUEUE_OPTIONS, }, - } + }, ); worker.on("completed", (job) => { diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts new file mode 100644 index 000000000..5ea077ba4 --- /dev/null +++ b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { mockDb } = vi.hoisted(() => ({ + mockDb: { + webhookCall: { + findMany: vi.fn(), + deleteMany: vi.fn(), + }, + }, +})); + +vi.mock("~/server/db", () => ({ db: mockDb })); +vi.mock("~/server/redis", () => ({ + getRedis: vi.fn(() => ({})), + BULL_PREFIX: "test", +})); +vi.mock("~/server/logger/log", () => ({ + logger: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, +})); +vi.mock("bullmq", () => ({ + Queue: class { + upsertJobScheduler = vi.fn().mockResolvedValue(undefined); + }, + Worker: class { + on = vi.fn(); + }, +})); + +import { deleteOldWebhookCalls } from "~/server/jobs/webhook-cleanup-job"; + +const ids = (count: number, from = 0) => + Array.from({ length: count }, (_, i) => ({ id: `call_${from + i}` })); + +describe("deleteOldWebhookCalls", () => { + const cutoff = new Date("2026-10-17T00:00:00Z"); + + beforeEach(() => { + mockDb.webhookCall.deleteMany.mockImplementation( + async ({ where }: any) => ({ + count: where.id.in.length, + }), + ); + }); + + it("deletes old calls in batches until none are left", async () => { + mockDb.webhookCall.findMany + .mockResolvedValueOnce(ids(3, 0)) + .mockResolvedValueOnce(ids(3, 3)) + .mockResolvedValueOnce(ids(1, 6)); + + const result = await deleteOldWebhookCalls({ + cutoff, + batchSize: 3, + pauseMs: 0, + }); + + expect(result).toEqual({ deleted: 7, batches: 3, finished: true }); + // Only calls older than the cutoff, oldest first, one batch at a time. + expect(mockDb.webhookCall.findMany).toHaveBeenCalledWith({ + where: { createdAt: { lt: cutoff } }, + select: { id: true }, + orderBy: { createdAt: "asc" }, + take: 3, + }); + // Each DELETE is limited to the ids of its batch. + expect(mockDb.webhookCall.deleteMany).toHaveBeenNthCalledWith(1, { + where: { id: { in: ["call_0", "call_1", "call_2"] } }, + }); + expect(mockDb.webhookCall.deleteMany).toHaveBeenCalledTimes(3); + }); + + it("does nothing when there is nothing old enough", async () => { + mockDb.webhookCall.findMany.mockResolvedValue([]); + + const result = await deleteOldWebhookCalls({ cutoff, pauseMs: 0 }); + + expect(result).toEqual({ deleted: 0, batches: 0, finished: true }); + expect(mockDb.webhookCall.deleteMany).not.toHaveBeenCalled(); + }); + + it("stops at the time limit and leaves the rest for the next run", async () => { + mockDb.webhookCall.findMany.mockImplementation(async () => ids(2)); + let clock = 0; + const now = () => { + clock += 400; + return clock; + }; + + const result = await deleteOldWebhookCalls({ + cutoff, + batchSize: 2, + pauseMs: 0, + maxRunMs: 1000, + now, + }); + + expect(result.finished).toBe(false); + expect(result.batches).toBeLessThanOrEqual(3); + expect(result.deleted).toBe(result.batches * 2); + }); +}); From 104bdca284d28e86a803396d0aa970a3b4727c68 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 10:08:56 +1100 Subject: [PATCH 2/4] fix(web): only delete finished, untouched webhook calls, with per-batch timeouts, on cloud only - Delete only DELIVERED/FAILED/DISCARDED calls created and last updated over 30 days ago; the DELETE repeats these conditions, so a call retried after it was selected is kept. - Each batch runs in its own transaction with lock_timeout 5s and statement_timeout 60s; the deadline is checked before each batch; a failed batch stops the run, keeping what was already deleted. - Start the job on cloud only; self-hosted installs keep their history until there's a setting to opt in. - Log progress every 50 batches and the run's elapsed time; comments no longer claim the cleanup shrinks the database files. --- apps/web/src/instrumentation.ts | 8 +- apps/web/src/instrumentation.unit.test.ts | 15 ++- .../src/server/jobs/webhook-cleanup-job.ts | 95 +++++++++++---- .../jobs/webhook-cleanup-job.unit.test.ts | 109 +++++++++++++----- 4 files changed, 174 insertions(+), 53 deletions(-) diff --git a/apps/web/src/instrumentation.ts b/apps/web/src/instrumentation.ts index 24fe2794f..2ef2acae5 100644 --- a/apps/web/src/instrumentation.ts +++ b/apps/web/src/instrumentation.ts @@ -31,8 +31,12 @@ export async function register() { await initDomainVerificationJob(); - // Deletes webhook calls older than 30 days, in small batches, nightly. - await import("~/server/jobs/webhook-cleanup-job"); + // Deletes finished webhook calls older than 30 days, in small batches, + // nightly. Cloud only: self-hosted installs keep their history until + // there's a setting to opt in. + if (isCloud()) { + await import("~/server/jobs/webhook-cleanup-job"); + } } if (isEmailCleanupEnabled()) { diff --git a/apps/web/src/instrumentation.unit.test.ts b/apps/web/src/instrumentation.unit.test.ts index 99cb069cd..3225b7699 100644 --- a/apps/web/src/instrumentation.unit.test.ts +++ b/apps/web/src/instrumentation.unit.test.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const flags = vi.hoisted(() => ({ webhookCleanupImported: false })); +const flags = vi.hoisted(() => ({ + webhookCleanupImported: false, + isCloud: true, +})); const mocks = vi.hoisted(() => ({ init: vi.fn().mockResolvedValue(undefined), @@ -9,7 +12,7 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("~/utils/common", () => ({ - isCloud: () => true, + isCloud: () => flags.isCloud, isEmailCleanupEnabled: () => false, })); @@ -38,6 +41,7 @@ describe("instrumentation register", () => { beforeEach(() => { vi.resetModules(); flags.webhookCleanupImported = false; + flags.isCloud = true; vi.stubEnv("NEXT_RUNTIME", "nodejs"); vi.stubEnv("REDIS_URL", "redis://127.0.0.1:6380/15"); }); @@ -54,6 +58,13 @@ describe("instrumentation register", () => { expect(flags.webhookCleanupImported).toBe(true); }); + it("does not start the webhook cleanup job on self-hosted installs", async () => { + flags.isCloud = false; + const { register } = await import("~/instrumentation"); + await register(); + expect(flags.webhookCleanupImported).toBe(false); + }); + it("does not start the webhook cleanup job without a Redis URL", async () => { vi.stubEnv("REDIS_URL", ""); const { register } = await import("~/instrumentation"); diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.ts b/apps/web/src/server/jobs/webhook-cleanup-job.ts index 4afcf4e1e..d9f677fac 100644 --- a/apps/web/src/server/jobs/webhook-cleanup-job.ts +++ b/apps/web/src/server/jobs/webhook-cleanup-job.ts @@ -1,4 +1,5 @@ import { Queue, Worker } from "bullmq"; +import { WebhookCallStatus } from "@prisma/client"; import { subDays } from "date-fns"; import { db } from "~/server/db"; import { getRedis, BULL_PREFIX } from "~/server/redis"; @@ -11,15 +12,66 @@ import { logger } from "../logger/log"; const WEBHOOK_RETENTION_DAYS = 30; // Delete in small batches, each its own short transaction, with a pause in -// between. One big DELETE of months of rows writes a lot of WAL at once and -// holds a long transaction; small batches keep both bounded. A run stops after -// MAX_RUN_MS and the next night's run carries on. +// between, rather than one DELETE of months of rows (one long transaction and +// a burst of WAL). This limits the work per transaction and per night; it does +// not shrink the database files: deleted rows become reusable space after +// autovacuum. A run stops after MAX_RUN_MS and the next night's run carries on. const DELETE_BATCH_SIZE = 1000; const PAUSE_BETWEEN_BATCHES_MS = 250; const MAX_RUN_MS = 10 * 60 * 1000; +// Per batch: give up rather than wait on a lock or a slow statement. +const BATCH_LOCK_TIMEOUT = "5s"; +const BATCH_STATEMENT_TIMEOUT = "60s"; +const LOG_PROGRESS_EVERY_BATCHES = 50; + +// Only calls that are finished. A call that is pending or in progress (for +// example an old call someone pressed Retry on) is never deleted. +const FINISHED_STATUSES: WebhookCallStatus[] = [ + WebhookCallStatus.DELIVERED, + WebhookCallStatus.FAILED, + WebhookCallStatus.DISCARDED, +]; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** + * Deletes one batch of finished webhook calls that were created and last + * updated before the cutoff. The same conditions are checked again in the + * DELETE, so a call retried after it was selected is kept. + */ +async function deleteBatch(cutoff: Date, batchSize: number) { + const oldAndFinished = { + createdAt: { lt: cutoff }, + updatedAt: { lt: cutoff }, + status: { in: FINISHED_STATUSES }, + }; + + return db.$transaction(async (tx) => { + await tx.$executeRawUnsafe( + `SET LOCAL lock_timeout = '${BATCH_LOCK_TIMEOUT}'`, + ); + await tx.$executeRawUnsafe( + `SET LOCAL statement_timeout = '${BATCH_STATEMENT_TIMEOUT}'`, + ); + + // Uses the createdAt index; ids keep the DELETE small and exact. + const rows = await tx.webhookCall.findMany({ + where: oldAndFinished, + select: { id: true }, + orderBy: { createdAt: "asc" }, + take: batchSize, + }); + + if (rows.length === 0) return { selected: 0, deleted: 0 }; + + const result = await tx.webhookCall.deleteMany({ + where: { id: { in: rows.map((row) => row.id) }, ...oldAndFinished }, + }); + + return { selected: rows.length, deleted: result.count }; + }); +} + export async function deleteOldWebhookCalls({ cutoff, batchSize = DELETE_BATCH_SIZE, @@ -36,32 +88,31 @@ export async function deleteOldWebhookCalls({ const startedAt = now(); let deleted = 0; let batches = 0; + const progress = (finished: boolean) => ({ + deleted, + batches, + finished, + elapsedMs: now() - startedAt, + }); for (;;) { - // Uses the createdAt index; ids keep each DELETE small and exact. - const rows = await db.webhookCall.findMany({ - where: { createdAt: { lt: cutoff } }, - select: { id: true }, - orderBy: { createdAt: "asc" }, - take: batchSize, - }); - - if (rows.length === 0) { - return { deleted, batches, finished: true }; + // Check the deadline before starting each batch. + if (batches > 0 && now() - startedAt >= maxRunMs) { + return progress(false); } - const result = await db.webhookCall.deleteMany({ - where: { id: { in: rows.map((row) => row.id) } }, - }); - deleted += result.count; + const batch = await deleteBatch(cutoff, batchSize); + deleted += batch.deleted; + if (batch.selected === 0) return progress(true); batches += 1; - if (rows.length < batchSize) { - return { deleted, batches, finished: true }; - } + if (batch.selected < batchSize) return progress(true); - if (now() - startedAt >= maxRunMs) { - return { deleted, batches, finished: false }; + if (batches % LOG_PROGRESS_EVERY_BATCHES === 0) { + logger.info( + { ...progress(false), cutoff: cutoff.toISOString() }, + "[WebhookCleanupJob]: Still deleting old webhook calls", + ); } await sleep(pauseMs); diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts index 5ea077ba4..d22233e4c 100644 --- a/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts +++ b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts @@ -1,13 +1,21 @@ +import { WebhookCallStatus } from "@prisma/client"; import { beforeEach, describe, expect, it, vi } from "vitest"; -const { mockDb } = vi.hoisted(() => ({ - mockDb: { +const { mockTx, mockDb } = vi.hoisted(() => { + const mockTx = { + $executeRawUnsafe: vi.fn(), webhookCall: { findMany: vi.fn(), deleteMany: vi.fn(), }, - }, -})); + }; + return { + mockTx, + mockDb: { + $transaction: vi.fn(), + }, + }; +}); vi.mock("~/server/db", () => ({ db: mockDb })); vi.mock("~/server/redis", () => ({ @@ -33,17 +41,30 @@ const ids = (count: number, from = 0) => describe("deleteOldWebhookCalls", () => { const cutoff = new Date("2026-10-17T00:00:00Z"); + const oldAndFinished = { + createdAt: { lt: cutoff }, + updatedAt: { lt: cutoff }, + status: { + in: [ + WebhookCallStatus.DELIVERED, + WebhookCallStatus.FAILED, + WebhookCallStatus.DISCARDED, + ], + }, + }; beforeEach(() => { - mockDb.webhookCall.deleteMany.mockImplementation( + mockDb.$transaction.mockImplementation(async (fn: any) => fn(mockTx)); + mockTx.$executeRawUnsafe.mockResolvedValue(0); + mockTx.webhookCall.deleteMany.mockImplementation( async ({ where }: any) => ({ count: where.id.in.length, }), ); }); - it("deletes old calls in batches until none are left", async () => { - mockDb.webhookCall.findMany + it("deletes old, finished calls in batches until none are left", async () => { + mockTx.webhookCall.findMany .mockResolvedValueOnce(ids(3, 0)) .mockResolvedValueOnce(ids(3, 3)) .mockResolvedValueOnce(ids(1, 6)); @@ -54,37 +75,59 @@ describe("deleteOldWebhookCalls", () => { pauseMs: 0, }); - expect(result).toEqual({ deleted: 7, batches: 3, finished: true }); - // Only calls older than the cutoff, oldest first, one batch at a time. - expect(mockDb.webhookCall.findMany).toHaveBeenCalledWith({ - where: { createdAt: { lt: cutoff } }, + expect(result).toMatchObject({ deleted: 7, batches: 3, finished: true }); + // Only finished calls created and last updated before the cutoff. + expect(mockTx.webhookCall.findMany).toHaveBeenCalledWith({ + where: oldAndFinished, select: { id: true }, orderBy: { createdAt: "asc" }, take: 3, }); - // Each DELETE is limited to the ids of its batch. - expect(mockDb.webhookCall.deleteMany).toHaveBeenNthCalledWith(1, { - where: { id: { in: ["call_0", "call_1", "call_2"] } }, + // The DELETE repeats those conditions, so a call retried in between is kept. + expect(mockTx.webhookCall.deleteMany).toHaveBeenNthCalledWith(1, { + where: { id: { in: ["call_0", "call_1", "call_2"] }, ...oldAndFinished }, }); - expect(mockDb.webhookCall.deleteMany).toHaveBeenCalledTimes(3); + // Each batch runs in its own transaction with lock and statement timeouts. + expect(mockDb.$transaction).toHaveBeenCalledTimes(3); + expect(mockTx.$executeRawUnsafe).toHaveBeenCalledWith( + "SET LOCAL lock_timeout = '5s'", + ); + expect(mockTx.$executeRawUnsafe).toHaveBeenCalledWith( + "SET LOCAL statement_timeout = '60s'", + ); }); it("does nothing when there is nothing old enough", async () => { - mockDb.webhookCall.findMany.mockResolvedValue([]); + mockTx.webhookCall.findMany.mockResolvedValue([]); const result = await deleteOldWebhookCalls({ cutoff, pauseMs: 0 }); - expect(result).toEqual({ deleted: 0, batches: 0, finished: true }); - expect(mockDb.webhookCall.deleteMany).not.toHaveBeenCalled(); + expect(result).toMatchObject({ deleted: 0, batches: 0, finished: true }); + expect(mockTx.webhookCall.deleteMany).not.toHaveBeenCalled(); }); - it("stops at the time limit and leaves the rest for the next run", async () => { - mockDb.webhookCall.findMany.mockImplementation(async () => ids(2)); + it("counts only the calls actually deleted", async () => { + // Two of the three selected calls were retried before the DELETE ran. + mockTx.webhookCall.findMany.mockResolvedValueOnce(ids(3)); + mockTx.webhookCall.deleteMany.mockResolvedValueOnce({ count: 1 }); + + const result = await deleteOldWebhookCalls({ + cutoff, + batchSize: 5, + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 1, batches: 1, finished: true }); + }); + + it("doesn't start a batch after the time limit, and leaves the rest for the next run", async () => { + mockTx.webhookCall.findMany.mockImplementation(async () => ids(2)); let clock = 0; - const now = () => { - clock += 400; - return clock; - }; + const now = () => clock; + mockTx.webhookCall.deleteMany.mockImplementation(async ({ where }: any) => { + clock += 400; // each batch takes 400ms + return { count: where.id.in.length }; + }); const result = await deleteOldWebhookCalls({ cutoff, @@ -94,8 +137,20 @@ describe("deleteOldWebhookCalls", () => { now, }); - expect(result.finished).toBe(false); - expect(result.batches).toBeLessThanOrEqual(3); - expect(result.deleted).toBe(result.batches * 2); + // Batches start at 0, 400 and 800ms; at 1200ms the limit has passed. + expect(result).toMatchObject({ deleted: 6, batches: 3, finished: false }); + }); + + it("stops the run when a batch fails, keeping what was already deleted", async () => { + mockTx.webhookCall.findMany + .mockResolvedValueOnce(ids(2)) + .mockRejectedValueOnce( + new Error("canceling statement due to lock timeout"), + ); + + await expect( + deleteOldWebhookCalls({ cutoff, batchSize: 2, pauseMs: 0 }), + ).rejects.toThrow("lock timeout"); + expect(mockTx.webhookCall.deleteMany).toHaveBeenCalledTimes(1); }); }); From 2ca89ade850a50c7ec44633c7123eba75625933e Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 10:11:01 +1100 Subject: [PATCH 3/4] fix(web): give each cleanup batch a Prisma transaction limit above its SQL timeouts; test against postgres Prisma's default interactive-transaction timeout (5s) would have cut a slow batch off before its 60s statement_timeout, possibly failing on the same batch every night. Adds an integration test that runs the cleanup against a real Postgres (CI): only finished calls created and last updated over 30 days ago are deleted. --- .../webhook-cleanup-job.integration.test.ts | 121 ++++++++++++++++++ .../src/server/jobs/webhook-cleanup-job.ts | 6 +- .../jobs/webhook-cleanup-job.unit.test.ts | 7 +- 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts b/apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts new file mode 100644 index 000000000..45fa1e998 --- /dev/null +++ b/apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts @@ -0,0 +1,121 @@ +import { WebhookCallStatus } from "@prisma/client"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { db } from "~/server/db"; +import { createTeam } from "~/test/factories/core"; +import { + closeIntegrationConnections, + integrationEnabled, + resetDatabase, +} from "~/test/integration/helpers"; + +// Only the database is real here: the job module's queue and worker are not +// needed to test the deletion itself. +vi.mock("bullmq", () => ({ + Queue: class { + upsertJobScheduler = vi.fn().mockResolvedValue(undefined); + }, + Worker: class { + on = vi.fn(); + }, +})); + +import { deleteOldWebhookCalls } from "~/server/jobs/webhook-cleanup-job"; + +const describeIntegration = integrationEnabled ? describe : describe.skip; + +const DAY_MS = 24 * 60 * 60 * 1000; + +describeIntegration("webhook cleanup against postgres", () => { + let webhookId: string; + let teamId: number; + + async function createCall( + name: string, + status: WebhookCallStatus, + createdDaysAgo: number, + updatedDaysAgo: number, + ) { + const call = await db.webhookCall.create({ + data: { + webhookId, + teamId, + type: "email.sent", + payload: JSON.stringify({ name }), + status, + }, + }); + const createdAt = new Date(Date.now() - createdDaysAgo * DAY_MS); + const updatedAt = new Date(Date.now() - updatedDaysAgo * DAY_MS); + await db.$executeRaw`UPDATE "WebhookCall" SET "createdAt" = ${createdAt}, "updatedAt" = ${updatedAt} WHERE id = ${call.id}`; + return call.id; + } + + beforeEach(async () => { + await resetDatabase(); + const team = await createTeam(); + teamId = team.id; + const webhook = await db.webhook.create({ + data: { + teamId, + url: "https://example.com/hook", + secret: "whsec_test", + eventTypes: [], + }, + }); + webhookId = webhook.id; + }); + + afterAll(async () => { + await closeIntegrationConnections(); + }); + + it("deletes only finished calls that are over 30 days old and untouched since", async () => { + const deletedIds = [ + await createCall("old delivered 1", WebhookCallStatus.DELIVERED, 60, 60), + await createCall("old delivered 2", WebhookCallStatus.DELIVERED, 45, 45), + await createCall("old failed", WebhookCallStatus.FAILED, 50, 49), + await createCall("old discarded", WebhookCallStatus.DISCARDED, 40, 40), + ]; + const keptIds = [ + await createCall("old but pending", WebhookCallStatus.PENDING, 60, 60), + await createCall( + "old in progress", + WebhookCallStatus.IN_PROGRESS, + 60, + 60, + ), + await createCall( + "old, retried today", + WebhookCallStatus.DELIVERED, + 60, + 0, + ), + await createCall("recent", WebhookCallStatus.DELIVERED, 2, 2), + ]; + + const result = await deleteOldWebhookCalls({ + cutoff: new Date(Date.now() - 30 * DAY_MS), + batchSize: 3, + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 4, batches: 2, finished: true }); + const remaining = await db.webhookCall.findMany({ select: { id: true } }); + expect(remaining.map((row) => row.id).sort()).toEqual([...keptIds].sort()); + expect(deletedIds.every((id) => !remaining.some((r) => r.id === id))).toBe( + true, + ); + }); + + it("does nothing when nothing is old enough", async () => { + await createCall("recent", WebhookCallStatus.DELIVERED, 1, 1); + + const result = await deleteOldWebhookCalls({ + cutoff: new Date(Date.now() - 30 * DAY_MS), + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 0, batches: 0, finished: true }); + expect(await db.webhookCall.count()).toBe(1); + }); +}); diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.ts b/apps/web/src/server/jobs/webhook-cleanup-job.ts index d9f677fac..26f5f453d 100644 --- a/apps/web/src/server/jobs/webhook-cleanup-job.ts +++ b/apps/web/src/server/jobs/webhook-cleanup-job.ts @@ -22,6 +22,10 @@ const MAX_RUN_MS = 10 * 60 * 1000; // Per batch: give up rather than wait on a lock or a slow statement. const BATCH_LOCK_TIMEOUT = "5s"; const BATCH_STATEMENT_TIMEOUT = "60s"; +// Prisma's own interactive-transaction limits (its default timeout is 5s, +// which would cut a slow batch off first): room for the two statements above, +// and a short wait for a connection. +const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 }; const LOG_PROGRESS_EVERY_BATCHES = 50; // Only calls that are finished. A call that is pending or in progress (for @@ -69,7 +73,7 @@ async function deleteBatch(cutoff: Date, batchSize: number) { }); return { selected: rows.length, deleted: result.count }; - }); + }, BATCH_TRANSACTION_OPTIONS); } export async function deleteOldWebhookCalls({ diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts index d22233e4c..a6959bbcb 100644 --- a/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts +++ b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts @@ -87,8 +87,13 @@ describe("deleteOldWebhookCalls", () => { expect(mockTx.webhookCall.deleteMany).toHaveBeenNthCalledWith(1, { where: { id: { in: ["call_0", "call_1", "call_2"] }, ...oldAndFinished }, }); - // Each batch runs in its own transaction with lock and statement timeouts. + // Each batch runs in its own transaction with lock and statement timeouts, + // and Prisma's transaction limit set above them (its default is 5s). expect(mockDb.$transaction).toHaveBeenCalledTimes(3); + expect(mockDb.$transaction).toHaveBeenCalledWith(expect.any(Function), { + timeout: 130_000, + maxWait: 2_000, + }); expect(mockTx.$executeRawUnsafe).toHaveBeenCalledWith( "SET LOCAL lock_timeout = '5s'", ); From f829a979e5f7d85e8fee254a8c83b3b6fc97bdf5 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 10:18:46 +1100 Subject: [PATCH 4/4] test(web): register the cleanup-job mock per test, so the self-hosted check can fail --- apps/web/src/instrumentation.unit.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/web/src/instrumentation.unit.test.ts b/apps/web/src/instrumentation.unit.test.ts index 3225b7699..ceb0692df 100644 --- a/apps/web/src/instrumentation.unit.test.ts +++ b/apps/web/src/instrumentation.unit.test.ts @@ -32,14 +32,15 @@ vi.mock("~/server/jobs/campaign-scheduler-job", () => ({ CampaignSchedulerService: { start: mocks.start }, })); -vi.mock("~/server/jobs/webhook-cleanup-job", () => { - flags.webhookCleanupImported = true; - return {}; -}); - describe("instrumentation register", () => { beforeEach(() => { vi.resetModules(); + // Registered fresh for every test: a factory registered once with vi.mock + // is cached, so later imports wouldn't record that the job was loaded. + vi.doMock("~/server/jobs/webhook-cleanup-job", () => { + flags.webhookCleanupImported = true; + return {}; + }); flags.webhookCleanupImported = false; flags.isCloud = true; vi.stubEnv("NEXT_RUNTIME", "nodejs");