diff --git a/apps/web/src/instrumentation.ts b/apps/web/src/instrumentation.ts index c8043fe25..2ef2acae5 100644 --- a/apps/web/src/instrumentation.ts +++ b/apps/web/src/instrumentation.ts @@ -30,6 +30,13 @@ export async function register() { ); await initDomainVerificationJob(); + + // Deletes finished webhook calls older than 30 days, in small batches, + // nightly. Cloud only: self-hosted installs keep their history until + // there's a setting to opt in. + if (isCloud()) { + await import("~/server/jobs/webhook-cleanup-job"); + } } if (isEmailCleanupEnabled()) { diff --git a/apps/web/src/instrumentation.unit.test.ts b/apps/web/src/instrumentation.unit.test.ts new file mode 100644 index 000000000..ceb0692df --- /dev/null +++ b/apps/web/src/instrumentation.unit.test.ts @@ -0,0 +1,75 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const flags = vi.hoisted(() => ({ + webhookCleanupImported: false, + isCloud: true, +})); + +const mocks = vi.hoisted(() => ({ + init: vi.fn().mockResolvedValue(undefined), + initDomain: vi.fn().mockResolvedValue(undefined), + start: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("~/utils/common", () => ({ + isCloud: () => flags.isCloud, + isEmailCleanupEnabled: () => false, +})); + +vi.mock("~/server/service/email-queue-service", () => ({ + EmailQueueService: { init: mocks.init }, +})); + +vi.mock("~/server/jobs/usage-job", () => ({})); + +vi.mock("~/server/jobs/domain-verification-job", () => ({ + initDomainVerificationJob: mocks.initDomain, +})); + +vi.mock("~/server/jobs/cleanup-email-bodies", () => ({})); + +vi.mock("~/server/jobs/campaign-scheduler-job", () => ({ + CampaignSchedulerService: { start: mocks.start }, +})); + +describe("instrumentation register", () => { + beforeEach(() => { + vi.resetModules(); + // Registered fresh for every test: a factory registered once with vi.mock + // is cached, so later imports wouldn't record that the job was loaded. + vi.doMock("~/server/jobs/webhook-cleanup-job", () => { + flags.webhookCleanupImported = true; + return {}; + }); + flags.webhookCleanupImported = false; + flags.isCloud = true; + vi.stubEnv("NEXT_RUNTIME", "nodejs"); + vi.stubEnv("REDIS_URL", "redis://127.0.0.1:6380/15"); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("starts the webhook cleanup job when Redis is configured", async () => { + const { register } = await import("~/instrumentation"); + await register(); + expect(mocks.init).toHaveBeenCalledTimes(1); + expect(mocks.initDomain).toHaveBeenCalledTimes(1); + expect(flags.webhookCleanupImported).toBe(true); + }); + + it("does not start the webhook cleanup job on self-hosted installs", async () => { + flags.isCloud = false; + const { register } = await import("~/instrumentation"); + await register(); + expect(flags.webhookCleanupImported).toBe(false); + }); + + it("does not start the webhook cleanup job without a Redis URL", async () => { + vi.stubEnv("REDIS_URL", ""); + const { register } = await import("~/instrumentation"); + await register(); + expect(flags.webhookCleanupImported).toBe(false); + }); +}); diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts b/apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts new file mode 100644 index 000000000..45fa1e998 --- /dev/null +++ b/apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts @@ -0,0 +1,121 @@ +import { WebhookCallStatus } from "@prisma/client"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { db } from "~/server/db"; +import { createTeam } from "~/test/factories/core"; +import { + closeIntegrationConnections, + integrationEnabled, + resetDatabase, +} from "~/test/integration/helpers"; + +// Only the database is real here: the job module's queue and worker are not +// needed to test the deletion itself. +vi.mock("bullmq", () => ({ + Queue: class { + upsertJobScheduler = vi.fn().mockResolvedValue(undefined); + }, + Worker: class { + on = vi.fn(); + }, +})); + +import { deleteOldWebhookCalls } from "~/server/jobs/webhook-cleanup-job"; + +const describeIntegration = integrationEnabled ? describe : describe.skip; + +const DAY_MS = 24 * 60 * 60 * 1000; + +describeIntegration("webhook cleanup against postgres", () => { + let webhookId: string; + let teamId: number; + + async function createCall( + name: string, + status: WebhookCallStatus, + createdDaysAgo: number, + updatedDaysAgo: number, + ) { + const call = await db.webhookCall.create({ + data: { + webhookId, + teamId, + type: "email.sent", + payload: JSON.stringify({ name }), + status, + }, + }); + const createdAt = new Date(Date.now() - createdDaysAgo * DAY_MS); + const updatedAt = new Date(Date.now() - updatedDaysAgo * DAY_MS); + await db.$executeRaw`UPDATE "WebhookCall" SET "createdAt" = ${createdAt}, "updatedAt" = ${updatedAt} WHERE id = ${call.id}`; + return call.id; + } + + beforeEach(async () => { + await resetDatabase(); + const team = await createTeam(); + teamId = team.id; + const webhook = await db.webhook.create({ + data: { + teamId, + url: "https://example.com/hook", + secret: "whsec_test", + eventTypes: [], + }, + }); + webhookId = webhook.id; + }); + + afterAll(async () => { + await closeIntegrationConnections(); + }); + + it("deletes only finished calls that are over 30 days old and untouched since", async () => { + const deletedIds = [ + await createCall("old delivered 1", WebhookCallStatus.DELIVERED, 60, 60), + await createCall("old delivered 2", WebhookCallStatus.DELIVERED, 45, 45), + await createCall("old failed", WebhookCallStatus.FAILED, 50, 49), + await createCall("old discarded", WebhookCallStatus.DISCARDED, 40, 40), + ]; + const keptIds = [ + await createCall("old but pending", WebhookCallStatus.PENDING, 60, 60), + await createCall( + "old in progress", + WebhookCallStatus.IN_PROGRESS, + 60, + 60, + ), + await createCall( + "old, retried today", + WebhookCallStatus.DELIVERED, + 60, + 0, + ), + await createCall("recent", WebhookCallStatus.DELIVERED, 2, 2), + ]; + + const result = await deleteOldWebhookCalls({ + cutoff: new Date(Date.now() - 30 * DAY_MS), + batchSize: 3, + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 4, batches: 2, finished: true }); + const remaining = await db.webhookCall.findMany({ select: { id: true } }); + expect(remaining.map((row) => row.id).sort()).toEqual([...keptIds].sort()); + expect(deletedIds.every((id) => !remaining.some((r) => r.id === id))).toBe( + true, + ); + }); + + it("does nothing when nothing is old enough", async () => { + await createCall("recent", WebhookCallStatus.DELIVERED, 1, 1); + + const result = await deleteOldWebhookCalls({ + cutoff: new Date(Date.now() - 30 * DAY_MS), + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 0, batches: 0, finished: true }); + expect(await db.webhookCall.count()).toBe(1); + }); +}); diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.ts b/apps/web/src/server/jobs/webhook-cleanup-job.ts index 6eb64261b..26f5f453d 100644 --- a/apps/web/src/server/jobs/webhook-cleanup-job.ts +++ b/apps/web/src/server/jobs/webhook-cleanup-job.ts @@ -1,12 +1,128 @@ import { Queue, Worker } from "bullmq"; +import { WebhookCallStatus } from "@prisma/client"; import { subDays } from "date-fns"; import { db } from "~/server/db"; import { getRedis, BULL_PREFIX } from "~/server/redis"; -import { DEFAULT_QUEUE_OPTIONS, WEBHOOK_CLEANUP_QUEUE } from "../queue/queue-constants"; +import { + DEFAULT_QUEUE_OPTIONS, + WEBHOOK_CLEANUP_QUEUE, +} from "../queue/queue-constants"; import { logger } from "../logger/log"; const WEBHOOK_RETENTION_DAYS = 30; +// Delete in small batches, each its own short transaction, with a pause in +// between, rather than one DELETE of months of rows (one long transaction and +// a burst of WAL). This limits the work per transaction and per night; it does +// not shrink the database files: deleted rows become reusable space after +// autovacuum. A run stops after MAX_RUN_MS and the next night's run carries on. +const DELETE_BATCH_SIZE = 1000; +const PAUSE_BETWEEN_BATCHES_MS = 250; +const MAX_RUN_MS = 10 * 60 * 1000; +// Per batch: give up rather than wait on a lock or a slow statement. +const BATCH_LOCK_TIMEOUT = "5s"; +const BATCH_STATEMENT_TIMEOUT = "60s"; +// Prisma's own interactive-transaction limits (its default timeout is 5s, +// which would cut a slow batch off first): room for the two statements above, +// and a short wait for a connection. +const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 }; +const LOG_PROGRESS_EVERY_BATCHES = 50; + +// Only calls that are finished. A call that is pending or in progress (for +// example an old call someone pressed Retry on) is never deleted. +const FINISHED_STATUSES: WebhookCallStatus[] = [ + WebhookCallStatus.DELIVERED, + WebhookCallStatus.FAILED, + WebhookCallStatus.DISCARDED, +]; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Deletes one batch of finished webhook calls that were created and last + * updated before the cutoff. The same conditions are checked again in the + * DELETE, so a call retried after it was selected is kept. + */ +async function deleteBatch(cutoff: Date, batchSize: number) { + const oldAndFinished = { + createdAt: { lt: cutoff }, + updatedAt: { lt: cutoff }, + status: { in: FINISHED_STATUSES }, + }; + + return db.$transaction(async (tx) => { + await tx.$executeRawUnsafe( + `SET LOCAL lock_timeout = '${BATCH_LOCK_TIMEOUT}'`, + ); + await tx.$executeRawUnsafe( + `SET LOCAL statement_timeout = '${BATCH_STATEMENT_TIMEOUT}'`, + ); + + // Uses the createdAt index; ids keep the DELETE small and exact. + const rows = await tx.webhookCall.findMany({ + where: oldAndFinished, + select: { id: true }, + orderBy: { createdAt: "asc" }, + take: batchSize, + }); + + if (rows.length === 0) return { selected: 0, deleted: 0 }; + + const result = await tx.webhookCall.deleteMany({ + where: { id: { in: rows.map((row) => row.id) }, ...oldAndFinished }, + }); + + return { selected: rows.length, deleted: result.count }; + }, BATCH_TRANSACTION_OPTIONS); +} + +export async function deleteOldWebhookCalls({ + cutoff, + batchSize = DELETE_BATCH_SIZE, + pauseMs = PAUSE_BETWEEN_BATCHES_MS, + maxRunMs = MAX_RUN_MS, + now = () => Date.now(), +}: { + cutoff: Date; + batchSize?: number; + pauseMs?: number; + maxRunMs?: number; + now?: () => number; +}) { + const startedAt = now(); + let deleted = 0; + let batches = 0; + const progress = (finished: boolean) => ({ + deleted, + batches, + finished, + elapsedMs: now() - startedAt, + }); + + for (;;) { + // Check the deadline before starting each batch. + if (batches > 0 && now() - startedAt >= maxRunMs) { + return progress(false); + } + + const batch = await deleteBatch(cutoff, batchSize); + deleted += batch.deleted; + if (batch.selected === 0) return progress(true); + batches += 1; + + if (batch.selected < batchSize) return progress(true); + + if (batches % LOG_PROGRESS_EVERY_BATCHES === 0) { + logger.info( + { ...progress(false), cutoff: cutoff.toISOString() }, + "[WebhookCleanupJob]: Still deleting old webhook calls", + ); + } + + await sleep(pauseMs); + } +} + const webhookCleanupQueue = new Queue(WEBHOOK_CLEANUP_QUEUE, { connection: getRedis(), prefix: BULL_PREFIX, @@ -17,24 +133,20 @@ const worker = new Worker( WEBHOOK_CLEANUP_QUEUE, async () => { const cutoff = subDays(new Date(), WEBHOOK_RETENTION_DAYS); - const result = await db.webhookCall.deleteMany({ - where: { - createdAt: { - lt: cutoff, - }, - }, - }); + const result = await deleteOldWebhookCalls({ cutoff }); logger.info( - { deleted: result.count, cutoff: cutoff.toISOString() }, - "[WebhookCleanupJob]: Deleted old webhook calls", + { ...result, cutoff: cutoff.toISOString() }, + result.finished + ? "[WebhookCleanupJob]: Deleted old webhook calls" + : "[WebhookCleanupJob]: Deleted old webhook calls; more remain for the next run", ); }, { connection: getRedis(), prefix: BULL_PREFIX, skipVersionCheck: true, - } + }, ); await webhookCleanupQueue.upsertJobScheduler( @@ -47,7 +159,7 @@ await webhookCleanupQueue.upsertJobScheduler( opts: { ...DEFAULT_QUEUE_OPTIONS, }, - } + }, ); worker.on("completed", (job) => { diff --git a/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts new file mode 100644 index 000000000..a6959bbcb --- /dev/null +++ b/apps/web/src/server/jobs/webhook-cleanup-job.unit.test.ts @@ -0,0 +1,161 @@ +import { WebhookCallStatus } from "@prisma/client"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { mockTx, mockDb } = vi.hoisted(() => { + const mockTx = { + $executeRawUnsafe: vi.fn(), + webhookCall: { + findMany: vi.fn(), + deleteMany: vi.fn(), + }, + }; + return { + mockTx, + mockDb: { + $transaction: vi.fn(), + }, + }; +}); + +vi.mock("~/server/db", () => ({ db: mockDb })); +vi.mock("~/server/redis", () => ({ + getRedis: vi.fn(() => ({})), + BULL_PREFIX: "test", +})); +vi.mock("~/server/logger/log", () => ({ + logger: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, +})); +vi.mock("bullmq", () => ({ + Queue: class { + upsertJobScheduler = vi.fn().mockResolvedValue(undefined); + }, + Worker: class { + on = vi.fn(); + }, +})); + +import { deleteOldWebhookCalls } from "~/server/jobs/webhook-cleanup-job"; + +const ids = (count: number, from = 0) => + Array.from({ length: count }, (_, i) => ({ id: `call_${from + i}` })); + +describe("deleteOldWebhookCalls", () => { + const cutoff = new Date("2026-10-17T00:00:00Z"); + const oldAndFinished = { + createdAt: { lt: cutoff }, + updatedAt: { lt: cutoff }, + status: { + in: [ + WebhookCallStatus.DELIVERED, + WebhookCallStatus.FAILED, + WebhookCallStatus.DISCARDED, + ], + }, + }; + + beforeEach(() => { + mockDb.$transaction.mockImplementation(async (fn: any) => fn(mockTx)); + mockTx.$executeRawUnsafe.mockResolvedValue(0); + mockTx.webhookCall.deleteMany.mockImplementation( + async ({ where }: any) => ({ + count: where.id.in.length, + }), + ); + }); + + it("deletes old, finished calls in batches until none are left", async () => { + mockTx.webhookCall.findMany + .mockResolvedValueOnce(ids(3, 0)) + .mockResolvedValueOnce(ids(3, 3)) + .mockResolvedValueOnce(ids(1, 6)); + + const result = await deleteOldWebhookCalls({ + cutoff, + batchSize: 3, + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 7, batches: 3, finished: true }); + // Only finished calls created and last updated before the cutoff. + expect(mockTx.webhookCall.findMany).toHaveBeenCalledWith({ + where: oldAndFinished, + select: { id: true }, + orderBy: { createdAt: "asc" }, + take: 3, + }); + // The DELETE repeats those conditions, so a call retried in between is kept. + expect(mockTx.webhookCall.deleteMany).toHaveBeenNthCalledWith(1, { + where: { id: { in: ["call_0", "call_1", "call_2"] }, ...oldAndFinished }, + }); + // Each batch runs in its own transaction with lock and statement timeouts, + // and Prisma's transaction limit set above them (its default is 5s). + expect(mockDb.$transaction).toHaveBeenCalledTimes(3); + expect(mockDb.$transaction).toHaveBeenCalledWith(expect.any(Function), { + timeout: 130_000, + maxWait: 2_000, + }); + expect(mockTx.$executeRawUnsafe).toHaveBeenCalledWith( + "SET LOCAL lock_timeout = '5s'", + ); + expect(mockTx.$executeRawUnsafe).toHaveBeenCalledWith( + "SET LOCAL statement_timeout = '60s'", + ); + }); + + it("does nothing when there is nothing old enough", async () => { + mockTx.webhookCall.findMany.mockResolvedValue([]); + + const result = await deleteOldWebhookCalls({ cutoff, pauseMs: 0 }); + + expect(result).toMatchObject({ deleted: 0, batches: 0, finished: true }); + expect(mockTx.webhookCall.deleteMany).not.toHaveBeenCalled(); + }); + + it("counts only the calls actually deleted", async () => { + // Two of the three selected calls were retried before the DELETE ran. + mockTx.webhookCall.findMany.mockResolvedValueOnce(ids(3)); + mockTx.webhookCall.deleteMany.mockResolvedValueOnce({ count: 1 }); + + const result = await deleteOldWebhookCalls({ + cutoff, + batchSize: 5, + pauseMs: 0, + }); + + expect(result).toMatchObject({ deleted: 1, batches: 1, finished: true }); + }); + + it("doesn't start a batch after the time limit, and leaves the rest for the next run", async () => { + mockTx.webhookCall.findMany.mockImplementation(async () => ids(2)); + let clock = 0; + const now = () => clock; + mockTx.webhookCall.deleteMany.mockImplementation(async ({ where }: any) => { + clock += 400; // each batch takes 400ms + return { count: where.id.in.length }; + }); + + const result = await deleteOldWebhookCalls({ + cutoff, + batchSize: 2, + pauseMs: 0, + maxRunMs: 1000, + now, + }); + + // Batches start at 0, 400 and 800ms; at 1200ms the limit has passed. + expect(result).toMatchObject({ deleted: 6, batches: 3, finished: false }); + }); + + it("stops the run when a batch fails, keeping what was already deleted", async () => { + mockTx.webhookCall.findMany + .mockResolvedValueOnce(ids(2)) + .mockRejectedValueOnce( + new Error("canceling statement due to lock timeout"), + ); + + await expect( + deleteOldWebhookCalls({ cutoff, batchSize: 2, pauseMs: 0 }), + ).rejects.toThrow("lock timeout"); + expect(mockTx.webhookCall.deleteMany).toHaveBeenCalledTimes(1); + }); +});