Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions apps/web/src/instrumentation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ export async function register() {
);

await initDomainVerificationJob();

// Deletes finished webhook calls older than 30 days, in small batches,
// nightly. Cloud only: self-hosted installs keep their history until
// there's a setting to opt in.
if (isCloud()) {
await import("~/server/jobs/webhook-cleanup-job");
}
}

if (isEmailCleanupEnabled()) {
Expand Down
75 changes: 75 additions & 0 deletions apps/web/src/instrumentation.unit.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

const flags = vi.hoisted(() => ({
webhookCleanupImported: false,
isCloud: true,
}));

const mocks = vi.hoisted(() => ({
init: vi.fn().mockResolvedValue(undefined),
initDomain: vi.fn().mockResolvedValue(undefined),
start: vi.fn().mockResolvedValue(undefined),
}));

vi.mock("~/utils/common", () => ({
isCloud: () => flags.isCloud,
isEmailCleanupEnabled: () => false,
}));

vi.mock("~/server/service/email-queue-service", () => ({
EmailQueueService: { init: mocks.init },
}));

vi.mock("~/server/jobs/usage-job", () => ({}));

vi.mock("~/server/jobs/domain-verification-job", () => ({
initDomainVerificationJob: mocks.initDomain,
}));

vi.mock("~/server/jobs/cleanup-email-bodies", () => ({}));

vi.mock("~/server/jobs/campaign-scheduler-job", () => ({
CampaignSchedulerService: { start: mocks.start },
}));

describe("instrumentation register", () => {
beforeEach(() => {
vi.resetModules();
// Registered fresh for every test: a factory registered once with vi.mock
// is cached, so later imports wouldn't record that the job was loaded.
vi.doMock("~/server/jobs/webhook-cleanup-job", () => {
flags.webhookCleanupImported = true;
return {};
});
flags.webhookCleanupImported = false;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
flags.isCloud = true;
vi.stubEnv("NEXT_RUNTIME", "nodejs");
vi.stubEnv("REDIS_URL", "redis://127.0.0.1:6380/15");
});

afterEach(() => {
vi.unstubAllEnvs();
});

it("starts the webhook cleanup job when Redis is configured", async () => {
const { register } = await import("~/instrumentation");
await register();
expect(mocks.init).toHaveBeenCalledTimes(1);
expect(mocks.initDomain).toHaveBeenCalledTimes(1);
expect(flags.webhookCleanupImported).toBe(true);
});

it("does not start the webhook cleanup job on self-hosted installs", async () => {
flags.isCloud = false;
const { register } = await import("~/instrumentation");
await register();
expect(flags.webhookCleanupImported).toBe(false);
});

it("does not start the webhook cleanup job without a Redis URL", async () => {
vi.stubEnv("REDIS_URL", "");
const { register } = await import("~/instrumentation");
await register();
expect(flags.webhookCleanupImported).toBe(false);
});
});
121 changes: 121 additions & 0 deletions apps/web/src/server/jobs/webhook-cleanup-job.integration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { WebhookCallStatus } from "@prisma/client";
import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
import { db } from "~/server/db";
import { createTeam } from "~/test/factories/core";
import {
closeIntegrationConnections,
integrationEnabled,
resetDatabase,
} from "~/test/integration/helpers";

// Only the database is real here: the job module's queue and worker are not
// needed to test the deletion itself.
vi.mock("bullmq", () => ({
Queue: class {
upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
},
Worker: class {
on = vi.fn();
},
}));

import { deleteOldWebhookCalls } from "~/server/jobs/webhook-cleanup-job";

const describeIntegration = integrationEnabled ? describe : describe.skip;

const DAY_MS = 24 * 60 * 60 * 1000;

describeIntegration("webhook cleanup against postgres", () => {
let webhookId: string;
let teamId: number;

async function createCall(
name: string,
status: WebhookCallStatus,
createdDaysAgo: number,
updatedDaysAgo: number,
) {
const call = await db.webhookCall.create({
data: {
webhookId,
teamId,
type: "email.sent",
payload: JSON.stringify({ name }),
status,
},
});
const createdAt = new Date(Date.now() - createdDaysAgo * DAY_MS);
const updatedAt = new Date(Date.now() - updatedDaysAgo * DAY_MS);
await db.$executeRaw`UPDATE "WebhookCall" SET "createdAt" = ${createdAt}, "updatedAt" = ${updatedAt} WHERE id = ${call.id}`;
return call.id;
}

beforeEach(async () => {
await resetDatabase();
const team = await createTeam();
teamId = team.id;
const webhook = await db.webhook.create({
data: {
teamId,
url: "https://example.com/hook",
secret: "whsec_test",
eventTypes: [],
},
});
webhookId = webhook.id;
});

afterAll(async () => {
await closeIntegrationConnections();
});

it("deletes only finished calls that are over 30 days old and untouched since", async () => {
const deletedIds = [
await createCall("old delivered 1", WebhookCallStatus.DELIVERED, 60, 60),
await createCall("old delivered 2", WebhookCallStatus.DELIVERED, 45, 45),
await createCall("old failed", WebhookCallStatus.FAILED, 50, 49),
await createCall("old discarded", WebhookCallStatus.DISCARDED, 40, 40),
];
const keptIds = [
await createCall("old but pending", WebhookCallStatus.PENDING, 60, 60),
await createCall(
"old in progress",
WebhookCallStatus.IN_PROGRESS,
60,
60,
),
await createCall(
"old, retried today",
WebhookCallStatus.DELIVERED,
60,
0,
),
await createCall("recent", WebhookCallStatus.DELIVERED, 2, 2),
];

const result = await deleteOldWebhookCalls({
cutoff: new Date(Date.now() - 30 * DAY_MS),
batchSize: 3,
pauseMs: 0,
});

expect(result).toMatchObject({ deleted: 4, batches: 2, finished: true });
const remaining = await db.webhookCall.findMany({ select: { id: true } });
expect(remaining.map((row) => row.id).sort()).toEqual([...keptIds].sort());
expect(deletedIds.every((id) => !remaining.some((r) => r.id === id))).toBe(
true,
);
});

it("does nothing when nothing is old enough", async () => {
await createCall("recent", WebhookCallStatus.DELIVERED, 1, 1);

const result = await deleteOldWebhookCalls({
cutoff: new Date(Date.now() - 30 * DAY_MS),
pauseMs: 0,
});

expect(result).toMatchObject({ deleted: 0, batches: 0, finished: true });
expect(await db.webhookCall.count()).toBe(1);
});
});
136 changes: 124 additions & 12 deletions apps/web/src/server/jobs/webhook-cleanup-job.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,128 @@
import { Queue, Worker } from "bullmq";
import { WebhookCallStatus } from "@prisma/client";
import { subDays } from "date-fns";
import { db } from "~/server/db";
import { getRedis, BULL_PREFIX } from "~/server/redis";
import { DEFAULT_QUEUE_OPTIONS, WEBHOOK_CLEANUP_QUEUE } from "../queue/queue-constants";
import {
DEFAULT_QUEUE_OPTIONS,
WEBHOOK_CLEANUP_QUEUE,
} from "../queue/queue-constants";
import { logger } from "../logger/log";

const WEBHOOK_RETENTION_DAYS = 30;

// Delete in small batches, each its own short transaction, with a pause in
// between, rather than one DELETE of months of rows (one long transaction and
// a burst of WAL). This limits the work per transaction and per night; it does
// not shrink the database files: deleted rows become reusable space after
// autovacuum. A run stops after MAX_RUN_MS and the next night's run carries on.
const DELETE_BATCH_SIZE = 1000;
const PAUSE_BETWEEN_BATCHES_MS = 250;
const MAX_RUN_MS = 10 * 60 * 1000;
// Per batch: give up rather than wait on a lock or a slow statement.
const BATCH_LOCK_TIMEOUT = "5s";
const BATCH_STATEMENT_TIMEOUT = "60s";
// Prisma's own interactive-transaction limits (its default timeout is 5s,
// which would cut a slow batch off first): room for the two statements above,
// and a short wait for a connection.
const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 };
const LOG_PROGRESS_EVERY_BATCHES = 50;

// Only calls that are finished. A call that is pending or in progress (for
// example an old call someone pressed Retry on) is never deleted.
const FINISHED_STATUSES: WebhookCallStatus[] = [
WebhookCallStatus.DELIVERED,
WebhookCallStatus.FAILED,
WebhookCallStatus.DISCARDED,
];

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

/**
* Deletes one batch of finished webhook calls that were created and last
* updated before the cutoff. The same conditions are checked again in the
* DELETE, so a call retried after it was selected is kept.
*/
async function deleteBatch(cutoff: Date, batchSize: number) {
const oldAndFinished = {
createdAt: { lt: cutoff },
updatedAt: { lt: cutoff },
status: { in: FINISHED_STATUSES },
};

return db.$transaction(async (tx) => {
await tx.$executeRawUnsafe(
`SET LOCAL lock_timeout = '${BATCH_LOCK_TIMEOUT}'`,
);
await tx.$executeRawUnsafe(
`SET LOCAL statement_timeout = '${BATCH_STATEMENT_TIMEOUT}'`,
);

// Uses the createdAt index; ids keep the DELETE small and exact.
const rows = await tx.webhookCall.findMany({
where: oldAndFinished,
select: { id: true },
orderBy: { createdAt: "asc" },
take: batchSize,
});

if (rows.length === 0) return { selected: 0, deleted: 0 };

const result = await tx.webhookCall.deleteMany({
where: { id: { in: rows.map((row) => row.id) }, ...oldAndFinished },
});

return { selected: rows.length, deleted: result.count };
}, BATCH_TRANSACTION_OPTIONS);
}

export async function deleteOldWebhookCalls({
cutoff,
batchSize = DELETE_BATCH_SIZE,
pauseMs = PAUSE_BETWEEN_BATCHES_MS,
maxRunMs = MAX_RUN_MS,
now = () => Date.now(),
}: {
cutoff: Date;
batchSize?: number;
pauseMs?: number;
maxRunMs?: number;
now?: () => number;
}) {
const startedAt = now();
let deleted = 0;
let batches = 0;
const progress = (finished: boolean) => ({
deleted,
batches,
finished,
elapsedMs: now() - startedAt,
});

for (;;) {
// Check the deadline before starting each batch.
if (batches > 0 && now() - startedAt >= maxRunMs) {
return progress(false);
}

const batch = await deleteBatch(cutoff, batchSize);
deleted += batch.deleted;
if (batch.selected === 0) return progress(true);
batches += 1;

if (batch.selected < batchSize) return progress(true);

if (batches % LOG_PROGRESS_EVERY_BATCHES === 0) {
logger.info(
{ ...progress(false), cutoff: cutoff.toISOString() },
"[WebhookCleanupJob]: Still deleting old webhook calls",
);
}

await sleep(pauseMs);
}
}

const webhookCleanupQueue = new Queue(WEBHOOK_CLEANUP_QUEUE, {
connection: getRedis(),
prefix: BULL_PREFIX,
Expand All @@ -17,24 +133,20 @@ const worker = new Worker(
WEBHOOK_CLEANUP_QUEUE,
async () => {
const cutoff = subDays(new Date(), WEBHOOK_RETENTION_DAYS);
const result = await db.webhookCall.deleteMany({
where: {
createdAt: {
lt: cutoff,
},
},
});
const result = await deleteOldWebhookCalls({ cutoff });

logger.info(
{ deleted: result.count, cutoff: cutoff.toISOString() },
"[WebhookCleanupJob]: Deleted old webhook calls",
{ ...result, cutoff: cutoff.toISOString() },
result.finished
? "[WebhookCleanupJob]: Deleted old webhook calls"
: "[WebhookCleanupJob]: Deleted old webhook calls; more remain for the next run",
);
},
{
connection: getRedis(),
prefix: BULL_PREFIX,
skipVersionCheck: true,
}
},
);

await webhookCleanupQueue.upsertJobScheduler(
Expand All @@ -47,7 +159,7 @@ await webhookCleanupQueue.upsertJobScheduler(
opts: {
...DEFAULT_QUEUE_OPTIONS,
},
}
},
);

worker.on("completed", (job) => {
Expand Down
Loading
Loading