From a422334a309687c2f6883a30276db09c01cf3a47 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 22:17:08 +1100 Subject: [PATCH 1/3] feat(web): clear old email content in small batches, on cloud too, and explain it in the dashboard The email content cleanup (EMAIL_CLEANUP_DAYS) only ran on self-hosted installs, as one UPDATE over every old email. Cloud kept every email's html, text, attachments and headers forever: 3.7 GB of the 7.1 GB database. - Runs wherever EMAIL_CLEANUP_DAYS is set (cloud or self-hosted). Nothing changes until it's set. - Walks forward by createdAt with a Redis watermark, so each night only reads emails it hasn't looked at; clears up to 100,000 emails a night in batches of 500, each a short transaction with lock/statement timeouts. - Vacuums Email after a run (TRUNCATE FALSE, PARALLEL 0), so the freed space is reused instead of the table growing. - Never clears scheduled or queued emails, or ones scheduled after the cutoff; doesn't change updatedAt (the campaign page sorts by it). - Email details explains removed content instead of 'No content to preview'. Tested on Postgres 16 with 400k production-shaped emails: a night of 50k took 23 s, wrote 41-110 MB of WAL, and grew the table by ~10 MB, then 3 MB once vacuum let the space be reused. --- apps/docs/self-hosting/overview.mdx | 4 +- .../app/(dashboard)/emails/email-details.tsx | 26 +- apps/web/src/server/api/routers/email.ts | 6 +- .../cleanup-email-bodies.integration.test.ts | 183 +++++++++++ .../src/server/jobs/cleanup-email-bodies.ts | 285 +++++++++++++----- 5 files changed, 426 insertions(+), 78 deletions(-) create mode 100644 apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts diff --git a/apps/docs/self-hosting/overview.mdx b/apps/docs/self-hosting/overview.mdx index d453eb85a..70c814881 100644 --- a/apps/docs/self-hosting/overview.mdx +++ b/apps/docs/self-hosting/overview.mdx @@ -82,7 +82,9 @@ Add the following environment variables. - By default, stored email content stays in the database forever. To bound database growth, set `EMAIL_CLEANUP_DAYS` to a positive number of days. A daily job (midnight UTC) then clears the `text`, `html`, `attachments` and `headers` values for emails older than the cutoff. Delivery status and event history are kept. + By default, stored email content stays in the database forever. To bound database growth, set `EMAIL_CLEANUP_DAYS` to a positive number of days. A daily job (midnight UTC) then clears the `text`, `html`, `attachments` and `headers` values for emails older than the cutoff. Delivery status and event history are kept, and the dashboard explains that the content was removed. Scheduled and queued emails keep their content until they're sent. + + The job clears up to 100,000 emails a night in small batches, then vacuums the `Email` table so the freed space is reused. If you turn it on for an existing database, older emails are cleared over several nights. This doesn't shrink the database files on disk; it stops them from growing. ```env EMAIL_CLEANUP_DAYS=30 diff --git a/apps/web/src/app/(dashboard)/emails/email-details.tsx b/apps/web/src/app/(dashboard)/emails/email-details.tsx index 221184281..3066a416e 100644 --- a/apps/web/src/app/(dashboard)/emails/email-details.tsx +++ b/apps/web/src/app/(dashboard)/emails/email-details.tsx @@ -94,6 +94,8 @@ export default function EmailDetails({ emailId }: { emailId: string }) { @@ -138,12 +140,25 @@ export default function EmailDetails({ emailId }: { emailId: string }) { ); } +function isPastRetention( + createdAt: Date | string | null | undefined, + days: number | null | undefined, +) { + if (!createdAt || !days) return false; + const ageMs = Date.now() - new Date(createdAt).getTime(); + return ageMs >= days * 24 * 60 * 60 * 1000; +} + const EmailPreview = ({ html, text, + createdAt, + contentRetentionDays, }: { html: string | null | undefined; text: string | null | undefined; + createdAt?: Date | string | null; + contentRetentionDays?: number | null; }) => { const [show, setShow] = useState(false); @@ -166,8 +181,10 @@ const EmailPreview = ({ if (!srcDoc) { return (
- - No content to preview + + {isPastRetention(createdAt, contentRetentionDays) + ? `Email content is removed ${contentRetentionDays} days after sending. Its status and events are kept.` + : "No content to preview"}
); @@ -306,10 +323,7 @@ const getErrorMessage = (data: SesBounce) => { if (data.bounceType === "Permanent") { return BOUNCE_ERROR_MESSAGES[data.bounceType][ data.bounceSubType as - | "General" - | "NoEmail" - | "Suppressed" - | "OnAccountSuppressionList" + "General" | "NoEmail" | "Suppressed" | "OnAccountSuppressionList" ]; } else if (data.bounceType === "Transient") { return BOUNCE_ERROR_MESSAGES[data.bounceType][ diff --git a/apps/web/src/server/api/routers/email.ts b/apps/web/src/server/api/routers/email.ts index 787d2daa8..cdaa7a476 100644 --- a/apps/web/src/server/api/routers/email.ts +++ b/apps/web/src/server/api/routers/email.ts @@ -11,6 +11,7 @@ import { teamProcedure, } from "~/server/api/trpc"; import { db } from "~/server/db"; +import { env } from "~/env"; import { cancelEmail, updateEmail } from "~/server/service/email-service"; const statuses = Object.values(EmailStatus) as [EmailStatus]; @@ -246,7 +247,10 @@ export const emailRouter = createTRPCRouter({ }, }); - return email; + if (!email) return email; + + // Email content (html, text) is cleared after this many days, when set. + return { ...email, contentRetentionDays: env.EMAIL_CLEANUP_DAYS ?? null }; }), cancelEmail: emailProcedure.mutation(async ({ input }) => { diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts new file mode 100644 index 000000000..f5f1ec64e --- /dev/null +++ b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts @@ -0,0 +1,183 @@ +import { EmailStatus } from "@prisma/client"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { db } from "~/server/db"; +import { createTeam } from "~/test/factories/core"; +import { + closeIntegrationConnections, + integrationEnabled, + resetDatabase, +} from "~/test/integration/helpers"; + +// Only the database is real here; the job's queue and worker aren't needed. +vi.mock("bullmq", () => ({ + Queue: class { + upsertJobScheduler = vi.fn().mockResolvedValue(undefined); + }, + Worker: class { + on = vi.fn(); + }, +})); + +import { + CLEANUP_LIMITS, + clearOldEmailBodies, +} from "~/server/jobs/cleanup-email-bodies"; + +const describeIntegration = integrationEnabled ? describe : describe.skip; + +const DAY_MS = 24 * 60 * 60 * 1000; +const daysAgo = (days: number) => new Date(Date.now() - days * DAY_MS); + +describeIntegration("email content cleanup against postgres", () => { + let teamId: number; + const cutoff = daysAgo(90); + + async function createEmail( + name: string, + createdDaysAgo: number, + status: EmailStatus = EmailStatus.DELIVERED, + scheduledAt: Date | null = null, + ) { + const email = await db.email.create({ + data: { + teamId, + from: "news@example.com", + to: [`${name}@example.com`], + subject: name, + html: `

${name}

`, + text: name, + attachments: "[]", + headers: "{}", + latestStatus: status, + scheduledAt, + }, + }); + const createdAt = daysAgo(createdDaysAgo); + const updatedAt = new Date("2026-01-01T00:00:00Z"); + await db.$executeRaw`UPDATE "Email" SET "createdAt" = ${createdAt}, "updatedAt" = ${updatedAt} WHERE id = ${email.id}`; + return email.id; + } + + const contentOf = (id: string) => + db.email.findUniqueOrThrow({ + where: { id }, + select: { + html: true, + text: true, + attachments: true, + headers: true, + updatedAt: true, + subject: true, + latestStatus: true, + }, + }); + + beforeEach(async () => { + await resetDatabase(); + teamId = (await createTeam()).id; + }); + + afterAll(async () => { + await closeIntegrationConnections(); + }); + + it("clears content of emails older than the cutoff and keeps everything else", async () => { + const old = [ + await createEmail("old-1", 200), + await createEmail("old-2", 120), + await createEmail("old-bounced", 95, EmailStatus.BOUNCED), + ]; + const kept = [ + await createEmail("recent", 10), + await createEmail( + "old-but-scheduled", + 120, + EmailStatus.SCHEDULED, + daysAgo(-5), + ), + await createEmail("old-but-queued", 120, EmailStatus.QUEUED), + await createEmail( + "old-scheduled-after-cutoff", + 120, + EmailStatus.SENT, + daysAgo(30), + ), + ]; + const saved: string[] = []; + + const result = await clearOldEmailBodies({ + cutoff, + startAfter: null, + saveProgress: async (p) => { + saved.push(p.id); + }, + limits: { ...CLEANUP_LIMITS, batchSize: 2, pauseMs: 0 }, + }); + + expect(result).toMatchObject({ cleared: 3, finished: true }); + for (const id of old) { + const email = await contentOf(id); + expect(email).toMatchObject({ + html: null, + text: null, + attachments: null, + headers: null, + }); + // Status, subject and updatedAt are untouched. + expect(email.subject).not.toBeNull(); + expect(email.updatedAt.toISOString()).toBe("2026-01-01T00:00:00.000Z"); + } + for (const id of kept) { + expect((await contentOf(id)).html).not.toBeNull(); + } + expect(saved.length).toBeGreaterThan(0); + }); + + it("stops at the nightly cap and the next run carries on where it stopped", async () => { + for (let i = 0; i < 5; i++) await createEmail(`old-${i}`, 100 + i); + let position: { createdAt: Date; id: string } | null = null; + const limits = { + ...CLEANUP_LIMITS, + batchSize: 2, + maxRowsPerRun: 2, + pauseMs: 0, + }; + + const first = await clearOldEmailBodies({ + cutoff, + startAfter: null, + saveProgress: async (p) => { + position = p; + }, + limits, + }); + expect(first).toMatchObject({ cleared: 2, finished: false }); + expect(await db.email.count({ where: { html: null } })).toBe(2); + + const second = await clearOldEmailBodies({ + cutoff, + startAfter: position, + saveProgress: async (p) => { + position = p; + }, + limits: { ...limits, maxRowsPerRun: 100 }, + }); + expect(second).toMatchObject({ cleared: 3, finished: true }); + // The second run started after the first one's position: it didn't look + // at the two emails already cleared. + expect(second.looked).toBe(3); + expect(await db.email.count({ where: { html: null } })).toBe(5); + }); + + it("does nothing when no email is old enough", async () => { + await createEmail("recent", 5); + + const result = await clearOldEmailBodies({ + cutoff, + startAfter: null, + saveProgress: async () => undefined, + }); + + expect(result).toMatchObject({ looked: 0, cleared: 0, finished: true }); + }); +}); diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.ts b/apps/web/src/server/jobs/cleanup-email-bodies.ts index cb000bc4c..87b3dbc81 100644 --- a/apps/web/src/server/jobs/cleanup-email-bodies.ts +++ b/apps/web/src/server/jobs/cleanup-email-bodies.ts @@ -1,81 +1,226 @@ -import {Queue, Worker} from "bullmq"; -import {db} from "~/server/db"; -import {getRedis, BULL_PREFIX} from "~/server/redis"; -import {logger} from "../logger/log"; -import {DEFAULT_QUEUE_OPTIONS} from "../queue/queue-constants"; -import {env} from "~/env"; -import {isSelfHosted, isEmailCleanupEnabled} from "~/utils/common"; +import { Queue, Worker } from "bullmq"; +import { Prisma } from "@prisma/client"; +import { subDays } from "date-fns"; +import { db } from "~/server/db"; +import { getRedis, BULL_PREFIX, redisKey } from "~/server/redis"; +import { logger } from "../logger/log"; +import { DEFAULT_QUEUE_OPTIONS } from "../queue/queue-constants"; +import { env } from "~/env"; +import { isEmailCleanupEnabled } from "~/utils/common"; + +/** + * Clears the stored content (html, text, attachments, headers) of emails older + * than EMAIL_CLEANUP_DAYS. Delivery status and events are kept. Runs wherever + * EMAIL_CLEANUP_DAYS is set (self-hosted or cloud). + * + * Built for a large table on a nearly full disk: + * - Walks forward by createdAt and remembers how far it got (a Redis + * watermark), so each night only reads emails it hasn't looked at yet. + * - Clears at most MAX_ROWS_PER_RUN emails a night, in small batches, each a + * short transaction with lock and statement timeouts. + * - Vacuums "Email" after a run, so the space it freed is reused the next night + * instead of the table growing. This doesn't shrink the files on disk. + * - Never touches emails that are scheduled or queued, or whose scheduled time + * is after the cutoff: they still need their content to send. + * - Doesn't change updatedAt (the campaign page sorts recent emails by it). + */ const CLEANUP_QUEUE_NAME = "cleanup-email-bodies"; +const CLEANUP_CRON = "0 0 * * *"; // midnight UTC -const CLEANUP_CRON = "0 0 * * *"; // default: midnight UTC - -// Only initialize if self hosted and cleanup enabled -if (isSelfHosted() && isEmailCleanupEnabled()) { - const CLEANUP_DAYS = env.EMAIL_CLEANUP_DAYS!; - - /** - * Initialize Queue - */ - const cleanupQueue = new Queue(CLEANUP_QUEUE_NAME, { - connection: getRedis(), - prefix: BULL_PREFIX, - skipVersionCheck: true, - }); - - const worker = new Worker( - CLEANUP_QUEUE_NAME, - async () => { - logger.info(`[Cleanup] Starting cleanup for emails older than ${CLEANUP_DAYS} days...`); - - const cutoffDate = new Date(); - cutoffDate.setDate(cutoffDate.getDate() - CLEANUP_DAYS); - - const result = await db.email.updateMany({ - where: { - createdAt: {lt: cutoffDate}, - OR: [ - {text: {not: null}}, - {html: {not: null}}, - {attachments: {not: null}}, - {headers: {not: null}}, - ], - }, - data: { - text: null, - html: null, - attachments: null, - headers: null, - }, - }); - - logger.info(`[Cleanup] Emails cleaned: ${result.count}`); - }, - { - connection: getRedis(), - prefix: BULL_PREFIX, - skipVersionCheck: true, - } +export const CLEANUP_LIMITS = { + batchSize: 500, + maxRowsPerRun: 100_000, + maxRunMs: 15 * 60 * 1000, + pauseMs: 200, +}; +const BATCH_LOCK_TIMEOUT = "5s"; +const BATCH_STATEMENT_TIMEOUT = "60s"; +// Prisma's own interactive-transaction limit (default 5s) must sit above the +// SQL timeouts, or it would cut a slow batch off first. +const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 }; + +/** Emails created before this time have all been looked at. */ +export const WATERMARK_KEY = "email-body-cleanup:cleaned-before"; + +type Position = { createdAt: Date; id: string }; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Looks at the next batch of emails older than the cutoff, after `after`, and + * clears the content of the ones that still have some. + */ +async function clearBatch(cutoff: Date, after: Position | null, size: number) { + return db.$transaction(async (tx) => { + await tx.$executeRawUnsafe( + `SET LOCAL lock_timeout = '${BATCH_LOCK_TIMEOUT}'`, + ); + await tx.$executeRawUnsafe( + `SET LOCAL statement_timeout = '${BATCH_STATEMENT_TIMEOUT}'`, ); - await cleanupQueue.upsertJobScheduler( - "scheduled-email-cleanup", - { - pattern: CLEANUP_CRON, - tz: "UTC", + // Keyset over the createdAt index: no rescans of emails already looked at. + const rows = await tx.$queryRaw` + SELECT id, "createdAt" FROM "Email" + WHERE "createdAt" < ${cutoff} + ${ + after + ? Prisma.sql`AND "createdAt" >= ${after.createdAt} + AND ("createdAt" > ${after.createdAt} OR id > ${after.id})` + : Prisma.empty + } + ORDER BY "createdAt" ASC, id ASC + LIMIT ${size} + `; + + if (rows.length === 0) return { looked: 0, cleared: 0, last: null }; + + const cleared = await tx.$executeRaw` + UPDATE "Email" + SET html = NULL, text = NULL, attachments = NULL, headers = NULL + WHERE id IN (${Prisma.join(rows.map((row) => row.id))}) + AND "createdAt" < ${cutoff} + AND ("scheduledAt" IS NULL OR "scheduledAt" < ${cutoff}) + AND "latestStatus" NOT IN ('SCHEDULED', 'QUEUED') + AND (html IS NOT NULL OR text IS NOT NULL + OR attachments IS NOT NULL OR headers IS NOT NULL) + `; + + return { looked: rows.length, cleared, last: rows[rows.length - 1]! }; + }, BATCH_TRANSACTION_OPTIONS); +} + +export async function clearOldEmailBodies({ + cutoff, + startAfter, + saveProgress, + limits = CLEANUP_LIMITS, + now = () => Date.now(), +}: { + cutoff: Date; + /** Where the last run stopped, if anywhere. */ + startAfter: Position | null; + /** Called after every batch with how far this run has got. */ + saveProgress: (position: Position) => Promise; + limits?: typeof CLEANUP_LIMITS; + now?: () => number; +}) { + const startedAt = now(); + let position = startAfter; + let looked = 0; + let cleared = 0; + let batches = 0; + const result = (finished: boolean) => ({ + looked, + cleared, + batches, + finished, + elapsedMs: now() - startedAt, + }); + + for (;;) { + if ( + batches > 0 && + (cleared >= limits.maxRowsPerRun || now() - startedAt >= limits.maxRunMs) + ) { + return result(false); + } + + const batch = await clearBatch(cutoff, position, limits.batchSize); + if (!batch.last) return result(true); + + looked += batch.looked; + cleared += batch.cleared; + batches += 1; + position = batch.last; + await saveProgress(position); + + if (batch.looked < limits.batchSize) return result(true); + await sleep(limits.pauseMs); + } +} + +function parsePosition(raw: string | null): Position | null { + if (!raw) return null; + try { + const value = JSON.parse(raw) as { createdAt: string; id: string }; + const createdAt = new Date(value.createdAt); + if (Number.isNaN(createdAt.getTime()) || !value.id) return null; + return { createdAt, id: value.id }; + } catch { + return null; + } +} + +if (isEmailCleanupEnabled()) { + const cleanupDays = env.EMAIL_CLEANUP_DAYS!; + const watermarkKey = redisKey(WATERMARK_KEY); + + const cleanupQueue = new Queue(CLEANUP_QUEUE_NAME, { + connection: getRedis(), + prefix: BULL_PREFIX, + skipVersionCheck: true, + }); + + const worker = new Worker( + CLEANUP_QUEUE_NAME, + async () => { + const redis = getRedis(); + const cutoff = subDays(new Date(), cleanupDays); + const startAfter = parsePosition(await redis.get(watermarkKey)); + + const result = await clearOldEmailBodies({ + cutoff, + startAfter, + saveProgress: async (position) => { + await redis.set(watermarkKey, JSON.stringify(position)); }, + }); + + logger.info( { - opts: { - ...DEFAULT_QUEUE_OPTIONS, - }, + ...result, + cutoff: cutoff.toISOString(), + startedAfter: startAfter?.createdAt.toISOString() ?? null, + }, + result.finished + ? "[EmailCleanupJob]: Cleared old email content" + : "[EmailCleanupJob]: Cleared old email content; more remains for the next run", + ); + + if (result.cleared > 0) { + // Make the freed space reusable for tomorrow's batch. PARALLEL 0: + // parallel vacuum needs more shared memory than small containers + // have. TRUNCATE FALSE: avoids the brief exclusive lock at the end. + try { + await db.$executeRawUnsafe( + 'VACUUM (TRUNCATE FALSE, PARALLEL 0) "Email"', + ); + logger.info("[EmailCleanupJob]: Vacuumed Email"); + } catch (err) { + logger.warn({ err }, "[EmailCleanupJob]: Vacuum failed"); } - ); + } + }, + { + connection: getRedis(), + prefix: BULL_PREFIX, + skipVersionCheck: true, + }, + ); + + await cleanupQueue.upsertJobScheduler( + "scheduled-email-cleanup", + { pattern: CLEANUP_CRON, tz: "UTC" }, + { opts: { ...DEFAULT_QUEUE_OPTIONS } }, + ); - worker.on("completed", (job) => { - logger.info({jobId: job.id}, ` Email Body cleanup job completed`); - }); + worker.on("completed", (job) => { + logger.info({ jobId: job.id }, "[EmailCleanupJob]: Job completed"); + }); - worker.on("failed", (job, err) => { - logger.error({err, jobId: job?.id}, `Email Body cleanup job failed`); - }); + worker.on("failed", (job, err) => { + logger.error({ err, jobId: job?.id }, "[EmailCleanupJob]: Job failed"); + }); } From a1c52dc78959ff6d1c7c4b37e65ae1bc53b3c236 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 22:27:13 +1100 Subject: [PATCH 2/3] fix(web): anchor email retention on send time, recheck skipped emails, exact nightly cap, tune autovacuum instead of vacuuming from the app - Clear content only when an email was created AND last updated before the cutoff: sending updates the row, so an email that waited in a queue keeps its content for N days after it went out. - Emails looked at but not clearable yet go on a pending list (Redis set) that later nights check again by id, so the watermark passing them doesn't keep their content forever. - Each batch looks at no more rows than the nightly cap has left. - No VACUUM from the app (it had no timeout and ran on a pooled connection). Instead, a migration sets autovacuum_vacuum_scale_factor and autovacuum_analyze_scale_factor to 0.02 on Email, so autovacuum runs after each night's changes. On 400k production-shaped rows, two nights of 100k left the table 278 MB smaller with no dead tuples left. - Cutoff is exact (N x 24 h), not calendar days. --- apps/docs/self-hosting/overview.mdx | 4 +- .../migration.sql | 6 + .../cleanup-email-bodies.integration.test.ts | 189 +++++++++++++----- .../src/server/jobs/cleanup-email-bodies.ts | 187 ++++++++++++----- 4 files changed, 279 insertions(+), 107 deletions(-) create mode 100644 apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql diff --git a/apps/docs/self-hosting/overview.mdx b/apps/docs/self-hosting/overview.mdx index 70c814881..e31808b10 100644 --- a/apps/docs/self-hosting/overview.mdx +++ b/apps/docs/self-hosting/overview.mdx @@ -82,9 +82,9 @@ Add the following environment variables.
- By default, stored email content stays in the database forever. To bound database growth, set `EMAIL_CLEANUP_DAYS` to a positive number of days. A daily job (midnight UTC) then clears the `text`, `html`, `attachments` and `headers` values for emails older than the cutoff. Delivery status and event history are kept, and the dashboard explains that the content was removed. Scheduled and queued emails keep their content until they're sent. + By default, stored email content stays in the database forever. To bound database growth, set `EMAIL_CLEANUP_DAYS` to a positive number of days. A daily job (midnight UTC) then clears the `text`, `html`, `attachments` and `headers` values of emails sent more than that many days ago. Delivery status and event history are kept, and the dashboard explains that the content was removed. Scheduled and queued emails keep their content until they're sent. - The job clears up to 100,000 emails a night in small batches, then vacuums the `Email` table so the freed space is reused. If you turn it on for an existing database, older emails are cleared over several nights. This doesn't shrink the database files on disk; it stops them from growing. + The job clears up to 100,000 emails a night in small batches. If you turn it on for an existing database, older emails are cleared over several nights. Autovacuum then makes the freed space reusable, which slows how fast the database grows; it usually doesn't shrink the files on disk. ```env EMAIL_CLEANUP_DAYS=30 diff --git a/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql b/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql new file mode 100644 index 000000000..12ee7eae2 --- /dev/null +++ b/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql @@ -0,0 +1,6 @@ +-- Vacuum "Email" after ~2% of its rows change instead of the default 20%. +-- The email content cleanup rewrites up to 100k rows a night; this lets +-- autovacuum make that space reusable before the next night, instead of the +-- table growing. Only changes a table setting; no data is touched, and reads +-- and writes aren't blocked. +ALTER TABLE "Email" SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_analyze_scale_factor = 0.02); diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts index f5f1ec64e..3a9253c89 100644 --- a/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts +++ b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts @@ -21,22 +21,45 @@ vi.mock("bullmq", () => ({ import { CLEANUP_LIMITS, clearOldEmailBodies, + retentionCutoff, + type PendingStore, + type Position, } from "~/server/jobs/cleanup-email-bodies"; const describeIntegration = integrationEnabled ? describe : describe.skip; const DAY_MS = 24 * 60 * 60 * 1000; const daysAgo = (days: number) => new Date(Date.now() - days * DAY_MS); +const LONG_AGO = new Date("2025-06-01T00:00:00Z"); + +function memoryPending(): PendingStore & { ids: Set } { + const ids = new Set(); + return { + ids, + list: async (limit) => [...ids].slice(0, limit), + add: async (more) => more.forEach((id) => ids.add(id)), + remove: async (gone) => gone.forEach((id) => ids.delete(id)), + }; +} describeIntegration("email content cleanup against postgres", () => { let teamId: number; - const cutoff = daysAgo(90); + const cutoff = retentionCutoff(90); + const fast = { ...CLEANUP_LIMITS, batchSize: 2, pauseMs: 0 }; async function createEmail( name: string, - createdDaysAgo: number, - status: EmailStatus = EmailStatus.DELIVERED, - scheduledAt: Date | null = null, + { + created, + updated = LONG_AGO, + status = EmailStatus.DELIVERED, + scheduledAt = null, + }: { + created: Date; + updated?: Date; + status?: EmailStatus; + scheduledAt?: Date | null; + }, ) { const email = await db.email.create({ data: { @@ -52,9 +75,7 @@ describeIntegration("email content cleanup against postgres", () => { scheduledAt, }, }); - const createdAt = daysAgo(createdDaysAgo); - const updatedAt = new Date("2026-01-01T00:00:00Z"); - await db.$executeRaw`UPDATE "Email" SET "createdAt" = ${createdAt}, "updatedAt" = ${updatedAt} WHERE id = ${email.id}`; + await db.$executeRaw`UPDATE "Email" SET "createdAt" = ${created}, "updatedAt" = ${updated} WHERE id = ${email.id}`; return email.id; } @@ -68,9 +89,9 @@ describeIntegration("email content cleanup against postgres", () => { headers: true, updatedAt: true, subject: true, - latestStatus: true, }, }); + const clearedCount = () => db.email.count({ where: { html: null } }); beforeEach(async () => { await resetDatabase(); @@ -81,37 +102,42 @@ describeIntegration("email content cleanup against postgres", () => { await closeIntegrationConnections(); }); - it("clears content of emails older than the cutoff and keeps everything else", async () => { + it("clears content of emails sent over 90 days ago and keeps everything else", async () => { const old = [ - await createEmail("old-1", 200), - await createEmail("old-2", 120), - await createEmail("old-bounced", 95, EmailStatus.BOUNCED), + await createEmail("old-1", { created: daysAgo(200) }), + await createEmail("old-2", { created: daysAgo(120) }), + await createEmail("old-bounced", { + created: daysAgo(95), + status: EmailStatus.BOUNCED, + }), ]; const kept = [ - await createEmail("recent", 10), - await createEmail( - "old-but-scheduled", - 120, - EmailStatus.SCHEDULED, - daysAgo(-5), - ), - await createEmail("old-but-queued", 120, EmailStatus.QUEUED), - await createEmail( - "old-scheduled-after-cutoff", - 120, - EmailStatus.SENT, - daysAgo(30), - ), + await createEmail("recent", { + created: daysAgo(10), + updated: daysAgo(10), + }), + await createEmail("old-but-scheduled", { + created: daysAgo(120), + status: EmailStatus.SCHEDULED, + scheduledAt: daysAgo(-5), + }), + await createEmail("old-but-queued", { + created: daysAgo(120), + status: EmailStatus.QUEUED, + }), + // Created long ago but only sent 10 days ago: kept until 90 days after. + await createEmail("old-but-sent-late", { + created: daysAgo(120), + updated: daysAgo(10), + }), ]; - const saved: string[] = []; const result = await clearOldEmailBodies({ cutoff, startAfter: null, - saveProgress: async (p) => { - saved.push(p.id); - }, - limits: { ...CLEANUP_LIMITS, batchSize: 2, pauseMs: 0 }, + saveProgress: async () => undefined, + pending: memoryPending(), + limits: fast, }); expect(result).toMatchObject({ cleared: 3, finished: true }); @@ -123,25 +149,83 @@ describeIntegration("email content cleanup against postgres", () => { attachments: null, headers: null, }); - // Status, subject and updatedAt are untouched. expect(email.subject).not.toBeNull(); - expect(email.updatedAt.toISOString()).toBe("2026-01-01T00:00:00.000Z"); + // updatedAt is untouched (the campaign page sorts by it). + expect(email.updatedAt.toISOString()).toBe(LONG_AGO.toISOString()); } for (const id of kept) { expect((await contentOf(id)).html).not.toBeNull(); } - expect(saved.length).toBeGreaterThan(0); }); - it("stops at the nightly cap and the next run carries on where it stopped", async () => { - for (let i = 0; i < 5; i++) await createEmail(`old-${i}`, 100 + i); - let position: { createdAt: Date; id: string } | null = null; - const limits = { - ...CLEANUP_LIMITS, - batchSize: 2, - maxRowsPerRun: 2, - pauseMs: 0, - }; + it("checks skipped emails again on later nights and clears them once eligible", async () => { + const late = await createEmail("sent-late", { + created: daysAgo(120), + updated: daysAgo(10), + }); + await createEmail("old", { created: daysAgo(130) }); + const pending = memoryPending(); + let position: Position | null = null; + const run = () => + clearOldEmailBodies({ + cutoff, + startAfter: position, + saveProgress: async (p) => { + position = p; + }, + pending, + limits: fast, + }); + + await run(); + expect((await contentOf(late)).html).not.toBeNull(); + expect([...pending.ids]).toEqual([late]); + + // 80 days later it was sent over 90 days ago. (Simulated by moving its + // update time back rather than the clock forward.) + await db.$executeRaw`UPDATE "Email" SET "updatedAt" = ${daysAgo(91)} WHERE id = ${late}`; + const second = await run(); + + expect(second).toMatchObject({ pendingChecked: 1, pendingCleared: 1 }); + expect((await contentOf(late)).html).toBeNull(); + expect(pending.ids.size).toBe(0); + }); + + it("doesn't skip emails that share a creation time across batches", async () => { + const sameTime = daysAgo(100); + for (let i = 0; i < 5; i++) + await createEmail(`tie-${i}`, { created: sameTime }); + + let position: Position | null = null; + const pending = memoryPending(); + // One email per batch, stopping after each: every resume starts mid-tie. + for (let night = 0; night < 5; night++) { + await clearOldEmailBodies({ + cutoff, + startAfter: position, + saveProgress: async (p) => { + position = p; + }, + pending, + limits: { ...fast, batchSize: 1, maxRowsPerRun: 1 }, + }); + } + + expect(await clearedCount()).toBe(5); + }); + + it("stops exactly at the nightly cap, then carries on where it stopped", async () => { + for (let i = 0; i < 4; i++) { + await createEmail(`old-${i}`, { created: daysAgo(100 + i * 2) }); + // Interleaved with emails that can't be cleared yet. + await createEmail(`late-${i}`, { + created: daysAgo(101 + i * 2), + updated: daysAgo(5), + }); + } + let position: Position | null = null; + const pending = memoryPending(); + const limits = { ...fast, batchSize: 2, maxRowsPerRun: 3 }; const first = await clearOldEmailBodies({ cutoff, @@ -149,10 +233,12 @@ describeIntegration("email content cleanup against postgres", () => { saveProgress: async (p) => { position = p; }, + pending, limits, }); - expect(first).toMatchObject({ cleared: 2, finished: false }); - expect(await db.email.count({ where: { html: null } })).toBe(2); + expect(first.cleared).toBe(3); + expect(first.finished).toBe(false); + expect(await clearedCount()).toBe(3); const second = await clearOldEmailBodies({ cutoff, @@ -160,22 +246,23 @@ describeIntegration("email content cleanup against postgres", () => { saveProgress: async (p) => { position = p; }, + pending, limits: { ...limits, maxRowsPerRun: 100 }, }); - expect(second).toMatchObject({ cleared: 3, finished: true }); - // The second run started after the first one's position: it didn't look - // at the two emails already cleared. - expect(second.looked).toBe(3); - expect(await db.email.count({ where: { html: null } })).toBe(5); + expect(second.cleared).toBe(1); + expect(second.finished).toBe(true); + expect(await clearedCount()).toBe(4); + expect(pending.ids.size).toBe(4); }); it("does nothing when no email is old enough", async () => { - await createEmail("recent", 5); + await createEmail("recent", { created: daysAgo(5), updated: daysAgo(5) }); const result = await clearOldEmailBodies({ cutoff, startAfter: null, saveProgress: async () => undefined, + pending: memoryPending(), }); expect(result).toMatchObject({ looked: 0, cleared: 0, finished: true }); diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.ts b/apps/web/src/server/jobs/cleanup-email-bodies.ts index 87b3dbc81..9894b5a0a 100644 --- a/apps/web/src/server/jobs/cleanup-email-bodies.ts +++ b/apps/web/src/server/jobs/cleanup-email-bodies.ts @@ -1,6 +1,5 @@ import { Queue, Worker } from "bullmq"; import { Prisma } from "@prisma/client"; -import { subDays } from "date-fns"; import { db } from "~/server/db"; import { getRedis, BULL_PREFIX, redisKey } from "~/server/redis"; import { logger } from "../logger/log"; @@ -9,30 +8,39 @@ import { env } from "~/env"; import { isEmailCleanupEnabled } from "~/utils/common"; /** - * Clears the stored content (html, text, attachments, headers) of emails older - * than EMAIL_CLEANUP_DAYS. Delivery status and events are kept. Runs wherever - * EMAIL_CLEANUP_DAYS is set (self-hosted or cloud). + * Clears the stored content (html, text, attachments, headers) of emails sent + * more than EMAIL_CLEANUP_DAYS ago. Delivery status and events are kept. Runs + * wherever EMAIL_CLEANUP_DAYS is set (self-hosted or cloud). + * + * "Sent more than N days ago" means created AND last updated before the + * cutoff. Sending updates the email (Prisma sets updatedAt), so an email that + * waited in a queue is kept for N days after it went out. SES status updates + * use raw SQL and don't touch updatedAt, so they don't delay clearing. + * Scheduled and queued emails, and ones scheduled after the cutoff, are never + * cleared: they still need their content to send. * * Built for a large table on a nearly full disk: - * - Walks forward by createdAt and remembers how far it got (a Redis - * watermark), so each night only reads emails it hasn't looked at yet. - * - Clears at most MAX_ROWS_PER_RUN emails a night, in small batches, each a - * short transaction with lock and statement timeouts. - * - Vacuums "Email" after a run, so the space it freed is reused the next night - * instead of the table growing. This doesn't shrink the files on disk. - * - Never touches emails that are scheduled or queued, or whose scheduled time - * is after the cutoff: they still need their content to send. + * - Walks forward by createdAt and remembers how far it got, so each night + * reads only emails it hasn't looked at yet. + * - Emails it looked at but couldn't clear yet go on a "pending" list that is + * checked again (by id) on later nights. + * - Clears at most MAX_ROWS_PER_RUN a night, in small batches, each a short + * transaction with lock and statement timeouts. + * - Leaves vacuuming to autovacuum, which is tuned for "Email" in a migration + * so it runs after each night's changes and the freed space is reused. * - Doesn't change updatedAt (the campaign page sorts recent emails by it). */ const CLEANUP_QUEUE_NAME = "cleanup-email-bodies"; const CLEANUP_CRON = "0 0 * * *"; // midnight UTC +const DAY_MS = 24 * 60 * 60 * 1000; export const CLEANUP_LIMITS = { batchSize: 500, maxRowsPerRun: 100_000, maxRunMs: 15 * 60 * 1000, pauseMs: 200, + maxPendingPerRun: 5_000, }; const BATCH_LOCK_TIMEOUT = "5s"; const BATCH_STATEMENT_TIMEOUT = "60s"; @@ -40,18 +48,28 @@ const BATCH_STATEMENT_TIMEOUT = "60s"; // SQL timeouts, or it would cut a slow batch off first. const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 }; -/** Emails created before this time have all been looked at. */ +/** Emails created before this position have all been looked at. */ export const WATERMARK_KEY = "email-body-cleanup:cleaned-before"; +/** Ids of emails looked at but not clearable yet. */ +export const PENDING_KEY = "email-body-cleanup:pending"; + +export type Position = { createdAt: Date; id: string }; -type Position = { createdAt: Date; id: string }; +export type PendingStore = { + list: (limit: number) => Promise; + add: (ids: string[]) => Promise; + remove: (ids: string[]) => Promise; +}; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -/** - * Looks at the next batch of emails older than the cutoff, after `after`, and - * clears the content of the ones that still have some. - */ -async function clearBatch(cutoff: Date, after: Position | null, size: number) { +export function retentionCutoff(days: number, now = Date.now()) { + return new Date(now - days * DAY_MS); +} + +type Tx = Prisma.TransactionClient; + +async function withBatchTimeouts(fn: (tx: Tx) => Promise) { return db.$transaction(async (tx) => { await tx.$executeRawUnsafe( `SET LOCAL lock_timeout = '${BATCH_LOCK_TIMEOUT}'`, @@ -59,7 +77,42 @@ async function clearBatch(cutoff: Date, after: Position | null, size: number) { await tx.$executeRawUnsafe( `SET LOCAL statement_timeout = '${BATCH_STATEMENT_TIMEOUT}'`, ); + return fn(tx); + }, BATCH_TRANSACTION_OPTIONS); +} +/** + * Clears the content of the given emails if they're eligible, and returns how + * many were cleared and which still have content (not eligible yet). + */ +async function clearEligible(tx: Tx, ids: string[], cutoff: Date) { + const cleared = await tx.$executeRaw` + UPDATE "Email" + SET html = NULL, text = NULL, attachments = NULL, headers = NULL + WHERE id IN (${Prisma.join(ids)}) + AND "createdAt" < ${cutoff} + AND "updatedAt" < ${cutoff} + AND ("scheduledAt" IS NULL OR "scheduledAt" < ${cutoff}) + AND "latestStatus" NOT IN ('SCHEDULED', 'QUEUED') + AND (html IS NOT NULL OR text IS NOT NULL + OR attachments IS NOT NULL OR headers IS NOT NULL) + `; + const stillWithContent = await tx.$queryRaw<{ id: string }[]>` + SELECT id FROM "Email" + WHERE id IN (${Prisma.join(ids)}) + AND (html IS NOT NULL OR text IS NOT NULL + OR attachments IS NOT NULL OR headers IS NOT NULL) + `; + return { cleared, notYet: stillWithContent.map((row) => row.id) }; +} + +/** Looks at the next emails older than the cutoff, after `after`. */ +async function clearNextBatch( + cutoff: Date, + after: Position | null, + size: number, +) { + return withBatchTimeouts(async (tx) => { // Keyset over the createdAt index: no rescans of emails already looked at. const rows = await tx.$queryRaw` SELECT id, "createdAt" FROM "Email" @@ -73,28 +126,49 @@ async function clearBatch(cutoff: Date, after: Position | null, size: number) { ORDER BY "createdAt" ASC, id ASC LIMIT ${size} `; + if (rows.length === 0) { + return { looked: 0, cleared: 0, notYet: [] as string[], last: null }; + } + const { cleared, notYet } = await clearEligible( + tx, + rows.map((row) => row.id), + cutoff, + ); + return { + looked: rows.length, + cleared, + notYet, + last: rows[rows.length - 1]!, + }; + }); +} - if (rows.length === 0) return { looked: 0, cleared: 0, last: null }; - - const cleared = await tx.$executeRaw` - UPDATE "Email" - SET html = NULL, text = NULL, attachments = NULL, headers = NULL - WHERE id IN (${Prisma.join(rows.map((row) => row.id))}) - AND "createdAt" < ${cutoff} - AND ("scheduledAt" IS NULL OR "scheduledAt" < ${cutoff}) - AND "latestStatus" NOT IN ('SCHEDULED', 'QUEUED') - AND (html IS NOT NULL OR text IS NOT NULL - OR attachments IS NOT NULL OR headers IS NOT NULL) - `; - - return { looked: rows.length, cleared, last: rows[rows.length - 1]! }; - }, BATCH_TRANSACTION_OPTIONS); +/** Checks emails skipped on earlier nights again, by id. */ +async function clearPending( + cutoff: Date, + pending: PendingStore, + limits: typeof CLEANUP_LIMITS, +) { + const ids = await pending.list(limits.maxPendingPerRun); + let cleared = 0; + for (let i = 0; i < ids.length; i += limits.batchSize) { + const chunk = ids.slice(i, i + limits.batchSize); + const result = await withBatchTimeouts((tx) => + clearEligible(tx, chunk, cutoff), + ); + cleared += result.cleared; + // Keep only the ones that still have content; drop cleared or deleted ones. + const keep = new Set(result.notYet); + await pending.remove(chunk.filter((id) => !keep.has(id))); + } + return { checked: ids.length, cleared }; } export async function clearOldEmailBodies({ cutoff, startAfter, saveProgress, + pending, limits = CLEANUP_LIMITS, now = () => Date.now(), }: { @@ -103,40 +177,49 @@ export async function clearOldEmailBodies({ startAfter: Position | null; /** Called after every batch with how far this run has got. */ saveProgress: (position: Position) => Promise; + pending: PendingStore; limits?: typeof CLEANUP_LIMITS; now?: () => number; }) { const startedAt = now(); + const pendingResult = await clearPending(cutoff, pending, limits); + let position = startAfter; let looked = 0; - let cleared = 0; + let cleared = pendingResult.cleared; let batches = 0; const result = (finished: boolean) => ({ looked, cleared, batches, + pendingChecked: pendingResult.checked, + pendingCleared: pendingResult.cleared, finished, elapsedMs: now() - startedAt, }); for (;;) { + const remaining = limits.maxRowsPerRun - cleared; if ( - batches > 0 && - (cleared >= limits.maxRowsPerRun || now() - startedAt >= limits.maxRunMs) + remaining <= 0 || + (batches > 0 && now() - startedAt >= limits.maxRunMs) ) { return result(false); } - const batch = await clearBatch(cutoff, position, limits.batchSize); + // Never look at more rows than the nightly cap has left, so it holds exactly. + const size = Math.min(limits.batchSize, remaining); + const batch = await clearNextBatch(cutoff, position, size); if (!batch.last) return result(true); looked += batch.looked; cleared += batch.cleared; batches += 1; + if (batch.notYet.length > 0) await pending.add(batch.notYet); position = batch.last; await saveProgress(position); - if (batch.looked < limits.batchSize) return result(true); + if (batch.looked < size) return result(true); await sleep(limits.pauseMs); } } @@ -156,6 +239,7 @@ function parsePosition(raw: string | null): Position | null { if (isEmailCleanupEnabled()) { const cleanupDays = env.EMAIL_CLEANUP_DAYS!; const watermarkKey = redisKey(WATERMARK_KEY); + const pendingKey = redisKey(PENDING_KEY); const cleanupQueue = new Queue(CLEANUP_QUEUE_NAME, { connection: getRedis(), @@ -167,7 +251,7 @@ if (isEmailCleanupEnabled()) { CLEANUP_QUEUE_NAME, async () => { const redis = getRedis(); - const cutoff = subDays(new Date(), cleanupDays); + const cutoff = retentionCutoff(cleanupDays); const startAfter = parsePosition(await redis.get(watermarkKey)); const result = await clearOldEmailBodies({ @@ -176,6 +260,15 @@ if (isEmailCleanupEnabled()) { saveProgress: async (position) => { await redis.set(watermarkKey, JSON.stringify(position)); }, + pending: { + list: (limit) => redis.srandmember(pendingKey, limit), + add: async (ids) => { + if (ids.length) await redis.sadd(pendingKey, ...ids); + }, + remove: async (ids) => { + if (ids.length) await redis.srem(pendingKey, ...ids); + }, + }, }); logger.info( @@ -188,20 +281,6 @@ if (isEmailCleanupEnabled()) { ? "[EmailCleanupJob]: Cleared old email content" : "[EmailCleanupJob]: Cleared old email content; more remains for the next run", ); - - if (result.cleared > 0) { - // Make the freed space reusable for tomorrow's batch. PARALLEL 0: - // parallel vacuum needs more shared memory than small containers - // have. TRUNCATE FALSE: avoids the brief exclusive lock at the end. - try { - await db.$executeRawUnsafe( - 'VACUUM (TRUNCATE FALSE, PARALLEL 0) "Email"', - ); - logger.info("[EmailCleanupJob]: Vacuumed Email"); - } catch (err) { - logger.warn({ err }, "[EmailCleanupJob]: Vacuum failed"); - } - } }, { connection: getRedis(), From 7bea9793a2699c5718aa5be0e365783238dd8bd2 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 22:31:29 +1100 Subject: [PATCH 3/3] fix(web): fair, bounded rechecks for skipped emails, and a lock timeout on the autovacuum migration - Pending emails are a Redis sorted set scored by when each becomes eligible (90 days after it was sent; daily for scheduled/queued), so each is checked when it's due, not at random. - The recheck pass shares the run's row cap and time limit, with pauses. - The run result reports how many emails are still pending. - The autovacuum migration runs in a transaction with lock_timeout 10s, so a deploy can't wait indefinitely behind a VACUUM/ANALYZE/CIC on Email. - Production Redis is maxmemory-policy noeviction with no maxmemory, so the pending set can't be evicted on its own. --- .../migration.sql | 17 +- .../cleanup-email-bodies.integration.test.ts | 62 +++++-- .../src/server/jobs/cleanup-email-bodies.ts | 152 +++++++++++------- 3 files changed, 156 insertions(+), 75 deletions(-) diff --git a/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql b/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql index 12ee7eae2..a5b1bc1f2 100644 --- a/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql +++ b/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql @@ -1,6 +1,13 @@ --- Vacuum "Email" after ~2% of its rows change instead of the default 20%. --- The email content cleanup rewrites up to 100k rows a night; this lets --- autovacuum make that space reusable before the next night, instead of the --- table growing. Only changes a table setting; no data is touched, and reads --- and writes aren't blocked. +-- Encourage autovacuum to visit "Email" sooner: after ~2% of its rows change +-- instead of the default 20%. The email content cleanup rewrites up to 100k +-- rows a night; this makes that space reusable sooner instead of the table +-- growing. It only changes a table setting and touches no data. +-- +-- The ALTER needs a SHARE UPDATE EXCLUSIVE lock: normal reads and writes carry +-- on, but it waits for a running VACUUM, ANALYZE or CREATE INDEX CONCURRENTLY +-- on "Email". lock_timeout makes it give up after 10s instead of waiting +-- indefinitely during a deploy; the migration then fails and can be retried. +BEGIN; +SET LOCAL lock_timeout = '10s'; ALTER TABLE "Email" SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_analyze_scale_factor = 0.02); +COMMIT; diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts index 3a9253c89..8321c18f4 100644 --- a/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts +++ b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts @@ -32,13 +32,20 @@ const DAY_MS = 24 * 60 * 60 * 1000; const daysAgo = (days: number) => new Date(Date.now() - days * DAY_MS); const LONG_AGO = new Date("2025-06-01T00:00:00Z"); -function memoryPending(): PendingStore & { ids: Set } { - const ids = new Set(); +function memoryPending(): PendingStore & { ids: Map } { + const ids = new Map(); return { ids, - list: async (limit) => [...ids].slice(0, limit), - add: async (more) => more.forEach((id) => ids.add(id)), + due: async (now, limit) => + [...ids.entries()] + .filter(([, at]) => at <= now) + .sort((a, b) => a[1] - b[1]) + .slice(0, limit) + .map(([id]) => id), + schedule: async (entries) => + entries.forEach((entry) => ids.set(entry.id, entry.at)), remove: async (gone) => gone.forEach((id) => ids.delete(id)), + size: async () => ids.size, }; } @@ -158,7 +165,7 @@ describeIntegration("email content cleanup against postgres", () => { } }); - it("checks skipped emails again on later nights and clears them once eligible", async () => { + it("checks skipped emails again once they're due, and clears them then", async () => { const late = await createEmail("sent-late", { created: daysAgo(120), updated: daysAgo(10), @@ -166,31 +173,57 @@ describeIntegration("email content cleanup against postgres", () => { await createEmail("old", { created: daysAgo(130) }); const pending = memoryPending(); let position: Position | null = null; - const run = () => + const runAt = (clock: number) => clearOldEmailBodies({ - cutoff, + cutoff: retentionCutoff(90, clock), startAfter: position, saveProgress: async (p) => { position = p; }, pending, limits: fast, + now: () => clock, }); - await run(); + await runAt(Date.now()); expect((await contentOf(late)).html).not.toBeNull(); - expect([...pending.ids]).toEqual([late]); + // Due 90 days after it was sent, not before. + const dueAt = pending.ids.get(late)!; + expect(dueAt).toBeGreaterThan(Date.now() + 79 * DAY_MS); + expect(dueAt).toBeLessThan(Date.now() + 81 * DAY_MS); - // 80 days later it was sent over 90 days ago. (Simulated by moving its - // update time back rather than the clock forward.) - await db.$executeRaw`UPDATE "Email" SET "updatedAt" = ${daysAgo(91)} WHERE id = ${late}`; - const second = await run(); + // 70 days later: not due yet, not checked. + const early = await runAt(Date.now() + 70 * DAY_MS); + expect(early.pendingChecked).toBe(0); + expect((await contentOf(late)).html).not.toBeNull(); - expect(second).toMatchObject({ pendingChecked: 1, pendingCleared: 1 }); + // 81 days later it was sent over 90 days ago. + const later = await runAt(Date.now() + 81 * DAY_MS); + expect(later).toMatchObject({ pendingChecked: 1, pendingCleared: 1 }); expect((await contentOf(late)).html).toBeNull(); expect(pending.ids.size).toBe(0); }); + it("keeps the recheck pass within the nightly cap", async () => { + const pending = memoryPending(); + const ids: string[] = []; + for (let i = 0; i < 5; i++) + ids.push(await createEmail(`p-${i}`, { created: daysAgo(100) })); + await pending.schedule(ids.map((id) => ({ id, at: 0 }))); + + const result = await clearOldEmailBodies({ + cutoff, + startAfter: { createdAt: new Date(), id: "zzz" }, // nothing new to scan + saveProgress: async () => undefined, + pending, + limits: { ...fast, maxRowsPerRun: 2 }, + }); + + expect(result).toMatchObject({ cleared: 2, finished: false }); + expect(await clearedCount()).toBe(2); + expect(pending.ids.size).toBe(3); + }); + it("doesn't skip emails that share a creation time across batches", async () => { const sameTime = daysAgo(100); for (let i = 0; i < 5; i++) @@ -253,6 +286,7 @@ describeIntegration("email content cleanup against postgres", () => { expect(second.finished).toBe(true); expect(await clearedCount()).toBe(4); expect(pending.ids.size).toBe(4); + expect(second.pendingLeft).toBe(4); }); it("does nothing when no email is old enough", async () => { diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.ts b/apps/web/src/server/jobs/cleanup-email-bodies.ts index 9894b5a0a..9b58d67bd 100644 --- a/apps/web/src/server/jobs/cleanup-email-bodies.ts +++ b/apps/web/src/server/jobs/cleanup-email-bodies.ts @@ -22,12 +22,13 @@ import { isEmailCleanupEnabled } from "~/utils/common"; * Built for a large table on a nearly full disk: * - Walks forward by createdAt and remembers how far it got, so each night * reads only emails it hasn't looked at yet. - * - Emails it looked at but couldn't clear yet go on a "pending" list that is - * checked again (by id) on later nights. + * - Emails it looked at but couldn't clear yet go on a "pending" list, keyed + * by when each one becomes eligible, and are checked again (by id) once due. * - Clears at most MAX_ROWS_PER_RUN a night, in small batches, each a short * transaction with lock and statement timeouts. - * - Leaves vacuuming to autovacuum, which is tuned for "Email" in a migration - * so it runs after each night's changes and the freed space is reused. + * - Leaves vacuuming to autovacuum, which a migration tunes to visit "Email" + * sooner (after ~2% of rows change), so freed space is usually reusable by + * the next night. * - Doesn't change updatedAt (the campaign page sorts recent emails by it). */ @@ -50,15 +51,24 @@ const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 }; /** Emails created before this position have all been looked at. */ export const WATERMARK_KEY = "email-body-cleanup:cleaned-before"; -/** Ids of emails looked at but not clearable yet. */ +/** + * Emails looked at but not clearable yet: a sorted set of ids, scored by the + * time each should be checked again. Lives in the same Redis as BullMQ, which + * requires maxmemory-policy noeviction, so it isn't evicted on its own. + */ export const PENDING_KEY = "email-body-cleanup:pending"; +// Scheduled or queued emails have no known eligibility time: look again daily. +const RECHECK_UNKNOWN_MS = DAY_MS; export type Position = { createdAt: Date; id: string }; export type PendingStore = { - list: (limit: number) => Promise; - add: (ids: string[]) => Promise; + /** Ids due for a check at `now`, oldest due first. */ + due: (now: number, limit: number) => Promise; + /** Check these ids again at the given times. */ + schedule: (entries: { id: string; at: number }[]) => Promise; remove: (ids: string[]) => Promise; + size: () => Promise; }; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -83,9 +93,10 @@ async function withBatchTimeouts(fn: (tx: Tx) => Promise) { /** * Clears the content of the given emails if they're eligible, and returns how - * many were cleared and which still have content (not eligible yet). + * many were cleared and, for the ones that still have content, when to check + * them again. */ -async function clearEligible(tx: Tx, ids: string[], cutoff: Date) { +async function clearEligible(tx: Tx, ids: string[], cutoff: Date, now: number) { const cleared = await tx.$executeRaw` UPDATE "Email" SET html = NULL, text = NULL, attachments = NULL, headers = NULL @@ -97,13 +108,26 @@ async function clearEligible(tx: Tx, ids: string[], cutoff: Date) { AND (html IS NOT NULL OR text IS NOT NULL OR attachments IS NOT NULL OR headers IS NOT NULL) `; - const stillWithContent = await tx.$queryRaw<{ id: string }[]>` - SELECT id FROM "Email" + const stillWithContent = await tx.$queryRaw< + { id: string; latestStatus: string; anchor: Date }[] + >` + SELECT id, "latestStatus"::text AS "latestStatus", + GREATEST("createdAt", "updatedAt", COALESCE("scheduledAt", "createdAt")) AS anchor + FROM "Email" WHERE id IN (${Prisma.join(ids)}) AND (html IS NOT NULL OR text IS NOT NULL OR attachments IS NOT NULL OR headers IS NOT NULL) `; - return { cleared, notYet: stillWithContent.map((row) => row.id) }; + const retentionMs = now - cutoff.getTime(); + const notYet = stillWithContent.map((row) => ({ + id: row.id, + at: + row.latestStatus === "SCHEDULED" || row.latestStatus === "QUEUED" + ? now + RECHECK_UNKNOWN_MS + : // A minute's margin so it's past the cutoff when checked. + Math.max(row.anchor.getTime() + retentionMs + 60_000, now + 60_000), + })); + return { cleared, notYet }; } /** Looks at the next emails older than the cutoff, after `after`. */ @@ -111,6 +135,7 @@ async function clearNextBatch( cutoff: Date, after: Position | null, size: number, + now: number, ) { return withBatchTimeouts(async (tx) => { // Keyset over the createdAt index: no rescans of emails already looked at. @@ -127,12 +152,18 @@ async function clearNextBatch( LIMIT ${size} `; if (rows.length === 0) { - return { looked: 0, cleared: 0, notYet: [] as string[], last: null }; + return { + looked: 0, + cleared: 0, + notYet: [] as { id: string; at: number }[], + last: null, + }; } const { cleared, notYet } = await clearEligible( tx, rows.map((row) => row.id), cutoff, + now, ); return { looked: rows.length, @@ -143,27 +174,6 @@ async function clearNextBatch( }); } -/** Checks emails skipped on earlier nights again, by id. */ -async function clearPending( - cutoff: Date, - pending: PendingStore, - limits: typeof CLEANUP_LIMITS, -) { - const ids = await pending.list(limits.maxPendingPerRun); - let cleared = 0; - for (let i = 0; i < ids.length; i += limits.batchSize) { - const chunk = ids.slice(i, i + limits.batchSize); - const result = await withBatchTimeouts((tx) => - clearEligible(tx, chunk, cutoff), - ); - cleared += result.cleared; - // Keep only the ones that still have content; drop cleared or deleted ones. - const keep = new Set(result.notYet); - await pending.remove(chunk.filter((id) => !keep.has(id))); - } - return { checked: ids.length, cleared }; -} - export async function clearOldEmailBodies({ cutoff, startAfter, @@ -182,40 +192,64 @@ export async function clearOldEmailBodies({ now?: () => number; }) { const startedAt = now(); - const pendingResult = await clearPending(cutoff, pending, limits); - - let position = startAfter; let looked = 0; - let cleared = pendingResult.cleared; + let cleared = 0; let batches = 0; - const result = (finished: boolean) => ({ + let pendingChecked = 0; + let pendingCleared = 0; + let position = startAfter; + // One budget for both phases: rows cleared and time. + const budgetLeft = () => { + const outOfTime = batches > 0 && now() - startedAt >= limits.maxRunMs; + return outOfTime ? 0 : Math.max(0, limits.maxRowsPerRun - cleared); + }; + const result = async (finished: boolean) => ({ looked, cleared, batches, - pendingChecked: pendingResult.checked, - pendingCleared: pendingResult.cleared, + pendingChecked, + pendingCleared, + pendingLeft: await pending.size(), finished, elapsedMs: now() - startedAt, }); - for (;;) { - const remaining = limits.maxRowsPerRun - cleared; - if ( - remaining <= 0 || - (batches > 0 && now() - startedAt >= limits.maxRunMs) - ) { - return result(false); - } + // 1. Emails skipped on earlier nights that are now due for a check. + const dueIds = await pending.due(now(), limits.maxPendingPerRun); + for (let i = 0; i < dueIds.length;) { + const size = Math.min(limits.batchSize, budgetLeft()); + if (size === 0) return result(false); + const chunk = dueIds.slice(i, i + size); + i += chunk.length; + const checkedAt = now(); + const batch = await withBatchTimeouts((tx) => + clearEligible(tx, chunk, cutoff, checkedAt), + ); + batches += 1; + pendingChecked += chunk.length; + pendingCleared += batch.cleared; + cleared += batch.cleared; + // Reschedule the ones still not clearable; drop cleared or deleted ones. + await pending.schedule(batch.notYet); + const keep = new Set(batch.notYet.map((entry) => entry.id)); + await pending.remove(chunk.filter((id) => !keep.has(id))); + await sleep(limits.pauseMs); + } + // 2. Emails not looked at yet, oldest first. + for (;;) { // Never look at more rows than the nightly cap has left, so it holds exactly. - const size = Math.min(limits.batchSize, remaining); - const batch = await clearNextBatch(cutoff, position, size); + const size = Math.min(limits.batchSize, budgetLeft()); + if (size === 0) return result(false); + + const batch = await clearNextBatch(cutoff, position, size, now()); if (!batch.last) return result(true); looked += batch.looked; cleared += batch.cleared; batches += 1; - if (batch.notYet.length > 0) await pending.add(batch.notYet); + // Save the skipped ones before moving the watermark past them. + if (batch.notYet.length > 0) await pending.schedule(batch.notYet); position = batch.last; await saveProgress(position); @@ -261,13 +295,19 @@ if (isEmailCleanupEnabled()) { await redis.set(watermarkKey, JSON.stringify(position)); }, pending: { - list: (limit) => redis.srandmember(pendingKey, limit), - add: async (ids) => { - if (ids.length) await redis.sadd(pendingKey, ...ids); + due: (at, limit) => + redis.zrangebyscore(pendingKey, "-inf", at, "LIMIT", 0, limit), + schedule: async (entries) => { + if (!entries.length) return; + await redis.zadd( + pendingKey, + ...entries.flatMap((entry) => [entry.at, entry.id]), + ); }, remove: async (ids) => { - if (ids.length) await redis.srem(pendingKey, ...ids); + if (ids.length) await redis.zrem(pendingKey, ...ids); }, + size: () => redis.zcard(pendingKey), }, });