diff --git a/apps/docs/self-hosting/overview.mdx b/apps/docs/self-hosting/overview.mdx index d453eb85a..e31808b10 100644 --- a/apps/docs/self-hosting/overview.mdx +++ b/apps/docs/self-hosting/overview.mdx @@ -82,7 +82,9 @@ Add the following environment variables. - By default, stored email content stays in the database forever. To bound database growth, set `EMAIL_CLEANUP_DAYS` to a positive number of days. A daily job (midnight UTC) then clears the `text`, `html`, `attachments` and `headers` values for emails older than the cutoff. Delivery status and event history are kept. + By default, stored email content stays in the database forever. To bound database growth, set `EMAIL_CLEANUP_DAYS` to a positive number of days. A daily job (midnight UTC) then clears the `text`, `html`, `attachments` and `headers` values of emails sent more than that many days ago. Delivery status and event history are kept, and the dashboard explains that the content was removed. Scheduled and queued emails keep their content until they're sent. + + The job clears up to 100,000 emails a night in small batches. If you turn it on for an existing database, older emails are cleared over several nights. Autovacuum then makes the freed space reusable, which slows how fast the database grows; it usually doesn't shrink the files on disk. ```env EMAIL_CLEANUP_DAYS=30 diff --git a/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql b/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql new file mode 100644 index 000000000..a5b1bc1f2 --- /dev/null +++ b/apps/web/prisma/migrations/20261117120000_email_autovacuum_tuning/migration.sql @@ -0,0 +1,13 @@ +-- Encourage autovacuum to visit "Email" sooner: after ~2% of its rows change +-- instead of the default 20%. The email content cleanup rewrites up to 100k +-- rows a night; this makes that space reusable sooner instead of the table +-- growing. It only changes a table setting and touches no data. +-- +-- The ALTER needs a SHARE UPDATE EXCLUSIVE lock: normal reads and writes carry +-- on, but it waits for a running VACUUM, ANALYZE or CREATE INDEX CONCURRENTLY +-- on "Email". lock_timeout makes it give up after 10s instead of waiting +-- indefinitely during a deploy; the migration then fails and can be retried. +BEGIN; +SET LOCAL lock_timeout = '10s'; +ALTER TABLE "Email" SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_analyze_scale_factor = 0.02); +COMMIT; diff --git a/apps/web/src/app/(dashboard)/emails/email-details.tsx b/apps/web/src/app/(dashboard)/emails/email-details.tsx index 221184281..3066a416e 100644 --- a/apps/web/src/app/(dashboard)/emails/email-details.tsx +++ b/apps/web/src/app/(dashboard)/emails/email-details.tsx @@ -94,6 +94,8 @@ export default function EmailDetails({ emailId }: { emailId: string }) { @@ -138,12 +140,25 @@ export default function EmailDetails({ emailId }: { emailId: string }) { ); } +function isPastRetention( + createdAt: Date | string | null | undefined, + days: number | null | undefined, +) { + if (!createdAt || !days) return false; + const ageMs = Date.now() - new Date(createdAt).getTime(); + return ageMs >= days * 24 * 60 * 60 * 1000; +} + const EmailPreview = ({ html, text, + createdAt, + contentRetentionDays, }: { html: string | null | undefined; text: string | null | undefined; + createdAt?: Date | string | null; + contentRetentionDays?: number | null; }) => { const [show, setShow] = useState(false); @@ -166,8 +181,10 @@ const EmailPreview = ({ if (!srcDoc) { return (
- - No content to preview + + {isPastRetention(createdAt, contentRetentionDays) + ? `Email content is removed ${contentRetentionDays} days after sending. Its status and events are kept.` + : "No content to preview"}
); @@ -306,10 +323,7 @@ const getErrorMessage = (data: SesBounce) => { if (data.bounceType === "Permanent") { return BOUNCE_ERROR_MESSAGES[data.bounceType][ data.bounceSubType as - | "General" - | "NoEmail" - | "Suppressed" - | "OnAccountSuppressionList" + "General" | "NoEmail" | "Suppressed" | "OnAccountSuppressionList" ]; } else if (data.bounceType === "Transient") { return BOUNCE_ERROR_MESSAGES[data.bounceType][ diff --git a/apps/web/src/server/api/routers/email.ts b/apps/web/src/server/api/routers/email.ts index 787d2daa8..cdaa7a476 100644 --- a/apps/web/src/server/api/routers/email.ts +++ b/apps/web/src/server/api/routers/email.ts @@ -11,6 +11,7 @@ import { teamProcedure, } from "~/server/api/trpc"; import { db } from "~/server/db"; +import { env } from "~/env"; import { cancelEmail, updateEmail } from "~/server/service/email-service"; const statuses = Object.values(EmailStatus) as [EmailStatus]; @@ -246,7 +247,10 @@ export const emailRouter = createTRPCRouter({ }, }); - return email; + if (!email) return email; + + // Email content (html, text) is cleared after this many days, when set. + return { ...email, contentRetentionDays: env.EMAIL_CLEANUP_DAYS ?? null }; }), cancelEmail: emailProcedure.mutation(async ({ input }) => { diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts new file mode 100644 index 000000000..8321c18f4 --- /dev/null +++ b/apps/web/src/server/jobs/cleanup-email-bodies.integration.test.ts @@ -0,0 +1,304 @@ +import { EmailStatus } from "@prisma/client"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { db } from "~/server/db"; +import { createTeam } from "~/test/factories/core"; +import { + closeIntegrationConnections, + integrationEnabled, + resetDatabase, +} from "~/test/integration/helpers"; + +// Only the database is real here; the job's queue and worker aren't needed. +vi.mock("bullmq", () => ({ + Queue: class { + upsertJobScheduler = vi.fn().mockResolvedValue(undefined); + }, + Worker: class { + on = vi.fn(); + }, +})); + +import { + CLEANUP_LIMITS, + clearOldEmailBodies, + retentionCutoff, + type PendingStore, + type Position, +} from "~/server/jobs/cleanup-email-bodies"; + +const describeIntegration = integrationEnabled ? describe : describe.skip; + +const DAY_MS = 24 * 60 * 60 * 1000; +const daysAgo = (days: number) => new Date(Date.now() - days * DAY_MS); +const LONG_AGO = new Date("2025-06-01T00:00:00Z"); + +function memoryPending(): PendingStore & { ids: Map } { + const ids = new Map(); + return { + ids, + due: async (now, limit) => + [...ids.entries()] + .filter(([, at]) => at <= now) + .sort((a, b) => a[1] - b[1]) + .slice(0, limit) + .map(([id]) => id), + schedule: async (entries) => + entries.forEach((entry) => ids.set(entry.id, entry.at)), + remove: async (gone) => gone.forEach((id) => ids.delete(id)), + size: async () => ids.size, + }; +} + +describeIntegration("email content cleanup against postgres", () => { + let teamId: number; + const cutoff = retentionCutoff(90); + const fast = { ...CLEANUP_LIMITS, batchSize: 2, pauseMs: 0 }; + + async function createEmail( + name: string, + { + created, + updated = LONG_AGO, + status = EmailStatus.DELIVERED, + scheduledAt = null, + }: { + created: Date; + updated?: Date; + status?: EmailStatus; + scheduledAt?: Date | null; + }, + ) { + const email = await db.email.create({ + data: { + teamId, + from: "news@example.com", + to: [`${name}@example.com`], + subject: name, + html: `

${name}

`, + text: name, + attachments: "[]", + headers: "{}", + latestStatus: status, + scheduledAt, + }, + }); + await db.$executeRaw`UPDATE "Email" SET "createdAt" = ${created}, "updatedAt" = ${updated} WHERE id = ${email.id}`; + return email.id; + } + + const contentOf = (id: string) => + db.email.findUniqueOrThrow({ + where: { id }, + select: { + html: true, + text: true, + attachments: true, + headers: true, + updatedAt: true, + subject: true, + }, + }); + const clearedCount = () => db.email.count({ where: { html: null } }); + + beforeEach(async () => { + await resetDatabase(); + teamId = (await createTeam()).id; + }); + + afterAll(async () => { + await closeIntegrationConnections(); + }); + + it("clears content of emails sent over 90 days ago and keeps everything else", async () => { + const old = [ + await createEmail("old-1", { created: daysAgo(200) }), + await createEmail("old-2", { created: daysAgo(120) }), + await createEmail("old-bounced", { + created: daysAgo(95), + status: EmailStatus.BOUNCED, + }), + ]; + const kept = [ + await createEmail("recent", { + created: daysAgo(10), + updated: daysAgo(10), + }), + await createEmail("old-but-scheduled", { + created: daysAgo(120), + status: EmailStatus.SCHEDULED, + scheduledAt: daysAgo(-5), + }), + await createEmail("old-but-queued", { + created: daysAgo(120), + status: EmailStatus.QUEUED, + }), + // Created long ago but only sent 10 days ago: kept until 90 days after. + await createEmail("old-but-sent-late", { + created: daysAgo(120), + updated: daysAgo(10), + }), + ]; + + const result = await clearOldEmailBodies({ + cutoff, + startAfter: null, + saveProgress: async () => undefined, + pending: memoryPending(), + limits: fast, + }); + + expect(result).toMatchObject({ cleared: 3, finished: true }); + for (const id of old) { + const email = await contentOf(id); + expect(email).toMatchObject({ + html: null, + text: null, + attachments: null, + headers: null, + }); + expect(email.subject).not.toBeNull(); + // updatedAt is untouched (the campaign page sorts by it). + expect(email.updatedAt.toISOString()).toBe(LONG_AGO.toISOString()); + } + for (const id of kept) { + expect((await contentOf(id)).html).not.toBeNull(); + } + }); + + it("checks skipped emails again once they're due, and clears them then", async () => { + const late = await createEmail("sent-late", { + created: daysAgo(120), + updated: daysAgo(10), + }); + await createEmail("old", { created: daysAgo(130) }); + const pending = memoryPending(); + let position: Position | null = null; + const runAt = (clock: number) => + clearOldEmailBodies({ + cutoff: retentionCutoff(90, clock), + startAfter: position, + saveProgress: async (p) => { + position = p; + }, + pending, + limits: fast, + now: () => clock, + }); + + await runAt(Date.now()); + expect((await contentOf(late)).html).not.toBeNull(); + // Due 90 days after it was sent, not before. + const dueAt = pending.ids.get(late)!; + expect(dueAt).toBeGreaterThan(Date.now() + 79 * DAY_MS); + expect(dueAt).toBeLessThan(Date.now() + 81 * DAY_MS); + + // 70 days later: not due yet, not checked. + const early = await runAt(Date.now() + 70 * DAY_MS); + expect(early.pendingChecked).toBe(0); + expect((await contentOf(late)).html).not.toBeNull(); + + // 81 days later it was sent over 90 days ago. + const later = await runAt(Date.now() + 81 * DAY_MS); + expect(later).toMatchObject({ pendingChecked: 1, pendingCleared: 1 }); + expect((await contentOf(late)).html).toBeNull(); + expect(pending.ids.size).toBe(0); + }); + + it("keeps the recheck pass within the nightly cap", async () => { + const pending = memoryPending(); + const ids: string[] = []; + for (let i = 0; i < 5; i++) + ids.push(await createEmail(`p-${i}`, { created: daysAgo(100) })); + await pending.schedule(ids.map((id) => ({ id, at: 0 }))); + + const result = await clearOldEmailBodies({ + cutoff, + startAfter: { createdAt: new Date(), id: "zzz" }, // nothing new to scan + saveProgress: async () => undefined, + pending, + limits: { ...fast, maxRowsPerRun: 2 }, + }); + + expect(result).toMatchObject({ cleared: 2, finished: false }); + expect(await clearedCount()).toBe(2); + expect(pending.ids.size).toBe(3); + }); + + it("doesn't skip emails that share a creation time across batches", async () => { + const sameTime = daysAgo(100); + for (let i = 0; i < 5; i++) + await createEmail(`tie-${i}`, { created: sameTime }); + + let position: Position | null = null; + const pending = memoryPending(); + // One email per batch, stopping after each: every resume starts mid-tie. + for (let night = 0; night < 5; night++) { + await clearOldEmailBodies({ + cutoff, + startAfter: position, + saveProgress: async (p) => { + position = p; + }, + pending, + limits: { ...fast, batchSize: 1, maxRowsPerRun: 1 }, + }); + } + + expect(await clearedCount()).toBe(5); + }); + + it("stops exactly at the nightly cap, then carries on where it stopped", async () => { + for (let i = 0; i < 4; i++) { + await createEmail(`old-${i}`, { created: daysAgo(100 + i * 2) }); + // Interleaved with emails that can't be cleared yet. + await createEmail(`late-${i}`, { + created: daysAgo(101 + i * 2), + updated: daysAgo(5), + }); + } + let position: Position | null = null; + const pending = memoryPending(); + const limits = { ...fast, batchSize: 2, maxRowsPerRun: 3 }; + + const first = await clearOldEmailBodies({ + cutoff, + startAfter: null, + saveProgress: async (p) => { + position = p; + }, + pending, + limits, + }); + expect(first.cleared).toBe(3); + expect(first.finished).toBe(false); + expect(await clearedCount()).toBe(3); + + const second = await clearOldEmailBodies({ + cutoff, + startAfter: position, + saveProgress: async (p) => { + position = p; + }, + pending, + limits: { ...limits, maxRowsPerRun: 100 }, + }); + expect(second.cleared).toBe(1); + expect(second.finished).toBe(true); + expect(await clearedCount()).toBe(4); + expect(pending.ids.size).toBe(4); + expect(second.pendingLeft).toBe(4); + }); + + it("does nothing when no email is old enough", async () => { + await createEmail("recent", { created: daysAgo(5), updated: daysAgo(5) }); + + const result = await clearOldEmailBodies({ + cutoff, + startAfter: null, + saveProgress: async () => undefined, + pending: memoryPending(), + }); + + expect(result).toMatchObject({ looked: 0, cleared: 0, finished: true }); + }); +}); diff --git a/apps/web/src/server/jobs/cleanup-email-bodies.ts b/apps/web/src/server/jobs/cleanup-email-bodies.ts index cb000bc4c..9b58d67bd 100644 --- a/apps/web/src/server/jobs/cleanup-email-bodies.ts +++ b/apps/web/src/server/jobs/cleanup-email-bodies.ts @@ -1,81 +1,345 @@ -import {Queue, Worker} from "bullmq"; -import {db} from "~/server/db"; -import {getRedis, BULL_PREFIX} from "~/server/redis"; -import {logger} from "../logger/log"; -import {DEFAULT_QUEUE_OPTIONS} from "../queue/queue-constants"; -import {env} from "~/env"; -import {isSelfHosted, isEmailCleanupEnabled} from "~/utils/common"; +import { Queue, Worker } from "bullmq"; +import { Prisma } from "@prisma/client"; +import { db } from "~/server/db"; +import { getRedis, BULL_PREFIX, redisKey } from "~/server/redis"; +import { logger } from "../logger/log"; +import { DEFAULT_QUEUE_OPTIONS } from "../queue/queue-constants"; +import { env } from "~/env"; +import { isEmailCleanupEnabled } from "~/utils/common"; + +/** + * Clears the stored content (html, text, attachments, headers) of emails sent + * more than EMAIL_CLEANUP_DAYS ago. Delivery status and events are kept. Runs + * wherever EMAIL_CLEANUP_DAYS is set (self-hosted or cloud). + * + * "Sent more than N days ago" means created AND last updated before the + * cutoff. Sending updates the email (Prisma sets updatedAt), so an email that + * waited in a queue is kept for N days after it went out. SES status updates + * use raw SQL and don't touch updatedAt, so they don't delay clearing. + * Scheduled and queued emails, and ones scheduled after the cutoff, are never + * cleared: they still need their content to send. + * + * Built for a large table on a nearly full disk: + * - Walks forward by createdAt and remembers how far it got, so each night + * reads only emails it hasn't looked at yet. + * - Emails it looked at but couldn't clear yet go on a "pending" list, keyed + * by when each one becomes eligible, and are checked again (by id) once due. + * - Clears at most MAX_ROWS_PER_RUN a night, in small batches, each a short + * transaction with lock and statement timeouts. + * - Leaves vacuuming to autovacuum, which a migration tunes to visit "Email" + * sooner (after ~2% of rows change), so freed space is usually reusable by + * the next night. + * - Doesn't change updatedAt (the campaign page sorts recent emails by it). + */ const CLEANUP_QUEUE_NAME = "cleanup-email-bodies"; +const CLEANUP_CRON = "0 0 * * *"; // midnight UTC +const DAY_MS = 24 * 60 * 60 * 1000; -const CLEANUP_CRON = "0 0 * * *"; // default: midnight UTC - -// Only initialize if self hosted and cleanup enabled -if (isSelfHosted() && isEmailCleanupEnabled()) { - const CLEANUP_DAYS = env.EMAIL_CLEANUP_DAYS!; - - /** - * Initialize Queue - */ - const cleanupQueue = new Queue(CLEANUP_QUEUE_NAME, { - connection: getRedis(), - prefix: BULL_PREFIX, - skipVersionCheck: true, - }); - - const worker = new Worker( - CLEANUP_QUEUE_NAME, - async () => { - logger.info(`[Cleanup] Starting cleanup for emails older than ${CLEANUP_DAYS} days...`); - - const cutoffDate = new Date(); - cutoffDate.setDate(cutoffDate.getDate() - CLEANUP_DAYS); - - const result = await db.email.updateMany({ - where: { - createdAt: {lt: cutoffDate}, - OR: [ - {text: {not: null}}, - {html: {not: null}}, - {attachments: {not: null}}, - {headers: {not: null}}, - ], - }, - data: { - text: null, - html: null, - attachments: null, - headers: null, - }, - }); - - logger.info(`[Cleanup] Emails cleaned: ${result.count}`); - }, - { - connection: getRedis(), - prefix: BULL_PREFIX, - skipVersionCheck: true, - } +export const CLEANUP_LIMITS = { + batchSize: 500, + maxRowsPerRun: 100_000, + maxRunMs: 15 * 60 * 1000, + pauseMs: 200, + maxPendingPerRun: 5_000, +}; +const BATCH_LOCK_TIMEOUT = "5s"; +const BATCH_STATEMENT_TIMEOUT = "60s"; +// Prisma's own interactive-transaction limit (default 5s) must sit above the +// SQL timeouts, or it would cut a slow batch off first. +const BATCH_TRANSACTION_OPTIONS = { timeout: 130_000, maxWait: 2_000 }; + +/** Emails created before this position have all been looked at. */ +export const WATERMARK_KEY = "email-body-cleanup:cleaned-before"; +/** + * Emails looked at but not clearable yet: a sorted set of ids, scored by the + * time each should be checked again. Lives in the same Redis as BullMQ, which + * requires maxmemory-policy noeviction, so it isn't evicted on its own. + */ +export const PENDING_KEY = "email-body-cleanup:pending"; +// Scheduled or queued emails have no known eligibility time: look again daily. +const RECHECK_UNKNOWN_MS = DAY_MS; + +export type Position = { createdAt: Date; id: string }; + +export type PendingStore = { + /** Ids due for a check at `now`, oldest due first. */ + due: (now: number, limit: number) => Promise; + /** Check these ids again at the given times. */ + schedule: (entries: { id: string; at: number }[]) => Promise; + remove: (ids: string[]) => Promise; + size: () => Promise; +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export function retentionCutoff(days: number, now = Date.now()) { + return new Date(now - days * DAY_MS); +} + +type Tx = Prisma.TransactionClient; + +async function withBatchTimeouts(fn: (tx: Tx) => Promise) { + return db.$transaction(async (tx) => { + await tx.$executeRawUnsafe( + `SET LOCAL lock_timeout = '${BATCH_LOCK_TIMEOUT}'`, + ); + await tx.$executeRawUnsafe( + `SET LOCAL statement_timeout = '${BATCH_STATEMENT_TIMEOUT}'`, ); + return fn(tx); + }, BATCH_TRANSACTION_OPTIONS); +} - await cleanupQueue.upsertJobScheduler( - "scheduled-email-cleanup", - { - pattern: CLEANUP_CRON, - tz: "UTC", +/** + * Clears the content of the given emails if they're eligible, and returns how + * many were cleared and, for the ones that still have content, when to check + * them again. + */ +async function clearEligible(tx: Tx, ids: string[], cutoff: Date, now: number) { + const cleared = await tx.$executeRaw` + UPDATE "Email" + SET html = NULL, text = NULL, attachments = NULL, headers = NULL + WHERE id IN (${Prisma.join(ids)}) + AND "createdAt" < ${cutoff} + AND "updatedAt" < ${cutoff} + AND ("scheduledAt" IS NULL OR "scheduledAt" < ${cutoff}) + AND "latestStatus" NOT IN ('SCHEDULED', 'QUEUED') + AND (html IS NOT NULL OR text IS NOT NULL + OR attachments IS NOT NULL OR headers IS NOT NULL) + `; + const stillWithContent = await tx.$queryRaw< + { id: string; latestStatus: string; anchor: Date }[] + >` + SELECT id, "latestStatus"::text AS "latestStatus", + GREATEST("createdAt", "updatedAt", COALESCE("scheduledAt", "createdAt")) AS anchor + FROM "Email" + WHERE id IN (${Prisma.join(ids)}) + AND (html IS NOT NULL OR text IS NOT NULL + OR attachments IS NOT NULL OR headers IS NOT NULL) + `; + const retentionMs = now - cutoff.getTime(); + const notYet = stillWithContent.map((row) => ({ + id: row.id, + at: + row.latestStatus === "SCHEDULED" || row.latestStatus === "QUEUED" + ? now + RECHECK_UNKNOWN_MS + : // A minute's margin so it's past the cutoff when checked. + Math.max(row.anchor.getTime() + retentionMs + 60_000, now + 60_000), + })); + return { cleared, notYet }; +} + +/** Looks at the next emails older than the cutoff, after `after`. */ +async function clearNextBatch( + cutoff: Date, + after: Position | null, + size: number, + now: number, +) { + return withBatchTimeouts(async (tx) => { + // Keyset over the createdAt index: no rescans of emails already looked at. + const rows = await tx.$queryRaw` + SELECT id, "createdAt" FROM "Email" + WHERE "createdAt" < ${cutoff} + ${ + after + ? Prisma.sql`AND "createdAt" >= ${after.createdAt} + AND ("createdAt" > ${after.createdAt} OR id > ${after.id})` + : Prisma.empty + } + ORDER BY "createdAt" ASC, id ASC + LIMIT ${size} + `; + if (rows.length === 0) { + return { + looked: 0, + cleared: 0, + notYet: [] as { id: string; at: number }[], + last: null, + }; + } + const { cleared, notYet } = await clearEligible( + tx, + rows.map((row) => row.id), + cutoff, + now, + ); + return { + looked: rows.length, + cleared, + notYet, + last: rows[rows.length - 1]!, + }; + }); +} + +export async function clearOldEmailBodies({ + cutoff, + startAfter, + saveProgress, + pending, + limits = CLEANUP_LIMITS, + now = () => Date.now(), +}: { + cutoff: Date; + /** Where the last run stopped, if anywhere. */ + startAfter: Position | null; + /** Called after every batch with how far this run has got. */ + saveProgress: (position: Position) => Promise; + pending: PendingStore; + limits?: typeof CLEANUP_LIMITS; + now?: () => number; +}) { + const startedAt = now(); + let looked = 0; + let cleared = 0; + let batches = 0; + let pendingChecked = 0; + let pendingCleared = 0; + let position = startAfter; + // One budget for both phases: rows cleared and time. + const budgetLeft = () => { + const outOfTime = batches > 0 && now() - startedAt >= limits.maxRunMs; + return outOfTime ? 0 : Math.max(0, limits.maxRowsPerRun - cleared); + }; + const result = async (finished: boolean) => ({ + looked, + cleared, + batches, + pendingChecked, + pendingCleared, + pendingLeft: await pending.size(), + finished, + elapsedMs: now() - startedAt, + }); + + // 1. Emails skipped on earlier nights that are now due for a check. + const dueIds = await pending.due(now(), limits.maxPendingPerRun); + for (let i = 0; i < dueIds.length;) { + const size = Math.min(limits.batchSize, budgetLeft()); + if (size === 0) return result(false); + const chunk = dueIds.slice(i, i + size); + i += chunk.length; + const checkedAt = now(); + const batch = await withBatchTimeouts((tx) => + clearEligible(tx, chunk, cutoff, checkedAt), + ); + batches += 1; + pendingChecked += chunk.length; + pendingCleared += batch.cleared; + cleared += batch.cleared; + // Reschedule the ones still not clearable; drop cleared or deleted ones. + await pending.schedule(batch.notYet); + const keep = new Set(batch.notYet.map((entry) => entry.id)); + await pending.remove(chunk.filter((id) => !keep.has(id))); + await sleep(limits.pauseMs); + } + + // 2. Emails not looked at yet, oldest first. + for (;;) { + // Never look at more rows than the nightly cap has left, so it holds exactly. + const size = Math.min(limits.batchSize, budgetLeft()); + if (size === 0) return result(false); + + const batch = await clearNextBatch(cutoff, position, size, now()); + if (!batch.last) return result(true); + + looked += batch.looked; + cleared += batch.cleared; + batches += 1; + // Save the skipped ones before moving the watermark past them. + if (batch.notYet.length > 0) await pending.schedule(batch.notYet); + position = batch.last; + await saveProgress(position); + + if (batch.looked < size) return result(true); + await sleep(limits.pauseMs); + } +} + +function parsePosition(raw: string | null): Position | null { + if (!raw) return null; + try { + const value = JSON.parse(raw) as { createdAt: string; id: string }; + const createdAt = new Date(value.createdAt); + if (Number.isNaN(createdAt.getTime()) || !value.id) return null; + return { createdAt, id: value.id }; + } catch { + return null; + } +} + +if (isEmailCleanupEnabled()) { + const cleanupDays = env.EMAIL_CLEANUP_DAYS!; + const watermarkKey = redisKey(WATERMARK_KEY); + const pendingKey = redisKey(PENDING_KEY); + + const cleanupQueue = new Queue(CLEANUP_QUEUE_NAME, { + connection: getRedis(), + prefix: BULL_PREFIX, + skipVersionCheck: true, + }); + + const worker = new Worker( + CLEANUP_QUEUE_NAME, + async () => { + const redis = getRedis(); + const cutoff = retentionCutoff(cleanupDays); + const startAfter = parsePosition(await redis.get(watermarkKey)); + + const result = await clearOldEmailBodies({ + cutoff, + startAfter, + saveProgress: async (position) => { + await redis.set(watermarkKey, JSON.stringify(position)); + }, + pending: { + due: (at, limit) => + redis.zrangebyscore(pendingKey, "-inf", at, "LIMIT", 0, limit), + schedule: async (entries) => { + if (!entries.length) return; + await redis.zadd( + pendingKey, + ...entries.flatMap((entry) => [entry.at, entry.id]), + ); + }, + remove: async (ids) => { + if (ids.length) await redis.zrem(pendingKey, ...ids); + }, + size: () => redis.zcard(pendingKey), }, + }); + + logger.info( { - opts: { - ...DEFAULT_QUEUE_OPTIONS, - }, - } - ); + ...result, + cutoff: cutoff.toISOString(), + startedAfter: startAfter?.createdAt.toISOString() ?? null, + }, + result.finished + ? "[EmailCleanupJob]: Cleared old email content" + : "[EmailCleanupJob]: Cleared old email content; more remains for the next run", + ); + }, + { + connection: getRedis(), + prefix: BULL_PREFIX, + skipVersionCheck: true, + }, + ); + + await cleanupQueue.upsertJobScheduler( + "scheduled-email-cleanup", + { pattern: CLEANUP_CRON, tz: "UTC" }, + { opts: { ...DEFAULT_QUEUE_OPTIONS } }, + ); - worker.on("completed", (job) => { - logger.info({jobId: job.id}, ` Email Body cleanup job completed`); - }); + worker.on("completed", (job) => { + logger.info({ jobId: job.id }, "[EmailCleanupJob]: Job completed"); + }); - worker.on("failed", (job, err) => { - logger.error({err, jobId: job?.id}, `Email Body cleanup job failed`); - }); + worker.on("failed", (job, err) => { + logger.error({ err, jobId: job?.id }, "[EmailCleanupJob]: Job failed"); + }); }