diff --git a/apps/web/src/server/queue/requeue-job.integration.test.ts b/apps/web/src/server/queue/requeue-job.integration.test.ts index 34ad8b40..6272925f 100644 --- a/apps/web/src/server/queue/requeue-job.integration.test.ts +++ b/apps/web/src/server/queue/requeue-job.integration.test.ts @@ -101,6 +101,76 @@ describeIntegration("requeueJob with real BullMQ and Redis", () => { TEST_TIMEOUT_MS, ); + it( + "waits before retrying a job that failed moments ago, then retries it", + async () => { + startWorker(); + await queue.add( + "batch_1", + { callId: "batch_1" }, + { jobId: "batch_1", attempts: 1 }, + ); + await waitFor(async () => (await stateOf("batch_1")) === "failed"); + expect(attempts).toEqual([1]); + + shouldFail = false; + await expect( + requeueJob( + queue, + "batch_1", + { callId: "batch_1" }, + { jobId: "batch_1" }, + { retryFailedAfterMs: 60_000 }, + ), + ).resolves.toBe("failed-too-recently"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("batch_1")).toBe("failed"); + + await expect( + requeueJob( + queue, + "batch_1", + { callId: "batch_1" }, + { jobId: "batch_1" }, + { retryFailedAfterMs: 0 }, + ), + ).resolves.toBe("retried"); + await waitFor(async () => (await stateOf("batch_1")) === "missing"); + expect(attempts).toEqual([1, 1]); + }, + TEST_TIMEOUT_MS, + ); + + it( + "leaves a job that failed too long ago alone", + async () => { + startWorker(); + await queue.add( + "batch_2", + { callId: "batch_2" }, + { jobId: "batch_2", attempts: 1 }, + ); + await waitFor(async () => (await stateOf("batch_2")) === "failed"); + await new Promise((resolve) => setTimeout(resolve, 50)); + + shouldFail = false; + await expect( + requeueJob( + queue, + "batch_2", + { callId: "batch_2" }, + { jobId: "batch_2" }, + { maxFailedAgeMs: 10 }, + ), + ).resolves.toBe("failed-too-long-ago"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("batch_2")).toBe("failed"); + }, + TEST_TIMEOUT_MS, + ); + it( "adds a new job when none exists", async () => { diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index 6a9e437d..17f6174e 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -1,6 +1,22 @@ -import type { JobsOptions, Queue } from "bullmq"; +import type { Job, JobsOptions, Queue } from "bullmq"; -export type RequeueResult = "retried" | "added" | "already-queued"; +export type RequeueResult = + | "retried" + | "added" + | "already-queued" + | "failed-too-recently" + | "failed-too-long-ago"; + +// A Queue's type argument is either the job data or a Job type (as BullMQ allows). +type QueueData = + DataOrJob extends Job ? Data : DataOrJob; + +export type RequeueOptions = { + /** Leave a job that failed less than this long ago alone, for now. */ + retryFailedAfterMs?: number; + /** Leave a job that failed more than this long ago alone. */ + maxFailedAgeMs?: number; +}; /** * Runs a job again under a fixed job id. @@ -14,39 +30,74 @@ export type RequeueResult = "retried" | "added" | "already-queued"; * run anyway; * - otherwise adds a new job. * + * A failed job can be left alone for a while (`retryFailedAfterMs`) or for good + * once it is old (`maxFailedAgeMs`). Without options it is retried at once. + * * Use it only where a job is deliberately run again (for example a user pressing - * Retry), not on hot paths: it costs an extra Redis round trip. + * Retry, or a scheduler continuing work), not on hot paths: it costs an extra + * Redis round trip. * - * The result says what this call did, not that the job will run: if the job - * changes state at the same moment (another retry, or the 30-day cleanup), - * `retry()` throws, and the caller should let the user try again. It does not - * repair queue data that is out of sync. + * The result is a best-effort description, not a delivery guarantee + * ("already-queued" means an existing job was kept, or a vanished one was + * re-added). If the job changes state at the same moment (another retry, or + * the 30-day cleanup), `retry()` throws, and the caller should let the user + * try again. It does not repair queue data that is out of sync. */ -export async function requeueJob( - queue: Queue, +export async function requeueJob( + queue: Queue, name: string, - data: Data, + data: QueueData, opts: JobsOptions & { jobId: string }, + options: RequeueOptions = {}, ): Promise { const existing = await queue.getJob(opts.jobId); if (existing) { - const state = await existing.getState(); - - if (state === "failed" || state === "completed") { - await existing.retry(state, { - resetAttemptsMade: true, - resetAttemptsStarted: true, - }); - return "retried"; - } + // isFailed() and isCompleted() are single ZSCORE lookups. getState() would + // also scan the wait and active lists, which is too costly for callers that + // run on every scheduler tick. + const finishedState = (await existing.isFailed()) + ? "failed" + : (await existing.isCompleted()) + ? "completed" + : null; - if (state !== "unknown") { + if (!finishedState) { + // Still waiting, delayed or running: it will run anyway. add() is still + // called, in case the job finished and was removed during the checks + // above: BullMQ ignores it while a job with this id exists. + await (queue as Queue).add(name, data, opts); return "already-queued"; } + + if (finishedState === "failed") { + // finishedOn is missing if the job was fetched while still running and + // failed just after: treat that as a fresh failure, never an old one. + const failedForMs = existing.finishedOn + ? Date.now() - existing.finishedOn + : 0; + if ( + options.retryFailedAfterMs !== undefined && + failedForMs < options.retryFailedAfterMs + ) { + return "failed-too-recently"; + } + if ( + options.maxFailedAgeMs !== undefined && + failedForMs > options.maxFailedAgeMs + ) { + return "failed-too-long-ago"; + } + } + + await existing.retry(finishedState, { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + return "retried"; } - // Queue's name and data generics are wider than this helper needs. + // Queue's name and data generics are wider than this helper needs. await (queue as Queue).add(name, data, opts); return "added"; } diff --git a/apps/web/src/server/queue/requeue-job.unit.test.ts b/apps/web/src/server/queue/requeue-job.unit.test.ts new file mode 100644 index 00000000..b27cc2bb --- /dev/null +++ b/apps/web/src/server/queue/requeue-job.unit.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from "vitest"; +import { requeueJob } from "~/server/queue/requeue-job"; + +function queueWith(job: unknown) { + return { + getJob: vi.fn().mockResolvedValue(job), + add: vi.fn().mockResolvedValue(undefined), + } as any; +} + +describe("requeueJob", () => { + it("treats a failed job without a finish time as a fresh failure", async () => { + // Fetched while running, failed just after: finishedOn isn't on the snapshot. + const job = { + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), + finishedOn: undefined, + retry: vi.fn().mockResolvedValue(undefined), + }; + const options = { retryFailedAfterMs: 60_000, maxFailedAgeMs: 86_400_000 }; + + await expect( + requeueJob(queueWith(job), "n", {}, { jobId: "j" }, options), + ).resolves.toBe("failed-too-recently"); + expect(job.retry).not.toHaveBeenCalled(); + }); + + it("never calls a failure with no finish time too old to retry", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), + finishedOn: undefined, + retry: vi.fn().mockResolvedValue(undefined), + }; + + await expect( + requeueJob( + queueWith(job), + "n", + {}, + { jobId: "j" }, + { maxFailedAgeMs: 1 }, + ), + ).resolves.toBe("retried"); + }); +}); diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index da5d7cd9..5c3db73b 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -17,6 +17,7 @@ import { } from "../queue/queue-constants"; import { logger } from "../logger/log"; import { createWorkerHandler, TeamJob } from "../queue/bullmq-context"; +import { requeueJob } from "../queue/requeue-job"; import { SuppressionService } from "./suppression-service"; import { UnsendApiError } from "../public-api/api-error"; import { @@ -378,6 +379,11 @@ export async function sendCampaign(id: string) { }, }); + // Sending is an explicit choice: clear a batch that failed long ago, so the + // scheduler doesn't pause the campaign because of it (after the update, as + // in scheduleCampaign). + await CampaignBatchService.clearFailedBatch(id); + // Kick off first batch immediately (idempotent by jobId) await CampaignBatchService.queueBatch({ campaignId: id, @@ -475,6 +481,12 @@ export async function scheduleCampaign({ }, }); + // Scheduling is an explicit choice: clear a batch that failed long ago, so + // the scheduler doesn't pause the campaign because of it. This runs after the + // new schedule is saved, so the scheduler can't start an early batch in + // between. If it fails, the worst case is the campaign gets paused, not sent. + await CampaignBatchService.clearFailedBatch(campaign.id); + return { ok: true }; } @@ -522,6 +534,10 @@ export async function resumeCampaign({ }); } + // Resuming is an explicit choice, so it also clears a batch that failed long + // ago and that the scheduler no longer retries on its own. + await CampaignBatchService.clearFailedBatch(campaignId); + if (campaign.scheduledAt && campaign.scheduledAt.getTime() > Date.now()) { await db.campaign.update({ where: { id: campaignId }, @@ -1058,6 +1074,17 @@ export async function updateCampaignAnalytics( type CampaignBatchJob = TeamJob<{ campaignId: string }>; +// A failed batch is retried by the scheduler after a short wait, so a batch +// that keeps failing doesn't run on every 1.5s scheduler tick. +const CAMPAIGN_BATCH_RETRY_AFTER_MS = 60 * 1000; +// A batch that failed longer ago than this is not retried automatically, so an +// old campaign never starts sending again by surprise: the campaign is paused +// instead, and its owner can resume it. +const CAMPAIGN_BATCH_MAX_AUTO_RETRY_AGE_MS = 24 * 60 * 60 * 1000; + +const campaignBatchJobId = (campaignId: string) => + `campaign-batch-${campaignId}`; + export class CampaignBatchService { private static batchQueue = new Queue( CAMPAIGN_BATCH_QUEUE, @@ -1212,6 +1239,32 @@ export class CampaignBatchService { }, ); + static { + this.worker.on("failed", (job, err) => { + logger.error( + { err, campaignId: job?.data?.campaignId, jobId: job?.id }, + "[CampaignBatchService]: Campaign batch failed; the scheduler retries it after a minute if the campaign is still running", + ); + }); + } + + static async hasFailedBatch(campaignId: string) { + const job = await this.batchQueue.getJob(campaignBatchJobId(campaignId)); + return Boolean(job && (await job.isFailed())); + } + + /** Removes a kept failed batch job for a campaign, if there is one. */ + static async clearFailedBatch(campaignId: string) { + const job = await this.batchQueue.getJob(campaignBatchJobId(campaignId)); + if (job && (await job.isFailed())) { + await job.remove(); + logger.info( + { campaignId }, + "[CampaignBatchService]: Cleared a failed batch on resume", + ); + } + } + static async queueBatch({ campaignId, teamId, @@ -1219,13 +1272,24 @@ export class CampaignBatchService { campaignId: string; teamId?: number; }) { + // When the campaign was last changed, read before looking at the queue. + // Pausing below only applies if updatedAt is unchanged since, an optimistic + // guard against undoing a resume or reschedule made at the same moment. + let seenUpdatedAt: Date | undefined; + // Defensive check: avoid enqueue if window not elapsed (scheduler already enforces) try { const campaign = await db.campaign.findUnique({ where: { id: campaignId }, - select: { lastSentAt: true, batchWindowMinutes: true, status: true }, + select: { + lastSentAt: true, + batchWindowMinutes: true, + status: true, + updatedAt: true, + }, }); if (!campaign) return; + seenUpdatedAt = campaign.updatedAt; if (campaign.status === "PAUSED" || campaign.status === "SENT") return; const windowMin = campaign.batchWindowMinutes ?? 0; if (windowMin > 0 && campaign.lastSentAt) { @@ -1246,10 +1310,53 @@ export class CampaignBatchService { ); } - await this.batchQueue.add( + // A plain add() would do nothing after a failed batch: BullMQ keeps the + // failed job under this fixed id for 30 days, and the campaign would stop. + const result = await requeueJob( + this.batchQueue, `campaign-${campaignId}`, { campaignId, teamId }, - { jobId: `campaign-batch-${campaignId}`, ...DEFAULT_QUEUE_OPTIONS }, + { jobId: campaignBatchJobId(campaignId), ...DEFAULT_QUEUE_OPTIONS }, + { + retryFailedAfterMs: CAMPAIGN_BATCH_RETRY_AFTER_MS, + maxFailedAgeMs: CAMPAIGN_BATCH_MAX_AUTO_RETRY_AGE_MS, + }, ); + + if (result === "retried") { + logger.warn( + { campaignId }, + "[CampaignBatchService]: Retrying a failed campaign batch", + ); + } + + if ( + result === "failed-too-long-ago" && + seenUpdatedAt && + // Check again right before pausing: resume, schedule and send clear the + // failed batch, and that must win. (A clear landing in the moment between + // this check and the update below can still pause the campaign; that is + // safe, as nothing is sent, and the owner can resume it.) + (await this.hasFailedBatch(campaignId)) + ) { + // Pause in the database, not just skip: BullMQ deletes the failed job + // after 30 days, and the campaign would then restart on the next tick. + // The updatedAt check guards against undoing a resume or reschedule + // made while we were looking at the queue. + const paused = await db.campaign.updateMany({ + where: { + id: campaignId, + status: { in: ["SCHEDULED", "RUNNING"] }, + updatedAt: seenUpdatedAt, + }, + data: { status: "PAUSED" }, + }); + if (paused.count > 0) { + logger.warn( + { campaignId }, + "[CampaignBatchService]: Paused a campaign whose batch failed over a day ago; resume it to continue", + ); + } + } } } diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index 78a0062c..dd3fae57 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -26,18 +26,46 @@ const { mockDb, mockTx, mockUpdateContactSubscription } = vi.hoisted(() => { ), contact: { findUnique: vi.fn(), + findMany: vi.fn(), + count: vi.fn(), + }, + contactBook: { + findUnique: vi.fn(), + }, + domain: { + findUnique: vi.fn(), }, campaign: { findUnique: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), + }, + campaignEmail: { + findMany: vi.fn(), + create: vi.fn(), + }, + email: { + create: vi.fn(), }, }, mockUpdateContactSubscription: vi.fn(), }; }); -const { mockQueueAdd } = vi.hoisted(() => ({ +const { + mockQueueAdd, + mockQueueGetJob, + capturedBatchHandler, + mockCheckSuppressions, + mockQueueEmail, +} = vi.hoisted(() => ({ mockQueueAdd: vi.fn(), + mockQueueGetJob: vi.fn(), + capturedBatchHandler: { + handler: null as any, + }, + mockCheckSuppressions: vi.fn(), + mockQueueEmail: vi.fn(), })); vi.mock("~/server/db", () => ({ @@ -61,8 +89,15 @@ vi.mock("~/server/service/contact-service", () => ({ vi.mock("bullmq", () => ({ Queue: class { add = mockQueueAdd; + getJob = mockQueueGetJob; + }, + Worker: class { + on = vi.fn(); + constructor(queueName: string, handler: any) { + if (queueName === "campaign-batch") + capturedBatchHandler.handler = handler; + } }, - Worker: class {}, })); vi.mock("~/server/redis", () => ({ @@ -71,7 +106,7 @@ vi.mock("~/server/redis", () => ({ })); vi.mock("~/server/service/email-queue-service", () => ({ - EmailQueueService: {}, + EmailQueueService: { queueEmail: mockQueueEmail }, })); vi.mock("~/server/queue/bullmq-context", () => ({ @@ -79,7 +114,7 @@ vi.mock("~/server/queue/bullmq-context", () => ({ })); vi.mock("~/server/service/suppression-service", () => ({ - SuppressionService: {}, + SuppressionService: { checkMultipleEmails: mockCheckSuppressions }, })); vi.mock("~/server/service/domain-service", () => ({ @@ -99,6 +134,8 @@ vi.mock("~/server/logger/log", () => ({ import { CampaignBatchService, recordCampaignContactFailure, + resumeCampaign, + scheduleCampaign, subscribeContact, unsubscribeContact, } from "~/server/service/campaign-service"; @@ -225,6 +262,327 @@ describe("CampaignBatchService", () => { expect.objectContaining({ jobId: "campaign-batch-campaign_1" }), ); }); + + describe("after a batch failed", () => { + const failedJob = (failedSecondsAgo: number) => ({ + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), + finishedOn: Date.now() - failedSecondsAgo * 1000, + retry: vi.fn().mockResolvedValue(undefined), + }); + + const seenUpdatedAt = new Date("2026-11-16T10:00:00Z"); + + beforeEach(() => { + mockDb.campaign.findUnique.mockResolvedValue({ + lastSentAt: null, + batchWindowMinutes: 0, + status: "RUNNING", + updatedAt: seenUpdatedAt, + }); + }); + + it("retries the batch a minute later, with its attempts reset", async () => { + const job = failedJob(61); + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(mockQueueGetJob).toHaveBeenCalledWith("campaign-batch-campaign_1"); + expect(job.retry).toHaveBeenCalledWith("failed", { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("waits before retrying a batch that just failed", async () => { + const job = failedJob(10); + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(job.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("pauses a campaign whose batch failed over a day ago instead of restarting it", async () => { + const job = failedJob(25 * 60 * 60); + mockQueueGetJob.mockResolvedValue(job); + mockDb.campaign.updateMany.mockResolvedValue({ count: 1 }); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(job.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + // Only if nothing changed the campaign since it was read (e.g. a resume). + expect(mockDb.campaign.updateMany).toHaveBeenCalledWith({ + where: { + id: "campaign_1", + status: { in: ["SCHEDULED", "RUNNING"] }, + updatedAt: seenUpdatedAt, + }, + data: { status: "PAUSED" }, + }); + }); + + it("does not pause if the failed batch was cleared in the meantime", async () => { + // queueBatch sees the old failed job, then a reschedule clears it before + // the pause: the second look finds no job. + mockQueueGetJob + .mockResolvedValueOnce(failedJob(25 * 60 * 60)) + .mockResolvedValueOnce(undefined); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + + it("does not pause when it couldn't read the campaign first", async () => { + mockDb.campaign.findUnique.mockRejectedValue(new Error("db timeout")); + mockQueueGetJob.mockResolvedValue(failedJob(25 * 60 * 60)); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + + it("does not queue a second copy of a batch that is still running", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(false), + isCompleted: vi.fn().mockResolvedValue(false), + retry: vi.fn(), + }; + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(job.retry).not.toHaveBeenCalled(); + // add() is still sent with the same job id, in case the job finished + // during the checks; BullMQ ignores it while the job exists (see the + // requeueJob integration tests). + expect(mockQueueAdd).toHaveBeenCalledWith( + "campaign-campaign_1", + { campaignId: "campaign_1", teamId: 7 }, + expect.objectContaining({ jobId: "campaign-batch-campaign_1" }), + ); + }); + }); +}); + +describe("scheduleCampaign", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDb.campaign.findUnique.mockResolvedValue({ + id: "campaign_1", + teamId: 7, + status: "PAUSED", + contactBookId: "book_1", + content: null, + html: '

Hi Unsubscribe

', + }); + mockDb.contact.count.mockResolvedValue(3); + mockDb.campaign.update.mockResolvedValue({}); + }); + + it("clears a batch that failed long ago, so the campaign isn't paused again", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(true), + remove: vi.fn().mockResolvedValue(undefined), + }; + mockQueueGetJob.mockResolvedValue(job); + + await scheduleCampaign({ campaignId: "campaign_1", teamId: 7 }); + + expect(job.remove).toHaveBeenCalled(); + expect(mockDb.campaign.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "campaign_1" }, + data: expect.objectContaining({ status: "SCHEDULED" }), + }), + ); + // The new schedule is saved before the failed batch is cleared, so the + // scheduler can't start an early batch in between. + expect(mockDb.campaign.update.mock.invocationCallOrder[0]).toBeLessThan( + job.remove.mock.invocationCallOrder[0]!, + ); + }); +}); + +describe("resumeCampaign", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDb.campaign.findUnique.mockResolvedValue({ + id: "campaign_1", + teamId: 7, + scheduledAt: null, + }); + mockDb.campaign.update.mockResolvedValue({}); + }); + + it("clears a failed batch so the scheduler can continue the campaign", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(true), + remove: vi.fn().mockResolvedValue(undefined), + }; + mockQueueGetJob.mockResolvedValue(job); + + await resumeCampaign({ campaignId: "campaign_1", teamId: 7 }); + + expect(mockQueueGetJob).toHaveBeenCalledWith("campaign-batch-campaign_1"); + expect(job.remove).toHaveBeenCalled(); + expect(mockDb.campaign.update).toHaveBeenCalledWith({ + where: { id: "campaign_1" }, + data: { status: "RUNNING" }, + }); + }); + + it("leaves a batch that is still running alone", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(false), + remove: vi.fn(), + }; + mockQueueGetJob.mockResolvedValue(job); + + await resumeCampaign({ campaignId: "campaign_1", teamId: 7 }); + + expect(job.remove).not.toHaveBeenCalled(); + }); + + it("keeps the campaign paused if the failed batch can't be cleared", async () => { + mockQueueGetJob.mockResolvedValue({ + isFailed: vi.fn().mockResolvedValue(true), + remove: vi.fn().mockRejectedValue(new Error("Job is locked")), + }); + + await expect( + resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), + ).rejects.toThrow("Job is locked"); + + expect(mockDb.campaign.update).not.toHaveBeenCalled(); + }); +}); + +describe("campaign batch worker, re-running a batch that failed part-way", () => { + const contacts = ["contact_1", "contact_2", "contact_3"].map((id) => ({ + id, + email: `${id}@example.com`, + firstName: null, + lastName: null, + properties: {}, + contactBookId: "book_1", + subscribed: true, + })); + + beforeEach(() => { + vi.clearAllMocks(); + mockDb.campaign.findUnique.mockResolvedValue({ + id: "campaign_1", + teamId: 7, + contactBookId: "book_1", + domainId: 3, + status: "RUNNING", + scheduledAt: null, + batchSize: 500, + lastCursor: null, + content: null, + html: "

Hello

", + from: "news@example.com", + subject: "Hello", + replyTo: [], + cc: [], + bcc: [], + previewText: null, + }); + mockDb.contact.findMany.mockResolvedValue(contacts); + mockDb.contactBook.findUnique.mockResolvedValue({ variables: [] }); + mockDb.domain.findUnique.mockResolvedValue({ id: 3, region: "us-east-1" }); + // The failed run already emailed contact_1 before it stopped. + mockDb.campaignEmail.findMany.mockResolvedValue([ + { contactId: "contact_1" }, + ]); + mockCheckSuppressions.mockResolvedValue({}); + mockDb.email.create.mockImplementation(async ({ data }: any) => ({ + id: `email_${data.contactId}`, + })); + mockDb.campaignEmail.create.mockResolvedValue({}); + mockDb.campaign.update.mockResolvedValue({}); + mockQueueEmail.mockResolvedValue(undefined); + }); + + it("emails only the contacts that weren't emailed yet, once each, then moves on", async () => { + expect(capturedBatchHandler.handler).toBeTypeOf("function"); + + await capturedBatchHandler.handler({ + data: { campaignId: "campaign_1", teamId: 7 }, + }); + + expect(mockDb.email.create).toHaveBeenCalledTimes(2); + expect( + mockDb.email.create.mock.calls.map(([arg]: any) => arg.data.contactId), + ).toEqual(["contact_2", "contact_3"]); + expect(mockQueueEmail).toHaveBeenCalledTimes(2); + expect(mockQueueEmail.mock.calls.map(([emailId]) => emailId)).toEqual([ + "email_contact_2", + "email_contact_3", + ]); + expect(mockDb.campaign.update).toHaveBeenCalledWith({ + where: { id: "campaign_1" }, + data: { lastCursor: "contact_3", lastSentAt: expect.any(Date) }, + }); + }); + + it("never emails anyone twice when a batch fails after sending and runs again", async () => { + // A stand-in for the database: remembers which contacts got their email. + const linked = new Set(); + mockDb.campaignEmail.findMany.mockImplementation(async ({ where }: any) => + where.contactId.in + .filter((id: string) => linked.has(id)) + .map((contactId: string) => ({ contactId })), + ); + mockDb.campaignEmail.create.mockImplementation(async ({ data }: any) => { + linked.add(data.contactId); + return {}; + }); + // The first run fails at the very end, when saving its progress. + mockDb.campaign.update + .mockRejectedValueOnce(new Error("connection reset")) + .mockResolvedValue({}); + + const job = { data: { campaignId: "campaign_1", teamId: 7 } }; + await expect(capturedBatchHandler.handler(job)).rejects.toThrow( + "connection reset", + ); + expect(mockQueueEmail).toHaveBeenCalledTimes(3); + + // The retry starts from the same (unsaved) position. + await capturedBatchHandler.handler(job); + + expect(mockQueueEmail).toHaveBeenCalledTimes(3); + expect(mockDb.email.create).toHaveBeenCalledTimes(3); + expect(mockDb.campaign.update).toHaveBeenLastCalledWith({ + where: { id: "campaign_1" }, + data: { lastCursor: "contact_3", lastSentAt: expect.any(Date) }, + }); + }); }); describe("campaign contact subscription changes", () => { diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index 37e83d6e..269a0f87 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -768,7 +768,8 @@ describe("WebhookService.retryCall", () => { it("moves a call's failed job back to the queue with its attempts reset", async () => { const failedJob = { - getState: vi.fn().mockResolvedValue("failed"), + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn().mockResolvedValue(undefined), }; mockQueueGetJob.mockResolvedValue(failedJob); @@ -801,7 +802,8 @@ describe("WebhookService.retryCall", () => { it("leaves a call that is already queued or running alone", async () => { const delayedJob = { - getState: vi.fn().mockResolvedValue("delayed"), + isFailed: vi.fn().mockResolvedValue(false), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn(), }; mockQueueGetJob.mockResolvedValue(delayedJob); @@ -809,7 +811,13 @@ describe("WebhookService.retryCall", () => { await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); expect(delayedJob.retry).not.toHaveBeenCalled(); - expect(mockQueueAdd).not.toHaveBeenCalled(); + // Same job id: BullMQ ignores the add while the job exists (see the + // requeueJob integration tests). + expect(mockQueueAdd).toHaveBeenCalledWith( + "call_123", + { callId: "call_123", teamId: 77 }, + { jobId: "call_123" }, + ); }); it("puts the call back as it was when it can't be queued", async () => { @@ -826,7 +834,8 @@ describe("WebhookService.retryCall", () => { }; mockDb.webhookCall.findFirst.mockResolvedValue(failedCall); mockQueueGetJob.mockResolvedValue({ - getState: vi.fn().mockResolvedValue("failed"), + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn().mockRejectedValue(new Error("Missing key for job")), });