From c462f2ddb35c2990c3c04ed14ce17ef0cf8b1298 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:23:28 +1100 Subject: [PATCH 1/5] fix(web): keep a campaign sending after one of its batches fails Campaign batches use a fixed BullMQ job id, and failed jobs are kept for 30 days. BullMQ ignores add() for an id that exists, so after one failed batch the scheduler could never queue the next one: the campaign stayed RUNNING and its remaining recipients never got the email. Nothing was logged, because the batch worker had no failed handler. - queueBatch uses requeueJob: a failed batch is retried by the scheduler after a minute (not on every 1.5s tick), and a batch that failed more than a day ago is not retried automatically, so an old campaign never starts sending again by surprise. - Resuming a campaign clears a kept failed batch, so the owner can still continue an old one. - A failed batch is logged as an error. - requeueJob gains retryFailedAfterMs and maxFailedAgeMs options; webhook Retry doesn't use them, so its behaviour is unchanged. Supersedes #517. Co-authored-by: Polylane --- .../queue/requeue-job.integration.test.ts | 70 ++++++++++ apps/web/src/server/queue/requeue-job.ts | 51 ++++++- .../src/server/service/campaign-service.ts | 54 +++++++- .../service/campaign-service.unit.test.ts | 125 +++++++++++++++++- 4 files changed, 289 insertions(+), 11 deletions(-) diff --git a/apps/web/src/server/queue/requeue-job.integration.test.ts b/apps/web/src/server/queue/requeue-job.integration.test.ts index 34ad8b40c..6272925ff 100644 --- a/apps/web/src/server/queue/requeue-job.integration.test.ts +++ b/apps/web/src/server/queue/requeue-job.integration.test.ts @@ -101,6 +101,76 @@ describeIntegration("requeueJob with real BullMQ and Redis", () => { TEST_TIMEOUT_MS, ); + it( + "waits before retrying a job that failed moments ago, then retries it", + async () => { + startWorker(); + await queue.add( + "batch_1", + { callId: "batch_1" }, + { jobId: "batch_1", attempts: 1 }, + ); + await waitFor(async () => (await stateOf("batch_1")) === "failed"); + expect(attempts).toEqual([1]); + + shouldFail = false; + await expect( + requeueJob( + queue, + "batch_1", + { callId: "batch_1" }, + { jobId: "batch_1" }, + { retryFailedAfterMs: 60_000 }, + ), + ).resolves.toBe("failed-too-recently"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("batch_1")).toBe("failed"); + + await expect( + requeueJob( + queue, + "batch_1", + { callId: "batch_1" }, + { jobId: "batch_1" }, + { retryFailedAfterMs: 0 }, + ), + ).resolves.toBe("retried"); + await waitFor(async () => (await stateOf("batch_1")) === "missing"); + expect(attempts).toEqual([1, 1]); + }, + TEST_TIMEOUT_MS, + ); + + it( + "leaves a job that failed too long ago alone", + async () => { + startWorker(); + await queue.add( + "batch_2", + { callId: "batch_2" }, + { jobId: "batch_2", attempts: 1 }, + ); + await waitFor(async () => (await stateOf("batch_2")) === "failed"); + await new Promise((resolve) => setTimeout(resolve, 50)); + + shouldFail = false; + await expect( + requeueJob( + queue, + "batch_2", + { callId: "batch_2" }, + { jobId: "batch_2" }, + { maxFailedAgeMs: 10 }, + ), + ).resolves.toBe("failed-too-long-ago"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("batch_2")).toBe("failed"); + }, + TEST_TIMEOUT_MS, + ); + it( "adds a new job when none exists", async () => { diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index 6a9e437da..a32f229f2 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -1,6 +1,22 @@ -import type { JobsOptions, Queue } from "bullmq"; +import type { Job, JobsOptions, Queue } from "bullmq"; -export type RequeueResult = "retried" | "added" | "already-queued"; +export type RequeueResult = + | "retried" + | "added" + | "already-queued" + | "failed-too-recently" + | "failed-too-long-ago"; + +// A Queue's type argument is either the job data or a Job type (as BullMQ allows). +type QueueData = + DataOrJob extends Job ? Data : DataOrJob; + +export type RequeueOptions = { + /** Leave a job that failed less than this long ago alone, for now. */ + retryFailedAfterMs?: number; + /** Leave a job that failed more than this long ago alone. */ + maxFailedAgeMs?: number; +}; /** * Runs a job again under a fixed job id. @@ -14,25 +30,46 @@ export type RequeueResult = "retried" | "added" | "already-queued"; * run anyway; * - otherwise adds a new job. * + * A failed job can be left alone for a while (`retryFailedAfterMs`) or for good + * once it is old (`maxFailedAgeMs`). Without options it is retried at once. + * * Use it only where a job is deliberately run again (for example a user pressing - * Retry), not on hot paths: it costs an extra Redis round trip. + * Retry, or a scheduler continuing work), not on hot paths: it costs an extra + * Redis round trip. * * The result says what this call did, not that the job will run: if the job * changes state at the same moment (another retry, or the 30-day cleanup), * `retry()` throws, and the caller should let the user try again. It does not * repair queue data that is out of sync. */ -export async function requeueJob( - queue: Queue, +export async function requeueJob( + queue: Queue, name: string, - data: Data, + data: QueueData, opts: JobsOptions & { jobId: string }, + options: RequeueOptions = {}, ): Promise { const existing = await queue.getJob(opts.jobId); if (existing) { const state = await existing.getState(); + if (state === "failed") { + const failedForMs = Date.now() - (existing.finishedOn ?? 0); + if ( + options.retryFailedAfterMs !== undefined && + failedForMs < options.retryFailedAfterMs + ) { + return "failed-too-recently"; + } + if ( + options.maxFailedAgeMs !== undefined && + failedForMs > options.maxFailedAgeMs + ) { + return "failed-too-long-ago"; + } + } + if (state === "failed" || state === "completed") { await existing.retry(state, { resetAttemptsMade: true, @@ -46,7 +83,7 @@ export async function requeueJob( } } - // Queue's name and data generics are wider than this helper needs. + // Queue's name and data generics are wider than this helper needs. await (queue as Queue).add(name, data, opts); return "added"; } diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index da5d7cd98..e13d48283 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -17,6 +17,7 @@ import { } from "../queue/queue-constants"; import { logger } from "../logger/log"; import { createWorkerHandler, TeamJob } from "../queue/bullmq-context"; +import { requeueJob } from "../queue/requeue-job"; import { SuppressionService } from "./suppression-service"; import { UnsendApiError } from "../public-api/api-error"; import { @@ -522,6 +523,10 @@ export async function resumeCampaign({ }); } + // Resuming is an explicit choice, so it also clears a batch that failed long + // ago and that the scheduler no longer retries on its own. + await CampaignBatchService.clearFailedBatch(campaignId); + if (campaign.scheduledAt && campaign.scheduledAt.getTime() > Date.now()) { await db.campaign.update({ where: { id: campaignId }, @@ -1058,6 +1063,16 @@ export async function updateCampaignAnalytics( type CampaignBatchJob = TeamJob<{ campaignId: string }>; +// A failed batch is retried by the scheduler after a short wait, so a batch +// that keeps failing doesn't run on every 1.5s scheduler tick. +const CAMPAIGN_BATCH_RETRY_AFTER_MS = 60 * 1000; +// A batch that failed longer ago than this is not retried automatically, so an +// old campaign never starts sending again by surprise. Resuming it does. +const CAMPAIGN_BATCH_MAX_AUTO_RETRY_AGE_MS = 24 * 60 * 60 * 1000; + +const campaignBatchJobId = (campaignId: string) => + `campaign-batch-${campaignId}`; + export class CampaignBatchService { private static batchQueue = new Queue( CAMPAIGN_BATCH_QUEUE, @@ -1212,6 +1227,27 @@ export class CampaignBatchService { }, ); + static { + this.worker.on("failed", (job, err) => { + logger.error( + { err, campaignId: job?.data?.campaignId, jobId: job?.id }, + "[CampaignBatchService]: Campaign batch failed; the scheduler will retry it", + ); + }); + } + + /** Removes a kept failed batch job for a campaign, if there is one. */ + static async clearFailedBatch(campaignId: string) { + const job = await this.batchQueue.getJob(campaignBatchJobId(campaignId)); + if (job && (await job.isFailed())) { + await job.remove(); + logger.info( + { campaignId }, + "[CampaignBatchService]: Cleared a failed batch on resume", + ); + } + } + static async queueBatch({ campaignId, teamId, @@ -1246,10 +1282,24 @@ export class CampaignBatchService { ); } - await this.batchQueue.add( + // A plain add() would do nothing after a failed batch: BullMQ keeps the + // failed job under this fixed id for 30 days, and the campaign would stop. + const result = await requeueJob( + this.batchQueue, `campaign-${campaignId}`, { campaignId, teamId }, - { jobId: `campaign-batch-${campaignId}`, ...DEFAULT_QUEUE_OPTIONS }, + { jobId: campaignBatchJobId(campaignId), ...DEFAULT_QUEUE_OPTIONS }, + { + retryFailedAfterMs: CAMPAIGN_BATCH_RETRY_AFTER_MS, + maxFailedAgeMs: CAMPAIGN_BATCH_MAX_AUTO_RETRY_AGE_MS, + }, ); + + if (result === "retried") { + logger.warn( + { campaignId }, + "[CampaignBatchService]: Retrying a failed campaign batch", + ); + } } } diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index 78a0062c1..0781159ff 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -36,8 +36,9 @@ const { mockDb, mockTx, mockUpdateContactSubscription } = vi.hoisted(() => { }; }); -const { mockQueueAdd } = vi.hoisted(() => ({ +const { mockQueueAdd, mockQueueGetJob } = vi.hoisted(() => ({ mockQueueAdd: vi.fn(), + mockQueueGetJob: vi.fn(), })); vi.mock("~/server/db", () => ({ @@ -61,8 +62,11 @@ vi.mock("~/server/service/contact-service", () => ({ vi.mock("bullmq", () => ({ Queue: class { add = mockQueueAdd; + getJob = mockQueueGetJob; + }, + Worker: class { + on = vi.fn(); }, - Worker: class {}, })); vi.mock("~/server/redis", () => ({ @@ -99,6 +103,7 @@ vi.mock("~/server/logger/log", () => ({ import { CampaignBatchService, recordCampaignContactFailure, + resumeCampaign, subscribeContact, unsubscribeContact, } from "~/server/service/campaign-service"; @@ -225,6 +230,122 @@ describe("CampaignBatchService", () => { expect.objectContaining({ jobId: "campaign-batch-campaign_1" }), ); }); + + describe("after a batch failed", () => { + const failedJob = (failedSecondsAgo: number) => ({ + getState: vi.fn().mockResolvedValue("failed"), + finishedOn: Date.now() - failedSecondsAgo * 1000, + retry: vi.fn().mockResolvedValue(undefined), + }); + + beforeEach(() => { + mockDb.campaign.findUnique.mockResolvedValue({ + lastSentAt: null, + batchWindowMinutes: 0, + status: "RUNNING", + }); + }); + + it("retries the batch a minute later, with its attempts reset", async () => { + const job = failedJob(61); + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(mockQueueGetJob).toHaveBeenCalledWith("campaign-batch-campaign_1"); + expect(job.retry).toHaveBeenCalledWith("failed", { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("waits before retrying a batch that just failed", async () => { + const job = failedJob(10); + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(job.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("does not restart a campaign whose batch failed over a day ago", async () => { + const job = failedJob(25 * 60 * 60); + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(job.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("does not queue a second copy of a batch that is still running", async () => { + const job = { + getState: vi.fn().mockResolvedValue("active"), + retry: vi.fn(), + }; + mockQueueGetJob.mockResolvedValue(job); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(job.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + }); +}); + +describe("resumeCampaign", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDb.campaign.findUnique.mockResolvedValue({ + id: "campaign_1", + teamId: 7, + scheduledAt: null, + }); + mockDb.campaign.update.mockResolvedValue({}); + }); + + it("clears a failed batch so the scheduler can continue the campaign", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(true), + remove: vi.fn().mockResolvedValue(undefined), + }; + mockQueueGetJob.mockResolvedValue(job); + + await resumeCampaign({ campaignId: "campaign_1", teamId: 7 }); + + expect(mockQueueGetJob).toHaveBeenCalledWith("campaign-batch-campaign_1"); + expect(job.remove).toHaveBeenCalled(); + expect(mockDb.campaign.update).toHaveBeenCalledWith({ + where: { id: "campaign_1" }, + data: { status: "RUNNING" }, + }); + }); + + it("leaves a batch that is still running alone", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(false), + remove: vi.fn(), + }; + mockQueueGetJob.mockResolvedValue(job); + + await resumeCampaign({ campaignId: "campaign_1", teamId: 7 }); + + expect(job.remove).not.toHaveBeenCalled(); + }); }); describe("campaign contact subscription changes", () => { From c9cb830ab68a2585c2ec6ef50452928b23c5c2e0 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:28:06 +1100 Subject: [PATCH 2/5] fix(web): pause a campaign whose batch failed long ago, and check job state without list scans - A batch that failed more than a day ago now pauses its campaign in the database. Skipping it was not enough: BullMQ deletes the failed job after 30 days, and the campaign would then restart on the next tick. - requeueJob checks isFailed()/isCompleted() (single ZSCORE lookups) instead of getState(), which scans the wait and active lists; the campaign scheduler calls it every 1.5s per running campaign. - Tests: the real batch worker, re-run after a part-way failure, emails only the contacts not yet emailed, once each, then advances the cursor; resume keeps the campaign paused if the failed batch can't be cleared. --- apps/web/src/server/queue/requeue-job.ts | 33 +++-- .../src/server/service/campaign-service.ts | 20 ++- .../service/campaign-service.unit.test.ts | 135 +++++++++++++++++- .../service/webhook-service.unit.test.ts | 9 +- 4 files changed, 173 insertions(+), 24 deletions(-) diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index a32f229f2..c4c279c7f 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -52,9 +52,22 @@ export async function requeueJob( const existing = await queue.getJob(opts.jobId); if (existing) { - const state = await existing.getState(); + // isFailed() and isCompleted() are single ZSCORE lookups. getState() would + // also scan the wait and active lists, which is too costly for callers that + // run on every scheduler tick. + const finishedState = (await existing.isFailed()) + ? "failed" + : (await existing.isCompleted()) + ? "completed" + : null; - if (state === "failed") { + if (!finishedState) { + // Still waiting, delayed or running: it will run anyway, and add() would + // be ignored for this id. + return "already-queued"; + } + + if (finishedState === "failed") { const failedForMs = Date.now() - (existing.finishedOn ?? 0); if ( options.retryFailedAfterMs !== undefined && @@ -70,17 +83,11 @@ export async function requeueJob( } } - if (state === "failed" || state === "completed") { - await existing.retry(state, { - resetAttemptsMade: true, - resetAttemptsStarted: true, - }); - return "retried"; - } - - if (state !== "unknown") { - return "already-queued"; - } + await existing.retry(finishedState, { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + return "retried"; } // Queue's name and data generics are wider than this helper needs. diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index e13d48283..8f63ce5f6 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -1067,7 +1067,8 @@ type CampaignBatchJob = TeamJob<{ campaignId: string }>; // that keeps failing doesn't run on every 1.5s scheduler tick. const CAMPAIGN_BATCH_RETRY_AFTER_MS = 60 * 1000; // A batch that failed longer ago than this is not retried automatically, so an -// old campaign never starts sending again by surprise. Resuming it does. +// old campaign never starts sending again by surprise: the campaign is paused +// instead, and its owner can resume it. const CAMPAIGN_BATCH_MAX_AUTO_RETRY_AGE_MS = 24 * 60 * 60 * 1000; const campaignBatchJobId = (campaignId: string) => @@ -1231,7 +1232,7 @@ export class CampaignBatchService { this.worker.on("failed", (job, err) => { logger.error( { err, campaignId: job?.data?.campaignId, jobId: job?.id }, - "[CampaignBatchService]: Campaign batch failed; the scheduler will retry it", + "[CampaignBatchService]: Campaign batch failed; the scheduler retries it after a minute if the campaign is still running", ); }); } @@ -1301,5 +1302,20 @@ export class CampaignBatchService { "[CampaignBatchService]: Retrying a failed campaign batch", ); } + + if (result === "failed-too-long-ago") { + // Pause in the database, not just skip: BullMQ deletes the failed job + // after 30 days, and the campaign would then restart on the next tick. + const paused = await db.campaign.updateMany({ + where: { id: campaignId, status: { in: ["SCHEDULED", "RUNNING"] } }, + data: { status: "PAUSED" }, + }); + if (paused.count > 0) { + logger.warn( + { campaignId }, + "[CampaignBatchService]: Paused a campaign whose batch failed over a day ago; resume it to continue", + ); + } + } } } diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index 0781159ff..c0f39efa9 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -26,19 +26,45 @@ const { mockDb, mockTx, mockUpdateContactSubscription } = vi.hoisted(() => { ), contact: { findUnique: vi.fn(), + findMany: vi.fn(), + }, + contactBook: { + findUnique: vi.fn(), + }, + domain: { + findUnique: vi.fn(), }, campaign: { findUnique: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), + }, + campaignEmail: { + findMany: vi.fn(), + create: vi.fn(), + }, + email: { + create: vi.fn(), }, }, mockUpdateContactSubscription: vi.fn(), }; }); -const { mockQueueAdd, mockQueueGetJob } = vi.hoisted(() => ({ +const { + mockQueueAdd, + mockQueueGetJob, + capturedBatchHandler, + mockCheckSuppressions, + mockQueueEmail, +} = vi.hoisted(() => ({ mockQueueAdd: vi.fn(), mockQueueGetJob: vi.fn(), + capturedBatchHandler: { + handler: null as any, + }, + mockCheckSuppressions: vi.fn(), + mockQueueEmail: vi.fn(), })); vi.mock("~/server/db", () => ({ @@ -66,6 +92,10 @@ vi.mock("bullmq", () => ({ }, Worker: class { on = vi.fn(); + constructor(queueName: string, handler: any) { + if (queueName === "campaign-batch") + capturedBatchHandler.handler = handler; + } }, })); @@ -75,7 +105,7 @@ vi.mock("~/server/redis", () => ({ })); vi.mock("~/server/service/email-queue-service", () => ({ - EmailQueueService: {}, + EmailQueueService: { queueEmail: mockQueueEmail }, })); vi.mock("~/server/queue/bullmq-context", () => ({ @@ -83,7 +113,7 @@ vi.mock("~/server/queue/bullmq-context", () => ({ })); vi.mock("~/server/service/suppression-service", () => ({ - SuppressionService: {}, + SuppressionService: { checkMultipleEmails: mockCheckSuppressions }, })); vi.mock("~/server/service/domain-service", () => ({ @@ -233,7 +263,8 @@ describe("CampaignBatchService", () => { describe("after a batch failed", () => { const failedJob = (failedSecondsAgo: number) => ({ - getState: vi.fn().mockResolvedValue("failed"), + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), finishedOn: Date.now() - failedSecondsAgo * 1000, retry: vi.fn().mockResolvedValue(undefined), }); @@ -276,9 +307,10 @@ describe("CampaignBatchService", () => { expect(mockQueueAdd).not.toHaveBeenCalled(); }); - it("does not restart a campaign whose batch failed over a day ago", async () => { + it("pauses a campaign whose batch failed over a day ago instead of restarting it", async () => { const job = failedJob(25 * 60 * 60); mockQueueGetJob.mockResolvedValue(job); + mockDb.campaign.updateMany.mockResolvedValue({ count: 1 }); await CampaignBatchService.queueBatch({ campaignId: "campaign_1", @@ -287,11 +319,19 @@ describe("CampaignBatchService", () => { expect(job.retry).not.toHaveBeenCalled(); expect(mockQueueAdd).not.toHaveBeenCalled(); + expect(mockDb.campaign.updateMany).toHaveBeenCalledWith({ + where: { + id: "campaign_1", + status: { in: ["SCHEDULED", "RUNNING"] }, + }, + data: { status: "PAUSED" }, + }); }); it("does not queue a second copy of a batch that is still running", async () => { const job = { - getState: vi.fn().mockResolvedValue("active"), + isFailed: vi.fn().mockResolvedValue(false), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn(), }; mockQueueGetJob.mockResolvedValue(job); @@ -346,6 +386,89 @@ describe("resumeCampaign", () => { expect(job.remove).not.toHaveBeenCalled(); }); + + it("keeps the campaign paused if the failed batch can't be cleared", async () => { + mockQueueGetJob.mockResolvedValue({ + isFailed: vi.fn().mockResolvedValue(true), + remove: vi.fn().mockRejectedValue(new Error("Job is locked")), + }); + + await expect( + resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), + ).rejects.toThrow("Job is locked"); + + expect(mockDb.campaign.update).not.toHaveBeenCalled(); + }); +}); + +describe("campaign batch worker, re-running a batch that failed part-way", () => { + const contacts = ["contact_1", "contact_2", "contact_3"].map((id) => ({ + id, + email: `${id}@example.com`, + firstName: null, + lastName: null, + properties: {}, + contactBookId: "book_1", + subscribed: true, + })); + + beforeEach(() => { + vi.clearAllMocks(); + mockDb.campaign.findUnique.mockResolvedValue({ + id: "campaign_1", + teamId: 7, + contactBookId: "book_1", + domainId: 3, + status: "RUNNING", + scheduledAt: null, + batchSize: 500, + lastCursor: null, + content: null, + html: "

Hello

", + from: "news@example.com", + subject: "Hello", + replyTo: [], + cc: [], + bcc: [], + previewText: null, + }); + mockDb.contact.findMany.mockResolvedValue(contacts); + mockDb.contactBook.findUnique.mockResolvedValue({ variables: [] }); + mockDb.domain.findUnique.mockResolvedValue({ id: 3, region: "us-east-1" }); + // The failed run already emailed contact_1 before it stopped. + mockDb.campaignEmail.findMany.mockResolvedValue([ + { contactId: "contact_1" }, + ]); + mockCheckSuppressions.mockResolvedValue({}); + mockDb.email.create.mockImplementation(async ({ data }: any) => ({ + id: `email_${data.contactId}`, + })); + mockDb.campaignEmail.create.mockResolvedValue({}); + mockDb.campaign.update.mockResolvedValue({}); + mockQueueEmail.mockResolvedValue(undefined); + }); + + it("emails only the contacts that weren't emailed yet, once each, then moves on", async () => { + expect(capturedBatchHandler.handler).toBeTypeOf("function"); + + await capturedBatchHandler.handler({ + data: { campaignId: "campaign_1", teamId: 7 }, + }); + + expect(mockDb.email.create).toHaveBeenCalledTimes(2); + expect( + mockDb.email.create.mock.calls.map(([arg]: any) => arg.data.contactId), + ).toEqual(["contact_2", "contact_3"]); + expect(mockQueueEmail).toHaveBeenCalledTimes(2); + expect(mockQueueEmail.mock.calls.map(([emailId]) => emailId)).toEqual([ + "email_contact_2", + "email_contact_3", + ]); + expect(mockDb.campaign.update).toHaveBeenCalledWith({ + where: { id: "campaign_1" }, + data: { lastCursor: "contact_3", lastSentAt: expect.any(Date) }, + }); + }); }); describe("campaign contact subscription changes", () => { diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index 37e83d6ef..c4a8d42ef 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -768,7 +768,8 @@ describe("WebhookService.retryCall", () => { it("moves a call's failed job back to the queue with its attempts reset", async () => { const failedJob = { - getState: vi.fn().mockResolvedValue("failed"), + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn().mockResolvedValue(undefined), }; mockQueueGetJob.mockResolvedValue(failedJob); @@ -801,7 +802,8 @@ describe("WebhookService.retryCall", () => { it("leaves a call that is already queued or running alone", async () => { const delayedJob = { - getState: vi.fn().mockResolvedValue("delayed"), + isFailed: vi.fn().mockResolvedValue(false), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn(), }; mockQueueGetJob.mockResolvedValue(delayedJob); @@ -826,7 +828,8 @@ describe("WebhookService.retryCall", () => { }; mockDb.webhookCall.findFirst.mockResolvedValue(failedCall); mockQueueGetJob.mockResolvedValue({ - getState: vi.fn().mockResolvedValue("failed"), + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), retry: vi.fn().mockRejectedValue(new Error("Missing key for job")), }); From a169ea1a237ea3bf443430d41dec9fff65a0e16f Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:31:40 +1100 Subject: [PATCH 3/5] fix(web): let explicit send and schedule clear an old failed batch, and never let a stale pause undo a resume - sendCampaign and scheduleCampaign clear a kept failed batch, like resume, so a campaign the owner just rescheduled isn't paused again. - The scheduler's pause only applies if the campaign's updatedAt is unchanged since it was read, so it can't undo a resume or reschedule made at the same moment; without that read it doesn't pause at all. - requeueJob still sends add() when the job is neither failed nor completed, in case it finished during the checks (BullMQ ignores it while the job exists). - Tests: a batch that fails after sending and runs again emails nobody twice (checked to fail if the already-sent check is removed); schedule clears an old failed batch; no pause without a fresh read. --- apps/web/src/server/queue/requeue-job.ts | 6 +- .../src/server/service/campaign-service.ts | 30 +++++- .../service/campaign-service.unit.test.ts | 96 ++++++++++++++++++- .../service/webhook-service.unit.test.ts | 8 +- 4 files changed, 133 insertions(+), 7 deletions(-) diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index c4c279c7f..9d8fd133b 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -62,8 +62,10 @@ export async function requeueJob( : null; if (!finishedState) { - // Still waiting, delayed or running: it will run anyway, and add() would - // be ignored for this id. + // Still waiting, delayed or running: it will run anyway. add() is still + // called, in case the job finished and was removed during the checks + // above: BullMQ ignores it while a job with this id exists. + await (queue as Queue).add(name, data, opts); return "already-queued"; } diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index 8f63ce5f6..29fb54d40 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -368,6 +368,10 @@ export async function sendCampaign(id: string) { where: { contactBookId: campaign.contactBookId, subscribed: true }, }); + // Sending is an explicit choice: clear a batch that failed long ago, so the + // scheduler doesn't pause the campaign again because of it. + await CampaignBatchService.clearFailedBatch(id); + // Mark as scheduled (or keep running if already running), set totals and scheduledAt if not set await db.campaign.update({ where: { id }, @@ -465,6 +469,10 @@ export async function scheduleCampaign({ const shouldResetCursor = campaign.status === "DRAFT" || campaign.status === "SENT"; + // Scheduling is an explicit choice: clear a batch that failed long ago, so + // the scheduler doesn't pause the campaign again because of it. + await CampaignBatchService.clearFailedBatch(campaign.id); + await db.campaign.update({ where: { id: campaign.id }, data: { @@ -1256,13 +1264,23 @@ export class CampaignBatchService { campaignId: string; teamId?: number; }) { + // When the campaign was last changed, read before looking at the queue. + // Pausing below only applies if nothing changed it since (e.g. a resume). + let seenUpdatedAt: Date | undefined; + // Defensive check: avoid enqueue if window not elapsed (scheduler already enforces) try { const campaign = await db.campaign.findUnique({ where: { id: campaignId }, - select: { lastSentAt: true, batchWindowMinutes: true, status: true }, + select: { + lastSentAt: true, + batchWindowMinutes: true, + status: true, + updatedAt: true, + }, }); if (!campaign) return; + seenUpdatedAt = campaign.updatedAt; if (campaign.status === "PAUSED" || campaign.status === "SENT") return; const windowMin = campaign.batchWindowMinutes ?? 0; if (windowMin > 0 && campaign.lastSentAt) { @@ -1303,11 +1321,17 @@ export class CampaignBatchService { ); } - if (result === "failed-too-long-ago") { + if (result === "failed-too-long-ago" && seenUpdatedAt) { // Pause in the database, not just skip: BullMQ deletes the failed job // after 30 days, and the campaign would then restart on the next tick. + // The updatedAt check keeps this from undoing a resume or reschedule + // that happened while we were looking at the queue. const paused = await db.campaign.updateMany({ - where: { id: campaignId, status: { in: ["SCHEDULED", "RUNNING"] } }, + where: { + id: campaignId, + status: { in: ["SCHEDULED", "RUNNING"] }, + updatedAt: seenUpdatedAt, + }, data: { status: "PAUSED" }, }); if (paused.count > 0) { diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index c0f39efa9..d8b85e1ae 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -27,6 +27,7 @@ const { mockDb, mockTx, mockUpdateContactSubscription } = vi.hoisted(() => { contact: { findUnique: vi.fn(), findMany: vi.fn(), + count: vi.fn(), }, contactBook: { findUnique: vi.fn(), @@ -134,6 +135,7 @@ import { CampaignBatchService, recordCampaignContactFailure, resumeCampaign, + scheduleCampaign, subscribeContact, unsubscribeContact, } from "~/server/service/campaign-service"; @@ -269,11 +271,14 @@ describe("CampaignBatchService", () => { retry: vi.fn().mockResolvedValue(undefined), }); + const seenUpdatedAt = new Date("2026-11-16T10:00:00Z"); + beforeEach(() => { mockDb.campaign.findUnique.mockResolvedValue({ lastSentAt: null, batchWindowMinutes: 0, status: "RUNNING", + updatedAt: seenUpdatedAt, }); }); @@ -319,15 +324,29 @@ describe("CampaignBatchService", () => { expect(job.retry).not.toHaveBeenCalled(); expect(mockQueueAdd).not.toHaveBeenCalled(); + // Only if nothing changed the campaign since it was read (e.g. a resume). expect(mockDb.campaign.updateMany).toHaveBeenCalledWith({ where: { id: "campaign_1", status: { in: ["SCHEDULED", "RUNNING"] }, + updatedAt: seenUpdatedAt, }, data: { status: "PAUSED" }, }); }); + it("does not pause when it couldn't read the campaign first", async () => { + mockDb.campaign.findUnique.mockRejectedValue(new Error("db timeout")); + mockQueueGetJob.mockResolvedValue(failedJob(25 * 60 * 60)); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + it("does not queue a second copy of a batch that is still running", async () => { const job = { isFailed: vi.fn().mockResolvedValue(false), @@ -342,8 +361,49 @@ describe("CampaignBatchService", () => { }); expect(job.retry).not.toHaveBeenCalled(); - expect(mockQueueAdd).not.toHaveBeenCalled(); + // add() is still sent with the same job id, in case the job finished + // during the checks; BullMQ ignores it while the job exists (see the + // requeueJob integration tests). + expect(mockQueueAdd).toHaveBeenCalledWith( + "campaign-campaign_1", + { campaignId: "campaign_1", teamId: 7 }, + expect.objectContaining({ jobId: "campaign-batch-campaign_1" }), + ); + }); + }); +}); + +describe("scheduleCampaign", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDb.campaign.findUnique.mockResolvedValue({ + id: "campaign_1", + teamId: 7, + status: "PAUSED", + contactBookId: "book_1", + content: null, + html: '

Hi Unsubscribe

', }); + mockDb.contact.count.mockResolvedValue(3); + mockDb.campaign.update.mockResolvedValue({}); + }); + + it("clears a batch that failed long ago, so the campaign isn't paused again", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(true), + remove: vi.fn().mockResolvedValue(undefined), + }; + mockQueueGetJob.mockResolvedValue(job); + + await scheduleCampaign({ campaignId: "campaign_1", teamId: 7 }); + + expect(job.remove).toHaveBeenCalled(); + expect(mockDb.campaign.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "campaign_1" }, + data: expect.objectContaining({ status: "SCHEDULED" }), + }), + ); }); }); @@ -469,6 +529,40 @@ describe("campaign batch worker, re-running a batch that failed part-way", () => data: { lastCursor: "contact_3", lastSentAt: expect.any(Date) }, }); }); + + it("never emails anyone twice when a batch fails after sending and runs again", async () => { + // A stand-in for the database: remembers which contacts got their email. + const linked = new Set(); + mockDb.campaignEmail.findMany.mockImplementation(async ({ where }: any) => + where.contactId.in + .filter((id: string) => linked.has(id)) + .map((contactId: string) => ({ contactId })), + ); + mockDb.campaignEmail.create.mockImplementation(async ({ data }: any) => { + linked.add(data.contactId); + return {}; + }); + // The first run fails at the very end, when saving its progress. + mockDb.campaign.update + .mockRejectedValueOnce(new Error("connection reset")) + .mockResolvedValue({}); + + const job = { data: { campaignId: "campaign_1", teamId: 7 } }; + await expect(capturedBatchHandler.handler(job)).rejects.toThrow( + "connection reset", + ); + expect(mockQueueEmail).toHaveBeenCalledTimes(3); + + // The retry starts from the same (unsaved) position. + await capturedBatchHandler.handler(job); + + expect(mockQueueEmail).toHaveBeenCalledTimes(3); + expect(mockDb.email.create).toHaveBeenCalledTimes(3); + expect(mockDb.campaign.update).toHaveBeenLastCalledWith({ + where: { id: "campaign_1" }, + data: { lastCursor: "contact_3", lastSentAt: expect.any(Date) }, + }); + }); }); describe("campaign contact subscription changes", () => { diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index c4a8d42ef..269a0f87f 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -811,7 +811,13 @@ describe("WebhookService.retryCall", () => { await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); expect(delayedJob.retry).not.toHaveBeenCalled(); - expect(mockQueueAdd).not.toHaveBeenCalled(); + // Same job id: BullMQ ignores the add while the job exists (see the + // requeueJob integration tests). + expect(mockQueueAdd).toHaveBeenCalledWith( + "call_123", + { callId: "call_123", teamId: 77 }, + { jobId: "call_123" }, + ); }); it("puts the call back as it was when it can't be queued", async () => { From 9a14383352a1010f8414bace3ae28ebd10e39de5 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:34:06 +1100 Subject: [PATCH 4/5] fix(web): save the new schedule before clearing an old failed batch Clearing first left a moment where the scheduler could start an early batch for a running campaign being rescheduled. Saving the schedule first means the worst case on failure is a paused campaign, not an early send. Resume keeps its order: the campaign is still paused while the failed batch is cleared. --- apps/web/src/server/queue/requeue-job.ts | 9 ++++--- .../src/server/service/campaign-service.ts | 26 +++++++++++-------- .../service/campaign-service.unit.test.ts | 5 ++++ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index 9d8fd133b..0a44a5222 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -37,10 +37,11 @@ export type RequeueOptions = { * Retry, or a scheduler continuing work), not on hot paths: it costs an extra * Redis round trip. * - * The result says what this call did, not that the job will run: if the job - * changes state at the same moment (another retry, or the 30-day cleanup), - * `retry()` throws, and the caller should let the user try again. It does not - * repair queue data that is out of sync. + * The result is a best-effort description, not a delivery guarantee + * ("already-queued" means an existing job was kept, or a vanished one was + * re-added). If the job changes state at the same moment (another retry, or + * the 30-day cleanup), `retry()` throws, and the caller should let the user + * try again. It does not repair queue data that is out of sync. */ export async function requeueJob( queue: Queue, diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index 29fb54d40..84c224e2d 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -368,10 +368,6 @@ export async function sendCampaign(id: string) { where: { contactBookId: campaign.contactBookId, subscribed: true }, }); - // Sending is an explicit choice: clear a batch that failed long ago, so the - // scheduler doesn't pause the campaign again because of it. - await CampaignBatchService.clearFailedBatch(id); - // Mark as scheduled (or keep running if already running), set totals and scheduledAt if not set await db.campaign.update({ where: { id }, @@ -383,6 +379,11 @@ export async function sendCampaign(id: string) { }, }); + // Sending is an explicit choice: clear a batch that failed long ago, so the + // scheduler doesn't pause the campaign because of it (after the update, as + // in scheduleCampaign). + await CampaignBatchService.clearFailedBatch(id); + // Kick off first batch immediately (idempotent by jobId) await CampaignBatchService.queueBatch({ campaignId: id, @@ -469,10 +470,6 @@ export async function scheduleCampaign({ const shouldResetCursor = campaign.status === "DRAFT" || campaign.status === "SENT"; - // Scheduling is an explicit choice: clear a batch that failed long ago, so - // the scheduler doesn't pause the campaign again because of it. - await CampaignBatchService.clearFailedBatch(campaign.id); - await db.campaign.update({ where: { id: campaign.id }, data: { @@ -484,6 +481,12 @@ export async function scheduleCampaign({ }, }); + // Scheduling is an explicit choice: clear a batch that failed long ago, so + // the scheduler doesn't pause the campaign because of it. This runs after the + // new schedule is saved, so the scheduler can't start an early batch in + // between. If it fails, the worst case is the campaign gets paused, not sent. + await CampaignBatchService.clearFailedBatch(campaign.id); + return { ok: true }; } @@ -1265,7 +1268,8 @@ export class CampaignBatchService { teamId?: number; }) { // When the campaign was last changed, read before looking at the queue. - // Pausing below only applies if nothing changed it since (e.g. a resume). + // Pausing below only applies if updatedAt is unchanged since, an optimistic + // guard against undoing a resume or reschedule made at the same moment. let seenUpdatedAt: Date | undefined; // Defensive check: avoid enqueue if window not elapsed (scheduler already enforces) @@ -1324,8 +1328,8 @@ export class CampaignBatchService { if (result === "failed-too-long-ago" && seenUpdatedAt) { // Pause in the database, not just skip: BullMQ deletes the failed job // after 30 days, and the campaign would then restart on the next tick. - // The updatedAt check keeps this from undoing a resume or reschedule - // that happened while we were looking at the queue. + // The updatedAt check guards against undoing a resume or reschedule + // made while we were looking at the queue. const paused = await db.campaign.updateMany({ where: { id: campaignId, diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index d8b85e1ae..9fcde9f67 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -404,6 +404,11 @@ describe("scheduleCampaign", () => { data: expect.objectContaining({ status: "SCHEDULED" }), }), ); + // The new schedule is saved before the failed batch is cleared, so the + // scheduler can't start an early batch in between. + expect(mockDb.campaign.update.mock.invocationCallOrder[0]).toBeLessThan( + job.remove.mock.invocationCallOrder[0]!, + ); }); }); From 09f9f2efb02bdf83684add992b2fbfb238f1d8f4 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:46:11 +1100 Subject: [PATCH 5/5] fix(web): treat a failure without a finish time as fresh, and re-check before pausing a campaign - requeueJob: a job fetched while running that fails just after has no finishedOn on the snapshot; treat it as a fresh failure, never an old one, so it can't pause a campaign by mistake. - queueBatch looks at the failed batch again right before pausing, so a resume, schedule or send that cleared it wins. A clear landing between that check and the update can still pause; nothing is sent in that case. --- apps/web/src/server/queue/requeue-job.ts | 6 ++- .../src/server/queue/requeue-job.unit.test.ts | 46 +++++++++++++++++++ .../src/server/service/campaign-service.ts | 15 +++++- .../service/campaign-service.unit.test.ts | 15 ++++++ 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/server/queue/requeue-job.unit.test.ts diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index 0a44a5222..17f6174ee 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -71,7 +71,11 @@ export async function requeueJob( } if (finishedState === "failed") { - const failedForMs = Date.now() - (existing.finishedOn ?? 0); + // finishedOn is missing if the job was fetched while still running and + // failed just after: treat that as a fresh failure, never an old one. + const failedForMs = existing.finishedOn + ? Date.now() - existing.finishedOn + : 0; if ( options.retryFailedAfterMs !== undefined && failedForMs < options.retryFailedAfterMs diff --git a/apps/web/src/server/queue/requeue-job.unit.test.ts b/apps/web/src/server/queue/requeue-job.unit.test.ts new file mode 100644 index 000000000..b27cc2bb7 --- /dev/null +++ b/apps/web/src/server/queue/requeue-job.unit.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from "vitest"; +import { requeueJob } from "~/server/queue/requeue-job"; + +function queueWith(job: unknown) { + return { + getJob: vi.fn().mockResolvedValue(job), + add: vi.fn().mockResolvedValue(undefined), + } as any; +} + +describe("requeueJob", () => { + it("treats a failed job without a finish time as a fresh failure", async () => { + // Fetched while running, failed just after: finishedOn isn't on the snapshot. + const job = { + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), + finishedOn: undefined, + retry: vi.fn().mockResolvedValue(undefined), + }; + const options = { retryFailedAfterMs: 60_000, maxFailedAgeMs: 86_400_000 }; + + await expect( + requeueJob(queueWith(job), "n", {}, { jobId: "j" }, options), + ).resolves.toBe("failed-too-recently"); + expect(job.retry).not.toHaveBeenCalled(); + }); + + it("never calls a failure with no finish time too old to retry", async () => { + const job = { + isFailed: vi.fn().mockResolvedValue(true), + isCompleted: vi.fn().mockResolvedValue(false), + finishedOn: undefined, + retry: vi.fn().mockResolvedValue(undefined), + }; + + await expect( + requeueJob( + queueWith(job), + "n", + {}, + { jobId: "j" }, + { maxFailedAgeMs: 1 }, + ), + ).resolves.toBe("retried"); + }); +}); diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index 84c224e2d..5c3db73b3 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -1248,6 +1248,11 @@ export class CampaignBatchService { }); } + static async hasFailedBatch(campaignId: string) { + const job = await this.batchQueue.getJob(campaignBatchJobId(campaignId)); + return Boolean(job && (await job.isFailed())); + } + /** Removes a kept failed batch job for a campaign, if there is one. */ static async clearFailedBatch(campaignId: string) { const job = await this.batchQueue.getJob(campaignBatchJobId(campaignId)); @@ -1325,7 +1330,15 @@ export class CampaignBatchService { ); } - if (result === "failed-too-long-ago" && seenUpdatedAt) { + if ( + result === "failed-too-long-ago" && + seenUpdatedAt && + // Check again right before pausing: resume, schedule and send clear the + // failed batch, and that must win. (A clear landing in the moment between + // this check and the update below can still pause the campaign; that is + // safe, as nothing is sent, and the owner can resume it.) + (await this.hasFailedBatch(campaignId)) + ) { // Pause in the database, not just skip: BullMQ deletes the failed job // after 30 days, and the campaign would then restart on the next tick. // The updatedAt check guards against undoing a resume or reschedule diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index 9fcde9f67..dd3fae577 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -335,6 +335,21 @@ describe("CampaignBatchService", () => { }); }); + it("does not pause if the failed batch was cleared in the meantime", async () => { + // queueBatch sees the old failed job, then a reschedule clears it before + // the pause: the second look finds no job. + mockQueueGetJob + .mockResolvedValueOnce(failedJob(25 * 60 * 60)) + .mockResolvedValueOnce(undefined); + + await CampaignBatchService.queueBatch({ + campaignId: "campaign_1", + teamId: 7, + }); + + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + it("does not pause when it couldn't read the campaign first", async () => { mockDb.campaign.findUnique.mockRejectedValue(new Error("db timeout")); mockQueueGetJob.mockResolvedValue(failedJob(25 * 60 * 60));