diff --git a/apps/web/src/server/queue/requeue-job.integration.test.ts b/apps/web/src/server/queue/requeue-job.integration.test.ts new file mode 100644 index 000000000..34ad8b40c --- /dev/null +++ b/apps/web/src/server/queue/requeue-job.integration.test.ts @@ -0,0 +1,175 @@ +import { Queue, Worker } from "bullmq"; +import { afterAll, afterEach, beforeEach, describe, expect, it } from "vitest"; +import { DEFAULT_QUEUE_OPTIONS } from "~/server/queue/queue-constants"; +import { requeueJob } from "~/server/queue/requeue-job"; +import { getRedis } from "~/server/redis"; +import { + closeIntegrationConnections, + integrationEnabled, + resetRedis, +} from "~/test/integration/helpers"; + +const describeIntegration = integrationEnabled ? describe : describe.skip; + +const QUEUE_NAME = "requeue-job-integration"; +const PREFIX = "requeue-job-test"; +// Each test polls the queue a few times; leave room for a slow CI runner. +const TEST_TIMEOUT_MS = 15_000; + +type Data = { callId: string }; + +async function waitFor(check: () => Promise, timeoutMs = 4000) { + const end = Date.now() + timeoutMs; + while (Date.now() < end) { + if (await check()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("Timed out waiting for the queue"); +} + +describeIntegration("requeueJob with real BullMQ and Redis", () => { + let queue: Queue; + let worker: Worker | undefined; + // Attempt numbers seen by the worker, per run (attemptsMade + 1). + let attempts: number[]; + let shouldFail: boolean; + + function startWorker(handler?: () => Promise): Worker { + worker = new Worker( + QUEUE_NAME, + async (job) => { + attempts.push(job.attemptsMade + 1); + if (handler) return handler(); + if (shouldFail) throw new Error("endpoint returned 500"); + }, + { connection: getRedis(), prefix: PREFIX }, + ); + return worker; + } + + const stateOf = async (jobId: string) => + (await queue.getJob(jobId))?.getState() ?? "missing"; + + beforeEach(async () => { + await resetRedis(); + attempts = []; + shouldFail = true; + queue = new Queue(QUEUE_NAME, { + connection: getRedis(), + prefix: PREFIX, + defaultJobOptions: { + ...DEFAULT_QUEUE_OPTIONS, + attempts: 2, + backoff: { type: "fixed", delay: 10 }, + }, + }); + }); + + afterEach(async () => { + await worker?.close(); + worker = undefined; + await queue.close(); + }); + + afterAll(async () => { + await closeIntegrationConnections(); + }); + + it( + "runs a failed job again with its attempts reset, where a plain add() does nothing", + async () => { + startWorker(); + await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); + await waitFor(async () => (await stateOf("call_1")) === "failed"); + expect(attempts).toEqual([1, 2]); + + // The bug: the failed job is kept under the same id, so add() is ignored. + await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1, 2]); + expect(await stateOf("call_1")).toBe("failed"); + + shouldFail = false; + await expect( + requeueJob(queue, "call_1", { callId: "call_1" }, { jobId: "call_1" }), + ).resolves.toBe("retried"); + + // Completed jobs are removed (removeOnComplete), so the job disappears. + await waitFor(async () => (await stateOf("call_1")) === "missing"); + expect(attempts).toEqual([1, 2, 1]); + }, + TEST_TIMEOUT_MS, + ); + + it( + "adds a new job when none exists", + async () => { + shouldFail = false; + startWorker(); + + await expect( + requeueJob(queue, "call_2", { callId: "call_2" }, { jobId: "call_2" }), + ).resolves.toBe("added"); + + await waitFor(async () => attempts.length === 1); + await waitFor(async () => (await stateOf("call_2")) === "missing"); + expect(attempts).toEqual([1]); + }, + TEST_TIMEOUT_MS, + ); + + it( + "leaves a job that is waiting for its next attempt alone", + async () => { + startWorker(); + await queue.add( + "call_3", + { callId: "call_3" }, + { jobId: "call_3", backoff: { type: "fixed", delay: 60_000 } }, + ); + await waitFor(async () => (await stateOf("call_3")) === "delayed"); + + await expect( + requeueJob(queue, "call_3", { callId: "call_3" }, { jobId: "call_3" }), + ).resolves.toBe("already-queued"); + + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("call_3")).toBe("delayed"); + }, + TEST_TIMEOUT_MS, + ); + + it( + "leaves a running job alone and never runs it twice at once", + async () => { + let release: () => void = () => undefined; + const running = new Promise((resolve) => { + release = resolve; + }); + startWorker(() => running); + + // Always let the job finish, so closing the worker can't hang the suite. + try { + await queue.add("call_4", { callId: "call_4" }, { jobId: "call_4" }); + await waitFor(async () => (await stateOf("call_4")) === "active"); + + await expect( + requeueJob( + queue, + "call_4", + { callId: "call_4" }, + { jobId: "call_4" }, + ), + ).resolves.toBe("already-queued"); + } finally { + release(); + } + + await waitFor(async () => (await stateOf("call_4")) === "missing"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + }, + TEST_TIMEOUT_MS, + ); +}); diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts new file mode 100644 index 000000000..6a9e437da --- /dev/null +++ b/apps/web/src/server/queue/requeue-job.ts @@ -0,0 +1,52 @@ +import type { JobsOptions, Queue } from "bullmq"; + +export type RequeueResult = "retried" | "added" | "already-queued"; + +/** + * Runs a job again under a fixed job id. + * + * BullMQ ignores `add()` when a job with the same id already exists, in any + * state, and our queues keep failed jobs for 30 days. So a plain `add()` after + * a failure silently does nothing. This helper: + * - moves a retained failed (or kept completed) job back to waiting, with its + * attempt count reset, in one atomic step; + * - leaves a job that is still waiting, delayed or running alone, since it will + * run anyway; + * - otherwise adds a new job. + * + * Use it only where a job is deliberately run again (for example a user pressing + * Retry), not on hot paths: it costs an extra Redis round trip. + * + * The result says what this call did, not that the job will run: if the job + * changes state at the same moment (another retry, or the 30-day cleanup), + * `retry()` throws, and the caller should let the user try again. It does not + * repair queue data that is out of sync. + */ +export async function requeueJob( + queue: Queue, + name: string, + data: Data, + opts: JobsOptions & { jobId: string }, +): Promise { + const existing = await queue.getJob(opts.jobId); + + if (existing) { + const state = await existing.getState(); + + if (state === "failed" || state === "completed") { + await existing.retry(state, { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + return "retried"; + } + + if (state !== "unknown") { + return "already-queued"; + } + } + + // Queue's name and data generics are wider than this helper needs. + await (queue as Queue).add(name, data, opts); + return "added"; +} diff --git a/apps/web/src/server/service/webhook-service.ts b/apps/web/src/server/service/webhook-service.ts index 88eb51a4e..b11e07241 100644 --- a/apps/web/src/server/service/webhook-service.ts +++ b/apps/web/src/server/service/webhook-service.ts @@ -16,6 +16,7 @@ import { WEBHOOK_DISPATCH_QUEUE, } from "../queue/queue-constants"; import { createWorkerHandler, TeamJob } from "../queue/bullmq-context"; +import { requeueJob } from "../queue/requeue-job"; import { logger } from "../logger/log"; import { LimitService } from "./limit-service"; import { withPlanLimitLock } from "./plan-limit-lock"; @@ -89,6 +90,20 @@ export class WebhookQueueService { { jobId: callId }, ); } + + /** + * Queues a call again when a user presses Retry. A plain add() would do + * nothing after the call used up its attempts, because the failed job is kept + * under the same id (see requeueJob). + */ + public static async requeueCall(callId: string, teamId: number) { + return requeueJob( + this.queue, + callId, + { callId, teamId }, + { jobId: callId }, + ); + } } export class WebhookService { @@ -188,7 +203,31 @@ export class WebhookService { }, }); - await WebhookQueueService.enqueueCall(call.id, params.teamId); + try { + await WebhookQueueService.requeueCall(call.id, params.teamId); + } catch (error) { + // Nothing was queued: put the call back as it was, so it still shows as + // failed and can be retried again from the dashboard. Only undo our own + // reset (PENDING, attempt 0): once a worker picks the call up it sets + // attempt >= 1, and its result must not be overwritten. + await db.webhookCall.updateMany({ + where: { + id: call.id, + status: WebhookCallStatus.PENDING, + attempt: 0, + }, + data: { + status: call.status, + attempt: call.attempt, + nextAttemptAt: call.nextAttemptAt, + lastError: call.lastError, + responseStatus: call.responseStatus, + responseTimeMs: call.responseTimeMs, + responseText: call.responseText, + }, + }); + throw error; + } return call.id; } diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index 0fd352e49..37e83d6ef 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -7,6 +7,7 @@ const { mockLimitService, mockLogger, mockQueueAdd, + mockQueueGetJob, mockRedis, mockTxWebhookUpdate, } = vi.hoisted(() => ({ @@ -31,6 +32,7 @@ const { findMany: vi.fn(), findUnique: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), }, }, mockLimitService: { @@ -43,6 +45,7 @@ const { warn: vi.fn(), }, mockQueueAdd: vi.fn(), + mockQueueGetJob: vi.fn(), mockRedis: { eval: vi.fn(), set: vi.fn(), @@ -53,6 +56,7 @@ const { vi.mock("bullmq", () => ({ Queue: class { public add = mockQueueAdd; + public getJob = mockQueueGetJob; }, Worker: class { public on = vi.fn(); @@ -734,3 +738,155 @@ describe("WebhookService.emit domain filters", () => { expect(mockQueueAdd).toHaveBeenCalledTimes(2); }); }); + +describe("WebhookService.retryCall", () => { + const resetRow = { + status: WebhookCallStatus.PENDING, + attempt: 0, + nextAttemptAt: null, + lastError: null, + responseStatus: null, + responseTimeMs: null, + responseText: null, + }; + + beforeEach(() => { + mockDb.webhookCall.findFirst.mockReset(); + mockDb.webhookCall.update.mockReset(); + mockQueueAdd.mockReset(); + mockQueueGetJob.mockReset(); + mockDb.webhookCall.updateMany.mockReset(); + mockDb.webhookCall.updateMany.mockResolvedValue({ count: 1 }); + + mockDb.webhookCall.findFirst.mockResolvedValue({ + id: "call_123", + teamId: 77, + }); + mockDb.webhookCall.update.mockResolvedValue({}); + mockQueueAdd.mockResolvedValue(undefined); + }); + + it("moves a call's failed job back to the queue with its attempts reset", async () => { + const failedJob = { + getState: vi.fn().mockResolvedValue("failed"), + retry: vi.fn().mockResolvedValue(undefined), + }; + mockQueueGetJob.mockResolvedValue(failedJob); + + await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); + + expect(mockDb.webhookCall.update).toHaveBeenCalledWith({ + where: { id: "call_123" }, + data: resetRow, + }); + expect(mockQueueGetJob).toHaveBeenCalledWith("call_123"); + expect(failedJob.retry).toHaveBeenCalledWith("failed", { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("queues the call when no job is kept for it", async () => { + mockQueueGetJob.mockResolvedValue(undefined); + + await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); + + expect(mockQueueAdd).toHaveBeenCalledWith( + "call_123", + { callId: "call_123", teamId: 77 }, + { jobId: "call_123" }, + ); + }); + + it("leaves a call that is already queued or running alone", async () => { + const delayedJob = { + getState: vi.fn().mockResolvedValue("delayed"), + retry: vi.fn(), + }; + mockQueueGetJob.mockResolvedValue(delayedJob); + + await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); + + expect(delayedJob.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("puts the call back as it was when it can't be queued", async () => { + const failedCall = { + id: "call_123", + teamId: 77, + status: WebhookCallStatus.FAILED, + attempt: 6, + nextAttemptAt: null, + lastError: "HTTP 500", + responseStatus: 500, + responseTimeMs: 120, + responseText: "oops", + }; + mockDb.webhookCall.findFirst.mockResolvedValue(failedCall); + mockQueueGetJob.mockResolvedValue({ + getState: vi.fn().mockResolvedValue("failed"), + retry: vi.fn().mockRejectedValue(new Error("Missing key for job")), + }); + + await expect( + WebhookService.retryCall({ callId: "call_123", teamId: 77 }), + ).rejects.toThrow("Missing key for job"); + + expect(mockDb.webhookCall.update).toHaveBeenNthCalledWith(1, { + where: { id: "call_123" }, + data: resetRow, + }); + expect(mockDb.webhookCall.update).toHaveBeenCalledTimes(1); + expect(mockDb.webhookCall.updateMany).toHaveBeenCalledWith({ + where: { + id: "call_123", + status: WebhookCallStatus.PENDING, + attempt: 0, + }, + data: { + status: WebhookCallStatus.FAILED, + attempt: 6, + nextAttemptAt: null, + lastError: "HTTP 500", + responseStatus: 500, + responseTimeMs: 120, + responseText: "oops", + }, + }); + }); + + it("rejects a retry for a call that is not on the team", async () => { + mockDb.webhookCall.findFirst.mockResolvedValue(null); + + await expect( + WebhookService.retryCall({ callId: "missing", teamId: 77 }), + ).rejects.toThrow("Webhook call not found"); + + expect(mockQueueGetJob).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("does not look up existing jobs for new calls", async () => { + mockDb.webhook.findMany.mockResolvedValue([ + { id: "wh_1", teamId: 77, status: WebhookStatus.ACTIVE }, + ]); + mockDb.webhookCall.create.mockResolvedValue({ id: "call_new" }); + + await WebhookService.emit(77, "contact.created", { + id: "contact_1", + email: "test@example.com", + contactBookId: "book_1", + subscribed: true, + properties: {}, + firstName: null, + lastName: null, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }); + + expect(mockQueueAdd).toHaveBeenCalledTimes(1); + expect(mockQueueGetJob).not.toHaveBeenCalled(); + }); +});