From d523473cd4a538684c6894851172d7d74de52a0a Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 08:49:10 +1100 Subject: [PATCH 1/4] fix(web): make webhook Retry deliver a call that used up its attempts BullMQ ignores add() when a job with the same id exists, and webhook jobs are kept for 30 days after they fail. So pressing Retry on a call that had used up its attempts reset the row to PENDING but queued nothing. Retry now goes through requeueJob: a kept failed job is moved back to the queue with its attempts reset, in one atomic step; a job that is still waiting, delayed or running is left alone; otherwise a new job is added. New calls keep the plain add(), so the hot path does no extra Redis work. Supersedes #516, which removed the job before every add(), including for every new call, where BullMQ's remove scans the wait list. Co-authored-by: Polylane --- .../queue/requeue-job.integration.test.ts | 148 ++++++++++++++++++ apps/web/src/server/queue/requeue-job.ts | 47 ++++++ .../web/src/server/service/webhook-service.ts | 17 +- .../service/webhook-service.unit.test.ts | 108 +++++++++++++ 4 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/server/queue/requeue-job.integration.test.ts create mode 100644 apps/web/src/server/queue/requeue-job.ts diff --git a/apps/web/src/server/queue/requeue-job.integration.test.ts b/apps/web/src/server/queue/requeue-job.integration.test.ts new file mode 100644 index 000000000..611786278 --- /dev/null +++ b/apps/web/src/server/queue/requeue-job.integration.test.ts @@ -0,0 +1,148 @@ +import { Queue, Worker } from "bullmq"; +import { afterAll, afterEach, beforeEach, describe, expect, it } from "vitest"; +import { DEFAULT_QUEUE_OPTIONS } from "~/server/queue/queue-constants"; +import { requeueJob } from "~/server/queue/requeue-job"; +import { getRedis } from "~/server/redis"; +import { + closeIntegrationConnections, + integrationEnabled, + resetRedis, +} from "~/test/integration/helpers"; + +const describeIntegration = integrationEnabled ? describe : describe.skip; + +const QUEUE_NAME = "requeue-job-integration"; +const PREFIX = "requeue-job-test"; + +type Data = { callId: string }; + +async function waitFor(check: () => Promise, timeoutMs = 4000) { + const end = Date.now() + timeoutMs; + while (Date.now() < end) { + if (await check()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("Timed out waiting for the queue"); +} + +describeIntegration("requeueJob with real BullMQ and Redis", () => { + let queue: Queue; + let worker: Worker | undefined; + // Attempt numbers seen by the worker, per run (attemptsMade + 1). + let attempts: number[]; + let shouldFail: boolean; + + function startWorker(handler?: () => Promise): Worker { + worker = new Worker( + QUEUE_NAME, + async (job) => { + attempts.push(job.attemptsMade + 1); + if (handler) return handler(); + if (shouldFail) throw new Error("endpoint returned 500"); + }, + { connection: getRedis(), prefix: PREFIX }, + ); + return worker; + } + + const stateOf = async (jobId: string) => + (await queue.getJob(jobId))?.getState() ?? "missing"; + + beforeEach(async () => { + await resetRedis(); + attempts = []; + shouldFail = true; + queue = new Queue(QUEUE_NAME, { + connection: getRedis(), + prefix: PREFIX, + defaultJobOptions: { + ...DEFAULT_QUEUE_OPTIONS, + attempts: 2, + backoff: { type: "fixed", delay: 10 }, + }, + }); + }); + + afterEach(async () => { + await worker?.close(); + worker = undefined; + await queue.close(); + }); + + afterAll(async () => { + await closeIntegrationConnections(); + }); + + it("runs a failed job again with its attempts reset, where a plain add() does nothing", async () => { + startWorker(); + await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); + await waitFor(async () => (await stateOf("call_1")) === "failed"); + expect(attempts).toEqual([1, 2]); + + // The bug: the failed job is kept under the same id, so add() is ignored. + await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1, 2]); + expect(await stateOf("call_1")).toBe("failed"); + + shouldFail = false; + await expect( + requeueJob(queue, "call_1", { callId: "call_1" }, { jobId: "call_1" }), + ).resolves.toBe("retried"); + + // Completed jobs are removed (removeOnComplete), so the job disappears. + await waitFor(async () => (await stateOf("call_1")) === "missing"); + expect(attempts).toEqual([1, 2, 1]); + }); + + it("adds a new job when none exists", async () => { + shouldFail = false; + startWorker(); + + await expect( + requeueJob(queue, "call_2", { callId: "call_2" }, { jobId: "call_2" }), + ).resolves.toBe("added"); + + await waitFor(async () => attempts.length === 1); + await waitFor(async () => (await stateOf("call_2")) === "missing"); + expect(attempts).toEqual([1]); + }); + + it("leaves a job that is waiting for its next attempt alone", async () => { + startWorker(); + await queue.add( + "call_3", + { callId: "call_3" }, + { jobId: "call_3", backoff: { type: "fixed", delay: 60_000 } }, + ); + await waitFor(async () => (await stateOf("call_3")) === "delayed"); + + await expect( + requeueJob(queue, "call_3", { callId: "call_3" }, { jobId: "call_3" }), + ).resolves.toBe("already-queued"); + + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("call_3")).toBe("delayed"); + }); + + it("leaves a running job alone and never runs it twice at once", async () => { + let release: () => void = () => undefined; + const running = new Promise((resolve) => { + release = resolve; + }); + startWorker(() => running); + + await queue.add("call_4", { callId: "call_4" }, { jobId: "call_4" }); + await waitFor(async () => (await stateOf("call_4")) === "active"); + + await expect( + requeueJob(queue, "call_4", { callId: "call_4" }, { jobId: "call_4" }), + ).resolves.toBe("already-queued"); + + release(); + await waitFor(async () => (await stateOf("call_4")) === "missing"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + }); +}); diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts new file mode 100644 index 000000000..f8fd44e75 --- /dev/null +++ b/apps/web/src/server/queue/requeue-job.ts @@ -0,0 +1,47 @@ +import type { JobsOptions, Queue } from "bullmq"; + +export type RequeueResult = "retried" | "added" | "already-queued"; + +/** + * Runs a job again under a fixed job id. + * + * BullMQ ignores `add()` when a job with the same id already exists, in any + * state, and our queues keep failed jobs for 30 days. So a plain `add()` after + * a failure silently does nothing. This helper: + * - moves a retained failed (or kept completed) job back to waiting, with its + * attempt count reset, in one atomic step; + * - leaves a job that is still waiting, delayed or running alone, since it will + * run anyway; + * - otherwise adds a new job. + * + * Use it only where a job is deliberately run again (for example a user pressing + * Retry), not on hot paths: it costs an extra Redis round trip. + */ +export async function requeueJob( + queue: Queue, + name: string, + data: Data, + opts: JobsOptions & { jobId: string }, +): Promise { + const existing = await queue.getJob(opts.jobId); + + if (existing) { + const state = await existing.getState(); + + if (state === "failed" || state === "completed") { + await existing.retry(state, { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + return "retried"; + } + + if (state !== "unknown") { + return "already-queued"; + } + } + + // Queue's name and data generics are wider than this helper needs. + await (queue as Queue).add(name, data, opts); + return "added"; +} diff --git a/apps/web/src/server/service/webhook-service.ts b/apps/web/src/server/service/webhook-service.ts index 88eb51a4e..b337135eb 100644 --- a/apps/web/src/server/service/webhook-service.ts +++ b/apps/web/src/server/service/webhook-service.ts @@ -16,6 +16,7 @@ import { WEBHOOK_DISPATCH_QUEUE, } from "../queue/queue-constants"; import { createWorkerHandler, TeamJob } from "../queue/bullmq-context"; +import { requeueJob } from "../queue/requeue-job"; import { logger } from "../logger/log"; import { LimitService } from "./limit-service"; import { withPlanLimitLock } from "./plan-limit-lock"; @@ -89,6 +90,20 @@ export class WebhookQueueService { { jobId: callId }, ); } + + /** + * Queues a call again when a user presses Retry. A plain add() would do + * nothing after the call used up its attempts, because the failed job is kept + * under the same id (see requeueJob). + */ + public static async requeueCall(callId: string, teamId: number) { + return requeueJob( + this.queue, + callId, + { callId, teamId }, + { jobId: callId }, + ); + } } export class WebhookService { @@ -188,7 +203,7 @@ export class WebhookService { }, }); - await WebhookQueueService.enqueueCall(call.id, params.teamId); + await WebhookQueueService.requeueCall(call.id, params.teamId); return call.id; } diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index 0fd352e49..59e084308 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -7,6 +7,7 @@ const { mockLimitService, mockLogger, mockQueueAdd, + mockQueueGetJob, mockRedis, mockTxWebhookUpdate, } = vi.hoisted(() => ({ @@ -43,6 +44,7 @@ const { warn: vi.fn(), }, mockQueueAdd: vi.fn(), + mockQueueGetJob: vi.fn(), mockRedis: { eval: vi.fn(), set: vi.fn(), @@ -53,6 +55,7 @@ const { vi.mock("bullmq", () => ({ Queue: class { public add = mockQueueAdd; + public getJob = mockQueueGetJob; }, Worker: class { public on = vi.fn(); @@ -734,3 +737,108 @@ describe("WebhookService.emit domain filters", () => { expect(mockQueueAdd).toHaveBeenCalledTimes(2); }); }); + +describe("WebhookService.retryCall", () => { + const resetRow = { + status: WebhookCallStatus.PENDING, + attempt: 0, + nextAttemptAt: null, + lastError: null, + responseStatus: null, + responseTimeMs: null, + responseText: null, + }; + + beforeEach(() => { + mockDb.webhookCall.findFirst.mockReset(); + mockDb.webhookCall.update.mockReset(); + mockQueueAdd.mockReset(); + mockQueueGetJob.mockReset(); + + mockDb.webhookCall.findFirst.mockResolvedValue({ + id: "call_123", + teamId: 77, + }); + mockDb.webhookCall.update.mockResolvedValue({}); + mockQueueAdd.mockResolvedValue(undefined); + }); + + it("moves a call's failed job back to the queue with its attempts reset", async () => { + const failedJob = { + getState: vi.fn().mockResolvedValue("failed"), + retry: vi.fn().mockResolvedValue(undefined), + }; + mockQueueGetJob.mockResolvedValue(failedJob); + + await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); + + expect(mockDb.webhookCall.update).toHaveBeenCalledWith({ + where: { id: "call_123" }, + data: resetRow, + }); + expect(mockQueueGetJob).toHaveBeenCalledWith("call_123"); + expect(failedJob.retry).toHaveBeenCalledWith("failed", { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("queues the call when no job is kept for it", async () => { + mockQueueGetJob.mockResolvedValue(undefined); + + await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); + + expect(mockQueueAdd).toHaveBeenCalledWith( + "call_123", + { callId: "call_123", teamId: 77 }, + { jobId: "call_123" }, + ); + }); + + it("leaves a call that is already queued or running alone", async () => { + const delayedJob = { + getState: vi.fn().mockResolvedValue("delayed"), + retry: vi.fn(), + }; + mockQueueGetJob.mockResolvedValue(delayedJob); + + await WebhookService.retryCall({ callId: "call_123", teamId: 77 }); + + expect(delayedJob.retry).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("rejects a retry for a call that is not on the team", async () => { + mockDb.webhookCall.findFirst.mockResolvedValue(null); + + await expect( + WebhookService.retryCall({ callId: "missing", teamId: 77 }), + ).rejects.toThrow("Webhook call not found"); + + expect(mockQueueGetJob).not.toHaveBeenCalled(); + expect(mockQueueAdd).not.toHaveBeenCalled(); + }); + + it("does not look up existing jobs for new calls", async () => { + mockDb.webhook.findMany.mockResolvedValue([ + { id: "wh_1", teamId: 77, status: WebhookStatus.ACTIVE }, + ]); + mockDb.webhookCall.create.mockResolvedValue({ id: "call_new" }); + + await WebhookService.emit(77, "contact.created", { + id: "contact_1", + email: "test@example.com", + contactBookId: "book_1", + subscribed: true, + properties: {}, + firstName: null, + lastName: null, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }); + + expect(mockQueueAdd).toHaveBeenCalledTimes(1); + expect(mockQueueGetJob).not.toHaveBeenCalled(); + }); +}); From f41fc916b0d32dc064ac7f3f2b9f6ae56fe32689 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 08:51:00 +1100 Subject: [PATCH 2/4] test(web): keep the requeue integration tests from hanging or timing out on slow CI --- .../queue/requeue-job.integration.test.ts | 171 ++++++++++-------- apps/web/src/server/queue/requeue-job.ts | 5 + 2 files changed, 104 insertions(+), 72 deletions(-) diff --git a/apps/web/src/server/queue/requeue-job.integration.test.ts b/apps/web/src/server/queue/requeue-job.integration.test.ts index 611786278..34ad8b40c 100644 --- a/apps/web/src/server/queue/requeue-job.integration.test.ts +++ b/apps/web/src/server/queue/requeue-job.integration.test.ts @@ -13,6 +13,8 @@ const describeIntegration = integrationEnabled ? describe : describe.skip; const QUEUE_NAME = "requeue-job-integration"; const PREFIX = "requeue-job-test"; +// Each test polls the queue a few times; leave room for a slow CI runner. +const TEST_TIMEOUT_MS = 15_000; type Data = { callId: string }; @@ -73,76 +75,101 @@ describeIntegration("requeueJob with real BullMQ and Redis", () => { await closeIntegrationConnections(); }); - it("runs a failed job again with its attempts reset, where a plain add() does nothing", async () => { - startWorker(); - await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); - await waitFor(async () => (await stateOf("call_1")) === "failed"); - expect(attempts).toEqual([1, 2]); - - // The bug: the failed job is kept under the same id, so add() is ignored. - await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); - await new Promise((resolve) => setTimeout(resolve, 200)); - expect(attempts).toEqual([1, 2]); - expect(await stateOf("call_1")).toBe("failed"); - - shouldFail = false; - await expect( - requeueJob(queue, "call_1", { callId: "call_1" }, { jobId: "call_1" }), - ).resolves.toBe("retried"); - - // Completed jobs are removed (removeOnComplete), so the job disappears. - await waitFor(async () => (await stateOf("call_1")) === "missing"); - expect(attempts).toEqual([1, 2, 1]); - }); - - it("adds a new job when none exists", async () => { - shouldFail = false; - startWorker(); - - await expect( - requeueJob(queue, "call_2", { callId: "call_2" }, { jobId: "call_2" }), - ).resolves.toBe("added"); - - await waitFor(async () => attempts.length === 1); - await waitFor(async () => (await stateOf("call_2")) === "missing"); - expect(attempts).toEqual([1]); - }); - - it("leaves a job that is waiting for its next attempt alone", async () => { - startWorker(); - await queue.add( - "call_3", - { callId: "call_3" }, - { jobId: "call_3", backoff: { type: "fixed", delay: 60_000 } }, - ); - await waitFor(async () => (await stateOf("call_3")) === "delayed"); - - await expect( - requeueJob(queue, "call_3", { callId: "call_3" }, { jobId: "call_3" }), - ).resolves.toBe("already-queued"); - - await new Promise((resolve) => setTimeout(resolve, 200)); - expect(attempts).toEqual([1]); - expect(await stateOf("call_3")).toBe("delayed"); - }); - - it("leaves a running job alone and never runs it twice at once", async () => { - let release: () => void = () => undefined; - const running = new Promise((resolve) => { - release = resolve; - }); - startWorker(() => running); - - await queue.add("call_4", { callId: "call_4" }, { jobId: "call_4" }); - await waitFor(async () => (await stateOf("call_4")) === "active"); - - await expect( - requeueJob(queue, "call_4", { callId: "call_4" }, { jobId: "call_4" }), - ).resolves.toBe("already-queued"); - - release(); - await waitFor(async () => (await stateOf("call_4")) === "missing"); - await new Promise((resolve) => setTimeout(resolve, 200)); - expect(attempts).toEqual([1]); - }); + it( + "runs a failed job again with its attempts reset, where a plain add() does nothing", + async () => { + startWorker(); + await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); + await waitFor(async () => (await stateOf("call_1")) === "failed"); + expect(attempts).toEqual([1, 2]); + + // The bug: the failed job is kept under the same id, so add() is ignored. + await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" }); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1, 2]); + expect(await stateOf("call_1")).toBe("failed"); + + shouldFail = false; + await expect( + requeueJob(queue, "call_1", { callId: "call_1" }, { jobId: "call_1" }), + ).resolves.toBe("retried"); + + // Completed jobs are removed (removeOnComplete), so the job disappears. + await waitFor(async () => (await stateOf("call_1")) === "missing"); + expect(attempts).toEqual([1, 2, 1]); + }, + TEST_TIMEOUT_MS, + ); + + it( + "adds a new job when none exists", + async () => { + shouldFail = false; + startWorker(); + + await expect( + requeueJob(queue, "call_2", { callId: "call_2" }, { jobId: "call_2" }), + ).resolves.toBe("added"); + + await waitFor(async () => attempts.length === 1); + await waitFor(async () => (await stateOf("call_2")) === "missing"); + expect(attempts).toEqual([1]); + }, + TEST_TIMEOUT_MS, + ); + + it( + "leaves a job that is waiting for its next attempt alone", + async () => { + startWorker(); + await queue.add( + "call_3", + { callId: "call_3" }, + { jobId: "call_3", backoff: { type: "fixed", delay: 60_000 } }, + ); + await waitFor(async () => (await stateOf("call_3")) === "delayed"); + + await expect( + requeueJob(queue, "call_3", { callId: "call_3" }, { jobId: "call_3" }), + ).resolves.toBe("already-queued"); + + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + expect(await stateOf("call_3")).toBe("delayed"); + }, + TEST_TIMEOUT_MS, + ); + + it( + "leaves a running job alone and never runs it twice at once", + async () => { + let release: () => void = () => undefined; + const running = new Promise((resolve) => { + release = resolve; + }); + startWorker(() => running); + + // Always let the job finish, so closing the worker can't hang the suite. + try { + await queue.add("call_4", { callId: "call_4" }, { jobId: "call_4" }); + await waitFor(async () => (await stateOf("call_4")) === "active"); + + await expect( + requeueJob( + queue, + "call_4", + { callId: "call_4" }, + { jobId: "call_4" }, + ), + ).resolves.toBe("already-queued"); + } finally { + release(); + } + + await waitFor(async () => (await stateOf("call_4")) === "missing"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(attempts).toEqual([1]); + }, + TEST_TIMEOUT_MS, + ); }); diff --git a/apps/web/src/server/queue/requeue-job.ts b/apps/web/src/server/queue/requeue-job.ts index f8fd44e75..6a9e437da 100644 --- a/apps/web/src/server/queue/requeue-job.ts +++ b/apps/web/src/server/queue/requeue-job.ts @@ -16,6 +16,11 @@ export type RequeueResult = "retried" | "added" | "already-queued"; * * Use it only where a job is deliberately run again (for example a user pressing * Retry), not on hot paths: it costs an extra Redis round trip. + * + * The result says what this call did, not that the job will run: if the job + * changes state at the same moment (another retry, or the 30-day cleanup), + * `retry()` throws, and the caller should let the user try again. It does not + * repair queue data that is out of sync. */ export async function requeueJob( queue: Queue, From 26eb3f61cba376ad3d3917f4e8a73925e4edc7de Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:00:28 +1100 Subject: [PATCH 3/4] fix(web): keep a webhook call retryable when Retry can't queue it --- .../web/src/server/service/webhook-service.ts | 20 +++++++++- .../service/webhook-service.unit.test.ts | 40 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/apps/web/src/server/service/webhook-service.ts b/apps/web/src/server/service/webhook-service.ts index b337135eb..b660da511 100644 --- a/apps/web/src/server/service/webhook-service.ts +++ b/apps/web/src/server/service/webhook-service.ts @@ -203,7 +203,25 @@ export class WebhookService { }, }); - await WebhookQueueService.requeueCall(call.id, params.teamId); + try { + await WebhookQueueService.requeueCall(call.id, params.teamId); + } catch (error) { + // Nothing was queued: put the call back as it was, so it still shows as + // failed and can be retried again from the dashboard. + await db.webhookCall.update({ + where: { id: call.id }, + data: { + status: call.status, + attempt: call.attempt, + nextAttemptAt: call.nextAttemptAt, + lastError: call.lastError, + responseStatus: call.responseStatus, + responseTimeMs: call.responseTimeMs, + responseText: call.responseText, + }, + }); + throw error; + } return call.id; } diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index 59e084308..d14d14392 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -809,6 +809,46 @@ describe("WebhookService.retryCall", () => { expect(mockQueueAdd).not.toHaveBeenCalled(); }); + it("puts the call back as it was when it can't be queued", async () => { + const failedCall = { + id: "call_123", + teamId: 77, + status: WebhookCallStatus.FAILED, + attempt: 6, + nextAttemptAt: null, + lastError: "HTTP 500", + responseStatus: 500, + responseTimeMs: 120, + responseText: "oops", + }; + mockDb.webhookCall.findFirst.mockResolvedValue(failedCall); + mockQueueGetJob.mockResolvedValue({ + getState: vi.fn().mockResolvedValue("failed"), + retry: vi.fn().mockRejectedValue(new Error("Missing key for job")), + }); + + await expect( + WebhookService.retryCall({ callId: "call_123", teamId: 77 }), + ).rejects.toThrow("Missing key for job"); + + expect(mockDb.webhookCall.update).toHaveBeenNthCalledWith(1, { + where: { id: "call_123" }, + data: resetRow, + }); + expect(mockDb.webhookCall.update).toHaveBeenNthCalledWith(2, { + where: { id: "call_123" }, + data: { + status: WebhookCallStatus.FAILED, + attempt: 6, + nextAttemptAt: null, + lastError: "HTTP 500", + responseStatus: 500, + responseTimeMs: 120, + responseText: "oops", + }, + }); + }); + it("rejects a retry for a call that is not on the team", async () => { mockDb.webhookCall.findFirst.mockResolvedValue(null); From 4120c88c64cf9d5a87bf64d5ddbffa0cb1805d27 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Tue, 17 Nov 2026 09:07:20 +1100 Subject: [PATCH 4/4] fix(web): only undo Retry's own reset when queueing fails --- apps/web/src/server/service/webhook-service.ts | 12 +++++++++--- .../src/server/service/webhook-service.unit.test.ts | 12 ++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/apps/web/src/server/service/webhook-service.ts b/apps/web/src/server/service/webhook-service.ts index b660da511..b11e07241 100644 --- a/apps/web/src/server/service/webhook-service.ts +++ b/apps/web/src/server/service/webhook-service.ts @@ -207,9 +207,15 @@ export class WebhookService { await WebhookQueueService.requeueCall(call.id, params.teamId); } catch (error) { // Nothing was queued: put the call back as it was, so it still shows as - // failed and can be retried again from the dashboard. - await db.webhookCall.update({ - where: { id: call.id }, + // failed and can be retried again from the dashboard. Only undo our own + // reset (PENDING, attempt 0): once a worker picks the call up it sets + // attempt >= 1, and its result must not be overwritten. + await db.webhookCall.updateMany({ + where: { + id: call.id, + status: WebhookCallStatus.PENDING, + attempt: 0, + }, data: { status: call.status, attempt: call.attempt, diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts index d14d14392..37e83d6ef 100644 --- a/apps/web/src/server/service/webhook-service.unit.test.ts +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -32,6 +32,7 @@ const { findMany: vi.fn(), findUnique: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), }, }, mockLimitService: { @@ -754,6 +755,8 @@ describe("WebhookService.retryCall", () => { mockDb.webhookCall.update.mockReset(); mockQueueAdd.mockReset(); mockQueueGetJob.mockReset(); + mockDb.webhookCall.updateMany.mockReset(); + mockDb.webhookCall.updateMany.mockResolvedValue({ count: 1 }); mockDb.webhookCall.findFirst.mockResolvedValue({ id: "call_123", @@ -835,8 +838,13 @@ describe("WebhookService.retryCall", () => { where: { id: "call_123" }, data: resetRow, }); - expect(mockDb.webhookCall.update).toHaveBeenNthCalledWith(2, { - where: { id: "call_123" }, + expect(mockDb.webhookCall.update).toHaveBeenCalledTimes(1); + expect(mockDb.webhookCall.updateMany).toHaveBeenCalledWith({ + where: { + id: "call_123", + status: WebhookCallStatus.PENDING, + attempt: 0, + }, data: { status: WebhookCallStatus.FAILED, attempt: 6,