Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 175 additions & 0 deletions apps/web/src/server/queue/requeue-job.integration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
import { Queue, Worker } from "bullmq";
import { afterAll, afterEach, beforeEach, describe, expect, it } from "vitest";
import { DEFAULT_QUEUE_OPTIONS } from "~/server/queue/queue-constants";
import { requeueJob } from "~/server/queue/requeue-job";
import { getRedis } from "~/server/redis";
import {
closeIntegrationConnections,
integrationEnabled,
resetRedis,
} from "~/test/integration/helpers";

const describeIntegration = integrationEnabled ? describe : describe.skip;

const QUEUE_NAME = "requeue-job-integration";
const PREFIX = "requeue-job-test";
// Each test polls the queue a few times; leave room for a slow CI runner.
const TEST_TIMEOUT_MS = 15_000;

type Data = { callId: string };

async function waitFor(check: () => Promise<boolean>, timeoutMs = 4000) {
const end = Date.now() + timeoutMs;
while (Date.now() < end) {
if (await check()) return;
await new Promise((resolve) => setTimeout(resolve, 20));
}
throw new Error("Timed out waiting for the queue");
}

describeIntegration("requeueJob with real BullMQ and Redis", () => {
let queue: Queue<Data>;
let worker: Worker<Data> | undefined;
// Attempt numbers seen by the worker, per run (attemptsMade + 1).
let attempts: number[];
let shouldFail: boolean;

function startWorker(handler?: () => Promise<void>): Worker<Data> {
worker = new Worker<Data>(
QUEUE_NAME,
async (job) => {
attempts.push(job.attemptsMade + 1);
if (handler) return handler();
if (shouldFail) throw new Error("endpoint returned 500");
},
{ connection: getRedis(), prefix: PREFIX },
);
return worker;
}

const stateOf = async (jobId: string) =>
(await queue.getJob(jobId))?.getState() ?? "missing";

beforeEach(async () => {
await resetRedis();
attempts = [];
shouldFail = true;
queue = new Queue<Data>(QUEUE_NAME, {
connection: getRedis(),
prefix: PREFIX,
defaultJobOptions: {
...DEFAULT_QUEUE_OPTIONS,
attempts: 2,
backoff: { type: "fixed", delay: 10 },
},
});
});

afterEach(async () => {
await worker?.close();
worker = undefined;
await queue.close();
});

afterAll(async () => {
await closeIntegrationConnections();
});

it(
"runs a failed job again with its attempts reset, where a plain add() does nothing",
async () => {
startWorker();
await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" });
await waitFor(async () => (await stateOf("call_1")) === "failed");
expect(attempts).toEqual([1, 2]);

// The bug: the failed job is kept under the same id, so add() is ignored.
await queue.add("call_1", { callId: "call_1" }, { jobId: "call_1" });
await new Promise((resolve) => setTimeout(resolve, 200));
expect(attempts).toEqual([1, 2]);
expect(await stateOf("call_1")).toBe("failed");

shouldFail = false;
await expect(
requeueJob(queue, "call_1", { callId: "call_1" }, { jobId: "call_1" }),
).resolves.toBe("retried");

// Completed jobs are removed (removeOnComplete), so the job disappears.
await waitFor(async () => (await stateOf("call_1")) === "missing");
expect(attempts).toEqual([1, 2, 1]);
},
TEST_TIMEOUT_MS,
);

it(
"adds a new job when none exists",
async () => {
shouldFail = false;
startWorker();

await expect(
requeueJob(queue, "call_2", { callId: "call_2" }, { jobId: "call_2" }),
).resolves.toBe("added");

await waitFor(async () => attempts.length === 1);
await waitFor(async () => (await stateOf("call_2")) === "missing");
expect(attempts).toEqual([1]);
},
TEST_TIMEOUT_MS,
);

it(
"leaves a job that is waiting for its next attempt alone",
async () => {
startWorker();
await queue.add(
"call_3",
{ callId: "call_3" },
{ jobId: "call_3", backoff: { type: "fixed", delay: 60_000 } },
);
await waitFor(async () => (await stateOf("call_3")) === "delayed");

await expect(
requeueJob(queue, "call_3", { callId: "call_3" }, { jobId: "call_3" }),
).resolves.toBe("already-queued");

await new Promise((resolve) => setTimeout(resolve, 200));
expect(attempts).toEqual([1]);
expect(await stateOf("call_3")).toBe("delayed");
},
TEST_TIMEOUT_MS,
);

it(
"leaves a running job alone and never runs it twice at once",
async () => {
let release: () => void = () => undefined;
const running = new Promise<void>((resolve) => {
release = resolve;
});
startWorker(() => running);

// Always let the job finish, so closing the worker can't hang the suite.
try {
await queue.add("call_4", { callId: "call_4" }, { jobId: "call_4" });
await waitFor(async () => (await stateOf("call_4")) === "active");

await expect(
requeueJob(
queue,
"call_4",
{ callId: "call_4" },
{ jobId: "call_4" },
),
).resolves.toBe("already-queued");
} finally {
release();
}

await waitFor(async () => (await stateOf("call_4")) === "missing");
await new Promise((resolve) => setTimeout(resolve, 200));
expect(attempts).toEqual([1]);
},
TEST_TIMEOUT_MS,
);
});
52 changes: 52 additions & 0 deletions apps/web/src/server/queue/requeue-job.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import type { JobsOptions, Queue } from "bullmq";

export type RequeueResult = "retried" | "added" | "already-queued";

/**
* Runs a job again under a fixed job id.
*
* BullMQ ignores `add()` when a job with the same id already exists, in any
* state, and our queues keep failed jobs for 30 days. So a plain `add()` after
* a failure silently does nothing. This helper:
* - moves a retained failed (or kept completed) job back to waiting, with its
* attempt count reset, in one atomic step;
* - leaves a job that is still waiting, delayed or running alone, since it will
* run anyway;
* - otherwise adds a new job.
*
* Use it only where a job is deliberately run again (for example a user pressing
* Retry), not on hot paths: it costs an extra Redis round trip.
*
* The result says what this call did, not that the job will run: if the job
* changes state at the same moment (another retry, or the 30-day cleanup),
* `retry()` throws, and the caller should let the user try again. It does not
* repair queue data that is out of sync.
*/
export async function requeueJob<Data>(
queue: Queue<Data>,
name: string,
data: Data,
opts: JobsOptions & { jobId: string },
): Promise<RequeueResult> {
const existing = await queue.getJob(opts.jobId);

if (existing) {
const state = await existing.getState();

if (state === "failed" || state === "completed") {
await existing.retry(state, {
resetAttemptsMade: true,
resetAttemptsStarted: true,
});
return "retried";
}

if (state !== "unknown") {
return "already-queued";
}
}

// Queue<Data>'s name and data generics are wider than this helper needs.
await (queue as Queue).add(name, data, opts);
return "added";
}
41 changes: 40 additions & 1 deletion apps/web/src/server/service/webhook-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
WEBHOOK_DISPATCH_QUEUE,
} from "../queue/queue-constants";
import { createWorkerHandler, TeamJob } from "../queue/bullmq-context";
import { requeueJob } from "../queue/requeue-job";
import { logger } from "../logger/log";
import { LimitService } from "./limit-service";
import { withPlanLimitLock } from "./plan-limit-lock";
Expand Down Expand Up @@ -89,6 +90,20 @@ export class WebhookQueueService {
{ jobId: callId },
);
}

/**
* Queues a call again when a user presses Retry. A plain add() would do
* nothing after the call used up its attempts, because the failed job is kept
* under the same id (see requeueJob).
*/
public static async requeueCall(callId: string, teamId: number) {
return requeueJob(
this.queue,
callId,
{ callId, teamId },
{ jobId: callId },
);
}
}

export class WebhookService {
Expand Down Expand Up @@ -188,7 +203,31 @@ export class WebhookService {
},
});

await WebhookQueueService.enqueueCall(call.id, params.teamId);
try {
await WebhookQueueService.requeueCall(call.id, params.teamId);
} catch (error) {
// Nothing was queued: put the call back as it was, so it still shows as
// failed and can be retried again from the dashboard. Only undo our own
// reset (PENDING, attempt 0): once a worker picks the call up it sets
// attempt >= 1, and its result must not be overwritten.
await db.webhookCall.updateMany({
where: {
id: call.id,
status: WebhookCallStatus.PENDING,
attempt: 0,
},
data: {
status: call.status,
attempt: call.attempt,
nextAttemptAt: call.nextAttemptAt,
lastError: call.lastError,
responseStatus: call.responseStatus,
responseTimeMs: call.responseTimeMs,
responseText: call.responseText,
},
});
throw error;
}

return call.id;
}
Expand Down
Loading
Loading