Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions apps/web/src/server/queue/requeue-job.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,76 @@ describeIntegration("requeueJob with real BullMQ and Redis", () => {
TEST_TIMEOUT_MS,
);

it(
"waits before retrying a job that failed moments ago, then retries it",
async () => {
startWorker();
await queue.add(
"batch_1",
{ callId: "batch_1" },
{ jobId: "batch_1", attempts: 1 },
);
await waitFor(async () => (await stateOf("batch_1")) === "failed");
expect(attempts).toEqual([1]);

shouldFail = false;
await expect(
requeueJob(
queue,
"batch_1",
{ callId: "batch_1" },
{ jobId: "batch_1" },
{ retryFailedAfterMs: 60_000 },
),
).resolves.toBe("failed-too-recently");
await new Promise((resolve) => setTimeout(resolve, 200));
expect(attempts).toEqual([1]);
expect(await stateOf("batch_1")).toBe("failed");

await expect(
requeueJob(
queue,
"batch_1",
{ callId: "batch_1" },
{ jobId: "batch_1" },
{ retryFailedAfterMs: 0 },
),
).resolves.toBe("retried");
await waitFor(async () => (await stateOf("batch_1")) === "missing");
expect(attempts).toEqual([1, 1]);
},
TEST_TIMEOUT_MS,
);

it(
"leaves a job that failed too long ago alone",
async () => {
startWorker();
await queue.add(
"batch_2",
{ callId: "batch_2" },
{ jobId: "batch_2", attempts: 1 },
);
await waitFor(async () => (await stateOf("batch_2")) === "failed");
await new Promise((resolve) => setTimeout(resolve, 50));

shouldFail = false;
await expect(
requeueJob(
queue,
"batch_2",
{ callId: "batch_2" },
{ jobId: "batch_2" },
{ maxFailedAgeMs: 10 },
),
).resolves.toBe("failed-too-long-ago");
await new Promise((resolve) => setTimeout(resolve, 200));
expect(attempts).toEqual([1]);
expect(await stateOf("batch_2")).toBe("failed");
},
TEST_TIMEOUT_MS,
);

it(
"adds a new job when none exists",
async () => {
Expand Down
93 changes: 72 additions & 21 deletions apps/web/src/server/queue/requeue-job.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,22 @@
import type { JobsOptions, Queue } from "bullmq";
import type { Job, JobsOptions, Queue } from "bullmq";

export type RequeueResult = "retried" | "added" | "already-queued";
export type RequeueResult =
| "retried"
| "added"
| "already-queued"
| "failed-too-recently"
| "failed-too-long-ago";

// A Queue's type argument is either the job data or a Job type (as BullMQ allows).
type QueueData<DataOrJob> =
DataOrJob extends Job<infer Data, any, any> ? Data : DataOrJob;

export type RequeueOptions = {
/** Leave a job that failed less than this long ago alone, for now. */
retryFailedAfterMs?: number;
/** Leave a job that failed more than this long ago alone. */
maxFailedAgeMs?: number;
};

/**
* Runs a job again under a fixed job id.
Expand All @@ -14,39 +30,74 @@ export type RequeueResult = "retried" | "added" | "already-queued";
* run anyway;
* - otherwise adds a new job.
*
* A failed job can be left alone for a while (`retryFailedAfterMs`) or for good
* once it is old (`maxFailedAgeMs`). Without options it is retried at once.
*
* Use it only where a job is deliberately run again (for example a user pressing
* Retry), not on hot paths: it costs an extra Redis round trip.
* Retry, or a scheduler continuing work), not on hot paths: it costs an extra
* Redis round trip.
*
* The result says what this call did, not that the job will run: if the job
* changes state at the same moment (another retry, or the 30-day cleanup),
* `retry()` throws, and the caller should let the user try again. It does not
* repair queue data that is out of sync.
* The result is a best-effort description, not a delivery guarantee
* ("already-queued" means an existing job was kept, or a vanished one was
* re-added). If the job changes state at the same moment (another retry, or
* the 30-day cleanup), `retry()` throws, and the caller should let the user
* try again. It does not repair queue data that is out of sync.
*/
export async function requeueJob<Data>(
queue: Queue<Data>,
export async function requeueJob<DataOrJob>(
queue: Queue<DataOrJob>,
name: string,
data: Data,
data: QueueData<DataOrJob>,
opts: JobsOptions & { jobId: string },
options: RequeueOptions = {},
): Promise<RequeueResult> {
const existing = await queue.getJob(opts.jobId);

if (existing) {
const state = await existing.getState();

if (state === "failed" || state === "completed") {
await existing.retry(state, {
resetAttemptsMade: true,
resetAttemptsStarted: true,
});
return "retried";
}
// isFailed() and isCompleted() are single ZSCORE lookups. getState() would
// also scan the wait and active lists, which is too costly for callers that
// run on every scheduler tick.
const finishedState = (await existing.isFailed())
? "failed"
: (await existing.isCompleted())
? "completed"
: null;

if (state !== "unknown") {
if (!finishedState) {
// Still waiting, delayed or running: it will run anyway. add() is still
// called, in case the job finished and was removed during the checks
// above: BullMQ ignores it while a job with this id exists.
await (queue as Queue).add(name, data, opts);
return "already-queued";
}

if (finishedState === "failed") {
// finishedOn is missing if the job was fetched while still running and
// failed just after: treat that as a fresh failure, never an old one.
const failedForMs = existing.finishedOn
? Date.now() - existing.finishedOn
: 0;
if (
options.retryFailedAfterMs !== undefined &&
failedForMs < options.retryFailedAfterMs
) {
return "failed-too-recently";
}
if (
options.maxFailedAgeMs !== undefined &&
failedForMs > options.maxFailedAgeMs
) {
return "failed-too-long-ago";
}
}

await existing.retry(finishedState, {
resetAttemptsMade: true,
resetAttemptsStarted: true,
});
return "retried";
}

// Queue<Data>'s name and data generics are wider than this helper needs.
// Queue's name and data generics are wider than this helper needs.
await (queue as Queue).add(name, data, opts);
return "added";
}
46 changes: 46 additions & 0 deletions apps/web/src/server/queue/requeue-job.unit.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { describe, expect, it, vi } from "vitest";
import { requeueJob } from "~/server/queue/requeue-job";

function queueWith(job: unknown) {
return {
getJob: vi.fn().mockResolvedValue(job),
add: vi.fn().mockResolvedValue(undefined),
} as any;
}

describe("requeueJob", () => {
it("treats a failed job without a finish time as a fresh failure", async () => {
// Fetched while running, failed just after: finishedOn isn't on the snapshot.
const job = {
isFailed: vi.fn().mockResolvedValue(true),
isCompleted: vi.fn().mockResolvedValue(false),
finishedOn: undefined,
retry: vi.fn().mockResolvedValue(undefined),
};
const options = { retryFailedAfterMs: 60_000, maxFailedAgeMs: 86_400_000 };

await expect(
requeueJob(queueWith(job), "n", {}, { jobId: "j" }, options),
).resolves.toBe("failed-too-recently");
expect(job.retry).not.toHaveBeenCalled();
});

it("never calls a failure with no finish time too old to retry", async () => {
const job = {
isFailed: vi.fn().mockResolvedValue(true),
isCompleted: vi.fn().mockResolvedValue(false),
finishedOn: undefined,
retry: vi.fn().mockResolvedValue(undefined),
};

await expect(
requeueJob(
queueWith(job),
"n",
{},
{ jobId: "j" },
{ maxFailedAgeMs: 1 },
),
).resolves.toBe("retried");
});
});
Loading
Loading