From 9ba1a59d436808aebccba32ca1157150bd56289b Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Sat, 21 Nov 2026 12:49:21 +1100 Subject: [PATCH 1/4] feat(rules): pause or block teams on high bounce and complaint rates --- .../migration.sql | 53 +++ apps/web/prisma/schema.prisma | 138 ++++-- .../src/app/(dashboard)/admin/teams/page.tsx | 100 +++- .../app/(dashboard)/admin/waitlist/page.tsx | 2 +- .../src/components/SendingStatusBanner.tsx | 13 + .../src/components/payments/UpgradeModal.tsx | 2 + apps/web/src/env.js | 4 + apps/web/src/lib/constants/plans.ts | 1 + apps/web/src/providers/next-auth.tsx | 34 +- apps/web/src/providers/team-context.tsx | 1 + apps/web/src/server/api/routers/admin.ts | 94 +++- .../src/server/service/campaign-service.ts | 11 + .../src/server/service/email-queue-service.ts | 12 +- apps/web/src/server/service/limit-service.ts | 9 + .../server/service/limit-service.unit.test.ts | 14 + .../web/src/server/service/sending-cohorts.ts | 138 ++++++ .../service/sending-cohorts.unit.test.ts | 169 +++++++ .../server/service/sending-rules-service.ts | 442 ++++++++++++++++++ .../sending-rules-service.unit.test.ts | 427 +++++++++++++++++ apps/web/src/server/service/sending-rules.ts | 81 ++++ .../web/src/server/service/ses-hook-parser.ts | 188 +++++--- .../service/ses-hook-parser.unit.test.ts | 183 +++++++- 22 files changed, 1944 insertions(+), 172 deletions(-) create mode 100644 apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql create mode 100644 apps/web/src/server/service/sending-cohorts.ts create mode 100644 apps/web/src/server/service/sending-cohorts.unit.test.ts create mode 100644 apps/web/src/server/service/sending-rules-service.ts create mode 100644 apps/web/src/server/service/sending-rules-service.unit.test.ts create mode 100644 apps/web/src/server/service/sending-rules.ts diff --git a/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql b/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql new file mode 100644 index 000000000..1410ff1a0 --- /dev/null +++ b/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql @@ -0,0 +1,53 @@ +-- All changes below only add columns or tables, so they finish instantly, but +-- adding a column to "Email" and "Campaign" needs a brief lock. Give up after +-- 10s rather than queue behind a long query and hold up sending; the deploy +-- then fails (the old version keeps serving) and can simply be retried. +SET lock_timeout = '10s'; + +-- CreateEnum +CREATE TYPE "TeamActionType" AS ENUM ('ALERT', 'PAUSE', 'RESUME', 'BLOCK', 'UNBLOCK'); + +-- AlterTable +ALTER TABLE "Team" ADD COLUMN "sendingPausedAt" TIMESTAMP(3); + +-- AlterTable +ALTER TABLE "Email" ADD COLUMN "cohortDate" TEXT; + +-- AlterTable +ALTER TABLE "Campaign" ADD COLUMN "acceptedComplained" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "acceptedHardBounced" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "acceptedSends" INTEGER NOT NULL DEFAULT 0; + +-- CreateTable +CREATE TABLE "SendingCohort" ( + "teamId" INTEGER NOT NULL, + "date" TEXT NOT NULL, + "sent" INTEGER NOT NULL DEFAULT 0, + "hardBounced" INTEGER NOT NULL DEFAULT 0, + "complained" INTEGER NOT NULL DEFAULT 0, + + CONSTRAINT "SendingCohort_pkey" PRIMARY KEY ("teamId","date") +); + +-- CreateTable +CREATE TABLE "TeamAction" ( + "id" TEXT NOT NULL, + "teamId" INTEGER NOT NULL, + "action" "TeamActionType" NOT NULL, + "actor" TEXT NOT NULL, + "reason" TEXT NOT NULL, + "details" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "TeamAction_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "TeamAction_teamId_createdAt_idx" ON "TeamAction"("teamId", "createdAt"); + +-- AddForeignKey +ALTER TABLE "SendingCohort" ADD CONSTRAINT "SendingCohort_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TeamAction" ADD CONSTRAINT "TeamAction_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team"("id") ON DELETE CASCADE ON UPDATE CASCADE; + diff --git a/apps/web/prisma/schema.prisma b/apps/web/prisma/schema.prisma index 49e4c59ea..3a349331b 100644 --- a/apps/web/prisma/schema.prisma +++ b/apps/web/prisma/schema.prisma @@ -79,18 +79,18 @@ model VerificationToken { } model User { - id Int @id @default(autoincrement()) - name String? - email String? @unique - emailVerified DateTime? - image String? - isBetaUser Boolean @default(false) - isWaitlisted Boolean @default(false) - createdAt DateTime @default(now()) - accounts Account[] - sessions Session[] - teamUsers TeamUser[] - webhookEndpoints Webhook[] + id Int @id @default(autoincrement()) + name String? + email String? @unique + emailVerified DateTime? + image String? + isBetaUser Boolean @default(false) + isWaitlisted Boolean @default(false) + createdAt DateTime @default(now()) + accounts Account[] + sessions Session[] + teamUsers TeamUser[] + webhookEndpoints Webhook[] waitlistApplication WaitlistApplication? feedback Feedback[] } @@ -143,6 +143,8 @@ model Team { isBlocked Boolean @default(false) /// Teams created before paid-only signups may keep sending on the free plan. freePlanAllowed Boolean @default(false) + /// Set when sending is paused (e.g. bounce rate too high). Cleared on resume. + sendingPausedAt DateTime? teamUsers TeamUser[] domains Domain[] apiKeys ApiKey[] @@ -157,6 +159,48 @@ model Team { webhookEndpoints Webhook[] webhookCalls WebhookCall[] feedback Feedback[] + actions TeamAction[] + sendingCohorts SendingCohort[] +} + +/// Emails a team sent each day, with the hard bounces and complaints those +/// same emails caused (filed under the day they were sent). Used by the +/// sending rules, so rates always compare matching sends and outcomes. +model SendingCohort { + teamId Int + /// YYYY-MM-DD (UTC) the emails were sent + date String + sent Int @default(0) + hardBounced Int @default(0) + complained Int @default(0) + + team Team @relation(fields: [teamId], references: [id], onDelete: Cascade) + + @@id([teamId, date]) +} + +enum TeamActionType { + ALERT + PAUSE + RESUME + BLOCK + UNBLOCK +} + +/// Every pause, block and alert on a team, by the sending rules, an admin or the Guard. +model TeamAction { + id String @id @default(cuid()) + teamId Int + action TeamActionType + /// "rules", "guard" or "admin:" + actor String + reason String + details Json? + createdAt DateTime @default(now()) + + team Team @relation(fields: [teamId], references: [id], onDelete: Cascade) + + @@index([teamId, createdAt]) } model TeamInvite { @@ -295,6 +339,9 @@ model Email { contactId String? inReplyToId String? headers String? + /// Day (YYYY-MM-DD, UTC) this email was counted as sent for the sending + /// rules. Its bounces and complaints are filed under the same day. + cohortDate String? team Team @relation(fields: [teamId], references: [id], onDelete: Cascade) emailEvents EmailEvent[] @@ -387,37 +434,42 @@ enum CampaignStatus { } model Campaign { - id String @id @default(cuid()) - name String - teamId Int - from String - cc String[] - bcc String[] - replyTo String[] - domainId Int - subject String - previewText String? - html String? - content String? - contactBookId String? - scheduledAt DateTime? - total Int @default(0) - sent Int @default(0) - delivered Int @default(0) - opened Int @default(0) - clicked Int @default(0) - unsubscribed Int @default(0) - bounced Int @default(0) - hardBounced Int @default(0) - complained Int @default(0) - isApi Boolean @default(false) - status CampaignStatus @default(DRAFT) - batchSize Int @default(500) - batchWindowMinutes Int @default(0) - lastCursor String? - lastSentAt DateTime? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + id String @id @default(cuid()) + name String + teamId Int + from String + cc String[] + bcc String[] + replyTo String[] + domainId Int + subject String + previewText String? + html String? + content String? + contactBookId String? + scheduledAt DateTime? + total Int @default(0) + sent Int @default(0) + delivered Int @default(0) + opened Int @default(0) + clicked Int @default(0) + unsubscribed Int @default(0) + bounced Int @default(0) + hardBounced Int @default(0) + complained Int @default(0) + /// For the sending rules: emails counted as sent (marked with a cohort + /// day), and the hard bounces and complaints of those same emails. + acceptedSends Int @default(0) + acceptedHardBounced Int @default(0) + acceptedComplained Int @default(0) + isApi Boolean @default(false) + status CampaignStatus @default(DRAFT) + batchSize Int @default(500) + batchWindowMinutes Int @default(0) + lastCursor String? + lastSentAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt team Team @relation(fields: [teamId], references: [id], onDelete: Cascade) diff --git a/apps/web/src/app/(dashboard)/admin/teams/page.tsx b/apps/web/src/app/(dashboard)/admin/teams/page.tsx index ad1b812fc..846f7425d 100644 --- a/apps/web/src/app/(dashboard)/admin/teams/page.tsx +++ b/apps/web/src/app/(dashboard)/admin/teams/page.tsx @@ -34,7 +34,10 @@ import { isCloud } from "~/utils/common"; const searchSchema = z.object({ query: z - .string({ required_error: "Enter a team ID, name, domain, member email, or subscription ID" }) + .string({ + required_error: + "Enter a team ID, name, domain, member email, or subscription ID", + }) .trim() .min(1, "Enter a team ID, name, domain, member email, or subscription ID"), }); @@ -49,6 +52,7 @@ const updateSchema = z.object({ dailyEmailLimit: z.coerce.number().int().min(0).max(10_000_000), isBlocked: z.boolean(), freePlanAllowed: z.boolean(), + sendingPaused: z.boolean(), plan: z.enum(["FREE", "BASIC"]), }); @@ -70,6 +74,7 @@ export default function AdminTeamsPage() { dailyEmailLimit: 0, isBlocked: false, freePlanAllowed: false, + sendingPaused: false, plan: "FREE", }, }); @@ -81,6 +86,7 @@ export default function AdminTeamsPage() { dailyEmailLimit: team.dailyEmailLimit, isBlocked: team.isBlocked, freePlanAllowed: team.freePlanAllowed, + sendingPaused: Boolean(team.sendingPausedAt), plan: team.plan, }); } @@ -117,6 +123,7 @@ export default function AdminTeamsPage() { dailyEmailLimit: updated.dailyEmailLimit, isBlocked: updated.isBlocked, freePlanAllowed: updated.freePlanAllowed, + sendingPaused: Boolean(updated.sendingPausedAt), plan: updated.plan, }); toast.success("Team settings updated"); @@ -190,7 +197,10 @@ export default function AdminTeamsPage() {

Team

{team.name}

- ID #{team.id} • Created {formatDistanceToNow(new Date(team.createdAt), { addSuffix: true })} + ID #{team.id} • Created{" "} + {formatDistanceToNow(new Date(team.createdAt), { + addSuffix: true, + })}

@@ -198,12 +208,17 @@ export default function AdminTeamsPage() { {team.isBlocked ? "Blocked" : "Active"} + {team.sendingPausedAt ? ( + Sending paused + ) : null}
-

Members

+

+ Members +

{team.teamUsers.length ? ( team.teamUsers.map((member) => ( @@ -212,19 +227,27 @@ export default function AdminTeamsPage() { className="flex items-center justify-between rounded-md bg-background px-3 py-2 text-sm" >
-

{member.user.name ?? member.user.email}

-

{member.user.email}

+

+ {member.user.name ?? member.user.email} +

+

+ {member.user.email} +

{member.role}
)) ) : ( -

No members found.

+

+ No members found. +

)}
-

Domains

+

+ Domains +

{team.domains.length ? ( team.domains.map((domain) => ( @@ -233,7 +256,11 @@ export default function AdminTeamsPage() { className="flex items-center justify-between rounded-md bg-background px-3 py-2 text-sm" > {domain.name} - + {domain.status === "SUCCESS" ? "Verified" : domain.status.toLowerCase()} @@ -241,7 +268,9 @@ export default function AdminTeamsPage() {
)) ) : ( -

No domains connected.

+

+ No domains connected. +

)}
@@ -255,7 +284,10 @@ export default function AdminTeamsPage() {
- + )} /> + ( + + Sending paused + +
+ + + {field.value + ? "Emails fail until resumed" + : "Sending normally"} + +
+
+ +
+ )} + />
+ +
+

Recent actions

+ {team.actions.length === 0 ? ( +

+ No pauses, blocks or alerts yet. +

+ ) : ( +
    + {team.actions.map((action) => ( +
  • + + {new Date(action.createdAt).toLocaleString()} + + {action.action} + + by {action.actor} + + {action.reason} +
  • + ))} +
+ )} +
) : null} diff --git a/apps/web/src/app/(dashboard)/admin/waitlist/page.tsx b/apps/web/src/app/(dashboard)/admin/waitlist/page.tsx index f39982f6c..a91483b0c 100644 --- a/apps/web/src/app/(dashboard)/admin/waitlist/page.tsx +++ b/apps/web/src/app/(dashboard)/admin/waitlist/page.tsx @@ -194,7 +194,7 @@ export default function AdminWaitlistPage() { {userResult.waitlistApplication ? ( {userResult.waitlistApplication.status.toLowerCase()} · - submitted{" "} + updated{" "} {formatDistanceToNow( new Date(userResult.waitlistApplication.updatedAt), { addSuffix: true }, diff --git a/apps/web/src/components/SendingStatusBanner.tsx b/apps/web/src/components/SendingStatusBanner.tsx index 87cc76a47..5c6684fa3 100644 --- a/apps/web/src/components/SendingStatusBanner.tsx +++ b/apps/web/src/components/SendingStatusBanner.tsx @@ -23,6 +23,19 @@ export function SendingStatusBanner() { ); } + if (currentTeam.sendingPausedAt) { + return ( +
+

Sending is paused for this team.

+

+ Too many emails bounced or were marked as spam. Remove old or + unconfirmed addresses from your list, then reply to our email or write + to hey@usesend.com and we'll turn sending back on. +

+
+ ); + } + const onFreePlan = currentTeam.plan === "FREE" || !currentTeam.isActive; if (onFreePlan && !currentTeam.freePlanAllowed) { return ( diff --git a/apps/web/src/components/payments/UpgradeModal.tsx b/apps/web/src/components/payments/UpgradeModal.tsx index 698d93027..8c30153a6 100644 --- a/apps/web/src/components/payments/UpgradeModal.tsx +++ b/apps/web/src/components/payments/UpgradeModal.tsx @@ -38,6 +38,8 @@ export const UpgradeModal = () => { "You've reached the team member limit for your current plan.", [LimitReason.WEBHOOK]: "You've reached the webhook limit for your current plan.", + [LimitReason.EMAIL_SENDING_PAUSED]: + "Sending is paused on your account.", [LimitReason.EMAIL_PLAN_REQUIRED]: "Start sending for $10 a month.", [LimitReason.EMAIL_BLOCKED]: diff --git a/apps/web/src/env.js b/apps/web/src/env.js index ae57114bf..ec8b262e7 100644 --- a/apps/web/src/env.js +++ b/apps/web/src/env.js @@ -61,6 +61,9 @@ export const env = createEnv({ DISCORD_WEBHOOK_URL: z.string().optional(), /** Bearer token for the internal admin API (/api/admin). The API is off when unset. */ ADMIN_API_TOKEN: z.string().min(32).optional(), + // Bounce/complaint rules (cloud only). "alert" (the default) only posts to + // Discord; "enforce" pauses or blocks teams and emails them; "off" disables. + SENDING_RULES_MODE: z.enum(["off", "alert", "enforce"]).default("alert"), REDIS_URL: z.string(), REDIS_KEY_PREFIX: z.string().default(""), S3_COMPATIBLE_ACCESS_KEY: z.string().optional(), @@ -134,6 +137,7 @@ export const env = createEnv({ FOUNDER_EMAIL: process.env.FOUNDER_EMAIL, DISCORD_WEBHOOK_URL: process.env.DISCORD_WEBHOOK_URL, ADMIN_API_TOKEN: process.env.ADMIN_API_TOKEN || undefined, + SENDING_RULES_MODE: process.env.SENDING_RULES_MODE, REDIS_URL: process.env.REDIS_URL, REDIS_KEY_PREFIX: process.env.REDIS_KEY_PREFIX, FROM_EMAIL: process.env.FROM_EMAIL, diff --git a/apps/web/src/lib/constants/plans.ts b/apps/web/src/lib/constants/plans.ts index 957e0477b..bce15436c 100644 --- a/apps/web/src/lib/constants/plans.ts +++ b/apps/web/src/lib/constants/plans.ts @@ -7,6 +7,7 @@ export enum LimitReason { WEBHOOK = "WEBHOOK", EMAIL_BLOCKED = "EMAIL_BLOCKED", EMAIL_PLAN_REQUIRED = "EMAIL_PLAN_REQUIRED", + EMAIL_SENDING_PAUSED = "EMAIL_SENDING_PAUSED", EMAIL_DAILY_LIMIT_REACHED = "EMAIL_DAILY_LIMIT_REACHED", EMAIL_FREE_PLAN_MONTHLY_LIMIT_REACHED = "EMAIL_FREE_PLAN_MONTHLY_LIMIT_REACHED", } diff --git a/apps/web/src/providers/next-auth.tsx b/apps/web/src/providers/next-auth.tsx index a52b26fef..c006fda65 100644 --- a/apps/web/src/providers/next-auth.tsx +++ b/apps/web/src/providers/next-auth.tsx @@ -1,13 +1,12 @@ "use client"; -import React from "react"; +import React, { useEffect } from "react"; import type { Session } from "next-auth"; import { SessionProvider, useSession } from "next-auth/react"; import LoginPage from "~/app/login/login-page"; import { FullScreenLoading } from "~/components/FullScreenLoading"; -import { Rocket } from "lucide-react"; -import { WaitListForm } from "~/app/wait-list/waitlist-form"; +import { useRouter } from "next/navigation"; export type NextAuthProviderProps = { session?: Session | null | undefined; @@ -37,26 +36,17 @@ const AppAuthProvider = ({ children }: { children: React.ReactNode }) => { } if (session.user.isWaitlisted) { - return ( -
-
-
- - - -
-

You're on the waitlist

-

- Share a bit more context so we can prioritize your access. -

-
-
- - -
-
- ); + // The waitlist page shows their saved application, if any. + return ; } return <>{children}; }; + +const RedirectToWaitlist = () => { + const router = useRouter(); + useEffect(() => { + router.replace("/wait-list"); + }, [router]); + return ; +}; diff --git a/apps/web/src/providers/team-context.tsx b/apps/web/src/providers/team-context.tsx index 33b3afa34..df5b21d34 100644 --- a/apps/web/src/providers/team-context.tsx +++ b/apps/web/src/providers/team-context.tsx @@ -13,6 +13,7 @@ type Team = { isActive: boolean; isBlocked: boolean; freePlanAllowed: boolean; + sendingPausedAt?: Date | string | null; stripeCustomerId?: string | null; billingEmail?: string | null; }; diff --git a/apps/web/src/server/api/routers/admin.ts b/apps/web/src/server/api/routers/admin.ts index 71b829dae..f84a4fd1e 100644 --- a/apps/web/src/server/api/routers/admin.ts +++ b/apps/web/src/server/api/routers/admin.ts @@ -14,6 +14,7 @@ import { toPlainHtml } from "~/server/utils/email-content"; import { sesRegionSchema } from "~/lib/zod/ses-setting-schema"; import { TeamService } from "~/server/service/team-service"; import { WaitlistService } from "~/server/service/waitlist-service"; +import { unsentCampaignIds } from "~/server/service/sending-rules-service"; const waitlistUserSelection = { id: true, @@ -55,8 +56,20 @@ const teamAdminSelection = { dailyEmailLimit: true, isBlocked: true, freePlanAllowed: true, + sendingPausedAt: true, billingEmail: true, createdAt: true, + actions: { + orderBy: { createdAt: "desc" }, + take: 10, + select: { + id: true, + action: true, + actor: true, + reason: true, + createdAt: true, + }, + }, teamUsers: { select: { role: true, @@ -421,16 +434,85 @@ export const adminRouter = createTRPCRouter({ dailyEmailLimit: z.number().int().min(0).max(10_000_000), isBlocked: z.boolean(), freePlanAllowed: z.boolean(), + sendingPaused: z.boolean(), plan: z.enum(["FREE", "BASIC"]), }), ) - .mutation(async ({ input }) => { - const { teamId, ...data } = input; + .mutation(async ({ ctx, input }) => { + const { teamId, sendingPaused, ...data } = input; + + const actor = `admin:${ctx.session.user.email ?? ctx.session.user.id}`; + + const updatedTeam = await db.$transaction(async (tx) => { + // Lock the row so a concurrent pause or block by the sending rules + // can't be overwritten without being logged. + await tx.$queryRaw`SELECT id FROM "Team" WHERE id = ${teamId} FOR UPDATE`; + const existing = await tx.team.findUnique({ + where: { id: teamId }, + select: { isBlocked: true, sendingPausedAt: true }, + }); + if (!existing) { + throw new Error("Team not found"); + } + + const actions: Prisma.TeamActionCreateManyInput[] = []; + const resets = + (existing.isBlocked && !data.isBlocked) || + (Boolean(existing.sendingPausedAt) && !sendingPaused); + // Campaigns that already sent aren't judged on old totals after a reset. + const resetDetails = resets + ? { unsentCampaignIds: await unsentCampaignIds(tx, teamId) } + : undefined; + if (existing.isBlocked !== data.isBlocked) { + actions.push({ + teamId, + actor, + action: data.isBlocked ? "BLOCK" : "UNBLOCK", + reason: data.isBlocked + ? "Blocked by an admin" + : "Unblocked by an admin", + details: data.isBlocked ? undefined : resetDetails, + }); + } + if (Boolean(existing.sendingPausedAt) !== sendingPaused) { + actions.push({ + teamId, + actor, + action: sendingPaused ? "PAUSE" : "RESUME", + reason: sendingPaused + ? "Paused by an admin" + : "Resumed by an admin", + details: sendingPaused ? undefined : resetDetails, + }); + } + + // Log first, so the team returned below includes the new actions. + if (actions.length > 0) { + await tx.teamAction.createMany({ data: actions }); + } + + const stopsSending = + (data.isBlocked && !existing.isBlocked) || + (sendingPaused && !existing.sendingPausedAt); + if (stopsSending) { + await tx.campaign.updateMany({ + where: { teamId, status: { in: ["RUNNING", "SCHEDULED"] } }, + data: { status: "PAUSED" }, + }); + } + + const team = await tx.team.update({ + where: { id: teamId }, + data: { + ...data, + sendingPausedAt: sendingPaused + ? (existing.sendingPausedAt ?? new Date()) + : null, + }, + select: teamAdminSelection, + }); - const updatedTeam = await db.team.update({ - where: { id: teamId }, - data, - select: teamAdminSelection, + return team; }); // Sending reads the cached team, so refresh it for blocks to apply now. diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index 5c3db73b3..25349756e 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -534,6 +534,17 @@ export async function resumeCampaign({ }); } + const team = await db.team.findUnique({ + where: { id: teamId }, + select: { isBlocked: true, sendingPausedAt: true }, + }); + if (team?.isBlocked || team?.sendingPausedAt) { + throw new UnsendApiError({ + code: "FORBIDDEN", + message: "Sending is paused on your account, so campaigns can't resume", + }); + } + // Resuming is an explicit choice, so it also clears a batch that failed long // ago and that the scheduler no longer retries on its own. await CampaignBatchService.clearFailedBatch(campaignId); diff --git a/apps/web/src/server/service/email-queue-service.ts b/apps/web/src/server/service/email-queue-service.ts index 7924fac05..a26a29605 100644 --- a/apps/web/src/server/service/email-queue-service.ts +++ b/apps/web/src/server/service/email-queue-service.ts @@ -10,6 +10,7 @@ import { DEFAULT_QUEUE_OPTIONS } from "../queue/queue-constants"; import { logger } from "../logger/log"; import { createWorkerHandler, TeamJob } from "../queue/bullmq-context"; import { LimitService } from "./limit-service"; +import { saveSentEmail } from "./sending-cohorts"; import { BUILT_IN_CONTACT_VARIABLES, replaceContactVariables, @@ -458,10 +459,13 @@ async function executeEmail(job: QueueEmailJob) { `[EmailQueueService]: Email sent` ); - // Delete attachments and headers after sending the email - await db.email.update({ - where: { id: email.id }, - data: { sesEmailId: messageId, text, attachments: null, headers: null }, + // Delete attachments and headers after sending the email. On cloud this + // also counts the send for the sending rules. + await saveSentEmail(email, { + sesEmailId: messageId, + text, + attachments: null, + headers: null, }); } catch (error: any) { await db.emailEvent.create({ diff --git a/apps/web/src/server/service/limit-service.ts b/apps/web/src/server/service/limit-service.ts index e40bfab21..8a0d36e66 100644 --- a/apps/web/src/server/service/limit-service.ts +++ b/apps/web/src/server/service/limit-service.ts @@ -176,6 +176,15 @@ export class LimitService { }; } + // Paused by the sending rules (or an admin) until someone resumes it. + if (team.sendingPausedAt) { + return { + isLimitReached: true, + limit: 0, + reason: LimitReason.EMAIL_SENDING_PAUSED, + }; + } + // Teams created after paid-only signups need to pay to send, except test // emails to their own members. Strict `=== false` so a team cached before // this field existed isn't caught. diff --git a/apps/web/src/server/service/limit-service.unit.test.ts b/apps/web/src/server/service/limit-service.unit.test.ts index d9c2363ad..ecdb1ea13 100644 --- a/apps/web/src/server/service/limit-service.unit.test.ts +++ b/apps/web/src/server/service/limit-service.unit.test.ts @@ -116,6 +116,20 @@ describe("LimitService.checkEmailLimit paid-only signups", () => { }); }); + it("stops a paused team from sending", async () => { + mockGetTeamCached.mockResolvedValue( + team({ + freePlanAllowed: true, + sendingPausedAt: new Date().toISOString(), + }), + ); + + await expect(LimitService.checkEmailLimit(1)).resolves.toMatchObject({ + isLimitReached: true, + reason: LimitReason.EMAIL_SENDING_PAUSED, + }); + }); + it("reports a block before the missing plan", async () => { mockGetTeamCached.mockResolvedValue(team({ isBlocked: true })); diff --git a/apps/web/src/server/service/sending-cohorts.ts b/apps/web/src/server/service/sending-cohorts.ts new file mode 100644 index 000000000..b23c1da77 --- /dev/null +++ b/apps/web/src/server/service/sending-cohorts.ts @@ -0,0 +1,138 @@ +import type { Prisma } from "@prisma/client"; +import { env } from "~/env"; +import { db } from "~/server/db"; +import { logger } from "~/server/logger/log"; +import { getRedis, redisKey } from "~/server/redis"; + +// Sends and the hard bounces and complaints they cause, filed under the day +// each email was counted as sent. The day is stored on the email itself, so +// an outcome only counts for an email whose send was counted, on that day. + +const OUTCOME_CLAIM_SECONDS = 14 * 24 * 60 * 60; + +export type Outcome = "hardBounced" | "complained"; + +function day(date: Date) { + return date.toISOString().split("T")[0] as string; +} + +/** + * Saves the email after SES accepted it. On cloud, in the same transaction, + * counts the send in the team's cohort for today (and its campaign) and marks + * the email with that day. If counting fails, the email is still saved, + * unmarked, so its bounces and complaints won't count either. + */ +export async function saveSentEmail( + email: { id: string; teamId: number; campaignId: string | null }, + data: Prisma.EmailUpdateInput, +) { + if (!env.NEXT_PUBLIC_IS_CLOUD) { + await db.email.update({ where: { id: email.id }, data }); + return; + } + + const date = day(new Date()); + try { + await db.$transaction([ + db.sendingCohort.upsert({ + where: { teamId_date: { teamId: email.teamId, date } }, + create: { teamId: email.teamId, date, sent: 1 }, + update: { sent: { increment: 1 } }, + }), + ...(email.campaignId + ? [ + db.campaign.update({ + where: { id: email.campaignId }, + data: { acceptedSends: { increment: 1 } }, + }), + ] + : []), + db.email.update({ + where: { id: email.id }, + data: { ...data, cohortDate: date }, + }), + ]); + } catch (error) { + logger.error( + { err: error, emailId: email.id }, + "[SendingCohorts]: Counting the send failed", + ); + await db.email.update({ where: { id: email.id }, data }); + } +} + +export type OutcomeClaim = "claimed" | "duplicate" | "unavailable"; + +function outcomeKey(emailId: string, outcome: Outcome) { + return redisKey(`ses-outcome:${emailId}:${outcome}`); +} + +/** + * Claims a hard bounce or complaint for an email, so a repeated SES event + * counts once. Atomic across workers. "unavailable" means Redis failed: the + * caller still updates dashboards but not the sending-rule counters, since + * undercounting can't wrongly pause anyone. + */ +export async function claimOutcome( + emailId: string, + outcome: Outcome, +): Promise { + try { + const claimed = await getRedis().set( + outcomeKey(emailId, outcome), + "1", + "EX", + OUTCOME_CLAIM_SECONDS, + "NX", + ); + return claimed === "OK" ? "claimed" : "duplicate"; + } catch (error) { + logger.warn({ err: error, emailId }, "[SendingCohorts]: claim failed"); + return "unavailable"; + } +} + +/** Gives a claim back when the work it guarded failed, so a retry can redo it. */ +export async function releaseOutcome(emailId: string, outcome: Outcome) { + try { + await getRedis().del(outcomeKey(emailId, outcome)); + } catch (error) { + logger.warn({ err: error, emailId }, "[SendingCohorts]: release failed"); + } +} + +/** + * Files a hard bounce or complaint under the day its email was counted as + * sent. Emails without that mark (sent before this existed, by an older + * worker, or whose count failed) are skipped. Returns whether it counted. + */ +export async function recordOutcome( + email: { + teamId: number; + campaignId: string | null; + cohortDate: string | null; + }, + outcome: Outcome, +) { + if (!env.NEXT_PUBLIC_IS_CLOUD || !email.cohortDate) return false; + + const date = email.cohortDate; + const campaignField = + outcome === "hardBounced" ? "acceptedHardBounced" : "acceptedComplained"; + await db.$transaction([ + db.sendingCohort.upsert({ + where: { teamId_date: { teamId: email.teamId, date } }, + create: { teamId: email.teamId, date, [outcome]: 1 }, + update: { [outcome]: { increment: 1 } }, + }), + ...(email.campaignId + ? [ + db.campaign.updateMany({ + where: { id: email.campaignId }, + data: { [campaignField]: { increment: 1 } }, + }), + ] + : []), + ]); + return true; +} diff --git a/apps/web/src/server/service/sending-cohorts.unit.test.ts b/apps/web/src/server/service/sending-cohorts.unit.test.ts new file mode 100644 index 000000000..9f412bceb --- /dev/null +++ b/apps/web/src/server/service/sending-cohorts.unit.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { mockEnv, mockDb, redisStore, mockRedisSet } = vi.hoisted(() => { + const redisStore = new Set(); + return { + mockEnv: { NEXT_PUBLIC_IS_CLOUD: true }, + mockDb: { + $transaction: vi.fn(), + sendingCohort: { + upsert: vi.fn((args: unknown) => ({ op: "cohort", args })), + }, + campaign: { + update: vi.fn((args: unknown) => ({ op: "campaign", args })), + updateMany: vi.fn((args: unknown) => ({ + op: "campaignOutcome", + args, + })), + }, + email: { update: vi.fn((args: unknown) => ({ op: "email", args })) }, + }, + redisStore, + mockRedisSet: vi.fn(async (key: string) => { + if (redisStore.has(key)) return null; + redisStore.add(key); + return "OK"; + }), + }; +}); + +vi.mock("~/env", () => ({ env: mockEnv })); +vi.mock("~/server/db", () => ({ db: mockDb })); +vi.mock("~/server/logger/log", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); +vi.mock("~/server/redis", () => ({ + redisKey: (key: string) => key, + getRedis: () => ({ set: mockRedisSet }), +})); + +import { + claimOutcome, + recordOutcome, + saveSentEmail, +} from "~/server/service/sending-cohorts"; + +const sent = { sesEmailId: "ses_1", text: "hi", headers: null }; + +describe("sending cohorts", () => { + beforeEach(() => { + vi.clearAllMocks(); + redisStore.clear(); + mockEnv.NEXT_PUBLIC_IS_CLOUD = true; + mockDb.$transaction.mockResolvedValue([]); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function freezeDate() { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-09T10:00:00Z")); + } + + it("counts the send and marks the email in one transaction", async () => { + freezeDate(); + await saveSentEmail({ id: "e1", teamId: 3, campaignId: "c1" }, sent); + + const ops = mockDb.$transaction.mock.calls[0]?.[0]; + expect(ops.map((op: { op: string }) => op.op)).toEqual([ + "cohort", + "campaign", + "email", + ]); + expect(mockDb.sendingCohort.upsert).toHaveBeenCalledWith({ + where: { teamId_date: { teamId: 3, date: "2026-11-09" } }, + create: { teamId: 3, date: "2026-11-09", sent: 1 }, + update: { sent: { increment: 1 } }, + }); + expect(mockDb.campaign.update).toHaveBeenCalledWith({ + where: { id: "c1" }, + data: { acceptedSends: { increment: 1 } }, + }); + expect(mockDb.email.update).toHaveBeenCalledWith({ + where: { id: "e1" }, + data: { ...sent, cohortDate: "2026-11-09" }, + }); + }); + + it("still saves the email, unmarked, if counting fails", async () => { + freezeDate(); + mockDb.$transaction.mockRejectedValue(new Error("db busy")); + + await saveSentEmail({ id: "e1", teamId: 3, campaignId: null }, sent); + + expect(mockDb.email.update).toHaveBeenLastCalledWith({ + where: { id: "e1" }, + data: sent, + }); + }); + + it("files an outcome under the day its email was counted", async () => { + await expect( + recordOutcome( + { teamId: 3, campaignId: "c1", cohortDate: "2026-11-09" }, + "complained", + ), + ).resolves.toBe(true); + + expect(mockDb.campaign.updateMany).toHaveBeenCalledWith({ + where: { id: "c1" }, + data: { acceptedComplained: { increment: 1 } }, + }); + + expect(mockDb.sendingCohort.upsert).toHaveBeenCalledWith({ + where: { teamId_date: { teamId: 3, date: "2026-11-09" } }, + create: { teamId: 3, date: "2026-11-09", complained: 1 }, + update: { complained: { increment: 1 } }, + }); + }); + + it("skips outcomes of emails whose send wasn't counted", async () => { + await expect( + recordOutcome( + { teamId: 3, campaignId: null, cohortDate: null }, + "hardBounced", + ), + ).resolves.toBe(false); + + expect(mockDb.sendingCohort.upsert).not.toHaveBeenCalled(); + }); + + it("claims each outcome once per email", async () => { + await expect(claimOutcome("email_1", "complained")).resolves.toBe( + "claimed", + ); + await expect(claimOutcome("email_1", "complained")).resolves.toBe( + "duplicate", + ); + await expect(claimOutcome("email_1", "hardBounced")).resolves.toBe( + "claimed", + ); + }); + + it("reports when Redis can't check for duplicates", async () => { + mockRedisSet.mockRejectedValueOnce(new Error("redis down")); + + await expect(claimOutcome("email_2", "complained")).resolves.toBe( + "unavailable", + ); + }); + + it("only saves the email on self-hosted", async () => { + mockEnv.NEXT_PUBLIC_IS_CLOUD = false; + + await saveSentEmail({ id: "e1", teamId: 3, campaignId: null }, sent); + await recordOutcome( + { teamId: 3, campaignId: null, cohortDate: "2026-11-09" }, + "complained", + ); + + expect(mockDb.$transaction).not.toHaveBeenCalled(); + expect(mockDb.sendingCohort.upsert).not.toHaveBeenCalled(); + expect(mockDb.email.update).toHaveBeenCalledWith({ + where: { id: "e1" }, + data: sent, + }); + }); +}); diff --git a/apps/web/src/server/service/sending-rules-service.ts b/apps/web/src/server/service/sending-rules-service.ts new file mode 100644 index 000000000..e186baca1 --- /dev/null +++ b/apps/web/src/server/service/sending-rules-service.ts @@ -0,0 +1,442 @@ +import type { Prisma, TeamActionType } from "@prisma/client"; +import { env } from "~/env"; +import { db } from "~/server/db"; +import { logger } from "~/server/logger/log"; +import { sendMail } from "~/server/mailer"; +import { getRedis, redisKey } from "~/server/redis"; +import { toPlainHtml } from "~/server/utils/email-content"; +import { sendToDiscord } from "./notification-service"; +import { TeamService } from "./team-service"; +import { + judgeCounts, + SENDING_RULES, + type SendingCounts, + type Verdict, +} from "./sending-rules"; + +export { isSpamTrapBounce, judgeCounts, SENDING_RULES } from "./sending-rules"; + +const ALERT_COOLDOWN_SECONDS = 24 * 60 * 60; + +function toDateString(date: Date) { + return date.toISOString().split("T")[0] as string; +} + +type Client = Prisma.TransactionClient | typeof db; + +/** + * The last time an admin resumed or unblocked the team, and which of its + * campaigns hadn't sent anything yet at that moment. + */ +type Reset = { + id: string; + createdAt: Date; + unsentCampaignIds: string[]; +} | null; + +async function lastReset(teamId: number, client: Client = db): Promise { + const reset = await client.teamAction.findFirst({ + where: { teamId, action: { in: ["RESUME", "UNBLOCK"] } }, + orderBy: { createdAt: "desc" }, + select: { id: true, createdAt: true, details: true }, + }); + if (!reset) return null; + const details = reset.details as { unsentCampaignIds?: string[] } | null; + return { + id: reset.id, + createdAt: reset.createdAt, + unsentCampaignIds: details?.unsentCampaignIds ?? [], + }; +} + +/** + * Stored on an admin resume or unblock. Campaigns that had already sent are + * not judged on their totals again (late bounces from before the reset would + * otherwise count against no new sends); the team's rolling window still + * covers them. Campaigns that hadn't sent yet are judged in full. + */ +export async function unsentCampaignIds( + client: Client, + teamId: number, +): Promise { + const campaigns = await client.campaign.findMany({ + where: { teamId, sent: 0, acceptedSends: 0, status: { not: "SENT" } }, + select: { id: true }, + orderBy: { createdAt: "desc" }, + take: 1000, + }); + return campaigns.map((campaign) => campaign.id); +} + +/** + * Locks the team row and confirms no admin reset happened since this check + * started, so a check computed before an unblock can't re-block on old counts. + * The admin settings update takes the same lock. + */ +async function resetUnchanged( + tx: Prisma.TransactionClient, + teamId: number, + seen: Reset, +) { + await tx.$queryRaw`SELECT id FROM "Team" WHERE id = ${teamId} FOR UPDATE`; + const latest = await lastReset(teamId, tx); + return (latest?.id ?? null) === (seen?.id ?? null); +} + +/** Notices shouldn't hold up SES event processing, or fail it. */ +function inBackground(task: Promise, what: string) { + void task.catch((error: unknown) => + logger.error({ err: error }, `[SendingRules]: ${what} failed`), + ); +} + +/** + * First day counted, `days` back including today. After an admin resumes or + * unblocks a team, days up to and including that day no longer count. + */ +function windowStart(days: number, reset: Reset) { + const start = new Date(); + start.setUTCDate(start.getUTCDate() - (days - 1)); + let from = toDateString(start); + + if (reset) { + const dayAfter = new Date(reset.createdAt); + dayAfter.setUTCDate(dayAfter.getUTCDate() + 1); + const resetFrom = toDateString(dayAfter); + if (resetFrom > from) from = resetFrom; + } + + return from; +} + +/** + * Emails the team sent over the last 7 days, and the hard bounces and + * complaints those same emails caused, whenever they arrived. + */ +async function teamCounts( + teamId: number, + reset: Reset, +): Promise { + const result = await db.sendingCohort.aggregate({ + where: { + teamId, + date: { gte: windowStart(SENDING_RULES.windowDays, reset) }, + sent: { gt: 0 }, + }, + _sum: { sent: true, hardBounced: true, complained: true }, + }); + return { + sent: result._sum.sent ?? 0, + hardBounced: result._sum.hardBounced ?? 0, + complained: result._sum.complained ?? 0, + }; +} + +async function recordAction( + teamId: number, + action: TeamActionType, + reason: string, + details?: Prisma.InputJsonValue, +) { + await db.teamAction.create({ + data: { teamId, action, actor: "rules", reason, details }, + }); +} + +/** One alert per key per day, so a burst of bounces posts once. */ +async function claimAlert(key: string) { + const acquired = await getRedis().set( + redisKey(`sending-rules:alert:${key}`), + "1", + "EX", + ALERT_COOLDOWN_SECONDS, + "NX", + ); + return acquired === "OK"; +} + +async function teamLabel(teamId: number) { + const team = await db.team.findUnique({ + where: { id: teamId }, + select: { name: true }, + }); + return `${team?.name ?? "Unknown team"} (#${teamId})`; +} + +async function emailTeam(teamId: number, subject: string, text: string) { + const teamUsers = await TeamService.getTeamUsers(teamId); + const recipients = teamUsers + .map((teamUser) => teamUser.user?.email) + .filter((email): email is string => Boolean(email)); + + const results = await Promise.allSettled( + recipients.map((to) => + sendMail(to, subject, text, toPlainHtml(text), "hey@usesend.com"), + ), + ); + results.forEach((result) => { + if (result.status === "rejected") { + logger.error( + { err: result.reason, teamId }, + "[SendingRules]: Team email failed", + ); + } + }); +} + +async function notify(message: string) { + try { + await sendToDiscord(message); + } catch (error) { + logger.error({ err: error }, "[SendingRules]: Discord alert failed"); + } +} + +async function actOnTeam( + teamId: number, + verdict: Exclude, + details: Prisma.InputJsonValue, + reset: Reset, +) { + const label = await teamLabel(teamId); + + if (env.SENDING_RULES_MODE === "alert") { + if (!(await claimAlert(`team:${teamId}:${verdict.level}`))) return; + await recordAction( + teamId, + "ALERT", + `Would ${verdict.level}: ${verdict.reason}`, + details, + ); + inBackground( + notify(`[alert only] Would ${verdict.level} ${label}: ${verdict.reason}`), + "Discord alert", + ); + return; + } + + // Conditional updates make this safe when many events arrive at once: + // only the first one changes the team and sends the notices. + const acted = await db.$transaction(async (tx) => { + if (!(await resetUnchanged(tx, teamId, reset))) return false; + + const { count } = + verdict.level === "block" + ? await tx.team.updateMany({ + where: { id: teamId, isBlocked: false }, + data: { isBlocked: true }, + }) + : await tx.team.updateMany({ + where: { id: teamId, isBlocked: false, sendingPausedAt: null }, + data: { sendingPausedAt: new Date() }, + }); + if (count === 0) return false; + + await tx.campaign.updateMany({ + where: { teamId, status: { in: ["RUNNING", "SCHEDULED"] } }, + data: { status: "PAUSED" }, + }); + await tx.teamAction.create({ + data: { + teamId, + action: verdict.level === "block" ? "BLOCK" : "PAUSE", + actor: "rules", + reason: verdict.reason, + details, + }, + }); + return true; + }); + if (!acted) return; + + try { + await TeamService.refreshTeamCache(teamId); + } catch (error) { + // The cache expires within 2 minutes anyway. + logger.error( + { err: error, teamId }, + "[SendingRules]: Cache refresh failed", + ); + } + + const name = label.replace(/ \(#\d+\)$/, ""); + + if (verdict.level === "block") { + inBackground( + notify(`Blocked ${label}: ${verdict.reason}`), + "Discord alert", + ); + inBackground( + emailTeam( + teamId, + "useSend: sending is blocked on your account", + `Hi ${name} team,\n\nWe've blocked sending on your useSend account because ${verdict.reason}.\n\nEmails won't go out until we've reviewed the account with you. Reply to this email to talk to us.\n\nuseSend`, + ), + "Team email", + ); + } else { + inBackground(notify(`Paused ${label}: ${verdict.reason}`), "Discord alert"); + inBackground( + emailTeam( + teamId, + "useSend: we paused sending on your account", + `Hi ${name} team,\n\nWe paused sending on your useSend account because ${verdict.reason} over the last ${SENDING_RULES.windowDays} days.\n\nThis usually means some addresses on your list are old, mistyped, or never signed up to hear from you. Please remove them, then reply to this email and we'll turn sending back on.\n\nWhile sending is paused, new emails fail instead of going out.\n\nuseSend`, + ), + "Team email", + ); + } +} + +async function checkCampaign(teamId: number, campaignId: string, reset: Reset) { + const campaign = await db.campaign.findFirst({ + where: { id: campaignId, teamId }, + select: { + id: true, + name: true, + acceptedSends: true, + acceptedHardBounced: true, + acceptedComplained: true, + createdAt: true, + }, + }); + if (!campaign) return; + + // After an admin resume or unblock, campaigns that had already sent aren't + // judged on their totals again; the team's rolling window still applies. + if ( + reset && + campaign.createdAt < reset.createdAt && + !reset.unsentCampaignIds.includes(campaign.id) + ) { + return; + } + + // Only emails counted when we handed them to SES, and the bounces and + // complaints of those same emails, like the team check. + const verdict = judgeCounts({ + sent: campaign.acceptedSends, + hardBounced: campaign.acceptedHardBounced, + complained: campaign.acceptedComplained, + }); + if (verdict.level === "ok") return; + + const details = { scope: "campaign", campaignId: campaign.id }; + + if (verdict.level === "block") { + await actOnTeam( + teamId, + { + level: "block", + reason: `${verdict.reason} on campaign "${campaign.name}"`, + }, + details, + reset, + ); + return; + } + + // Once a campaign was paused and its owner resumed it, they've accepted the + // risk: only the block level stops it again. + const pausedBefore = await db.teamAction.findFirst({ + where: { + teamId, + action: env.SENDING_RULES_MODE === "alert" ? "ALERT" : "PAUSE", + details: { path: ["campaignId"], equals: campaign.id }, + }, + select: { id: true }, + }); + if (pausedBefore) return; + + const label = await teamLabel(teamId); + + if (env.SENDING_RULES_MODE === "alert") { + if (!(await claimAlert(`campaign:${campaign.id}`))) return; + await recordAction( + teamId, + "ALERT", + `Would pause campaign "${campaign.name}": ${verdict.reason}`, + details, + ); + inBackground( + notify( + `[alert only] Would pause campaign "${campaign.name}" of ${label}: ${verdict.reason}`, + ), + "Discord alert", + ); + return; + } + + const acted = await db.$transaction(async (tx) => { + if (!(await resetUnchanged(tx, teamId, reset))) return false; + + const { count } = await tx.campaign.updateMany({ + where: { id: campaign.id, status: { in: ["RUNNING", "SCHEDULED"] } }, + data: { status: "PAUSED" }, + }); + if (count === 0) return false; + await tx.teamAction.create({ + data: { + teamId, + action: "PAUSE", + actor: "rules", + reason: `Paused campaign "${campaign.name}": ${verdict.reason}`, + details, + }, + }); + return true; + }); + if (!acted) return; + + inBackground( + notify(`Paused campaign "${campaign.name}" of ${label}: ${verdict.reason}`), + "Discord alert", + ); + inBackground( + emailTeam( + teamId, + `useSend: we paused your campaign "${campaign.name}"`, + `Hi there,\n\nWe paused your campaign "${campaign.name}" because ${verdict.reason}.\n\nPlease check where these contacts came from before you resume it. If it keeps going and gets worse, we'll have to block sending on your account.\n\nuseSend`, + ), + "Team email", + ); +} + +/** + * Runs after every hard bounce and complaint (cloud only). Pauses or blocks + * the team or campaign when its rates cross the limits, emails the team and + * posts to Discord. SENDING_RULES_MODE "alert" only posts; "off" disables it. + */ +export async function checkSendingHealth(params: { + teamId: number; + campaignId?: string | null; + spamTrap?: boolean; +}) { + if (!env.NEXT_PUBLIC_IS_CLOUD || env.SENDING_RULES_MODE === "off") return; + + const { teamId, campaignId } = params; + const reset = await lastReset(teamId); + + if (params.spamTrap) { + await actOnTeam( + teamId, + { + level: "block", + reason: + "an email reached a spam trap, an address found only on bought, scraped or old lists", + }, + { scope: "spamTrap", campaignId: campaignId ?? null }, + reset, + ); + return; + } + + if (campaignId) { + await checkCampaign(teamId, campaignId, reset); + } + + const counts = await teamCounts(teamId, reset); + const verdict = judgeCounts(counts); + if (verdict.level !== "ok") { + await actOnTeam(teamId, verdict, { scope: "team", ...counts }, reset); + } +} diff --git a/apps/web/src/server/service/sending-rules-service.unit.test.ts b/apps/web/src/server/service/sending-rules-service.unit.test.ts new file mode 100644 index 000000000..9e7ae100a --- /dev/null +++ b/apps/web/src/server/service/sending-rules-service.unit.test.ts @@ -0,0 +1,427 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { + mockEnv, + mockDb, + redisStore, + mockSendToDiscord, + mockSendMail, + mockRefreshTeamCache, +} = vi.hoisted(() => ({ + mockEnv: { + NEXT_PUBLIC_IS_CLOUD: true, + SENDING_RULES_MODE: "enforce" as "off" | "alert" | "enforce", + }, + mockDb: { + $transaction: vi.fn(), + $queryRaw: vi.fn(), + team: { findUnique: vi.fn(), updateMany: vi.fn() }, + campaign: { findFirst: vi.fn(), findMany: vi.fn(), updateMany: vi.fn() }, + teamAction: { findFirst: vi.fn(), create: vi.fn() }, + sendingCohort: { aggregate: vi.fn() }, + }, + redisStore: new Set(), + mockSendToDiscord: vi.fn(), + mockSendMail: vi.fn(), + mockRefreshTeamCache: vi.fn(), +})); + +vi.mock("~/env", () => ({ env: mockEnv })); +vi.mock("~/server/db", () => ({ db: mockDb })); +vi.mock("~/server/logger/log", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); +vi.mock("~/server/mailer", () => ({ sendMail: mockSendMail })); +vi.mock("~/server/redis", () => ({ + redisKey: (key: string) => key, + getRedis: () => ({ + set: vi.fn(async (key: string) => { + if (redisStore.has(key)) return null; + redisStore.add(key); + return "OK"; + }), + }), +})); +vi.mock("~/server/service/notification-service", () => ({ + sendToDiscord: mockSendToDiscord, +})); +vi.mock("~/server/service/team-service", () => ({ + TeamService: { + refreshTeamCache: mockRefreshTeamCache, + getTeamUsers: vi.fn(async () => [ + { user: { email: "owner@acme.com" } }, + { user: { email: null } }, + ]), + }, +})); + +import { + checkSendingHealth, + isSpamTrapBounce, + judgeCounts, +} from "~/server/service/sending-rules-service"; + +/** Lets the notices that run in the background finish. */ +async function flush() { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +function usage(sent: number, hardBounced: number, complained = 0) { + mockDb.sendingCohort.aggregate.mockResolvedValue({ + _sum: { sent, hardBounced, complained }, + }); +} + +describe("judgeCounts", () => { + it("needs both the rate and a minimum count", () => { + expect(judgeCounts({ sent: 10, hardBounced: 4, complained: 0 }).level).toBe( + "ok", + ); + expect( + judgeCounts({ sent: 1000, hardBounced: 25, complained: 0 }).level, + ).toBe("ok"); + expect( + judgeCounts({ sent: 1000, hardBounced: 26, complained: 0 }).level, + ).toBe("pause"); + expect( + judgeCounts({ sent: 1000, hardBounced: 51, complained: 0 }).level, + ).toBe("block"); + expect( + judgeCounts({ sent: 100, hardBounced: 9, complained: 0 }).level, + ).toBe("pause"); + }); + + it("checks spam complaints", () => { + expect( + judgeCounts({ sent: 1000, hardBounced: 0, complained: 1 }).level, + ).toBe("ok"); + expect( + judgeCounts({ sent: 1000, hardBounced: 0, complained: 2 }).level, + ).toBe("pause"); + expect( + judgeCounts({ sent: 1000, hardBounced: 0, complained: 4 }).level, + ).toBe("block"); + expect( + judgeCounts({ sent: 100_000, hardBounced: 0, complained: 50 }).level, + ).toBe("ok"); + }); + + it("explains the reason in plain words", () => { + expect(judgeCounts({ sent: 1000, hardBounced: 30, complained: 0 })).toEqual( + { + level: "pause", + reason: "too many emails bounced: 3.00% (30 of 1,000)", + }, + ); + }); +}); + +describe("isSpamTrapBounce", () => { + it("spots spam traps and honeypots in bounce diagnostics", () => { + expect( + isSpamTrapBounce({ + bouncedRecipients: [ + { diagnosticCode: "550 5.7.1 Message rejected: spam-trap hit" }, + ], + }), + ).toBe(true); + expect( + isSpamTrapBounce({ + bouncedRecipients: [{ diagnosticCode: "550 honeypot address" }], + }), + ).toBe(true); + expect( + isSpamTrapBounce({ + bouncedRecipients: [{ diagnosticCode: "550 5.1.1 user unknown" }], + }), + ).toBe(false); + expect(isSpamTrapBounce(undefined)).toBe(false); + }); +}); + +describe("checkSendingHealth", () => { + beforeEach(() => { + vi.clearAllMocks(); + redisStore.clear(); + mockEnv.NEXT_PUBLIC_IS_CLOUD = true; + mockEnv.SENDING_RULES_MODE = "enforce"; + mockDb.$transaction.mockImplementation( + async (fn: (tx: typeof mockDb) => unknown) => fn(mockDb), + ); + mockDb.team.findUnique.mockResolvedValue({ name: "Acme" }); + mockDb.team.updateMany.mockResolvedValue({ count: 1 }); + mockDb.campaign.updateMany.mockResolvedValue({ count: 1 }); + mockDb.teamAction.findFirst.mockResolvedValue(null); + usage(1000, 0, 0); + }); + + it("does nothing while rates are healthy", async () => { + await checkSendingHealth({ teamId: 1 }); + + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockSendToDiscord).not.toHaveBeenCalled(); + }); + + it("pauses the team, its campaigns, and tells everyone", async () => { + usage(1000, 30); + + await checkSendingHealth({ teamId: 1 }); + await flush(); + + expect(mockDb.$transaction).toHaveBeenCalledTimes(1); + expect(mockDb.team.updateMany).toHaveBeenCalledWith({ + where: { id: 1, isBlocked: false, sendingPausedAt: null }, + data: { sendingPausedAt: expect.any(Date) }, + }); + expect(mockDb.campaign.updateMany).toHaveBeenCalledWith({ + where: { teamId: 1, status: { in: ["RUNNING", "SCHEDULED"] } }, + data: { status: "PAUSED" }, + }); + expect(mockRefreshTeamCache).toHaveBeenCalledWith(1); + expect(mockDb.teamAction.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + teamId: 1, + action: "PAUSE", + actor: "rules", + }), + }); + expect(mockSendToDiscord).toHaveBeenCalledWith( + expect.stringContaining("Paused Acme (#1)"), + ); + expect(mockSendMail).toHaveBeenCalledTimes(1); + expect(mockSendMail.mock.calls[0]?.[0]).toBe("owner@acme.com"); + }); + + it("acts only once when the team was already paused", async () => { + usage(1000, 30); + mockDb.team.updateMany.mockResolvedValue({ count: 0 }); + + await checkSendingHealth({ teamId: 1 }); + + expect(mockDb.teamAction.create).not.toHaveBeenCalled(); + expect(mockSendToDiscord).not.toHaveBeenCalled(); + expect(mockSendMail).not.toHaveBeenCalled(); + }); + + it("blocks at the higher level", async () => { + usage(1000, 0, 4); + + await checkSendingHealth({ teamId: 1 }); + + expect(mockDb.team.updateMany).toHaveBeenCalledWith({ + where: { id: 1, isBlocked: false }, + data: { isBlocked: true }, + }); + expect(mockDb.teamAction.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ action: "BLOCK" }), + }); + }); + + it("blocks at once on a spam trap", async () => { + await checkSendingHealth({ teamId: 1, spamTrap: true }); + + expect(mockDb.team.updateMany).toHaveBeenCalledWith({ + where: { id: 1, isBlocked: false }, + data: { isBlocked: true }, + }); + expect(mockDb.sendingCohort.aggregate).not.toHaveBeenCalled(); + }); + + it("only alerts, once a day, in alert mode", async () => { + mockEnv.SENDING_RULES_MODE = "alert"; + usage(1000, 30); + + await checkSendingHealth({ teamId: 1 }); + await checkSendingHealth({ teamId: 1 }); + await flush(); + + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + expect(mockSendMail).not.toHaveBeenCalled(); + expect(mockDb.teamAction.create).toHaveBeenCalledTimes(1); + expect(mockDb.teamAction.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ action: "ALERT" }), + }); + expect(mockSendToDiscord).toHaveBeenCalledTimes(1); + expect(mockSendToDiscord).toHaveBeenCalledWith( + expect.stringContaining("[alert only] Would pause"), + ); + }); + + it("pauses a bad campaign without pausing the team", async () => { + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_1", + name: "November news", + acceptedSends: 400, + acceptedHardBounced: 20, + acceptedComplained: 0, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + + await checkSendingHealth({ teamId: 1, campaignId: "cmp_1" }); + await flush(); + + expect(mockDb.campaign.updateMany).toHaveBeenCalledWith({ + where: { id: "cmp_1", status: { in: ["RUNNING", "SCHEDULED"] } }, + data: { status: "PAUSED" }, + }); + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockSendMail.mock.calls[0]?.[1]).toContain("November news"); + }); + + it("doesn't pause a campaign again after its owner resumed it", async () => { + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_1", + name: "November news", + acceptedSends: 400, + acceptedHardBounced: 20, + acceptedComplained: 0, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + mockDb.teamAction.findFirst.mockImplementation(async (args: any) => + args.where.details ? { id: "act_1" } : null, + ); + + await checkSendingHealth({ teamId: 1, campaignId: "cmp_1" }); + + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + + it("counts campaign sends at send time, not when SES reports them", async () => { + // 10 bounces against 10,000 emails counted when SES accepted them, + // even though SES has only reported 10 sends so far. + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_1", + name: "Big send", + acceptedSends: 10_000, + acceptedHardBounced: 10, + acceptedComplained: 0, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + + await checkSendingHealth({ teamId: 1, campaignId: "cmp_1" }); + + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + + it("doesn't judge an already-sent campaign on old totals after an unblock", async () => { + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_1", + name: "Old blast", + acceptedSends: 100_000, + acceptedHardBounced: 0, + acceptedComplained: 3, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + mockDb.teamAction.findFirst.mockImplementation(async (args: any) => + args.where.action?.in + ? { + id: "reset_1", + createdAt: new Date("2026-11-05T00:00:00Z"), + details: { unsentCampaignIds: [] }, + } + : null, + ); + + await checkSendingHealth({ teamId: 1, campaignId: "cmp_1" }); + + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + }); + + it("still checks a draft that first sends after an unblock", async () => { + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_draft", + name: "Draft", + acceptedSends: 400, + acceptedHardBounced: 200, + acceptedComplained: 0, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + mockDb.teamAction.findFirst.mockImplementation(async (args: any) => + args.where.action?.in + ? { + id: "reset_1", + createdAt: new Date("2026-11-05T00:00:00Z"), + details: { unsentCampaignIds: ["cmp_draft"] }, + } + : null, + ); + + await checkSendingHealth({ teamId: 1, campaignId: "cmp_draft" }); + + expect(mockDb.team.updateMany).toHaveBeenCalledWith({ + where: { id: 1, isBlocked: false }, + data: { isBlocked: true }, + }); + }); + + it("compares sends with the bounces and complaints they caused", async () => { + await checkSendingHealth({ teamId: 1 }); + + const where = mockDb.sendingCohort.aggregate.mock.calls[0]?.[0].where; + expect(where.teamId).toBe(1); + // Only days we counted sends for; outcomes filed under their send day. + expect(where.sent).toEqual({ gt: 0 }); + }); + + it("doesn't re-block on old counts when an admin unblocked mid-check", async () => { + usage(1000, 0, 4); + // No reset when the check starts; one appears before it acts. + mockDb.teamAction.findFirst.mockResolvedValueOnce(null).mockResolvedValue({ + id: "reset_new", + createdAt: new Date(), + details: null, + }); + + await checkSendingHealth({ teamId: 1 }); + + expect(mockDb.$queryRaw).toHaveBeenCalled(); + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockDb.teamAction.create).not.toHaveBeenCalled(); + }); + + it("keeps processing when a notice fails", async () => { + usage(1000, 30); + mockSendToDiscord.mockRejectedValue(new Error("discord down")); + mockSendMail.mockRejectedValue(new Error("smtp down")); + + await expect(checkSendingHealth({ teamId: 1 })).resolves.toBeUndefined(); + await flush(); + + expect(mockDb.teamAction.create).toHaveBeenCalledTimes(1); + }); + + it("ignores days up to an admin resume", async () => { + const resumedAt = new Date(); + mockDb.teamAction.findFirst.mockResolvedValue({ createdAt: resumedAt }); + + await checkSendingHealth({ teamId: 1 }); + + const tomorrow = new Date(resumedAt); + tomorrow.setUTCDate(tomorrow.getUTCDate() + 1); + expect(mockDb.sendingCohort.aggregate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + teamId: 1, + date: { gte: tomorrow.toISOString().split("T")[0] }, + sent: { gt: 0 }, + }, + }), + ); + }); + + it("is off on self-hosted and when switched off", async () => { + usage(1000, 500); + mockEnv.NEXT_PUBLIC_IS_CLOUD = false; + await checkSendingHealth({ teamId: 1, spamTrap: true }); + + mockEnv.NEXT_PUBLIC_IS_CLOUD = true; + mockEnv.SENDING_RULES_MODE = "off"; + await checkSendingHealth({ teamId: 1, spamTrap: true }); + + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockDb.sendingCohort.aggregate).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/server/service/sending-rules.ts b/apps/web/src/server/service/sending-rules.ts new file mode 100644 index 000000000..96c981924 --- /dev/null +++ b/apps/web/src/server/service/sending-rules.ts @@ -0,0 +1,81 @@ +// Pure bounce/complaint rules, kept free of imports so the SES event parser +// and tests can use them without loading the database or mailer. + +/** + * Bounce and complaint limits. A level needs both the rate and a minimum + * count, so a single bounce or complaint never trips anything. They sit below + * the rates at which AWS reviews our whole SES account (5% / 0.1%). + */ +export const SENDING_RULES = { + /** Days of sends judged, with the bounces and complaints they caused. */ + windowDays: 7, + pause: { + bounceRate: 0.025, + minBounces: 5, + complaintRate: 0.0008, + minComplaints: 2, + }, + block: { + bounceRate: 0.05, + minBounces: 10, + complaintRate: 0.003, + minComplaints: 3, + }, +} as const; + +const SPAM_TRAP_PATTERN = /spam[\s_-]?trap|honey[\s_-]?pot/i; + +export type SendingCounts = { + sent: number; + hardBounced: number; + complained: number; +}; + +export type Verdict = + { level: "ok" } | { level: "pause" | "block"; reason: string }; + +function percent(rate: number) { + return `${(rate * 100).toFixed(2)}%`; +} + +/** Applies the limits to a set of counts. Block is checked before pause. */ +export function judgeCounts(counts: SendingCounts): Verdict { + const sent = Math.max(counts.sent, counts.hardBounced, counts.complained, 1); + const bounceRate = counts.hardBounced / sent; + const complaintRate = counts.complained / sent; + + for (const level of ["block", "pause"] as const) { + const limits = SENDING_RULES[level]; + if ( + counts.hardBounced >= limits.minBounces && + bounceRate > limits.bounceRate + ) { + return { + level, + reason: `too many emails bounced: ${percent(bounceRate)} (${counts.hardBounced.toLocaleString()} of ${sent.toLocaleString()})`, + }; + } + if ( + counts.complained >= limits.minComplaints && + complaintRate > limits.complaintRate + ) { + return { + level, + reason: `too many people marked your emails as spam: ${percent(complaintRate)} (${counts.complained.toLocaleString()} of ${sent.toLocaleString()})`, + }; + } + } + + return { level: "ok" }; +} + +/** A hard bounce whose diagnostic names a spam trap or honeypot. */ +export function isSpamTrapBounce(bounce: unknown): boolean { + const recipients = ( + bounce as { bouncedRecipients?: Array<{ diagnosticCode?: string }> } + )?.bouncedRecipients; + if (!Array.isArray(recipients)) return false; + return recipients.some((recipient) => + SPAM_TRAP_PATTERN.test(recipient?.diagnosticCode ?? ""), + ); +} diff --git a/apps/web/src/server/service/ses-hook-parser.ts b/apps/web/src/server/service/ses-hook-parser.ts index 34d32c559..d18477e87 100644 --- a/apps/web/src/server/service/ses-hook-parser.ts +++ b/apps/web/src/server/service/ses-hook-parser.ts @@ -3,6 +3,7 @@ import { SuppressionReason, UnsubscribeReason, type Email, + type Prisma, } from "@prisma/client"; import { type EmailBasePayload, @@ -31,6 +32,9 @@ import { getChildLogger, logger, withLogger } from "../logger/log"; import { randomUUID } from "crypto"; import { SuppressionService } from "./suppression-service"; import { WebhookService } from "./webhook-service"; +import { checkSendingHealth } from "./sending-rules-service"; +import { isSpamTrapBounce } from "./sending-rules"; +import { claimOutcome, recordOutcome, releaseOutcome } from "./sending-cohorts"; export async function parseSesHook(data: SesEvent) { const mailStatus = getEmailStatus(data); @@ -204,76 +208,106 @@ export async function parseSesHook(data: SesEvent) { const isDuplicateEngagement = Boolean(existingMailEvent) && isEngagementEvent; - if ( - !isDuplicateEngagement && - [ - "DELIVERED", - "OPENED", - "CLICKED", - "BOUNCED", - "COMPLAINED", - "SENT", - ].includes(mailStatus) - ) { - logger.info("Updating daily email usage"); - const updateField = mailStatus.toLowerCase(); - - await db.dailyEmailUsage.upsert({ - where: { - teamId_domainId_date_type: { - teamId: email.teamId, - domainId: email.domainId ?? 0, - date: today, - type: email.campaignId ? "MARKETING" : "TRANSACTIONAL", - }, - }, - create: { - teamId: email.teamId, - domainId: email.domainId ?? 0, - date: today, - type: email.campaignId ? "MARKETING" : "TRANSACTIONAL", - delivered: updateField === "delivered" ? 1 : 0, - opened: updateField === "opened" ? 1 : 0, - clicked: updateField === "clicked" ? 1 : 0, - bounced: updateField === "bounced" ? 1 : 0, - complained: updateField === "complained" ? 1 : 0, - sent: updateField === "sent" ? 1 : 0, - hardBounced: isHardBounced ? 1 : 0, - }, - update: { - [updateField]: { - increment: 1, - }, - ...(isHardBounced ? { hardBounced: { increment: 1 } } : {}), - }, - }); + // SES can deliver the same hard bounce or complaint more than once. Count + // each only once per email: the sending rules and dashboards rely on it. + const outcome = isHardBounced + ? ("hardBounced" as const) + : mailStatus === EmailStatus.COMPLAINED + ? ("complained" as const) + : null; + const claim = outcome ? await claimOutcome(email.id, outcome) : null; + // "unavailable" (Redis failed) still updates the dashboards, as before; + // only a confirmed duplicate is skipped. + const isDuplicateOutcome = claim === "duplicate"; + try { if ( - isHardBounced || - updateField === "complained" || - updateField === "delivered" + !isDuplicateEngagement && + !isDuplicateOutcome && + [ + "DELIVERED", + "OPENED", + "CLICKED", + "BOUNCED", + "COMPLAINED", + "SENT", + ].includes(mailStatus) ) { - logger.info("Updating cumulated metrics"); - const cumulatedField = isHardBounced ? "hardBounced" : updateField; - await db.cumulatedMetrics.upsert({ - where: { - teamId_domainId: { + logger.info("Updating daily email usage"); + const updateField = mailStatus.toLowerCase(); + + // Both counters in one transaction, so a failure can't leave one + // counted and the other not (which a replay would then double). + const usageWrites: Prisma.PrismaPromise[] = [ + db.dailyEmailUsage.upsert({ + where: { + teamId_domainId_date_type: { + teamId: email.teamId, + domainId: email.domainId ?? 0, + date: today, + type: email.campaignId ? "MARKETING" : "TRANSACTIONAL", + }, + }, + create: { teamId: email.teamId, domainId: email.domainId ?? 0, + date: today, + type: email.campaignId ? "MARKETING" : "TRANSACTIONAL", + delivered: updateField === "delivered" ? 1 : 0, + opened: updateField === "opened" ? 1 : 0, + clicked: updateField === "clicked" ? 1 : 0, + bounced: updateField === "bounced" ? 1 : 0, + complained: updateField === "complained" ? 1 : 0, + sent: updateField === "sent" ? 1 : 0, + hardBounced: isHardBounced ? 1 : 0, }, - }, - update: { - [cumulatedField]: { - increment: BigInt(1), + update: { + [updateField]: { + increment: 1, + }, + ...(isHardBounced ? { hardBounced: { increment: 1 } } : {}), }, - }, - create: { - teamId: email.teamId, - domainId: email.domainId ?? 0, - [cumulatedField]: BigInt(1), - }, - }); + }), + ]; + + if ( + isHardBounced || + updateField === "complained" || + updateField === "delivered" + ) { + logger.info("Updating cumulated metrics"); + const cumulatedField = isHardBounced ? "hardBounced" : updateField; + usageWrites.push( + db.cumulatedMetrics.upsert({ + where: { + teamId_domainId: { + teamId: email.teamId, + domainId: email.domainId ?? 0, + }, + }, + update: { + [cumulatedField]: { + increment: BigInt(1), + }, + }, + create: { + teamId: email.teamId, + domainId: email.domainId ?? 0, + [cumulatedField]: BigInt(1), + }, + }), + ); + } + + await db.$transaction(usageWrites); } + } catch (error) { + // Nothing was counted, so give the claim back: a redelivery or replay of + // this event can then count it instead of finding it claimed. + if (outcome && claim === "claimed") { + await releaseOutcome(email.id, outcome); + } + throw error; } if (email.campaignId) { @@ -299,6 +333,33 @@ export async function parseSesHook(data: SesEvent) { } } + // Rule counters only for an event we know we haven't counted before. + if (outcome && claim === "claimed") { + try { + await recordOutcome(email, outcome); + } catch (error) { + // Keep the claim: if the write did commit, releasing it could count the + // outcome twice. Undercounting can't wrongly pause anyone. + logger.error( + { err: error, teamId: email.teamId }, + "[SesHookParser]: Counting the outcome failed", + ); + } + try { + await checkSendingHealth({ + teamId: email.teamId, + campaignId: email.campaignId, + spamTrap: isHardBounced && isSpamTrapBounce(data.bounce), + }); + } catch (error) { + // Never fail event processing because of the rules. + logger.error( + { err: error, teamId: email.teamId }, + "[SesHookParser]: Sending rules check failed", + ); + } + } + logger.info("Creating email event"); await db.emailEvent.create({ @@ -412,8 +473,7 @@ function normalizeBounceSubType( subType: SesBounce["bounceSubType"] | undefined, ): EmailBounceSubType { const normalized = subType?.replace(/\s+/g, "") as - | EmailBounceSubType - | undefined; + EmailBounceSubType | undefined; const validSubTypes: EmailBounceSubType[] = [ "General", diff --git a/apps/web/src/server/service/ses-hook-parser.unit.test.ts b/apps/web/src/server/service/ses-hook-parser.unit.test.ts index d658378a1..1affa7f87 100644 --- a/apps/web/src/server/service/ses-hook-parser.unit.test.ts +++ b/apps/web/src/server/service/ses-hook-parser.unit.test.ts @@ -2,29 +2,40 @@ import { EmailStatus } from "@prisma/client"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { SesEvent } from "~/types/aws-types"; -const { mockDb, mockUpdateCampaignAnalytics, mockWebhookEmit } = vi.hoisted( - () => ({ - mockDb: { - $executeRaw: vi.fn(), - email: { - findUnique: vi.fn(), - update: vi.fn(), - }, - emailEvent: { - findFirst: vi.fn(), - create: vi.fn(), - }, - dailyEmailUsage: { - upsert: vi.fn(), - }, - cumulatedMetrics: { - upsert: vi.fn(), - }, +const { + mockDb, + mockUpdateCampaignAnalytics, + mockWebhookEmit, + mockCheckSendingHealth, + mockClaimOutcome, + mockRecordOutcome, + mockReleaseOutcome, +} = vi.hoisted(() => ({ + mockReleaseOutcome: vi.fn(), + mockCheckSendingHealth: vi.fn(), + mockClaimOutcome: vi.fn(), + mockRecordOutcome: vi.fn(), + mockDb: { + $executeRaw: vi.fn(), + $transaction: vi.fn((ops: Promise[]) => Promise.all(ops)), + email: { + findUnique: vi.fn(), + update: vi.fn(), }, - mockUpdateCampaignAnalytics: vi.fn(), - mockWebhookEmit: vi.fn(), - }), -); + emailEvent: { + findFirst: vi.fn(), + create: vi.fn(), + }, + dailyEmailUsage: { + upsert: vi.fn(), + }, + cumulatedMetrics: { + upsert: vi.fn(), + }, + }, + mockUpdateCampaignAnalytics: vi.fn(), + mockWebhookEmit: vi.fn(), +})); vi.mock("~/server/db", () => ({ db: mockDb, @@ -47,6 +58,16 @@ vi.mock("~/server/service/webhook-service", () => ({ }, })); +vi.mock("~/server/service/sending-cohorts", () => ({ + claimOutcome: mockClaimOutcome, + recordOutcome: mockRecordOutcome, + releaseOutcome: mockReleaseOutcome, +})); + +vi.mock("~/server/service/sending-rules-service", () => ({ + checkSendingHealth: mockCheckSendingHealth, +})); + vi.mock("~/server/service/suppression-service", () => ({ SuppressionService: { addSuppression: vi.fn(), @@ -184,3 +205,121 @@ describe("parseSesHook dashboard engagement usage", () => { }, ); }); + +describe("parseSesHook sending rules", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDb.email.findUnique.mockResolvedValue(email); + mockDb.emailEvent.findFirst.mockResolvedValue(null); + mockDb.emailEvent.create.mockResolvedValue({}); + mockDb.dailyEmailUsage.upsert.mockResolvedValue({}); + mockDb.cumulatedMetrics.upsert.mockResolvedValue({}); + mockWebhookEmit.mockResolvedValue(undefined); + mockClaimOutcome.mockResolvedValue("claimed"); + }); + + function bounceEvent(diagnosticCode: string): SesEvent { + const event = buildEvent("Open"); + delete event.open; + event.eventType = "Bounce"; + event.bounce = { + bounceType: "Permanent", + bounceSubType: "General", + bouncedRecipients: [ + { emailAddress: "recipient@example.com", diagnosticCode }, + ], + timestamp: "2026-07-13T01:00:00.000Z", + feedbackId: "feedback_1", + } as SesEvent["bounce"]; + return event; + } + + it("checks the team's sending health after a hard bounce", async () => { + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockCheckSendingHealth).toHaveBeenCalledWith({ + teamId: 7, + campaignId: null, + spamTrap: false, + }); + }); + + it("flags spam trap bounces", async () => { + await parseSesHook(bounceEvent("550 spamtrap hit")); + + expect(mockCheckSendingHealth).toHaveBeenCalledWith( + expect.objectContaining({ spamTrap: true }), + ); + }); + + it("keeps processing the event if the check fails", async () => { + mockCheckSendingHealth.mockRejectedValue(new Error("db down")); + + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockDb.emailEvent.create).toHaveBeenCalled(); + }); + + it("files the bounce against its email's counted send", async () => { + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockClaimOutcome).toHaveBeenCalledWith(email.id, "hardBounced"); + expect(mockRecordOutcome).toHaveBeenCalledWith( + expect.objectContaining({ id: email.id, teamId: 7 }), + "hardBounced", + ); + }); + + it("counts a repeated hard bounce only once", async () => { + mockClaimOutcome.mockResolvedValue("duplicate"); + + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockDb.dailyEmailUsage.upsert).not.toHaveBeenCalled(); + expect(mockRecordOutcome).not.toHaveBeenCalled(); + expect(mockCheckSendingHealth).not.toHaveBeenCalled(); + expect(mockDb.emailEvent.create).toHaveBeenCalled(); + }); + + it("still updates dashboards when Redis can't check for duplicates", async () => { + mockClaimOutcome.mockResolvedValue("unavailable"); + + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockDb.dailyEmailUsage.upsert).toHaveBeenCalled(); + expect(mockRecordOutcome).not.toHaveBeenCalled(); + }); + + it("writes both usage counters in one transaction", async () => { + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockDb.$transaction).toHaveBeenCalledTimes(1); + expect(mockDb.$transaction.mock.calls[0]?.[0]).toHaveLength(2); + }); + + it("keeps the claim if the rule counter fails, so it can't count twice", async () => { + mockRecordOutcome.mockRejectedValueOnce(new Error("db busy")); + + await parseSesHook(bounceEvent("550 5.1.1 user unknown")); + + expect(mockReleaseOutcome).not.toHaveBeenCalled(); + expect(mockDb.emailEvent.create).toHaveBeenCalled(); + }); + + it("gives the claim back if saving the counts fails, so a replay counts it", async () => { + mockDb.dailyEmailUsage.upsert.mockRejectedValue(new Error("db busy")); + + await expect( + parseSesHook(bounceEvent("550 5.1.1 user unknown")), + ).rejects.toThrow("db busy"); + + expect(mockReleaseOutcome).toHaveBeenCalledWith(email.id, "hardBounced"); + }); + + it("doesn't run on opens", async () => { + mockDb.emailEvent.findFirst.mockResolvedValue(null); + await parseSesHook(buildEvent("Open")); + + expect(mockCheckSendingHealth).not.toHaveBeenCalled(); + }); +}); From 022fdef853fad65a77216ca6ab7f450c6396dfef Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Sat, 21 Nov 2026 12:51:54 +1100 Subject: [PATCH 2/4] fix(rules): default to alert-only, re-date the migration with a lock timeout, cover paused resume --- .../migration.sql | 2 +- .../service/campaign-service.unit.test.ts | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql b/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql index 1410ff1a0..2e15af7da 100644 --- a/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql +++ b/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql @@ -1,7 +1,7 @@ -- All changes below only add columns or tables, so they finish instantly, but -- adding a column to "Email" and "Campaign" needs a brief lock. Give up after -- 10s rather than queue behind a long query and hold up sending; the deploy --- then fails (the old version keeps serving) and can simply be retried. +-- then fails (the old version keeps serving); mark it rolled back and retry. SET lock_timeout = '10s'; -- CreateEnum diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index dd3fae577..1ad729cf5 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -35,6 +35,9 @@ const { mockDb, mockTx, mockUpdateContactSubscription } = vi.hoisted(() => { domain: { findUnique: vi.fn(), }, + team: { + findUnique: vi.fn(), + }, campaign: { findUnique: vi.fn(), update: vi.fn(), @@ -435,9 +438,40 @@ describe("resumeCampaign", () => { teamId: 7, scheduledAt: null, }); + mockDb.team.findUnique.mockResolvedValue({ + isBlocked: false, + sendingPausedAt: null, + }); mockDb.campaign.update.mockResolvedValue({}); }); + it("won't resume while sending is paused on the team", async () => { + mockDb.team.findUnique.mockResolvedValue({ + isBlocked: false, + sendingPausedAt: new Date(), + }); + + await expect( + resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), + ).rejects.toThrow("Sending is paused"); + + expect(mockQueueGetJob).not.toHaveBeenCalled(); + expect(mockDb.campaign.update).not.toHaveBeenCalled(); + }); + + it("won't resume while the team is blocked", async () => { + mockDb.team.findUnique.mockResolvedValue({ + isBlocked: true, + sendingPausedAt: null, + }); + + await expect( + resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), + ).rejects.toThrow("Sending is paused"); + + expect(mockDb.campaign.update).not.toHaveBeenCalled(); + }); + it("clears a failed batch so the scheduler can continue the campaign", async () => { const job = { isFailed: vi.fn().mockResolvedValue(true), From f6402f191b614f5a0ea7230b80dec81c78fb9403 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Sat, 21 Nov 2026 12:57:52 +1100 Subject: [PATCH 3/4] fix(rules): only an explicit enforce acts; one short-lock transaction for the migration; don't count outcomes past the dedupe window --- .../migration.sql | 15 +++-- .../web/src/server/service/sending-cohorts.ts | 12 ++++ .../service/sending-cohorts.unit.test.ts | 19 ++++++ .../server/service/sending-rules-service.ts | 17 +++-- .../sending-rules-service.unit.test.ts | 66 ++++++++++++++++++- 5 files changed, 119 insertions(+), 10 deletions(-) diff --git a/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql b/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql index 2e15af7da..80360f709 100644 --- a/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql +++ b/apps/web/prisma/migrations/20261121120000_sending_rules_and_team_actions/migration.sql @@ -1,8 +1,12 @@ --- All changes below only add columns or tables, so they finish instantly, but --- adding a column to "Email" and "Campaign" needs a brief lock. Give up after --- 10s rather than queue behind a long query and hold up sending; the deploy --- then fails (the old version keeps serving); mark it rolled back and retry. -SET lock_timeout = '10s'; +-- Only adds columns and tables: no table is rewritten, so this runs in +-- milliseconds. Adding columns to "Team", "Email" and "Campaign" still needs a +-- brief exclusive lock on each, held until COMMIT. Give up after 3s per lock +-- rather than queue behind a long query and hold up sending. On failure the +-- whole transaction rolls back, the old deployment keeps serving, and the +-- migration can be marked rolled back and retried. Avoid deploying while the +-- nightly cleanup runs (00:00-00:05 UTC). +BEGIN; +SET LOCAL lock_timeout = '3s'; -- CreateEnum CREATE TYPE "TeamActionType" AS ENUM ('ALERT', 'PAUSE', 'RESUME', 'BLOCK', 'UNBLOCK'); @@ -51,3 +55,4 @@ ALTER TABLE "SendingCohort" ADD CONSTRAINT "SendingCohort_teamId_fkey" FOREIGN K -- AddForeignKey ALTER TABLE "TeamAction" ADD CONSTRAINT "TeamAction_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team"("id") ON DELETE CASCADE ON UPDATE CASCADE; +COMMIT; diff --git a/apps/web/src/server/service/sending-cohorts.ts b/apps/web/src/server/service/sending-cohorts.ts index b23c1da77..448b0343f 100644 --- a/apps/web/src/server/service/sending-cohorts.ts +++ b/apps/web/src/server/service/sending-cohorts.ts @@ -10,6 +10,14 @@ import { getRedis, redisKey } from "~/server/redis"; const OUTCOME_CLAIM_SECONDS = 14 * 24 * 60 * 60; +/** + * Outcomes are counted only for emails sent within this many days, so a claim + * is always still held while its outcome can count: a replay after the claim + * expires can't count the same bounce or complaint twice. Later ones are + * skipped, which only undercounts. The team window is 7 days anyway. + */ +export const OUTCOME_COUNT_DAYS = 13; + export type Outcome = "hardBounced" | "complained"; function day(date: Date) { @@ -116,6 +124,10 @@ export async function recordOutcome( ) { if (!env.NEXT_PUBLIC_IS_CLOUD || !email.cohortDate) return false; + const oldest = new Date(); + oldest.setUTCDate(oldest.getUTCDate() - OUTCOME_COUNT_DAYS); + if (email.cohortDate < day(oldest)) return false; + const date = email.cohortDate; const campaignField = outcome === "hardBounced" ? "acceptedHardBounced" : "acceptedComplained"; diff --git a/apps/web/src/server/service/sending-cohorts.unit.test.ts b/apps/web/src/server/service/sending-cohorts.unit.test.ts index 9f412bceb..85955bd21 100644 --- a/apps/web/src/server/service/sending-cohorts.unit.test.ts +++ b/apps/web/src/server/service/sending-cohorts.unit.test.ts @@ -100,6 +100,7 @@ describe("sending cohorts", () => { }); it("files an outcome under the day its email was counted", async () => { + freezeDate(); await expect( recordOutcome( { teamId: 3, campaignId: "c1", cohortDate: "2026-11-09" }, @@ -119,6 +120,24 @@ describe("sending cohorts", () => { }); }); + it("skips outcomes of emails sent too long ago to still hold a claim", async () => { + freezeDate(); + await expect( + recordOutcome( + { teamId: 3, campaignId: "c1", cohortDate: "2026-10-26" }, + "hardBounced", + ), + ).resolves.toBe(false); + await expect( + recordOutcome( + { teamId: 3, campaignId: "c1", cohortDate: "2026-10-27" }, + "hardBounced", + ), + ).resolves.toBe(true); + + expect(mockDb.sendingCohort.upsert).toHaveBeenCalledTimes(1); + }); + it("skips outcomes of emails whose send wasn't counted", async () => { await expect( recordOutcome( diff --git a/apps/web/src/server/service/sending-rules-service.ts b/apps/web/src/server/service/sending-rules-service.ts index e186baca1..f5a3ddc80 100644 --- a/apps/web/src/server/service/sending-rules-service.ts +++ b/apps/web/src/server/service/sending-rules-service.ts @@ -18,6 +18,14 @@ export { isSpamTrapBounce, judgeCounts, SENDING_RULES } from "./sending-rules"; const ALERT_COOLDOWN_SECONDS = 24 * 60 * 60; +/** + * Only an explicit "enforce" may pause, block or email a team. Anything else, + * including an unset value when env validation is skipped, only alerts. + */ +function enforcing() { + return env.SENDING_RULES_MODE === "enforce"; +} + function toDateString(date: Date) { return date.toISOString().split("T")[0] as string; } @@ -200,7 +208,7 @@ async function actOnTeam( ) { const label = await teamLabel(teamId); - if (env.SENDING_RULES_MODE === "alert") { + if (!enforcing()) { if (!(await claimAlert(`team:${teamId}:${verdict.level}`))) return; await recordAction( teamId, @@ -340,7 +348,7 @@ async function checkCampaign(teamId: number, campaignId: string, reset: Reset) { const pausedBefore = await db.teamAction.findFirst({ where: { teamId, - action: env.SENDING_RULES_MODE === "alert" ? "ALERT" : "PAUSE", + action: enforcing() ? "PAUSE" : "ALERT", details: { path: ["campaignId"], equals: campaign.id }, }, select: { id: true }, @@ -349,7 +357,7 @@ async function checkCampaign(teamId: number, campaignId: string, reset: Reset) { const label = await teamLabel(teamId); - if (env.SENDING_RULES_MODE === "alert") { + if (!enforcing()) { if (!(await claimAlert(`campaign:${campaign.id}`))) return; await recordAction( teamId, @@ -404,7 +412,8 @@ async function checkCampaign(teamId: number, campaignId: string, reset: Reset) { /** * Runs after every hard bounce and complaint (cloud only). Pauses or blocks * the team or campaign when its rates cross the limits, emails the team and - * posts to Discord. SENDING_RULES_MODE "alert" only posts; "off" disables it. + * posts to Discord, but only when SENDING_RULES_MODE is "enforce". "off" + * disables it; any other value (including unset) only posts alerts. */ export async function checkSendingHealth(params: { teamId: number; diff --git a/apps/web/src/server/service/sending-rules-service.unit.test.ts b/apps/web/src/server/service/sending-rules-service.unit.test.ts index 9e7ae100a..6b437d461 100644 --- a/apps/web/src/server/service/sending-rules-service.unit.test.ts +++ b/apps/web/src/server/service/sending-rules-service.unit.test.ts @@ -10,7 +10,7 @@ const { } = vi.hoisted(() => ({ mockEnv: { NEXT_PUBLIC_IS_CLOUD: true, - SENDING_RULES_MODE: "enforce" as "off" | "alert" | "enforce", + SENDING_RULES_MODE: "enforce" as "off" | "alert" | "enforce" | undefined, }, mockDb: { $transaction: vi.fn(), @@ -412,6 +412,70 @@ describe("checkSendingHealth", () => { ); }); + describe("when the mode isn't set (env validation skipped)", () => { + beforeEach(() => { + mockEnv.SENDING_RULES_MODE = undefined; + }); + + async function expectAlertOnly() { + await flush(); + expect(mockDb.$transaction).not.toHaveBeenCalled(); + expect(mockDb.team.updateMany).not.toHaveBeenCalled(); + expect(mockDb.campaign.updateMany).not.toHaveBeenCalled(); + expect(mockRefreshTeamCache).not.toHaveBeenCalled(); + expect(mockSendMail).not.toHaveBeenCalled(); + expect(mockDb.teamAction.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ action: "ALERT" }), + }); + expect(mockSendToDiscord).toHaveBeenCalledWith( + expect.stringContaining("[alert only]"), + ); + } + + it("only alerts on a spam trap", async () => { + await checkSendingHealth({ teamId: 1, spamTrap: true }); + await expectAlertOnly(); + }); + + it("only alerts when the team crosses the pause level", async () => { + usage(1000, 30); + await checkSendingHealth({ teamId: 1 }); + await expectAlertOnly(); + }); + + it("only alerts when the team crosses the block level", async () => { + usage(1000, 0, 4); + await checkSendingHealth({ teamId: 1 }); + await expectAlertOnly(); + }); + + it("only alerts on a campaign over the pause level", async () => { + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_1", + name: "November news", + acceptedSends: 400, + acceptedHardBounced: 20, + acceptedComplained: 0, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + await checkSendingHealth({ teamId: 1, campaignId: "cmp_1" }); + await expectAlertOnly(); + }); + + it("only alerts on a campaign over the block level", async () => { + mockDb.campaign.findFirst.mockResolvedValue({ + id: "cmp_1", + name: "November news", + acceptedSends: 400, + acceptedHardBounced: 40, + acceptedComplained: 0, + createdAt: new Date("2026-11-01T00:00:00Z"), + }); + await checkSendingHealth({ teamId: 1, campaignId: "cmp_1" }); + await expectAlertOnly(); + }); + }); + it("is off on self-hosted and when switched off", async () => { usage(1000, 500); mockEnv.NEXT_PUBLIC_IS_CLOUD = false; From b93150f115747353558affa2f263b938493573e9 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Sat, 21 Nov 2026 13:02:35 +1100 Subject: [PATCH 4/4] fix(campaigns): check the team and resume under the team row lock --- .../src/server/service/campaign-service.ts | 49 ++++++++++++------- .../service/campaign-service.unit.test.ts | 39 +++++++++++++-- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/apps/web/src/server/service/campaign-service.ts b/apps/web/src/server/service/campaign-service.ts index 25349756e..1da38318e 100644 --- a/apps/web/src/server/service/campaign-service.ts +++ b/apps/web/src/server/service/campaign-service.ts @@ -516,6 +516,22 @@ export async function pauseCampaign({ return { ok: true }; } +async function assertTeamCanSend( + client: Pick, + teamId: number, +) { + const team = await client.team.findUnique({ + where: { id: teamId }, + select: { isBlocked: true, sendingPausedAt: true }, + }); + if (team?.isBlocked || team?.sendingPausedAt) { + throw new UnsendApiError({ + code: "FORBIDDEN", + message: "Sending is paused on your account, so campaigns can't resume", + }); + } +} + export async function resumeCampaign({ campaignId, teamId, @@ -534,32 +550,27 @@ export async function resumeCampaign({ }); } - const team = await db.team.findUnique({ - where: { id: teamId }, - select: { isBlocked: true, sendingPausedAt: true }, - }); - if (team?.isBlocked || team?.sendingPausedAt) { - throw new UnsendApiError({ - code: "FORBIDDEN", - message: "Sending is paused on your account, so campaigns can't resume", - }); - } + await assertTeamCanSend(db, teamId); // Resuming is an explicit choice, so it also clears a batch that failed long // ago and that the scheduler no longer retries on its own. await CampaignBatchService.clearFailedBatch(campaignId); - if (campaign.scheduledAt && campaign.scheduledAt.getTime() > Date.now()) { - await db.campaign.update({ - where: { id: campaignId }, - data: { status: "SCHEDULED" }, - }); - } else { - await db.campaign.update({ + const status = + campaign.scheduledAt && campaign.scheduledAt.getTime() > Date.now() + ? "SCHEDULED" + : "RUNNING"; + + // Check again and write the status under the team row lock that pausing and + // blocking take, so a pause landing in between can't be overwritten. + await db.$transaction(async (tx) => { + await tx.$queryRaw`SELECT id FROM "Team" WHERE id = ${teamId} FOR UPDATE`; + await assertTeamCanSend(tx, teamId); + await tx.campaign.update({ where: { id: campaignId }, - data: { status: "RUNNING" }, + data: { status }, }); - } + }); return { ok: true }; } diff --git a/apps/web/src/server/service/campaign-service.unit.test.ts b/apps/web/src/server/service/campaign-service.unit.test.ts index 1ad729cf5..1db72c907 100644 --- a/apps/web/src/server/service/campaign-service.unit.test.ts +++ b/apps/web/src/server/service/campaign-service.unit.test.ts @@ -4,6 +4,13 @@ import { UnsubscribeReason } from "@prisma/client"; const { mockDb, mockTx, mockUpdateContactSubscription } = vi.hoisted(() => { const mockTx = { + $queryRaw: vi.fn(), + team: { + findUnique: vi.fn(), + }, + campaign: { + update: vi.fn(), + }, campaignEmail: { findUnique: vi.fn(), create: vi.fn(), @@ -442,7 +449,11 @@ describe("resumeCampaign", () => { isBlocked: false, sendingPausedAt: null, }); - mockDb.campaign.update.mockResolvedValue({}); + mockTx.team.findUnique.mockResolvedValue({ + isBlocked: false, + sendingPausedAt: null, + }); + mockTx.campaign.update.mockResolvedValue({}); }); it("won't resume while sending is paused on the team", async () => { @@ -456,7 +467,7 @@ describe("resumeCampaign", () => { ).rejects.toThrow("Sending is paused"); expect(mockQueueGetJob).not.toHaveBeenCalled(); - expect(mockDb.campaign.update).not.toHaveBeenCalled(); + expect(mockTx.campaign.update).not.toHaveBeenCalled(); }); it("won't resume while the team is blocked", async () => { @@ -469,7 +480,25 @@ describe("resumeCampaign", () => { resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), ).rejects.toThrow("Sending is paused"); - expect(mockDb.campaign.update).not.toHaveBeenCalled(); + expect(mockTx.campaign.update).not.toHaveBeenCalled(); + }); + + it("won't overwrite a pause that lands after the first check", async () => { + mockQueueGetJob.mockResolvedValue(null); + mockTx.team.findUnique.mockResolvedValue({ + isBlocked: false, + sendingPausedAt: new Date(), + }); + + await expect( + resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), + ).rejects.toThrow("Sending is paused"); + + expect(mockTx.$queryRaw).toHaveBeenCalled(); + expect(mockTx.$queryRaw.mock.invocationCallOrder[0]).toBeLessThan( + mockTx.team.findUnique.mock.invocationCallOrder[0]!, + ); + expect(mockTx.campaign.update).not.toHaveBeenCalled(); }); it("clears a failed batch so the scheduler can continue the campaign", async () => { @@ -483,7 +512,7 @@ describe("resumeCampaign", () => { expect(mockQueueGetJob).toHaveBeenCalledWith("campaign-batch-campaign_1"); expect(job.remove).toHaveBeenCalled(); - expect(mockDb.campaign.update).toHaveBeenCalledWith({ + expect(mockTx.campaign.update).toHaveBeenCalledWith({ where: { id: "campaign_1" }, data: { status: "RUNNING" }, }); @@ -511,7 +540,7 @@ describe("resumeCampaign", () => { resumeCampaign({ campaignId: "campaign_1", teamId: 7 }), ).rejects.toThrow("Job is locked"); - expect(mockDb.campaign.update).not.toHaveBeenCalled(); + expect(mockTx.campaign.update).not.toHaveBeenCalled(); }); });