Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions prisma/migrations/20261117120000_add_user_created_at/migration.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- Fail fast rather than queue every query on "User" behind this migration while the table is busy
SET lock_timeout = '5s';

-- Users had no creation time. Both statements only change the catalog, so they are instant:
-- existing rows stay NULL (estimated later by the backfillUserCreatedAt data migration in
-- src/migrations, which never touches a set value) and new rows get the time they are created.
-- If it times out, run `prisma migrate resolve --rolled-back 20261117120000_add_user_created_at`
-- and deploy again: both statements are safe to repeat.

-- AlterTable
ALTER TABLE "User" ADD COLUMN IF NOT EXISTS "createdAt" TIMESTAMP(3);

ALTER TABLE "User" ALTER COLUMN "createdAt" SET DEFAULT CURRENT_TIMESTAMP;
3 changes: 3 additions & 0 deletions prisma/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ model User {
/// Set when the person deleted their account. The row stays, without their name or email, so
/// expenses and balances they share with others stay intact.
deletedAt DateTime?
/// When the row was created. Rows from before this column existed hold an estimate (their
/// earliest recorded activity) or null; see migration 20261117120000_add_user_created_at.
createdAt DateTime? @default(now())
accounts Account[]
sessions Session[]
groups Group[] @relation("GroupCreatedBy")
Expand Down
11 changes: 11 additions & 0 deletions src/instrumentation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,17 @@ export async function register() {
validateAuthEnv();

const { env } = await import('./env');

if ('phase-production-build' !== process.env.NEXT_PHASE) {
const [{ runDataMigrations }, { db }] = await Promise.all([
import('./migrations'),
import('./server/db'),
]);
// Not awaited: the server starts serving (and passes its healthcheck) while these run.
// It logs its own errors and never rejects.
void runDataMigrations(db);
}

if (!env.RECURRING_RUNNER_DISABLED && 'phase-production-build' !== process.env.NEXT_PHASE) {
const { startRecurringRunner } = await import('./server/api/services/recurringRunner');
startRecurringRunner(
Expand Down
89 changes: 89 additions & 0 deletions src/migrations/backfillUserCreatedAt.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { type PrismaClient } from '@prisma/client';

import { getCreatedAtExactSince } from '~/server/userCreatedAt';

export const BACKFILL_BATCH_SIZE = 500;
const STATEMENT_TIMEOUT_MS = 5000;
const LOCK_TIMEOUT_MS = 2000;

type Db = Pick<PrismaClient, '$transaction' | '$queryRaw'>;

/**
* Estimates `User.createdAt` for users from before the column existed, the rows still NULL.
*
* The estimate is the earliest of the timestamps that are written at the moment the user did or
* was linked to something, so they already existed then:
* - expenses they added (`addedBy`), notes they wrote, recurring rules they set up
* - their balances and friendships, created when the user was first involved
* - their push devices' `updatedAt` (a token moving to another user keeps its `createdAt`)
* - their sessions' expiry minus 30 days, the shortest session length (app sessions last longer)
* - `emailVerified`, set when they sign in by email
* - feedback they sent from the app (imported feedback has external, email-matched times)
* Left out because they can predate the user: expenses they only pay for or share (payers and
* participants can be edited in later), groups they own (ownership moves when an owner leaves)
* and group membership (no timestamp). Estimates are capped at when the column was added.
*
* Users with none of these stay NULL. Works through users in id order, `batchSize` per short
* transaction, and only ever fills NULLs: safe to run again and never overwrites an exact time.
*/
export const backfillUserCreatedAt = async (
db: Db,
{ batchSize = BACKFILL_BATCH_SIZE }: { batchSize?: number } = {},
) => {
const cap = (await getCreatedAtExactSince(db)) ?? new Date();
const capAt = cap.toISOString().slice(0, 23);

let afterId = 0;
let batches = 0;
let updated = 0;

for (;;) {
const ids = (
await db.$queryRaw<{ id: number }[]>`
SELECT "id" FROM "User"
WHERE "createdAt" IS NULL AND "id" > ${afterId}
ORDER BY "id" LIMIT ${batchSize}
`
).map(({ id }) => id);

if (0 === ids.length) {
return { batches, updated };
}

afterId = ids.at(-1)!;
batches += 1;
updated += await db.$transaction(async (tx) => {
await tx.$executeRawUnsafe(`SET LOCAL statement_timeout = ${STATEMENT_TIMEOUT_MS}`);
await tx.$executeRawUnsafe(`SET LOCAL lock_timeout = ${LOCK_TIMEOUT_MS}`);
return tx.$executeRaw`
WITH "seen" AS (
SELECT "addedBy" AS "userId", "createdAt" AS "at" FROM "Expense"
WHERE "addedBy" = ANY(${ids})
UNION ALL SELECT "createdById", "createdAt" FROM "ExpenseNote"
WHERE "createdById" = ANY(${ids})
UNION ALL SELECT "createdById", "createdAt" FROM "RecurringExpense"
WHERE "createdById" = ANY(${ids})
UNION ALL SELECT "userId", "createdAt" FROM "Balance" WHERE "userId" = ANY(${ids})
UNION ALL SELECT "userId", "createdAt" FROM "Friendship" WHERE "userId" = ANY(${ids})
UNION ALL SELECT "friendId", "createdAt" FROM "Friendship" WHERE "friendId" = ANY(${ids})
UNION ALL SELECT "userId", "updatedAt" FROM "DevicePushToken" WHERE "userId" = ANY(${ids})
UNION ALL SELECT "userId", "expires" - INTERVAL '30 days' FROM "Session"
WHERE "userId" = ANY(${ids})
UNION ALL SELECT "userId", "createdAt" FROM "Feedback"
WHERE "userId" = ANY(${ids}) AND "source" <> 'import'
UNION ALL SELECT "id", "emailVerified" FROM "User" WHERE "id" = ANY(${ids})
),
"estimate" AS (
SELECT "userId", LEAST(MIN("at"), ${capAt}::timestamp) AS "at"
FROM "seen"
WHERE "at" IS NOT NULL
GROUP BY "userId"
)
UPDATE "User" u
SET "createdAt" = e."at"
FROM "estimate" e
WHERE u."id" = e."userId" AND u."createdAt" IS NULL
`;
});
}
};
78 changes: 78 additions & 0 deletions src/migrations/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import { type PrismaClient } from '@prisma/client';

import { backfillUserCreatedAt } from './backfillUserCreatedAt';

/**
* Data migrations: changes to existing rows that are too slow or too involved for a SQL migration,
* run in the background after the server starts (see `instrumentation.ts`).
*
* Like upstream SplitPro's runner, the version reached is kept in AppMetadata and each migration
* runs once, in order, setting the version when it completes. The key is our own, not upstream's
* `schema_version`, whose runner treats a missing value as a pre-2.0.0 database.
*
* A failing migration is logged and leaves the version as it was, so it runs again on the next
* start; it never stops the server. Migrations must be safe to run again.
*/

export const DATA_VERSION_KEY = 'cloud_data_version';

type Db = Pick<PrismaClient, '$transaction' | '$queryRaw' | 'appMetadata'>;

export type DataMigration = {
/** Semver-like, compared as strings like upstream's runner */
version: string;
name: string;
run: (db: Db) => Promise<unknown>;
};

export const USER_CREATED_AT_BACKFILL_VERSION = '1.0.0';

export const DATA_MIGRATIONS: DataMigration[] = [
{
version: USER_CREATED_AT_BACKFILL_VERSION,
name: 'Estimate User.createdAt',
run: (db) => backfillUserCreatedAt(db),
},
];

const getVersion = async (db: Db) =>
(await db.appMetadata.findUnique({ where: { key: DATA_VERSION_KEY } }))?.value ?? null;

const setVersion = (db: Db, version: string) =>
db.appMetadata.upsert({
where: { key: DATA_VERSION_KEY },
update: { value: version },
create: { key: DATA_VERSION_KEY, value: version },
});

/** Only an error's name and Prisma or Node code: messages can contain data. */
const safeErrorDetails = (error: unknown) => {
if (!(error instanceof Error)) {
return { name: typeof error };
}
const { code } = error as { code?: unknown };
return { name: error.name, code: 'string' === typeof code ? code : undefined };
};

/** Runs the pending data migrations. Never throws. */
export const runDataMigrations = async (db: Db, migrations = DATA_MIGRATIONS) => {
try {
const current = await getVersion(db);
const pending = migrations.filter(({ version }) => null === current || version > current);

// One after another: each one starts from the previous one's result
for (const migration of pending) {
const startedAt = Date.now();
const result = await migration.run(db);
await setVersion(db, migration.version);
console.log(`Data migration ${migration.version} (${migration.name}) done`, {
ms: Date.now() - startedAt,
result,
});
}
return true;
} catch (error) {
console.error('Data migration failed, will retry on next start', safeErrorDetails(error));
return false;
}
};
1 change: 1 addition & 0 deletions src/pages/add.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ const AddPage: NextPageWithUser<{
...user,
emailVerified: null,
deletedAt: null,
createdAt: null,
name: user.name ?? null,
email: user.email ?? null,
image: user.image ?? null,
Expand Down
16 changes: 16 additions & 0 deletions src/pages/api/admin/stats.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import { adminApiRoute } from '~/server/adminApi';
import { adminStatsQuery, getAdminStats } from '~/server/api/services/adminStats';
import { db } from '~/server/db';

// Read by `adminApiRoute`, once the request is authorised
export const config = { api: { bodyParser: false } };

/**
* GET /api/admin/stats?days=<1-365, default 90>
* Read-only growth numbers: totals, daily counts for the last `days` UTC days, and distinct active
* users for the last 12 ISO weeks and 6 months. See `adminStats.ts` for what each number means.
*/
export default adminApiRoute(['GET'], async (req, res) => {
const { days } = adminStatsQuery.parse(req.query);
res.status(200).json(await getAdminStats(db, { days }));
});
Loading
Loading