test: close every database pool a test file opens

scripts/test-server-sharded.ts

@@ -1,12 +1,16 @@
 export {};
 
 /**
- * `server/tests` in shards, because the whole directory in one process no longer runs.
+ * `server/tests` in shards: faster wall-clock, and headroom under PostgreSQL's connection ceiling.
  *
- * Around a hundred of those files open a module-scope connection pool. PostgreSQL's default
- * `max_connections` is 100, so a single `bun test server/tests` exhausts the server's connection
- * slots during the import phase and dies with `sorry, too many clients already` BEFORE it reports
- * anything. That is a harness ceiling, not a regression, and it is never a reason to change source.
+ * Around a hundred of those files open a module-scope connection pool. Constructing one costs no
+ * connection: the driver opens the pool's `max` (two by default) on the file's first query, and
+ * a pool nobody closes holds its slots for the rest of the run. PostgreSQL's default
+ * `max_connections` is 100, so a single `bun test server/tests` crosses that ceiling as a function
+ * of how many files have RUN, not how many have been imported, and dies with `sorry, too many
+ * clients already` BEFORE it reports anything. That is why #785 has every file close its pool and
+ * a guard test keep the pairing: an unclosed pool is a defect in the source, not a harness
+ * ceiling.
  *
  * Shards keep the pools-per-process count under the ceiling. The suite is split into fixed-size
  * groups of files and each group runs in its own `bun test` invocation, so each process opens its

server/tests/account-mode.integration.test.ts

@@ -1,5 +1,5 @@
 // server/tests/account-mode.integration.test.ts
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
 import { createAuditStore } from "../src/audit";
@@ -406,3 +406,7 @@ describe("switching back to Personal", () => {
     expect(rules[0]?.revokedAt).not.toBeNull();
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/app-access-routes.integration.test.ts

@@ -373,6 +373,7 @@ afterAll(async () => {
   await database
     .delete(users)
     .where(inArray(users.id, [person, other, admin, maker]));
+  await database.$client.close();
 });
 
 describe("what each app a Bot holds is, for the person asking", () => {

server/tests/audit-cursor.integration.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "bun:test";
+import { afterAll, describe, expect, test } from "bun:test";
 import { sql } from "drizzle-orm";
 import { createApp } from "../src/app";
 import { createAuditReader } from "../src/audit";
@@ -158,3 +158,7 @@ describe("a cursor over rows written within one millisecond", () => {
     expect(await walk(targetId)).toEqual(expected);
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/audit-initiator.integration.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "bun:test";
+import { afterAll, describe, expect, test } from "bun:test";
 import { eq } from "drizzle-orm";
 import {
   auditQueryFromUrl,
@@ -291,3 +291,7 @@ describe("what started a run", () => {
     expect(reattribute()).rejects.toThrow();
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/audit-retention.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { eq, sql } from "drizzle-orm";
 import postgres from "postgres";
 import { sweepAuditTrail } from "../src/audit-retention";
@@ -309,3 +309,7 @@ describe("keeping the audit trail to a retention policy", () => {
     expect(deleted).toBe(1);
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/bot-lifecycle.integration.test.ts

@@ -229,6 +229,7 @@ afterAll(async () => {
   await database
     .delete(users)
     .where(inArray(users.id, [owner, other, `${prefix}-chooser`]));
+  await database.$client.close();
 });
 
 describe("pause", () => {

server/tests/brokered-account-mode-backfill.integration.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "bun:test";
+import { afterAll, describe, expect, test } from "bun:test";
 import { eq, inArray, sql } from "drizzle-orm";
 import { createDatabase } from "../src/db/client";
 import { mcpServers, mcpTools } from "../src/db/schema";
@@ -175,3 +175,7 @@ describe("the account-mode backfill migration", () => {
     });
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/component-store.integration.test.ts

@@ -70,6 +70,7 @@ afterAll(async () => {
     .where(inArray(componentExclusions.componentName, names));
   await database.delete(components).where(inArray(components.name, names));
   await database.delete(agents).where(inArray(agents.id, [...fixtureAgentIds]));
+  await database.$client.close();
 });
 
 describe("deciding whether a Bot may use a component", () => {

server/tests/composio-connection-kinds.test.ts

@@ -904,6 +904,7 @@ afterEach(() => useComposioClient(null));
 
 afterAll(async () => {
   await clean();
+  await database.$client.close();
 });
 
 /** Every action Composio was asked to run, so "nothing reached the vendor" is an assertion. */

server/tests/composio-connections.test.ts

@@ -1122,6 +1122,7 @@ afterEach(() => useComposioClient(null));
 afterAll(async () => {
   await clean();
   await cleanForeign();
+  await database.$client.close();
 });
 
 /**

server/tests/computer-page-frame-store.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { eq } from "drizzle-orm";
 import { createPageFrameStore } from "../src/computer/page-frames";
 import { createDatabase } from "../src/db/client";
@@ -208,3 +208,7 @@ describe("kept page frames", () => {
     expect(await store.load(id, "")).toBe(null);
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/computer-session-guard-replicas.integration.test.ts

@@ -106,6 +106,7 @@ afterAll(async () => {
   await database
     .delete(computerSnapshot)
     .where(eq(computerSnapshot.computerId, botId));
+  await database.$client.close();
 });
 
 test("replica B refuses a ref that replica A took before the container was replaced", async () => {

server/tests/computer-snapshot-store.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import type { SnapshotElement } from "../src/computer/schema";
 import {
   createSnapshotStore,
@@ -147,3 +147,7 @@ describe("a snapshot taken on one server", () => {
     );
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/credential-rotation-leak.integration.test.ts

@@ -84,6 +84,7 @@ afterAll(async () => {
   for (const id of created.splice(0)) {
     await database.delete(credentials).where(eq(credentials.id, id));
   }
+  await database.$client.close();
 });
 
 describe("a rotation the database refuses", () => {

server/tests/credentials.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
 import { createApp } from "../src/app";
@@ -865,3 +865,7 @@ describe("admin credential API", () => {
     expect(await rotate.text()).not.toContain("rotated-secret");
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/database-pool-closing.test.ts

@@ -0,0 +1,41 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "bun:test";
+
+/**
+ * Every pool a test file opens is closed by the same file (#785).
+ *
+ * The suite runs in a single process, so a module-scope pool nobody closes holds its two
+ * connections for the rest of the run. PostgreSQL's ceiling is 100, and when it is crossed the
+ * failure lands on whoever added the next file, not on anything they did wrong: #770's branch
+ * saw 114 failures and 402 `sorry, too many clients already`, none of them about the change
+ * under review.
+ *
+ * A pairing rule, checked against the file's own text: every `const <name> = createDatabase(`
+ * needs a `<name>.$client.close()` (or the `$client.end(` some files already use) somewhere in
+ * the same file. `server/tests` is outside `tsc`, so no type error would notice file 118 quietly
+ * reintroducing the leak; this is what notices. The close must sit in the file's LAST top-level
+ * `afterAll` (or one registered after every other hook): Bun runs `afterAll` hooks
+ * first-registered-first, and a pool closed before a cleanup's deletes takes those deletes with
+ * it.
+ */
+describe("every database pool a test file opens is closed", () => {
+  test("each createDatabase variable is closed or ended in its own file", () => {
+    const offenders: string[] = [];
+    for (const file of readdirSync(import.meta.dir)
+      .filter((name) => name.endsWith(".test.ts"))
+      .sort()) {
+      const text = readFileSync(join(import.meta.dir, file), "utf8");
+      for (const match of text.matchAll(/const (\w+) = createDatabase\(/g)) {
+        const name = match[1]!;
+        if (
+          !text.includes(`${name}.$client.close()`) &&
+          !new RegExp(`\\b${name}\\.\\$client\\.end\\(`).test(text)
+        ) {
+          offenders.push(`${file}: ${name}`);
+        }
+      }
+    }
+    expect(offenders).toEqual([]);
+  });
+});

server/tests/database.test.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "bun:test";
+import { afterAll, expect, test } from "bun:test";
 import { createDatabase } from "../src/db/client";
 import { TEST_POOL } from "./support/database";
 
@@ -10,3 +10,7 @@ test("creates a typed database boundary without opening a query", () => {
 
   expect(database.query.users).toBeDefined();
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/headless-components.integration.test.ts

@@ -82,6 +82,7 @@ afterAll(async () => {
   await database
     .delete(agents)
     .where(inArray(agents.id, [allowedBot, deniedBot]));
+  await database.$client.close();
 });
 
 function lifecycle(input: RunAgentInput, body: BaseEvent[]): BaseEvent[] {

server/tests/jsonb-encoding.integration.test.ts

@@ -23,6 +23,7 @@ const agentId = `agent_jsonb_${suite}`;
 
 afterAll(async () => {
   await database.delete(agents).where(eq(agents.id, agentId));
+  await database.$client.close();
 });
 
 describe("what a jsonb column actually stores", () => {

server/tests/last-sign-in-backfill.integration.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "bun:test";
+import { afterAll, describe, expect, test } from "bun:test";
 import { inArray, sql } from "drizzle-orm";
 import { createDatabase } from "../src/db/client";
 import { sessions, users } from "../src/db/schema";
@@ -112,3 +112,7 @@ describe("the backfill migration", () => {
     expect(observed.get(neverHere)).toBeNull();
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/mcp-grant-removal.integration.test.ts

@@ -104,6 +104,7 @@ afterAll(async () => {
     .delete(mcpServers)
     .where(inArray(mcpServers.id, [serverId, keptServerId]));
   await database.delete(agents).where(eq(agents.id, botId));
+  await database.$client.close();
 });
 
 test("removing an app takes its grants with it, and adding it back grants nothing", async () => {

server/tests/offboarding-retirement.integration.test.ts

@@ -29,6 +29,7 @@ afterAll(async () => {
     .where(eq(revokedAccess.email, memberEmail));
   await database.delete(sessions).where(eq(sessions.userId, memberId));
   await database.delete(users).where(inArray(users.id, [adminId, memberId]));
+  await database.$client.close();
 });
 
 function appFor(retire: () => Promise<{ retired: number }>) {

server/tests/owner-app-grants.integration.test.ts

@@ -261,6 +261,7 @@ beforeEach(async () => {
 afterAll(async () => {
   await clear();
   await database.delete(users).where(inArray(users.id, people));
+  await database.$client.close();
 });
 
 describe("an owner giving their own app to their own Bot", () => {

server/tests/page-frame-retention.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { eq, sql } from "drizzle-orm";
 import { startRetentionSweeps } from "../src/audit-retention";
 import {
@@ -120,3 +120,7 @@ describe("page frame retention", () => {
     expect(await kept()).toHaveLength(0);
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/people-last-sign-in.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { eq, inArray } from "drizzle-orm";
 import { stampSignIn } from "../src/auth";
 import { createDatabase } from "../src/db/client";
@@ -129,3 +129,7 @@ describe("when somebody was last here", () => {
     ).resolves.toBeUndefined();
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/people-paging.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { inArray } from "drizzle-orm";
 import { createDatabase } from "../src/db/client";
 import { sessions, users } from "../src/db/schema";
@@ -193,3 +193,7 @@ describe("reading the people in a deployment", () => {
     expect(await store.find("nobody-by-that-id")).toBeUndefined();
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});

server/tests/plugin-credential-binding.integration.test.ts

@@ -134,6 +134,7 @@ afterAll(async () => {
   await database
     .delete(credentials)
     .where(inArray(credentials.id, [ownCredentialId, otherCredentialId]));
+  await database.$client.close();
 });
 
 async function storedUrl(id: string) {

server/tests/plugin-curated-credential.integration.test.ts

@@ -120,6 +120,7 @@ afterAll(async () => {
         personalCredentialId,
       ]),
     );
+  await database.$client.close();
 });
 
 describe("a curated server may only be pointed at its own kind of credential", () => {

server/tests/plugin-routes.integration.test.ts

@@ -145,6 +145,7 @@ afterAll(async () => {
         ownCredentialId,
       ]),
     );
+  await database.$client.close();
 });
 
 async function serverRow() {

server/tests/plugin-store.integration.test.ts

@@ -4437,6 +4437,7 @@ afterAll(async () => {
     .where(eq(brokeredConnections.app, `sweep_witness_${suite}`));
   await database.delete(agents).where(eq(agents.id, "bot_helper"));
   await database.delete(users).where(eq(users.id, "user_leaver"));
+  await database.$client.close();
 });
 
 test("a Composio connection row survives the person being deleted", async () => {

server/tests/plugin-user-credential.integration.test.ts

@@ -360,6 +360,7 @@ afterAll(async () => {
   await database.delete(agents).where(eq(agents.id, botId));
   await database.delete(users).where(eq(users.id, askerId));
   await database.delete(users).where(eq(users.id, otherId));
+  await database.$client.close();
 });
 
 describe("a person who has not connected", () => {

server/tests/policy-durability.integration.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
 import { eq } from "drizzle-orm";
 import {
   createPolicyStore,
@@ -113,3 +113,7 @@ describe("a boundary set while running", () => {
     expect(store.get()).toEqual(configured);
   });
 });
+
+afterAll(async () => {
+  await database.$client.close();
+});