fix(db): bound every session with a statement timeout

Without a statement timeout a single stuck query or lock wait holds one of
the 20 pool slots indefinitely; enough of them permanently exhaust the pool
and every DB-backed route stops serving. Apply statement_timeout and
idle_in_transaction_session_timeout (default 15s, env-overridable, 0 to
disable) at connection establishment so every pooled session is bounded.

Correct the stale pool-sizing comment: production max_connections is 200,
not 100, and the web and worker containers share the one Postgres.
This commit is contained in:
Marc-André Bombeck committed 2026-06-20 16:44:35 +02:00
1 parent 4851f6534b
commit 108fcaab4e
2 files changed
+113 -9

No files matched your search

+64 -1
View File
@@ -16,7 +16,7 @@
*/
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { getPoolMax } from "../db";
import { buildSessionOptions, getPoolMax, getStatementTimeoutMs } from "../db";
describe("getPoolMax", () => {
const originalEnv = process.env.DATABASE_POOL_MAX;
@@ -62,3 +62,66 @@ describe("getPoolMax", () => {
expect(getPoolMax()).toBeGreaterThanOrEqual(20);
});
});
// A-1 — a per-session statement timeout so a single stuck query can't hold a
// pool slot forever and starve every DB-backed route.
describe("getStatementTimeoutMs", () => {
const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS;
beforeEach(() => {
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
});
afterEach(() => {
if (originalEnv === undefined) {
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
} else {
process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv;
}
});
it("defaults to 15000ms when the env var is unset", () => {
expect(getStatementTimeoutMs()).toBe(15_000);
});
it("honours a positive override", () => {
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "30000";
expect(getStatementTimeoutMs()).toBe(30_000);
});
it("allows 0 to disable the timeout (legacy unbounded behaviour)", () => {
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0";
expect(getStatementTimeoutMs()).toBe(0);
});
it("falls back to the default on a malformed or negative value", () => {
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "not-a-number";
expect(getStatementTimeoutMs()).toBe(15_000);
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "-5";
expect(getStatementTimeoutMs()).toBe(15_000);
});
});
describe("buildSessionOptions", () => {
const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS;
afterEach(() => {
if (originalEnv === undefined) {
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
} else {
process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv;
}
});
it("emits both session timeouts as libpq startup options by default", () => {
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
const opts = buildSessionOptions();
expect(opts).toContain("-c statement_timeout=15000");
expect(opts).toContain("-c idle_in_transaction_session_timeout=15000");
});
it("returns undefined (no options) when the timeout is disabled", () => {
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0";
expect(buildSessionOptions()).toBeUndefined();
});
});
+49 -8
View File
@@ -13,14 +13,14 @@ const globalForPrisma = globalThis as unknown as {
* (`.planning/round-v1439-empirical-trace.md` § B2) showed thick
* `/api/analytics` holding ≥ 8 of the 10 default pool slots for
* 6.5 s on the maintainer's 347k-row tenant, starving every other Wave-B and
* Wave-C useQuery fan-out for the duration. Coolify's Postgres
* container runs the stock `max_connections` 100 ceiling, so a
* 20-slot Node-side pool sits well under the server's hard cap with
* plenty of headroom for concurrent power-users (≤ 5 simultaneous
* tenants × 20 = 100). Complements the W-POOL `p-limit(4)` cap on
* the analytics fan-out itself — the bounded concurrency keeps any
* single analytics call to ≤ 4 slots, so the remaining 16 stay
* available for every other dashboard query.
* Wave-C useQuery fan-out for the duration. The production Postgres runs a
* `max_connections` of 200 (raised from the stock 100), so a 20-slot
* Node-side pool sits well under the server's hard cap with plenty of
* headroom for concurrent power-users. Note the web + worker containers
* share this one Postgres, so the effective ceiling covers both pools.
* Complements the W-POOL `p-limit(4)` cap on the analytics fan-out itself —
* the bounded concurrency keeps any single analytics call to ≤ 4 slots, so
* the remaining 16 stay available for every other dashboard query.
*
* Env-overridable via `DATABASE_POOL_MAX` so an operator running
* with a smaller Postgres `max_connections` can dial down without a
@@ -35,10 +35,51 @@ export function getPoolMax(): number {
return 20;
}
/**
* Per-session DB statement timeout in milliseconds.
*
* Without this a single pathological query or lock wait holds one of the
* (default 20) pool slots indefinitely; 20 such queries permanently exhaust
* the pool and every DB-backed route stops serving — the single highest-impact
* availability gap (A-1). `statement_timeout` caps how long the server runs one
* statement; `idle_in_transaction_session_timeout` reaps a connection wedged
* mid-transaction (a client that opened a tx and stalled). Both are applied at
* connection-establishment via the libpq `options` startup parameter, so every
* pooled session inherits them.
*
* Env-overridable via `DATABASE_STATEMENT_TIMEOUT_MS`. Set to `0` to disable
* (legacy unbounded behaviour). Default 15 s — generous for the rollup-backed
* read paths while still bounding a runaway query well before it can starve the
* pool. Heavy admin one-shots (drain/backfill) run on the worker, not the web
* pool, and can dial their own timeout if needed.
*/
export function getStatementTimeoutMs(): number {
const raw = process.env.DATABASE_STATEMENT_TIMEOUT_MS;
if (raw !== undefined && raw !== "") {
const parsed = Number.parseInt(raw, 10);
if (Number.isFinite(parsed) && parsed >= 0) return parsed;
}
return 15_000;
}
/**
* Build the libpq `options` startup string applying the session timeouts, or
* `undefined` when disabled (timeout 0). Passed straight through `PrismaPg` to
* the underlying `pg.Pool`, which forwards it as the connection's `options`
* startup parameter so every session is timeout-bounded from the first query.
*/
export function buildSessionOptions(): string | undefined {
const timeoutMs = getStatementTimeoutMs();
if (timeoutMs <= 0) return undefined;
return `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs}`;
}
function createPrismaClient() {
const sessionOptions = buildSessionOptions();
const adapter = new PrismaPg({
connectionString: process.env.DATABASE_URL!,
max: getPoolMax(),
...(sessionOptions ? { options: sessionOptions } : {}),
});
return new PrismaClient({ adapter });
}