From 108fcaab4e98b4277454c1b24950721b6a4836ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 20 Jun 2026 16:44:35 +0200 Subject: [PATCH] fix(db): bound every session with a statement timeout Without a statement timeout a single stuck query or lock wait holds one of the 20 pool slots indefinitely; enough of them permanently exhaust the pool and every DB-backed route stops serving. Apply statement_timeout and idle_in_transaction_session_timeout (default 15s, env-overridable, 0 to disable) at connection establishment so every pooled session is bounded. Correct the stale pool-sizing comment: production max_connections is 200, not 100, and the web and worker containers share the one Postgres. --- src/lib/__tests__/db.test.ts | 65 +++++++++++++++++++++++++++++++++++- src/lib/db.ts | 57 ++++++++++++++++++++++++++----- 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/src/lib/__tests__/db.test.ts b/src/lib/__tests__/db.test.ts index d433f534d..ec872c51f 100644 --- a/src/lib/__tests__/db.test.ts +++ b/src/lib/__tests__/db.test.ts @@ -16,7 +16,7 @@ */ import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { getPoolMax } from "../db"; +import { buildSessionOptions, getPoolMax, getStatementTimeoutMs } from "../db"; describe("getPoolMax", () => { const originalEnv = process.env.DATABASE_POOL_MAX; @@ -62,3 +62,66 @@ describe("getPoolMax", () => { expect(getPoolMax()).toBeGreaterThanOrEqual(20); }); }); + +// A-1 — a per-session statement timeout so a single stuck query can't hold a +// pool slot forever and starve every DB-backed route. +describe("getStatementTimeoutMs", () => { + const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS; + + beforeEach(() => { + delete process.env.DATABASE_STATEMENT_TIMEOUT_MS; + }); + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.DATABASE_STATEMENT_TIMEOUT_MS; + } else { + process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv; + } + }); + + it("defaults to 15000ms when the env var is unset", () => { + expect(getStatementTimeoutMs()).toBe(15_000); + }); + + it("honours a positive override", () => { + process.env.DATABASE_STATEMENT_TIMEOUT_MS = "30000"; + expect(getStatementTimeoutMs()).toBe(30_000); + }); + + it("allows 0 to disable the timeout (legacy unbounded behaviour)", () => { + process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0"; + expect(getStatementTimeoutMs()).toBe(0); + }); + + it("falls back to the default on a malformed or negative value", () => { + process.env.DATABASE_STATEMENT_TIMEOUT_MS = "not-a-number"; + expect(getStatementTimeoutMs()).toBe(15_000); + process.env.DATABASE_STATEMENT_TIMEOUT_MS = "-5"; + expect(getStatementTimeoutMs()).toBe(15_000); + }); +}); + +describe("buildSessionOptions", () => { + const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS; + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.DATABASE_STATEMENT_TIMEOUT_MS; + } else { + process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv; + } + }); + + it("emits both session timeouts as libpq startup options by default", () => { + delete process.env.DATABASE_STATEMENT_TIMEOUT_MS; + const opts = buildSessionOptions(); + expect(opts).toContain("-c statement_timeout=15000"); + expect(opts).toContain("-c idle_in_transaction_session_timeout=15000"); + }); + + it("returns undefined (no options) when the timeout is disabled", () => { + process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0"; + expect(buildSessionOptions()).toBeUndefined(); + }); +}); diff --git a/src/lib/db.ts b/src/lib/db.ts index 7097d0a83..01ab1607f 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -13,14 +13,14 @@ const globalForPrisma = globalThis as unknown as { * (`.planning/round-v1439-empirical-trace.md` § B2) showed thick * `/api/analytics` holding ≥ 8 of the 10 default pool slots for * 6.5 s on the maintainer's 347k-row tenant, starving every other Wave-B and - * Wave-C useQuery fan-out for the duration. Coolify's Postgres - * container runs the stock `max_connections` 100 ceiling, so a - * 20-slot Node-side pool sits well under the server's hard cap with - * plenty of headroom for concurrent power-users (≤ 5 simultaneous - * tenants × 20 = 100). Complements the W-POOL `p-limit(4)` cap on - * the analytics fan-out itself — the bounded concurrency keeps any - * single analytics call to ≤ 4 slots, so the remaining 16 stay - * available for every other dashboard query. + * Wave-C useQuery fan-out for the duration. The production Postgres runs a + * `max_connections` of 200 (raised from the stock 100), so a 20-slot + * Node-side pool sits well under the server's hard cap with plenty of + * headroom for concurrent power-users. Note the web + worker containers + * share this one Postgres, so the effective ceiling covers both pools. + * Complements the W-POOL `p-limit(4)` cap on the analytics fan-out itself — + * the bounded concurrency keeps any single analytics call to ≤ 4 slots, so + * the remaining 16 stay available for every other dashboard query. * * Env-overridable via `DATABASE_POOL_MAX` so an operator running * with a smaller Postgres `max_connections` can dial down without a @@ -35,10 +35,51 @@ export function getPoolMax(): number { return 20; } +/** + * Per-session DB statement timeout in milliseconds. + * + * Without this a single pathological query or lock wait holds one of the + * (default 20) pool slots indefinitely; 20 such queries permanently exhaust + * the pool and every DB-backed route stops serving — the single highest-impact + * availability gap (A-1). `statement_timeout` caps how long the server runs one + * statement; `idle_in_transaction_session_timeout` reaps a connection wedged + * mid-transaction (a client that opened a tx and stalled). Both are applied at + * connection-establishment via the libpq `options` startup parameter, so every + * pooled session inherits them. + * + * Env-overridable via `DATABASE_STATEMENT_TIMEOUT_MS`. Set to `0` to disable + * (legacy unbounded behaviour). Default 15 s — generous for the rollup-backed + * read paths while still bounding a runaway query well before it can starve the + * pool. Heavy admin one-shots (drain/backfill) run on the worker, not the web + * pool, and can dial their own timeout if needed. + */ +export function getStatementTimeoutMs(): number { + const raw = process.env.DATABASE_STATEMENT_TIMEOUT_MS; + if (raw !== undefined && raw !== "") { + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed >= 0) return parsed; + } + return 15_000; +} + +/** + * Build the libpq `options` startup string applying the session timeouts, or + * `undefined` when disabled (timeout 0). Passed straight through `PrismaPg` to + * the underlying `pg.Pool`, which forwards it as the connection's `options` + * startup parameter so every session is timeout-bounded from the first query. + */ +export function buildSessionOptions(): string | undefined { + const timeoutMs = getStatementTimeoutMs(); + if (timeoutMs <= 0) return undefined; + return `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs}`; +} + function createPrismaClient() { + const sessionOptions = buildSessionOptions(); const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL!, max: getPoolMax(), + ...(sessionOptions ? { options: sessionOptions } : {}), }); return new PrismaClient({ adapter }); }