mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 05:12:14 +00:00
fix(db): bound every session with a statement timeout
Without a statement timeout a single stuck query or lock wait holds one of the 20 pool slots indefinitely; enough of them permanently exhaust the pool and every DB-backed route stops serving. Apply statement_timeout and idle_in_transaction_session_timeout (default 15s, env-overridable, 0 to disable) at connection establishment so every pooled session is bounded. Correct the stale pool-sizing comment: production max_connections is 200, not 100, and the web and worker containers share the one Postgres.
This commit is contained in:
1 parent
4851f6534b
commit
108fcaab4e
2 files changed
+113
-9
No files matched your search
@@ -16,7 +16,7 @@
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { getPoolMax } from "../db";
|
||||
import { buildSessionOptions, getPoolMax, getStatementTimeoutMs } from "../db";
|
||||
|
||||
describe("getPoolMax", () => {
|
||||
const originalEnv = process.env.DATABASE_POOL_MAX;
|
||||
@@ -62,3 +62,66 @@ describe("getPoolMax", () => {
|
||||
expect(getPoolMax()).toBeGreaterThanOrEqual(20);
|
||||
});
|
||||
});
|
||||
|
||||
// A-1 — a per-session statement timeout so a single stuck query can't hold a
|
||||
// pool slot forever and starve every DB-backed route.
|
||||
describe("getStatementTimeoutMs", () => {
|
||||
const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalEnv === undefined) {
|
||||
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv;
|
||||
}
|
||||
});
|
||||
|
||||
it("defaults to 15000ms when the env var is unset", () => {
|
||||
expect(getStatementTimeoutMs()).toBe(15_000);
|
||||
});
|
||||
|
||||
it("honours a positive override", () => {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "30000";
|
||||
expect(getStatementTimeoutMs()).toBe(30_000);
|
||||
});
|
||||
|
||||
it("allows 0 to disable the timeout (legacy unbounded behaviour)", () => {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0";
|
||||
expect(getStatementTimeoutMs()).toBe(0);
|
||||
});
|
||||
|
||||
it("falls back to the default on a malformed or negative value", () => {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "not-a-number";
|
||||
expect(getStatementTimeoutMs()).toBe(15_000);
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "-5";
|
||||
expect(getStatementTimeoutMs()).toBe(15_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildSessionOptions", () => {
|
||||
const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalEnv === undefined) {
|
||||
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv;
|
||||
}
|
||||
});
|
||||
|
||||
it("emits both session timeouts as libpq startup options by default", () => {
|
||||
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
const opts = buildSessionOptions();
|
||||
expect(opts).toContain("-c statement_timeout=15000");
|
||||
expect(opts).toContain("-c idle_in_transaction_session_timeout=15000");
|
||||
});
|
||||
|
||||
it("returns undefined (no options) when the timeout is disabled", () => {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0";
|
||||
expect(buildSessionOptions()).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+49
-8
@@ -13,14 +13,14 @@ const globalForPrisma = globalThis as unknown as {
|
||||
* (`.planning/round-v1439-empirical-trace.md` § B2) showed thick
|
||||
* `/api/analytics` holding ≥ 8 of the 10 default pool slots for
|
||||
* 6.5 s on the maintainer's 347k-row tenant, starving every other Wave-B and
|
||||
* Wave-C useQuery fan-out for the duration. Coolify's Postgres
|
||||
* container runs the stock `max_connections` 100 ceiling, so a
|
||||
* 20-slot Node-side pool sits well under the server's hard cap with
|
||||
* plenty of headroom for concurrent power-users (≤ 5 simultaneous
|
||||
* tenants × 20 = 100). Complements the W-POOL `p-limit(4)` cap on
|
||||
* the analytics fan-out itself — the bounded concurrency keeps any
|
||||
* single analytics call to ≤ 4 slots, so the remaining 16 stay
|
||||
* available for every other dashboard query.
|
||||
* Wave-C useQuery fan-out for the duration. The production Postgres runs a
|
||||
* `max_connections` of 200 (raised from the stock 100), so a 20-slot
|
||||
* Node-side pool sits well under the server's hard cap with plenty of
|
||||
* headroom for concurrent power-users. Note the web + worker containers
|
||||
* share this one Postgres, so the effective ceiling covers both pools.
|
||||
* Complements the W-POOL `p-limit(4)` cap on the analytics fan-out itself —
|
||||
* the bounded concurrency keeps any single analytics call to ≤ 4 slots, so
|
||||
* the remaining 16 stay available for every other dashboard query.
|
||||
*
|
||||
* Env-overridable via `DATABASE_POOL_MAX` so an operator running
|
||||
* with a smaller Postgres `max_connections` can dial down without a
|
||||
@@ -35,10 +35,51 @@ export function getPoolMax(): number {
|
||||
return 20;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-session DB statement timeout in milliseconds.
|
||||
*
|
||||
* Without this a single pathological query or lock wait holds one of the
|
||||
* (default 20) pool slots indefinitely; 20 such queries permanently exhaust
|
||||
* the pool and every DB-backed route stops serving — the single highest-impact
|
||||
* availability gap (A-1). `statement_timeout` caps how long the server runs one
|
||||
* statement; `idle_in_transaction_session_timeout` reaps a connection wedged
|
||||
* mid-transaction (a client that opened a tx and stalled). Both are applied at
|
||||
* connection-establishment via the libpq `options` startup parameter, so every
|
||||
* pooled session inherits them.
|
||||
*
|
||||
* Env-overridable via `DATABASE_STATEMENT_TIMEOUT_MS`. Set to `0` to disable
|
||||
* (legacy unbounded behaviour). Default 15 s — generous for the rollup-backed
|
||||
* read paths while still bounding a runaway query well before it can starve the
|
||||
* pool. Heavy admin one-shots (drain/backfill) run on the worker, not the web
|
||||
* pool, and can dial their own timeout if needed.
|
||||
*/
|
||||
export function getStatementTimeoutMs(): number {
|
||||
const raw = process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
if (raw !== undefined && raw !== "") {
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (Number.isFinite(parsed) && parsed >= 0) return parsed;
|
||||
}
|
||||
return 15_000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the libpq `options` startup string applying the session timeouts, or
|
||||
* `undefined` when disabled (timeout 0). Passed straight through `PrismaPg` to
|
||||
* the underlying `pg.Pool`, which forwards it as the connection's `options`
|
||||
* startup parameter so every session is timeout-bounded from the first query.
|
||||
*/
|
||||
export function buildSessionOptions(): string | undefined {
|
||||
const timeoutMs = getStatementTimeoutMs();
|
||||
if (timeoutMs <= 0) return undefined;
|
||||
return `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs}`;
|
||||
}
|
||||
|
||||
function createPrismaClient() {
|
||||
const sessionOptions = buildSessionOptions();
|
||||
const adapter = new PrismaPg({
|
||||
connectionString: process.env.DATABASE_URL!,
|
||||
max: getPoolMax(),
|
||||
...(sessionOptions ? { options: sessionOptions } : {}),
|
||||
});
|
||||
return new PrismaClient({ adapter });
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user