From cefc8c15c14011ede55223524e87d7eabc6771fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 11 Oct 2026 12:15:03 +0200 Subject: [PATCH] fix(db): pin every connection to UTC and say so at boot when it is not Stored times are UTC in timestamp columns without a zone, and Postgres reads now(), CURRENT_TIMESTAMP and offset-free timestamptz parameters through the session time zone. On a database whose default is not UTC every expiry check ran off by the offset. The Prisma pool, pg-boss, the lock and the maintenance connections now send TimeZone=UTC as a startup option, or SET it on connect when the startup options are disabled for a pooler. A boot check reads SHOW TimeZone and warns on anything else. --- src/instrumentation.ts | 11 +++ src/lib/__tests__/db.test.ts | 39 +++++++++- .../boot/__tests__/time-zone-check.test.ts | 59 +++++++++++++++ src/lib/boot/time-zone-check.ts | 75 +++++++++++++++++++ src/lib/db-session.ts | 69 +++++++++++++++++ src/lib/db.ts | 36 ++++----- .../measurement-maintenance-session.test.ts | 1 + src/lib/jobs/boss-instance.ts | 4 + src/lib/jobs/job-lock.ts | 7 +- src/lib/jobs/measurement-maintenance.ts | 2 +- 10 files changed, 281 insertions(+), 22 deletions(-) create mode 100644 src/lib/boot/__tests__/time-zone-check.test.ts create mode 100644 src/lib/boot/time-zone-check.ts create mode 100644 src/lib/db-session.ts diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 9a3e2cb25..78981c5e6 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -28,6 +28,17 @@ export async function register() { console.error("[instrumentation] Encryption key check crashed:", err); } + // Every connection is pinned to UTC; say clearly if one is not. + try { + const [{ runBootTimeZoneCheck }, { prisma }] = await Promise.all([ + import("@/lib/boot/time-zone-check"), + import("@/lib/db"), + ]); + await runBootTimeZoneCheck(prisma); + } catch { + // A diagnostic must never break boot. + } + // Every Node process watches its own event loop — web-only // containers included; a stall report from the process serving // requests is the whole point. diff --git a/src/lib/__tests__/db.test.ts b/src/lib/__tests__/db.test.ts index 5cf7c93e6..9ca117bee 100644 --- a/src/lib/__tests__/db.test.ts +++ b/src/lib/__tests__/db.test.ts @@ -7,6 +7,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildSessionOptions, getConnectionBudget, + pinTimeZoneOnConnect, + SET_SESSION_TIME_ZONE, + timeZoneSessionOptions, getPgBossPoolMax, getPoolConnectionTimeoutMs, getPrismaPoolMax, @@ -135,14 +138,14 @@ describe("buildSessionOptions", () => { it("keeps work_mem when the timeout is disabled", () => { process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0"; delete process.env.DATABASE_WORK_MEM; - expect(buildSessionOptions()).toBe("-c work_mem=16MB"); + expect(buildSessionOptions()).toBe("-c work_mem=16MB -c TimeZone=UTC"); }); it("sets the default work_mem beside the timeouts", () => { delete process.env.DATABASE_STATEMENT_TIMEOUT_MS; delete process.env.DATABASE_WORK_MEM; expect(buildSessionOptions()).toBe( - "-c statement_timeout=60000 -c idle_in_transaction_session_timeout=60000 -c work_mem=16MB", + "-c statement_timeout=60000 -c idle_in_transaction_session_timeout=60000 -c work_mem=16MB -c TimeZone=UTC", ); }); @@ -166,6 +169,38 @@ describe("buildSessionOptions", () => { }); }); +// Every session runs in UTC whatever the database default says: the stored +// `timestamp` values are UTC and `now()` reads through the session zone. +describe("session time zone pin", () => { + afterEach(() => { + delete process.env.DATABASE_SESSION_OPTIONS_DISABLED; + }); + + it("rides the startup options of every pool", () => { + expect(buildSessionOptions()).toContain("-c TimeZone=UTC"); + expect(timeZoneSessionOptions()).toBe("-c TimeZone=UTC"); + }); + + it("adds no connect hook while the startup options carry the pin", () => { + const on = vi.fn(); + pinTimeZoneOnConnect({ on } as never); + expect(on).not.toHaveBeenCalled(); + }); + + it("pins by statement on connect when the startup options are off", async () => { + process.env.DATABASE_SESSION_OPTIONS_DISABLED = "true"; + expect(timeZoneSessionOptions()).toBeUndefined(); + const on = vi.fn(); + pinTimeZoneOnConnect({ on } as never); + expect(on).toHaveBeenCalledWith("connect", expect.any(Function)); + const query = vi.fn().mockResolvedValue(undefined); + const hook = on.mock.calls[0][1] as (client: unknown) => void; + hook({ query }); + expect(query).toHaveBeenCalledWith(SET_SESSION_TIME_ZONE); + expect(SET_SESSION_TIME_ZONE).toBe("SET TIME ZONE 'UTC'"); + }); +}); + describe("process-wide Prisma client", () => { const globalSlot = globalThis as unknown as { prisma?: unknown }; diff --git a/src/lib/boot/__tests__/time-zone-check.test.ts b/src/lib/boot/__tests__/time-zone-check.test.ts new file mode 100644 index 000000000..bce14204c --- /dev/null +++ b/src/lib/boot/__tests__/time-zone-check.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + checkSessionTimeZone, + isUtcTimeZone, + runBootTimeZoneCheck, +} from "../time-zone-check"; + +function clientAnswering(rows: unknown) { + return { $queryRawUnsafe: vi.fn().mockResolvedValue(rows) }; +} + +describe("boot time zone check", () => { + afterEach(() => vi.restoreAllMocks()); + + it("accepts the spellings Postgres reports for UTC", () => { + for (const name of ["UTC", "Etc/UTC", "utc", "UCT", "Zulu"]) { + expect(isUtcTimeZone(name)).toBe(true); + } + for (const name of ["Europe/Berlin", "GMT+1", "America/New_York"]) { + expect(isUtcTimeZone(name)).toBe(false); + } + }); + + it("stays quiet on a UTC session", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const client = clientAnswering([{ TimeZone: "UTC" }]); + expect(await runBootTimeZoneCheck(client)).toEqual({ + state: "utc", + timeZone: "UTC", + }); + expect(client.$queryRawUnsafe).toHaveBeenCalledWith("SHOW TimeZone"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("warns, naming the zone and the fix, on any other zone", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const outcome = await runBootTimeZoneCheck( + clientAnswering([{ TimeZone: "Europe/Berlin" }]), + ); + expect(outcome).toEqual({ state: "mismatch", timeZone: "Europe/Berlin" }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain('"Europe/Berlin"'); + expect(warn.mock.calls[0][0]).toContain("SET timezone TO 'UTC'"); + }); + + it("never throws when the database cannot answer", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const client = { + $queryRawUnsafe: vi.fn().mockRejectedValue(new Error("down")), + }; + expect(await checkSessionTimeZone(client)).toEqual({ + state: "error", + message: "down", + }); + await runBootTimeZoneCheck(client); + expect(warn.mock.calls[0][0]).toContain("skipped: down"); + }); +}); diff --git a/src/lib/boot/time-zone-check.ts b/src/lib/boot/time-zone-check.ts new file mode 100644 index 000000000..45a23ba60 --- /dev/null +++ b/src/lib/boot/time-zone-check.ts @@ -0,0 +1,75 @@ +/** + * Boot check: does a pooled connection really run in UTC? + * + * HealthLog pins `TimeZone=UTC` on every connection it opens (see + * `@/lib/db-session`). This reads the setting back on one pooled connection + * and says so in the log when it is anything else: a pooler that drops the + * startup options, a role or proxy that overrides the setting. The expiry + * checks of passkey challenges, sessions and one-time links read the stored + * UTC timestamps through that zone, so a mismatch shows up as "expired" + * straight away. The check never throws and never stops the boot. + */ + +const UTC_NAMES = /^(etc\/)?(utc|uct|universal|zulu)$/i; + +export interface TimeZoneCheckClient { + $queryRawUnsafe(query: string): Promise; +} + +export type TimeZoneCheckOutcome = + | { state: "utc"; timeZone: string } + | { state: "mismatch"; timeZone: string } + | { state: "error"; message: string }; + +export function isUtcTimeZone(name: string): boolean { + return UTC_NAMES.test(name.trim()); +} + +export async function checkSessionTimeZone( + client: TimeZoneCheckClient, +): Promise { + try { + const rows = + await client.$queryRawUnsafe>>( + "SHOW TimeZone", + ); + const value = rows[0] ? Object.values(rows[0])[0] : undefined; + const timeZone = typeof value === "string" ? value : String(value); + return isUtcTimeZone(timeZone) + ? { state: "utc", timeZone } + : { state: "mismatch", timeZone }; + } catch (err) { + return { + state: "error", + message: err instanceof Error ? err.message : String(err), + }; + } +} + +export function timeZoneMismatchMessage(timeZone: string): string { + return ( + `[boot] The database session runs in time zone "${timeZone}", not UTC. ` + + "HealthLog stores times as UTC and pins every connection to UTC, but " + + "something between the app and the database overrode it. Expiry checks " + + "(passkey challenges, sessions, one-time links) will be off by the zone " + + "offset. Fix: ALTER DATABASE SET timezone TO 'UTC', and if a " + + "connection pooler drops the startup options, set " + + "DATABASE_SESSION_OPTIONS_DISABLED=true so the app pins the zone by " + + "statement instead." + ); +} + +/** Run the check and log the outcome. Resolves the outcome for tests. */ +export async function runBootTimeZoneCheck( + client: TimeZoneCheckClient, +): Promise { + const outcome = await checkSessionTimeZone(client); + if (outcome.state === "mismatch") { + console.warn(timeZoneMismatchMessage(outcome.timeZone)); + } else if (outcome.state === "error") { + console.warn( + `[boot] Database time zone check skipped: ${outcome.message.slice(0, 300)}`, + ); + } + return outcome; +} diff --git a/src/lib/db-session.ts b/src/lib/db-session.ts new file mode 100644 index 000000000..e9b0e8323 --- /dev/null +++ b/src/lib/db-session.ts @@ -0,0 +1,69 @@ +/** + * Session settings shared by every Postgres connection HealthLog opens: the + * Prisma pool, pg-boss's pool and the dedicated lock and maintenance clients. + * Kept apart from `db.ts` so a side connection can use them without loading + * the Prisma client. + */ +import type { Pool, PoolClient } from "pg"; + +/** + * Whether the app may send the libpq `options` startup parameter at all. + * + * PgBouncer refuses a connection that carries a startup parameter it does not + * know (`unsupported startup parameter: options`) unless the operator lists + * it in `ignore_startup_parameters`. Up to v1.41 such a host could set the + * statement timeout to 0, which dropped the whole string; v1.42 always sends + * `work_mem`, so that no longer worked. `DATABASE_SESSION_OPTIONS_DISABLED` + * (1, true or yes) sends none of the session settings; the pooler or the + * database's own defaults then apply. + */ +export function sessionOptionsDisabled(): boolean { + const raw = process.env.DATABASE_SESSION_OPTIONS_DISABLED?.trim(); + return raw !== undefined && /^(1|true|yes)$/i.test(raw); +} + +/** + * The session time zone every connection HealthLog opens runs in. + * + * Almost every `DateTime` column is `timestamp(3)` WITHOUT time zone and + * holds UTC. Postgres converts between that type and `now()` / + * `CURRENT_TIMESTAMP` / `timestamptz` through the session time zone, and the + * driver sends a `Date` parameter as an offset-free UTC string that a + * `timestamptz` column reads in the session time zone. On a database whose + * default is anything but UTC (a shared instance set to Europe/Berlin, say) + * every expiry check then ran hours off, a passkey challenge expired before + * the prompt closed, and `timestamptz` writes landed shifted. Pinning the + * session removes the dependence for every query at once, whatever the + * database or role default says. + */ +export const SESSION_TIME_ZONE = "UTC"; + +export const TIME_ZONE_OPTION = `-c TimeZone=${SESSION_TIME_ZONE}`; + +/** + * The statement that pins the time zone where the startup `options` cannot + * be sent (`DATABASE_SESSION_OPTIONS_DISABLED`, a connection pooler). + * PgBouncer tracks `TimeZone` per client and replays it onto whichever server + * connection serves the next transaction, so a `SET` on connect holds in + * transaction pooling too. + */ +export const SET_SESSION_TIME_ZONE = `SET TIME ZONE '${SESSION_TIME_ZONE}'`; + +/** The startup option alone, for the side connections (pg-boss, locks). */ +export function timeZoneSessionOptions(): string | undefined { + return sessionOptionsDisabled() ? undefined : TIME_ZONE_OPTION; +} + +/** + * Pin the time zone on every new connection of `pool` by statement, when the + * startup options are off. pg queues the statement ahead of the first query + * the new client is handed out for, so no query runs before it. + */ +export function pinTimeZoneOnConnect(pool: Pool): void { + if (!sessionOptionsDisabled()) return; + pool.on("connect", (client: PoolClient) => { + client.query(SET_SESSION_TIME_ZONE).catch((err: unknown) => { + console.error("[db] Could not pin the session time zone:", err); + }); + }); +} diff --git a/src/lib/db.ts b/src/lib/db.ts index bfa815ae1..be64fb5b7 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,5 +1,19 @@ import { Prisma, PrismaClient } from "@/generated/prisma/client"; import { PrismaPg } from "@prisma/adapter-pg"; +import { Pool } from "pg"; +import { + pinTimeZoneOnConnect, + sessionOptionsDisabled, + TIME_ZONE_OPTION, +} from "@/lib/db-session"; + +export { + pinTimeZoneOnConnect, + SESSION_TIME_ZONE, + SET_SESSION_TIME_ZONE, + sessionOptionsDisabled, + timeZoneSessionOptions, +} from "@/lib/db-session"; const globalForPrisma = globalThis as unknown as { prisma: PrismaClient | undefined; @@ -108,22 +122,6 @@ export function getWorkMem(): string { return raw && WORK_MEM_SHAPE.test(raw) ? raw : DEFAULT_WORK_MEM; } -/** - * Whether the app may send the libpq `options` startup parameter at all. - * - * PgBouncer refuses a connection that carries a startup parameter it does not - * know (`unsupported startup parameter: options`) unless the operator lists - * it in `ignore_startup_parameters`. Up to v1.41 such a host could set the - * statement timeout to 0, which dropped the whole string; v1.42 always sends - * `work_mem`, so that no longer worked. `DATABASE_SESSION_OPTIONS_DISABLED` - * (1, true or yes) sends none of the session settings; the pooler or the - * database's own defaults then apply. - */ -export function sessionOptionsDisabled(): boolean { - const raw = process.env.DATABASE_SESSION_OPTIONS_DISABLED?.trim(); - return raw !== undefined && /^(1|true|yes)$/i.test(raw); -} - /** * Build the libpq `options` startup string applying the session settings. * Passed straight through `PrismaPg` to the underlying `pg.Pool`, which @@ -143,16 +141,18 @@ export function buildSessionOptions(): string | undefined { timeoutMs > 0 ? `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs} ` : ""; - return `${timeouts}-c work_mem=${getWorkMem()}`; + return `${timeouts}-c work_mem=${getWorkMem()} ${TIME_ZONE_OPTION}`; } function createPrismaClient() { - const adapter = new PrismaPg({ + const pool = new Pool({ connectionString: process.env.DATABASE_URL!, max: getPrismaPoolMax(), connectionTimeoutMillis: getPoolConnectionTimeoutMs(), options: buildSessionOptions(), }); + pinTimeZoneOnConnect(pool); + const adapter = new PrismaPg(pool, { disposeExternalPool: true }); return new PrismaClient({ adapter }); } diff --git a/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts b/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts index 93ba20002..213c9c5ed 100644 --- a/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts +++ b/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts @@ -26,6 +26,7 @@ describe("maintenance session settings behind a pooler", () => { "SET maintenance_work_mem = '128MB'", "SET max_parallel_maintenance_workers = '0'", "SET lock_timeout = '300000'", + "SET TimeZone = 'UTC'", ]); }); it("keeps maintenance in one process so it needs no /dev/shm segment", () => { diff --git a/src/lib/jobs/boss-instance.ts b/src/lib/jobs/boss-instance.ts index b807477d5..438e85818 100644 --- a/src/lib/jobs/boss-instance.ts +++ b/src/lib/jobs/boss-instance.ts @@ -8,6 +8,7 @@ import { PgBoss } from "pg-boss"; import { getPgBossPoolMax, getPoolConnectionTimeoutMs } from "@/lib/db"; +import { timeZoneSessionOptions } from "@/lib/db-session"; const BOSS_KEY = "__healthlog_pgboss__" as const; const BOSS_START_KEY = "__healthlog_pgboss_start__" as const; @@ -76,6 +77,9 @@ function databaseOptions(connectionString: string) { connectionString, max: getPgBossPoolMax(), connectionTimeoutMillis: getPoolConnectionTimeoutMs(), + // pg-boss keeps its own tables in `timestamptz`, so its queue math does + // not depend on the zone; the pin keeps every HealthLog session alike. + options: timeZoneSessionOptions(), } as const; } diff --git a/src/lib/jobs/job-lock.ts b/src/lib/jobs/job-lock.ts index 711fe471d..eb85441d7 100644 --- a/src/lib/jobs/job-lock.ts +++ b/src/lib/jobs/job-lock.ts @@ -19,6 +19,7 @@ * One extra connection per running locked job. */ import { Client } from "pg"; +import { timeZoneSessionOptions } from "@/lib/db-session"; import { caughtAs } from "@/lib/logging/signal"; /** What a guarded run came to. */ @@ -50,7 +51,11 @@ export async function withJobLock( connectionString: string | undefined = process.env.DATABASE_URL, options: JobLockOptions = {}, ): Promise> { - const client = new Client({ connectionString, keepAlive: true }); + const client = new Client({ + connectionString, + keepAlive: true, + options: timeZoneSessionOptions(), + }); // An idle client whose server goes away emits `error`; unhandled, that // takes the whole worker down. The run itself notices the database is // gone through its own queries. diff --git a/src/lib/jobs/measurement-maintenance.ts b/src/lib/jobs/measurement-maintenance.ts index 72a7c6a03..34eed50d9 100644 --- a/src/lib/jobs/measurement-maintenance.ts +++ b/src/lib/jobs/measurement-maintenance.ts @@ -77,7 +77,7 @@ export interface MeasurementMaintenancePayload { * whatever its container settings. */ export const MAINTENANCE_SESSION_OPTIONS = - "-c statement_timeout=0 -c idle_in_transaction_session_timeout=0 -c maintenance_work_mem=128MB -c max_parallel_maintenance_workers=0 -c lock_timeout=300000"; + "-c statement_timeout=0 -c idle_in_transaction_session_timeout=0 -c maintenance_work_mem=128MB -c max_parallel_maintenance_workers=0 -c lock_timeout=300000 -c TimeZone=UTC"; /** * The same settings as `SET` statements, for a host behind a connection