From a10bebdc43bb3c97143d819df5b846198b2c6255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 3 Nov 2026 23:31:37 +0100 Subject: [PATCH] fix(db): give each database its own app role and keep the queue schema the owner's Roles are cluster-wide. Two HealthLog databases on one server shared healthlog_app: each start set its own password, locking the other instance out, and the role held grants on both databases. The role is now healthlog_app__, marked with a comment naming its database, its password is derived per database, and a role holding an explicit CONNECT on another database is not used. The role owned the pgboss schema, so it could plant a trigger, a rule or a function body that a later owner session (a rollback, DB_APP_ROLE=off, a fallback start) would run with the owner's rights. pg-boss changes its schema only on install, on migration and for partitioned queues, which the app does not use. The start now installs and migrates the schema as the owner with pg-boss's own plans, and the worker runs pg-boss with migrate: false when it does not own the schema. The role gets rows only, read-only on pgboss.bam (SQL the migration runner executes) and on the version column. Every start, on every path, takes back what a HealthLog role owns there, drops triggers and rules, and after a take-back restores the queue functions to pg-boss's own definitions and retires the legacy role. Bloated queue indexes, which pg-boss only reports when it does not own them, are rebuilt at start. --- scripts/db-app-role.mjs | 816 +++++++++++++++++++------- src/lib/jobs/boss-instance.ts | 21 +- src/lib/jobs/reminder-worker.ts | 18 +- tests/integration/db-app-role.test.ts | 358 +++++++++-- 4 files changed, 948 insertions(+), 265 deletions(-) diff --git a/scripts/db-app-role.mjs b/scripts/db-app-role.mjs index 73b9ca2f5..12c9db441 100644 --- a/scripts/db-app-role.mjs +++ b/scripts/db-app-role.mjs @@ -6,10 +6,19 @@ * `DATABASE_URL` names. The long-running web and worker processes do not need * the owner: they read and write rows, run the queue, and maintain one table. * After the migrations, `docker-entrypoint.sh` runs this script as the owner. - * It ensures a login role, `healthlog_app`, that can do exactly that, and - * hands the entrypoint a `DATABASE_URL` for it; the entrypoint then starts the - * app with that URL in place of the owner's, so the owner's credentials never - * reach the app's environment. + * It ensures a login role for this database, `healthlog_app__` + * ({@link appRoleName}), that can do exactly that, and hands the entrypoint a + * `DATABASE_URL` for it; the entrypoint then starts the app with that URL in + * place of the owner's, so the owner's credentials never reach the app's + * environment. + * + * One role per database. Roles are cluster-wide, so two HealthLog databases on + * one server (production and staging on a shared Postgres) sharing one role + * name would each set their own password on every start, locking the other + * out, and the role would hold grants on both, letting either instance read + * the other's database. The name carries the database's name and a hash of + * it; the marker comment names the database too. A role that holds an explicit + * CONNECT grant on another database is refused. * * What the role can do: * - SELECT, INSERT, UPDATE, DELETE on every table of the app's schema, and @@ -17,38 +26,66 @@ * only: the app reads the migration history, it never writes it. * - MAINTAIN on `measurements` (Postgres 17+): the maintenance job's * `VACUUM (ANALYZE)` and `REINDEX INDEX CONCURRENTLY`. - * - Own the `pgboss` schema and everything in it. pg-boss creates and - * migrates its own tables, functions and per-queue partitions at runtime, - * so it needs DDL there; it is the queue's private schema and nothing of - * the app's data lives in it. + * - Rows of the queue schema `pgboss`, which stays the owner's. See + * "The queue schema" below. * - * What it cannot do: create, alter or drop anything in the app's schema; - * create schemas, databases or roles; act as a superuser, so no - * `COPY ... FROM PROGRAM`, no `pg_read_file`, no untrusted languages; write - * the migration history. `CREATE` on the app's schema is revoked from PUBLIC - * as well (the default since Postgres 15; a database first created on an - * older major still grants it), where the owner may do so. + * What it cannot do: create, alter or drop anything in either schema; create + * triggers, rules or functions; create schemas, databases or roles; act as a + * superuser, so no `COPY ... FROM PROGRAM`, no `pg_read_file`, no untrusted + * languages; write the migration history. `CREATE` on the app's schema is + * revoked from PUBLIC as well (the default since Postgres 15; a database first + * created on an older major still grants it), where the owner may do so. * - * Every boot re-applies it, inside one transaction under an advisory lock, so - * a table a new migration created is covered before the app starts, a web and - * a worker container booting together do not race, and a restored database - * (whose objects `pg_restore --no-owner` hands to the owner) is put right. - * Grants only touch what lacks them, so a boot after the first writes + * The queue schema. pg-boss changes its schema in three places: its install, + * its migrations (and the index builds they queue in `pgboss.bam`), and + * queues created with `partition: true`. HealthLog creates no partitioned + * queue, and with `persistQueueStats` off pg-boss creates no stats + * partitions, so a running app only reads and writes rows. This script + * installs and migrates the schema as the owner before the app starts + * ({@link prepareQueueSchema}), with pg-boss's own plans and its index builds + * run inline, and the worker starts pg-boss with `migrate: false` when it + * does not own the schema (`reminder-worker.ts`). The role gets rows only, + * and not all of them: `pgboss.bam` holds SQL that pg-boss runs as whoever + * runs its migrations, so the role may only read it, and of + * `pgboss.version` it may update the timestamps but not the version. Owning + * nothing in the schema, the role cannot plant a trigger, a rule or a + * function body that a later owner session (a rollback to an older release, + * `DB_APP_ROLE=off`, a container that fell back to the owner) would run with + * the owner's rights. + * + * A pre-release build handed the schema to a cluster-wide `healthlog_app` + * role. Every start therefore also takes back, as the owner, anything in the + * schema a HealthLog-managed role owns, drops every trigger and rule there + * (pg-boss defines none), and when it took something back compares each + * function with pg-boss's own definition, built in a scratch schema, and + * restores the ones that differ ({@link hardenQueueSchema}). It runs on every + * path, `DB_APP_ROLE=off` and the fallback included. The legacy role loses + * its grants in this database. + * + * pg-boss rebuilds bloated indexes only on tables its connection owns; under + * the role it reports them instead. The start rebuilds them as the owner + * ({@link rebuildBloatedQueueIndexes}), each under a short lock wait. + * + * Every boot re-applies the grants, inside one transaction under an advisory + * lock, so a table a new migration created is covered before the app starts, + * a web and a worker container booting together do not race, and a restored + * database (whose objects `pg_restore --no-owner` hands to the owner) is put + * right. Grants only touch what lacks them, so a boot after the first writes * almost nothing to the catalog. * - * The password is derived, not stored: HMAC-SHA256 of a fixed label under - * API_TOKEN_HMAC_KEY. Every container of one instance carries that key, so a - * separate worker container derives the same password without a new - * variable. It is set as a SCRAM verifier computed here, so the plaintext - * never travels to the server, where a statement log could keep it. The role - * is marked with a comment; a role of the same name without the mark is not - * HealthLog's, and is left alone. + * The password is derived, not stored: HMAC-SHA256 of a label naming the + * database under API_TOKEN_HMAC_KEY. Every container of one instance carries + * that key, so a separate worker container derives the same password without + * a new variable. It is set as a SCRAM verifier computed here, so the + * plaintext never travels to the server, where a statement log could keep it. + * The role is marked with a comment; a role of the same name without the + * mark is not HealthLog's, and is left alone. * * Fail open, loudly. When the owner cannot do this (a managed or external * database whose user may not create roles, Postgres before 17, a pg_hba.conf * that does not let the new role in, a lock that did not come in time), the * script says why in one notice and the app runs as the owner, as it did - * before. `DB_APP_ROLE=off` skips it for an operator who manages roles + * before. `DB_APP_ROLE=off` skips the role for an operator who manages roles * themselves, including one whose `DATABASE_URL` already names a restricted role. * * Why Postgres 17: the maintenance job vacuums and rebuilds the indexes of @@ -70,29 +107,50 @@ * not printed. Exit 0 in every case the app should start; nothing written to * descriptor 3 means: stay on the owner. * - * Plain ESM with no imports beyond Node, `pg` and the shared dump module, so - * it runs from the runtime image like the other entrypoint scripts. + * Plain ESM with no imports beyond Node, `pg`, `pg-boss` and the shared dump + * module, so it runs from the runtime image like the other entrypoint scripts. */ import { createHash, createHmac, pbkdf2Sync, randomBytes } from "node:crypto"; import { writeSync } from "node:fs"; +import { createRequire } from "node:module"; import { pathToFileURL } from "node:url"; +import { + getConstructionPlans, + getIndexBloatPlans, + getMigrationPlans, +} from "pg-boss"; + import { connect } from "../src/lib/database-backup/dump-core.mjs"; import { CONCURRENT_COPY, RETIRED_MEASUREMENT_INDEXES, } from "../src/lib/jobs/retired-measurement-indexes.mjs"; -/** The role's name. Cluster-wide, like every Postgres role. */ -export const APP_ROLE = "healthlog_app"; +/** Every role this script creates starts with this. */ +export const APP_ROLE_PREFIX = "healthlog_app_"; -/** The mark on a role this script created. */ -export const ROLE_COMMENT = - "Managed by HealthLog: the app's least-privilege login (scripts/db-app-role.mjs). Recreated and re-granted on every start."; +/** The cluster-wide role a pre-release build created, retired here. */ +export const LEGACY_APP_ROLE = "healthlog_app"; + +/** The start of every comment this script puts on a role. */ +export const MANAGED_MARK = "Managed by HealthLog"; /** The schema pg-boss keeps its queue in (its default; HealthLog sets none). */ export const QUEUE_SCHEMA = "pgboss"; +/** The queue schema version this image's pg-boss installs and expects. */ +export const QUEUE_SCHEMA_VERSION = Number( + createRequire(import.meta.url)("pg-boss/package.json").pgboss.schema, +); + +/** + * Queue tables the role may only read: `bam` holds SQL that pg-boss's + * migration runner executes, `version` decides which migrations run (its + * timestamp columns stay writable, below). + */ +export const QUEUE_READ_ONLY_TABLES = Object.freeze(["bam", "version"]); + /** Tables the role may maintain (VACUUM, ANALYZE, REINDEX). */ export const MAINTAINED_TABLES = Object.freeze(["measurements"]); @@ -100,7 +158,7 @@ export const MAINTAINED_TABLES = Object.freeze(["measurements"]); export const READ_ONLY_TABLES = Object.freeze(["_prisma_migrations"]); /** The label the password is derived under. A new label is a new password. */ -const PASSWORD_LABEL = "healthlog database app role v1"; +const PASSWORD_LABEL = "healthlog database app role v2"; /** The first major with the MAINTAIN privilege. */ export const MIN_SERVER_VERSION_NUM = 170000; @@ -108,9 +166,13 @@ export const MIN_SERVER_VERSION_NUM = 170000; /** How long the setup waits for a lock before staying on the owner. */ const SETUP_LOCK_TIMEOUT = "15s"; -/** How long a boot-time index drop waits for its lock. */ +/** How long a boot-time index drop or rebuild waits for its lock. */ export const BOOT_DROP_LOCK_TIMEOUT = "5s"; +/** The advisory lock every start of every database takes for its setup. */ +const SETUP_LOCK = + "SELECT pg_advisory_xact_lock(hashtext('healthlog.db-app-role'))"; + /** An identifier this script splices into SQL: lower-case catalog names only. */ const IDENTIFIER = /^[a-z_][a-z0-9_]{0,62}$/; @@ -126,10 +188,38 @@ function ident(name) { return `"${name}"`; } -/** The role's password: 64 hex characters, the same in every container. */ -export function appRolePassword(hmacKey) { +/** Double-quote any identifier, doubling its quotes (database names). */ +function quoteAny(name) { + return `"${String(name).replace(/"/g, '""')}"`; +} + +/** + * The role's name for a database: the prefix, the database's name cut to + * letters, digits and underscores (at most 32), and eight hex digits of its + * SHA-256, so two databases whose names clean up alike still differ. At most + * 55 characters, under Postgres's 63. + */ +export function appRoleName(database) { + const clean = String(database) + .toLowerCase() + .replace(/[^a-z0-9_]+/g, "_") + .replace(/^_+|_+$/g, "") + .slice(0, 32); + const hash = createHash("sha256").update(String(database)).digest("hex"); + return `${APP_ROLE_PREFIX}${clean ? `${clean}_` : ""}${hash.slice(0, 8)}`; +} + +/** The mark on the role this script created for `database`. */ +export function roleComment(database) { + return `${MANAGED_MARK} for database ${JSON.stringify(String(database))}: the app's least-privilege login (scripts/db-app-role.mjs). Re-granted on every start.`; +} + +/** The role's password for a database: 64 hex characters, the same in every container. */ +export function appRolePassword(hmacKey, database) { if (!hmacKey) throw new Error("API_TOKEN_HMAC_KEY is not set"); - return createHmac("sha256", hmacKey).update(PASSWORD_LABEL).digest("hex"); + return createHmac("sha256", hmacKey) + .update(`${PASSWORD_LABEL}:${database}`) + .digest("hex"); } /** @@ -150,9 +240,9 @@ export function scramVerifier( } /** The owner's URL with the role's name and password: same host, database and parameters. */ -export function appRoleUrl(ownerUrl, password) { +export function appRoleUrl(ownerUrl, role, password) { const url = new URL(ownerUrl); - url.username = APP_ROLE; + url.username = role; url.password = password; return url.toString(); } @@ -177,22 +267,24 @@ function errorText(err) { * cannot. Read-only. * * @param {{ query: Function }} client - * @returns {Promise<{ ok: true, superuser: boolean } | { ok: false, reason: string, detail: string }>} + * @returns {Promise<{ ok: true, superuser: boolean, role: string, database: string } | { ok: false, reason: string, detail: string }>} */ export async function checkPreconditions(client) { const { rows } = await client.query( `SELECT current_setting('server_version_num')::int AS version, r.rolsuper AS superuser, r.rolcreaterole AS createrole, - current_user AS me + current_user AS me, + current_database() AS database FROM pg_roles r WHERE r.rolname = current_user`, ); const me = rows[0]; - if (me.me === APP_ROLE) { + const name = appRoleName(me.database); + if (me.me === name) { return { ok: false, reason: "already_app_role", - detail: `DATABASE_URL already names ${APP_ROLE}`, + detail: `DATABASE_URL already names ${name}`, }; } if (me.version < MIN_SERVER_VERSION_NUM) { @@ -212,45 +304,359 @@ export async function checkPreconditions(client) { const existing = await client.query( `SELECT shobj_description(r.oid, 'pg_authid') AS comment, pg_has_role(r.oid, current_user, 'MEMBER') AS member_of_owner, - r.rolsuper OR r.rolreplication OR r.rolbypassrls AS elevated + r.rolsuper OR r.rolreplication OR r.rolbypassrls AS elevated, + ARRAY( + SELECT d.datname::text FROM pg_database d, aclexplode(d.datacl) a + WHERE d.datname <> current_database() + AND a.grantee = r.oid AND a.privilege_type = 'CONNECT' + ORDER BY d.datname + ) AS other_databases FROM pg_roles r WHERE r.rolname = $1`, - [APP_ROLE], + [name], ); const role = existing.rows[0]; - if (role && role.comment !== ROLE_COMMENT) { + if (role && role.comment !== roleComment(me.database)) { return { ok: false, reason: "foreign_role", - detail: `a role named ${APP_ROLE} exists that HealthLog did not create`, + detail: `a role named ${name} exists that HealthLog did not create for this database`, }; } if (role?.member_of_owner) { return { ok: false, reason: "role_inherits_owner", - detail: `${APP_ROLE} is a member of ${me.me}, so it would hold the owner's rights`, + detail: `${name} is a member of ${me.me}, so it would hold the owner's rights`, }; } if (role?.elevated && !me.superuser) { return { ok: false, reason: "role_elevated", - detail: `${APP_ROLE} has SUPERUSER, REPLICATION or BYPASSRLS, and only a superuser can take it away`, + detail: `${name} has SUPERUSER, REPLICATION or BYPASSRLS, and only a superuser can take it away`, }; } - return { ok: true, superuser: me.superuser }; + if (role?.other_databases?.length > 0) { + return { + ok: false, + reason: "role_reaches_other_database", + detail: `${name} holds CONNECT on another database (${role.other_databases.join(", ")}); revoke it there first`, + }; + } + return { + ok: true, + superuser: me.superuser, + role: name, + database: me.database, + }; +} + +/** + * The queue schema's state, read-only: whether pg-boss is installed and at + * which version. + * + * @param {{ query: Function }} client + */ +async function queueSchemaVersion(client) { + const { rows } = await client.query( + `SELECT to_regclass($1) IS NOT NULL AS installed`, + [`${QUEUE_SCHEMA}.version`], + ); + if (!rows[0].installed) return null; + const { rows: version } = await client.query( + `SELECT max(version)::int AS version FROM ${ident(QUEUE_SCHEMA)}.version`, + ); + return version[0]?.version ?? null; +} + +/** + * Make the queue schema the owner's alone, on every start and every path. + * + * Takes back, in one transaction, every object in the schema a + * HealthLog-managed role owns (the pre-release build handed the schema to + * one), retires the legacy cluster-wide role's grants here, and drops every + * trigger and rule in the schema (pg-boss defines none; anything there was + * planted). When it took something back, the functions are compared with + * pg-boss's own afterwards ({@link restoreQueueFunctions}). + * + * @param {{ query: Function }} client connected as the owner + * @returns {Promise<{ reclaimed: number, dropped: string[] }>} + */ +export async function hardenQueueSchema(client) { + const { rows: present } = await client.query( + `SELECT to_regnamespace($1) IS NOT NULL AS present`, + [QUEUE_SCHEMA], + ); + const { rows: legacy } = await client.query( + `SELECT 1 FROM pg_roles r + WHERE r.rolname = $1 AND shobj_description(r.oid, 'pg_authid') LIKE $2`, + [LEGACY_APP_ROLE, `${MANAGED_MARK}%`], + ); + if (!present[0].present && legacy.length === 0) { + return { reclaimed: 0, dropped: [] }; + } + const queue = ident(QUEUE_SCHEMA); + let reclaimed = 0; + const dropped = []; + await client.query("BEGIN"); + try { + await client.query(`SET LOCAL lock_timeout = '${SETUP_LOCK_TIMEOUT}'`); + await client.query(SETUP_LOCK); + if (present[0].present) { + // Owned by a role HealthLog manages: any of them, this database's or + // another's, or the legacy one. + const managed = `SELECT oid FROM pg_roles + WHERE shobj_description(oid, 'pg_authid') LIKE '${MANAGED_MARK}%'`; + const { rows: schemaOwned } = await client.query( + `SELECT 1 FROM pg_namespace WHERE nspname = $1 AND nspowner IN (${managed})`, + [QUEUE_SCHEMA], + ); + if (schemaOwned.length > 0) { + await client.query(`ALTER SCHEMA ${queue} OWNER TO CURRENT_USER`); + reclaimed += 1; + } + const { rows: relations } = await client.query( + `SELECT c.oid::regclass::text AS name, c.relkind AS kind + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 + AND c.relowner IN (${managed}) + AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S') + AND NOT (c.relkind = 'S' AND EXISTS ( + SELECT 1 FROM pg_depend d + WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid + AND d.deptype IN ('a', 'i')))`, + [QUEUE_SCHEMA], + ); + const kinds = { + S: "SEQUENCE", + v: "VIEW", + m: "MATERIALIZED VIEW", + f: "FOREIGN TABLE", + }; + for (const { name, kind } of relations) { + // `regclass::text` is the catalog's own quoting of the name. + await client.query( + `ALTER ${kinds[kind] ?? "TABLE"} ${name} OWNER TO CURRENT_USER`, + ); + reclaimed += 1; + } + const { rows: routines } = await client.query( + `SELECT p.oid::regprocedure::text AS signature + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = $1 AND p.proowner IN (${managed})`, + [QUEUE_SCHEMA], + ); + for (const { signature } of routines) { + await client.query(`ALTER ROUTINE ${signature} OWNER TO CURRENT_USER`); + reclaimed += 1; + } + const { rows: types } = await client.query( + `SELECT t.oid::regtype::text AS name, t.typtype AS kind + FROM pg_type t + JOIN pg_namespace n ON n.oid = t.typnamespace + LEFT JOIN pg_class c ON c.oid = t.typrelid + WHERE n.nspname = $1 AND t.typowner IN (${managed}) AND t.typelem = 0 + AND (t.typtype IN ('e', 'd', 'r') OR (t.typtype = 'c' AND c.relkind = 'c'))`, + [QUEUE_SCHEMA], + ); + for (const { name, kind } of types) { + await client.query( + `ALTER ${kind === "d" ? "DOMAIN" : "TYPE"} ${name} OWNER TO CURRENT_USER`, + ); + reclaimed += 1; + } + + const { rows: triggers } = await client.query( + `SELECT t.tgname AS name, t.tgrelid::regclass::text AS on_table + FROM pg_trigger t + JOIN pg_class c ON c.oid = t.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND NOT t.tgisinternal`, + [QUEUE_SCHEMA], + ); + for (const { name, on_table: onTable } of triggers) { + await client.query(`DROP TRIGGER ${quoteAny(name)} ON ${onTable}`); + dropped.push(`trigger ${name}`); + } + const { rows: rules } = await client.query( + `SELECT r.rulename AS name, r.ev_class::regclass::text AS on_table + FROM pg_rewrite r + JOIN pg_class c ON c.oid = r.ev_class + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND r.rulename <> '_RETURN'`, + [QUEUE_SCHEMA], + ); + for (const { name, on_table: onTable } of rules) { + await client.query(`DROP RULE ${quoteAny(name)} ON ${onTable}`); + dropped.push(`rule ${name}`); + } + } + if (legacy.length > 0) { + // Its grants in this database; the role itself goes once no database + // holds any (DROP ROLE fails while another one does). + // REASSIGN first: DROP OWNED drops what the role still owns, and + // nothing it owns here may go with it. + await client.query( + `REASSIGN OWNED BY ${ident(LEGACY_APP_ROLE)} TO CURRENT_USER`, + ); + await client.query(`DROP OWNED BY ${ident(LEGACY_APP_ROLE)}`); + } + await client.query("COMMIT"); + } catch (err) { + await client.query("ROLLBACK").catch(() => {}); + throw err; + } + if (legacy.length > 0) { + await client + .query(`DROP ROLE ${ident(LEGACY_APP_ROLE)}`) + .catch(() => undefined); + } + return { reclaimed, dropped }; +} + +/** The functions of a schema by signature (without the schema), each as `CREATE OR REPLACE` text. */ +async function functionDefinitions(client, schema) { + const { rows } = await client.query( + `SELECT p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')' AS signature, + p.oid::regprocedure::text AS qualified, + pg_get_functiondef(p.oid) AS definition + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = $1`, + [schema], + ); + return new Map(rows.map((r) => [r.signature, r])); +} + +/** + * After a take-back: compare every function in the queue schema with the + * one pg-boss itself installs, built in a scratch schema, and restore any + * that differs; drop a function pg-boss does not define. Run as the owner. + * + * @param {{ query: Function }} client connected as the owner, outside a transaction + * @returns {Promise} the functions restored or dropped + */ +export async function restoreQueueFunctions(client) { + const scratch = "healthlog_pgboss_reference"; + const changed = []; + await client.query(`DROP SCHEMA IF EXISTS ${ident(scratch)} CASCADE`); + try { + await client.query(getConstructionPlans(scratch)); + const reference = await functionDefinitions(client, scratch); + const installed = await functionDefinitions(client, QUEUE_SCHEMA); + // The scratch name appears qualified and as a literal (catalog + // lookups); it is unique, so every occurrence is the schema's name. + const asQueue = (text) => text.split(scratch).join(QUEUE_SCHEMA); + await client.query("BEGIN"); + try { + await client.query(SETUP_LOCK); + for (const [signature, ref] of reference) { + const want = asQueue(ref.definition); + if (installed.get(signature)?.definition === want) continue; + await client.query(want); + changed.push(signature); + } + for (const [signature, own] of installed) { + if (reference.has(signature)) continue; + await client.query(`DROP ROUTINE ${own.qualified}`); + changed.push(`${signature} (dropped)`); + } + await client.query("COMMIT"); + } catch (err) { + await client.query("ROLLBACK").catch(() => {}); + throw err; + } + } finally { + await client.query(`DROP SCHEMA IF EXISTS ${ident(scratch)} CASCADE`); + } + return changed; +} + +/** + * Install or migrate the queue schema as the owner, with pg-boss's own plans, + * so the app (as the role, `migrate: false`) finds it current. A migration's + * concurrent index builds, which pg-boss would leave to its runner, are run + * here one after another. + * + * @param {{ query: Function }} client connected as the owner, outside a transaction + * @returns {Promise<"current" | "installed" | "migrated" | "newer">} + */ +export async function prepareQueueSchema(client) { + const version = await queueSchemaVersion(client); + if (version === null) { + await client.query(getConstructionPlans(QUEUE_SCHEMA)); + return "installed"; + } + if (version === QUEUE_SCHEMA_VERSION) return "current"; + if (version > QUEUE_SCHEMA_VERSION) return "newer"; + const [transactional, ...concurrent] = getMigrationPlans( + QUEUE_SCHEMA, + version, + ).split(/\n(?=-- inlined from )/); + await client.query(transactional); + for (const statement of concurrent) { + await client.query(statement.trim().replace(/;$/, "")); + } + return "migrated"; +} + +/** + * Rebuild, as the owner, the queue indexes pg-boss's own check calls bloated + * (under the role it may only report them). One at a time, each waiting at + * most {@link BOOT_DROP_LOCK_TIMEOUT} for its lock; a rebuild that fails drops + * its half-built copy and leaves the index for the next start. + * + * @param {{ query: Function }} client connected as the owner, outside a transaction + * @returns {Promise<{ rebuilt: string[], left: string[] }>} + */ +export async function rebuildBloatedQueueIndexes(client) { + const rebuilt = []; + const left = []; + if ((await queueSchemaVersion(client)) === null) return { rebuilt, left }; + const { rows } = await client.query(getIndexBloatPlans(QUEUE_SCHEMA)); + if (rows.length === 0) return { rebuilt, left }; + const { rows: setting } = await client.query( + `SELECT current_setting('lock_timeout') AS value`, + ); + try { + await client.query(`SET lock_timeout = '${BOOT_DROP_LOCK_TIMEOUT}'`); + for (const { name } of rows) { + const index = `${ident(QUEUE_SCHEMA)}.${ident(name)}`; + try { + await client.query(`REINDEX INDEX CONCURRENTLY ${index}`); + rebuilt.push(name); + } catch (err) { + left.push(name); + await client + .query( + `DROP INDEX CONCURRENTLY IF EXISTS ${ident(QUEUE_SCHEMA)}.${ident(`${name}_ccnew`)}`, + ) + .catch(() => undefined); + say( + `notice: queue index ${name} was not rebuilt (${errorText(err)}); the next start tries again`, + ); + } + } + } finally { + await client.query(`SELECT set_config('lock_timeout', $1, false)`, [ + setting[0]?.value ?? "0", + ]); + } + return { rebuilt, left }; } /** * Create or update the role and apply every grant, in one transaction. - * Call it on a client connected as the owner, after the migrations. + * Call it on a client connected as the owner, after the migrations and + * {@link prepareQueueSchema}. * * @param {{ query: Function }} client - * @param {{ password: string, schema?: string, superuser: boolean }} options + * @param {{ role: string, database: string, password: string, schema?: string, superuser: boolean }} options */ export async function applyAppRole(client, options) { const schema = options.schema ?? "public"; - const role = ident(APP_ROLE); + const name = options.role; + const role = ident(name); const app = ident(schema); const queue = ident(QUEUE_SCHEMA); // The verifier is a base64/`$`/`:` string; it never contains a quote. @@ -260,13 +666,11 @@ export async function applyAppRole(client, options) { await client.query("BEGIN"); try { await client.query(`SET LOCAL lock_timeout = '${SETUP_LOCK_TIMEOUT}'`); - await client.query( - `SELECT pg_advisory_xact_lock(hashtext('healthlog.db-app-role'))`, - ); + await client.query(SETUP_LOCK); const { rows: present } = await client.query( `SELECT 1 FROM pg_roles WHERE rolname = $1`, - [APP_ROLE], + [name], ); // The attributes a superuser alone may set are named only by one; a // role a CREATEROLE user creates never has them (checked beforehand). @@ -284,24 +688,15 @@ export async function applyAppRole(client, options) { ); } await client.query( - `COMMENT ON ROLE ${role} IS '${ROLE_COMMENT.replace(/'/g, "''")}'`, + `COMMENT ON ROLE ${role} IS '${roleComment(options.database).replace(/'/g, "''")}'`, ); - // A non-superuser owner hands objects to the role only when it may act - // as it (Postgres 16+), and must keep reading what the role owns: the - // dump before a migration reads the queue schema too. The membership - // runs owner -> role, never the other way. - if (!options.superuser) { - await client.query( - `GRANT ${role} TO CURRENT_USER WITH INHERIT TRUE, SET TRUE`, - ); - } + // A non-superuser owner may change a role's password and comment only + // with ADMIN on it, which creating it gave. It needs no membership: the + // role owns nothing. - const { rows: db } = await client.query( - `SELECT current_database() AS name`, - ); await client.query( - `GRANT CONNECT, TEMPORARY ON DATABASE "${String(db[0].name).replace(/"/g, '""')}" TO ${role}`, + `GRANT CONNECT, TEMPORARY ON DATABASE ${quoteAny(options.database)} TO ${role}`, ); // The app's schema: rows, never structure. @@ -319,51 +714,7 @@ export async function applyAppRole(client, options) { `notice: the schema ${schema} is not the database user's to change; PUBLIC may still create objects in it`, ); } - - const { rows: tables } = await client.query( - `SELECT c.relname AS name - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 AND c.relkind IN ('r', 'p', 'v', 'm', 'f') - AND c.relname <> ALL($3::text[]) - AND NOT (has_table_privilege($2, c.oid, 'SELECT') - AND has_table_privilege($2, c.oid, 'INSERT') - AND has_table_privilege($2, c.oid, 'UPDATE') - AND has_table_privilege($2, c.oid, 'DELETE'))`, - [schema, APP_ROLE, READ_ONLY_TABLES], - ); - for (const { name } of tables) { - await client.query( - `GRANT SELECT, INSERT, UPDATE, DELETE ON ${app}.${ident(name)} TO ${role}`, - ); - } - for (const name of READ_ONLY_TABLES) { - const { rows } = await client.query( - `SELECT to_regclass($1) IS NOT NULL AS present`, - [`${app}.${ident(name)}`], - ); - if (!rows[0].present) continue; - await client.query( - `REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON ${app}.${ident(name)} FROM ${role}`, - ); - await client.query(`GRANT SELECT ON ${app}.${ident(name)} TO ${role}`); - } - - const { rows: sequences } = await client.query( - `SELECT c.relname AS name - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 AND c.relkind = 'S' - AND NOT (has_sequence_privilege($2, c.oid, 'USAGE') - AND has_sequence_privilege($2, c.oid, 'SELECT') - AND has_sequence_privilege($2, c.oid, 'UPDATE'))`, - [schema, APP_ROLE], - ); - for (const { name } of sequences) { - await client.query( - `GRANT USAGE, SELECT, UPDATE ON SEQUENCE ${app}.${ident(name)} TO ${role}`, - ); - } + await grantRows(client, { schema, role: name, readOnly: READ_ONLY_TABLES }); // Objects the owner creates later in this schema (the migrations of the // next update) are granted at creation as well. The per-boot pass above @@ -375,27 +726,44 @@ export async function applyAppRole(client, options) { `ALTER DEFAULT PRIVILEGES IN SCHEMA ${app} GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO ${role}`, ); - for (const name of MAINTAINED_TABLES) { + for (const table of MAINTAINED_TABLES) { const { rows } = await client.query( `SELECT to_regclass($1) IS NOT NULL AS present`, - [`${app}.${ident(name)}`], + [`${app}.${ident(table)}`], ); if (rows[0].present) { await client.query( - `GRANT MAINTAIN ON ${app}.${ident(name)} TO ${role}`, + `GRANT MAINTAIN ON ${app}.${ident(table)} TO ${role}`, ); } } - // The queue's schema belongs to the role: pg-boss creates its tables, - // functions and partitions there at runtime. On the first boot after an - // update, and after a restore, everything in it still belongs to the - // owner and is handed over once. - await client.query( - `CREATE SCHEMA IF NOT EXISTS ${queue} AUTHORIZATION ${role}`, + // The queue schema: rows only, and only read on the tables whose + // contents decide what pg-boss's migration runner executes. No default + // privileges here: a table a later pg-boss adds is granted by the next + // start's pass, after this script migrated it, so a new table of that + // kind is not writable before anyone has looked at it. + await client.query(`GRANT USAGE ON SCHEMA ${queue} TO ${role}`); + await client.query(`REVOKE CREATE ON SCHEMA ${queue} FROM ${role}`); + await grantRows(client, { + schema: QUEUE_SCHEMA, + role: name, + readOnly: QUEUE_READ_ONLY_TABLES, + }); + // pg-boss stamps its maintenance times on `version`; the version itself + // stays the owner's to write. + const { rows: stamps } = await client.query( + `SELECT a.attname AS name FROM pg_attribute a + WHERE a.attrelid = to_regclass($1) AND a.attnum > 0 AND NOT a.attisdropped + AND a.attname <> 'version' + AND NOT has_column_privilege($2, a.attrelid, a.attnum, 'UPDATE')`, + [`${QUEUE_SCHEMA}.version`, name], ); - await client.query(`ALTER SCHEMA ${queue} OWNER TO ${role}`); - await transferQueueObjects(client, role); + if (stamps.length > 0) { + await client.query( + `GRANT UPDATE (${stamps.map((c) => ident(c.name)).join(", ")}) ON ${queue}.${ident("version")} TO ${role}`, + ); + } await client.query("COMMIT"); } catch (err) { @@ -404,66 +772,54 @@ export async function applyAppRole(client, options) { } } -/** Hand every object in the queue schema the role does not own to it. */ -async function transferQueueObjects(client, role) { - const { rows: relations } = await client.query( - `SELECT c.oid::regclass::text AS name, c.relkind AS kind +/** + * Grant rows on every table of a schema that lacks them, and read only on + * `readOnly` (any write it holds there is taken away); sequences alike. + */ +async function grantRows(client, { schema, role: name, readOnly }) { + const role = ident(name); + const where = ident(schema); + const { rows: tables } = await client.query( + `SELECT c.relname AS name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 - AND c.relowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) - AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S') - -- A sequence a column owns follows its table. - AND NOT (c.relkind = 'S' AND EXISTS ( - SELECT 1 FROM pg_depend d - WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid - AND d.deptype IN ('a', 'i'))) - ORDER BY c.relkind = 'r' DESC, c.oid`, - [QUEUE_SCHEMA, APP_ROLE], + WHERE n.nspname = $1 AND c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND c.relname <> ALL($3::text[]) + AND NOT (has_table_privilege($2, c.oid, 'SELECT') + AND has_table_privilege($2, c.oid, 'INSERT') + AND has_table_privilege($2, c.oid, 'UPDATE') + AND has_table_privilege($2, c.oid, 'DELETE'))`, + [schema, name, readOnly], ); - for (const { name, kind } of relations) { - const what = - kind === "S" - ? "SEQUENCE" - : kind === "v" - ? "VIEW" - : kind === "m" - ? "MATERIALIZED VIEW" - : kind === "f" - ? "FOREIGN TABLE" - : "TABLE"; - // `regclass::text` is the catalog's own quoting of the name. - await client.query(`ALTER ${what} ${name} OWNER TO ${role}`); - } - - const { rows: routines } = await client.query( - `SELECT p.oid::regprocedure::text AS signature - FROM pg_proc p - JOIN pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname = $1 - AND p.proowner <> (SELECT oid FROM pg_roles WHERE rolname = $2)`, - [QUEUE_SCHEMA, APP_ROLE], - ); - for (const { signature } of routines) { - await client.query(`ALTER ROUTINE ${signature} OWNER TO ${role}`); - } - - // Enums, domains and free-standing composite types; a table's row type - // and an array type follow their owner object. - const { rows: types } = await client.query( - `SELECT t.oid::regtype::text AS name, t.typtype AS kind - FROM pg_type t - JOIN pg_namespace n ON n.oid = t.typnamespace - LEFT JOIN pg_class c ON c.oid = t.typrelid - WHERE n.nspname = $1 - AND t.typowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) - AND t.typelem = 0 - AND (t.typtype IN ('e', 'd', 'r') OR (t.typtype = 'c' AND c.relkind = 'c'))`, - [QUEUE_SCHEMA, APP_ROLE], - ); - for (const { name, kind } of types) { + for (const { name: table } of tables) { await client.query( - `ALTER ${kind === "d" ? "DOMAIN" : "TYPE"} ${name} OWNER TO ${role}`, + `GRANT SELECT, INSERT, UPDATE, DELETE ON ${where}.${ident(table)} TO ${role}`, + ); + } + for (const table of readOnly) { + const { rows } = await client.query( + `SELECT to_regclass($1) IS NOT NULL AS present`, + [`${where}.${ident(table)}`], + ); + if (!rows[0].present) continue; + await client.query( + `REVOKE INSERT, UPDATE, DELETE, TRUNCATE, TRIGGER, REFERENCES ON ${where}.${ident(table)} FROM ${role}`, + ); + await client.query(`GRANT SELECT ON ${where}.${ident(table)} TO ${role}`); + } + const { rows: sequences } = await client.query( + `SELECT c.relname AS name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = 'S' + AND NOT (has_sequence_privilege($2, c.oid, 'USAGE') + AND has_sequence_privilege($2, c.oid, 'SELECT') + AND has_sequence_privilege($2, c.oid, 'UPDATE'))`, + [schema, name], + ); + for (const { name: sequence } of sequences) { + await client.query( + `GRANT USAGE, SELECT, UPDATE ON SEQUENCE ${where}.${ident(sequence)} TO ${role}`, ); } } @@ -523,12 +879,12 @@ export async function dropOwnerOnlyIndexes(client) { } /** Log in as the role once, so a pg_hba.conf that refuses it shows here. */ -async function proveLogin(url) { +async function proveLogin(url, role) { const client = await connect(url); try { const { rows } = await client.query(`SELECT current_user AS me`); - if (rows[0].me !== APP_ROLE) { - throw new Error(`logged in as ${rows[0].me}, not ${APP_ROLE}`); + if (rows[0].me !== role) { + throw new Error(`logged in as ${rows[0].me}, not ${role}`); } } finally { await client.end().catch(() => {}); @@ -545,6 +901,38 @@ function handOver(url) { } } +/** + * The queue schema back to the owner alone, on any path. A failure is a + * notice, never a stop: on the role path the setup then falls back. + * + * @returns {Promise} whether the schema is in the owner's hands + */ +async function secureQueue(client) { + try { + const { reclaimed, dropped } = await hardenQueueSchema(client); + if (dropped.length > 0) { + say( + `warning: dropped ${dropped.join(", ")} in the queue schema; pg-boss defines none`, + ); + } + if (reclaimed > 0) { + say( + `took back ${reclaimed} object(s) in the queue schema from a HealthLog role`, + ); + const changed = await restoreQueueFunctions(client); + if (changed.length > 0) { + say( + `warning: restored queue function(s) ${changed.join(", ")} to pg-boss's own definition`, + ); + } + } + return true; + } catch (err) { + say(`notice: the queue schema check failed (${errorText(err)})`); + return false; + } +} + /** * The boot step. Returns the role's URL when the app should use it, or null * to stay on the owner. @@ -555,13 +943,8 @@ function handOver(url) { export async function ensureAppRole(env = process.env) { const ownerUrl = env.DATABASE_URL; const mode = (env.DB_APP_ROLE ?? "").trim().toLowerCase(); - if (mode === "off" || mode === "false") { - say( - `DB_APP_ROLE=off: the app connects as ${userOf(ownerUrl)}, the user DATABASE_URL names`, - ); - return null; - } - if (mode && mode !== "auto" && mode !== "on") { + const off = mode === "off" || mode === "false"; + if (mode && !off && mode !== "auto" && mode !== "on") { say( `notice: DB_APP_ROLE="${mode}" is not auto or off; treating it as auto`, ); @@ -571,7 +954,7 @@ export async function ensureAppRole(env = process.env) { const user = userOf(ownerUrl); const fallBack = (detail) => { say( - `notice: the app connects as ${user}, the user DATABASE_URL names, not as the restricted role ${APP_ROLE}: ${detail}. ` + + `notice: the app connects as ${user}, the user DATABASE_URL names, not as a restricted role: ${detail}. ` + "This is how every earlier release ran. " + `If ${user} is a restricted role you manage yourself, DB_APP_ROLE=off silences this; otherwise see docs/self-hosting/database-role.md.`, ); @@ -582,21 +965,40 @@ export async function ensureAppRole(env = process.env) { try { client = await connect(ownerUrl); } catch (err) { + if (off) return null; return fallBack(`could not connect to set it up (${errorText(err)})`); } try { + // Every path, before anything runs as the owner against the queue. + const secured = await secureQueue(client); + if (off) { + say( + `DB_APP_ROLE=off: the app connects as ${user}, the user DATABASE_URL names`, + ); + return null; + } const pre = await checkPreconditions(client); if (!pre.ok) { if (pre.reason === "already_app_role") { - say(`DATABASE_URL already names ${APP_ROLE}; nothing to set up`); + say(`${pre.detail}; nothing to set up`); return null; } return fallBack(pre.detail); } - const password = appRolePassword(env.API_TOKEN_HMAC_KEY); + if (!secured) return fallBack("the queue schema could not be checked"); + const password = appRolePassword(env.API_TOKEN_HMAC_KEY, pre.database); const schema = new URL(ownerUrl).searchParams.get("schema") || "public"; try { + const queue = await prepareQueueSchema(client); + if (queue === "newer") { + return fallBack( + "the queue schema is newer than this image's pg-boss (a rollback?), and only the owner may run on it", + ); + } + if (queue !== "current") say(`queue schema ${queue} as the owner`); await applyAppRole(client, { + role: pre.role, + database: pre.database, password, schema, superuser: pre.superuser, @@ -604,23 +1006,33 @@ export async function ensureAppRole(env = process.env) { } catch (err) { return fallBack(`setting it up failed (${errorText(err)})`); } - const url = appRoleUrl(ownerUrl, password); + const url = appRoleUrl(ownerUrl, pre.role, password); try { - await proveLogin(url); + await proveLogin(url, pre.role); } catch (err) { return fallBack( - `${APP_ROLE} cannot log in (${errorText(err)}); a pg_hba.conf that lists users by name needs a line for it`, + `${pre.role} cannot log in (${errorText(err)}); a pg_hba.conf that lists users by name needs a line for it`, ); } const { dropped } = await dropOwnerOnlyIndexes(client); if (dropped.length > 0) { say(`dropped index ${dropped.join(", ")} on measurements`); } + const { rebuilt } = await rebuildBloatedQueueIndexes(client).catch( + (err) => { + say(`notice: the queue index check failed (${errorText(err)})`); + return { rebuilt: [] }; + }, + ); + if (rebuilt.length > 0) { + say(`rebuilt queue index ${rebuilt.join(", ")}`); + } say( - `the app connects as ${APP_ROLE}: rows and the queue, no schema changes (DB_APP_ROLE=off turns this off)`, + `the app connects as ${pre.role}: rows and the queue, no schema changes (DB_APP_ROLE=off turns this off)`, ); return url; } catch (err) { + if (off) return null; return fallBack(`the check failed (${errorText(err)})`); } finally { await client.end().catch(() => {}); diff --git a/src/lib/jobs/boss-instance.ts b/src/lib/jobs/boss-instance.ts index 169021e24..6fae7610e 100644 --- a/src/lib/jobs/boss-instance.ts +++ b/src/lib/jobs/boss-instance.ts @@ -89,20 +89,25 @@ function databaseOptions( } /** - * The worker's pg-boss. `createSchema` says whether pg-boss's first install - * may run `CREATE SCHEMA IF NOT EXISTS pgboss`. Postgres checks the database's - * CREATE privilege before it checks whether the schema exists, so under the - * app's restricted role (`scripts/db-app-role.mjs`), which may not create - * schemas, that statement fails even though the boot already created the - * schema for it. The caller passes false when the schema is there. + * The worker's pg-boss. `ownsQueueSchema` is false when the connection does + * not own the `pgboss` schema, which is how the app's restricted role + * (`scripts/db-app-role.mjs`) runs: the boot installed and migrated the + * schema as the owner, so pg-boss neither installs, migrates nor runs the + * index builds a migration queues (`migrate: false`), and never sends + * `CREATE SCHEMA IF NOT EXISTS` (`createSchema: false`), which Postgres + * refuses without the database's CREATE privilege even when the schema is + * there. With `migrate: false`, pg-boss refuses to start on a schema that is + * not at its version, rather than run on it. */ export function createWorkerBoss( connectionString: string, - options: { createSchema?: boolean } = {}, + options: { ownsQueueSchema?: boolean } = {}, ): PgBoss { return new PgBoss({ ...databaseOptions(connectionString, "worker"), - ...(options.createSchema === false ? { createSchema: false } : {}), + ...(options.ownsQueueSchema === false + ? { createSchema: false, migrate: false } + : {}), }); } diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index 72991a313..4c251b7c4 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -48,13 +48,17 @@ export async function startReminderWorker() { return; } - // The queue schema exists once a worker has started, and on every boot - // the restricted database role runs under (the entrypoint creates it for - // that role); only a fresh database without either needs pg-boss to - // create it. - const [{ present }] = await prisma.$queryRaw<{ present: boolean }[]>` - SELECT to_regnamespace('pgboss') IS NOT NULL AS present`; - const boss = createWorkerBoss(DATABASE_URL, { createSchema: !present }); + // Under the restricted database role the boot installed and migrated the + // queue schema as its owner, and the role owns nothing there; pg-boss then + // only reads and writes rows. A connection that owns the schema, or a fresh + // database without one, lets pg-boss install and migrate it itself. + const [{ owned }] = await prisma.$queryRaw<{ owned: boolean | null }[]>` + SELECT pg_has_role(current_user, n.nspowner, 'USAGE') AS owned + FROM (SELECT 1) one + LEFT JOIN pg_namespace n ON n.nspname = 'pgboss'`; + const boss = createWorkerBoss(DATABASE_URL, { + ownsQueueSchema: owned !== false, + }); boss.on("error", (error: unknown) => { workerLog("error", "boss emitted error", error); diff --git a/tests/integration/db-app-role.test.ts b/tests/integration/db-app-role.test.ts index 0be78cf29..9df0538f4 100644 --- a/tests/integration/db-app-role.test.ts +++ b/tests/integration/db-app-role.test.ts @@ -2,44 +2,68 @@ * The app's restricted database role, against a real Postgres. * * `scripts/db-app-role.mjs` runs at boot as the owner and hands the app a URL - * for `healthlog_app`. Proven here: the role does what the app does (Prisma - * reads and writes, pg-boss installing its schema and creating a queue - * partition, the measurement maintenance's VACUUM and concurrent rebuild), - * and is refused what it must not do (DDL in the app's schema, roles, - * schemas, server files and programs, the migration history). A second run is - * a no-op that keeps the same password, the index the role may not drop is - * dropped by the next run as the owner, and an owner that may not create - * roles leaves the app on the owner with a notice. + * for this database's own role (`healthlog_app__`). Proven + * here: + * + * - The role does what the app does: Prisma reads and writes, the queue + * lifecycle the worker runs (start with `migrate: false`, create a new + * queue, send, fetch, complete, schedule, delete a queue, a supervise + * pass), the measurement maintenance's VACUUM and concurrent rebuild. + * - It is refused what it must not do: DDL in either schema, triggers, + * functions, roles, schemas, server files and programs, the migration + * history, and the two queue tables whose contents pg-boss executes or + * migrates by. + * - It owns nothing in the queue schema, which the boot installs as the + * owner, and the boot takes back, from a pre-release state, a schema a + * HealthLog role owned, with a planted trigger and a replaced function. + * - Two databases on one server get two roles that do not lock each other + * out and cannot read each other's data; a role that holds CONNECT on + * another database is refused. + * - A second run is a no-op that keeps the password; an owner that may not + * create roles, and DB_APP_ROLE=off, leave the app on the owner. * * On Postgres 16 (the nightly `INTEGRATION_PG_IMAGE` run) the script stays * on the owner by design; the file proves that instead. */ import { PrismaPg } from "@prisma/adapter-pg"; import { Client } from "pg"; -import { PgBoss } from "pg-boss"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { PrismaClient } from "@/generated/prisma/client"; +import { createWorkerBoss } from "@/lib/jobs/boss-instance"; import { MAINTENANCE_SESSION_OPTIONS, RETIRED_MEASUREMENT_INDEXES, runMeasurementMaintenance, } from "@/lib/jobs/measurement-maintenance"; import { - APP_ROLE, + LEGACY_APP_ROLE, + MANAGED_MARK, + appRoleName, appRolePassword, ensureAppRole, + restoreQueueFunctions, } from "../../scripts/db-app-role.mjs"; +import { TEMPLATE_DATABASE } from "./worker-database"; const HMAC_KEY = "a".repeat(64); +const OTHER_HMAC_KEY = "b".repeat(64); const PROBE_OWNER = "app_role_probe_owner"; const USER_ID = "user-app-role-probe"; const RETIRED = RETIRED_MEASUREMENT_INDEXES[0]; +const OTHER_DB = `app_role_probe_other_${process.pid}`; -function ownerUrl(): string { +function ownerUrl(database?: string): string { const url = process.env.DATABASE_URL; if (!url) throw new Error("DATABASE_URL is not set for the integration run"); - return url; + if (!database) return url; + const u = new URL(url); + u.pathname = `/${database}`; + return u.toString(); +} + +function databaseOf(url: string): string { + return decodeURIComponent(new URL(url).pathname.slice(1)); } function withoutSchema(url: string): string { @@ -83,23 +107,46 @@ async function sqlState(db: Client, sql: string): Promise { } } +/** Objects in the queue schema not owned by `owner`. */ +async function queueObjectsNotOwnedBy( + db: Client, + owner: string, +): Promise { + const { rows } = await db.query<{ n: string }>( + `SELECT ( + (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'pgboss' AND pg_get_userbyid(c.relowner) <> $1) + + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'pgboss' AND pg_get_userbyid(p.proowner) <> $1) + + (SELECT count(*) FROM pg_namespace WHERE nspname = 'pgboss' + AND pg_get_userbyid(nspowner) <> $1))::text AS n`, + [owner], + ); + return Number(rows[0].n); +} + let owner: Client; +let ownerName = ""; let serverVersion = 0; +const roleA = () => appRoleName(databaseOf(ownerUrl())); +const roleB = appRoleName(OTHER_DB); beforeAll(async () => { owner = await connect(ownerUrl()); - const { rows } = await owner.query<{ v: number }>( - `SELECT current_setting('server_version_num')::int AS v`, + const { rows } = await owner.query<{ v: number; me: string }>( + `SELECT current_setting('server_version_num')::int AS v, current_user AS me`, ); serverVersion = rows[0].v; + ownerName = rows[0].me; // Each file starts from a worker database without the queue schema. await owner.query(`DROP SCHEMA IF EXISTS pgboss CASCADE`); }); afterAll(async () => { - // The role is cluster-wide: hand back what it owns here and drop it, so no - // other file or run sees it. - for (const role of [APP_ROLE, PROBE_OWNER]) { + await owner.query(`DROP DATABASE IF EXISTS "${OTHER_DB}" WITH (FORCE)`); + // Roles are cluster-wide: hand back what they hold here and drop them, so + // no other file or run sees them. + for (const role of [roleA(), roleB, LEGACY_APP_ROLE, PROBE_OWNER]) { const { rows } = await owner.query( `SELECT 1 FROM pg_roles WHERE rolname = $1`, [role], @@ -114,8 +161,24 @@ afterAll(async () => { await owner.end(); }); +describe("the role's name", () => { + it("is per database, under 63 characters, and stable", () => { + expect(appRoleName("healthlog")).toMatch( + /^healthlog_app_healthlog_[0-9a-f]{8}$/, + ); + expect(appRoleName("healthlog")).toBe(appRoleName("healthlog")); + expect(appRoleName("healthlog")).not.toBe(appRoleName("healthlog-staging")); + // Names that clean up alike still differ by their hash. + expect(appRoleName("a-b")).not.toBe(appRoleName("a_b")); + expect(appRoleName("x".repeat(63)).length).toBeLessThanOrEqual(63); + expect(appRolePassword(HMAC_KEY, "a")).not.toBe( + appRolePassword(HMAC_KEY, "b"), + ); + }); +}); + describe("DB_APP_ROLE=off", () => { - it("stays on the owner without touching the database", async () => { + it("stays on the owner and creates no role", async () => { const { url, said } = await boot({ DATABASE_URL: ownerUrl(), API_TOKEN_HMAC_KEY: HMAC_KEY, @@ -125,7 +188,7 @@ describe("DB_APP_ROLE=off", () => { expect(said).toContain("DB_APP_ROLE=off"); const { rows } = await owner.query( `SELECT 1 FROM pg_roles WHERE rolname = $1`, - [APP_ROLE], + [roleA()], ); expect(rows).toHaveLength(0); }); @@ -149,7 +212,9 @@ describe("an owner that may not create roles", () => { expect(said).toContain("may not create roles"); } expect(said).not.toContain("probe-owner-pw"); - expect(said).not.toContain(appRolePassword(HMAC_KEY)); + expect(said).not.toContain( + appRolePassword(HMAC_KEY, databaseOf(ownerUrl())), + ); }); }); @@ -183,8 +248,11 @@ describe("the restricted role (Postgres 17+)", () => { API_TOKEN_HMAC_KEY: HMAC_KEY, }); expect(first.url, first.said).not.toBeNull(); - expect(first.said).toContain(`the app connects as ${APP_ROLE}`); - expect(first.said).not.toContain(appRolePassword(HMAC_KEY)); + expect(first.said).toContain(`the app connects as ${roleA()}`); + expect(first.said).toContain("queue schema installed as the owner"); + expect(first.said).not.toContain( + appRolePassword(HMAC_KEY, databaseOf(ownerUrl())), + ); // The first run drops it as the owner; put it back for the role's turn. expect(first.said).toContain(`dropped index ${RETIRED}`); await owner.query( @@ -198,12 +266,14 @@ describe("the restricted role (Postgres 17+)", () => { await app?.end(); }); - it("keeps the owner's host, database and parameters", async (ctx) => { + it("keeps the owner's host, database and parameters", (ctx) => { if (serverVersion < 170000) ctx.skip(); const owned = new URL(ownerUrl()); const handed = new URL(appUrl); - expect(handed.username).toBe(APP_ROLE); - expect(handed.password).toBe(appRolePassword(HMAC_KEY)); + expect(handed.username).toBe(roleA()); + expect(handed.password).toBe( + appRolePassword(HMAC_KEY, databaseOf(ownerUrl())), + ); expect(handed.host).toBe(owned.host); expect(handed.pathname).toBe(owned.pathname); expect(handed.search).toBe(owned.search); @@ -248,33 +318,35 @@ describe("the restricted role (Postgres 17+)", () => { } }); - it("runs pg-boss: install, a queue partition, a job", async (ctx) => { + it("runs the worker's queue lifecycle owning nothing in the queue schema", async (ctx) => { if (serverVersion < 170000) ctx.skip(); - const boss = new PgBoss({ - connectionString: withoutSchema(appUrl), - createSchema: false, - supervise: false, - schedule: false, + expect(await queueObjectsNotOwnedBy(owner, ownerName)).toBe(0); + const errors: unknown[] = []; + // The options the worker passes when it does not own the schema. + const boss = createWorkerBoss(withoutSchema(appUrl), { + ownsQueueSchema: false, }); - boss.on("error", () => {}); + boss.on("error", (err: unknown) => errors.push(err)); await boss.start(); try { - await boss.createQueue("app-role-probe", { policy: "standard" }); - const id = await boss.send("app-role-probe", { n: 1 }); - const [job] = await boss.fetch("app-role-probe"); + // A queue no earlier boot created: a row, no table. + const queue = `app-role-probe-${Date.now()}`; + await boss.createQueue(queue, { policy: "standard" }); + await boss.createQueue(`${queue}-singleton`, { policy: "singleton" }); + const id = await boss.send(queue, { n: 1 }); + const [job] = await boss.fetch(queue); expect(job?.id).toBe(id); - await boss.complete("app-role-probe", job.id); + await boss.complete(queue, job.id); + await boss.schedule(queue, "0 3 * * *", {}, { tz: "Europe/Berlin" }); + await boss.unschedule(queue); + await boss.supervise(); + await boss.deleteQueue(`${queue}-singleton`); } finally { await boss.stop({ graceful: false }); } - const { rows } = await owner.query<{ foreign: string }>( - `SELECT count(*)::text AS foreign FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'pgboss' AND c.relkind IN ('r', 'p') - AND pg_get_userbyid(c.relowner) <> $1`, - [APP_ROLE], - ); - expect(rows[0].foreign).toBe("0"); + expect(errors).toEqual([]); + // Still nothing of the role's in the schema. + expect(await queueObjectsNotOwnedBy(owner, ownerName)).toBe(0); }); it("vacuums and rebuilds measurements, and leaves the drop to the owner", async (ctx) => { @@ -302,7 +374,7 @@ describe("the restricted role (Postgres 17+)", () => { } }); - it("is refused schema changes, roles, schemas, server files and the migration history", async (ctx) => { + it("is refused schema changes, triggers, functions, roles, server files and the queue's executed tables", async (ctx) => { if (serverVersion < 170000) ctx.skip(); const refused = { dropTable: await sqlState(app, `DROP TABLE "users"`), @@ -324,9 +396,32 @@ describe("the restricted role (Postgres 17+)", () => { app, `DELETE FROM _prisma_migrations WHERE false`, ), - becomeOwner: await sqlState( + becomeOwner: await sqlState(app, `SET ROLE "${ownerName}"`), + queueTrigger: await sqlState( app, - `SET ROLE ${new URL(ownerUrl()).username}`, + `CREATE TRIGGER probe BEFORE INSERT ON pgboss.job_common + FOR EACH ROW EXECUTE FUNCTION pgboss.job_now()`, + ), + queueFunction: await sqlState( + app, + `CREATE FUNCTION pgboss.probe() RETURNS int LANGUAGE sql AS 'SELECT 1'`, + ), + replaceQueueFunction: await sqlState( + app, + `CREATE OR REPLACE FUNCTION pgboss.job_now() RETURNS timestamptz LANGUAGE sql AS 'SELECT now()'`, + ), + queueTable: await sqlState(app, `CREATE TABLE pgboss.probe (x int)`), + alterQueueTable: await sqlState( + app, + `ALTER TABLE pgboss.queue ADD COLUMN probe text`, + ), + writeBam: await sqlState( + app, + `INSERT INTO pgboss.bam (name, version, status, command) VALUES ('probe', 1, 'pending', 'SELECT 1')`, + ), + writeQueueVersion: await sqlState( + app, + `UPDATE pgboss.version SET version = version`, ), }; expect(refused).toEqual({ @@ -341,7 +436,18 @@ describe("the restricted role (Postgres 17+)", () => { dropIndex: "42501", writeMigrations: "42501", becomeOwner: "42501", + queueTrigger: "42501", + queueFunction: "42501", + replaceQueueFunction: "42501", + queueTable: "42501", + alterQueueTable: "42501", + writeBam: "42501", + writeQueueVersion: "42501", }); + // pg-boss's own maintenance stamps stay writable. + expect( + await sqlState(app, `UPDATE pgboss.version SET cron_on = cron_on`), + ).toBeNull(); const { rows } = await app.query( `SELECT rolsuper, rolcreaterole, rolcreatedb, rolinherit, rolreplication, rolbypassrls FROM pg_roles WHERE rolname = current_user`, @@ -356,10 +462,165 @@ describe("the restricted role (Postgres 17+)", () => { }); }); - it("is idempotent: a second boot keeps the password and drops what the role could not", async (ctx) => { + it("does not let the worker migrate the queue schema as the role", async (ctx) => { + if (serverVersion < 170000) ctx.skip(); + const { rows } = await owner.query<{ v: number }>( + `SELECT max(version)::int AS v FROM pgboss.version`, + ); + await owner.query(`UPDATE pgboss.version SET version = $1`, [ + rows[0].v - 1, + ]); + try { + const boss = createWorkerBoss(withoutSchema(appUrl), { + ownsQueueSchema: false, + }); + boss.on("error", () => {}); + await expect(boss.start()).rejects.toThrow(/requires migrations/); + await boss.stop({ graceful: false }).catch(() => {}); + } finally { + await owner.query(`UPDATE pgboss.version SET version = $1`, [rows[0].v]); + } + }); + + it("leaves pg-boss's own functions alone on an untouched schema", async (ctx) => { + if (serverVersion < 170000) ctx.skip(); + expect(await restoreQueueFunctions(owner)).toEqual([]); + }); + + it("gives a second database on the server its own role, without locking the first out", async (ctx) => { + if (serverVersion < 170000) ctx.skip(); + await owner.query( + `SELECT pg_advisory_lock(hashtext('healthlog-integration-clone'))`, + ); + try { + await owner.query( + `CREATE DATABASE "${OTHER_DB}" TEMPLATE "${TEMPLATE_DATABASE}"`, + ); + } finally { + await owner.query( + `SELECT pg_advisory_unlock(hashtext('healthlog-integration-clone'))`, + ); + } + const other = await boot({ + DATABASE_URL: ownerUrl(OTHER_DB), + API_TOKEN_HMAC_KEY: OTHER_HMAC_KEY, + }); + expect(other.url, other.said).not.toBeNull(); + expect(new URL(other.url!).username).toBe(roleB); + expect(roleB).not.toBe(roleA()); + + // Both log in to their own database, the first after the second's boot. + for (const [url, role] of [ + [appUrl, roleA()], + [other.url!, roleB], + ] as const) { + const db = await connect(url); + const { rows } = await db.query(`SELECT current_user AS me`); + expect(rows[0].me).toBe(role); + await db.query(`SELECT count(*) FROM users`); + await db.end(); + } + + // Each is refused the other's data and queue. + for (const [url, database] of [ + [appUrl, OTHER_DB], + [other.url!, databaseOf(ownerUrl())], + ] as const) { + const u = new URL(url); + u.pathname = `/${database}`; + const db = await connect(u.toString()); + expect(await sqlState(db, `SELECT count(*) FROM users`)).toBe("42501"); + expect(await sqlState(db, `SELECT count(*) FROM pgboss.job`)).toBe( + "42501", + ); + expect(await sqlState(db, `CREATE TABLE public.probe (x int)`)).toBe( + "42501", + ); + await db.end(); + } + + // A role that holds CONNECT on another database is not used. + await owner.query( + `GRANT CONNECT ON DATABASE "${OTHER_DB}" TO "${roleA()}"`, + ); + try { + const refused = await boot({ + DATABASE_URL: ownerUrl(), + API_TOKEN_HMAC_KEY: HMAC_KEY, + }); + expect(refused.url).toBeNull(); + expect(refused.said).toContain( + `holds CONNECT on another database (${OTHER_DB})`, + ); + } finally { + await owner.query( + `REVOKE CONNECT ON DATABASE "${OTHER_DB}" FROM "${roleA()}"`, + ); + } + }); + + it("takes the queue schema back from a pre-release role, with what it planted", async (ctx) => { + if (serverVersion < 170000) ctx.skip(); + const { rows: before } = await owner.query<{ src: string }>( + `SELECT prosrc AS src FROM pg_proc WHERE oid = 'pgboss.job_now()'::regprocedure`, + ); + // The state a pre-release build left: the schema and everything in it + // owned by the cluster-wide role, which could then plant code. + await owner.query( + `CREATE ROLE "${LEGACY_APP_ROLE}" NOLOGIN; + COMMENT ON ROLE "${LEGACY_APP_ROLE}" IS '${MANAGED_MARK}: the app''s least-privilege login (scripts/db-app-role.mjs). Recreated and re-granted on every start.'; + GRANT SELECT ON users TO "${LEGACY_APP_ROLE}"; + ALTER SCHEMA pgboss OWNER TO "${LEGACY_APP_ROLE}"; + ALTER TABLE pgboss.job_common OWNER TO "${LEGACY_APP_ROLE}"; + ALTER FUNCTION pgboss.job_now() OWNER TO "${LEGACY_APP_ROLE}"; + CREATE OR REPLACE FUNCTION pgboss.job_now() RETURNS timestamptz + LANGUAGE sql AS 'SELECT now() - interval ''1 day'''; + CREATE FUNCTION pgboss.planted() RETURNS trigger LANGUAGE plpgsql + AS 'BEGIN RETURN NEW; END'; + ALTER FUNCTION pgboss.planted() OWNER TO "${LEGACY_APP_ROLE}"; + CREATE TRIGGER planted BEFORE INSERT ON pgboss.job_common + FOR EACH ROW EXECUTE FUNCTION pgboss.planted(); + CREATE RULE planted AS ON DELETE TO pgboss.queue DO INSTEAD NOTHING;`, + ); + + const next = await boot({ + DATABASE_URL: ownerUrl(), + API_TOKEN_HMAC_KEY: HMAC_KEY, + }); + expect(next.url, next.said).toBe(appUrl); + expect(next.said).toContain("took back"); + expect(next.said).toContain("trigger planted"); + expect(next.said).toContain("rule planted"); + expect(await queueObjectsNotOwnedBy(owner, ownerName)).toBe(0); + const { rows: after } = await owner.query<{ src: string }>( + `SELECT prosrc AS src FROM pg_proc WHERE oid = 'pgboss.job_now()'::regprocedure`, + ); + expect(after[0].src).toBe(before[0].src); + const { rows: leftovers } = await owner.query( + `SELECT to_regprocedure('pgboss.planted()') AS planted, + (SELECT count(*)::int FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'pgboss' AND NOT t.tgisinternal) AS triggers, + (SELECT count(*)::int FROM pg_roles WHERE rolname = $1) AS legacy`, + [LEGACY_APP_ROLE], + ); + expect(leftovers[0]).toEqual({ planted: null, triggers: 0, legacy: 0 }); + // Taken back, not dropped: the queue's tables are all still there. + const { rows: tables } = await owner.query( + `SELECT to_regclass('pgboss.job_common') IS NOT NULL AS common, + (SELECT count(*)::int FROM pgboss.queue) > 0 AS queues`, + ); + expect(tables[0]).toEqual({ common: true, queues: true }); + }); + + it("is idempotent: a later boot keeps the password and drops what the role could not", async (ctx) => { if (serverVersion < 170000) ctx.skip(); // A table a later migration adds: the next boot grants it. await owner.query(`CREATE TABLE public.app_role_probe_new (x int)`); + // The boot before this one dropped it; put it back once more. + await owner.query( + `CREATE INDEX IF NOT EXISTS "${RETIRED}" ON "measurements" ("user_id", "type", "measured_at")`, + ); try { const second = await boot({ DATABASE_URL: ownerUrl(), @@ -375,6 +636,7 @@ describe("the restricted role (Postgres 17+)", () => { }); expect(third.url).toBe(appUrl); expect(third.said).not.toContain("dropped index"); + expect(third.said).not.toContain("took back"); const { rows } = await owner.query( `SELECT to_regclass($1) IS NULL AS gone`, [RETIRED],