diff --git a/Dockerfile b/Dockerfile index 2630d356c..625feff6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -212,11 +212,13 @@ COPY --from=builder /app/src/generated ./src/generated # application graph (`@/…`, the generated Prisma client) run from a source # checkout instead. `src/__tests__/container-scripts-ship-guard.test.ts` holds # the runbooks and the entrypoint against this list. `pending-migrations.mjs` -# is the entrypoint's own: it logs the pending migration count before -# `migrate deploy`. Root-owned on purpose: the app user can run them but not +# and `migrate-deploy.mjs` are the entrypoint's own: the first logs the +# pending migration count, the second runs `migrate deploy` with its +# lock-timeout retry. Root-owned on purpose: the app user can run them but not # rewrite them. COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs +COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts # Prove the reset CLI's imports resolve in this tree: with every import found @@ -224,7 +226,8 @@ COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-int RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:" # The boot notice parses, and the `pg` it loads on a real boot resolves here. RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \ - node --input-type=module -e "await import('pg')" + node --input-type=module -e "await import('pg')" && \ + node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')" # Install the migration CLI, its config dependency, and a pinned launcher for # the maintenance scripts already shipped in the standalone tree. The Prisma diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 8eff71a6b..b33d52002 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -88,9 +88,12 @@ echo "HealthLog: Database is reachable." # Say how many migrations this image is about to apply, with a reminder to # back up first. Reporting only: it never stops the boot. node scripts/pending-migrations.mjs || true +# `migrate-deploy.mjs` runs `prisma migrate deploy` and retries only a +# migration that timed out on a lock and is marked safe to re-run; any other +# failure stops the boot with Prisma's exit code (docs/ops/deploy.md). echo "HealthLog: Running database migrations..." NODE_PATH=/opt/prisma-cli/node_modules \ - node /opt/prisma-cli/node_modules/prisma/build/index.js migrate deploy + node scripts/migrate-deploy.mjs echo "HealthLog: Migrations complete." # ── Start application ──────────────────────────────────────── diff --git a/docs/ops/deploy.md b/docs/ops/deploy.md index 1bf418cdc..89ac3d380 100644 --- a/docs/ops/deploy.md +++ b/docs/ops/deploy.md @@ -40,6 +40,55 @@ not reachable". The entrypoint also runs the migrations on every boot, so step 2 is optional; it is there for when you want to see the migration result before the new container takes traffic. +### A migration that timed out on a lock + +Some migrations bound how long they wait for a table lock, so that a +long-running holder (most often an autovacuum that is preventing +transaction-ID wraparound, which does not step aside) cannot queue every +write to that table behind the migration. Migration `0412` waits at most ten +seconds for `measurements`. When the wait runs out, the boot log shows +Prisma's report followed by a line from the entrypoint: + +``` +Error: P3018 +... +Migration name: 0412_measurement_hot_updates + +Database error code: 55P03 +... +HealthLog: migration 0412_measurement_hot_updates timed out waiting for a lock (55P03) and changed nothing; marking it rolled back and retrying in 15s (retry 1 of 5). +``` + +What happens on its own: such a migration is written so that a timed-out +attempt changes nothing, and it says so in its file +(`-- @retry-on-lock-timeout`). The entrypoint then marks that one migration +rolled back (`prisma migrate resolve --rolled-back`) and runs +`migrate deploy` again after 15, 30, 60, 120 and 240 seconds. The app starts +as soon as an attempt gets the lock. A later boot that finds the attempt still +on record (Prisma's `P3009`) reads the cause from `_prisma_migrations` and +does the same. Nothing else is retried: any other failure, and a lock timeout +in a migration that does not carry the marker, stops the boot exactly as +before. + +If all five retries time out, the log ends with +`HealthLog: ERROR — migration … timed out on a lock 6 times in a row; giving +up.` and the container exits. Something has held the table for about eight +minutes. Find it: + +```sh +docker compose exec db psql -U healthlog -c " + SELECT pid, backend_type, state, now() - xact_start AS running, query + FROM pg_stat_activity + WHERE pid IN (SELECT pid FROM pg_locks + WHERE relation = 'measurements'::regclass)" +``` + +An `autovacuum: VACUUM … (to prevent wraparound)` row finishes on its own; +wait for it to disappear and start the container again (a restart policy does +that for you, and each boot retries). Anything else is a session you can +judge: a stuck client may be ended with `SELECT pg_terminate_backend()`. +You never need to run `migrate resolve` by hand for this case. + ### v1.11.0 notes Migrations `0110`–`0114` are additive (new tables / columns for the diff --git a/prisma/migrations/0411_encrypt_insights_cached_text/migration.sql b/prisma/migrations/0411_encrypt_insights_cached_text/migration.sql index dfd6ff1e1..6718709f4 100644 --- a/prisma/migrations/0411_encrypt_insights_cached_text/migration.sql +++ b/prisma/migrations/0411_encrypt_insights_cached_text/migration.sql @@ -16,7 +16,7 @@ -- -- Additive only: no table rewrite, no data change. -ALTER TABLE "users" ADD COLUMN "insights_cached_text_encrypted" BYTEA; +ALTER TABLE "users" ADD COLUMN IF NOT EXISTS "insights_cached_text_encrypted" BYTEA; -- Partial index on exactly the backfill's discovery predicate, so the boot -- scan is index-only and shrinks to nothing as the backfill converges (the diff --git a/prisma/migrations/0412_measurement_hot_updates/migration.sql b/prisma/migrations/0412_measurement_hot_updates/migration.sql index 6846cee4f..d49f488c7 100644 --- a/prisma/migrations/0412_measurement_hot_updates/migration.sql +++ b/prisma/migrations/0412_measurement_hot_updates/migration.sql @@ -28,13 +28,22 @@ -- `ALTER TABLE ... SET` SHARE UPDATE EXCLUSIVE; both wait for an autovacuum -- that is preventing wraparound, which does not step aside, and while they -- wait every write to `measurements` queues behind them, including the ones --- the running release still serves during a deploy. `migrate deploy` runs the --- whole file as one transaction, so `SET LOCAL` bounds the wait for this file --- alone: past ten seconds the migration fails and rolls back whole, the --- writes go on, and the next deploy tries again once `prisma migrate resolve --- --rolled-back 0412_measurement_hot_updates` has cleared the failed attempt. - -SET LOCAL lock_timeout = '10s'; +-- the running release still serves during a deploy. The wait is bounded at +-- ten seconds; past that the attempt fails with SQLSTATE 55P03 and the queued +-- writes go on. +-- +-- `migrate deploy` does not run a migration file inside one transaction: each +-- statement commits on its own, so a top-level `SET LOCAL` does nothing and a +-- file that fails half-way stays half-applied. The two statements that take a +-- lock therefore run inside one `DO` block, which is a single statement and +-- so commits or rolls back whole, and the timeout is set inside it with +-- `set_config(..., true)`, scoped to that statement. The function before it +-- is `CREATE OR REPLACE` and harmless to run again. A lock timeout leaves the +-- file safe to apply once more, which is what the marker below declares: +-- `scripts/migrate-deploy.mjs` marks the attempt rolled back and retries it +-- with a backoff (docs/ops/deploy.md, "A migration that timed out on a lock"). +-- +-- @retry-on-lock-timeout CREATE OR REPLACE FUNCTION "measurements_keep_stamps_on_noop"() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -48,13 +57,19 @@ BEGIN END; $$; -CREATE TRIGGER "measurements_noop_update_keeps_stamps" - BEFORE UPDATE ON "measurements" - FOR EACH ROW - EXECUTE FUNCTION "measurements_keep_stamps_on_noop"(); +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '10s', true); -ALTER TABLE "measurements" SET ( - fillfactor = 85, - autovacuum_vacuum_scale_factor = 0.05, - autovacuum_analyze_scale_factor = 0.05 -); + CREATE TRIGGER "measurements_noop_update_keeps_stamps" + BEFORE UPDATE ON "measurements" + FOR EACH ROW + EXECUTE FUNCTION "measurements_keep_stamps_on_noop"(); + + ALTER TABLE "measurements" SET ( + fillfactor = 85, + autovacuum_vacuum_scale_factor = 0.05, + autovacuum_analyze_scale_factor = 0.05 + ); +END +$migration$; diff --git a/prisma/migrations/0413_strava_backfill_resume/migration.sql b/prisma/migrations/0413_strava_backfill_resume/migration.sql index 821446acc..8d11102ef 100644 --- a/prisma/migrations/0413_strava_backfill_resume/migration.sql +++ b/prisma/migrations/0413_strava_backfill_resume/migration.sql @@ -9,4 +9,4 @@ -- Nullable and unset for every existing row: an unfinished backfill starts -- once more from the newest page and records its place from then on. -ALTER TABLE "users" ADD COLUMN "strava_backfill_resume_before" TIMESTAMP(3); +ALTER TABLE "users" ADD COLUMN IF NOT EXISTS "strava_backfill_resume_before" TIMESTAMP(3); diff --git a/scripts/__tests__/migrate-deploy.test.ts b/scripts/__tests__/migrate-deploy.test.ts new file mode 100644 index 000000000..a63edc96b --- /dev/null +++ b/scripts/__tests__/migrate-deploy.test.ts @@ -0,0 +1,283 @@ +/** + * The boot's `migrate deploy` wrapper (`scripts/migrate-deploy.mjs`): which + * failures it retries, and the loop it runs around Prisma. + * + * The report texts below are what Prisma 7's `migrate deploy` printed against + * a real Postgres 16 for a migration that hit its `lock_timeout` (P3018), and + * for the boot after that (P3009). The loop is driven end to end through a + * stand-in CLI that plays those reports back, so no database is needed here; + * `tests/integration/migrate-deploy-lock-retry.test.ts` runs the real CLI. + */ +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + decideLockRetry, + lockTimeoutInReport, + retryDelays, +} from "../migrate-deploy.mjs"; + +const P3018_LOCK = `Applying migration \`0412_measurement_hot_updates\` +Error: P3018 + +A migration failed to apply. New migrations cannot be applied before the error is recovered from. Read more about how to resolve migration issues in a production database: https://pris.ly/d/migrate-resolve + +Migration name: 0412_measurement_hot_updates + +Database error code: 55P03 + +Database error: +ERROR: canceling statement due to lock timeout +`; + +const P3018_UNIQUE = P3018_LOCK.replace("55P03", "23505").replace( + "canceling statement due to lock timeout", + 'duplicate key value violates unique constraint "x"', +); + +const P3009 = `Error: P3009 + +migrate found failed migrations in the target database, new migrations will not be applied. Read more about how to resolve migration issues in a production database: https://pris.ly/d/migrate-resolve +The \`0412_measurement_hot_updates\` migration started at 2026-10-20 08:38:22.114155 UTC failed +`; + +const MARKED = "-- @retry-on-lock-timeout\nDO $$ BEGIN END $$;\n"; +const UNMARKED = "ALTER TABLE x ADD COLUMN y int;\n"; + +const reader = + (sql: string | null) => + (_name: string): string | null => + sql; + +describe("which failure is retried", () => { + it("reads the failed migration and its SQLSTATE from Prisma's report", () => { + expect(lockTimeoutInReport(P3018_LOCK)).toBe( + "0412_measurement_hot_updates", + ); + expect(lockTimeoutInReport(P3018_UNIQUE)).toBeNull(); + expect(lockTimeoutInReport(P3009)).toBeNull(); + expect(lockTimeoutInReport(null)).toBeNull(); + }); + + it("retries a lock timeout of a migration marked safe to re-run", () => { + expect( + decideLockRetry({ + output: P3018_LOCK, + failedRows: null, + readMigration: reader(MARKED), + }), + ).toEqual({ retry: true, migration: "0412_measurement_hot_updates" }); + }); + + it("does not retry a lock timeout of an unmarked migration", () => { + const decision = decideLockRetry({ + output: P3018_LOCK, + failedRows: null, + readMigration: reader(UNMARKED), + }); + expect(decision.retry).toBe(false); + expect(decision).toMatchObject({ + reason: expect.stringContaining("not marked safe to re-run"), + }); + }); + + it("does not retry any other database error", () => { + expect( + decideLockRetry({ + output: P3018_UNIQUE, + failedRows: null, + readMigration: reader(MARKED), + }), + ).toEqual({ retry: false, reason: "not a lock timeout" }); + }); + + it("does not retry a migration the image does not ship", () => { + expect( + decideLockRetry({ + output: P3018_LOCK, + failedRows: null, + readMigration: reader(null), + }).retry, + ).toBe(false); + }); + + it("on the boot after a failure, reads the cause from the stored row", () => { + const failedRows = [ + { migration_name: "0412_measurement_hot_updates", logs: P3018_LOCK }, + ]; + expect( + decideLockRetry({ + output: P3009, + failedRows, + readMigration: reader(MARKED), + }), + ).toEqual({ retry: true, migration: "0412_measurement_hot_updates" }); + // A stored failure of another kind stays fatal. + expect( + decideLockRetry({ + output: P3009, + failedRows: [{ ...failedRows[0], logs: P3018_UNIQUE }], + readMigration: reader(MARKED), + }).retry, + ).toBe(false); + // Two failed rows, or rows that could not be read: not a state to guess at. + expect( + decideLockRetry({ + output: P3009, + failedRows: [failedRows[0], failedRows[0]], + readMigration: reader(MARKED), + }).retry, + ).toBe(false); + expect( + decideLockRetry({ + output: P3009, + failedRows: null, + readMigration: reader(MARKED), + }).retry, + ).toBe(false); + }); + + it("refuses a migration name that is not a directory name", () => { + expect( + decideLockRetry({ + output: P3018_LOCK.replace( + "Migration name: 0412_measurement_hot_updates", + "Migration name: ../../etc", + ), + failedRows: null, + readMigration: reader(MARKED), + }).retry, + ).toBe(false); + }); + + it("reads the backoff from the environment and falls back on nonsense", () => { + expect(retryDelays(undefined)).toEqual([15, 30, 60, 120, 240]); + expect(retryDelays("0, 1")).toEqual([0, 1]); + expect(retryDelays("a,1")).toEqual([15, 30, 60, 120, 240]); + }); +}); + +describe("the retry loop", () => { + const dirs: string[] = []; + afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true }); + }); + + /** + * A stand-in Prisma CLI: each `migrate deploy` prints the next scripted + * report and exits with its code; every call is appended to `calls.log`. + */ + function setup(reports: Array<[number, string]>, sql: string) { + const dir = mkdtempSync(join(tmpdir(), "hl-migrate-deploy-")); + dirs.push(dir); + const migrations = join(dir, "migrations"); + mkdirSync(join(migrations, "0412_measurement_hot_updates"), { + recursive: true, + }); + writeFileSync( + join(migrations, "0412_measurement_hot_updates", "migration.sql"), + sql, + ); + writeFileSync(join(dir, "reports.json"), JSON.stringify(reports)); + const cli = join(dir, "fake-prisma.mjs"); + writeFileSync( + cli, + `import { appendFileSync, readFileSync, writeFileSync, existsSync } from "node:fs"; +const dir = ${JSON.stringify(dir)}; +const args = process.argv.slice(2); +appendFileSync(dir + "/calls.log", args.join(" ") + "\\n"); +if (args[0] === "migrate" && args[1] === "deploy") { + const n = existsSync(dir + "/n") ? Number(readFileSync(dir + "/n", "utf8")) : 0; + writeFileSync(dir + "/n", String(n + 1)); + const [code, text] = JSON.parse(readFileSync(dir + "/reports.json", "utf8"))[n]; + process.stderr.write(text); + process.exit(code); +} +process.exit(0); +`, + ); + const run = () => + spawnSync( + process.execPath, + [join(process.cwd(), "scripts/migrate-deploy.mjs"), "--schema", "x"], + { + encoding: "utf8", + env: { + ...process.env, + DATABASE_URL: "", + HEALTHLOG_PRISMA_CLI: cli, + HEALTHLOG_MIGRATIONS_DIR: migrations, + HEALTHLOG_MIGRATE_LOCK_RETRY_DELAYS: "0,0", + }, + }, + ); + const calls = () => + existsSync(join(dir, "calls.log")) + ? readFileSync(join(dir, "calls.log"), "utf8").trim().split("\n") + : []; + return { run, calls }; + } + + it("marks the timed-out migration rolled back and deploys again", () => { + const { run, calls } = setup( + [ + [1, P3018_LOCK], + [0, "All migrations have been successfully applied."], + ], + MARKED, + ); + const result = run(); + expect(result.status).toBe(0); + expect(calls()).toEqual([ + "migrate deploy --schema x", + "migrate resolve --rolled-back 0412_measurement_hot_updates --schema x", + "migrate deploy --schema x", + ]); + expect(result.stdout).toContain("timed out waiting for a lock (55P03)"); + }); + + it("gives up after the last delay with Prisma's exit code", () => { + const { run, calls } = setup( + [ + [1, P3018_LOCK], + [1, P3018_LOCK], + [1, P3018_LOCK], + ], + MARKED, + ); + const result = run(); + expect(result.status).toBe(1); + expect(calls().filter((c) => c.startsWith("migrate deploy"))).toHaveLength( + 3, + ); + expect(calls().filter((c) => c.startsWith("migrate resolve"))).toHaveLength( + 2, + ); + expect(result.stderr).toContain("giving up"); + }); + + it("stops at the first failure that is not a lock timeout", () => { + const { run, calls } = setup([[1, P3018_UNIQUE]], MARKED); + const result = run(); + expect(result.status).toBe(1); + expect(calls()).toEqual(["migrate deploy --schema x"]); + }); + + it("stops at a lock timeout of a migration that is not marked", () => { + const { run, calls } = setup([[1, P3018_LOCK]], UNMARKED); + const result = run(); + expect(result.status).toBe(1); + expect(calls()).toEqual(["migrate deploy --schema x"]); + expect(result.stderr).toContain("not marked safe to re-run"); + }); +}); diff --git a/scripts/migrate-deploy.mjs b/scripts/migrate-deploy.mjs new file mode 100644 index 000000000..23abb5f9a --- /dev/null +++ b/scripts/migrate-deploy.mjs @@ -0,0 +1,239 @@ +/** + * `prisma migrate deploy` for the container boot, with one narrow retry: a + * migration that gave up waiting for a lock. + * + * Run by `docker-entrypoint.sh` as `node scripts/migrate-deploy.mjs`. + * + * A migration that bounds its lock wait (`lock_timeout`) fails with SQLSTATE + * 55P03 when something holds the table longer, typically an autovacuum that + * is preventing wraparound and does not step aside. Prisma then records the + * attempt as failed, and every later `migrate deploy` stops at P3009 until an + * operator runs `prisma migrate resolve --rolled-back`: under a restart + * policy that is a container that never comes up. + * + * `migrate deploy` does not wrap a migration file in a transaction, so a + * failed file is not necessarily rolled back: statements before the one that + * failed stay applied. Marking such an attempt rolled back and running it + * again is only safe when the file was written for it. A migration declares + * that with a `-- @retry-on-lock-timeout` line, which its author adds only + * when every statement before the lock-taking one is harmless to repeat and + * the lock-taking statements form one statement (a `DO` block) that rolls + * back whole. `src/__tests__/migration-lock-timeout-guard.test.ts` holds the + * migrations to that. + * + * So: on a 55P03 failure of a migration carrying the marker, mark that one + * migration rolled back and run `migrate deploy` again after a backoff (15, + * 30, 60, 120, 240 seconds). Any other failure, a lock timeout of a migration + * without the marker, or a fifth lock timeout in a row, ends the boot with + * Prisma's exit code exactly as before. The failure is read from Prisma's + * own report (`Migration name:` and `Database error code:`), or, on a boot + * after a failed one (P3009), from the `logs` Prisma stored on the failed + * row in `_prisma_migrations`. + * + * Environment: + * HEALTHLOG_PRISMA_CLI Prisma's CLI entry (default: the image's copy) + * HEALTHLOG_MIGRATIONS_DIR where the migration files are (default: + * prisma/migrations under the working directory) + * HEALTHLOG_MIGRATE_LOCK_RETRY_DELAYS + * comma-separated backoff in seconds + * Arguments are passed through to every Prisma call (`--schema …`). + */ +import { spawn } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const DEFAULT_RETRY_DELAYS_S = [15, 30, 60, 120, 240]; + +/** The line a migration carries when a lock-timeout retry is safe for it. */ +export const RETRY_MARKER = /^--\s*@retry-on-lock-timeout\s*$/m; + +const LOCK_TIMEOUT = "55P03"; +const MIGRATION_NAME = /^[A-Za-z0-9_]+$/; + +/** + * The migration a P3018 report says failed on a lock timeout, or null. + * Prisma prints `Migration name: ` and `Database error code: ` + * for a migration that failed to apply. + */ +export function lockTimeoutInReport(text) { + if (typeof text !== "string") return null; + const name = /^Migration name: (\S+)\s*$/m.exec(text)?.[1]; + const code = /^Database error code: (\S+)\s*$/m.exec(text)?.[1]; + return name && code === LOCK_TIMEOUT ? name : null; +} + +/** Whether Prisma refused to start because a failed migration is on record. */ +function isFailedMigrationOnRecord(output) { + return /\bP3009\b/.test(output); +} + +/** + * Decide whether a failed `migrate deploy` is the one failure retried here. + * + * @param {object} input + * @param {string} input.output everything Prisma printed + * @param {Array<{migration_name: string, logs: string | null}> | null} input.failedRows + * the unfinished, not-rolled-back rows of `_prisma_migrations`, or null + * when they were not read + * @param {(name: string) => string | null} input.readMigration + * the migration's SQL, or null when the image does not ship it + * @returns {{ retry: true, migration: string } | { retry: false, reason: string }} + */ +export function decideLockRetry({ output, failedRows, readMigration }) { + let migration = lockTimeoutInReport(output); + if (!migration && isFailedMigrationOnRecord(output) && failedRows) { + // More than one failed row is not a state this boot can reason about. + if (failedRows.length === 1) { + migration = lockTimeoutInReport(failedRows[0].logs); + } + } + if (!migration) { + return { retry: false, reason: "not a lock timeout" }; + } + if (!MIGRATION_NAME.test(migration)) { + return { retry: false, reason: `unexpected migration name ${migration}` }; + } + const sql = readMigration(migration); + if (sql === null) { + return { retry: false, reason: `${migration} is not in this image` }; + } + if (!RETRY_MARKER.test(sql)) { + return { + retry: false, + reason: `${migration} timed out on a lock but is not marked safe to re-run`, + }; + } + return { retry: true, migration }; +} + +/** The backoff schedule, from the environment or the default. */ +export function retryDelays(raw) { + if (!raw) return DEFAULT_RETRY_DELAYS_S; + const delays = raw.split(",").map((part) => Number(part.trim())); + return delays.every((d) => Number.isFinite(d) && d >= 0) + ? delays + : DEFAULT_RETRY_DELAYS_S; +} + +function runPrisma(cli, args) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [cli, ...args], { + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (chunk) => { + output += chunk; + process.stdout.write(chunk); + }); + child.stderr.on("data", (chunk) => { + output += chunk; + process.stderr.write(chunk); + }); + child.on("error", (error) => { + output += String(error); + resolve({ code: 1, output }); + }); + child.on("close", (code) => resolve({ code: code ?? 1, output })); + }); +} + +/** The failed rows of `_prisma_migrations`, or null when they cannot be read. */ +async function readFailedRows(connectionString) { + if (!connectionString) return null; + try { + const { default: pg } = await import("pg"); + const url = new URL(connectionString); + const schema = url.searchParams.get("schema") || "public"; + url.searchParams.delete("schema"); + const client = new pg.Client({ + connectionString: url.toString(), + connectionTimeoutMillis: 5_000, + statement_timeout: 5_000, + }); + await client.connect(); + try { + const { rows } = await client.query( + `SELECT migration_name, logs FROM ${client.escapeIdentifier(schema)}._prisma_migrations + WHERE finished_at IS NULL AND rolled_back_at IS NULL`, + ); + return rows; + } finally { + await client.end().catch(() => {}); + } + } catch { + return null; + } +} + +function migrationReader(dir) { + return (name) => { + try { + return readFileSync(join(dir, name, "migration.sql"), "utf8"); + } catch { + return null; + } + }; +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function main({ + cli = process.env.HEALTHLOG_PRISMA_CLI || + "/opt/prisma-cli/node_modules/prisma/build/index.js", + migrationsDir = process.env.HEALTHLOG_MIGRATIONS_DIR || + join(process.cwd(), "prisma", "migrations"), + delays = retryDelays(process.env.HEALTHLOG_MIGRATE_LOCK_RETRY_DELAYS), + extraArgs = process.argv.slice(2), +} = {}) { + const readMigration = migrationReader(migrationsDir); + for (let attempt = 0; ; attempt++) { + const deploy = await runPrisma(cli, ["migrate", "deploy", ...extraArgs]); + if (deploy.code === 0) return 0; + + const failedRows = isFailedMigrationOnRecord(deploy.output) + ? await readFailedRows(process.env.DATABASE_URL) + : null; + const decision = decideLockRetry({ + output: deploy.output, + failedRows, + readMigration, + }); + if (!decision.retry) { + if (decision.reason !== "not a lock timeout") { + console.error( + `HealthLog: not retrying the migration: ${decision.reason}.`, + ); + } + return deploy.code; + } + if (attempt >= delays.length) { + console.error( + `HealthLog: ERROR — migration ${decision.migration} timed out on a lock ` + + `${attempt + 1} times in a row; giving up. Something holds the table ` + + "for minutes: see docs/ops/deploy.md, " + + '"A migration that timed out on a lock".', + ); + return deploy.code; + } + const delay = delays[attempt]; + console.log( + `HealthLog: migration ${decision.migration} timed out waiting for a lock ` + + `(55P03) and changed nothing; marking it rolled back and retrying in ` + + `${delay}s (retry ${attempt + 1} of ${delays.length}).`, + ); + const resolve = await runPrisma(cli, [ + "migrate", + "resolve", + "--rolled-back", + decision.migration, + ...extraArgs, + ]); + if (resolve.code !== 0) return resolve.code; + await sleep(delay * 1000); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + process.exitCode = await main(); +} diff --git a/src/__tests__/migration-lock-timeout-guard.test.ts b/src/__tests__/migration-lock-timeout-guard.test.ts new file mode 100644 index 000000000..6afe05a6e --- /dev/null +++ b/src/__tests__/migration-lock-timeout-guard.test.ts @@ -0,0 +1,128 @@ +/** + * Structural guard: how a migration bounds its lock wait, and when the boot + * may retry it. + * + * `prisma migrate deploy` runs a migration file statement by statement, each + * committing on its own; it does not wrap the file in a transaction. Two + * consequences were written into this repository the other way round: + * + * - A top-level `SET LOCAL lock_timeout` does nothing (there is no + * transaction for it to be local to), so the migration waits as long as + * the lock is held, and every write to the table queues behind it. + * - A file that fails half-way stays half-applied, so marking it rolled back + * and running it again can fail on what already exists, or apply a change + * twice. + * + * So a lock wait is bounded inside a `DO` block with + * `set_config('lock_timeout', …, true)`, which is one statement and rolls back + * whole; and the `-- @retry-on-lock-timeout` marker, which lets + * `scripts/migrate-deploy.mjs` retry a 55P03 failure on boot, is allowed only + * on a file whose every other statement is harmless to run again. + */ +import { readdirSync, readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { RETRY_MARKER } from "../../scripts/migrate-deploy.mjs"; + +const MIGRATIONS_DIR = join(process.cwd(), "prisma", "migrations"); + +function migrations(): Array<{ name: string; sql: string }> { + return readdirSync(MIGRATIONS_DIR, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join(MIGRATIONS_DIR, entry.name, "migration.sql")) + .filter((file) => existsSync(file)) + .map((file) => ({ + name: file.split("/").at(-2) ?? file, + sql: readFileSync(file, "utf8"), + })); +} + +/** The statements of a file, comments removed, dollar-quoted bodies blanked. */ +function topLevelStatements(sql: string): string[] { + return sql + .replace(/--[^\n]*/g, "") + .replace(/\$([A-Za-z_]*)\$[\s\S]*?\$\1\$/g, "$BODY$") + .split(";") + .map((statement) => statement.trim().replace(/\s+/g, " ")) + .filter((statement) => statement.length > 0); +} + +/** Statements that leave the database as they found it when run twice. */ +const REPEATABLE = [ + /^DO \$BODY\$$/i, + /^CREATE OR REPLACE (FUNCTION|VIEW|PROCEDURE) /i, + /^CREATE (UNIQUE )?INDEX IF NOT EXISTS /i, + /^CREATE TABLE IF NOT EXISTS /i, + /^ALTER TABLE "?\w+"? ADD COLUMN IF NOT EXISTS /i, + /^DROP (INDEX|TRIGGER|FUNCTION|TABLE) IF EXISTS /i, +]; + +/** A session setting changed outside a transaction-scoped `DO` body. */ +const TOP_LEVEL_TIMEOUT = + /^SET (LOCAL |SESSION )?(lock_timeout|statement_timeout)\b|^SET LOCAL\b/i; + +function violations(name: string, sql: string): string[] { + const found: string[] = []; + const statements = topLevelStatements(sql); + for (const statement of statements) { + if (TOP_LEVEL_TIMEOUT.test(statement)) { + found.push(`${name}: top-level "${statement}" has no effect`); + } + } + const boundsLockWait = /set_config\(\s*'lock_timeout'/i.test(sql); + const marked = RETRY_MARKER.test(sql); + if (boundsLockWait && !marked) { + found.push( + `${name}: bounds its lock wait but lacks -- @retry-on-lock-timeout`, + ); + } + if (marked) { + for (const statement of statements) { + if (!REPEATABLE.some((pattern) => pattern.test(statement))) { + found.push( + `${name}: marked retry-safe, but "${statement.slice(0, 60)}" is not repeatable`, + ); + } + } + } + return found; +} + +describe("migration lock waits", () => { + it("bounds lock waits where they take effect and marks only repeatable files", () => { + const found = migrations().flatMap(({ name, sql }) => + violations(name, sql), + ); + expect(found, found.join("\n")).toEqual([]); + }); + + it("sees the one migration that bounds its wait today", () => { + const marked = migrations() + .filter(({ sql }) => RETRY_MARKER.test(sql)) + .map(({ name }) => name); + expect(marked).toContain("0412_measurement_hot_updates"); + }); + + it("catches each shape it exists to refuse", () => { + expect(violations("a", "SET LOCAL lock_timeout = '10s';")).toHaveLength(1); + expect( + violations( + "b", + "DO $x$ BEGIN PERFORM set_config('lock_timeout', '1s', true); END $x$;", + ), + ).toHaveLength(1); + expect( + violations( + "c", + "-- @retry-on-lock-timeout\nCREATE TRIGGER t BEFORE UPDATE ON m FOR EACH ROW EXECUTE FUNCTION f();\nDO $x$ BEGIN PERFORM set_config('lock_timeout', '1s', true); END $x$;", + ), + ).toEqual([expect.stringContaining("is not repeatable")]); + expect( + violations( + "d", + "-- @retry-on-lock-timeout\nCREATE OR REPLACE FUNCTION f() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END; $$;\nDO $x$ BEGIN PERFORM set_config('lock_timeout', '1s', true); END $x$;", + ), + ).toEqual([]); + }); +}); diff --git a/tests/integration/measurement-hot-update.test.ts b/tests/integration/measurement-hot-update.test.ts index 731809db6..7b5144c1c 100644 --- a/tests/integration/measurement-hot-update.test.ts +++ b/tests/integration/measurement-hot-update.test.ts @@ -8,11 +8,10 @@ * here against Postgres: the no-op update keeps `updated_at` and * `sync_version` and is counted as HOT in the transaction's own statistics, * a real change still moves both, the nightly check reads the table's - * statistics and leaves the vacuuming to autovacuum, and the migration gives - * up on a lock it cannot get instead of queueing every writer behind it. + * statistics and leaves the vacuuming to autovacuum. The migration's lock + * wait is proven through the real `migrate deploy` in + * `migrate-deploy-lock-retry.test.ts`. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; import { Client } from "pg"; import { beforeEach, describe, expect, it } from "vitest"; @@ -121,46 +120,3 @@ describe("the nightly check", () => { } }); }); - -describe("the migration's lock wait", () => { - it("fails within its lock timeout while another session holds the table", async () => { - // The migration's own text under fresh names, run the way `migrate - // deploy` runs a file: one multi-statement query, one transaction. - const sql = readFileSync( - join( - process.cwd(), - "prisma/migrations/0412_measurement_hot_updates/migration.sql", - ), - "utf8", - ) - .replaceAll("measurements_keep_stamps_on_noop", "lock_probe_fn") - .replaceAll("measurements_noop_update_keeps_stamps", "lock_probe_trg"); - const holder = new Client({ connectionString: process.env.DATABASE_URL }); - const migrator = new Client({ connectionString: process.env.DATABASE_URL }); - await holder.connect(); - await migrator.connect(); - try { - // What an anti-wraparound autovacuum holds, and does not give up. - await holder.query("BEGIN"); - await holder.query( - "LOCK TABLE measurements IN SHARE UPDATE EXCLUSIVE MODE", - ); - const startedAt = Date.now(); - const error = await migrator.query(sql).then( - () => null, - (err: { code?: string }) => err, - ); - expect(error?.code).toBe("55P03"); - expect(Date.now() - startedAt).toBeLessThan(30_000); - // Rolled back whole: nothing of the probe is left behind. - const { rows } = await migrator.query( - "SELECT to_regproc('lock_probe_fn') AS fn", - ); - expect(rows[0].fn).toBeNull(); - } finally { - await holder.query("ROLLBACK").catch(() => {}); - await holder.end(); - await migrator.end(); - } - }, 60_000); -}); diff --git a/tests/integration/migrate-deploy-lock-retry.test.ts b/tests/integration/migrate-deploy-lock-retry.test.ts new file mode 100644 index 000000000..b9678feaf --- /dev/null +++ b/tests/integration/migrate-deploy-lock-retry.test.ts @@ -0,0 +1,259 @@ +/** + * The boot's `migrate deploy` (`scripts/migrate-deploy.mjs`) against the real + * Prisma CLI and Postgres: migration 0412's lock wait, and the retry after it. + * + * Runs 0412's own text (its ten-second timeout cut to one) in a schema of its + * own, on a minimal `measurements` table, while another session holds the + * lock an anti-wraparound autovacuum would hold. Prisma runs a migration file + * statement by statement, so this is the only place the timeout and the + * all-or-nothing `DO` block are proven the way they will run on a host; a + * multi-statement `pg` query is one implicit transaction and would prove both + * even for a file that cannot deliver them. + */ +import { spawn } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Client } from "pg"; +import { afterAll, beforeEach, describe, expect, it } from "vitest"; + +const SCHEMA = "lock_retry_probe"; +const NAME = "0412_measurement_hot_updates"; +const ROOT = process.cwd(); + +const dirs: string[] = []; +afterAll(async () => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true }); + const db = await connect(); + await db.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await db.end(); +}); + +function baseUrl(): string { + const url = process.env.DATABASE_URL; + if (!url) throw new Error("DATABASE_URL is not set for the integration run"); + return url; +} + +function probeUrl(): string { + const url = new URL(baseUrl()); + url.searchParams.set("schema", SCHEMA); + return url.toString(); +} + +async function connect(): Promise { + const url = new URL(baseUrl()); + url.searchParams.delete("schema"); + const client = new Client({ connectionString: url.toString() }); + await client.connect(); + return client; +} + +/** + * A migrations directory whose first migration creates the probe table + * (Prisma will not migrate a schema it did not build) and is applied here. + */ +async function migrationsDir(): Promise { + const dir = mkdtempSync(join(tmpdir(), "hl-lock-retry-")); + dirs.push(dir); + writeFileSync( + join(dir, "schema.prisma"), + 'datasource db {\n provider = "postgresql"\n}\n', + ); + mkdirSync(join(dir, "migrations", "0001_probe_table"), { recursive: true }); + cpSync( + join(ROOT, "prisma", "migrations", "migration_lock.toml"), + join(dir, "migrations", "migration_lock.toml"), + ); + writeFileSync( + join(dir, "migrations", "0001_probe_table", "migration.sql"), + `CREATE TABLE "measurements" ( + "id" integer PRIMARY KEY, "value" integer, + "updated_at" timestamp(3), "sync_version" integer);`, + ); + const setup = await migrateDeploy(dir); + expect(setup.code, setup.stderr).toBe(0); + return dir; +} + +/** Add 0412's own text, its timeout cut to one second, optionally unmarked. */ +function add0412(dir: string, { marked }: { marked: boolean }): void { + const original = readFileSync( + join(ROOT, "prisma", "migrations", NAME, "migration.sql"), + "utf8", + ); + let sql = original.replace("'lock_timeout', '10s'", "'lock_timeout', '1s'"); + expect( + sql, + "0412 still sets its timeout the way this test shortens it", + ).not.toBe(original); + if (!marked) sql = sql.replace(/^-- @retry-on-lock-timeout\n/m, ""); + mkdirSync(join(dir, "migrations", NAME), { recursive: true }); + writeFileSync(join(dir, "migrations", NAME, "migration.sql"), sql); +} + +function migrateDeploy( + dir: string, +): Promise<{ code: number; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn( + process.execPath, + [ + join(ROOT, "scripts", "migrate-deploy.mjs"), + "--schema", + join(dir, "schema.prisma"), + ], + { + cwd: ROOT, + env: { + ...process.env, + DATABASE_URL: probeUrl(), + HEALTHLOG_PRISMA_CLI: join( + ROOT, + "node_modules", + "prisma", + "build", + "index.js", + ), + HEALTHLOG_MIGRATIONS_DIR: join(dir, "migrations"), + HEALTHLOG_MIGRATE_LOCK_RETRY_DELAYS: "2,2,2", + }, + }, + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("close", (code) => resolve({ code: code ?? 1, stdout, stderr })); + }); +} + +/** Hold what an anti-wraparound autovacuum holds on the probe table. */ +async function holdLock(): Promise { + const holder = await connect(); + await holder.query("BEGIN"); + await holder.query( + `LOCK TABLE ${SCHEMA}.measurements IN SHARE UPDATE EXCLUSIVE MODE`, + ); + return holder; +} + +async function state(db: Client) { + const rows = await db.query( + `SELECT migration_name, finished_at IS NOT NULL AS finished, + rolled_back_at IS NOT NULL AS rolled_back, logs + FROM ${SCHEMA}._prisma_migrations ORDER BY started_at`, + ); + const trigger = await db.query( + `SELECT 1 FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND t.tgname = 'measurements_noop_update_keeps_stamps'`, + [SCHEMA], + ); + const options = await db.query( + `SELECT c.reloptions FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = 'measurements'`, + [SCHEMA], + ); + return { + migrations: rows.rows, + trigger: trigger.rowCount === 1, + reloptions: (options.rows[0]?.reloptions ?? []) as string[], + }; +} + +beforeEach(async () => { + const db = await connect(); + await db.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await db.end(); +}); + +describe("migration 0412 under a held lock", () => { + it("gives up within its timeout, changes nothing, and the boot retries it", async () => { + const dir = await migrationsDir(); + add0412(dir, { marked: true }); + const holder = await holdLock(); + // The first attempt meets the lock; the holder lets go during the backoff. + const release = setTimeout(() => { + void holder.query("ROLLBACK"); + }, 2_500); + const startedAt = Date.now(); + const result = await migrateDeploy(dir); + clearTimeout(release); + await holder.query("ROLLBACK").catch(() => {}); + await holder.end(); + + expect(result.code, result.stderr).toBe(0); + expect(result.stderr).toContain("Database error code: 55P03"); + expect(result.stdout).toContain( + `migration ${NAME} timed out waiting for a lock (55P03)`, + ); + expect(Date.now() - startedAt).toBeLessThan(45_000); + + const db = await connect(); + const after = await state(db); + await db.end(); + expect( + after.migrations.map((m) => [ + m.migration_name, + m.finished, + m.rolled_back, + ]), + ).toEqual([ + ["0001_probe_table", true, false], + [NAME, false, true], + [NAME, true, false], + ]); + expect(after.trigger).toBe(true); + expect(after.reloptions).toContain("fillfactor=85"); + }, 90_000); + + it("leaves nothing half-applied, stays fatal unmarked, and a later boot recovers", async () => { + const dir = await migrationsDir(); + add0412(dir, { marked: false }); + const holder = await holdLock(); + let result; + try { + result = await migrateDeploy(dir); + } finally { + await holder.query("ROLLBACK").catch(() => {}); + await holder.end(); + } + expect(result.code).not.toBe(0); + expect(result.stderr).toContain("not marked safe to re-run"); + + const db = await connect(); + const failed = await state(db); + // The lock-taking statements rolled back as one: no trigger, no options. + expect(failed.trigger).toBe(false); + expect(failed.reloptions).toEqual([]); + expect(failed.migrations).toHaveLength(2); + expect(failed.migrations[1]).toMatchObject({ + migration_name: NAME, + finished: false, + rolled_back: false, + }); + expect(failed.migrations[1].logs).toContain("Database error code: 55P03"); + + // The next image ships the marked file; its boot meets P3009 and reads + // the cause from the stored row. + add0412(dir, { marked: true }); + const recovered = await migrateDeploy(dir); + expect(recovered.code, recovered.stderr).toBe(0); + expect(recovered.stderr).toContain("P3009"); + const after = await state(db); + await db.end(); + expect(after.trigger).toBe(true); + expect(after.migrations.at(-1)).toMatchObject({ + finished: true, + rolled_back: false, + }); + }, 90_000); +});