diff --git a/prisma/migrations/0486_database_backup_runs/migration.sql b/prisma/migrations/0486_database_backup_runs/migration.sql new file mode 100644 index 000000000..37bd430af --- /dev/null +++ b/prisma/migrations/0486_database_backup_runs/migration.sql @@ -0,0 +1,30 @@ +-- The nightly whole-database dump records each run: whether a dump was +-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not. +-- Instance metadata with no account; the admin backups card and the backup +-- alerts read the newest rows. +-- +-- The table is new, so nothing waits on its lock in practice; the wait is +-- bounded in a DO block anyway, and every statement is safe to run again, so +-- the boot retries the file on a lock timeout. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + CREATE TABLE IF NOT EXISTS "database_backup_runs" ( + "id" TEXT NOT NULL, + "started_at" TIMESTAMP(3) NOT NULL, + "finished_at" TIMESTAMP(3), + "status" TEXT NOT NULL, + "file_name" TEXT, + "dump_bytes" BIGINT, + "local_result" TEXT NOT NULL, + "offhost_result" TEXT NOT NULL, + "error" TEXT, + CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id") + ); + CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx" + ON "database_backup_runs"("started_at"); +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8397e0cb5..ae2d0c292 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5929,6 +5929,35 @@ model OffhostPurgeRequest { @@map("offhost_purge_requests") } +/// One run of the nightly whole-database dump (the `database-backup` job): +/// whether a dump was written on this server and copied off-host, and why not +/// when it was not. Instance metadata, not an account's data. Written by +/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the +/// database row of the admin backups card and the backup alerts. A row left +/// `running` by a process that died is read as a failed run once the job's +/// expiry has passed. +model DatabaseBackupRun { + id String @id @default(cuid()) + startedAt DateTime @map("started_at") + finishedAt DateTime? @map("finished_at") + /// `running`, `succeeded`, `failed` or `skipped`. + status String + /// The dump's file name, once one was written and checked. + fileName String? @map("file_name") + /// Size of the dump in bytes. + dumpBytes BigInt? @map("dump_bytes") + /// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable + /// directory, the dump went off-host only), or `failed`. + localResult String @map("local_result") + /// `uploaded`, `not_configured` or `failed`. + offhostResult String @map("offhost_result") + /// Why the run failed or was skipped, trimmed. Never key material. + error String? + + @@index([startedAt]) + @@map("database_backup_runs") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/src/lib/boot/__tests__/readiness-summary.test.ts b/src/lib/boot/__tests__/readiness-summary.test.ts index fc23b92ee..003ade157 100644 --- a/src/lib/boot/__tests__/readiness-summary.test.ts +++ b/src/lib/boot/__tests__/readiness-summary.test.ts @@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => { expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined(); }); }); + +describe("collectReadiness — the database dump", () => { + const offhost = { + BACKUP_S3_ENDPOINT: "https://s3.example.test", + BACKUP_S3_BUCKET: "b", + BACKUP_S3_ACCESS_KEY: "a", + BACKUP_S3_SECRET_KEY: "s", + BACKUP_ENCRYPTION_KEY: HEX32, + }; + const dumpLine = ( + env: Record, + writable: boolean, + ) => + collectReadiness(env, () => writable).lines.find( + (l) => l.label === "Database dump", + ); + + it("names the schedule, the directory and the copies kept", () => { + const line = dumpLine(baseEnv(), true); + expect(line?.status).toBe("ok"); + expect(line?.detail).toBe( + "nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7", + ); + expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain( + "copied off-host", + ); + }); + + it("warns when nothing can be kept on the server", () => { + expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("goes off-host only"), + }); + expect(dumpLine(baseEnv(), false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("no database dump can be taken"), + }); + }); + + it("warns when it is switched off, and names a schedule it ignored", () => { + expect( + dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true), + ).toMatchObject({ status: "warn" }); + const report = collectReadiness( + { ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" }, + () => true, + ); + expect( + report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail, + ).toBe("not a five-field cron; using 30 1 * * *"); + expect(report.hasBlocker).toBe(false); + }); +}); diff --git a/src/lib/boot/readiness-summary.ts b/src/lib/boot/readiness-summary.ts index 14e707157..e8907cb29 100644 --- a/src/lib/boot/readiness-summary.ts +++ b/src/lib/boot/readiness-summary.ts @@ -19,6 +19,13 @@ import { LEGACY_ANY_HOST_WARNING, legacyAnyHostConfigured, } from "@/lib/ai/local-host-allowlist"; +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, + resolveBackupSchedule, + resolveDailyKeep, +} from "@/lib/database-backup/dump-core.mjs"; export interface ReadinessLine { /** Short label, e.g. "ENCRYPTION_KEY". */ @@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean { return keys.some((k) => (env[k] ?? "").trim().length > 0); } -/** Inspect the environment and build the readiness report. Pure + side-effect free. */ -export function collectReadiness(env: Env = process.env): ReadinessReport { +/** + * The nightly database dump's line. The dump is what rebuilds a lost server, + * so a host where it cannot be kept anywhere is said out loud at every boot, + * and so is one where it only goes off-host. + */ +export function databaseBackupReadiness( + env: Env, + writable: (dir: string) => boolean, +): ReadinessLine[] { + const label = "Database dump"; + if (!databaseBackupEnabled(env)) { + return [ + { + label, + status: "warn", + detail: + "DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way", + }, + ]; + } + const lines: ReadinessLine[] = []; + const schedule = resolveBackupSchedule(env); + if (!schedule.valid) { + lines.push({ + label: "DB_BACKUP_SCHEDULE", + status: "warn", + detail: `not a five-field cron; using ${schedule.cron}`, + }); + } + const dir = resolveBackupDir(env); + const offhost = allSet(env, [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]); + if (writable(dir)) { + lines.push({ + label, + status: "ok", + detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`, + }); + } else if (offhost) { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } else { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } + return lines; +} + +/** + * Inspect the environment and build the readiness report. Pure apart from + * `writable`, the probe of the dump directory, which tests replace. + */ +export function collectReadiness( + env: Env = process.env, + writable: (dir: string) => boolean = isWritableDirectory, +): ReadinessReport { const lines: ReadinessLine[] = []; // --- Core secrets (boot blockers) --------------------------------------- @@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport { }); } + lines.push(...databaseBackupReadiness(env, writable)); + // --- Optional subsystems (informational) -------------------------------- const optional: Array<[string, boolean]> = [ ["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])], diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 92cead915..211e00af6 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly> = { HostMetric: "instance-wide host telemetry, not attributable to an account", OffhostBackupKeyUse: "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + DatabaseBackupRun: + "the instance's ledger of nightly whole-database dumps; describes the server, not any account", OffhostPurgeRequest: "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: diff --git a/src/lib/database-backup/__tests__/dump-core.test.ts b/src/lib/database-backup/__tests__/dump-core.test.ts new file mode 100644 index 000000000..fad98e4a3 --- /dev/null +++ b/src/lib/database-backup/__tests__/dump-core.test.ts @@ -0,0 +1,630 @@ +/** + * The rules of the whole-database dump that need no database: names, + * retention, the manifest, key fingerprints, row-count comparison, the + * connection the client tools get, and the off-host envelope read back by the + * restore's own reader. + * + * The fingerprints and the envelope are checked against the app's own code + * (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy + * of the rule: the restore script cannot import TypeScript, so these are the + * tests that keep its plain-JS reading of both from drifting. + */ +import { + mkdtempSync, + createReadStream, + rmSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + applyRetention, + buildManifest, + compareKeyFingerprints, + compareRowCounts, + connectionFromUrl, + decodeKeyMaterial, + decryptOffhostFile, + dumpBaseName, + formatDumpTimestamp, + keyFingerprints, + listDumpSets, + offhostObjectKey, + parseDumpFileName, + parseManifest, + resolveBackupSchedule, + resolveDailyKeep, + selectForDeletion, + type DumpSet, +} from "../dump-core.mjs"; +import type { S3Like } from "@/lib/jobs/offhost-backup"; + +/** Every byte of a stream, in order. */ +async function collect(stream: AsyncIterable): Promise { + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array)); + return Buffer.concat(chunks); +} + +const HEX_A = "a".repeat(64); +const HEX_B = "b".repeat(64); +const HEX_C = "c".repeat(64); +const HEX_D = "d".repeat(64); + +const dirs: string[] = []; +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-")); + dirs.push(dir); + return dir; +} +afterEach(() => { + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); + vi.unstubAllEnvs(); +}); + +describe("dump names", () => { + it("names a dump by its UTC instant and kind", () => { + const at = new Date("2026-10-25T01:30:07.123Z"); + expect(formatDumpTimestamp(at)).toBe("20261025T013007Z"); + expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily"); + expect(dumpBaseName("pre-migration", at)).toBe( + "healthlog-20261025T013007Z-pre-migration", + ); + }); + + it("parses every file it writes, and back to the same instant", () => { + const base = "healthlog-20261025T013007Z-daily"; + const cases: Array<[string, string, string]> = [ + [`${base}.dump`, "dump", "final"], + [`${base}.manifest.json`, "manifest", "final"], + [`${base}.dump.partial`, "dump", "partial"], + [`${base}.manifest.json.partial`, "manifest", "partial"], + [`${base}.dump.enc`, "dump", "encrypted"], + [`${base}.manifest.json.enc`, "manifest", "encrypted"], + ]; + for (const [name, role, variant] of cases) { + const parsed = parseDumpFileName(name); + expect(parsed, name).not.toBeNull(); + expect(parsed?.base).toBe(base); + expect(parsed?.kind).toBe("daily"); + expect(parsed?.role).toBe(role); + expect(parsed?.variant).toBe(variant); + expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z"); + } + expect( + parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind, + ).toBe("pre-migration"); + }); + + it("refuses every name it did not write", () => { + for (const name of [ + "healthlog.dump", + "backup-20261025T013007Z-daily.dump", + "healthlog-20261025T013007Z-weekly.dump", + "healthlog-20261025-daily.dump", + "healthlog-20261025T013007Z-daily.sql", + "healthlog-20261025T013007Z-daily.dump.bak", + "healthlog-20261399T013007Z-daily.dump", + "notes.txt", + ]) { + expect(parseDumpFileName(name), name).toBeNull(); + } + }); +}); + +describe("configuration", () => { + it("keeps at least one daily dump and ignores what is not a number", () => { + expect(resolveDailyKeep({})).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7); + }); + + it("takes a five-field cron and says when it ignored one", () => { + expect(resolveBackupSchedule({})).toEqual({ + cron: "30 1 * * *", + valid: true, + configured: false, + }); + expect( + resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }), + ).toEqual({ + cron: "0 2 * * 1-5", + valid: true, + configured: true, + }); + for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) { + expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({ + cron: "30 1 * * *", + valid: false, + configured: true, + }); + } + }); +}); + +/** A dump set as `listDumpSets` reports it, for the pure retention rule. */ +function set( + kind: "daily" | "pre-migration", + iso: string, + parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {}, + now = Date.parse("2026-10-25T12:00:00Z"), +): DumpSet { + const base = dumpBaseName(kind, new Date(iso)); + return { + base, + kind, + timestamp: new Date(iso), + dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 }, + manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`, + encryptedDump: null, + encryptedManifest: null, + leftovers: + parts.partialAgeMs === undefined + ? [] + : [ + { + path: `/b/${base}.dump.partial`, + mtimeMs: now - parts.partialAgeMs, + }, + ], + }; +} + +describe("retention", () => { + const now = Date.parse("2026-10-25T12:00:00Z"); + + it("keeps the newest N daily dumps and removes the rest, both files each", () => { + const sets = [1, 2, 3, 4, 5].map((d) => + set("daily", `2026-10-2${d}T01:30:00Z`), + ); + const doomed = selectForDeletion(sets, { + keep: { daily: 3, "pre-migration": 2 }, + now, + }); + expect(doomed.sort()).toEqual( + [ + "/b/healthlog-20261021T013000Z-daily.dump", + "/b/healthlog-20261021T013000Z-daily.manifest.json", + "/b/healthlog-20261022T013000Z-daily.dump", + "/b/healthlog-20261022T013000Z-daily.manifest.json", + ].sort(), + ); + }); + + it("never removes the newest good dump, whatever keep says", () => { + const sets = [ + set("daily", "2026-10-24T01:30:00Z"), + set("daily", "2026-10-23T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 0, "pre-migration": 0 }, + now, + }); + expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump"); + expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump"); + }); + + it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => { + const sets = [ + set("daily", "2026-10-25T01:30:00Z", { + dump: false, + partialAgeMs: 60_000, + }), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([]); + }); + + it("keeps two pre-migration dumps apart from the daily count", () => { + const sets = [ + set("pre-migration", "2026-10-20T08:00:00Z"), + set("pre-migration", "2026-10-10T08:00:00Z"), + set("pre-migration", "2026-10-01T08:00:00Z"), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261001T080000Z-pre-migration.dump", + "/b/healthlog-20261001T080000Z-pre-migration.manifest.json", + ]); + }); + + it("removes a .partial only once it is older than twelve hours", () => { + const fresh = set("daily", "2026-10-25T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 60 * 60_000, + }); + const stale = set("daily", "2026-10-23T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 13 * 60 * 60_000, + }); + const doomed = selectForDeletion([fresh, stale], { + keep: { daily: 7, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261023T013000Z-daily.dump.partial", + ]); + }); + + it("works on a real directory and leaves files it did not write alone", async () => { + const dir = tempDir(); + const names: string[] = []; + for (let day = 15; day <= 24; day++) { + const base = `healthlog-202610${day}T013000Z-daily`; + names.push(`${base}.dump`, `${base}.manifest.json`); + } + // Two that are not ours, and an off-host copy an operator put here. + names.push( + "my-own-dump.dump", + "healthlog-20261001T013000Z-daily.dump.bak", + "healthlog-20261002T013000Z-daily.dump.enc", + ); + for (const name of names) writeFileSync(join(dir, name), "x"); + // A crash's leftovers: one old, one from a run that may still be going. + const oldPartial = join( + dir, + "healthlog-20261003T013000Z-daily.dump.partial", + ); + writeFileSync(oldPartial, "x"); + const day = 24 * 60 * 60; + utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day); + writeFileSync( + join(dir, "healthlog-20261025T013000Z-daily.dump.partial"), + "x", + ); + + const removed = await applyRetention(dir, { dailyKeep: 7 }); + expect(removed).toHaveLength(7); + const left = (await readdir(dir)).sort(); + expect(left).toContain("my-own-dump.dump"); + expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak"); + expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc"); + expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial"); + expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial"); + expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual( + [18, 19, 20, 21, 22, 23, 24].map( + (d) => `healthlog-202610${d}T013000Z-daily.dump`, + ), + ); + expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily"); + }); +}); + +describe("key fingerprints", () => { + it("are the fingerprints the app's own key code gives, for the map and the single key", async () => { + vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B })); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2"); + vi.stubEnv("ENCRYPTION_KEY", ""); + const crypto = await import("@/lib/crypto"); + crypto._resetCryptoCacheForTests(); + const prints = keyFingerprints(process.env); + expect(prints.encryption.activeKeyId).toBe("v2"); + expect(prints.encryption.keys).toEqual([ + { id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }, + { id: "v2", fingerprint: crypto.getKeyFingerprint("v2") }, + ]); + + vi.stubEnv("ENCRYPTION_KEYS", ""); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", ""); + const base64 = Buffer.from(HEX_C, "hex").toString("base64"); + vi.stubEnv("ENCRYPTION_KEY", base64); + crypto._resetCryptoCacheForTests(); + const single = keyFingerprints(process.env); + expect(single.encryption).toEqual({ + activeKeyId: "v1", + keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }], + }); + crypto._resetCryptoCacheForTests(); + }); + + it("names the off-host key the way an off-host object records it", async () => { + const { offhostKeyId } = await import("@/lib/jobs/offhost-backup"); + const prints = keyFingerprints({ + ENCRYPTION_KEY: HEX_A, + BACKUP_ENCRYPTION_KEY: HEX_D, + BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`, + API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789", + }); + expect(prints.backupEncryptionKey).toBe( + offhostKeyId(Buffer.from(HEX_D, "hex")), + ); + expect(prints.backupEncryptionPreviousKeys).toEqual([ + offhostKeyId(Buffer.from(HEX_C, "hex")), + ]); + expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/); + }); + + it("never carries key material", () => { + const env = { + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }), + BACKUP_ENCRYPTION_KEY: HEX_B, + API_TOKEN_HMAC_KEY: HEX_C, + }; + const text = JSON.stringify(keyFingerprints(env)); + for (const secret of [HEX_A, HEX_B, HEX_C]) { + expect(text).not.toContain(secret.slice(0, 16)); + } + }); + + it("decodes keys the way the app does, and nothing that is not a key", () => { + expect(decodeKeyMaterial(HEX_A)?.length).toBe(32); + expect( + decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length, + ).toBe(32); + expect(decodeKeyMaterial("")).toBeNull(); + expect(decodeKeyMaterial("not a key!")).toBeNull(); + }); +}); + +describe("comparing a dump's keys with the environment's", () => { + const written = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + + it("finds nothing when every key is there", () => { + const same = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + expect(compareKeyFingerprints(written, same)).toEqual({ + findings: [], + refuse: false, + }); + }); + + it("refuses when the active key is different, and says which", () => { + const wrong = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + const result = compareKeyFingerprints(written, wrong); + expect(result.refuse).toBe(true); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]).toMatchObject({ + key: "ENCRYPTION_KEYS[v2]", + state: "different", + severity: "refuse", + }); + }); + + it("refuses when the active key is missing, warns for a retired one", () => { + const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A }); + const result = compareKeyFingerprints(written, onlyOld); + expect(result.refuse).toBe(true); + const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f])); + expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({ + state: "missing", + severity: "refuse", + }); + expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined(); + expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + + const noRetired = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }), + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-two", + }); + const second = compareKeyFingerprints(written, noRetired); + expect(second.refuse).toBe(false); + expect(second.findings.map((f) => [f.key, f.severity])).toEqual([ + ["ENCRYPTION_KEYS[v1]", "warn"], + ["API_TOKEN_HMAC_KEY", "warn"], + ]); + }); +}); + +describe("the manifest", () => { + const manifest = buildManifest({ + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: new Date("2026-10-25T01:30:00Z"), + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + server: { version: "18.6", versionNum: 180006 }, + pgDump: "pg_dump (PostgreSQL) 18.6", + archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 }, + env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" }, + }); + + it("carries what a restore and an operator need", () => { + expect(manifest).toMatchObject({ + format: "healthlog-database-dump", + formatVersion: 1, + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: "2026-10-25T01:30:00.000Z", + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + postgres: { serverVersion: "18.6", serverVersionNum: 180006 }, + dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + }); + expect(Object.keys(manifest.rowCounts)).toEqual([ + "pgboss.job", + "public.a", + "public.users", + ]); + expect(manifest.keys.encryption.keys[0].id).toBe("v1"); + }); + + it("reads back, and refuses anything else", () => { + expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual( + manifest, + ); + expect(() => parseManifest({ format: "something-else" })).toThrow( + /not a HealthLog/, + ); + expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow(); + expect(() => parseManifest(null)).toThrow(); + }); +}); + +describe("row counts after a restore", () => { + it("names every table that is missing, extra or different", () => { + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2 }, + { "public.a": 1, "public.b": 2 }, + ), + ).toEqual([]); + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2, "public.c": 5 }, + { "public.a": 1, "public.b": 0, "public.d": 1 }, + ), + ).toEqual([ + "public.b: 2 rows in the dump, 0 restored", + "public.c: in the dump with 5 rows, missing now", + "public.d: restored but not in the dump's manifest", + ]); + }); +}); + +describe("the connection the client tools get", () => { + it("drops Prisma's parameters and keeps the password out of the arguments", () => { + const c = connectionFromUrl( + "postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require", + ); + expect(c.pgEnv).toEqual({ + PGHOST: "db", + PGPORT: "6543", + PGUSER: "health@log", + PGPASSWORD: "p@ss/w", + PGDATABASE: "hl", + PGSSLMODE: "require", + }); + expect(c.schema).toBe("app"); + const cs = String(c.clientConfig.connectionString); + expect(cs).not.toContain("connection_limit"); + expect(cs).not.toContain("schema="); + expect(cs).toContain("sslmode=require"); + }); +}); + +describe("an off-host copy read back by the restore", () => { + /** The uploader the job uses, into memory. */ + async function upload(objectKey: string, key: Buffer, body: Buffer) { + const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup"); + const stored = new Map(); + const s3: S3Like = { + putStream: async (k: string, stream: Readable) => { + stored.set(k, await collect(stream)); + }, + putObject: async () => {}, + getObject: async () => Buffer.alloc(0), + headObject: async () => false, + listObjects: async () => [], + deleteObject: async () => {}, + }; + await uploadEncryptedBackup(s3, objectKey, key, async (write) => { + // Several pieces, as the job streams the file. + for (let i = 0; i < body.length; i += 1000) { + await write(body.subarray(i, i + 1000)); + } + }); + return stored.get(objectKey) as Buffer; + } + + it("opens what the job's uploader sealed, byte for byte", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + + await decryptOffhostFile({ + inPath: join(dir, name), + outPath: join(dir, "opened.dump"), + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex"), key], + }); + expect( + (await collect(createReadStream(join(dir, "opened.dump")))).equals(body), + ).toBe(true); + }); + + it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from("PGDMP fake archive bytes".repeat(100)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + const out = join(dir, "opened.dump"); + + // Renamed: the object key is bound as associated data. + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey( + "healthlog-20261024T013000Z-daily.dump.enc", + ), + keys: [key], + }), + ).rejects.toThrow(/does not open/); + expect(await readdir(dir)).toEqual([name]); + + // Altered. + const altered = Buffer.from(sealed); + altered[altered.length - 40] ^= 1; + writeFileSync(join(dir, name), altered); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [key], + }), + ).rejects.toThrow(); + expect(await readdir(dir)).toEqual([name]); + + // A key it was not written under. + writeFileSync(join(dir, name), sealed); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex")], + }), + ).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/); + expect(await readdir(dir)).toEqual([name]); + }); +}); diff --git a/src/lib/database-backup/dump-core.mjs b/src/lib/database-backup/dump-core.mjs new file mode 100644 index 000000000..14f13d2ca --- /dev/null +++ b/src/lib/database-backup/dump-core.mjs @@ -0,0 +1,1183 @@ +/** + * The whole-database dump: naming, writing, checking, keeping and reading it + * back. One module for three callers that cannot share TypeScript: + * + * - the nightly job in the app (`src/lib/database-backup/run.ts`), bundled; + * - the pre-migration dump the entrypoint takes (`scripts/db-dump.mjs`); + * - the restore (`scripts/db-restore.mjs`), which runs in the image with + * plain `node` on a server that has nothing but the dump and the keys. + * + * It is plain ESM with no imports beyond Node and `pg`, for the same reason + * `src/lib/auth/argon2-params.mjs` is: the production image strips `tsx`, and + * a restore that needs a source checkout is not a restore. Every rule that + * decides whether a dump is good, which files may be deleted and whether the + * keys fit lives here once, so the writer and the reader cannot drift. + * + * What a dump is: `pg_dump --format=custom` of the whole database (every + * schema, pg-boss included), named `healthlog--.dump`, + * with `healthlog--.manifest.json` beside it. A dump is + * good only when both exist under their final names: the archive is written + * as `.partial`, read back with `pg_restore --list`, the manifest is written + * and renamed first, and the archive is renamed last. A run that dies at any + * point leaves nothing that looks good. + * + * The row counts in the manifest come from the same snapshot `pg_dump` reads + * (`pg_export_snapshot()` + `--snapshot`), so a dump taken while the app is + * writing still restores to exactly the counts it claims. + */ +import { spawn } from "node:child_process"; +import { createDecipheriv, createHash } from "node:crypto"; +import { + accessSync, + closeSync, + constants as fsConstants, + createReadStream, + createWriteStream, + mkdirSync, + openSync, + readSync, + readdirSync, + statSync, +} from "node:fs"; +import { readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; + +/** Where dumps are written when `DB_BACKUP_DIR` is unset. */ +export const DEFAULT_BACKUP_DIR = "/var/lib/healthlog/backups"; + +/** Daily dumps kept when `DB_BACKUP_KEEP` is unset. */ +export const DEFAULT_DAILY_KEEP = 7; + +/** Pre-migration dumps kept: the one before this update and the one before. */ +export const PRE_MIGRATION_KEEP = 2; + +/** The cron the nightly dump runs on when `DB_BACKUP_SCHEDULE` is unset. */ +export const DEFAULT_BACKUP_SCHEDULE = "30 1 * * *"; + +/** The kinds of dump this module writes. */ +export const DUMP_KINDS = /** @type {const} */ (["daily", "pre-migration"]); + +/** The manifest's own format marker. */ +export const MANIFEST_FORMAT = "healthlog-database-dump"; +export const MANIFEST_FORMAT_VERSION = 1; + +/** The bucket prefix off-host database dumps live under. */ +export const OFFHOST_DATABASE_PREFIX = "database/"; + +/** + * A leftover `.partial` or an orphaned manifest younger than this may belong + * to a run that is still going (another container, the entrypoint), so it is + * left alone. + */ +const STALE_LEFTOVER_MS = 12 * 60 * 60 * 1000; + +const NAME_PATTERN = + /^healthlog-(\d{8}T\d{6}Z)-(daily|pre-migration)\.(dump|manifest\.json)(\.partial|\.enc)?$/; + +/** + * @typedef {"daily" | "pre-migration"} DumpKind + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * role: "dump" | "manifest"; + * variant: "final" | "partial" | "encrypted"; + * }} ParsedDumpName + */ + +/** `20261025T013000Z` for an instant, in UTC. */ +export function formatDumpTimestamp(date) { + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[-:]/g, ""); +} + +/** + * The shared stem of a dump and its manifest. + * + * @param {DumpKind} kind + * @param {Date} date + */ +export function dumpBaseName(kind, date) { + return `healthlog-${formatDumpTimestamp(date)}-${kind}`; +} + +/** @param {string} base */ +export function dumpFileName(base) { + return `${base}.dump`; +} + +/** @param {string} base */ +export function manifestFileName(base) { + return `${base}.manifest.json`; +} + +/** + * Parse a file name this module writes. Anything else answers null, which is + * what keeps retention away from files it did not write. + * + * @param {string} name + * @returns {ParsedDumpName | null} + */ +export function parseDumpFileName(name) { + const m = NAME_PATTERN.exec(name); + if (!m) return null; + const [, stamp, kind, role, suffix] = m; + const iso = `${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`; + const timestamp = new Date(iso); + if (Number.isNaN(timestamp.getTime())) return null; + return { + base: `healthlog-${stamp}-${kind}`, + kind: /** @type {DumpKind} */ (kind), + timestamp, + role: role === "dump" ? "dump" : "manifest", + variant: + suffix === ".partial" + ? "partial" + : suffix === ".enc" + ? "encrypted" + : "final", + }; +} + +// ── Configuration ────────────────────────────────────────────────────────── + +/** @typedef {Record} Env */ + +/** @param {string | undefined} raw */ +function isOff(raw) { + const v = (raw ?? "").trim().toLowerCase(); + return v === "false" || v === "0" || v === "no" || v === "off"; +} + +/** `DB_BACKUP_ENABLED`, on unless it says off. @param {Env} env */ +export function databaseBackupEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_ENABLED); +} + +/** `DB_BACKUP_BEFORE_MIGRATE`, on unless it says off. @param {Env} env */ +export function preMigrationDumpEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_BEFORE_MIGRATE); +} + +/** `DB_BACKUP_DIR`, or the image's default. @param {Env} env */ +export function resolveBackupDir(env = process.env) { + return (env.DB_BACKUP_DIR ?? "").trim() || DEFAULT_BACKUP_DIR; +} + +/** + * `DB_BACKUP_KEEP`: how many daily dumps stay on disk. At least one, so the + * newest good dump is never deleted; a value that is not a whole number + * keeps the default. + * + * @param {Env} env + */ +export function resolveDailyKeep(env = process.env) { + const raw = (env.DB_BACKUP_KEEP ?? "").trim(); + if (!/^\d+$/.test(raw)) return DEFAULT_DAILY_KEEP; + return Math.min(Math.max(Number.parseInt(raw, 10), 1), 366); +} + +const CRON_FIELD = /^[0-9*,/-]+$/; + +/** + * `DB_BACKUP_SCHEDULE` when it is a five-field cron of digits, `*`, `,`, `/` + * and `-`, otherwise the default. `valid` says which, so the boot summary can + * name a schedule it ignored. + * + * @param {Env} env + * @returns {{ cron: string; valid: boolean; configured: boolean }} + */ +export function resolveBackupSchedule(env = process.env) { + const raw = (env.DB_BACKUP_SCHEDULE ?? "").trim(); + if (!raw) { + return { cron: DEFAULT_BACKUP_SCHEDULE, valid: true, configured: false }; + } + const fields = raw.split(/\s+/); + const valid = fields.length === 5 && fields.every((f) => CRON_FIELD.test(f)); + return { + cron: valid ? fields.join(" ") : DEFAULT_BACKUP_SCHEDULE, + valid, + configured: true, + }; +} + +/** + * Whether `dir` exists (or can be created) and this process may write in it. + * Never throws. + * + * @param {string} dir + */ +export function isWritableDirectory(dir) { + try { + mkdirSync(dir, { recursive: true }); + if (!statSync(dir).isDirectory()) return false; + accessSync(dir, fsConstants.W_OK | fsConstants.X_OK); + return true; + } catch { + return false; + } +} + +// ── Keys ─────────────────────────────────────────────────────────────────── + +/** + * Decode a 32-byte key the way the app does (`decodeKey` in + * `src/lib/crypto.ts`): 64 hex characters, or base64 of 32 bytes. Shorter hex + * is padded with its own SHA-256 there, outside production only; it is + * decoded the same way here so a development instance's fingerprints match + * what its admin page shows. Null when the value is not a key at all. + * + * @param {string | undefined} raw + * @returns {Buffer | null} + */ +export function decodeKeyMaterial(raw) { + const value = (raw ?? "").trim(); + if (!value) return null; + if (/^[0-9a-fA-F]{64}$/.test(value)) return Buffer.from(value, "hex"); + if (/^[A-Za-z0-9+/=]+$/.test(value)) { + const buf = Buffer.from(value, "base64"); + if (buf.length === 32) return buf; + } + if (/^[0-9a-fA-F]+$/.test(value) && value.length >= 32) { + const padded = + value + createHash("sha256").update(value, "utf8").digest("hex"); + return Buffer.from(padded.slice(0, 64), "hex"); + } + return null; +} + +/** + * The public fingerprint of a key: the first twelve hex characters of + * SHA-256 over its bytes. The same rule as `fingerprintKeyBytes` in + * `src/lib/crypto.ts` (what the admin's key-backup step shows) and as the key + * id an off-host object records, so one number names one key everywhere. + * + * @param {Buffer} bytes + */ +export function fingerprintBytes(bytes) { + return createHash("sha256").update(bytes).digest("hex").slice(0, 12); +} + +/** + * @typedef {{ + * encryption: { + * activeKeyId: string | null; + * keys: Array<{ id: string; fingerprint: string }>; + * }; + * backupEncryptionKey: string | null; + * backupEncryptionPreviousKeys: string[]; + * apiTokenHmacKey: string | null; + * }} KeyFingerprints + */ + +/** + * Fingerprints of every key a restored database needs, read from an + * environment. Never the key material. A malformed value is left out, which + * the comparison then reports as missing. + * + * `ENCRYPTION_KEY` joins the map as `v1` unless the map names `v1` itself, + * exactly as the app's key loader does. + * + * @param {Env} env + * @returns {KeyFingerprints} + */ +export function keyFingerprints(env = process.env) { + /** @type {Map} */ + const keys = new Map(); + let activeKeyId = null; + const rawMap = (env.ENCRYPTION_KEYS ?? "").trim(); + if (rawMap) { + try { + const parsed = JSON.parse(rawMap); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + for (const [id, value] of Object.entries(parsed)) { + const bytes = + typeof value === "string" ? decodeKeyMaterial(value) : null; + if (bytes) keys.set(id, fingerprintBytes(bytes)); + } + } + } catch { + // Malformed JSON: no keys from the map. The entrypoint refuses to boot + // on it anyway; here it only means nothing matches. + } + activeKeyId = (env.ENCRYPTION_ACTIVE_KEY_ID ?? "").trim() || null; + if (!activeKeyId && keys.size === 1) activeKeyId = [...keys.keys()][0]; + } + const legacy = decodeKeyMaterial(env.ENCRYPTION_KEY); + if (legacy && !keys.has("v1")) keys.set("v1", fingerprintBytes(legacy)); + if (!activeKeyId && legacy) activeKeyId = "v1"; + + const backup = decodeKeyMaterial(env.BACKUP_ENCRYPTION_KEY); + const previous = (env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "") + .split(",") + .map((raw) => decodeKeyMaterial(raw)) + .filter((b) => b !== null) + .map((b) => fingerprintBytes(/** @type {Buffer} */ (b))); + const hmac = (env.API_TOKEN_HMAC_KEY ?? "").trim(); + + return { + encryption: { + activeKeyId, + keys: [...keys.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, fingerprint]) => ({ id, fingerprint })), + }, + backupEncryptionKey: backup ? fingerprintBytes(backup) : null, + backupEncryptionPreviousKeys: previous, + // Used as a string by the app (`createHmac("sha256", key)`), so its + // fingerprint is over the string's bytes. + apiTokenHmacKey: hmac ? fingerprintBytes(Buffer.from(hmac, "utf8")) : null, + }; +} + +/** + * @typedef {{ + * key: string; + * state: "missing" | "different"; + * severity: "refuse" | "warn"; + * detail: string; + * }} KeyFinding + */ + +/** + * Compare the keys a dump was written with against the keys an environment + * holds. The active encryption key is the one that refuses: without it the + * app's start-up key check fails and nothing encrypted reads. Every other + * difference is a warning that says what stops working. + * + * @param {KeyFingerprints} expected the manifest's + * @param {KeyFingerprints} actual the environment's + * @returns {{ findings: KeyFinding[]; refuse: boolean }} + */ +export function compareKeyFingerprints(expected, actual) { + /** @type {KeyFinding[]} */ + const findings = []; + const have = new Map( + actual.encryption.keys.map((k) => [k.id, k.fingerprint]), + ); + const active = expected.encryption.activeKeyId; + for (const { id, fingerprint } of expected.encryption.keys) { + const mine = have.get(id); + if (mine === fingerprint) continue; + const isActive = id === active; + findings.push({ + key: `ENCRYPTION_KEYS[${id}]`, + state: mine === undefined ? "missing" : "different", + severity: isActive ? "refuse" : "warn", + detail: isActive + ? `the dump's active encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `configured with a different value (fingerprint ${mine})`}; nothing encrypted in this database can be read without it` + : `encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `different here (fingerprint ${mine})`}; rows still sealed under it will not open`, + }); + } + if ( + expected.backupEncryptionKey && + expected.backupEncryptionKey !== actual.backupEncryptionKey + ) { + findings.push({ + key: "BACKUP_ENCRYPTION_KEY", + state: actual.backupEncryptionKey ? "different" : "missing", + severity: "warn", + detail: `the off-host backup key (fingerprint ${expected.backupEncryptionKey}) is ${actual.backupEncryptionKey ? `different here (fingerprint ${actual.backupEncryptionKey})` : "not configured"}; off-host copies written before the restore will not open`, + }); + } + if ( + expected.apiTokenHmacKey && + expected.apiTokenHmacKey !== actual.apiTokenHmacKey + ) { + findings.push({ + key: "API_TOKEN_HMAC_KEY", + state: actual.apiTokenHmacKey ? "different" : "missing", + severity: "warn", + detail: `API_TOKEN_HMAC_KEY (fingerprint ${expected.apiTokenHmacKey}) is ${actual.apiTokenHmacKey ? `different here (fingerprint ${actual.apiTokenHmacKey})` : "not configured"}; every session, app token and share link will stop working, no data is lost`, + }); + } + return { + findings, + refuse: findings.some((f) => f.severity === "refuse"), + }; +} + +// ── Connection ───────────────────────────────────────────────────────────── + +/** + * Query parameters libpq understands, by the environment variable that + * carries each to `pg_dump` / `pg_restore`. `DATABASE_URL` also carries + * Prisma's own (`schema`, `connection_limit`, `pool_timeout`), which libpq + * rejects, so the URL is never handed over as it is. + */ +const LIBPQ_PARAMS = { + host: "PGHOST", + sslmode: "PGSSLMODE", + sslcert: "PGSSLCERT", + sslkey: "PGSSLKEY", + sslrootcert: "PGSSLROOTCERT", + connect_timeout: "PGCONNECT_TIMEOUT", + application_name: "PGAPPNAME", +}; + +/** + * @typedef {{ + * pgEnv: Record; + * clientConfig: Record; + * schema: string; + * }} Connection + */ + +/** + * The connection `DATABASE_URL` describes, as libpq environment variables + * (the password travels there, never in a process argument) and as a `pg` + * client config. + * + * @param {string} databaseUrl + * @returns {Connection} + */ +export function connectionFromUrl(databaseUrl) { + const url = new URL(databaseUrl); + const host = decodeURIComponent(url.hostname).replace(/^\[|\]$/g, ""); + /** @type {Record} */ + const pgEnv = { + PGHOST: host, + PGPORT: url.port || "5432", + PGUSER: decodeURIComponent(url.username), + PGPASSWORD: decodeURIComponent(url.password), + PGDATABASE: decodeURIComponent(url.pathname.replace(/^\//, "")), + }; + for (const [param, value] of url.searchParams) { + const name = /** @type {Record} */ (LIBPQ_PARAMS)[param]; + if (name) pgEnv[name] = value; + } + const clean = new URL(databaseUrl); + for (const param of [...clean.searchParams.keys()]) { + if (!(param in LIBPQ_PARAMS)) clean.searchParams.delete(param); + } + return { + pgEnv, + clientConfig: { + connectionString: clean.toString(), + connectionTimeoutMillis: 10_000, + }, + schema: url.searchParams.get("schema") || "public", + }; +} + +/** + * A connected `pg` client for `DATABASE_URL`. + * + * @param {string} databaseUrl + */ +export async function connect(databaseUrl) { + const { default: pg } = await import("pg"); + const client = new pg.Client(connectionFromUrl(databaseUrl).clientConfig); + await client.connect(); + // The server may carry a default statement timeout (the app sets one per + // session through `options`); counting a large table must not hit it. + await client.query("SET statement_timeout = 0"); + return client; +} + +// ── Facts about a database ───────────────────────────────────────────────── + +/** + * @typedef {{ query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> }} Queryable + */ + +/** Tables outside the system schemas, `schema.table`, sorted. @param {Queryable} client */ +export async function listUserTables(client) { + const { rows } = await client.query( + `SELECT format('%I.%I', n.nspname, c.relname) AS name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg\\_%' + ORDER BY 1`, + ); + return rows.map((r) => /** @type {string} */ (r.name)); +} + +/** + * Exact row counts of every user table, keyed `schema.table`. + * + * @param {Queryable} client + * @returns {Promise>} + */ +export async function countRows(client) { + /** @type {Record} */ + const counts = {}; + for (const table of await listUserTables(client)) { + const { rows } = await client.query( + `SELECT count(*)::bigint AS n FROM ${table}`, + ); + counts[table] = Number(rows[0].n); + } + return counts; +} + +/** + * The newest finished migration, or null on a database Prisma never touched. + * + * @param {Queryable} client + * @param {string} schema + */ +export async function latestMigration(client, schema) { + const table = await client.query( + "SELECT to_regclass(format('%I._prisma_migrations', $1::text)) AS t", + [schema], + ); + if (!table.rows[0]?.t) return null; + const { rows } = await client.query( + `SELECT migration_name FROM ${table.rows[0].t} + WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL + ORDER BY migration_name DESC LIMIT 1`, + ); + return rows[0]?.migration_name ?? null; +} + +/** @param {Queryable} client */ +export async function serverVersion(client) { + const { rows } = await client.query( + "SELECT current_setting('server_version') AS v, current_setting('server_version_num')::int AS n", + ); + return { version: String(rows[0].v), versionNum: Number(rows[0].n) }; +} + +// ── Running the Postgres client tools ────────────────────────────────────── + +/** + * Run one Postgres client tool. Standard output goes to `stdout` when given + * (a file stream) and is collected otherwise; standard input comes from + * `stdin` when given. Rejects with the tool's own last lines on a non-zero + * exit, and kills it at `deadline` (epoch ms). + * + * @param {string} tool + * @param {string[]} args + * @param {{ + * env?: Record; + * stdout?: NodeJS.WritableStream; + * stdin?: NodeJS.ReadableStream; + * deadline?: number; + * }} [options] + * @returns {Promise} + */ +export function runTool(tool, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(tool, args, { + env: { ...process.env, ...(options.env ?? {}) }, + stdio: [options.stdin ? "pipe" : "ignore", "pipe", "pipe"], + }); + /** @type {Buffer[]} */ + const out = []; + let err = ""; + let timedOut = false; + const timer = + options.deadline !== undefined + ? setTimeout( + () => { + timedOut = true; + child.kill("SIGTERM"); + }, + Math.max(0, options.deadline - Date.now()), + ) + : null; + if (options.stdout) { + // A destination that fails (a full disk) must stop the tool: unpiped, + // it would block on a full pipe until its deadline, or forever. + options.stdout.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdout.pipe(options.stdout); + } else { + child.stdout.on("data", (c) => out.push(c)); + } + child.stderr.on("data", (c) => { + err = (err + c.toString("utf8")).slice(-4000); + }); + if (options.stdin && child.stdin) { + options.stdin.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdin.on("error", () => { + // The tool closed its input early; its exit code says why. + }); + options.stdin.pipe(child.stdin); + } + child.on("error", (e) => { + if (timer) clearTimeout(timer); + reject( + /** @type {NodeJS.ErrnoException} */ (e).code === "ENOENT" + ? new Error( + `${tool} is not installed in this environment (the HealthLog image ships it; on another host install the PostgreSQL client tools)`, + ) + : e, + ); + }); + child.on("close", (code, signal) => { + if (timer) clearTimeout(timer); + if (timedOut) { + reject(new Error(`${tool} ran past its time limit and was stopped`)); + } else if (code === 0) { + resolve(Buffer.concat(out).toString("utf8")); + } else { + const tail = err.trim().split("\n").slice(-3).join(" | "); + reject( + new Error( + `${tool} exited ${code ?? signal}${tail ? `: ${tail}` : ""}`, + ), + ); + } + }); + }); +} + +/** `pg_dump (PostgreSQL) 18.6`, or null when it cannot be asked. */ +export async function pgDumpVersion() { + try { + return (await runTool("pg_dump", ["--version"])).trim(); + } catch { + return null; + } +} + +/** + * Read an archive's table of contents back. A file `pg_restore` cannot list + * is not a dump, whatever its name. Answers the number of entries. + * + * @param {string} path + */ +export async function verifyArchive(path) { + const listing = await runTool("pg_restore", ["--list"], { + stdin: createReadStream(path), + }); + const entries = listing + .split("\n") + .filter((line) => line.trim() && !line.startsWith(";")).length; + if (entries === 0) throw new Error("the archive lists no entries"); + return entries; +} + +/** SHA-256 and size of a file, streamed. @param {string} path */ +export async function hashFile(path) { + const hash = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + bytes += chunk.length; + } + return { sha256: hash.digest("hex"), bytes }; +} + +// ── Writing a dump ───────────────────────────────────────────────────────── + +/** + * @typedef {{ + * format: string; + * formatVersion: number; + * kind: DumpKind; + * file: string; + * createdAt: string; + * appVersion: string | null; + * buildSha: string | null; + * latestMigration: string | null; + * postgres: { serverVersion: string; serverVersionNum: number; pgDump: string | null }; + * dump: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * keys: KeyFingerprints; + * }} DumpManifest + */ + +/** + * Build the manifest of one dump. Pure, so the shape is tested on its own. + * + * @param {{ + * kind: DumpKind; + * file: string; + * createdAt: Date; + * appVersion?: string | null; + * buildSha?: string | null; + * latestMigration: string | null; + * server: { version: string; versionNum: number }; + * pgDump: string | null; + * archive: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * env: Env; + * }} input + * @returns {DumpManifest} + */ +export function buildManifest(input) { + return { + format: MANIFEST_FORMAT, + formatVersion: MANIFEST_FORMAT_VERSION, + kind: input.kind, + file: input.file, + createdAt: input.createdAt.toISOString(), + appVersion: input.appVersion || null, + buildSha: input.buildSha || null, + latestMigration: input.latestMigration, + postgres: { + serverVersion: input.server.version, + serverVersionNum: input.server.versionNum, + pgDump: input.pgDump, + }, + dump: { ...input.archive }, + rowCounts: Object.fromEntries( + Object.entries(input.rowCounts).sort(([a], [b]) => a.localeCompare(b)), + ), + keys: keyFingerprints(input.env), + }; +} + +/** + * Check a parsed manifest has the fields a restore reads. + * + * @param {unknown} value + * @returns {DumpManifest} + */ +export function parseManifest(value) { + const m = /** @type {Record} */ (value); + if ( + !m || + m.format !== MANIFEST_FORMAT || + m.formatVersion !== MANIFEST_FORMAT_VERSION || + typeof m.file !== "string" || + typeof m.dump?.sha256 !== "string" || + typeof m.dump?.bytes !== "number" || + typeof m.rowCounts !== "object" || + !m.keys?.encryption + ) { + throw new Error("not a HealthLog database dump manifest"); + } + return /** @type {DumpManifest} */ (m); +} + +/** + * Take one dump into `dir`, with its manifest, and answer both paths. + * + * The archive and the row counts read one exported snapshot. The archive is + * written as `.partial`, hashed and listed back; the manifest is written as + * `.partial` and renamed; the archive is renamed last. On any failure every + * file this call created is removed and the error is rethrown. + * + * @param {{ + * databaseUrl: string; + * dir: string; + * kind: DumpKind; + * now?: Date; + * env?: Env; + * appVersion?: string | null; + * buildSha?: string | null; + * deadline?: number; + * }} options + * @returns {Promise<{ base: string; dumpPath: string; manifestPath: string; manifest: DumpManifest }>} + */ +export async function createDatabaseDump(options) { + const now = options.now ?? new Date(); + const env = options.env ?? process.env; + const base = dumpBaseName(options.kind, now); + const dumpPath = join(options.dir, dumpFileName(base)); + const manifestPath = join(options.dir, manifestFileName(base)); + const partialDump = `${dumpPath}.partial`; + const partialManifest = `${manifestPath}.partial`; + const connection = connectionFromUrl(options.databaseUrl); + // Two runs in the same second would share a name. The second one must not + // replace, or on failure remove, what the first one wrote. + for (const taken of [dumpPath, manifestPath]) { + if (await statOrNull(taken)) { + throw new Error(`a dump named ${basename(taken)} already exists`); + } + } + + const client = await connect(options.databaseUrl); + let finished = false; + /** The files this call created, and only those, go on a failure. */ + const created = []; + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY"); + const { rows } = await client.query( + "SELECT pg_export_snapshot() AS snapshot", + ); + const snapshot = String(rows[0].snapshot); + + const out = createWriteStream(partialDump, { flags: "wx", mode: 0o600 }); + const opened = new Promise((resolve, reject) => { + out.once("open", resolve); + out.once("error", reject); + }); + await opened; + created.push(partialDump); + const closed = new Promise((resolve, reject) => { + out.once("close", resolve); + out.once("error", reject); + }); + // Awaited below on success; on a failure the run is already rejecting + // and this must not surface a second time as unhandled. + closed.catch(() => {}); + // `pg_dump` reads the snapshot in its own session while this one counts + // the rows of the same snapshot; the queries on this client run one + // after another (a `pg` client does not overlap them). + const facts = async () => ({ + counts: await countRows(client), + server: await serverVersion(client), + migration: await latestMigration(client, connection.schema), + }); + const [, { counts, server, migration }, pgDump] = await Promise.all([ + runTool( + "pg_dump", + [ + "--format=custom", + "--no-password", + `--snapshot=${snapshot}`, + "--lock-wait-timeout=120000", + ], + { env: connection.pgEnv, stdout: out, deadline: options.deadline }, + ), + facts(), + pgDumpVersion(), + ]); + await closed; + await client.query("COMMIT"); + + const { sha256, bytes } = await hashFile(partialDump); + const entries = await verifyArchive(partialDump); + const manifest = buildManifest({ + kind: options.kind, + file: dumpFileName(base), + createdAt: now, + appVersion: options.appVersion, + buildSha: options.buildSha, + latestMigration: migration, + server, + pgDump, + archive: { bytes, sha256, entries }, + rowCounts: counts, + env, + }); + await writeFile(partialManifest, `${JSON.stringify(manifest, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); + created.push(partialManifest); + await rename(partialManifest, manifestPath); + created.push(manifestPath); + await rename(partialDump, dumpPath); + finished = true; + return { base, dumpPath, manifestPath, manifest }; + } finally { + await client.end().catch(() => {}); + if (!finished) { + // Only what this call created: a `wx` open that failed created nothing. + await Promise.all(created.map((p) => rm(p, { force: true }))); + } + } +} + +// ── What is on disk, and what may go ─────────────────────────────────────── + +/** + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * dump: { path: string; bytes: number } | null; + * manifest: string | null; + * encryptedDump: string | null; + * encryptedManifest: string | null; + * leftovers: Array<{ path: string; mtimeMs: number }>; + * }} DumpSet + */ + +/** + * Every dump this module wrote into `dir`, newest first, grouped by stem. + * Files with any other name are not looked at. + * + * @param {string} dir + * @returns {DumpSet[]} + */ +export function listDumpSets(dir) { + /** @type {Map} */ + const sets = new Map(); + let names = []; + try { + names = readdirSync(dir); + } catch { + return []; + } + for (const name of names) { + const parsed = parseDumpFileName(name); + if (!parsed) continue; + const path = join(dir, name); + let info; + try { + info = statSync(path); + } catch { + continue; + } + if (!info.isFile()) continue; + const set = sets.get(parsed.base) ?? { + base: parsed.base, + kind: parsed.kind, + timestamp: parsed.timestamp, + dump: null, + manifest: null, + encryptedDump: null, + encryptedManifest: null, + leftovers: [], + }; + if (parsed.variant === "partial") { + set.leftovers.push({ path, mtimeMs: info.mtimeMs }); + } else if (parsed.variant === "encrypted") { + if (parsed.role === "dump") set.encryptedDump = path; + else set.encryptedManifest = path; + } else if (parsed.role === "dump") { + set.dump = { path, bytes: info.size }; + } else { + set.manifest = path; + } + sets.set(parsed.base, set); + } + return [...sets.values()].sort( + (a, b) => b.timestamp.getTime() - a.timestamp.getTime(), + ); +} + +/** A set is good when its archive and its manifest both carry final names. @param {DumpSet} set */ +export function isGoodSet(set) { + return set.dump !== null && set.manifest !== null; +} + +/** + * Which files retention removes. Pure. + * + * Per kind, the newest `keep` good dumps stay (at least one, so the newest + * good dump is never removed) and every older good dump goes, archive and + * manifest together. A `.partial` or a manifest without its archive goes once + * it is older than twelve hours, so a run still in progress keeps its files. + * A final-named archive without a manifest is not something this module + * leaves behind, so it is not touched. Encrypted copies (`.enc`) an operator + * placed here to restore from are never touched either. + * + * @param {DumpSet[]} sets + * @param {{ keep: Record; now: number }} options + * @returns {string[]} + */ +export function selectForDeletion(sets, options) { + /** @type {string[]} */ + const remove = []; + for (const kind of DUMP_KINDS) { + const keep = Math.max(1, options.keep[kind] ?? 1); + const good = sets + .filter((s) => s.kind === kind && isGoodSet(s)) + .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); + for (const set of good.slice(keep)) { + remove.push( + /** @type {{ path: string }} */ (set.dump).path, + /** @type {string} */ (set.manifest), + ); + } + } + for (const set of sets) { + for (const leftover of set.leftovers) { + if (options.now - leftover.mtimeMs > STALE_LEFTOVER_MS) { + remove.push(leftover.path); + } + } + if (set.manifest && !set.dump && set.leftovers.length === 0) { + try { + if (options.now - statSync(set.manifest).mtimeMs > STALE_LEFTOVER_MS) { + remove.push(set.manifest); + } + } catch { + // Gone already. + } + } + } + return remove; +} + +/** + * Apply retention to `dir`. Answers the paths removed. + * + * @param {string} dir + * @param {{ dailyKeep?: number; now?: number }} [options] + */ +export async function applyRetention(dir, options = {}) { + const doomed = selectForDeletion(listDumpSets(dir), { + keep: { + daily: options.dailyKeep ?? DEFAULT_DAILY_KEEP, + "pre-migration": PRE_MIGRATION_KEEP, + }, + now: options.now ?? Date.now(), + }); + for (const path of doomed) await rm(path, { force: true }); + return doomed; +} + +/** @param {string} path */ +export async function readManifestFile(path) { + return parseManifest(JSON.parse(await readFile(path, "utf8"))); +} + +// ── Off-host copies ──────────────────────────────────────────────────────── + +/** + * The bucket key of an off-host database object. It is also the associated + * data its encryption is bound to, so a copy cannot be passed off under + * another name. + * + * @param {string} fileName e.g. `healthlog-…-daily.dump.enc` + */ +export function offhostObjectKey(fileName) { + return `${OFFHOST_DATABASE_PREFIX}${fileName}`; +} + +const ENVELOPE_MAGIC = "HLBK"; +const ENVELOPE_KEYED = 0x04; +const AAD_PREFIX = "healthlog/offhost-backup/v4|"; +const IV_LENGTH = 12; +const TAG_LENGTH = 16; + +/** + * Open an off-host object written in the version-4 envelope + * (`src/lib/jobs/offhost-backup.ts`: `HLBK`, 0x04, key id, IV, gzip + * ciphertext, tag, the object key as associated data) into `outPath`, + * streaming, without holding the file. + * + * GCM's tag sits at the end, so the plaintext is written before the tag is + * checked; it goes to `outPath` and is removed again unless the tag verifies. + * Nothing reads `outPath` before this resolves. + * + * @param {{ inPath: string; outPath: string; objectKey: string; keys: Buffer[] }} options + */ +export async function decryptOffhostFile(options) { + const size = statSync(options.inPath).size; + const fd = openSync(options.inPath, "r"); + let header; + let tag; + try { + const head = Buffer.alloc(Math.min(size, 6 + 255 + IV_LENGTH)); + readSync(fd, head, 0, head.length, 0); + if ( + head.subarray(0, 4).toString("latin1") !== ENVELOPE_MAGIC || + head[4] !== ENVELOPE_KEYED + ) { + throw new Error( + "not an off-host database copy (unknown envelope); expected a .dump.enc written by HealthLog", + ); + } + const idLen = head[5]; + const keyId = head.subarray(6, 6 + idLen).toString("latin1"); + const iv = head.subarray(6 + idLen, 6 + idLen + IV_LENGTH); + header = { keyId, iv, length: 6 + idLen + IV_LENGTH }; + tag = Buffer.alloc(TAG_LENGTH); + readSync(fd, tag, 0, TAG_LENGTH, size - TAG_LENGTH); + } finally { + closeSync(fd); + } + if (size < header.length + TAG_LENGTH) { + throw new Error("the off-host copy is truncated"); + } + const key = options.keys.find((k) => fingerprintBytes(k) === header.keyId); + if (!key) { + throw new Error( + `the off-host copy was written under backup key ${header.keyId}, which is neither BACKUP_ENCRYPTION_KEY nor in BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + ); + } + const decipher = createDecipheriv("aes-256-gcm", key, header.iv); + decipher.setAAD(Buffer.from(`${AAD_PREFIX}${options.objectKey}`, "utf8")); + decipher.setAuthTag(tag); + try { + await pipeline( + createReadStream(options.inPath, { + start: header.length, + end: size - TAG_LENGTH - 1, + }), + decipher, + createGunzip(), + createWriteStream(options.outPath, { flags: "wx", mode: 0o600 }), + ); + } catch (err) { + await rm(options.outPath, { force: true }); + const reason = err instanceof Error ? err.message : String(err); + throw new Error( + /auth|unable to authenticate/i.test(reason) + ? "the off-host copy does not open with this key under this name (altered, renamed, or written with another key)" + : `could not open the off-host copy: ${reason}`, + ); + } +} + +/** + * The off-host keys an environment holds, current first. + * + * @param {Env} env + * @returns {Buffer[]} + */ +export function offhostKeysFromEnv(env = process.env) { + return [ + env.BACKUP_ENCRYPTION_KEY, + ...(env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "").split(","), + ] + .map((raw) => decodeKeyMaterial(raw)) + .filter((k) => k !== null) + .map((k) => /** @type {Buffer} */ (k)); +} + +// ── Restore ──────────────────────────────────────────────────────────────── + +/** + * Compare the counts a restore produced with the manifest's. Every table the + * manifest names must be there with its count, and no table may be there that + * the manifest does not name. + * + * @param {Record} expected + * @param {Record} actual + * @returns {string[]} one line per difference + */ +export function compareRowCounts(expected, actual) { + /** @type {string[]} */ + const problems = []; + for (const [table, count] of Object.entries(expected)) { + if (!(table in actual)) { + problems.push(`${table}: in the dump with ${count} rows, missing now`); + } else if (actual[table] !== count) { + problems.push( + `${table}: ${count} rows in the dump, ${actual[table]} restored`, + ); + } + } + for (const table of Object.keys(actual)) { + if (!(table in expected)) { + problems.push(`${table}: restored but not in the dump's manifest`); + } + } + return problems; +} + +/** + * Drop every non-system schema and recreate an empty `public`, so a restore + * that failed its check leaves the database as empty as it found it. + * + * @param {Queryable} client + */ +export async function emptyDatabase(client) { + const { rows } = await client.query( + `SELECT quote_ident(nspname) AS s FROM pg_namespace + WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'`, + ); + for (const { s } of rows) await client.query(`DROP SCHEMA ${s} CASCADE`); + await client.query("CREATE SCHEMA IF NOT EXISTS public"); +} + +/** Stat helper that answers null for a missing file. @param {string} path */ +export async function statOrNull(path) { + try { + return await stat(path); + } catch { + return null; + } +} diff --git a/src/lib/database-backup/health.ts b/src/lib/database-backup/health.ts new file mode 100644 index 000000000..2039df249 --- /dev/null +++ b/src/lib/database-backup/health.ts @@ -0,0 +1,117 @@ +/** + * Whether the nightly whole-database dump is happening, as facts a page or an + * alert can act on. + * + * Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of + * this process: whether `DB_BACKUP_DIR` is writable here, and whether the + * off-host bucket is configured. A ledger row that says a copy went off-host + * outranks this process's environment, for the reason the off-host card gives + * (a split install can hand the bucket variables to the worker only). + */ +import { prisma } from "@/lib/db"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; + +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, +} from "./dump-core.mjs"; + +export interface DatabaseBackupHealth { + /** `DB_BACKUP_ENABLED` is not off. */ + enabled: boolean; + /** The directory dumps are kept in (`DB_BACKUP_DIR`). */ + directory: string; + /** This process can write there. False means dumps only go off-host. */ + localDirWritable: boolean; + /** The off-host bucket is configured, or a dump has been uploaded to it. */ + offhostConfigured: boolean; + /** The newest dump that was kept on the server or reached the bucket. */ + lastGood: { + at: string; + fileName: string; + sizeBytes: number | null; + local: boolean; + offhost: boolean; + } | null; + /** The newest run that failed, or that a dying process left unfinished. */ + lastFailure: { at: string; error: string } | null; +} + +/** + * A run still `running` this long after it started was cut off by its + * process: the job's expiry, past which pg-boss has failed it anyway. + */ +const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000; + +export async function getDatabaseBackupHealth( + now: Date = new Date(), + env: NodeJS.ProcessEnv = process.env, +): Promise { + const directory = resolveBackupDir(env); + const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all( + [ + prisma.databaseBackupRun.findFirst({ + where: { + fileName: { not: null }, + OR: [{ localResult: "written" }, { offhostResult: "uploaded" }], + }, + orderBy: { startedAt: "desc" }, + select: { + startedAt: true, + finishedAt: true, + fileName: true, + dumpBytes: true, + localResult: true, + offhostResult: true, + }, + }), + prisma.databaseBackupRun.findFirst({ + where: { status: "failed" }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true, finishedAt: true, error: true }, + }), + prisma.databaseBackupRun.findFirst({ + where: { + status: "running", + startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) }, + }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true }, + }), + prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }), + ], + ); + + const failures = [ + lastFailed && { + at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(), + error: lastFailed.error ?? "failed without a reason", + }, + lastAbandoned && { + at: lastAbandoned.startedAt.toISOString(), + error: "the run did not finish (the process stopped under it)", + }, + ] + .filter((f): f is { at: string; error: string } => Boolean(f)) + .sort((a, b) => b.at.localeCompare(a.at)); + + return { + enabled: databaseBackupEnabled(env), + directory, + localDirWritable: isWritableDirectory(directory), + offhostConfigured: offhostBackupConfigured() || everUploaded > 0, + lastGood: lastGood + ? { + at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(), + fileName: lastGood.fileName ?? "", + sizeBytes: + lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes), + local: lastGood.localResult === "written", + offhost: lastGood.offhostResult === "uploaded", + } + : null, + lastFailure: failures[0] ?? null, + }; +} diff --git a/src/lib/database-backup/run.ts b/src/lib/database-backup/run.ts new file mode 100644 index 000000000..4f3e3d99e --- /dev/null +++ b/src/lib/database-backup/run.ts @@ -0,0 +1,243 @@ +/** + * One run of the nightly whole-database dump. + * + * The dump is the artifact that rebuilds a lost server: every account, every + * login, every grant and managed profile, the job queue and the instance + * settings, in one `pg_dump --format=custom` file with a manifest beside it + * (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the + * bundled compose file) and, when the off-host bucket is configured, uploaded + * under `database/` in the same envelope the per-account copies use, sealed + * with `BACKUP_ENCRYPTION_KEY`. + * + * Without a writable directory (an install that runs the image without the + * volume) the dump is written to a temporary directory, uploaded and removed: + * the off-host copy still happens, nothing is kept on the server, and the + * ledger and the boot summary say so. With neither, the run is skipped and + * recorded as such. + * + * Every run leaves one `DatabaseBackupRun` row: started, then finished with + * what happened on each leg. `getDatabaseBackupHealth` reads them. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { + getS3Client, + loadOffhostConfig, + uploadEncryptedBackup, + type S3Like, +} from "@/lib/jobs/offhost-backup"; +import { annotate } from "@/lib/logging/context"; +import packageJson from "../../../package.json"; + +import { + applyRetention, + createDatabaseDump, + databaseBackupEnabled, + isWritableDirectory, + offhostObjectKey, + resolveBackupDir, + resolveDailyKeep, +} from "./dump-core.mjs"; + +/** How long ledger rows are kept. A run a day; a year and a bit of history. */ +const LEDGER_RETENTION_DAYS = 400; + +export type DatabaseBackupLocalResult = + "written" | "no_directory" | "failed" | "skipped"; +export type DatabaseBackupOffhostResult = + "uploaded" | "not_configured" | "failed" | "skipped"; + +export interface DatabaseBackupReport { + status: "succeeded" | "failed" | "skipped"; + fileName: string | null; + dumpBytes: number | null; + local: DatabaseBackupLocalResult; + offhost: DatabaseBackupOffhostResult; + /** Dumps and manifests retention removed. */ + removed: number; + error: string | null; +} + +export interface RunDatabaseBackupOptions { + now?: Date; + /** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */ + deadline?: number; + env?: NodeJS.ProcessEnv; + /** A bucket double for tests; the configured bucket otherwise. */ + s3?: S3Like; +} + +/** The running build's version, as `/api/version` answers it. */ +function appVersion(env: NodeJS.ProcessEnv): string { + return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version; +} + +/** An error as one line for the ledger and the admin page. */ +function reason(err: unknown): string { + const text = err instanceof Error ? err.message : String(err); + return text.replace(/\s+/g, " ").trim().slice(0, 500); +} + +/** Upload one local file as an off-host object, streamed. */ +async function uploadFile( + s3: S3Like, + key: Buffer, + path: string, +): Promise { + return uploadEncryptedBackup( + s3, + offhostObjectKey(`${basename(path)}.enc`), + key, + async (write) => { + for await (const chunk of createReadStream(path)) { + await write(chunk as Buffer); + } + }, + ); +} + +/** + * Take tonight's dump, keep it, copy it off-host, prune the old ones, and + * record what happened. Never throws: a failure is the report's `status`. + */ +export async function runDatabaseBackup( + prisma: PrismaClient, + options: RunDatabaseBackupOptions = {}, +): Promise { + const env = options.env ?? process.env; + const now = options.now ?? new Date(); + const dir = resolveBackupDir(env); + const localWritable = isWritableDirectory(dir); + const offhostConfig = (() => { + try { + return loadOffhostConfig(); + } catch { + // A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the + // same value with their own message; here it reads as an upload error. + return "invalid" as const; + } + })(); + + // Each leg reads "failed" until it has succeeded, so a run the process died + // under says so once its expiry has passed. + const report: DatabaseBackupReport = { + status: "failed", + fileName: null, + dumpBytes: null, + local: localWritable ? "failed" : "no_directory", + offhost: offhostConfig ? "failed" : "not_configured", + removed: 0, + error: null, + }; + const run = await prisma.databaseBackupRun.create({ + data: { + startedAt: now, + status: "running", + localResult: report.local, + offhostResult: report.offhost, + }, + select: { id: true }, + }); + + let scratch: string | null = null; + try { + if (!databaseBackupEnabled(env)) { + report.status = "skipped"; + report.local = "skipped"; + report.offhost = "skipped"; + report.error = "DB_BACKUP_ENABLED is false"; + } else if (!localWritable && !offhostConfig) { + report.status = "skipped"; + report.error = `no writable backup directory (${dir}) and no off-host bucket configured`; + } else { + const target = localWritable + ? dir + : (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-"))); + const dump = await createDatabaseDump({ + databaseUrl: env.DATABASE_URL ?? "", + dir: target, + kind: "daily", + now, + env, + appVersion: appVersion(env), + buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null, + deadline: options.deadline, + }); + if (localWritable) report.local = "written"; + report.fileName = basename(dump.dumpPath); + report.dumpBytes = dump.manifest.dump.bytes; + + let uploadError: string | null = null; + if (offhostConfig === "invalid") { + uploadError = + "BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64"; + } else if (offhostConfig) { + try { + const s3 = options.s3 ?? (await getS3Client(offhostConfig)); + // The manifest goes up first: a dump object in the bucket always + // has the manifest a restore checks it against. + await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath); + await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath); + report.offhost = "uploaded"; + } catch (err) { + uploadError = `off-host upload failed: ${reason(err)}`; + } + } + + if (localWritable) { + const removed = await applyRetention(dir, { + dailyKeep: resolveDailyKeep(env), + now: now.getTime(), + }); + report.removed = removed.length; + } + + // Good when the dump exists somewhere it survives: on the volume, or in + // the bucket. An upload that failed beside a kept dump still fails the + // run, because the operator asked for both. + report.status = uploadError ? "failed" : "succeeded"; + report.error = uploadError; + } + } catch (err) { + report.status = "failed"; + report.error = reason(err); + } finally { + if (scratch) await rm(scratch, { recursive: true, force: true }); + } + + await prisma.databaseBackupRun.update({ + where: { id: run.id }, + data: { + finishedAt: new Date(), + status: report.status, + fileName: report.fileName, + dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes), + localResult: report.local, + offhostResult: report.offhost, + error: report.error, + }, + }); + await prisma.databaseBackupRun.deleteMany({ + where: { + startedAt: { + lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000), + }, + }, + }); + + annotate({ + action: { name: "database.backup.run" }, + meta: { + status: report.status, + local: report.local, + offhost: report.offhost, + dump_bytes: report.dumpBytes ?? 0, + removed: report.removed, + }, + }); + return report; +} diff --git a/src/lib/jobs/database-backup-queue.ts b/src/lib/jobs/database-backup-queue.ts new file mode 100644 index 000000000..5285095eb --- /dev/null +++ b/src/lib/jobs/database-backup-queue.ts @@ -0,0 +1,40 @@ +/** + * The queue facts of the nightly whole-database dump: its name, its schedule + * and its send options. Apart from the job so the worker registry can read + * them without loading it (`job-registry-graph.test.ts`). + */ +import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; +import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs"; + +export const DATABASE_BACKUP_QUEUE = "database-backup"; + +/** + * `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the + * zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30, + * an hour before the off-host account copies so the two uploads do not meet. + * A value that is not a five-field cron keeps the default, and the boot + * readiness summary says so. + */ +export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron; + +/** + * Four hours. `pg_dump` of a large instance takes minutes, the upload of the + * dump about as long as the dump; the window is there to be impossible to hit + * for a working run. The handler kills `pg_dump` at three quarters of it + * (`jobDeadline`), so a hung dump ends as a failed run with a reason rather + * than an expired job. + */ +export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; + +/** + * What the nightly schedule sends with: the expiry, the backups' heartbeat, + * and one retry. A dump that failed for a passing reason (a lock it waited + * out, a bucket that answered 503) gets a second attempt; one that fails + * again waits for the next night, and the failure stays on the admin page. + */ +export const DATABASE_BACKUP_SEND_OPTIONS = { + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, + retryLimit: 1, + retryDelay: 15 * 60, +} as const; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 250838b7d..ab79d94c1 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "continued", "conversation_titles_migrated", "created", + // The nightly whole-database dump: where it went and how big. + "database_backup_bytes", + "database_backup_local", + "database_backup_offhost", + "database_backup_removed", + "database_backup_status", "daily_rows_retired", "days_consolidated", "days_rebuilt", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index cf6fbf1ba..f34ded204 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -494,9 +494,9 @@ function createEnvelopeStream( return { stream, bytes: () => written }; } -/** Produces the backup JSON in pieces. Every piece is written in order. */ +/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */ export type BackupJsonProducer = ( - write: (chunk: string) => Promise, + write: (chunk: string | Buffer) => Promise, ) => Promise; export interface UploadBackupOptions { @@ -572,9 +572,9 @@ export async function uploadEncryptedBackup( }, ); - const write = async (chunk: string): Promise => { + const write = async (chunk: string | Buffer): Promise => { if (failure) throw failure; - if (gzip.write(chunk, "utf8")) return; + if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding await new Promise((resolve, reject) => { const onDrain = (): void => { gzip.off("error", onError); diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 28ed26810..9a2503c1b 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key- import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run"; import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue"; import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue"; @@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly> = { exclusive: "lockedPass", why: "Serialises every account's whole record.", }, + "database-backup": { + runtime: "long", + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"], + stop: { + file: "lib/jobs/reminder/backup-handlers.ts", + fn: "handleDatabaseBackup", + }, + exclusive: "lockedPass", + why: "Runs pg_dump over the whole database and uploads the dump off-host.", + }, "data-backup-offhost": { runtime: "long", expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 70429c156..6c84a3eaf 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -1,5 +1,6 @@ /** - * The housekeeping queues: backups (weekly in-database, nightly off-host), + * The housekeeping queues: backups (the nightly whole-database dump, weekly + * in-database account copies, nightly off-host), * the restore drill and restores, every ledger and retention cleanup, the * medication maintenance passes, the imports (Apple Health, Health Connect, * medication intake), the converging encryption backfills, the document vault @@ -47,6 +48,11 @@ import { DATA_BACKUP_QUEUE, DATA_BACKUP_SEND_OPTIONS, } from "@/lib/jobs/data-backup-policy"; +import { + DATABASE_BACKUP_CRON, + DATABASE_BACKUP_QUEUE, + DATABASE_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/database-backup-queue"; import { DAWARICH_SYNC_CRON, DAWARICH_SYNC_QUEUE, @@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [ lane: true, load: async () => (await backups()).handleDataBackup, }), + // The whole-database dump: the copy that rebuilds a lost server. Locked, so + // a retry that starts while a run is still dumping waits its turn. + defineJob({ + queue: DATABASE_BACKUP_QUEUE, + cron: DATABASE_BACKUP_CRON, + scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + load: async () => (await backups()).handleDatabaseBackup, + }), prune( RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index c8bc8fa83..845f03927 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -5,7 +5,7 @@ * schedules, and boss.work registrations. */ import { type Job, type JobWithMetadata } from "pg-boss"; -import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget"; import { recordError } from "@/lib/jobs/worker-status"; import { BackupBlobTooLargeError, @@ -31,6 +31,37 @@ import { } from "@/lib/jobs/backup-pass-attempts"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +/** + * The nightly whole-database dump (`database-backup`). + * + * One `pg_dump` in a child process, so it is not in the heavy lane: the heap + * holds a part of the upload at a time, not the database. `pg_dump` is stopped + * at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs + * on a lock ends as a failed run with its reason rather than an expired job. + * A skipped run (switched off, nowhere to put it) completes; the admin page + * and the boot summary carry why. + */ +export async function handleDatabaseBackup( + jobs: Job>[], +): Promise { + return withBackgroundEvent("job.database_backup", async () => { + const { runDatabaseBackup } = await import("@/lib/database-backup/run"); + const report = await runDatabaseBackup(getWorkerPrisma(), { + deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined, + }); + const did = { + database_backup_status: report.status, + database_backup_local: report.local, + database_backup_offhost: report.offhost, + database_backup_bytes: report.dumpBytes ?? 0, + database_backup_removed: report.removed, + }; + return report.status === "failed" + ? jobFailed("database dump failed", report.error ?? undefined, did) + : jobDone(did); + }); +} + export interface DataBackupPayload { triggeredAt: string; } diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index d90560c84..c95d658a6 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -99,6 +99,7 @@ const subsystemSurface = { "audit-log-cleanup": { audience: "system" }, "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, + "database-backup": { audience: "system" }, "data-restore-drill": { audience: "system" }, "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, diff --git a/tests/integration/__golden__/queue-registration.json b/tests/integration/__golden__/queue-registration.json index d85705a21..31a41afe6 100644 --- a/tests/integration/__golden__/queue-registration.json +++ b/tests/integration/__golden__/queue-registration.json @@ -1,5 +1,5 @@ { - "registeredQueueCount": 122, + "registeredQueueCount": 123, "registeredQueues": [ "achievement-unlock-sweep", "apple-health-import", @@ -23,6 +23,7 @@ "data-backup", "data-backup-offhost", "data-restore-drill", + "database-backup", "dawarich-sync", "dense-intraday-hourly-rebuild", "dense-intraday-retention", @@ -455,6 +456,21 @@ "retry_delay_max": null, "retry_limit": 2 }, + { + "dead_letter": null, + "deletion_seconds": 604800, + "expire_seconds": 900, + "heartbeat_seconds": null, + "name": "database-backup", + "notify": false, + "partition": false, + "policy": "standard", + "retention_seconds": 1209600, + "retry_backoff": false, + "retry_delay": 0, + "retry_delay_max": null, + "retry_limit": 2 + }, { "dead_letter": null, "deletion_seconds": 604800, @@ -2137,6 +2153,21 @@ }, "timezone": "Europe/Berlin" }, + { + "cron": "30 1 * * *", + "data": {}, + "key": "", + "kind": "cron", + "name": "database-backup", + "options": { + "expireInSeconds": 14400, + "heartbeatSeconds": 300, + "retryDelay": 900, + "retryLimit": 1, + "tz": "Europe/Berlin" + }, + "timezone": "Europe/Berlin" + }, { "cron": "20 3 * * *", "data": {}, @@ -3022,6 +3053,13 @@ "pollingIntervalSeconds": 30 } }, + { + "queue": "database-backup", + "options": { + "localConcurrency": 1, + "pollingIntervalSeconds": 30 + } + }, { "queue": "dawarich-sync", "options": { diff --git a/tests/integration/database-backup.test.ts b/tests/integration/database-backup.test.ts new file mode 100644 index 000000000..58e778188 --- /dev/null +++ b/tests/integration/database-backup.test.ts @@ -0,0 +1,511 @@ +/** + * The nightly whole-database dump against a real Postgres: the job writes a + * dump and its manifest, records the run, copies both off-host in the + * envelope the restore opens, keeps only what retention allows; and the + * restore puts the dump into an empty database and refuses one with tables. + * + * `pg_dump` and `pg_restore` are the test container's own. A workstation may + * have none, or one of another major than the server (which `pg_dump` + * refuses), so the test puts two small forwarders on PATH that run the tool + * inside the database container with `docker exec`. Everything else, the + * snapshot, the counts, the manifest, the ledger and the restore's checks, + * runs exactly as in the image. + */ +import { execFileSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; + +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +import { prisma } from "@/lib/db"; +import { getDatabaseBackupHealth } from "@/lib/database-backup/health"; +import { runDatabaseBackup } from "@/lib/database-backup/run"; +import { + connect, + countRows, + decryptOffhostFile, + offhostObjectKey, + readManifestFile, +} from "@/lib/database-backup/dump-core.mjs"; + +import { main as restoreMain } from "../../scripts/db-restore.mjs"; +import { shippedMigrations } from "../../scripts/pending-migrations.mjs"; + +const OFFHOST_KEY = "e".repeat(64); +const work = mkdtempSync(join(tmpdir(), "hl-db-backup-")); +const backups = join(work, "backups"); +const shims = join(work, "bin"); +const originalPath = process.env.PATH; + +/** The id of the container that serves DATABASE_URL. */ +function containerId(): string { + const port = new URL(process.env.DATABASE_URL ?? "").port; + const rows = execFileSync( + "docker", + ["ps", "--format", "{{.ID}} {{.Ports}}"], + { + encoding: "utf8", + }, + ).split("\n"); + const row = rows.find((r) => r.includes(`:${port}->5432/tcp`)); + if (!row) throw new Error(`no container publishes port ${port}`); + return row.split(" ")[0]; +} + +/** + * `pg_dump` / `pg_restore` that run inside the database container. The + * connection comes from the PG* variables the core sets; host and port become + * the container's own. A file argument that exists on this machine is sent on + * standard input, which both tools read when no file is named. + */ +function writeShims(container: string): void { + for (const tool of ["pg_dump", "pg_restore"]) { + const path = join(shims, tool); + writeFileSync( + path, + `#!/usr/bin/env node +const { spawn } = require("node:child_process"); +const { createReadStream, existsSync } = require("node:fs"); +const args = process.argv.slice(2); +let input = null; +if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) { + input = args.pop(); +} +const env = []; +for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) { + if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]); +} +env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432"); +const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], { + stdio: ["pipe", "inherit", "inherit"], +}); +(input ? createReadStream(input) : process.stdin).pipe(child.stdin); +child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0))); +`, + ); + chmodSync(path, 0o755); + } +} + +/** A bucket in memory, with the one call the job's uploader uses. */ +function memoryBucket() { + const objects = new Map(); + return { + objects, + putStream: async (key: string, body: Readable) => { + const chunks: Buffer[] = []; + for await (const c of body) chunks.push(c as Buffer); + objects.set(key, Buffer.concat(chunks)); + }, + }; +} + +function backupEnv(extra: Record = {}): NodeJS.ProcessEnv { + return { ...process.env, DB_BACKUP_DIR: backups, ...extra }; +} + +beforeAll(async () => { + const { mkdirSync } = await import("node:fs"); + mkdirSync(shims, { recursive: true }); + mkdirSync(backups, { recursive: true }); + writeShims(containerId()); + process.env.PATH = `${shims}:${originalPath}`; + for (const name of ["dump-a", "dump-b", "dump-c"]) { + await prisma.user.create({ + data: { username: `${name}-${Date.now()}` }, + }); + } +}); + +afterAll(() => { + process.env.PATH = originalPath; + vi.unstubAllEnvs(); + rmSync(work, { recursive: true, force: true }); +}); + +describe("the nightly database dump", () => { + let dumpFile = ""; + + it("writes a checked dump and manifest, records the run and copies both off-host", async () => { + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid"); + vi.stubEnv("BACKUP_S3_BUCKET", "b"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "s"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY); + const bucket = memoryBucket(); + const now = new Date("2026-10-20T01:30:00Z"); + + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv(), + s3: bucket as never, + }); + expect(report).toMatchObject({ + status: "succeeded", + local: "written", + offhost: "uploaded", + error: null, + fileName: "healthlog-20261020T013000Z-daily.dump", + }); + dumpFile = join(backups, report.fileName ?? ""); + + // Both files, under their final names, and nothing half-written. + expect(readdirSync(backups).sort()).toEqual([ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]); + expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe( + "PGDMP", + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261020T013000Z-daily.manifest.json"), + ); + expect(manifest.kind).toBe("daily"); + expect(manifest.dump.bytes).toBe(report.dumpBytes); + expect(manifest.latestMigration).toBe( + shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1), + ); + expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100); + expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0); + expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/); + + // The ledger row and what the admin page reads from it. + const runs = await prisma.databaseBackupRun.findMany({ + where: { startedAt: now }, + }); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ + status: "succeeded", + fileName: "healthlog-20261020T013000Z-daily.dump", + localResult: "written", + offhostResult: "uploaded", + error: null, + }); + expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes); + const health = await getDatabaseBackupHealth( + new Date("2026-10-20T02:00:00Z"), + backupEnv(), + ); + expect(health).toMatchObject({ + enabled: true, + localDirWritable: true, + offhostConfigured: true, + lastGood: { + fileName: "healthlog-20261020T013000Z-daily.dump", + sizeBytes: manifest.dump.bytes, + local: true, + offhost: true, + }, + }); + + // The off-host pair opens, with the restore's own reader, to the very + // bytes on disk. + expect([...bucket.objects.keys()].sort()).toEqual([ + "database/healthlog-20261020T013000Z-daily.dump.enc", + "database/healthlog-20261020T013000Z-daily.manifest.json.enc", + ]); + const opened = join(work, "opened"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(opened); + for (const name of [ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]) { + writeFileSync( + join(opened, `${name}.enc`), + bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer, + ); + await decryptOffhostFile({ + inPath: join(opened, `${name}.enc`), + outPath: join(opened, name), + objectKey: offhostObjectKey(`${name}.enc`), + keys: [Buffer.from(OFFHOST_KEY, "hex")], + }); + expect( + readFileSync(join(opened, name)).equals( + readFileSync(join(backups, name)), + ), + ).toBe(true); + } + vi.unstubAllEnvs(); + }); + + it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => { + for (const day of ["21", "22", "23"]) { + const report = await runDatabaseBackup(prisma, { + now: new Date(`2026-10-${day}T01:30:00Z`), + env: backupEnv({ DB_BACKUP_KEEP: "2" }), + }); + expect(report.status).toBe("succeeded"); + expect(report.offhost).toBe("not_configured"); + } + expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([ + "healthlog-20261022T013000Z-daily.dump", + "healthlog-20261023T013000Z-daily.dump", + ]); + }); + + it("records a failed dump, leaves nothing that looks good, and the page says so", async () => { + const before = readdirSync(backups).sort(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.password = "not-the-password"; + const now = new Date("2026-10-24T01:30:00Z"); + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv({ DATABASE_URL: url.toString() }), + }); + expect(report.status).toBe("failed"); + expect(report.local).toBe("failed"); + expect(report.error).toMatch(/password authentication failed/); + expect(readdirSync(backups).sort()).toEqual(before); + const health = await getDatabaseBackupHealth( + new Date("2026-10-24T02:00:00Z"), + backupEnv(), + ); + expect(health.lastFailure?.error).toMatch(/password authentication/); + expect(health.lastGood?.fileName).toBe( + "healthlog-20261023T013000Z-daily.dump", + ); + }); + + it("skips, and says why, with nowhere to put a dump", async () => { + const report = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T03:00:00Z"), + env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }), + }); + expect(report).toMatchObject({ + status: "skipped", + local: "no_directory", + offhost: "not_configured", + }); + expect(report.error).toMatch(/no writable backup directory/); + const off = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T04:00:00Z"), + env: backupEnv({ DB_BACKUP_ENABLED: "false" }), + }); + expect(off).toMatchObject({ + status: "skipped", + local: "skipped", + offhost: "skipped", + }); + }); + + describe("restoring it", () => { + const target = `restore_${process.pid}_${Date.now()}`; + let targetUrl = ""; + + beforeAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`CREATE DATABASE ${target}`); + await admin.end(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.pathname = `/${target}`; + targetUrl = url.toString(); + }); + + afterAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`); + await admin.end(); + }); + + it("refuses the database the dump came from, which has tables", async () => { + const errors: string[] = []; + const spy = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const code = await restoreMain(["latest"], backupEnv()); + spy.mockRestore(); + expect(code).toBe(1); + expect(errors.join("\n")).toMatch(/only into an empty database/); + }); + + it("restores the newest dump into an empty database with every row count", async () => { + const lines: string[] = []; + const spy = vi + .spyOn(console, "log") + .mockImplementation((l: string) => void lines.push(String(l))); + const code = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + spy.mockRestore(); + expect(code, lines.join("\n")).toBe(0); + expect(lines.join("\n")).toMatch( + /all \d+ tables hold the row counts the manifest names/, + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261023T013000Z-daily.manifest.json"), + ); + const client = await connect(targetUrl); + try { + expect(await countRows(client)).toEqual(manifest.rowCounts); + const { rows } = await client.query( + "SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'", + ); + expect(rows[0].n).toBeGreaterThanOrEqual(3); + } finally { + await client.end(); + } + // Restoring again into the now full database is refused. + const again = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + expect(again).toBe(1); + expect(existsSync(dumpFile)).toBe(false); // the first dump aged out + }); + }); +}); + +describe("the dump before migrating", () => { + const preDir = join(work, "pre"); + const appDir = join(work, "app"); + const failing = join(work, "failing-bin"); + let cwd = ""; + + beforeAll(async () => { + const { cpSync, mkdirSync } = await import("node:fs"); + // An image that ships one migration the database has not applied. + mkdirSync(join(appDir, "prisma"), { recursive: true }); + cpSync( + join(process.cwd(), "prisma", "migrations"), + join(appDir, "prisma", "migrations"), + { recursive: true }, + ); + mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet")); + writeFileSync( + join( + appDir, + "prisma", + "migrations", + "9999_not_applied_yet", + "migration.sql", + ), + "", + ); + mkdirSync(preDir, { recursive: true }); + mkdirSync(failing, { recursive: true }); + writeFileSync( + join(failing, "pg_dump"), + "#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n", + ); + chmodSync(join(failing, "pg_dump"), 0o755); + cwd = process.cwd(); + process.chdir(appDir); + }); + + afterAll(() => { + process.chdir(cwd); + process.env.PATH = `${shims}:${originalPath}`; + }); + + async function preMigration(env: Record) { + const { preMigrationDump } = await import("../../scripts/db-dump.mjs"); + const errors: string[] = []; + const spyErr = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const spyLog = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + const code = await preMigrationDump({ ...process.env, ...env }); + return { code, errors: errors.join("\n") }; + } finally { + spyErr.mockRestore(); + spyLog.mockRestore(); + } + } + + it("writes a checked dump with its manifest, and keeps the last two", async () => { + for (let i = 0; i < 3; i++) { + if (i > 0) await new Promise((r) => setTimeout(r, 1100)); + const { code } = await preMigration({ DB_BACKUP_DIR: preDir }); + expect(code).toBe(0); + } + const names = readdirSync(preDir).sort(); + expect(names).toHaveLength(4); + expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength( + 2, + ); + const manifest = await readManifestFile( + join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""), + ); + expect(manifest.kind).toBe("pre-migration"); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + }); + + it("does nothing when switched off", async () => { + const before = readdirSync(preDir); + const { code } = await preMigration({ + DB_BACKUP_DIR: preDir, + DB_BACKUP_BEFORE_MIGRATE: "false", + }); + expect(code).toBe(0); + expect(readdirSync(preDir)).toEqual(before); + }); + + it("warns and lets the boot go on without a writable directory", async () => { + const { code, errors } = await preMigration({ + DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here", + }); + expect(code).toBe(0); + expect(errors).toMatch( + /is not writable, so no dump is taken before the 1 pending migration/, + ); + }); + + it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => { + // A second of its own, so the name cannot meet the last dump's. + await new Promise((r) => setTimeout(r, 1100)); + const before = readdirSync(preDir).sort(); + process.env.PATH = `${failing}:${originalPath}`; + const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir }); + process.env.PATH = `${shims}:${originalPath}`; + expect(code).toBe(1); + expect(errors).toMatch( + /the database dump before migrating failed: .*No space left/, + ); + expect(readdirSync(preDir).sort()).toEqual(before); + }); +}); + +describe("two dumps in the same second", () => { + it("refuses the second and leaves the first whole", async () => { + const { createDatabaseDump } = + await import("@/lib/database-backup/dump-core.mjs"); + const dir = join(work, "same-second"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(dir); + const now = new Date("2026-10-26T01:30:00Z"); + const options = { + databaseUrl: process.env.DATABASE_URL ?? "", + dir, + kind: "daily" as const, + now, + }; + await createDatabaseDump(options); + const before = readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]); + await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/); + expect( + readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]), + ).toEqual(before); + }); +});