From 6d3a1582fa3b75224d75254a950c4fb53b50ca19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:51:39 +0100 Subject: [PATCH 01/10] feat(backup): take a nightly whole-database dump with a manifest The per-account copies roll one account back on the same server; nothing the app made could rebuild a lost one. A scheduled job now runs pg_dump over the whole database (every schema, pg-boss included) into DB_BACKUP_DIR as healthlog--daily.dump, with a manifest beside it: app version and build, newest migration, Postgres version, size and SHA-256, exact row counts of every table, and the fingerprints of the keys the database needs. The counts read the same exported snapshot pg_dump reads, so a dump taken while the app writes still matches its manifest. The archive is written as .partial, listed back with pg_restore --list, and renamed after its manifest; a failed run leaves nothing named like a good dump. Retention removes only files this job wrote, oldest first, and never the newest good one (DB_BACKUP_KEEP, default 7; two pre-migration dumps). With the off-host bucket configured, each dump and its manifest go up under database/ in the envelope the account copies use, streamed. Every run lands in the new database_backup_runs ledger (migration 0486), and getDatabaseBackupHealth() answers the last good dump, the last failure, whether the directory is writable and whether off-host is configured. The boot readiness summary says where dumps go, or that none can be kept. The dump, naming, retention and restore rules live in one plain ESM module so the image's own scripts can share them without tsx. --- .../0486_database_backup_runs/migration.sql | 30 + prisma/schema.prisma | 29 + .../boot/__tests__/readiness-summary.test.ts | 53 + src/lib/boot/readiness-summary.ts | 78 +- src/lib/data-wipe/wipe-plan.ts | 2 + .../__tests__/dump-core.test.ts | 630 +++++++++ src/lib/database-backup/dump-core.mjs | 1183 +++++++++++++++++ src/lib/database-backup/health.ts | 117 ++ src/lib/database-backup/run.ts | 243 ++++ src/lib/jobs/database-backup-queue.ts | 40 + src/lib/jobs/job-outcome.ts | 6 + src/lib/jobs/offhost-backup.ts | 8 +- src/lib/jobs/queue-runtime.ts | 12 + src/lib/jobs/registry/maintenance.ts | 18 +- src/lib/jobs/reminder/backup-handlers.ts | 33 +- src/lib/jobs/subsystem-surface.ts | 1 + .../__golden__/queue-registration.json | 40 +- tests/integration/database-backup.test.ts | 511 +++++++ 18 files changed, 3025 insertions(+), 9 deletions(-) create mode 100644 prisma/migrations/0486_database_backup_runs/migration.sql create mode 100644 src/lib/database-backup/__tests__/dump-core.test.ts create mode 100644 src/lib/database-backup/dump-core.mjs create mode 100644 src/lib/database-backup/health.ts create mode 100644 src/lib/database-backup/run.ts create mode 100644 src/lib/jobs/database-backup-queue.ts create mode 100644 tests/integration/database-backup.test.ts diff --git a/prisma/migrations/0486_database_backup_runs/migration.sql b/prisma/migrations/0486_database_backup_runs/migration.sql new file mode 100644 index 000000000..37bd430af --- /dev/null +++ b/prisma/migrations/0486_database_backup_runs/migration.sql @@ -0,0 +1,30 @@ +-- The nightly whole-database dump records each run: whether a dump was +-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not. +-- Instance metadata with no account; the admin backups card and the backup +-- alerts read the newest rows. +-- +-- The table is new, so nothing waits on its lock in practice; the wait is +-- bounded in a DO block anyway, and every statement is safe to run again, so +-- the boot retries the file on a lock timeout. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + CREATE TABLE IF NOT EXISTS "database_backup_runs" ( + "id" TEXT NOT NULL, + "started_at" TIMESTAMP(3) NOT NULL, + "finished_at" TIMESTAMP(3), + "status" TEXT NOT NULL, + "file_name" TEXT, + "dump_bytes" BIGINT, + "local_result" TEXT NOT NULL, + "offhost_result" TEXT NOT NULL, + "error" TEXT, + CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id") + ); + CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx" + ON "database_backup_runs"("started_at"); +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8397e0cb5..ae2d0c292 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5929,6 +5929,35 @@ model OffhostPurgeRequest { @@map("offhost_purge_requests") } +/// One run of the nightly whole-database dump (the `database-backup` job): +/// whether a dump was written on this server and copied off-host, and why not +/// when it was not. Instance metadata, not an account's data. Written by +/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the +/// database row of the admin backups card and the backup alerts. A row left +/// `running` by a process that died is read as a failed run once the job's +/// expiry has passed. +model DatabaseBackupRun { + id String @id @default(cuid()) + startedAt DateTime @map("started_at") + finishedAt DateTime? @map("finished_at") + /// `running`, `succeeded`, `failed` or `skipped`. + status String + /// The dump's file name, once one was written and checked. + fileName String? @map("file_name") + /// Size of the dump in bytes. + dumpBytes BigInt? @map("dump_bytes") + /// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable + /// directory, the dump went off-host only), or `failed`. + localResult String @map("local_result") + /// `uploaded`, `not_configured` or `failed`. + offhostResult String @map("offhost_result") + /// Why the run failed or was skipped, trimmed. Never key material. + error String? + + @@index([startedAt]) + @@map("database_backup_runs") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/src/lib/boot/__tests__/readiness-summary.test.ts b/src/lib/boot/__tests__/readiness-summary.test.ts index fc23b92ee..003ade157 100644 --- a/src/lib/boot/__tests__/readiness-summary.test.ts +++ b/src/lib/boot/__tests__/readiness-summary.test.ts @@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => { expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined(); }); }); + +describe("collectReadiness — the database dump", () => { + const offhost = { + BACKUP_S3_ENDPOINT: "https://s3.example.test", + BACKUP_S3_BUCKET: "b", + BACKUP_S3_ACCESS_KEY: "a", + BACKUP_S3_SECRET_KEY: "s", + BACKUP_ENCRYPTION_KEY: HEX32, + }; + const dumpLine = ( + env: Record, + writable: boolean, + ) => + collectReadiness(env, () => writable).lines.find( + (l) => l.label === "Database dump", + ); + + it("names the schedule, the directory and the copies kept", () => { + const line = dumpLine(baseEnv(), true); + expect(line?.status).toBe("ok"); + expect(line?.detail).toBe( + "nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7", + ); + expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain( + "copied off-host", + ); + }); + + it("warns when nothing can be kept on the server", () => { + expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("goes off-host only"), + }); + expect(dumpLine(baseEnv(), false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("no database dump can be taken"), + }); + }); + + it("warns when it is switched off, and names a schedule it ignored", () => { + expect( + dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true), + ).toMatchObject({ status: "warn" }); + const report = collectReadiness( + { ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" }, + () => true, + ); + expect( + report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail, + ).toBe("not a five-field cron; using 30 1 * * *"); + expect(report.hasBlocker).toBe(false); + }); +}); diff --git a/src/lib/boot/readiness-summary.ts b/src/lib/boot/readiness-summary.ts index 14e707157..e8907cb29 100644 --- a/src/lib/boot/readiness-summary.ts +++ b/src/lib/boot/readiness-summary.ts @@ -19,6 +19,13 @@ import { LEGACY_ANY_HOST_WARNING, legacyAnyHostConfigured, } from "@/lib/ai/local-host-allowlist"; +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, + resolveBackupSchedule, + resolveDailyKeep, +} from "@/lib/database-backup/dump-core.mjs"; export interface ReadinessLine { /** Short label, e.g. "ENCRYPTION_KEY". */ @@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean { return keys.some((k) => (env[k] ?? "").trim().length > 0); } -/** Inspect the environment and build the readiness report. Pure + side-effect free. */ -export function collectReadiness(env: Env = process.env): ReadinessReport { +/** + * The nightly database dump's line. The dump is what rebuilds a lost server, + * so a host where it cannot be kept anywhere is said out loud at every boot, + * and so is one where it only goes off-host. + */ +export function databaseBackupReadiness( + env: Env, + writable: (dir: string) => boolean, +): ReadinessLine[] { + const label = "Database dump"; + if (!databaseBackupEnabled(env)) { + return [ + { + label, + status: "warn", + detail: + "DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way", + }, + ]; + } + const lines: ReadinessLine[] = []; + const schedule = resolveBackupSchedule(env); + if (!schedule.valid) { + lines.push({ + label: "DB_BACKUP_SCHEDULE", + status: "warn", + detail: `not a five-field cron; using ${schedule.cron}`, + }); + } + const dir = resolveBackupDir(env); + const offhost = allSet(env, [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]); + if (writable(dir)) { + lines.push({ + label, + status: "ok", + detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`, + }); + } else if (offhost) { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } else { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } + return lines; +} + +/** + * Inspect the environment and build the readiness report. Pure apart from + * `writable`, the probe of the dump directory, which tests replace. + */ +export function collectReadiness( + env: Env = process.env, + writable: (dir: string) => boolean = isWritableDirectory, +): ReadinessReport { const lines: ReadinessLine[] = []; // --- Core secrets (boot blockers) --------------------------------------- @@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport { }); } + lines.push(...databaseBackupReadiness(env, writable)); + // --- Optional subsystems (informational) -------------------------------- const optional: Array<[string, boolean]> = [ ["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])], diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 92cead915..211e00af6 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly> = { HostMetric: "instance-wide host telemetry, not attributable to an account", OffhostBackupKeyUse: "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + DatabaseBackupRun: + "the instance's ledger of nightly whole-database dumps; describes the server, not any account", OffhostPurgeRequest: "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: diff --git a/src/lib/database-backup/__tests__/dump-core.test.ts b/src/lib/database-backup/__tests__/dump-core.test.ts new file mode 100644 index 000000000..fad98e4a3 --- /dev/null +++ b/src/lib/database-backup/__tests__/dump-core.test.ts @@ -0,0 +1,630 @@ +/** + * The rules of the whole-database dump that need no database: names, + * retention, the manifest, key fingerprints, row-count comparison, the + * connection the client tools get, and the off-host envelope read back by the + * restore's own reader. + * + * The fingerprints and the envelope are checked against the app's own code + * (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy + * of the rule: the restore script cannot import TypeScript, so these are the + * tests that keep its plain-JS reading of both from drifting. + */ +import { + mkdtempSync, + createReadStream, + rmSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + applyRetention, + buildManifest, + compareKeyFingerprints, + compareRowCounts, + connectionFromUrl, + decodeKeyMaterial, + decryptOffhostFile, + dumpBaseName, + formatDumpTimestamp, + keyFingerprints, + listDumpSets, + offhostObjectKey, + parseDumpFileName, + parseManifest, + resolveBackupSchedule, + resolveDailyKeep, + selectForDeletion, + type DumpSet, +} from "../dump-core.mjs"; +import type { S3Like } from "@/lib/jobs/offhost-backup"; + +/** Every byte of a stream, in order. */ +async function collect(stream: AsyncIterable): Promise { + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array)); + return Buffer.concat(chunks); +} + +const HEX_A = "a".repeat(64); +const HEX_B = "b".repeat(64); +const HEX_C = "c".repeat(64); +const HEX_D = "d".repeat(64); + +const dirs: string[] = []; +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-")); + dirs.push(dir); + return dir; +} +afterEach(() => { + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); + vi.unstubAllEnvs(); +}); + +describe("dump names", () => { + it("names a dump by its UTC instant and kind", () => { + const at = new Date("2026-10-25T01:30:07.123Z"); + expect(formatDumpTimestamp(at)).toBe("20261025T013007Z"); + expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily"); + expect(dumpBaseName("pre-migration", at)).toBe( + "healthlog-20261025T013007Z-pre-migration", + ); + }); + + it("parses every file it writes, and back to the same instant", () => { + const base = "healthlog-20261025T013007Z-daily"; + const cases: Array<[string, string, string]> = [ + [`${base}.dump`, "dump", "final"], + [`${base}.manifest.json`, "manifest", "final"], + [`${base}.dump.partial`, "dump", "partial"], + [`${base}.manifest.json.partial`, "manifest", "partial"], + [`${base}.dump.enc`, "dump", "encrypted"], + [`${base}.manifest.json.enc`, "manifest", "encrypted"], + ]; + for (const [name, role, variant] of cases) { + const parsed = parseDumpFileName(name); + expect(parsed, name).not.toBeNull(); + expect(parsed?.base).toBe(base); + expect(parsed?.kind).toBe("daily"); + expect(parsed?.role).toBe(role); + expect(parsed?.variant).toBe(variant); + expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z"); + } + expect( + parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind, + ).toBe("pre-migration"); + }); + + it("refuses every name it did not write", () => { + for (const name of [ + "healthlog.dump", + "backup-20261025T013007Z-daily.dump", + "healthlog-20261025T013007Z-weekly.dump", + "healthlog-20261025-daily.dump", + "healthlog-20261025T013007Z-daily.sql", + "healthlog-20261025T013007Z-daily.dump.bak", + "healthlog-20261399T013007Z-daily.dump", + "notes.txt", + ]) { + expect(parseDumpFileName(name), name).toBeNull(); + } + }); +}); + +describe("configuration", () => { + it("keeps at least one daily dump and ignores what is not a number", () => { + expect(resolveDailyKeep({})).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7); + }); + + it("takes a five-field cron and says when it ignored one", () => { + expect(resolveBackupSchedule({})).toEqual({ + cron: "30 1 * * *", + valid: true, + configured: false, + }); + expect( + resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }), + ).toEqual({ + cron: "0 2 * * 1-5", + valid: true, + configured: true, + }); + for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) { + expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({ + cron: "30 1 * * *", + valid: false, + configured: true, + }); + } + }); +}); + +/** A dump set as `listDumpSets` reports it, for the pure retention rule. */ +function set( + kind: "daily" | "pre-migration", + iso: string, + parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {}, + now = Date.parse("2026-10-25T12:00:00Z"), +): DumpSet { + const base = dumpBaseName(kind, new Date(iso)); + return { + base, + kind, + timestamp: new Date(iso), + dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 }, + manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`, + encryptedDump: null, + encryptedManifest: null, + leftovers: + parts.partialAgeMs === undefined + ? [] + : [ + { + path: `/b/${base}.dump.partial`, + mtimeMs: now - parts.partialAgeMs, + }, + ], + }; +} + +describe("retention", () => { + const now = Date.parse("2026-10-25T12:00:00Z"); + + it("keeps the newest N daily dumps and removes the rest, both files each", () => { + const sets = [1, 2, 3, 4, 5].map((d) => + set("daily", `2026-10-2${d}T01:30:00Z`), + ); + const doomed = selectForDeletion(sets, { + keep: { daily: 3, "pre-migration": 2 }, + now, + }); + expect(doomed.sort()).toEqual( + [ + "/b/healthlog-20261021T013000Z-daily.dump", + "/b/healthlog-20261021T013000Z-daily.manifest.json", + "/b/healthlog-20261022T013000Z-daily.dump", + "/b/healthlog-20261022T013000Z-daily.manifest.json", + ].sort(), + ); + }); + + it("never removes the newest good dump, whatever keep says", () => { + const sets = [ + set("daily", "2026-10-24T01:30:00Z"), + set("daily", "2026-10-23T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 0, "pre-migration": 0 }, + now, + }); + expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump"); + expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump"); + }); + + it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => { + const sets = [ + set("daily", "2026-10-25T01:30:00Z", { + dump: false, + partialAgeMs: 60_000, + }), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([]); + }); + + it("keeps two pre-migration dumps apart from the daily count", () => { + const sets = [ + set("pre-migration", "2026-10-20T08:00:00Z"), + set("pre-migration", "2026-10-10T08:00:00Z"), + set("pre-migration", "2026-10-01T08:00:00Z"), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261001T080000Z-pre-migration.dump", + "/b/healthlog-20261001T080000Z-pre-migration.manifest.json", + ]); + }); + + it("removes a .partial only once it is older than twelve hours", () => { + const fresh = set("daily", "2026-10-25T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 60 * 60_000, + }); + const stale = set("daily", "2026-10-23T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 13 * 60 * 60_000, + }); + const doomed = selectForDeletion([fresh, stale], { + keep: { daily: 7, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261023T013000Z-daily.dump.partial", + ]); + }); + + it("works on a real directory and leaves files it did not write alone", async () => { + const dir = tempDir(); + const names: string[] = []; + for (let day = 15; day <= 24; day++) { + const base = `healthlog-202610${day}T013000Z-daily`; + names.push(`${base}.dump`, `${base}.manifest.json`); + } + // Two that are not ours, and an off-host copy an operator put here. + names.push( + "my-own-dump.dump", + "healthlog-20261001T013000Z-daily.dump.bak", + "healthlog-20261002T013000Z-daily.dump.enc", + ); + for (const name of names) writeFileSync(join(dir, name), "x"); + // A crash's leftovers: one old, one from a run that may still be going. + const oldPartial = join( + dir, + "healthlog-20261003T013000Z-daily.dump.partial", + ); + writeFileSync(oldPartial, "x"); + const day = 24 * 60 * 60; + utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day); + writeFileSync( + join(dir, "healthlog-20261025T013000Z-daily.dump.partial"), + "x", + ); + + const removed = await applyRetention(dir, { dailyKeep: 7 }); + expect(removed).toHaveLength(7); + const left = (await readdir(dir)).sort(); + expect(left).toContain("my-own-dump.dump"); + expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak"); + expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc"); + expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial"); + expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial"); + expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual( + [18, 19, 20, 21, 22, 23, 24].map( + (d) => `healthlog-202610${d}T013000Z-daily.dump`, + ), + ); + expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily"); + }); +}); + +describe("key fingerprints", () => { + it("are the fingerprints the app's own key code gives, for the map and the single key", async () => { + vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B })); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2"); + vi.stubEnv("ENCRYPTION_KEY", ""); + const crypto = await import("@/lib/crypto"); + crypto._resetCryptoCacheForTests(); + const prints = keyFingerprints(process.env); + expect(prints.encryption.activeKeyId).toBe("v2"); + expect(prints.encryption.keys).toEqual([ + { id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }, + { id: "v2", fingerprint: crypto.getKeyFingerprint("v2") }, + ]); + + vi.stubEnv("ENCRYPTION_KEYS", ""); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", ""); + const base64 = Buffer.from(HEX_C, "hex").toString("base64"); + vi.stubEnv("ENCRYPTION_KEY", base64); + crypto._resetCryptoCacheForTests(); + const single = keyFingerprints(process.env); + expect(single.encryption).toEqual({ + activeKeyId: "v1", + keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }], + }); + crypto._resetCryptoCacheForTests(); + }); + + it("names the off-host key the way an off-host object records it", async () => { + const { offhostKeyId } = await import("@/lib/jobs/offhost-backup"); + const prints = keyFingerprints({ + ENCRYPTION_KEY: HEX_A, + BACKUP_ENCRYPTION_KEY: HEX_D, + BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`, + API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789", + }); + expect(prints.backupEncryptionKey).toBe( + offhostKeyId(Buffer.from(HEX_D, "hex")), + ); + expect(prints.backupEncryptionPreviousKeys).toEqual([ + offhostKeyId(Buffer.from(HEX_C, "hex")), + ]); + expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/); + }); + + it("never carries key material", () => { + const env = { + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }), + BACKUP_ENCRYPTION_KEY: HEX_B, + API_TOKEN_HMAC_KEY: HEX_C, + }; + const text = JSON.stringify(keyFingerprints(env)); + for (const secret of [HEX_A, HEX_B, HEX_C]) { + expect(text).not.toContain(secret.slice(0, 16)); + } + }); + + it("decodes keys the way the app does, and nothing that is not a key", () => { + expect(decodeKeyMaterial(HEX_A)?.length).toBe(32); + expect( + decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length, + ).toBe(32); + expect(decodeKeyMaterial("")).toBeNull(); + expect(decodeKeyMaterial("not a key!")).toBeNull(); + }); +}); + +describe("comparing a dump's keys with the environment's", () => { + const written = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + + it("finds nothing when every key is there", () => { + const same = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + expect(compareKeyFingerprints(written, same)).toEqual({ + findings: [], + refuse: false, + }); + }); + + it("refuses when the active key is different, and says which", () => { + const wrong = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + const result = compareKeyFingerprints(written, wrong); + expect(result.refuse).toBe(true); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]).toMatchObject({ + key: "ENCRYPTION_KEYS[v2]", + state: "different", + severity: "refuse", + }); + }); + + it("refuses when the active key is missing, warns for a retired one", () => { + const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A }); + const result = compareKeyFingerprints(written, onlyOld); + expect(result.refuse).toBe(true); + const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f])); + expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({ + state: "missing", + severity: "refuse", + }); + expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined(); + expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + + const noRetired = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }), + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-two", + }); + const second = compareKeyFingerprints(written, noRetired); + expect(second.refuse).toBe(false); + expect(second.findings.map((f) => [f.key, f.severity])).toEqual([ + ["ENCRYPTION_KEYS[v1]", "warn"], + ["API_TOKEN_HMAC_KEY", "warn"], + ]); + }); +}); + +describe("the manifest", () => { + const manifest = buildManifest({ + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: new Date("2026-10-25T01:30:00Z"), + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + server: { version: "18.6", versionNum: 180006 }, + pgDump: "pg_dump (PostgreSQL) 18.6", + archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 }, + env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" }, + }); + + it("carries what a restore and an operator need", () => { + expect(manifest).toMatchObject({ + format: "healthlog-database-dump", + formatVersion: 1, + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: "2026-10-25T01:30:00.000Z", + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + postgres: { serverVersion: "18.6", serverVersionNum: 180006 }, + dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + }); + expect(Object.keys(manifest.rowCounts)).toEqual([ + "pgboss.job", + "public.a", + "public.users", + ]); + expect(manifest.keys.encryption.keys[0].id).toBe("v1"); + }); + + it("reads back, and refuses anything else", () => { + expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual( + manifest, + ); + expect(() => parseManifest({ format: "something-else" })).toThrow( + /not a HealthLog/, + ); + expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow(); + expect(() => parseManifest(null)).toThrow(); + }); +}); + +describe("row counts after a restore", () => { + it("names every table that is missing, extra or different", () => { + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2 }, + { "public.a": 1, "public.b": 2 }, + ), + ).toEqual([]); + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2, "public.c": 5 }, + { "public.a": 1, "public.b": 0, "public.d": 1 }, + ), + ).toEqual([ + "public.b: 2 rows in the dump, 0 restored", + "public.c: in the dump with 5 rows, missing now", + "public.d: restored but not in the dump's manifest", + ]); + }); +}); + +describe("the connection the client tools get", () => { + it("drops Prisma's parameters and keeps the password out of the arguments", () => { + const c = connectionFromUrl( + "postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require", + ); + expect(c.pgEnv).toEqual({ + PGHOST: "db", + PGPORT: "6543", + PGUSER: "health@log", + PGPASSWORD: "p@ss/w", + PGDATABASE: "hl", + PGSSLMODE: "require", + }); + expect(c.schema).toBe("app"); + const cs = String(c.clientConfig.connectionString); + expect(cs).not.toContain("connection_limit"); + expect(cs).not.toContain("schema="); + expect(cs).toContain("sslmode=require"); + }); +}); + +describe("an off-host copy read back by the restore", () => { + /** The uploader the job uses, into memory. */ + async function upload(objectKey: string, key: Buffer, body: Buffer) { + const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup"); + const stored = new Map(); + const s3: S3Like = { + putStream: async (k: string, stream: Readable) => { + stored.set(k, await collect(stream)); + }, + putObject: async () => {}, + getObject: async () => Buffer.alloc(0), + headObject: async () => false, + listObjects: async () => [], + deleteObject: async () => {}, + }; + await uploadEncryptedBackup(s3, objectKey, key, async (write) => { + // Several pieces, as the job streams the file. + for (let i = 0; i < body.length; i += 1000) { + await write(body.subarray(i, i + 1000)); + } + }); + return stored.get(objectKey) as Buffer; + } + + it("opens what the job's uploader sealed, byte for byte", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + + await decryptOffhostFile({ + inPath: join(dir, name), + outPath: join(dir, "opened.dump"), + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex"), key], + }); + expect( + (await collect(createReadStream(join(dir, "opened.dump")))).equals(body), + ).toBe(true); + }); + + it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from("PGDMP fake archive bytes".repeat(100)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + const out = join(dir, "opened.dump"); + + // Renamed: the object key is bound as associated data. + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey( + "healthlog-20261024T013000Z-daily.dump.enc", + ), + keys: [key], + }), + ).rejects.toThrow(/does not open/); + expect(await readdir(dir)).toEqual([name]); + + // Altered. + const altered = Buffer.from(sealed); + altered[altered.length - 40] ^= 1; + writeFileSync(join(dir, name), altered); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [key], + }), + ).rejects.toThrow(); + expect(await readdir(dir)).toEqual([name]); + + // A key it was not written under. + writeFileSync(join(dir, name), sealed); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex")], + }), + ).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/); + expect(await readdir(dir)).toEqual([name]); + }); +}); diff --git a/src/lib/database-backup/dump-core.mjs b/src/lib/database-backup/dump-core.mjs new file mode 100644 index 000000000..14f13d2ca --- /dev/null +++ b/src/lib/database-backup/dump-core.mjs @@ -0,0 +1,1183 @@ +/** + * The whole-database dump: naming, writing, checking, keeping and reading it + * back. One module for three callers that cannot share TypeScript: + * + * - the nightly job in the app (`src/lib/database-backup/run.ts`), bundled; + * - the pre-migration dump the entrypoint takes (`scripts/db-dump.mjs`); + * - the restore (`scripts/db-restore.mjs`), which runs in the image with + * plain `node` on a server that has nothing but the dump and the keys. + * + * It is plain ESM with no imports beyond Node and `pg`, for the same reason + * `src/lib/auth/argon2-params.mjs` is: the production image strips `tsx`, and + * a restore that needs a source checkout is not a restore. Every rule that + * decides whether a dump is good, which files may be deleted and whether the + * keys fit lives here once, so the writer and the reader cannot drift. + * + * What a dump is: `pg_dump --format=custom` of the whole database (every + * schema, pg-boss included), named `healthlog--.dump`, + * with `healthlog--.manifest.json` beside it. A dump is + * good only when both exist under their final names: the archive is written + * as `.partial`, read back with `pg_restore --list`, the manifest is written + * and renamed first, and the archive is renamed last. A run that dies at any + * point leaves nothing that looks good. + * + * The row counts in the manifest come from the same snapshot `pg_dump` reads + * (`pg_export_snapshot()` + `--snapshot`), so a dump taken while the app is + * writing still restores to exactly the counts it claims. + */ +import { spawn } from "node:child_process"; +import { createDecipheriv, createHash } from "node:crypto"; +import { + accessSync, + closeSync, + constants as fsConstants, + createReadStream, + createWriteStream, + mkdirSync, + openSync, + readSync, + readdirSync, + statSync, +} from "node:fs"; +import { readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; + +/** Where dumps are written when `DB_BACKUP_DIR` is unset. */ +export const DEFAULT_BACKUP_DIR = "/var/lib/healthlog/backups"; + +/** Daily dumps kept when `DB_BACKUP_KEEP` is unset. */ +export const DEFAULT_DAILY_KEEP = 7; + +/** Pre-migration dumps kept: the one before this update and the one before. */ +export const PRE_MIGRATION_KEEP = 2; + +/** The cron the nightly dump runs on when `DB_BACKUP_SCHEDULE` is unset. */ +export const DEFAULT_BACKUP_SCHEDULE = "30 1 * * *"; + +/** The kinds of dump this module writes. */ +export const DUMP_KINDS = /** @type {const} */ (["daily", "pre-migration"]); + +/** The manifest's own format marker. */ +export const MANIFEST_FORMAT = "healthlog-database-dump"; +export const MANIFEST_FORMAT_VERSION = 1; + +/** The bucket prefix off-host database dumps live under. */ +export const OFFHOST_DATABASE_PREFIX = "database/"; + +/** + * A leftover `.partial` or an orphaned manifest younger than this may belong + * to a run that is still going (another container, the entrypoint), so it is + * left alone. + */ +const STALE_LEFTOVER_MS = 12 * 60 * 60 * 1000; + +const NAME_PATTERN = + /^healthlog-(\d{8}T\d{6}Z)-(daily|pre-migration)\.(dump|manifest\.json)(\.partial|\.enc)?$/; + +/** + * @typedef {"daily" | "pre-migration"} DumpKind + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * role: "dump" | "manifest"; + * variant: "final" | "partial" | "encrypted"; + * }} ParsedDumpName + */ + +/** `20261025T013000Z` for an instant, in UTC. */ +export function formatDumpTimestamp(date) { + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[-:]/g, ""); +} + +/** + * The shared stem of a dump and its manifest. + * + * @param {DumpKind} kind + * @param {Date} date + */ +export function dumpBaseName(kind, date) { + return `healthlog-${formatDumpTimestamp(date)}-${kind}`; +} + +/** @param {string} base */ +export function dumpFileName(base) { + return `${base}.dump`; +} + +/** @param {string} base */ +export function manifestFileName(base) { + return `${base}.manifest.json`; +} + +/** + * Parse a file name this module writes. Anything else answers null, which is + * what keeps retention away from files it did not write. + * + * @param {string} name + * @returns {ParsedDumpName | null} + */ +export function parseDumpFileName(name) { + const m = NAME_PATTERN.exec(name); + if (!m) return null; + const [, stamp, kind, role, suffix] = m; + const iso = `${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`; + const timestamp = new Date(iso); + if (Number.isNaN(timestamp.getTime())) return null; + return { + base: `healthlog-${stamp}-${kind}`, + kind: /** @type {DumpKind} */ (kind), + timestamp, + role: role === "dump" ? "dump" : "manifest", + variant: + suffix === ".partial" + ? "partial" + : suffix === ".enc" + ? "encrypted" + : "final", + }; +} + +// ── Configuration ────────────────────────────────────────────────────────── + +/** @typedef {Record} Env */ + +/** @param {string | undefined} raw */ +function isOff(raw) { + const v = (raw ?? "").trim().toLowerCase(); + return v === "false" || v === "0" || v === "no" || v === "off"; +} + +/** `DB_BACKUP_ENABLED`, on unless it says off. @param {Env} env */ +export function databaseBackupEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_ENABLED); +} + +/** `DB_BACKUP_BEFORE_MIGRATE`, on unless it says off. @param {Env} env */ +export function preMigrationDumpEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_BEFORE_MIGRATE); +} + +/** `DB_BACKUP_DIR`, or the image's default. @param {Env} env */ +export function resolveBackupDir(env = process.env) { + return (env.DB_BACKUP_DIR ?? "").trim() || DEFAULT_BACKUP_DIR; +} + +/** + * `DB_BACKUP_KEEP`: how many daily dumps stay on disk. At least one, so the + * newest good dump is never deleted; a value that is not a whole number + * keeps the default. + * + * @param {Env} env + */ +export function resolveDailyKeep(env = process.env) { + const raw = (env.DB_BACKUP_KEEP ?? "").trim(); + if (!/^\d+$/.test(raw)) return DEFAULT_DAILY_KEEP; + return Math.min(Math.max(Number.parseInt(raw, 10), 1), 366); +} + +const CRON_FIELD = /^[0-9*,/-]+$/; + +/** + * `DB_BACKUP_SCHEDULE` when it is a five-field cron of digits, `*`, `,`, `/` + * and `-`, otherwise the default. `valid` says which, so the boot summary can + * name a schedule it ignored. + * + * @param {Env} env + * @returns {{ cron: string; valid: boolean; configured: boolean }} + */ +export function resolveBackupSchedule(env = process.env) { + const raw = (env.DB_BACKUP_SCHEDULE ?? "").trim(); + if (!raw) { + return { cron: DEFAULT_BACKUP_SCHEDULE, valid: true, configured: false }; + } + const fields = raw.split(/\s+/); + const valid = fields.length === 5 && fields.every((f) => CRON_FIELD.test(f)); + return { + cron: valid ? fields.join(" ") : DEFAULT_BACKUP_SCHEDULE, + valid, + configured: true, + }; +} + +/** + * Whether `dir` exists (or can be created) and this process may write in it. + * Never throws. + * + * @param {string} dir + */ +export function isWritableDirectory(dir) { + try { + mkdirSync(dir, { recursive: true }); + if (!statSync(dir).isDirectory()) return false; + accessSync(dir, fsConstants.W_OK | fsConstants.X_OK); + return true; + } catch { + return false; + } +} + +// ── Keys ─────────────────────────────────────────────────────────────────── + +/** + * Decode a 32-byte key the way the app does (`decodeKey` in + * `src/lib/crypto.ts`): 64 hex characters, or base64 of 32 bytes. Shorter hex + * is padded with its own SHA-256 there, outside production only; it is + * decoded the same way here so a development instance's fingerprints match + * what its admin page shows. Null when the value is not a key at all. + * + * @param {string | undefined} raw + * @returns {Buffer | null} + */ +export function decodeKeyMaterial(raw) { + const value = (raw ?? "").trim(); + if (!value) return null; + if (/^[0-9a-fA-F]{64}$/.test(value)) return Buffer.from(value, "hex"); + if (/^[A-Za-z0-9+/=]+$/.test(value)) { + const buf = Buffer.from(value, "base64"); + if (buf.length === 32) return buf; + } + if (/^[0-9a-fA-F]+$/.test(value) && value.length >= 32) { + const padded = + value + createHash("sha256").update(value, "utf8").digest("hex"); + return Buffer.from(padded.slice(0, 64), "hex"); + } + return null; +} + +/** + * The public fingerprint of a key: the first twelve hex characters of + * SHA-256 over its bytes. The same rule as `fingerprintKeyBytes` in + * `src/lib/crypto.ts` (what the admin's key-backup step shows) and as the key + * id an off-host object records, so one number names one key everywhere. + * + * @param {Buffer} bytes + */ +export function fingerprintBytes(bytes) { + return createHash("sha256").update(bytes).digest("hex").slice(0, 12); +} + +/** + * @typedef {{ + * encryption: { + * activeKeyId: string | null; + * keys: Array<{ id: string; fingerprint: string }>; + * }; + * backupEncryptionKey: string | null; + * backupEncryptionPreviousKeys: string[]; + * apiTokenHmacKey: string | null; + * }} KeyFingerprints + */ + +/** + * Fingerprints of every key a restored database needs, read from an + * environment. Never the key material. A malformed value is left out, which + * the comparison then reports as missing. + * + * `ENCRYPTION_KEY` joins the map as `v1` unless the map names `v1` itself, + * exactly as the app's key loader does. + * + * @param {Env} env + * @returns {KeyFingerprints} + */ +export function keyFingerprints(env = process.env) { + /** @type {Map} */ + const keys = new Map(); + let activeKeyId = null; + const rawMap = (env.ENCRYPTION_KEYS ?? "").trim(); + if (rawMap) { + try { + const parsed = JSON.parse(rawMap); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + for (const [id, value] of Object.entries(parsed)) { + const bytes = + typeof value === "string" ? decodeKeyMaterial(value) : null; + if (bytes) keys.set(id, fingerprintBytes(bytes)); + } + } + } catch { + // Malformed JSON: no keys from the map. The entrypoint refuses to boot + // on it anyway; here it only means nothing matches. + } + activeKeyId = (env.ENCRYPTION_ACTIVE_KEY_ID ?? "").trim() || null; + if (!activeKeyId && keys.size === 1) activeKeyId = [...keys.keys()][0]; + } + const legacy = decodeKeyMaterial(env.ENCRYPTION_KEY); + if (legacy && !keys.has("v1")) keys.set("v1", fingerprintBytes(legacy)); + if (!activeKeyId && legacy) activeKeyId = "v1"; + + const backup = decodeKeyMaterial(env.BACKUP_ENCRYPTION_KEY); + const previous = (env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "") + .split(",") + .map((raw) => decodeKeyMaterial(raw)) + .filter((b) => b !== null) + .map((b) => fingerprintBytes(/** @type {Buffer} */ (b))); + const hmac = (env.API_TOKEN_HMAC_KEY ?? "").trim(); + + return { + encryption: { + activeKeyId, + keys: [...keys.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, fingerprint]) => ({ id, fingerprint })), + }, + backupEncryptionKey: backup ? fingerprintBytes(backup) : null, + backupEncryptionPreviousKeys: previous, + // Used as a string by the app (`createHmac("sha256", key)`), so its + // fingerprint is over the string's bytes. + apiTokenHmacKey: hmac ? fingerprintBytes(Buffer.from(hmac, "utf8")) : null, + }; +} + +/** + * @typedef {{ + * key: string; + * state: "missing" | "different"; + * severity: "refuse" | "warn"; + * detail: string; + * }} KeyFinding + */ + +/** + * Compare the keys a dump was written with against the keys an environment + * holds. The active encryption key is the one that refuses: without it the + * app's start-up key check fails and nothing encrypted reads. Every other + * difference is a warning that says what stops working. + * + * @param {KeyFingerprints} expected the manifest's + * @param {KeyFingerprints} actual the environment's + * @returns {{ findings: KeyFinding[]; refuse: boolean }} + */ +export function compareKeyFingerprints(expected, actual) { + /** @type {KeyFinding[]} */ + const findings = []; + const have = new Map( + actual.encryption.keys.map((k) => [k.id, k.fingerprint]), + ); + const active = expected.encryption.activeKeyId; + for (const { id, fingerprint } of expected.encryption.keys) { + const mine = have.get(id); + if (mine === fingerprint) continue; + const isActive = id === active; + findings.push({ + key: `ENCRYPTION_KEYS[${id}]`, + state: mine === undefined ? "missing" : "different", + severity: isActive ? "refuse" : "warn", + detail: isActive + ? `the dump's active encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `configured with a different value (fingerprint ${mine})`}; nothing encrypted in this database can be read without it` + : `encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `different here (fingerprint ${mine})`}; rows still sealed under it will not open`, + }); + } + if ( + expected.backupEncryptionKey && + expected.backupEncryptionKey !== actual.backupEncryptionKey + ) { + findings.push({ + key: "BACKUP_ENCRYPTION_KEY", + state: actual.backupEncryptionKey ? "different" : "missing", + severity: "warn", + detail: `the off-host backup key (fingerprint ${expected.backupEncryptionKey}) is ${actual.backupEncryptionKey ? `different here (fingerprint ${actual.backupEncryptionKey})` : "not configured"}; off-host copies written before the restore will not open`, + }); + } + if ( + expected.apiTokenHmacKey && + expected.apiTokenHmacKey !== actual.apiTokenHmacKey + ) { + findings.push({ + key: "API_TOKEN_HMAC_KEY", + state: actual.apiTokenHmacKey ? "different" : "missing", + severity: "warn", + detail: `API_TOKEN_HMAC_KEY (fingerprint ${expected.apiTokenHmacKey}) is ${actual.apiTokenHmacKey ? `different here (fingerprint ${actual.apiTokenHmacKey})` : "not configured"}; every session, app token and share link will stop working, no data is lost`, + }); + } + return { + findings, + refuse: findings.some((f) => f.severity === "refuse"), + }; +} + +// ── Connection ───────────────────────────────────────────────────────────── + +/** + * Query parameters libpq understands, by the environment variable that + * carries each to `pg_dump` / `pg_restore`. `DATABASE_URL` also carries + * Prisma's own (`schema`, `connection_limit`, `pool_timeout`), which libpq + * rejects, so the URL is never handed over as it is. + */ +const LIBPQ_PARAMS = { + host: "PGHOST", + sslmode: "PGSSLMODE", + sslcert: "PGSSLCERT", + sslkey: "PGSSLKEY", + sslrootcert: "PGSSLROOTCERT", + connect_timeout: "PGCONNECT_TIMEOUT", + application_name: "PGAPPNAME", +}; + +/** + * @typedef {{ + * pgEnv: Record; + * clientConfig: Record; + * schema: string; + * }} Connection + */ + +/** + * The connection `DATABASE_URL` describes, as libpq environment variables + * (the password travels there, never in a process argument) and as a `pg` + * client config. + * + * @param {string} databaseUrl + * @returns {Connection} + */ +export function connectionFromUrl(databaseUrl) { + const url = new URL(databaseUrl); + const host = decodeURIComponent(url.hostname).replace(/^\[|\]$/g, ""); + /** @type {Record} */ + const pgEnv = { + PGHOST: host, + PGPORT: url.port || "5432", + PGUSER: decodeURIComponent(url.username), + PGPASSWORD: decodeURIComponent(url.password), + PGDATABASE: decodeURIComponent(url.pathname.replace(/^\//, "")), + }; + for (const [param, value] of url.searchParams) { + const name = /** @type {Record} */ (LIBPQ_PARAMS)[param]; + if (name) pgEnv[name] = value; + } + const clean = new URL(databaseUrl); + for (const param of [...clean.searchParams.keys()]) { + if (!(param in LIBPQ_PARAMS)) clean.searchParams.delete(param); + } + return { + pgEnv, + clientConfig: { + connectionString: clean.toString(), + connectionTimeoutMillis: 10_000, + }, + schema: url.searchParams.get("schema") || "public", + }; +} + +/** + * A connected `pg` client for `DATABASE_URL`. + * + * @param {string} databaseUrl + */ +export async function connect(databaseUrl) { + const { default: pg } = await import("pg"); + const client = new pg.Client(connectionFromUrl(databaseUrl).clientConfig); + await client.connect(); + // The server may carry a default statement timeout (the app sets one per + // session through `options`); counting a large table must not hit it. + await client.query("SET statement_timeout = 0"); + return client; +} + +// ── Facts about a database ───────────────────────────────────────────────── + +/** + * @typedef {{ query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> }} Queryable + */ + +/** Tables outside the system schemas, `schema.table`, sorted. @param {Queryable} client */ +export async function listUserTables(client) { + const { rows } = await client.query( + `SELECT format('%I.%I', n.nspname, c.relname) AS name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg\\_%' + ORDER BY 1`, + ); + return rows.map((r) => /** @type {string} */ (r.name)); +} + +/** + * Exact row counts of every user table, keyed `schema.table`. + * + * @param {Queryable} client + * @returns {Promise>} + */ +export async function countRows(client) { + /** @type {Record} */ + const counts = {}; + for (const table of await listUserTables(client)) { + const { rows } = await client.query( + `SELECT count(*)::bigint AS n FROM ${table}`, + ); + counts[table] = Number(rows[0].n); + } + return counts; +} + +/** + * The newest finished migration, or null on a database Prisma never touched. + * + * @param {Queryable} client + * @param {string} schema + */ +export async function latestMigration(client, schema) { + const table = await client.query( + "SELECT to_regclass(format('%I._prisma_migrations', $1::text)) AS t", + [schema], + ); + if (!table.rows[0]?.t) return null; + const { rows } = await client.query( + `SELECT migration_name FROM ${table.rows[0].t} + WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL + ORDER BY migration_name DESC LIMIT 1`, + ); + return rows[0]?.migration_name ?? null; +} + +/** @param {Queryable} client */ +export async function serverVersion(client) { + const { rows } = await client.query( + "SELECT current_setting('server_version') AS v, current_setting('server_version_num')::int AS n", + ); + return { version: String(rows[0].v), versionNum: Number(rows[0].n) }; +} + +// ── Running the Postgres client tools ────────────────────────────────────── + +/** + * Run one Postgres client tool. Standard output goes to `stdout` when given + * (a file stream) and is collected otherwise; standard input comes from + * `stdin` when given. Rejects with the tool's own last lines on a non-zero + * exit, and kills it at `deadline` (epoch ms). + * + * @param {string} tool + * @param {string[]} args + * @param {{ + * env?: Record; + * stdout?: NodeJS.WritableStream; + * stdin?: NodeJS.ReadableStream; + * deadline?: number; + * }} [options] + * @returns {Promise} + */ +export function runTool(tool, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(tool, args, { + env: { ...process.env, ...(options.env ?? {}) }, + stdio: [options.stdin ? "pipe" : "ignore", "pipe", "pipe"], + }); + /** @type {Buffer[]} */ + const out = []; + let err = ""; + let timedOut = false; + const timer = + options.deadline !== undefined + ? setTimeout( + () => { + timedOut = true; + child.kill("SIGTERM"); + }, + Math.max(0, options.deadline - Date.now()), + ) + : null; + if (options.stdout) { + // A destination that fails (a full disk) must stop the tool: unpiped, + // it would block on a full pipe until its deadline, or forever. + options.stdout.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdout.pipe(options.stdout); + } else { + child.stdout.on("data", (c) => out.push(c)); + } + child.stderr.on("data", (c) => { + err = (err + c.toString("utf8")).slice(-4000); + }); + if (options.stdin && child.stdin) { + options.stdin.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdin.on("error", () => { + // The tool closed its input early; its exit code says why. + }); + options.stdin.pipe(child.stdin); + } + child.on("error", (e) => { + if (timer) clearTimeout(timer); + reject( + /** @type {NodeJS.ErrnoException} */ (e).code === "ENOENT" + ? new Error( + `${tool} is not installed in this environment (the HealthLog image ships it; on another host install the PostgreSQL client tools)`, + ) + : e, + ); + }); + child.on("close", (code, signal) => { + if (timer) clearTimeout(timer); + if (timedOut) { + reject(new Error(`${tool} ran past its time limit and was stopped`)); + } else if (code === 0) { + resolve(Buffer.concat(out).toString("utf8")); + } else { + const tail = err.trim().split("\n").slice(-3).join(" | "); + reject( + new Error( + `${tool} exited ${code ?? signal}${tail ? `: ${tail}` : ""}`, + ), + ); + } + }); + }); +} + +/** `pg_dump (PostgreSQL) 18.6`, or null when it cannot be asked. */ +export async function pgDumpVersion() { + try { + return (await runTool("pg_dump", ["--version"])).trim(); + } catch { + return null; + } +} + +/** + * Read an archive's table of contents back. A file `pg_restore` cannot list + * is not a dump, whatever its name. Answers the number of entries. + * + * @param {string} path + */ +export async function verifyArchive(path) { + const listing = await runTool("pg_restore", ["--list"], { + stdin: createReadStream(path), + }); + const entries = listing + .split("\n") + .filter((line) => line.trim() && !line.startsWith(";")).length; + if (entries === 0) throw new Error("the archive lists no entries"); + return entries; +} + +/** SHA-256 and size of a file, streamed. @param {string} path */ +export async function hashFile(path) { + const hash = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + bytes += chunk.length; + } + return { sha256: hash.digest("hex"), bytes }; +} + +// ── Writing a dump ───────────────────────────────────────────────────────── + +/** + * @typedef {{ + * format: string; + * formatVersion: number; + * kind: DumpKind; + * file: string; + * createdAt: string; + * appVersion: string | null; + * buildSha: string | null; + * latestMigration: string | null; + * postgres: { serverVersion: string; serverVersionNum: number; pgDump: string | null }; + * dump: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * keys: KeyFingerprints; + * }} DumpManifest + */ + +/** + * Build the manifest of one dump. Pure, so the shape is tested on its own. + * + * @param {{ + * kind: DumpKind; + * file: string; + * createdAt: Date; + * appVersion?: string | null; + * buildSha?: string | null; + * latestMigration: string | null; + * server: { version: string; versionNum: number }; + * pgDump: string | null; + * archive: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * env: Env; + * }} input + * @returns {DumpManifest} + */ +export function buildManifest(input) { + return { + format: MANIFEST_FORMAT, + formatVersion: MANIFEST_FORMAT_VERSION, + kind: input.kind, + file: input.file, + createdAt: input.createdAt.toISOString(), + appVersion: input.appVersion || null, + buildSha: input.buildSha || null, + latestMigration: input.latestMigration, + postgres: { + serverVersion: input.server.version, + serverVersionNum: input.server.versionNum, + pgDump: input.pgDump, + }, + dump: { ...input.archive }, + rowCounts: Object.fromEntries( + Object.entries(input.rowCounts).sort(([a], [b]) => a.localeCompare(b)), + ), + keys: keyFingerprints(input.env), + }; +} + +/** + * Check a parsed manifest has the fields a restore reads. + * + * @param {unknown} value + * @returns {DumpManifest} + */ +export function parseManifest(value) { + const m = /** @type {Record} */ (value); + if ( + !m || + m.format !== MANIFEST_FORMAT || + m.formatVersion !== MANIFEST_FORMAT_VERSION || + typeof m.file !== "string" || + typeof m.dump?.sha256 !== "string" || + typeof m.dump?.bytes !== "number" || + typeof m.rowCounts !== "object" || + !m.keys?.encryption + ) { + throw new Error("not a HealthLog database dump manifest"); + } + return /** @type {DumpManifest} */ (m); +} + +/** + * Take one dump into `dir`, with its manifest, and answer both paths. + * + * The archive and the row counts read one exported snapshot. The archive is + * written as `.partial`, hashed and listed back; the manifest is written as + * `.partial` and renamed; the archive is renamed last. On any failure every + * file this call created is removed and the error is rethrown. + * + * @param {{ + * databaseUrl: string; + * dir: string; + * kind: DumpKind; + * now?: Date; + * env?: Env; + * appVersion?: string | null; + * buildSha?: string | null; + * deadline?: number; + * }} options + * @returns {Promise<{ base: string; dumpPath: string; manifestPath: string; manifest: DumpManifest }>} + */ +export async function createDatabaseDump(options) { + const now = options.now ?? new Date(); + const env = options.env ?? process.env; + const base = dumpBaseName(options.kind, now); + const dumpPath = join(options.dir, dumpFileName(base)); + const manifestPath = join(options.dir, manifestFileName(base)); + const partialDump = `${dumpPath}.partial`; + const partialManifest = `${manifestPath}.partial`; + const connection = connectionFromUrl(options.databaseUrl); + // Two runs in the same second would share a name. The second one must not + // replace, or on failure remove, what the first one wrote. + for (const taken of [dumpPath, manifestPath]) { + if (await statOrNull(taken)) { + throw new Error(`a dump named ${basename(taken)} already exists`); + } + } + + const client = await connect(options.databaseUrl); + let finished = false; + /** The files this call created, and only those, go on a failure. */ + const created = []; + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY"); + const { rows } = await client.query( + "SELECT pg_export_snapshot() AS snapshot", + ); + const snapshot = String(rows[0].snapshot); + + const out = createWriteStream(partialDump, { flags: "wx", mode: 0o600 }); + const opened = new Promise((resolve, reject) => { + out.once("open", resolve); + out.once("error", reject); + }); + await opened; + created.push(partialDump); + const closed = new Promise((resolve, reject) => { + out.once("close", resolve); + out.once("error", reject); + }); + // Awaited below on success; on a failure the run is already rejecting + // and this must not surface a second time as unhandled. + closed.catch(() => {}); + // `pg_dump` reads the snapshot in its own session while this one counts + // the rows of the same snapshot; the queries on this client run one + // after another (a `pg` client does not overlap them). + const facts = async () => ({ + counts: await countRows(client), + server: await serverVersion(client), + migration: await latestMigration(client, connection.schema), + }); + const [, { counts, server, migration }, pgDump] = await Promise.all([ + runTool( + "pg_dump", + [ + "--format=custom", + "--no-password", + `--snapshot=${snapshot}`, + "--lock-wait-timeout=120000", + ], + { env: connection.pgEnv, stdout: out, deadline: options.deadline }, + ), + facts(), + pgDumpVersion(), + ]); + await closed; + await client.query("COMMIT"); + + const { sha256, bytes } = await hashFile(partialDump); + const entries = await verifyArchive(partialDump); + const manifest = buildManifest({ + kind: options.kind, + file: dumpFileName(base), + createdAt: now, + appVersion: options.appVersion, + buildSha: options.buildSha, + latestMigration: migration, + server, + pgDump, + archive: { bytes, sha256, entries }, + rowCounts: counts, + env, + }); + await writeFile(partialManifest, `${JSON.stringify(manifest, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); + created.push(partialManifest); + await rename(partialManifest, manifestPath); + created.push(manifestPath); + await rename(partialDump, dumpPath); + finished = true; + return { base, dumpPath, manifestPath, manifest }; + } finally { + await client.end().catch(() => {}); + if (!finished) { + // Only what this call created: a `wx` open that failed created nothing. + await Promise.all(created.map((p) => rm(p, { force: true }))); + } + } +} + +// ── What is on disk, and what may go ─────────────────────────────────────── + +/** + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * dump: { path: string; bytes: number } | null; + * manifest: string | null; + * encryptedDump: string | null; + * encryptedManifest: string | null; + * leftovers: Array<{ path: string; mtimeMs: number }>; + * }} DumpSet + */ + +/** + * Every dump this module wrote into `dir`, newest first, grouped by stem. + * Files with any other name are not looked at. + * + * @param {string} dir + * @returns {DumpSet[]} + */ +export function listDumpSets(dir) { + /** @type {Map} */ + const sets = new Map(); + let names = []; + try { + names = readdirSync(dir); + } catch { + return []; + } + for (const name of names) { + const parsed = parseDumpFileName(name); + if (!parsed) continue; + const path = join(dir, name); + let info; + try { + info = statSync(path); + } catch { + continue; + } + if (!info.isFile()) continue; + const set = sets.get(parsed.base) ?? { + base: parsed.base, + kind: parsed.kind, + timestamp: parsed.timestamp, + dump: null, + manifest: null, + encryptedDump: null, + encryptedManifest: null, + leftovers: [], + }; + if (parsed.variant === "partial") { + set.leftovers.push({ path, mtimeMs: info.mtimeMs }); + } else if (parsed.variant === "encrypted") { + if (parsed.role === "dump") set.encryptedDump = path; + else set.encryptedManifest = path; + } else if (parsed.role === "dump") { + set.dump = { path, bytes: info.size }; + } else { + set.manifest = path; + } + sets.set(parsed.base, set); + } + return [...sets.values()].sort( + (a, b) => b.timestamp.getTime() - a.timestamp.getTime(), + ); +} + +/** A set is good when its archive and its manifest both carry final names. @param {DumpSet} set */ +export function isGoodSet(set) { + return set.dump !== null && set.manifest !== null; +} + +/** + * Which files retention removes. Pure. + * + * Per kind, the newest `keep` good dumps stay (at least one, so the newest + * good dump is never removed) and every older good dump goes, archive and + * manifest together. A `.partial` or a manifest without its archive goes once + * it is older than twelve hours, so a run still in progress keeps its files. + * A final-named archive without a manifest is not something this module + * leaves behind, so it is not touched. Encrypted copies (`.enc`) an operator + * placed here to restore from are never touched either. + * + * @param {DumpSet[]} sets + * @param {{ keep: Record; now: number }} options + * @returns {string[]} + */ +export function selectForDeletion(sets, options) { + /** @type {string[]} */ + const remove = []; + for (const kind of DUMP_KINDS) { + const keep = Math.max(1, options.keep[kind] ?? 1); + const good = sets + .filter((s) => s.kind === kind && isGoodSet(s)) + .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); + for (const set of good.slice(keep)) { + remove.push( + /** @type {{ path: string }} */ (set.dump).path, + /** @type {string} */ (set.manifest), + ); + } + } + for (const set of sets) { + for (const leftover of set.leftovers) { + if (options.now - leftover.mtimeMs > STALE_LEFTOVER_MS) { + remove.push(leftover.path); + } + } + if (set.manifest && !set.dump && set.leftovers.length === 0) { + try { + if (options.now - statSync(set.manifest).mtimeMs > STALE_LEFTOVER_MS) { + remove.push(set.manifest); + } + } catch { + // Gone already. + } + } + } + return remove; +} + +/** + * Apply retention to `dir`. Answers the paths removed. + * + * @param {string} dir + * @param {{ dailyKeep?: number; now?: number }} [options] + */ +export async function applyRetention(dir, options = {}) { + const doomed = selectForDeletion(listDumpSets(dir), { + keep: { + daily: options.dailyKeep ?? DEFAULT_DAILY_KEEP, + "pre-migration": PRE_MIGRATION_KEEP, + }, + now: options.now ?? Date.now(), + }); + for (const path of doomed) await rm(path, { force: true }); + return doomed; +} + +/** @param {string} path */ +export async function readManifestFile(path) { + return parseManifest(JSON.parse(await readFile(path, "utf8"))); +} + +// ── Off-host copies ──────────────────────────────────────────────────────── + +/** + * The bucket key of an off-host database object. It is also the associated + * data its encryption is bound to, so a copy cannot be passed off under + * another name. + * + * @param {string} fileName e.g. `healthlog-…-daily.dump.enc` + */ +export function offhostObjectKey(fileName) { + return `${OFFHOST_DATABASE_PREFIX}${fileName}`; +} + +const ENVELOPE_MAGIC = "HLBK"; +const ENVELOPE_KEYED = 0x04; +const AAD_PREFIX = "healthlog/offhost-backup/v4|"; +const IV_LENGTH = 12; +const TAG_LENGTH = 16; + +/** + * Open an off-host object written in the version-4 envelope + * (`src/lib/jobs/offhost-backup.ts`: `HLBK`, 0x04, key id, IV, gzip + * ciphertext, tag, the object key as associated data) into `outPath`, + * streaming, without holding the file. + * + * GCM's tag sits at the end, so the plaintext is written before the tag is + * checked; it goes to `outPath` and is removed again unless the tag verifies. + * Nothing reads `outPath` before this resolves. + * + * @param {{ inPath: string; outPath: string; objectKey: string; keys: Buffer[] }} options + */ +export async function decryptOffhostFile(options) { + const size = statSync(options.inPath).size; + const fd = openSync(options.inPath, "r"); + let header; + let tag; + try { + const head = Buffer.alloc(Math.min(size, 6 + 255 + IV_LENGTH)); + readSync(fd, head, 0, head.length, 0); + if ( + head.subarray(0, 4).toString("latin1") !== ENVELOPE_MAGIC || + head[4] !== ENVELOPE_KEYED + ) { + throw new Error( + "not an off-host database copy (unknown envelope); expected a .dump.enc written by HealthLog", + ); + } + const idLen = head[5]; + const keyId = head.subarray(6, 6 + idLen).toString("latin1"); + const iv = head.subarray(6 + idLen, 6 + idLen + IV_LENGTH); + header = { keyId, iv, length: 6 + idLen + IV_LENGTH }; + tag = Buffer.alloc(TAG_LENGTH); + readSync(fd, tag, 0, TAG_LENGTH, size - TAG_LENGTH); + } finally { + closeSync(fd); + } + if (size < header.length + TAG_LENGTH) { + throw new Error("the off-host copy is truncated"); + } + const key = options.keys.find((k) => fingerprintBytes(k) === header.keyId); + if (!key) { + throw new Error( + `the off-host copy was written under backup key ${header.keyId}, which is neither BACKUP_ENCRYPTION_KEY nor in BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + ); + } + const decipher = createDecipheriv("aes-256-gcm", key, header.iv); + decipher.setAAD(Buffer.from(`${AAD_PREFIX}${options.objectKey}`, "utf8")); + decipher.setAuthTag(tag); + try { + await pipeline( + createReadStream(options.inPath, { + start: header.length, + end: size - TAG_LENGTH - 1, + }), + decipher, + createGunzip(), + createWriteStream(options.outPath, { flags: "wx", mode: 0o600 }), + ); + } catch (err) { + await rm(options.outPath, { force: true }); + const reason = err instanceof Error ? err.message : String(err); + throw new Error( + /auth|unable to authenticate/i.test(reason) + ? "the off-host copy does not open with this key under this name (altered, renamed, or written with another key)" + : `could not open the off-host copy: ${reason}`, + ); + } +} + +/** + * The off-host keys an environment holds, current first. + * + * @param {Env} env + * @returns {Buffer[]} + */ +export function offhostKeysFromEnv(env = process.env) { + return [ + env.BACKUP_ENCRYPTION_KEY, + ...(env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "").split(","), + ] + .map((raw) => decodeKeyMaterial(raw)) + .filter((k) => k !== null) + .map((k) => /** @type {Buffer} */ (k)); +} + +// ── Restore ──────────────────────────────────────────────────────────────── + +/** + * Compare the counts a restore produced with the manifest's. Every table the + * manifest names must be there with its count, and no table may be there that + * the manifest does not name. + * + * @param {Record} expected + * @param {Record} actual + * @returns {string[]} one line per difference + */ +export function compareRowCounts(expected, actual) { + /** @type {string[]} */ + const problems = []; + for (const [table, count] of Object.entries(expected)) { + if (!(table in actual)) { + problems.push(`${table}: in the dump with ${count} rows, missing now`); + } else if (actual[table] !== count) { + problems.push( + `${table}: ${count} rows in the dump, ${actual[table]} restored`, + ); + } + } + for (const table of Object.keys(actual)) { + if (!(table in expected)) { + problems.push(`${table}: restored but not in the dump's manifest`); + } + } + return problems; +} + +/** + * Drop every non-system schema and recreate an empty `public`, so a restore + * that failed its check leaves the database as empty as it found it. + * + * @param {Queryable} client + */ +export async function emptyDatabase(client) { + const { rows } = await client.query( + `SELECT quote_ident(nspname) AS s FROM pg_namespace + WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'`, + ); + for (const { s } of rows) await client.query(`DROP SCHEMA ${s} CASCADE`); + await client.query("CREATE SCHEMA IF NOT EXISTS public"); +} + +/** Stat helper that answers null for a missing file. @param {string} path */ +export async function statOrNull(path) { + try { + return await stat(path); + } catch { + return null; + } +} diff --git a/src/lib/database-backup/health.ts b/src/lib/database-backup/health.ts new file mode 100644 index 000000000..2039df249 --- /dev/null +++ b/src/lib/database-backup/health.ts @@ -0,0 +1,117 @@ +/** + * Whether the nightly whole-database dump is happening, as facts a page or an + * alert can act on. + * + * Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of + * this process: whether `DB_BACKUP_DIR` is writable here, and whether the + * off-host bucket is configured. A ledger row that says a copy went off-host + * outranks this process's environment, for the reason the off-host card gives + * (a split install can hand the bucket variables to the worker only). + */ +import { prisma } from "@/lib/db"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; + +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, +} from "./dump-core.mjs"; + +export interface DatabaseBackupHealth { + /** `DB_BACKUP_ENABLED` is not off. */ + enabled: boolean; + /** The directory dumps are kept in (`DB_BACKUP_DIR`). */ + directory: string; + /** This process can write there. False means dumps only go off-host. */ + localDirWritable: boolean; + /** The off-host bucket is configured, or a dump has been uploaded to it. */ + offhostConfigured: boolean; + /** The newest dump that was kept on the server or reached the bucket. */ + lastGood: { + at: string; + fileName: string; + sizeBytes: number | null; + local: boolean; + offhost: boolean; + } | null; + /** The newest run that failed, or that a dying process left unfinished. */ + lastFailure: { at: string; error: string } | null; +} + +/** + * A run still `running` this long after it started was cut off by its + * process: the job's expiry, past which pg-boss has failed it anyway. + */ +const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000; + +export async function getDatabaseBackupHealth( + now: Date = new Date(), + env: NodeJS.ProcessEnv = process.env, +): Promise { + const directory = resolveBackupDir(env); + const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all( + [ + prisma.databaseBackupRun.findFirst({ + where: { + fileName: { not: null }, + OR: [{ localResult: "written" }, { offhostResult: "uploaded" }], + }, + orderBy: { startedAt: "desc" }, + select: { + startedAt: true, + finishedAt: true, + fileName: true, + dumpBytes: true, + localResult: true, + offhostResult: true, + }, + }), + prisma.databaseBackupRun.findFirst({ + where: { status: "failed" }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true, finishedAt: true, error: true }, + }), + prisma.databaseBackupRun.findFirst({ + where: { + status: "running", + startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) }, + }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true }, + }), + prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }), + ], + ); + + const failures = [ + lastFailed && { + at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(), + error: lastFailed.error ?? "failed without a reason", + }, + lastAbandoned && { + at: lastAbandoned.startedAt.toISOString(), + error: "the run did not finish (the process stopped under it)", + }, + ] + .filter((f): f is { at: string; error: string } => Boolean(f)) + .sort((a, b) => b.at.localeCompare(a.at)); + + return { + enabled: databaseBackupEnabled(env), + directory, + localDirWritable: isWritableDirectory(directory), + offhostConfigured: offhostBackupConfigured() || everUploaded > 0, + lastGood: lastGood + ? { + at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(), + fileName: lastGood.fileName ?? "", + sizeBytes: + lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes), + local: lastGood.localResult === "written", + offhost: lastGood.offhostResult === "uploaded", + } + : null, + lastFailure: failures[0] ?? null, + }; +} diff --git a/src/lib/database-backup/run.ts b/src/lib/database-backup/run.ts new file mode 100644 index 000000000..4f3e3d99e --- /dev/null +++ b/src/lib/database-backup/run.ts @@ -0,0 +1,243 @@ +/** + * One run of the nightly whole-database dump. + * + * The dump is the artifact that rebuilds a lost server: every account, every + * login, every grant and managed profile, the job queue and the instance + * settings, in one `pg_dump --format=custom` file with a manifest beside it + * (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the + * bundled compose file) and, when the off-host bucket is configured, uploaded + * under `database/` in the same envelope the per-account copies use, sealed + * with `BACKUP_ENCRYPTION_KEY`. + * + * Without a writable directory (an install that runs the image without the + * volume) the dump is written to a temporary directory, uploaded and removed: + * the off-host copy still happens, nothing is kept on the server, and the + * ledger and the boot summary say so. With neither, the run is skipped and + * recorded as such. + * + * Every run leaves one `DatabaseBackupRun` row: started, then finished with + * what happened on each leg. `getDatabaseBackupHealth` reads them. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { + getS3Client, + loadOffhostConfig, + uploadEncryptedBackup, + type S3Like, +} from "@/lib/jobs/offhost-backup"; +import { annotate } from "@/lib/logging/context"; +import packageJson from "../../../package.json"; + +import { + applyRetention, + createDatabaseDump, + databaseBackupEnabled, + isWritableDirectory, + offhostObjectKey, + resolveBackupDir, + resolveDailyKeep, +} from "./dump-core.mjs"; + +/** How long ledger rows are kept. A run a day; a year and a bit of history. */ +const LEDGER_RETENTION_DAYS = 400; + +export type DatabaseBackupLocalResult = + "written" | "no_directory" | "failed" | "skipped"; +export type DatabaseBackupOffhostResult = + "uploaded" | "not_configured" | "failed" | "skipped"; + +export interface DatabaseBackupReport { + status: "succeeded" | "failed" | "skipped"; + fileName: string | null; + dumpBytes: number | null; + local: DatabaseBackupLocalResult; + offhost: DatabaseBackupOffhostResult; + /** Dumps and manifests retention removed. */ + removed: number; + error: string | null; +} + +export interface RunDatabaseBackupOptions { + now?: Date; + /** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */ + deadline?: number; + env?: NodeJS.ProcessEnv; + /** A bucket double for tests; the configured bucket otherwise. */ + s3?: S3Like; +} + +/** The running build's version, as `/api/version` answers it. */ +function appVersion(env: NodeJS.ProcessEnv): string { + return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version; +} + +/** An error as one line for the ledger and the admin page. */ +function reason(err: unknown): string { + const text = err instanceof Error ? err.message : String(err); + return text.replace(/\s+/g, " ").trim().slice(0, 500); +} + +/** Upload one local file as an off-host object, streamed. */ +async function uploadFile( + s3: S3Like, + key: Buffer, + path: string, +): Promise { + return uploadEncryptedBackup( + s3, + offhostObjectKey(`${basename(path)}.enc`), + key, + async (write) => { + for await (const chunk of createReadStream(path)) { + await write(chunk as Buffer); + } + }, + ); +} + +/** + * Take tonight's dump, keep it, copy it off-host, prune the old ones, and + * record what happened. Never throws: a failure is the report's `status`. + */ +export async function runDatabaseBackup( + prisma: PrismaClient, + options: RunDatabaseBackupOptions = {}, +): Promise { + const env = options.env ?? process.env; + const now = options.now ?? new Date(); + const dir = resolveBackupDir(env); + const localWritable = isWritableDirectory(dir); + const offhostConfig = (() => { + try { + return loadOffhostConfig(); + } catch { + // A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the + // same value with their own message; here it reads as an upload error. + return "invalid" as const; + } + })(); + + // Each leg reads "failed" until it has succeeded, so a run the process died + // under says so once its expiry has passed. + const report: DatabaseBackupReport = { + status: "failed", + fileName: null, + dumpBytes: null, + local: localWritable ? "failed" : "no_directory", + offhost: offhostConfig ? "failed" : "not_configured", + removed: 0, + error: null, + }; + const run = await prisma.databaseBackupRun.create({ + data: { + startedAt: now, + status: "running", + localResult: report.local, + offhostResult: report.offhost, + }, + select: { id: true }, + }); + + let scratch: string | null = null; + try { + if (!databaseBackupEnabled(env)) { + report.status = "skipped"; + report.local = "skipped"; + report.offhost = "skipped"; + report.error = "DB_BACKUP_ENABLED is false"; + } else if (!localWritable && !offhostConfig) { + report.status = "skipped"; + report.error = `no writable backup directory (${dir}) and no off-host bucket configured`; + } else { + const target = localWritable + ? dir + : (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-"))); + const dump = await createDatabaseDump({ + databaseUrl: env.DATABASE_URL ?? "", + dir: target, + kind: "daily", + now, + env, + appVersion: appVersion(env), + buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null, + deadline: options.deadline, + }); + if (localWritable) report.local = "written"; + report.fileName = basename(dump.dumpPath); + report.dumpBytes = dump.manifest.dump.bytes; + + let uploadError: string | null = null; + if (offhostConfig === "invalid") { + uploadError = + "BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64"; + } else if (offhostConfig) { + try { + const s3 = options.s3 ?? (await getS3Client(offhostConfig)); + // The manifest goes up first: a dump object in the bucket always + // has the manifest a restore checks it against. + await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath); + await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath); + report.offhost = "uploaded"; + } catch (err) { + uploadError = `off-host upload failed: ${reason(err)}`; + } + } + + if (localWritable) { + const removed = await applyRetention(dir, { + dailyKeep: resolveDailyKeep(env), + now: now.getTime(), + }); + report.removed = removed.length; + } + + // Good when the dump exists somewhere it survives: on the volume, or in + // the bucket. An upload that failed beside a kept dump still fails the + // run, because the operator asked for both. + report.status = uploadError ? "failed" : "succeeded"; + report.error = uploadError; + } + } catch (err) { + report.status = "failed"; + report.error = reason(err); + } finally { + if (scratch) await rm(scratch, { recursive: true, force: true }); + } + + await prisma.databaseBackupRun.update({ + where: { id: run.id }, + data: { + finishedAt: new Date(), + status: report.status, + fileName: report.fileName, + dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes), + localResult: report.local, + offhostResult: report.offhost, + error: report.error, + }, + }); + await prisma.databaseBackupRun.deleteMany({ + where: { + startedAt: { + lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000), + }, + }, + }); + + annotate({ + action: { name: "database.backup.run" }, + meta: { + status: report.status, + local: report.local, + offhost: report.offhost, + dump_bytes: report.dumpBytes ?? 0, + removed: report.removed, + }, + }); + return report; +} diff --git a/src/lib/jobs/database-backup-queue.ts b/src/lib/jobs/database-backup-queue.ts new file mode 100644 index 000000000..5285095eb --- /dev/null +++ b/src/lib/jobs/database-backup-queue.ts @@ -0,0 +1,40 @@ +/** + * The queue facts of the nightly whole-database dump: its name, its schedule + * and its send options. Apart from the job so the worker registry can read + * them without loading it (`job-registry-graph.test.ts`). + */ +import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; +import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs"; + +export const DATABASE_BACKUP_QUEUE = "database-backup"; + +/** + * `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the + * zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30, + * an hour before the off-host account copies so the two uploads do not meet. + * A value that is not a five-field cron keeps the default, and the boot + * readiness summary says so. + */ +export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron; + +/** + * Four hours. `pg_dump` of a large instance takes minutes, the upload of the + * dump about as long as the dump; the window is there to be impossible to hit + * for a working run. The handler kills `pg_dump` at three quarters of it + * (`jobDeadline`), so a hung dump ends as a failed run with a reason rather + * than an expired job. + */ +export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; + +/** + * What the nightly schedule sends with: the expiry, the backups' heartbeat, + * and one retry. A dump that failed for a passing reason (a lock it waited + * out, a bucket that answered 503) gets a second attempt; one that fails + * again waits for the next night, and the failure stays on the admin page. + */ +export const DATABASE_BACKUP_SEND_OPTIONS = { + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, + retryLimit: 1, + retryDelay: 15 * 60, +} as const; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 250838b7d..ab79d94c1 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "continued", "conversation_titles_migrated", "created", + // The nightly whole-database dump: where it went and how big. + "database_backup_bytes", + "database_backup_local", + "database_backup_offhost", + "database_backup_removed", + "database_backup_status", "daily_rows_retired", "days_consolidated", "days_rebuilt", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index cf6fbf1ba..f34ded204 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -494,9 +494,9 @@ function createEnvelopeStream( return { stream, bytes: () => written }; } -/** Produces the backup JSON in pieces. Every piece is written in order. */ +/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */ export type BackupJsonProducer = ( - write: (chunk: string) => Promise, + write: (chunk: string | Buffer) => Promise, ) => Promise; export interface UploadBackupOptions { @@ -572,9 +572,9 @@ export async function uploadEncryptedBackup( }, ); - const write = async (chunk: string): Promise => { + const write = async (chunk: string | Buffer): Promise => { if (failure) throw failure; - if (gzip.write(chunk, "utf8")) return; + if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding await new Promise((resolve, reject) => { const onDrain = (): void => { gzip.off("error", onError); diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 28ed26810..9a2503c1b 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key- import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run"; import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue"; import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue"; @@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly> = { exclusive: "lockedPass", why: "Serialises every account's whole record.", }, + "database-backup": { + runtime: "long", + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"], + stop: { + file: "lib/jobs/reminder/backup-handlers.ts", + fn: "handleDatabaseBackup", + }, + exclusive: "lockedPass", + why: "Runs pg_dump over the whole database and uploads the dump off-host.", + }, "data-backup-offhost": { runtime: "long", expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 70429c156..6c84a3eaf 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -1,5 +1,6 @@ /** - * The housekeeping queues: backups (weekly in-database, nightly off-host), + * The housekeeping queues: backups (the nightly whole-database dump, weekly + * in-database account copies, nightly off-host), * the restore drill and restores, every ledger and retention cleanup, the * medication maintenance passes, the imports (Apple Health, Health Connect, * medication intake), the converging encryption backfills, the document vault @@ -47,6 +48,11 @@ import { DATA_BACKUP_QUEUE, DATA_BACKUP_SEND_OPTIONS, } from "@/lib/jobs/data-backup-policy"; +import { + DATABASE_BACKUP_CRON, + DATABASE_BACKUP_QUEUE, + DATABASE_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/database-backup-queue"; import { DAWARICH_SYNC_CRON, DAWARICH_SYNC_QUEUE, @@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [ lane: true, load: async () => (await backups()).handleDataBackup, }), + // The whole-database dump: the copy that rebuilds a lost server. Locked, so + // a retry that starts while a run is still dumping waits its turn. + defineJob({ + queue: DATABASE_BACKUP_QUEUE, + cron: DATABASE_BACKUP_CRON, + scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + load: async () => (await backups()).handleDatabaseBackup, + }), prune( RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index c8bc8fa83..845f03927 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -5,7 +5,7 @@ * schedules, and boss.work registrations. */ import { type Job, type JobWithMetadata } from "pg-boss"; -import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget"; import { recordError } from "@/lib/jobs/worker-status"; import { BackupBlobTooLargeError, @@ -31,6 +31,37 @@ import { } from "@/lib/jobs/backup-pass-attempts"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +/** + * The nightly whole-database dump (`database-backup`). + * + * One `pg_dump` in a child process, so it is not in the heavy lane: the heap + * holds a part of the upload at a time, not the database. `pg_dump` is stopped + * at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs + * on a lock ends as a failed run with its reason rather than an expired job. + * A skipped run (switched off, nowhere to put it) completes; the admin page + * and the boot summary carry why. + */ +export async function handleDatabaseBackup( + jobs: Job>[], +): Promise { + return withBackgroundEvent("job.database_backup", async () => { + const { runDatabaseBackup } = await import("@/lib/database-backup/run"); + const report = await runDatabaseBackup(getWorkerPrisma(), { + deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined, + }); + const did = { + database_backup_status: report.status, + database_backup_local: report.local, + database_backup_offhost: report.offhost, + database_backup_bytes: report.dumpBytes ?? 0, + database_backup_removed: report.removed, + }; + return report.status === "failed" + ? jobFailed("database dump failed", report.error ?? undefined, did) + : jobDone(did); + }); +} + export interface DataBackupPayload { triggeredAt: string; } diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index d90560c84..c95d658a6 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -99,6 +99,7 @@ const subsystemSurface = { "audit-log-cleanup": { audience: "system" }, "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, + "database-backup": { audience: "system" }, "data-restore-drill": { audience: "system" }, "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, diff --git a/tests/integration/__golden__/queue-registration.json b/tests/integration/__golden__/queue-registration.json index d85705a21..31a41afe6 100644 --- a/tests/integration/__golden__/queue-registration.json +++ b/tests/integration/__golden__/queue-registration.json @@ -1,5 +1,5 @@ { - "registeredQueueCount": 122, + "registeredQueueCount": 123, "registeredQueues": [ "achievement-unlock-sweep", "apple-health-import", @@ -23,6 +23,7 @@ "data-backup", "data-backup-offhost", "data-restore-drill", + "database-backup", "dawarich-sync", "dense-intraday-hourly-rebuild", "dense-intraday-retention", @@ -455,6 +456,21 @@ "retry_delay_max": null, "retry_limit": 2 }, + { + "dead_letter": null, + "deletion_seconds": 604800, + "expire_seconds": 900, + "heartbeat_seconds": null, + "name": "database-backup", + "notify": false, + "partition": false, + "policy": "standard", + "retention_seconds": 1209600, + "retry_backoff": false, + "retry_delay": 0, + "retry_delay_max": null, + "retry_limit": 2 + }, { "dead_letter": null, "deletion_seconds": 604800, @@ -2137,6 +2153,21 @@ }, "timezone": "Europe/Berlin" }, + { + "cron": "30 1 * * *", + "data": {}, + "key": "", + "kind": "cron", + "name": "database-backup", + "options": { + "expireInSeconds": 14400, + "heartbeatSeconds": 300, + "retryDelay": 900, + "retryLimit": 1, + "tz": "Europe/Berlin" + }, + "timezone": "Europe/Berlin" + }, { "cron": "20 3 * * *", "data": {}, @@ -3022,6 +3053,13 @@ "pollingIntervalSeconds": 30 } }, + { + "queue": "database-backup", + "options": { + "localConcurrency": 1, + "pollingIntervalSeconds": 30 + } + }, { "queue": "dawarich-sync", "options": { diff --git a/tests/integration/database-backup.test.ts b/tests/integration/database-backup.test.ts new file mode 100644 index 000000000..58e778188 --- /dev/null +++ b/tests/integration/database-backup.test.ts @@ -0,0 +1,511 @@ +/** + * The nightly whole-database dump against a real Postgres: the job writes a + * dump and its manifest, records the run, copies both off-host in the + * envelope the restore opens, keeps only what retention allows; and the + * restore puts the dump into an empty database and refuses one with tables. + * + * `pg_dump` and `pg_restore` are the test container's own. A workstation may + * have none, or one of another major than the server (which `pg_dump` + * refuses), so the test puts two small forwarders on PATH that run the tool + * inside the database container with `docker exec`. Everything else, the + * snapshot, the counts, the manifest, the ledger and the restore's checks, + * runs exactly as in the image. + */ +import { execFileSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; + +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +import { prisma } from "@/lib/db"; +import { getDatabaseBackupHealth } from "@/lib/database-backup/health"; +import { runDatabaseBackup } from "@/lib/database-backup/run"; +import { + connect, + countRows, + decryptOffhostFile, + offhostObjectKey, + readManifestFile, +} from "@/lib/database-backup/dump-core.mjs"; + +import { main as restoreMain } from "../../scripts/db-restore.mjs"; +import { shippedMigrations } from "../../scripts/pending-migrations.mjs"; + +const OFFHOST_KEY = "e".repeat(64); +const work = mkdtempSync(join(tmpdir(), "hl-db-backup-")); +const backups = join(work, "backups"); +const shims = join(work, "bin"); +const originalPath = process.env.PATH; + +/** The id of the container that serves DATABASE_URL. */ +function containerId(): string { + const port = new URL(process.env.DATABASE_URL ?? "").port; + const rows = execFileSync( + "docker", + ["ps", "--format", "{{.ID}} {{.Ports}}"], + { + encoding: "utf8", + }, + ).split("\n"); + const row = rows.find((r) => r.includes(`:${port}->5432/tcp`)); + if (!row) throw new Error(`no container publishes port ${port}`); + return row.split(" ")[0]; +} + +/** + * `pg_dump` / `pg_restore` that run inside the database container. The + * connection comes from the PG* variables the core sets; host and port become + * the container's own. A file argument that exists on this machine is sent on + * standard input, which both tools read when no file is named. + */ +function writeShims(container: string): void { + for (const tool of ["pg_dump", "pg_restore"]) { + const path = join(shims, tool); + writeFileSync( + path, + `#!/usr/bin/env node +const { spawn } = require("node:child_process"); +const { createReadStream, existsSync } = require("node:fs"); +const args = process.argv.slice(2); +let input = null; +if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) { + input = args.pop(); +} +const env = []; +for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) { + if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]); +} +env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432"); +const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], { + stdio: ["pipe", "inherit", "inherit"], +}); +(input ? createReadStream(input) : process.stdin).pipe(child.stdin); +child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0))); +`, + ); + chmodSync(path, 0o755); + } +} + +/** A bucket in memory, with the one call the job's uploader uses. */ +function memoryBucket() { + const objects = new Map(); + return { + objects, + putStream: async (key: string, body: Readable) => { + const chunks: Buffer[] = []; + for await (const c of body) chunks.push(c as Buffer); + objects.set(key, Buffer.concat(chunks)); + }, + }; +} + +function backupEnv(extra: Record = {}): NodeJS.ProcessEnv { + return { ...process.env, DB_BACKUP_DIR: backups, ...extra }; +} + +beforeAll(async () => { + const { mkdirSync } = await import("node:fs"); + mkdirSync(shims, { recursive: true }); + mkdirSync(backups, { recursive: true }); + writeShims(containerId()); + process.env.PATH = `${shims}:${originalPath}`; + for (const name of ["dump-a", "dump-b", "dump-c"]) { + await prisma.user.create({ + data: { username: `${name}-${Date.now()}` }, + }); + } +}); + +afterAll(() => { + process.env.PATH = originalPath; + vi.unstubAllEnvs(); + rmSync(work, { recursive: true, force: true }); +}); + +describe("the nightly database dump", () => { + let dumpFile = ""; + + it("writes a checked dump and manifest, records the run and copies both off-host", async () => { + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid"); + vi.stubEnv("BACKUP_S3_BUCKET", "b"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "s"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY); + const bucket = memoryBucket(); + const now = new Date("2026-10-20T01:30:00Z"); + + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv(), + s3: bucket as never, + }); + expect(report).toMatchObject({ + status: "succeeded", + local: "written", + offhost: "uploaded", + error: null, + fileName: "healthlog-20261020T013000Z-daily.dump", + }); + dumpFile = join(backups, report.fileName ?? ""); + + // Both files, under their final names, and nothing half-written. + expect(readdirSync(backups).sort()).toEqual([ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]); + expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe( + "PGDMP", + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261020T013000Z-daily.manifest.json"), + ); + expect(manifest.kind).toBe("daily"); + expect(manifest.dump.bytes).toBe(report.dumpBytes); + expect(manifest.latestMigration).toBe( + shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1), + ); + expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100); + expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0); + expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/); + + // The ledger row and what the admin page reads from it. + const runs = await prisma.databaseBackupRun.findMany({ + where: { startedAt: now }, + }); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ + status: "succeeded", + fileName: "healthlog-20261020T013000Z-daily.dump", + localResult: "written", + offhostResult: "uploaded", + error: null, + }); + expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes); + const health = await getDatabaseBackupHealth( + new Date("2026-10-20T02:00:00Z"), + backupEnv(), + ); + expect(health).toMatchObject({ + enabled: true, + localDirWritable: true, + offhostConfigured: true, + lastGood: { + fileName: "healthlog-20261020T013000Z-daily.dump", + sizeBytes: manifest.dump.bytes, + local: true, + offhost: true, + }, + }); + + // The off-host pair opens, with the restore's own reader, to the very + // bytes on disk. + expect([...bucket.objects.keys()].sort()).toEqual([ + "database/healthlog-20261020T013000Z-daily.dump.enc", + "database/healthlog-20261020T013000Z-daily.manifest.json.enc", + ]); + const opened = join(work, "opened"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(opened); + for (const name of [ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]) { + writeFileSync( + join(opened, `${name}.enc`), + bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer, + ); + await decryptOffhostFile({ + inPath: join(opened, `${name}.enc`), + outPath: join(opened, name), + objectKey: offhostObjectKey(`${name}.enc`), + keys: [Buffer.from(OFFHOST_KEY, "hex")], + }); + expect( + readFileSync(join(opened, name)).equals( + readFileSync(join(backups, name)), + ), + ).toBe(true); + } + vi.unstubAllEnvs(); + }); + + it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => { + for (const day of ["21", "22", "23"]) { + const report = await runDatabaseBackup(prisma, { + now: new Date(`2026-10-${day}T01:30:00Z`), + env: backupEnv({ DB_BACKUP_KEEP: "2" }), + }); + expect(report.status).toBe("succeeded"); + expect(report.offhost).toBe("not_configured"); + } + expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([ + "healthlog-20261022T013000Z-daily.dump", + "healthlog-20261023T013000Z-daily.dump", + ]); + }); + + it("records a failed dump, leaves nothing that looks good, and the page says so", async () => { + const before = readdirSync(backups).sort(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.password = "not-the-password"; + const now = new Date("2026-10-24T01:30:00Z"); + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv({ DATABASE_URL: url.toString() }), + }); + expect(report.status).toBe("failed"); + expect(report.local).toBe("failed"); + expect(report.error).toMatch(/password authentication failed/); + expect(readdirSync(backups).sort()).toEqual(before); + const health = await getDatabaseBackupHealth( + new Date("2026-10-24T02:00:00Z"), + backupEnv(), + ); + expect(health.lastFailure?.error).toMatch(/password authentication/); + expect(health.lastGood?.fileName).toBe( + "healthlog-20261023T013000Z-daily.dump", + ); + }); + + it("skips, and says why, with nowhere to put a dump", async () => { + const report = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T03:00:00Z"), + env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }), + }); + expect(report).toMatchObject({ + status: "skipped", + local: "no_directory", + offhost: "not_configured", + }); + expect(report.error).toMatch(/no writable backup directory/); + const off = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T04:00:00Z"), + env: backupEnv({ DB_BACKUP_ENABLED: "false" }), + }); + expect(off).toMatchObject({ + status: "skipped", + local: "skipped", + offhost: "skipped", + }); + }); + + describe("restoring it", () => { + const target = `restore_${process.pid}_${Date.now()}`; + let targetUrl = ""; + + beforeAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`CREATE DATABASE ${target}`); + await admin.end(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.pathname = `/${target}`; + targetUrl = url.toString(); + }); + + afterAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`); + await admin.end(); + }); + + it("refuses the database the dump came from, which has tables", async () => { + const errors: string[] = []; + const spy = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const code = await restoreMain(["latest"], backupEnv()); + spy.mockRestore(); + expect(code).toBe(1); + expect(errors.join("\n")).toMatch(/only into an empty database/); + }); + + it("restores the newest dump into an empty database with every row count", async () => { + const lines: string[] = []; + const spy = vi + .spyOn(console, "log") + .mockImplementation((l: string) => void lines.push(String(l))); + const code = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + spy.mockRestore(); + expect(code, lines.join("\n")).toBe(0); + expect(lines.join("\n")).toMatch( + /all \d+ tables hold the row counts the manifest names/, + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261023T013000Z-daily.manifest.json"), + ); + const client = await connect(targetUrl); + try { + expect(await countRows(client)).toEqual(manifest.rowCounts); + const { rows } = await client.query( + "SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'", + ); + expect(rows[0].n).toBeGreaterThanOrEqual(3); + } finally { + await client.end(); + } + // Restoring again into the now full database is refused. + const again = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + expect(again).toBe(1); + expect(existsSync(dumpFile)).toBe(false); // the first dump aged out + }); + }); +}); + +describe("the dump before migrating", () => { + const preDir = join(work, "pre"); + const appDir = join(work, "app"); + const failing = join(work, "failing-bin"); + let cwd = ""; + + beforeAll(async () => { + const { cpSync, mkdirSync } = await import("node:fs"); + // An image that ships one migration the database has not applied. + mkdirSync(join(appDir, "prisma"), { recursive: true }); + cpSync( + join(process.cwd(), "prisma", "migrations"), + join(appDir, "prisma", "migrations"), + { recursive: true }, + ); + mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet")); + writeFileSync( + join( + appDir, + "prisma", + "migrations", + "9999_not_applied_yet", + "migration.sql", + ), + "", + ); + mkdirSync(preDir, { recursive: true }); + mkdirSync(failing, { recursive: true }); + writeFileSync( + join(failing, "pg_dump"), + "#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n", + ); + chmodSync(join(failing, "pg_dump"), 0o755); + cwd = process.cwd(); + process.chdir(appDir); + }); + + afterAll(() => { + process.chdir(cwd); + process.env.PATH = `${shims}:${originalPath}`; + }); + + async function preMigration(env: Record) { + const { preMigrationDump } = await import("../../scripts/db-dump.mjs"); + const errors: string[] = []; + const spyErr = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const spyLog = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + const code = await preMigrationDump({ ...process.env, ...env }); + return { code, errors: errors.join("\n") }; + } finally { + spyErr.mockRestore(); + spyLog.mockRestore(); + } + } + + it("writes a checked dump with its manifest, and keeps the last two", async () => { + for (let i = 0; i < 3; i++) { + if (i > 0) await new Promise((r) => setTimeout(r, 1100)); + const { code } = await preMigration({ DB_BACKUP_DIR: preDir }); + expect(code).toBe(0); + } + const names = readdirSync(preDir).sort(); + expect(names).toHaveLength(4); + expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength( + 2, + ); + const manifest = await readManifestFile( + join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""), + ); + expect(manifest.kind).toBe("pre-migration"); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + }); + + it("does nothing when switched off", async () => { + const before = readdirSync(preDir); + const { code } = await preMigration({ + DB_BACKUP_DIR: preDir, + DB_BACKUP_BEFORE_MIGRATE: "false", + }); + expect(code).toBe(0); + expect(readdirSync(preDir)).toEqual(before); + }); + + it("warns and lets the boot go on without a writable directory", async () => { + const { code, errors } = await preMigration({ + DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here", + }); + expect(code).toBe(0); + expect(errors).toMatch( + /is not writable, so no dump is taken before the 1 pending migration/, + ); + }); + + it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => { + // A second of its own, so the name cannot meet the last dump's. + await new Promise((r) => setTimeout(r, 1100)); + const before = readdirSync(preDir).sort(); + process.env.PATH = `${failing}:${originalPath}`; + const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir }); + process.env.PATH = `${shims}:${originalPath}`; + expect(code).toBe(1); + expect(errors).toMatch( + /the database dump before migrating failed: .*No space left/, + ); + expect(readdirSync(preDir).sort()).toEqual(before); + }); +}); + +describe("two dumps in the same second", () => { + it("refuses the second and leaves the first whole", async () => { + const { createDatabaseDump } = + await import("@/lib/database-backup/dump-core.mjs"); + const dir = join(work, "same-second"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(dir); + const now = new Date("2026-10-26T01:30:00Z"); + const options = { + databaseUrl: process.env.DATABASE_URL ?? "", + dir, + kind: "daily" as const, + now, + }; + await createDatabaseDump(options); + const before = readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]); + await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/); + expect( + readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]), + ).toEqual(before); + }); +}); From e665f17f521285f144828af8ce6bac49456a8e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:52:07 +0100 Subject: [PATCH 02/10] feat(backup): dump before migrating, and restore a dump into an empty database The image now carries postgresql18-client (pg_dump and pg_restore 18.6) and a /var/lib/healthlog/backups directory owned by the app user, which the bundled compose file backs with a new `backups` volume for `app` and the commented `app-worker`; a fresh named volume inherits that owner. Before `migrate deploy`, when migrations are pending on a database that has applied some, the entrypoint writes healthlog-