From e665f17f521285f144828af8ce6bac49456a8e83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= <mbombeck@gmail.com>
Date: Sun, 25 Oct 2026 12:52:07 +0100
Subject: [PATCH] feat(backup): dump before migrating, and restore a dump into
 an empty database

The image now carries postgresql18-client (pg_dump and pg_restore 18.6)
and a /var/lib/healthlog/backups directory owned by the app user, which
the bundled compose file backs with a new `backups` volume for `app` and
the commented `app-worker`; a fresh named volume inherits that owner.

Before `migrate deploy`, when migrations are pending on a database that
has applied some, the entrypoint writes healthlog-<time>-pre-migration.dump
with its manifest and keeps the last two. A failed dump on a writable
directory stops the boot and names DB_BACKUP_BEFORE_MIGRATE=false; without
a writable directory it warns and continues.

`docker compose run --rm db-restore <file|latest|list>` runs
scripts/db-restore.mjs in the app image with plain node. It opens an
off-host .dump.enc with BACKUP_ENCRYPTION_KEY (or a previous key), checks
the archive against the manifest's SHA-256, compares the keys in the
environment with the manifest's fingerprints and refuses on a different
active encryption key unless --ignore-key-mismatch, refuses a database
that holds any table, restores in one transaction, runs ANALYZE, and
compares every table's row count with the manifest, emptying the database
again on a difference.

DB_BACKUP_ENABLED, DB_BACKUP_DIR, DB_BACKUP_SCHEDULE, DB_BACKUP_KEEP and
DB_BACKUP_BEFORE_MIGRATE join the compose whitelist, both example files
and the env manifest.
---
 .env.example                                  |  25 ++
 .env.production.example                       |  25 ++
 Dockerfile                                    |  30 +-
 docker-compose.yml                            |  67 +++-
 docker-entrypoint.sh                          |   9 +
 scripts/__tests__/db-restore.test.ts          | 155 ++++++++
 scripts/db-dump.mjs                           | 156 +++++++++
 scripts/db-restore.mjs                        | 331 ++++++++++++++++++
 scripts/env-manifest.json                     |  32 ++
 scripts/pending-migrations.mjs                |  13 +-
 .../compose-db-upgrade-guard.test.ts          |   2 +
 11 files changed, 830 insertions(+), 15 deletions(-)
 create mode 100644 scripts/__tests__/db-restore.test.ts
 create mode 100644 scripts/db-dump.mjs
 create mode 100644 scripts/db-restore.mjs

diff --git a/.env.example b/.env.example
index 092422769..254f84bfb 100644
--- a/.env.example
+++ b/.env.example
@@ -277,6 +277,31 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32"
 # BACKUP_ENCRYPTION_KEY=""          # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY
 # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation
 
+# -----------------------------------------------------------------------------
+# Database dumps (optional, on by default)
+# -----------------------------------------------------------------------------
+# The whole-database dump that rebuilds a lost server: nightly, and before
+# every update that brings migrations. Written to DB_BACKUP_DIR with a
+# manifest beside it (row counts, key fingerprints, versions); with the
+# off-host bucket above set, each nightly dump is also uploaded under
+# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not
+# encrypted: they sit beside the database volume, in the same trust zone.
+# Restore: `docker compose run --rm db-restore latest`. See
+# docs/ops/disaster-recovery.md.
+#
+# Set "false" to stop the nightly dump.
+# DB_BACKUP_ENABLED="true"
+# Where dumps are written. The bundled compose file mounts the `backups`
+# volume here; change it only together with that mount.
+# DB_BACKUP_DIR="/var/lib/healthlog/backups"
+# Five-field cron, in Europe/Berlin time like every scheduled job.
+# DB_BACKUP_SCHEDULE="30 1 * * *"
+# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept
+# besides.
+# DB_BACKUP_KEEP="7"
+# Set "false" to start an update without the dump before its migrations.
+# DB_BACKUP_BEFORE_MIGRATE="true"
+
 # -----------------------------------------------------------------------------
 # Stored backups in the database (optional)
 # -----------------------------------------------------------------------------
diff --git a/.env.production.example b/.env.production.example
index ecf31e293..2495f5d22 100644
--- a/.env.production.example
+++ b/.env.production.example
@@ -414,6 +414,31 @@ API_TOKEN_HMAC_KEY="<openssl rand -hex 32>"
 # key once the lifecycle rule has retired every object written under it.
 # BACKUP_ENCRYPTION_PREVIOUS_KEYS=""
 
+# -----------------------------------------------------------------------------
+# Database dumps (optional, on by default)
+# -----------------------------------------------------------------------------
+# The whole-database dump that rebuilds a lost server: nightly, and before
+# every update that brings migrations. Written to DB_BACKUP_DIR with a
+# manifest beside it (row counts, key fingerprints, versions); with the
+# off-host bucket above set, each nightly dump is also uploaded under
+# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not
+# encrypted: they sit beside the database volume, in the same trust zone.
+# Restore: `docker compose run --rm db-restore latest`. See
+# docs/ops/disaster-recovery.md.
+#
+# Set "false" to stop the nightly dump.
+# DB_BACKUP_ENABLED="true"
+# Where dumps are written. The bundled compose file mounts the `backups`
+# volume here; change it only together with that mount.
+# DB_BACKUP_DIR="/var/lib/healthlog/backups"
+# Five-field cron, in Europe/Berlin time like every scheduled job.
+# DB_BACKUP_SCHEDULE="30 1 * * *"
+# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept
+# besides.
+# DB_BACKUP_KEEP="7"
+# Set "false" to start an update without the dump before its migrations.
+# DB_BACKUP_BEFORE_MIGRATE="true"
+
 # -----------------------------------------------------------------------------
 # Stored backups in the database (optional)
 # -----------------------------------------------------------------------------
diff --git a/Dockerfile b/Dockerfile
index b09dba475..033b08005 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -77,7 +77,13 @@ FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb
 # security patches from the Alpine repo override it deliberately —
 # CVE-2026-14456 in openssl was the first instance the container scan
 # caught between base-image rebuilds).
-RUN apk add --no-cache tzdata && apk upgrade --no-cache
+#
+# `postgresql18-client` carries `pg_dump` and `pg_restore` for the nightly
+# database dump, the pre-migration dump in the entrypoint and the
+# `db-restore` service (docs/ops/disaster-recovery.md). Major 18 matches the
+# bundled database; a client dumps servers of its own major and older, so it
+# also serves an install still on its own Postgres 16.
+RUN apk add --no-cache tzdata postgresql18-client && apk upgrade --no-cache
 WORKDIR /app
 
 ENV NODE_ENV=production
@@ -111,6 +117,13 @@ ENV NEXT_PUBLIC_APP_BUILD_SHA=$NEXT_PUBLIC_APP_BUILD_SHA
 RUN addgroup --system --gid 1001 nodejs && \
     adduser --system --uid 1001 nextjs
 
+# Where the database dumps go (`DB_BACKUP_DIR`). Owned by the app user, so a
+# fresh named volume mounted here starts with that owner: Docker fills an
+# empty named volume from the image directory it covers, ownership included.
+RUN mkdir -p /var/lib/healthlog/backups && \
+    chown nextjs:nodejs /var/lib/healthlog/backups && \
+    chmod 700 /var/lib/healthlog/backups
+
 # Copy built assets
 COPY --from=builder /app/public ./public
 # The server reads its message catalogs from here on first use, one locale at
@@ -233,11 +246,16 @@ COPY --from=builder /app/src/generated ./src/generated
 # the runbooks and the entrypoint against this list. `pending-migrations.mjs`
 # and `migrate-deploy.mjs` are the entrypoint's own: the first logs the
 # pending migration count, the second runs `migrate deploy` with its
-# lock-timeout retry. Root-owned on purpose: the app user can run them but not
-# rewrite them.
+# lock-timeout retry. `db-dump.mjs` takes the dump before migrating, and
+# `db-restore.mjs` is the `db-restore` service; both share
+# `src/lib/database-backup/dump-core.mjs` with the app. Root-owned on purpose:
+# the app user can run them but not rewrite them.
 COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs
 COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs
 COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs
+COPY --from=builder /app/scripts/db-dump.mjs ./scripts/db-dump.mjs
+COPY --from=builder /app/scripts/db-restore.mjs ./scripts/db-restore.mjs
+COPY --from=builder /app/src/lib/database-backup/dump-core.mjs ./src/lib/database-backup/dump-core.mjs
 COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs
 COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts
 # Prove the reset CLI's imports resolve in this tree: with every import found
@@ -247,6 +265,12 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:"
 RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \
     node --input-type=module -e "await import('pg')" && \
     node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')"
+# The dump and restore scripts load, and the client tools they run are the
+# major the bundled database runs.
+RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \
+    node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \
+    pg_dump --version | grep -q "(PostgreSQL) 18\." && \
+    pg_restore --version | grep -q "(PostgreSQL) 18\."
 
 # Install the migration CLI, its config dependency, and a pinned launcher for
 # the maintenance scripts already shipped in the standalone tree. The Prisma
diff --git a/docker-compose.yml b/docker-compose.yml
index cdfbfeea6..676522118 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -14,9 +14,11 @@
 # and login silently fails. This is the single most common self-host snag.
 # Leave it unset (or true) only when a TLS reverse proxy serves HTTPS.
 #
-# Persistent state lives in exactly one named volume (`postgres-data`,
-# Postgres' data). The app container is stateless, so a backup of the database
-# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to
+# Persistent state lives in the named volume `postgres-data` (Postgres' data).
+# The app container is stateless, so a backup of the database is a backup of
+# everything. `backups` holds the database dumps the app takes every night and
+# before every migration (docs/ops/disaster-recovery.md); keep a copy of them
+# off this host, or set the off-host bucket below. `pgdata` is where Postgres 16 kept the data up to
 # v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once
 # and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked
 # into the image at build time or mounted read-only from the host and
@@ -147,6 +149,20 @@ x-healthlog-env: &healthlog_env
   # serves anyway. A last resort for a refusal you are certain is false: it
   # disables a safety check. See docs/self-hosting/getting-started.md.
   ENCRYPTION_KEY_CHECK: "${ENCRYPTION_KEY_CHECK:-}"
+  # The whole-database dump (docs/ops/disaster-recovery.md). Nightly at
+  # DB_BACKUP_SCHEDULE (five-field cron in Europe/Berlin time, default
+  # "30 1 * * *") into DB_BACKUP_DIR (default /var/lib/healthlog/backups, the
+  # `backups` volume below), keeping DB_BACKUP_KEEP daily dumps (default 7)
+  # and the last two taken before a migration. With the off-host bucket set,
+  # each nightly dump is also uploaded under `database/`, sealed with
+  # BACKUP_ENCRYPTION_KEY. DB_BACKUP_ENABLED=false stops the nightly dump;
+  # DB_BACKUP_BEFORE_MIGRATE=false skips the dump before migrations. Change
+  # DB_BACKUP_DIR only together with the volume mount.
+  DB_BACKUP_ENABLED: "${DB_BACKUP_ENABLED:-}"
+  DB_BACKUP_DIR: "${DB_BACKUP_DIR:-}"
+  DB_BACKUP_SCHEDULE: "${DB_BACKUP_SCHEDULE:-}"
+  DB_BACKUP_KEEP: "${DB_BACKUP_KEEP:-}"
+  DB_BACKUP_BEFORE_MIGRATE: "${DB_BACKUP_BEFORE_MIGRATE:-}"
   BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}"
   BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}"
   BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}"
@@ -475,6 +491,11 @@ services:
       - "3000:3000"
     environment:
       <<: *healthlog_env
+    # The database dumps (DB_BACKUP_DIR). Without this mount the app still
+    # copies its nightly dump off-host when the bucket is set, but keeps none
+    # here and takes none before a migration; the boot log says so.
+    volumes:
+      - backups:/var/lib/healthlog/backups
     # Bring your own GeoLite2 databases. The published image only carries
     # MMDBs if the build had a MaxMind licence key, so on `:latest` the
     # offline tier is usually absent and every login IP goes to the online
@@ -484,9 +505,9 @@ services:
     # matches GEOLITE2_DIR's default (/opt/geolite2), so this works as-is
     # once uncommented. Mounting at a different container path also works,
     # just also set GEOLITE2_DIR in .env to match it.
-    # Steps: docs/self-hosting/geolite2.md.
+    # Steps: docs/self-hosting/geolite2.md. The line goes under `volumes:`
+    # above, beside the backups line:
     #
-    # volumes:
     #   - ./geolite2:/opt/geolite2:ro
     depends_on:
       db:
@@ -563,6 +584,9 @@ services:
   #   environment:
   #     <<: *healthlog_env
   #     HEALTHLOG_PROCESS_TYPE: "worker"
+  #   # The worker takes the nightly database dump, so it needs the volume.
+  #   volumes:
+  #     - backups:/var/lib/healthlog/backups
   #   depends_on:
   #     db:
   #       condition: service_healthy
@@ -776,6 +800,39 @@ services:
         mv "$$MOVING" "$$MARK"
         log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md"
 
+  # Restores a database dump into an EMPTY database (docs/ops/disaster-recovery.md).
+  # Never started by `docker compose up`; run it by name:
+  #
+  #   docker compose run --rm db-restore list
+  #   docker compose run --rm db-restore latest
+  #   docker compose run --rm db-restore healthlog-<time>-daily.dump
+  #
+  # It checks the dump against its manifest and the keys in .env against the
+  # keys the dump was written with, refuses a database that already holds
+  # tables, restores in one transaction and compares every table's row count
+  # with the manifest. Start the app only after it says "Restored".
+  # An off-host copy (`database/…dump.enc` and its `.manifest.json.enc`)
+  # restores the same way once both files are in the backups volume, or from a
+  # directory you mount: `docker compose run --rm -v ./from-bucket:/restore:ro
+  # db-restore /restore/healthlog-<time>-daily.dump.enc`.
+  db-restore:
+    image: ghcr.io/mbombeck/healthlog${HEALTHLOG_IMAGE_REF:-:latest}
+    profiles: ["restore"]
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    environment:
+      <<: *healthlog_env
+    volumes:
+      - backups:/var/lib/healthlog/backups
+    depends_on:
+      db:
+        condition: service_healthy
+    entrypoint: ["node", "scripts/db-restore.mjs"]
+    command: ["list"]
+
 volumes:
   postgres-data:
   pgdata:
+  backups:
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index b33d52002..a375985bc 100644
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -88,6 +88,15 @@ echo "HealthLog: Database is reachable."
 # Say how many migrations this image is about to apply, with a reminder to
 # back up first. Reporting only: it never stops the boot.
 node scripts/pending-migrations.mjs || true
+# When migrations are pending, dump the database into DB_BACKUP_DIR first, so
+# the state before them can be restored (docs/ops/disaster-recovery.md). No
+# writable directory: a warning, and the boot goes on. A writable directory
+# and a failed dump: the boot stops here, before anything is migrated.
+if ! node scripts/db-dump.mjs pre-migration; then
+  echo "HealthLog: ERROR — not migrating without the dump. Free disk space or fix the error above and start again." >&2
+  echo "HealthLog: to migrate without a dump this once, set DB_BACKUP_BEFORE_MIGRATE=false." >&2
+  exit 1
+fi
 # `migrate-deploy.mjs` runs `prisma migrate deploy` and retries only a
 # migration that timed out on a lock and is marked safe to re-run; any other
 # failure stops the boot with Prisma's exit code (docs/ops/deploy.md).
diff --git a/scripts/__tests__/db-restore.test.ts b/scripts/__tests__/db-restore.test.ts
new file mode 100644
index 000000000..89849d4cb
--- /dev/null
+++ b/scripts/__tests__/db-restore.test.ts
@@ -0,0 +1,155 @@
+/**
+ * The restore's refusals, which all come before it writes anything.
+ *
+ * The database is a stand-in here: `connect` answers a client whose only use
+ * is to be asked which tables exist, and `runTool` (which would run
+ * `pg_restore`) is a spy that must never be called on a refusal. The real
+ * restore into a real empty database, and its refusal of a real database with
+ * tables, are in `tests/integration/database-backup.test.ts` and in the
+ * compose drill (`scripts/compose-dr-drill.sh`).
+ */
+import { createHash } from "node:crypto";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const tablesInTarget = vi.fn(async (): Promise<string[]> => []);
+const runTool = vi.fn(async () => "");
+const end = vi.fn(async () => {});
+
+vi.mock("../../src/lib/database-backup/dump-core.mjs", async (original) => ({
+  ...(await original<Record<string, unknown>>()),
+  connect: async () => ({ query: async () => ({ rows: [] }), end }),
+  listUserTables: () => tablesInTarget(),
+  runTool: (...args: unknown[]) => runTool(...(args as [])),
+}));
+
+import { buildManifest } from "../../src/lib/database-backup/dump-core.mjs";
+import { main } from "../db-restore.mjs";
+
+const KEY_A = "a".repeat(64);
+const KEY_B = "b".repeat(64);
+const BASE = "healthlog-20261025T013000Z-daily";
+
+let dir: string;
+let errors: string[];
+let lines: string[];
+
+function env(extra: Record<string, string> = {}) {
+  return {
+    DATABASE_URL: "postgresql://healthlog:pw@db:5432/healthlog?schema=public",
+    DB_BACKUP_DIR: dir,
+    ENCRYPTION_KEY: KEY_A,
+    API_TOKEN_HMAC_KEY: "hmac",
+    ...extra,
+  };
+}
+
+/** A dump file and a manifest that describes it, written with KEY_A. */
+function writeDump(body = Buffer.from("PGDMP not really an archive")) {
+  writeFileSync(join(dir, `${BASE}.dump`), body);
+  const manifest = buildManifest({
+    kind: "daily",
+    file: `${BASE}.dump`,
+    createdAt: new Date("2026-10-25T01:30:00Z"),
+    appVersion: "1.46.0",
+    buildSha: null,
+    latestMigration: "0486_database_backup_runs",
+    server: { version: "18.6", versionNum: 180006 },
+    pgDump: null,
+    archive: {
+      bytes: body.length,
+      sha256: createHash("sha256").update(body).digest("hex"),
+      entries: 1,
+    },
+    rowCounts: { "public.users": 2 },
+    env: { ENCRYPTION_KEY: KEY_A, API_TOKEN_HMAC_KEY: "hmac" },
+  });
+  writeFileSync(
+    join(dir, `${BASE}.manifest.json`),
+    JSON.stringify(manifest, null, 2),
+  );
+}
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), "hl-db-restore-"));
+  errors = [];
+  lines = [];
+  vi.spyOn(console, "error").mockImplementation((line: string) => {
+    errors.push(String(line));
+  });
+  vi.spyOn(console, "log").mockImplementation((line: string) => {
+    lines.push(String(line));
+  });
+  tablesInTarget.mockResolvedValue([]);
+  runTool.mockClear();
+});
+afterEach(() => {
+  rmSync(dir, { recursive: true, force: true });
+  vi.restoreAllMocks();
+});
+
+describe("db-restore refuses before it writes", () => {
+  it("refuses a target database that already holds tables", async () => {
+    writeDump();
+    tablesInTarget.mockResolvedValue(["public.users", "public.sessions"]);
+    expect(await main(["latest"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(
+      /already holds 2 tables .*only into an empty database/,
+    );
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("refuses when the active encryption key is not the dump's, and names it", async () => {
+    writeDump();
+    expect(await main(["latest"], env({ ENCRYPTION_KEY: KEY_B }))).toBe(1);
+    expect(lines.join("\n")).toMatch(/KEY MISMATCH: ENCRYPTION_KEYS\[v1\]/);
+    expect(errors.join("\n")).toMatch(/refusing to restore/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("goes past the key check with --ignore-key-mismatch, and still refuses a full database", async () => {
+    writeDump();
+    tablesInTarget.mockResolvedValue(["public.users"]);
+    expect(
+      await main(
+        ["latest", "--ignore-key-mismatch"],
+        env({ ENCRYPTION_KEY: KEY_B }),
+      ),
+    ).toBe(1);
+    expect(errors.join("\n")).toMatch(/already holds 1 table /);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("refuses a dump that does not match its manifest", async () => {
+    writeDump();
+    writeFileSync(join(dir, `${BASE}.dump`), "PGDMP altered");
+    expect(await main(["latest"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/does not match its manifest/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("refuses a dump without its manifest, and a file it did not write", async () => {
+    writeFileSync(join(dir, `${BASE}.dump`), "x");
+    expect(await main([`${BASE}.dump`], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/manifest.json is missing/);
+    writeFileSync(join(dir, "other.dump"), "x");
+    expect(await main(["other.dump"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/not a HealthLog database dump/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("says when there is nothing to restore, and how it is used", async () => {
+    expect(await main(["latest"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/no restorable dump/);
+    expect(await main([], env())).toBe(2);
+  });
+
+  it("lists what is there", async () => {
+    writeDump();
+    expect(await main(["list"], env())).toBe(0);
+    expect(lines.join("\n")).toContain(`${BASE}.dump  daily`);
+  });
+});
diff --git a/scripts/db-dump.mjs b/scripts/db-dump.mjs
new file mode 100644
index 000000000..3cf873666
--- /dev/null
+++ b/scripts/db-dump.mjs
@@ -0,0 +1,156 @@
+/**
+ * The pre-migration dump: a whole-database dump the entrypoint takes before
+ * `prisma migrate deploy` applies anything.
+ *
+ *   node scripts/db-dump.mjs pre-migration
+ *
+ * A migration is not undone by going back to the previous image, and the
+ * bundled compose file pulls `:latest` and migrates on every start. So when
+ * this image brings migrations the database has not applied, it first writes
+ * `healthlog-<timestamp>-pre-migration.dump` with its manifest into
+ * `DB_BACKUP_DIR` (`src/lib/database-backup/dump-core.mjs`), and keeps the
+ * newest two.
+ *
+ * Exit 0: the dump was written, or there was nothing to do (no pending
+ * migration, a new database, `DB_BACKUP_BEFORE_MIGRATE=false`), or there is no
+ * writable directory to put it in, which is said as a warning and is not a
+ * reason to keep the app down. Exit 1: the directory is writable and the dump
+ * failed. The entrypoint then stops the boot and names the switch that skips
+ * the dump, because migrating without the way back the operator was promised
+ * is the one thing this step exists to prevent.
+ *
+ * Fast on purpose: one snapshot, `pg_dump`, a count per table for the
+ * manifest, `pg_restore --list` to prove the file reads. Nothing else.
+ */
+import { readFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import {
+  applyRetention,
+  connect,
+  createDatabaseDump,
+  isWritableDirectory,
+  latestMigration,
+  listUserTables,
+  connectionFromUrl,
+  preMigrationDumpEnabled,
+  resolveBackupDir,
+  resolveDailyKeep,
+} from "../src/lib/database-backup/dump-core.mjs";
+import { pendingMigrations } from "./pending-migrations.mjs";
+
+const log = (line) => console.log(`HealthLog: ${line}`);
+const warn = (line) => console.error(`HealthLog: WARNING — ${line}`);
+
+/** The image's version: the build argument, else package.json beside it. */
+function imageVersion(env) {
+  const fromEnv = (env.NEXT_PUBLIC_APP_VERSION ?? "").trim();
+  if (fromEnv) return fromEnv;
+  try {
+    return JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8"))
+      .version;
+  } catch {
+    return null;
+  }
+}
+
+/** Megabytes as an operator reads them. */
+function mb(bytes) {
+  return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}
+
+export async function preMigrationDump(env = process.env) {
+  if (!preMigrationDumpEnabled(env)) {
+    log("DB_BACKUP_BEFORE_MIGRATE=false, no dump before migrating.");
+    return 0;
+  }
+  const databaseUrl = env.DATABASE_URL;
+  if (!databaseUrl) return 0;
+
+  let pending;
+  try {
+    pending = await pendingMigrations({
+      connectionString: databaseUrl,
+      migrationsDir: join(process.cwd(), "prisma", "migrations"),
+    });
+  } catch (error) {
+    warn(
+      `could not count pending migrations (${error instanceof Error ? error.message : String(error)}); no dump before migrating.`,
+    );
+    return 0;
+  }
+  if (pending.length === 0) return 0;
+
+  // A database no migration has touched holds nothing to lose.
+  const client = await connect(databaseUrl);
+  let applied;
+  let tables;
+  try {
+    applied = await latestMigration(
+      client,
+      connectionFromUrl(databaseUrl).schema,
+    );
+    tables = (await listUserTables(client)).length;
+  } finally {
+    await client.end().catch(() => {});
+  }
+  if (applied === null && tables === 0) {
+    log("new database, no dump before the first migrations.");
+    return 0;
+  }
+
+  const dir = resolveBackupDir(env);
+  if (!isWritableDirectory(dir)) {
+    warn(
+      `${dir} is not writable, so no dump is taken before the ${pending.length} pending migration(s). ` +
+        "Mount the backups volume (docker-compose.yml) to keep a way back; continuing.",
+    );
+    return 0;
+  }
+
+  log(
+    `taking a database dump before ${pending.length} pending migration(s) into ${dir}...`,
+  );
+  const started = Date.now();
+  try {
+    const { dumpPath, manifest } = await createDatabaseDump({
+      databaseUrl,
+      dir,
+      kind: "pre-migration",
+      env,
+      appVersion: imageVersion(env),
+      buildSha: (env.NEXT_PUBLIC_APP_BUILD_SHA ?? "").trim() || null,
+    });
+    log(
+      `pre-migration dump written: ${basename(dumpPath)} (${mb(manifest.dump.bytes)}, ` +
+        `${Object.keys(manifest.rowCounts).length} tables, ${Math.round((Date.now() - started) / 1000)} s).`,
+    );
+  } catch (error) {
+    console.error(
+      `HealthLog: ERROR — the database dump before migrating failed: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    return 1;
+  }
+  try {
+    await applyRetention(dir, { dailyKeep: resolveDailyKeep(env) });
+  } catch (error) {
+    warn(
+      `could not remove old dumps in ${dir} (${error instanceof Error ? error.message : String(error)}).`,
+    );
+  }
+  return 0;
+}
+
+async function main() {
+  const kind = process.argv[2];
+  if (kind !== "pre-migration") {
+    console.error("db-dump: usage: node scripts/db-dump.mjs pre-migration");
+    process.exit(2);
+  }
+  process.exit(await preMigrationDump());
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+  await main();
+}
diff --git a/scripts/db-restore.mjs b/scripts/db-restore.mjs
new file mode 100644
index 000000000..9718792e7
--- /dev/null
+++ b/scripts/db-restore.mjs
@@ -0,0 +1,331 @@
+/**
+ * Restore a whole-database dump into an empty database.
+ *
+ *   docker compose run --rm db-restore list [directory]
+ *   docker compose run --rm db-restore latest
+ *   docker compose run --rm db-restore <file> [--ignore-key-mismatch]
+ *
+ * Runs in the app image with plain `node` (no `tsx`, no source checkout), as
+ * the `db-restore` compose service, so a new server needs only the compose
+ * file, the keys and the dump. `<file>` is a name in `DB_BACKUP_DIR` or a
+ * path inside the container: a `.dump` written on a server, or a `.dump.enc`
+ * downloaded from the off-host bucket (`database/` prefix), with its manifest
+ * (`.manifest.json` or `.manifest.json.enc`) beside it.
+ *
+ * In this order, and it stops at the first thing that is wrong, before it
+ * writes anything:
+ *   1. an off-host copy is opened with BACKUP_ENCRYPTION_KEY (or a key in
+ *      BACKUP_ENCRYPTION_PREVIOUS_KEYS) into a temporary file;
+ *   2. the dump's SHA-256 and size must equal the manifest's;
+ *   3. the keys in this environment are compared with the manifest's
+ *      fingerprints; each one missing or different is named, and a mismatch
+ *      of the active encryption key refuses (`--ignore-key-mismatch` restores
+ *      anyway, for an operator who knows the key is lost);
+ *   4. the target database must hold no table at all. This never overwrites.
+ * Then `pg_restore` in one transaction, `ANALYZE`, and a count of every table
+ * against the manifest. A count that differs empties the database again and
+ * fails, so the app is never started on a restore that does not match.
+ *
+ * Exit 0 restored (or listed), 1 refused or failed, 2 usage.
+ */
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, dirname, isAbsolute, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import {
+  compareKeyFingerprints,
+  compareRowCounts,
+  connect,
+  countRows,
+  decryptOffhostFile,
+  emptyDatabase,
+  hashFile,
+  isGoodSet,
+  isWritableDirectory,
+  keyFingerprints,
+  listDumpSets,
+  listUserTables,
+  offhostKeysFromEnv,
+  offhostObjectKey,
+  parseDumpFileName,
+  parseManifest,
+  resolveBackupDir,
+  runTool,
+  statOrNull,
+  connectionFromUrl,
+} from "../src/lib/database-backup/dump-core.mjs";
+import { shippedMigrations } from "./pending-migrations.mjs";
+
+const out = (line = "") => console.log(line);
+const say = (line) => console.log(`db-restore: ${line}`);
+
+class Refusal extends Error {}
+
+function mb(bytes) {
+  return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}
+
+/**
+ * Whether this image knows the dump's newest migration. A dump from a newer
+ * release restored under an older image leaves the app with a schema its code
+ * does not expect, so the restore says which image to run.
+ */
+function imageKnowsMigration(name) {
+  if (!name) return true;
+  try {
+    return shippedMigrations(
+      join(process.cwd(), "prisma", "migrations"),
+    ).includes(name);
+  } catch {
+    return true;
+  }
+}
+
+function listCommand(dir) {
+  const sets = listDumpSets(dir);
+  if (sets.length === 0) {
+    say(`no dumps in ${dir}`);
+    return 0;
+  }
+  out(`Dumps in ${dir}, newest first:`);
+  for (const set of sets) {
+    const state = isGoodSet(set)
+      ? `${mb(set.dump.bytes)}`
+      : set.encryptedDump
+        ? "off-host copy (.enc)"
+        : set.dump
+          ? "no manifest, not restorable"
+          : "incomplete";
+    out(
+      `  ${set.base}.dump${set.encryptedDump && !set.dump ? ".enc" : ""}  ${set.kind}  ${set.timestamp.toISOString()}  ${state}`,
+    );
+  }
+  return 0;
+}
+
+/**
+ * The dump and manifest to restore: a local pair, or an encrypted pair
+ * opened into `scratch`.
+ */
+async function resolveInput(arg, dir, env) {
+  if (arg === "latest") {
+    const set = listDumpSets(dir).find(
+      (s) =>
+        isGoodSet(s) ||
+        (s.encryptedDump !== null &&
+          (s.encryptedManifest !== null || s.manifest !== null)),
+    );
+    if (!set) throw new Refusal(`no restorable dump in ${dir}`);
+    arg = set.dump ? set.dump.path : set.encryptedDump;
+  }
+  const path = isAbsolute(arg) ? arg : join(dir, arg);
+  if (!(await statOrNull(path))) throw new Refusal(`${path} does not exist`);
+  const name = basename(path);
+  const parsed = parseDumpFileName(name);
+  if (!parsed || parsed.role !== "dump" || parsed.variant === "partial") {
+    throw new Refusal(
+      `${name} is not a HealthLog database dump (healthlog-<time>-<kind>.dump or .dump.enc)`,
+    );
+  }
+  const folder = dirname(path);
+  const plainManifest = join(folder, `${parsed.base}.manifest.json`);
+  const encManifest = `${plainManifest}.enc`;
+
+  if (parsed.variant === "final") {
+    if (!(await statOrNull(plainManifest))) {
+      throw new Refusal(
+        `${basename(plainManifest)} is missing beside ${name}; a dump is restored only with its manifest`,
+      );
+    }
+    return {
+      dumpPath: path,
+      manifest: parseManifest(
+        JSON.parse(await readFile(plainManifest, "utf8")),
+      ),
+      scratch: null,
+    };
+  }
+
+  // An off-host copy. Open it next to the backups if there is room to write
+  // there, in the container's temp directory otherwise.
+  const keys = offhostKeysFromEnv(env);
+  if (keys.length === 0) {
+    throw new Refusal(
+      "this is an off-host copy and BACKUP_ENCRYPTION_KEY is not set; it cannot be opened without it",
+    );
+  }
+  const scratch = await mkdtemp(
+    join(isWritableDirectory(dir) ? dir : tmpdir(), ".restore-"),
+  );
+  try {
+    let manifest;
+    if (await statOrNull(encManifest)) {
+      const opened = join(scratch, `${parsed.base}.manifest.json`);
+      await decryptOffhostFile({
+        inPath: encManifest,
+        outPath: opened,
+        objectKey: offhostObjectKey(basename(encManifest)),
+        keys,
+      });
+      manifest = parseManifest(JSON.parse(await readFile(opened, "utf8")));
+    } else if (await statOrNull(plainManifest)) {
+      manifest = parseManifest(
+        JSON.parse(await readFile(plainManifest, "utf8")),
+      );
+    } else {
+      throw new Refusal(
+        `${basename(encManifest)} is missing beside ${name}; download it from the bucket too`,
+      );
+    }
+    say(`opening the off-host copy ${name}...`);
+    const dumpPath = join(scratch, `${parsed.base}.dump`);
+    await decryptOffhostFile({
+      inPath: path,
+      outPath: dumpPath,
+      objectKey: offhostObjectKey(name),
+      keys,
+    });
+    return { dumpPath, manifest, scratch };
+  } catch (error) {
+    await rm(scratch, { recursive: true, force: true });
+    throw error instanceof Refusal ? error : new Refusal(error.message);
+  }
+}
+
+async function restoreCommand(arg, flags, env) {
+  const dir = resolveBackupDir(env);
+  const databaseUrl = env.DATABASE_URL;
+  if (!databaseUrl) throw new Refusal("DATABASE_URL is not set");
+
+  const { dumpPath, manifest, scratch } = await resolveInput(arg, dir, env);
+  try {
+    say(
+      `dump ${manifest.file}: taken ${manifest.createdAt} by HealthLog ${manifest.appVersion ?? "(unknown version)"}, ` +
+        `newest migration ${manifest.latestMigration ?? "(none)"}, Postgres ${manifest.postgres.serverVersion}, ` +
+        `${Object.keys(manifest.rowCounts).length} tables`,
+    );
+    if (!imageKnowsMigration(manifest.latestMigration)) {
+      say(
+        `warning: this image does not know the dump's newest migration ${manifest.latestMigration}; ` +
+          `start the app with HealthLog ${manifest.appVersion ?? "the version that wrote it"} or newer (HEALTHLOG_IMAGE_REF), not this image`,
+      );
+    }
+    if (basename(dumpPath) !== manifest.file) {
+      throw new Refusal(
+        `the manifest describes ${manifest.file}, not ${basename(dumpPath)}`,
+      );
+    }
+
+    // 2. The file is the one the manifest describes.
+    const { sha256, bytes } = await hashFile(dumpPath);
+    if (sha256 !== manifest.dump.sha256 || bytes !== manifest.dump.bytes) {
+      throw new Refusal(
+        `the dump does not match its manifest (SHA-256 ${sha256.slice(0, 12)}…, ${bytes} bytes; ` +
+          `the manifest says ${manifest.dump.sha256.slice(0, 12)}…, ${manifest.dump.bytes} bytes). It is damaged or not this dump.`,
+      );
+    }
+    say(`checksum matches the manifest (${mb(bytes)})`);
+
+    // 3. The keys.
+    const keys = compareKeyFingerprints(manifest.keys, keyFingerprints(env));
+    if (keys.findings.length === 0) {
+      say("every key the dump was written with is configured here");
+    }
+    for (const finding of keys.findings) {
+      say(
+        `${finding.severity === "refuse" ? "KEY MISMATCH" : "warning"}: ${finding.key}: ${finding.detail}`,
+      );
+    }
+    if (keys.refuse && !flags.has("--ignore-key-mismatch")) {
+      throw new Refusal(
+        "refusing to restore: set the encryption keys this dump was written with (see above). " +
+          "Only if that key is lost for good, run again with --ignore-key-mismatch; everything encrypted will stay unreadable.",
+      );
+    }
+
+    // 4. Never over existing data.
+    const client = await connect(databaseUrl);
+    try {
+      const tables = await listUserTables(client);
+      if (tables.length > 0) {
+        throw new Refusal(
+          `the target database already holds ${tables.length} ${tables.length === 1 ? "table" : "tables"} (${tables.slice(0, 3).join(", ")}${tables.length > 3 ? ", …" : ""}). ` +
+            "A restore goes only into an empty database: start it on a new, empty database volume, and do not start the app before the restore.",
+        );
+      }
+
+      say("restoring (one transaction)...");
+      const started = Date.now();
+      await runTool(
+        "pg_restore",
+        [
+          "--no-owner",
+          "--no-privileges",
+          "--single-transaction",
+          "--exit-on-error",
+          "--no-password",
+          `--dbname=${connectionFromUrl(databaseUrl).pgEnv.PGDATABASE}`,
+          dumpPath,
+        ],
+        { env: connectionFromUrl(databaseUrl).pgEnv },
+      );
+      say(`restored in ${Math.round((Date.now() - started) / 1000)} s`);
+
+      await client.query("ANALYZE");
+      say("planner statistics gathered");
+
+      const problems = compareRowCounts(
+        manifest.rowCounts,
+        await countRows(client),
+      );
+      if (problems.length > 0) {
+        for (const line of problems.slice(0, 20)) say(`row count: ${line}`);
+        await emptyDatabase(client);
+        throw new Refusal(
+          `${problems.length} table(s) do not match the manifest; the database was emptied again. Do not start the app on it.`,
+        );
+      }
+      say(
+        `all ${Object.keys(manifest.rowCounts).length} tables hold the row counts the manifest names`,
+      );
+    } finally {
+      await client.end().catch(() => {});
+    }
+    out();
+    out(
+      `Restored ${manifest.file}. Start the app with the image this dump came from or a newer one ` +
+        `(HealthLog ${manifest.appVersion ?? "?"}, newest migration ${manifest.latestMigration ?? "none"}): docker compose up -d`,
+    );
+    return 0;
+  } finally {
+    if (scratch) await rm(scratch, { recursive: true, force: true });
+  }
+}
+
+export async function main(argv = process.argv.slice(2), env = process.env) {
+  const flags = new Set(argv.filter((a) => a.startsWith("--")));
+  const [command] = argv.filter((a) => !a.startsWith("--"));
+  if (!command) {
+    console.error(
+      "db-restore: usage: db-restore list [directory] | latest | <file> [--ignore-key-mismatch]",
+    );
+    return 2;
+  }
+  try {
+    if (command === "list") {
+      const [, other] = argv.filter((a) => !a.startsWith("--"));
+      return listCommand(other ?? resolveBackupDir(env));
+    }
+    return await restoreCommand(command, flags, env);
+  } catch (error) {
+    console.error(
+      `db-restore: ${error instanceof Refusal ? "" : "failed: "}${error instanceof Error ? error.message : String(error)}`,
+    );
+    return 1;
+  }
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+  process.exit(await main());
+}
diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json
index 890c6327c..9380b4fea 100644
--- a/scripts/env-manifest.json
+++ b/scripts/env-manifest.json
@@ -360,6 +360,38 @@
         }
       ]
     },
+    {
+      "name": "Database dumps",
+      "description": "The nightly whole-database dump and the dump before migrations (docs/ops/disaster-recovery.md). Optional: each has a default, and the bundled compose file mounts the `backups` volume at the default directory.",
+      "required": false,
+      "variables": [
+        {
+          "name": "DB_BACKUP_ENABLED",
+          "purpose": "Set false to stop the nightly database dump. On when unset.",
+          "example": "true"
+        },
+        {
+          "name": "DB_BACKUP_DIR",
+          "purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. Without a writable directory the nightly dump only goes off-host and no dump is taken before migrations.",
+          "example": "/var/lib/healthlog/backups"
+        },
+        {
+          "name": "DB_BACKUP_SCHEDULE",
+          "purpose": "Five-field cron for the nightly dump, in Europe/Berlin time like every scheduled job. Default 30 1 * * * (01:30).",
+          "example": "30 1 * * *"
+        },
+        {
+          "name": "DB_BACKUP_KEEP",
+          "purpose": "How many nightly dumps stay in DB_BACKUP_DIR (default 7, at least 1). The last two dumps taken before a migration are kept besides.",
+          "example": "7"
+        },
+        {
+          "name": "DB_BACKUP_BEFORE_MIGRATE",
+          "purpose": "Set false to start without the dump the entrypoint takes before applying migrations. On when unset; a failed dump stops the start until it works or this is false.",
+          "example": "true"
+        }
+      ]
+    },
     {
       "name": "Operational windows and cookie flags",
       "description": "Values the product states back to a user or that decide whether a session cookie is accepted. Optional: each has a documented default. They are listed so `pnpm check-env` reports them, and so the compose whitelist and the example files can be held in step with this manifest.",
diff --git a/scripts/pending-migrations.mjs b/scripts/pending-migrations.mjs
index 387e062dc..c39a12605 100644
--- a/scripts/pending-migrations.mjs
+++ b/scripts/pending-migrations.mjs
@@ -1,12 +1,11 @@
 /**
  * Boot-time notice before `prisma migrate deploy`: how many migrations this
- * image is about to apply, and a reminder to have a backup first.
+ * image is about to apply, and that a dump is taken first
+ * (`scripts/db-dump.mjs`, which imports `pendingMigrations` from here).
  *
  * Run by `docker-entrypoint.sh` as `node scripts/pending-migrations.mjs`.
  * It only reports. A migration that changes data cannot be undone by pulling
- * the previous image, so an operator who reads the count in the log still has
- * a moment to stop the container and take a `pg_dump`; the entrypoint does
- * not wait for that.
+ * the previous image; the dump the entrypoint takes next is the way back.
  *
  * Best effort by design: any failure to count (no `_prisma_migrations` table
  * yet is not a failure, it means every migration is pending) prints one line
@@ -67,9 +66,9 @@ export function pendingNotice(count) {
   const noun = count === 1 ? "migration" : "migrations";
   return (
     `HealthLog: ${count} database ${noun} pending. ` +
-    "Back up the database before updating (pg_dump, see the Updates section " +
-    "of docs/self-hosting/getting-started.md); a migration is not undone by " +
-    "going back to the previous image."
+    "Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is " +
+    "written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); " +
+    "a migration is not undone by going back to the previous image."
   );
 }
 
diff --git a/src/__tests__/compose-db-upgrade-guard.test.ts b/src/__tests__/compose-db-upgrade-guard.test.ts
index c4f892a10..1ab117a85 100644
--- a/src/__tests__/compose-db-upgrade-guard.test.ts
+++ b/src/__tests__/compose-db-upgrade-guard.test.ts
@@ -58,7 +58,9 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
   it("keeps the new data where the Postgres 18 image expects it", () => {
     // Mounted at .../data, as 16 had it, the 18 image refuses to start.
     expect(db.volumes).toEqual(["postgres-data:/var/lib/postgresql"]);
+    // `backups` holds the app's database dumps, not database files.
     expect(Object.keys(compose.volumes).sort()).toEqual([
+      "backups",
       "pgdata",
       "postgres-data",
     ]);