Merge the release into the backup health work and read the real dump health

The readiness model now takes the database dump's state from
getDatabaseBackupHealth, and the dump test reaches the off-host key id
where it moved.

scripts/__tests__/db-restore.test.ts

@@ -0,0 +1,161 @@
+/**
+ * The restore's refusals, which all come before it writes anything.
+ *
+ * The database is a stand-in here: `connect` answers a client whose only use
+ * is to be asked which tables exist, and `runTool` (which would run
+ * `pg_restore`) is a spy that must never be called on a refusal. The real
+ * restore into a real empty database, and its refusal of a real database with
+ * tables, are in `tests/integration/database-backup.test.ts` and in the
+ * compose drill (`scripts/compose-dr-drill.sh`).
+ */
+import { createHash } from "node:crypto";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const tablesInTarget = vi.fn(async (): Promise<string[]> => []);
+const runTool = vi.fn(async () => "");
+const end = vi.fn(async () => {});
+
+vi.mock("../../src/lib/database-backup/dump-core.mjs", async (original) => ({
+  ...(await original<Record<string, unknown>>()),
+  connect: async () => ({ query: async () => ({ rows: [] }), end }),
+  listUserTables: () => tablesInTarget(),
+  runTool: (...args: unknown[]) => runTool(...(args as [])),
+}));
+
+import { buildManifest } from "../../src/lib/database-backup/dump-core.mjs";
+import { main } from "../db-restore.mjs";
+
+const KEY_A = "a".repeat(64);
+const KEY_B = "b".repeat(64);
+const BASE = "healthlog-20261025T013000Z-daily";
+
+let dir: string;
+let errors: string[];
+let lines: string[];
+
+function env(extra: Record<string, string> = {}) {
+  return {
+    DATABASE_URL: "postgresql://healthlog:pw@db:5432/healthlog?schema=public",
+    DB_BACKUP_DIR: dir,
+    ENCRYPTION_KEY: KEY_A,
+    API_TOKEN_HMAC_KEY: "hmac",
+    ...extra,
+  };
+}
+
+/** A dump file and a manifest that describes it, written with KEY_A. */
+function writeDump(body = Buffer.from("PGDMP not really an archive")) {
+  writeFileSync(join(dir, `${BASE}.dump`), body);
+  const manifest = buildManifest({
+    kind: "daily",
+    file: `${BASE}.dump`,
+    createdAt: new Date("2026-10-25T01:30:00Z"),
+    appVersion: "1.46.0",
+    buildSha: null,
+    latestMigration: "0486_database_backup_runs",
+    server: { version: "18.6", versionNum: 180006 },
+    pgDump: null,
+    archive: {
+      bytes: body.length,
+      sha256: createHash("sha256").update(body).digest("hex"),
+      entries: 1,
+    },
+    rowCounts: { "public.users": 2 },
+    env: { ENCRYPTION_KEY: KEY_A, API_TOKEN_HMAC_KEY: "hmac" },
+  });
+  writeFileSync(
+    join(dir, `${BASE}.manifest.json`),
+    JSON.stringify(manifest, null, 2),
+  );
+}
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), "hl-db-restore-"));
+  errors = [];
+  lines = [];
+  vi.spyOn(console, "error").mockImplementation((line: string) => {
+    errors.push(String(line));
+  });
+  vi.spyOn(console, "log").mockImplementation((line: string) => {
+    lines.push(String(line));
+  });
+  tablesInTarget.mockResolvedValue([]);
+  runTool.mockClear();
+});
+afterEach(() => {
+  rmSync(dir, { recursive: true, force: true });
+  vi.restoreAllMocks();
+});
+
+describe("db-restore refuses before it writes", () => {
+  it("refuses a target database that already holds tables", async () => {
+    writeDump();
+    tablesInTarget.mockResolvedValue(["public.users", "public.sessions"]);
+    expect(await main(["latest"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(
+      /already holds 2 tables .*only into an empty database/,
+    );
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("refuses when the active encryption key is not the dump's, and names it", async () => {
+    writeDump();
+    expect(await main(["latest"], env({ ENCRYPTION_KEY: KEY_B }))).toBe(1);
+    expect(lines.join("\n")).toMatch(/KEY MISMATCH: ENCRYPTION_KEYS\[v1\]/);
+    expect(errors.join("\n")).toMatch(/refusing to restore/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("goes past the key check with --ignore-key-mismatch, and still refuses a full database", async () => {
+    writeDump();
+    tablesInTarget.mockResolvedValue(["public.users"]);
+    expect(
+      await main(
+        ["latest", "--ignore-key-mismatch"],
+        env({ ENCRYPTION_KEY: KEY_B }),
+      ),
+    ).toBe(1);
+    expect(errors.join("\n")).toMatch(/already holds 1 table /);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("refuses a dump that does not match its manifest", async () => {
+    writeDump();
+    writeFileSync(join(dir, `${BASE}.dump`), "PGDMP altered");
+    expect(await main(["latest"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/does not match its manifest/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("refuses a dump without its manifest, and a file it did not write", async () => {
+    writeFileSync(join(dir, `${BASE}.dump`), "x");
+    expect(await main([`${BASE}.dump`], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/manifest.json is missing/);
+    writeFileSync(join(dir, "other.dump"), "x");
+    expect(await main(["other.dump"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/not a HealthLog database dump/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("says when there is nothing to restore, and how it is used", async () => {
+    expect(await main(["latest"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/no restorable dump/);
+    expect(await main([], env())).toBe(2);
+  });
+
+  it("refuses --from-bucket without a configured bucket, before it connects", async () => {
+    expect(await main(["latest", "--from-bucket"], env())).toBe(1);
+    expect(errors.join("\n")).toMatch(/--from-bucket needs BACKUP_S3_ENDPOINT/);
+    expect(runTool).not.toHaveBeenCalled();
+  });
+
+  it("lists what is there", async () => {
+    writeDump();
+    expect(await main(["list"], env())).toBe(0);
+    expect(lines.join("\n")).toContain(`${BASE}.dump  daily`);
+  });
+});

scripts/compose-db-upgrade-check.sh

@@ -21,7 +21,9 @@ ROOT=$(git rev-parse --show-toplevel)
 # The last release whose compose file ran Postgres 16.
 OLD_REF=${OLD_REF:-v1.45.0}
 PROJECT=hl-db-upgrade-check
-PORT=${CHECK_DB_PORT:-55416}
+# A free port unless CHECK_DB_PORT names one, so the check never collides with
+# another stack on the same machine.
+PORT=${CHECK_DB_PORT:-$(node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})")}
 WORK=$(mktemp -d)
 
 export PRISMA_HIDE_UPDATE_MESSAGE=1
@@ -63,11 +65,40 @@ run_upgrade() {
 }
 upgrade_log() { docker logs healthlog-db-upgrade 2>&1 | grep '^db-upgrade:' || true; }
 
+# The container can report healthy before its published port forwards (Docker
+# Desktop and OrbStack set the forward up a moment later), and a migrate
+# started in that gap failed with "Can't reach database server". So wait
+# until the port answers, for at most thirty seconds, and retry the migrate
+# within the same bound.
+wait_for_port() {
+  local i=0
+  until node -e "
+    const s = require('node:net').connect(Number(process.argv[1]), '127.0.0.1');
+    s.on('connect', () => { s.end(); process.exit(0); });
+    s.on('error', () => process.exit(1));
+  " "$PORT" 2>/dev/null; do
+    i=$((i + 1))
+    [ "$i" -lt 30 ] || return 1
+    sleep 1
+  done
+}
+migrate_old() {
+  local started=$SECONDS
+  until DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
+    node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do
+    if [ $((SECONDS - started)) -ge 30 ] || ! grep -q "P1001" "$WORK/migrate.log"; then
+      cat "$WORK/migrate.log" >&2
+      return 1
+    fi
+    sleep 1
+  done
+}
+
 down_all
 step "a v1.45 database on Postgres 16"
-old up -d --wait db >/dev/null 2>&1
-DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
-  node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >/dev/null
+old up -d --wait db >"$WORK/up.log" 2>&1 || { cat "$WORK/up.log" >&2; fail "the v1.45 database did not start"; }
+wait_for_port || fail "port $PORT did not answer within thirty seconds"
+migrate_old || fail "the migrations did not apply to the v1.45 database"
 old exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -q <<'SQL'
 CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
 CREATE SCHEMA check_probe;

scripts/compose-dr-drill.sh

@@ -0,0 +1,346 @@
+#!/usr/bin/env bash
+# Plays the documented disaster recovery end to end against real containers:
+# a server is lost with everything on it, and a new one is rebuilt from the
+# nightly database dump and the keys alone (docs/ops/disaster-recovery.md).
+#
+#   1. Build the app image from this checkout and start the bundled compose
+#      stack with fixed throwaway keys, and an S3-compatible bucket outside
+#      it (Versity S3 Gateway, pinned) as the off-host target.
+#   2. Seed real data: the demo seed (documents, encrypted notes, years of
+#      readings) and a Guardian with a managed profile, written by the app's
+#      own code; then, through the API, a reading with an encrypted note.
+#   3. Let the scheduled `database-backup` job take the dump (its cron is set
+#      to every minute here) and wait for its ledger row: kept on the server
+#      and uploaded to the bucket.
+#   4. DESTROY every volume of the stack, the local dumps included. The
+#      bucket lives outside the stack and survives, as an off-host copy does.
+#   5. Start a new stack and restore with the keys alone:
+#      DRILL_SOURCE=bucket (the default, the disaster path)
+#        `docker compose run --rm db-restore latest --from-bucket`;
+#      DRILL_SOURCE=local (the dump files had been copied off the host)
+#        the files go into the new backups volume, then
+#        `docker compose run --rm db-restore latest`.
+#   6. Count every table against the manifest, independently of the restore.
+#   7. Boot the app on the restored database and check: the health endpoint,
+#      the demo login, the encrypted note read back through the API, a
+#      document opened, and the Guardian still seeing the managed profile.
+#
+# DRILL_MUTATION breaks one step on purpose, and the drill must then fail
+# (a check that cannot fail proves nothing):
+#   wrong-key         restore with a different active encryption key
+#   wrong-backup-key  restore from the bucket with a different
+#                     BACKUP_ENCRYPTION_KEY (DRILL_SOURCE=bucket)
+#   skip-table        a dump whose archive lacks one table's rows, the
+#                     manifest's checksum fixed up and its counts left true
+#                     (DRILL_SOURCE=local)
+#   non-empty-target  the new app starts and migrates before the restore
+#
+# Needs Docker, openssl, pnpm and the repo's node_modules (for the seeds). Run
+# it from anywhere in the repo: `bash scripts/compose-dr-drill.sh`.
+# DRILL_IMAGE names an image to use instead of building one. The ports are
+# free ones picked at start unless DRILL_APP_PORT, DRILL_DB_PORT or
+# DRILL_BUCKET_PORT name them.
+set -euo pipefail
+
+ROOT=$(git rev-parse --show-toplevel)
+PROJECT=hl-dr-drill
+free_port() {
+  node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})"
+}
+APP_PORT=${DRILL_APP_PORT:-$(free_port)}
+DB_PORT=${DRILL_DB_PORT:-$(free_port)}
+BUCKET_PORT=${DRILL_BUCKET_PORT:-$(free_port)}
+IMAGE=${DRILL_IMAGE:-}
+MUTATION=${DRILL_MUTATION:-}
+SOURCE=${DRILL_SOURCE:-bucket}
+OFFSITE=$PROJECT-offsite
+BUCKET=$PROJECT-bucket
+BUCKET_IMAGE=versity/versitygw:v1.8.0@sha256:30292fc2eeacc67a36993b01f7a7a5e3361a19cced0e80c1d71cfa2a4b0a2499
+WORK=$(mktemp -d)
+BASE_URL="http://127.0.0.1:$APP_PORT"
+
+case "$SOURCE/$MUTATION" in
+  bucket/ | local/ | */wrong-key | */non-empty-target | bucket/wrong-backup-key | local/skip-table) ;;
+  *)
+    echo "DRILL_SOURCE=$SOURCE with DRILL_MUTATION=$MUTATION is not a drill" >&2
+    exit 2
+    ;;
+esac
+
+export PRISMA_HIDE_UPDATE_MESSAGE=1
+export POSTGRES_PASSWORD
+POSTGRES_PASSWORD=$(openssl rand -hex 16)
+# Fixed throwaway keys: the restore gets these and nothing else of the lost
+# host. Two encryption keys, the second active, so a retired key is carried.
+export ENCRYPTION_KEY=""
+export ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"2222222222222222222222222222222222222222222222222222222222222222"}'
+export ENCRYPTION_ACTIVE_KEY_ID=v2
+export API_TOKEN_HMAC_KEY=3333333333333333333333333333333333333333333333333333333333333333
+export SESSION_COOKIE_SECURE=false
+export NEXT_PUBLIC_APP_URL=$BASE_URL APP_URL=$BASE_URL
+export DB_BACKUP_SCHEDULE="* * * * *"
+# The off-host bucket, reached from the stack by its container name on a
+# network of its own that outlives the stack, as a real bucket does.
+export BACKUP_S3_ENDPOINT=http://$BUCKET:7070
+export BACKUP_S3_BUCKET=healthlog-backups
+export BACKUP_S3_ACCESS_KEY=drill-access
+export BACKUP_S3_SECRET_KEY=drill-secret-0123456789
+export BACKUP_S3_REGION=us-east-1
+export BACKUP_ENCRYPTION_KEY=5555555555555555555555555555555555555555555555555555555555555555
+
+step() { echo "== $*"; }
+fail() {
+  echo "FAIL: $*" >&2
+  docker logs "$PROJECT-app" 2>&1 | tail -40 >&2 || true
+  exit 1
+}
+
+if [ -z "$IMAGE" ]; then
+  IMAGE=healthlog-dr-drill:local
+  step "building the app image from this checkout"
+  docker build -q \
+    --build-arg NEXT_PUBLIC_APP_VERSION="drill-$(git -C "$ROOT" rev-parse --short HEAD)" \
+    --build-arg NEXT_PUBLIC_APP_BUILD_SHA="$(git -C "$ROOT" rev-parse --short HEAD)" \
+    -t "$IMAGE" "$ROOT" >/dev/null
+fi
+
+# The bundled compose file, with this image, its own names and ports, so the
+# drill never meets another stack on the same machine, and the app and the
+# restore on the bucket's network.
+cat >"$WORK/drill.yml" <<YAML
+services:
+  app:
+    image: $IMAGE
+    pull_policy: never
+    container_name: $PROJECT-app
+    ports: !override
+      - "127.0.0.1:$APP_PORT:3000"
+    networks: [default, bucket]
+  db:
+    container_name: $PROJECT-db
+    ports:
+      - "127.0.0.1:$DB_PORT:5432"
+  db-upgrade:
+    container_name: $PROJECT-db-upgrade
+  db-restore:
+    image: $IMAGE
+    pull_policy: never
+    networks: [default, bucket]
+networks:
+  bucket:
+    external: true
+    name: $BUCKET
+YAML
+
+stack() { docker compose -p "$PROJECT" -f "$ROOT/docker-compose.yml" -f "$WORK/drill.yml" "$@"; }
+sql() { docker exec "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$1"; }
+teardown() {
+  stack down -v --remove-orphans >/dev/null 2>&1 || true
+  docker volume rm "$OFFSITE" >/dev/null 2>&1 || true
+  docker rm -f "$BUCKET" >/dev/null 2>&1 || true
+  docker volume rm "$BUCKET-data" >/dev/null 2>&1 || true
+  docker network rm "$BUCKET" >/dev/null 2>&1 || true
+}
+trap 'teardown; rm -rf "$WORK"' EXIT
+
+wait_healthy() {
+  local i=0
+  until [ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/api/health")" = 200 ]; do
+    i=$((i + 1))
+    [ "$i" -lt 150 ] || fail "the app did not report healthy within five minutes"
+    docker inspect -f '{{.State.Running}}' "$PROJECT-app" 2>/dev/null | grep -q true ||
+      fail "the app container stopped"
+    sleep 2
+  done
+}
+
+# POST/GET as a signed-in account, with its own cookie jar.
+login() {
+  local jar=$1 user=$2 password=$3
+  local code
+  code=$(curl -s -o "$WORK/login.json" -w '%{http_code}' -c "$jar" \
+    -H 'content-type: application/json' \
+    -d "{\"email\":\"$user\",\"password\":\"$password\"}" "$BASE_URL/api/auth/login")
+  [ "$code" = 200 ] || fail "login as $user answered $code: $(cat "$WORK/login.json")"
+}
+api() { curl -s -b "$1" -w '\n%{http_code}' "${@:2}"; }
+json() { node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const v=JSON.parse(s);console.log((${1})(v))})"; }
+
+# A shell in the app image as root, with volumes, for moving the dump files
+# the way an operator copies them; `cp -p` keeps the app user as owner.
+as_root() {
+  local command=$1 mounts=()
+  shift
+  for volume in "$@"; do mounts+=(-v "$volume"); done
+  docker run --rm --user 0:0 --entrypoint sh "${mounts[@]}" "$IMAGE" -c "$command"
+}
+
+teardown
+
+step "an off-host bucket, outside the stack"
+docker network create "$BUCKET" >/dev/null
+docker run -d --name "$BUCKET" --network "$BUCKET" -p "127.0.0.1:$BUCKET_PORT:7070" \
+  -v "$BUCKET-data:/data" --entrypoint sh "$BUCKET_IMAGE" -c \
+  "mkdir -p /data/$BACKUP_S3_BUCKET && exec versitygw --access $BACKUP_S3_ACCESS_KEY --secret $BACKUP_S3_SECRET_KEY posix /data" \
+  >/dev/null || fail "the bucket did not start"
+
+step "a running server with real data"
+stack up -d app >"$WORK/up.log" 2>&1 || { tail -20 "$WORK/up.log" >&2; fail "the stack did not start"; }
+wait_healthy
+DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$DB_PORT/healthlog?schema=public"
+export DATABASE_URL
+(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/seed-demo.ts) >"$WORK/seed.log" 2>&1 ||
+  { tail -20 "$WORK/seed.log" >&2; fail "the demo seed failed"; }
+IDS=$(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/dr-drill-seed.ts | tail -1)
+PROFILE_ID=$(echo "$IDS" | json 'v=>v.profileId')
+unset DATABASE_URL
+
+login "$WORK/demo.jar" demo demo123demo123
+NOTE="drill note $(openssl rand -hex 6)"
+OUT=$(api "$WORK/demo.jar" -H 'content-type: application/json' \
+  -d "{\"type\":\"WEIGHT\",\"value\":81.4,\"measuredAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"notes\":\"$NOTE\"}" \
+  "$BASE_URL/api/measurements")
+[ "$(echo "$OUT" | tail -1)" = 201 ] || [ "$(echo "$OUT" | tail -1)" = 200 ] ||
+  fail "creating a reading with a note answered: $OUT"
+READING_ID=$(echo "$OUT" | sed '$d' | json 'v=>v.data.id')
+[ "$(sql "SELECT count(*) FROM measurements WHERE id = '$READING_ID' AND position('$NOTE' in measurements::text) > 0")" = 0 ] ||
+  fail "the note is stored in plain text"
+DOCUMENTS=$(sql "SELECT count(*) FROM inbound_documents WHERE deleted_at IS NULL")
+[ "$DOCUMENTS" -gt 0 ] || fail "the seed wrote no documents"
+SEEDED_AT=$(sql "SELECT now()")
+
+step "the scheduled job takes the dump and uploads it"
+i=0
+until DUMP=$(sql "SELECT file_name FROM database_backup_runs WHERE status = 'succeeded' AND local_result = 'written' AND offhost_result = 'uploaded' AND started_at > '$SEEDED_AT'::timestamptz ORDER BY started_at DESC LIMIT 1") && [ -n "$DUMP" ]; do
+  i=$((i + 1))
+  [ "$i" -lt 120 ] || fail "no succeeded and uploaded database-backup run within four minutes: $(sql "SELECT status, local_result, offhost_result, error FROM database_backup_runs ORDER BY started_at DESC LIMIT 3")"
+  sleep 2
+done
+BASE=${DUMP%.dump}
+echo "   $DUMP"
+
+if [ "$SOURCE" = local ]; then
+  step "the dump files are copied off the host"
+  docker volume create "$OFFSITE" >/dev/null
+  as_root "cp -p /from/$BASE.dump /from/$BASE.manifest.json /to/" \
+    "${PROJECT}_backups:/from:ro" "$OFFSITE:/to" ||
+    fail "could not copy the dump off the host"
+fi
+
+step "the host is lost: every volume of the stack is destroyed"
+stack down -v --remove-orphans >/dev/null 2>&1
+[ -z "$(docker volume ls -q --filter "name=^${PROJECT}_")" ] || fail "a volume of the old stack survived"
+
+if [ "$MUTATION" = skip-table ]; then
+  step "MUTATION: the dump loses the rows of public.measurements"
+  # A scratch Postgres restores the real dump and dumps it again without that
+  # table's rows; the files stay the app user's, read and written by root.
+  docker run --rm -v "$OFFSITE:/d" -e BASE="$BASE" --entrypoint sh postgres:18.6-alpine -ec '
+    gosu postgres initdb -D /tmp/pg >/dev/null
+    gosu postgres pg_ctl -D /tmp/pg -o "-c listen_addresses=" -w start >/dev/null
+    gosu postgres createdb scratch
+    gosu postgres pg_restore --no-owner --no-privileges -d scratch <"/d/$BASE.dump"
+    gosu postgres pg_dump -Fc --exclude-table-data=public.measurements scratch >"/d/$BASE.dump"
+    sum=$(sha256sum "/d/$BASE.dump" | cut -d" " -f1)
+    size=$(wc -c <"/d/$BASE.dump")
+    sed -i "s/\"sha256\": \"[0-9a-f]*\"/\"sha256\": \"$sum\"/; s/\"bytes\": [0-9]*/\"bytes\": $size/" "/d/$BASE.manifest.json"
+    chown 1001:1001 "/d/$BASE.dump"' || fail "could not build the mutated dump"
+fi
+
+step "a new server: the compose file, the keys and the $SOURCE copy"
+if [ "$MUTATION" = wrong-key ]; then
+  echo "   MUTATION: the restore gets a different active encryption key"
+  ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"4444444444444444444444444444444444444444444444444444444444444444"}'
+fi
+if [ "$MUTATION" = wrong-backup-key ]; then
+  echo "   MUTATION: the restore gets a different BACKUP_ENCRYPTION_KEY"
+  BACKUP_ENCRYPTION_KEY=6666666666666666666666666666666666666666666666666666666666666666
+fi
+if [ "$SOURCE" = local ]; then
+  # Mounting the image's directory fills the new volume with the app user's
+  # ownership; then the two files go in.
+  stack run --rm --no-deps --entrypoint true db-restore >/dev/null 2>&1
+  as_root "cp -p /from/* /to/" "$OFFSITE:/from:ro" "${PROJECT}_backups:/to" ||
+    fail "could not put the dump on the new server"
+  RESTORE=(latest)
+else
+  RESTORE=(latest --from-bucket)
+fi
+if [ "$MUTATION" = non-empty-target ]; then
+  echo "   MUTATION: the new app starts (and migrates) before the restore"
+  stack up -d app >/dev/null 2>&1
+  wait_healthy
+fi
+stack up -d db >/dev/null 2>&1
+if ! stack run --rm db-restore "${RESTORE[@]}" 2>&1 | tee "$WORK/restore.log"; then
+  fail "the restore failed"
+fi
+grep -q "^Restored $DUMP" "$WORK/restore.log" || fail "the restore did not say it restored $DUMP"
+
+step "every table against the manifest, counted apart from the restore"
+if [ "$SOURCE" = local ]; then
+  as_root "cat /d/$BASE.manifest.json" "$OFFSITE:/d:ro" >"$WORK/manifest.json"
+else
+  # From the bucket, opened with the restore's own reader.
+  (cd "$ROOT" && node --input-type=module -e '
+    import { writeFileSync } from "node:fs";
+    import { join } from "node:path";
+    const core = await import(join(process.cwd(), "src/lib/database-backup/dump-core.mjs"));
+    const [port, base, work] = process.argv.slice(1);
+    const bucket = await core.openBucket({ ...core.bucketConfigFromEnv(process.env), endpoint: `http://127.0.0.1:${port}` });
+    const name = `${base}.manifest.json.enc`;
+    await bucket.download(core.offhostObjectKey(name), join(work, name));
+    await core.decryptOffhostFile({ inPath: join(work, name), outPath: join(work, "manifest.json"), objectKey: core.offhostObjectKey(name), keys: core.offhostKeysFromEnv(process.env) });
+  ' "$BUCKET_PORT" "$BASE" "$WORK") || fail "could not read the manifest from the bucket"
+fi
+node -e '
+  const m = require(process.argv[1]);
+  const q = String.fromCharCode(39);
+  const lit = (t) => q + t.split(q).join(q + q) + q;
+  console.log(Object.keys(m.rowCounts).map((t) => `SELECT ${lit(t)} || chr(9) || count(*) FROM ${t}`).join(" UNION ALL ") + ";");
+' "$WORK/manifest.json" >"$WORK/count.sql"
+docker exec -i "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atq <"$WORK/count.sql" >"$WORK/counts.tsv"
+node -e '
+  const fs = require("node:fs");
+  const m = require(process.argv[1]);
+  const got = Object.fromEntries(fs.readFileSync(process.argv[2], "utf8").trim().split("\n").map((l) => l.split("\t")).map(([t, n]) => [t, Number(n)]));
+  const bad = Object.entries(m.rowCounts).filter(([t, n]) => got[t] !== n);
+  if (bad.length) { console.error(bad.map(([t, n]) => `${t}: manifest ${n}, database ${got[t]}`).join("\n")); process.exit(1); }
+  if (m.rowCounts["public.users"] < 3) { console.error("fewer than three accounts in the dump"); process.exit(1); }
+  const c = m.rowCounts;
+  console.log(`   ${Object.keys(c).length} tables, ${Object.values(c).reduce((a, b) => a + b, 0)} rows: ` +
+    `${c["public.users"]} accounts, ${c["public.account_grants"]} grants, ${c["public.measurements"]} readings, ` +
+    `${c["public.inbound_documents"]} documents, ${c["pgboss.job"] ?? 0} queued jobs`);
+' "$WORK/manifest.json" "$WORK/counts.tsv" || fail "the restored row counts differ from the manifest"
+
+step "the app on the restored database"
+stack up -d app >/dev/null 2>&1 || fail "the app did not start on the restored database"
+wait_healthy
+[ "$(curl -s "$BASE_URL/api/health" | json 'v=>v.status ?? v.data?.status')" = ok ] || fail "health is not ok"
+
+login "$WORK/demo2.jar" demo demo123demo123
+OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/measurements/$READING_ID")
+[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "reading the note back answered: $OUT"
+[ "$(echo "$OUT" | sed '$d' | json 'v=>v.data.notes')" = "$NOTE" ] || fail "the encrypted note did not read back"
+echo "   the encrypted note reads back"
+
+OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/documents/inbound")
+[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing documents answered: $OUT"
+DOC_ID=$(echo "$OUT" | sed '$d' | json 'v=>(v.data.documents ?? v.data.items ?? v.data)[0].id')
+CODE=$(curl -s -b "$WORK/demo2.jar" -o "$WORK/doc.bin" -w '%{http_code}' "$BASE_URL/api/documents/inbound/$DOC_ID/original")
+[ "$CODE" = 200 ] || fail "opening document $DOC_ID answered $CODE"
+[ "$(head -c 4 "$WORK/doc.bin")" = "%PDF" ] || fail "document $DOC_ID did not open to a PDF"
+echo "   a document opens ($(wc -c <"$WORK/doc.bin" | tr -d ' ') bytes)"
+
+login "$WORK/guardian.jar" drill-guardian drill-guardian-password-1
+OUT=$(api "$WORK/guardian.jar" "$BASE_URL/api/account/grants")
+[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing the Guardian's grants answered: $OUT"
+echo "$OUT" | sed '$d' | node -e "
+  let s=''; process.stdin.on('data', d => s += d).on('end', () => {
+    const received = JSON.parse(s).data.received ?? [];
+    const found = received.some((g) => JSON.stringify(g).includes('$PROFILE_ID'));
+    process.exit(found ? 0 : 1);
+  });" || fail "the Guardian no longer sees the managed profile"
+echo "   the Guardian still sees the managed profile"
+
+echo "OK: the server was rebuilt from the $SOURCE copy of its dump and the keys alone"

scripts/db-dump.mjs

@@ -0,0 +1,156 @@
+/**
+ * The pre-migration dump: a whole-database dump the entrypoint takes before
+ * `prisma migrate deploy` applies anything.
+ *
+ *   node scripts/db-dump.mjs pre-migration
+ *
+ * A migration is not undone by going back to the previous image, and the
+ * bundled compose file pulls `:latest` and migrates on every start. So when
+ * this image brings migrations the database has not applied, it first writes
+ * `healthlog-<timestamp>-pre-migration.dump` with its manifest into
+ * `DB_BACKUP_DIR` (`src/lib/database-backup/dump-core.mjs`), and keeps the
+ * newest two.
+ *
+ * Exit 0: the dump was written, or there was nothing to do (no pending
+ * migration, a new database, `DB_BACKUP_BEFORE_MIGRATE=false`), or there is no
+ * writable directory to put it in, which is said as a warning and is not a
+ * reason to keep the app down. Exit 1: the directory is writable and the dump
+ * failed. The entrypoint then stops the boot and names the switch that skips
+ * the dump, because migrating without the way back the operator was promised
+ * is the one thing this step exists to prevent.
+ *
+ * Fast on purpose: one snapshot, `pg_dump`, a count per table for the
+ * manifest, `pg_restore --list` to prove the file reads. Nothing else.
+ */
+import { readFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import {
+  applyRetention,
+  connect,
+  createDatabaseDump,
+  isWritableDirectory,
+  latestMigration,
+  listUserTables,
+  connectionFromUrl,
+  preMigrationDumpEnabled,
+  resolveBackupDir,
+  resolveDailyKeep,
+} from "../src/lib/database-backup/dump-core.mjs";
+import { pendingMigrations } from "./pending-migrations.mjs";
+
+const log = (line) => console.log(`HealthLog: ${line}`);
+const warn = (line) => console.error(`HealthLog: WARNING — ${line}`);
+
+/** The image's version: the build argument, else package.json beside it. */
+function imageVersion(env) {
+  const fromEnv = (env.NEXT_PUBLIC_APP_VERSION ?? "").trim();
+  if (fromEnv) return fromEnv;
+  try {
+    return JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8"))
+      .version;
+  } catch {
+    return null;
+  }
+}
+
+/** Megabytes as an operator reads them. */
+function mb(bytes) {
+  return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}
+
+export async function preMigrationDump(env = process.env) {
+  if (!preMigrationDumpEnabled(env)) {
+    log("DB_BACKUP_BEFORE_MIGRATE=false, no dump before migrating.");
+    return 0;
+  }
+  const databaseUrl = env.DATABASE_URL;
+  if (!databaseUrl) return 0;
+
+  let pending;
+  try {
+    pending = await pendingMigrations({
+      connectionString: databaseUrl,
+      migrationsDir: join(process.cwd(), "prisma", "migrations"),
+    });
+  } catch (error) {
+    warn(
+      `could not count pending migrations (${error instanceof Error ? error.message : String(error)}); no dump before migrating.`,
+    );
+    return 0;
+  }
+  if (pending.length === 0) return 0;
+
+  // A database no migration has touched holds nothing to lose.
+  const client = await connect(databaseUrl);
+  let applied;
+  let tables;
+  try {
+    applied = await latestMigration(
+      client,
+      connectionFromUrl(databaseUrl).schema,
+    );
+    tables = (await listUserTables(client)).length;
+  } finally {
+    await client.end().catch(() => {});
+  }
+  if (applied === null && tables === 0) {
+    log("new database, no dump before the first migrations.");
+    return 0;
+  }
+
+  const dir = resolveBackupDir(env);
+  if (!isWritableDirectory(dir)) {
+    warn(
+      `${dir} is not writable, so no dump is taken before the ${pending.length} pending migration(s). ` +
+        "Mount the backups volume (docker-compose.yml) to keep a way back; continuing.",
+    );
+    return 0;
+  }
+
+  log(
+    `taking a database dump before ${pending.length} pending migration(s) into ${dir}...`,
+  );
+  const started = Date.now();
+  try {
+    const { dumpPath, manifest } = await createDatabaseDump({
+      databaseUrl,
+      dir,
+      kind: "pre-migration",
+      env,
+      appVersion: imageVersion(env),
+      buildSha: (env.NEXT_PUBLIC_APP_BUILD_SHA ?? "").trim() || null,
+    });
+    log(
+      `pre-migration dump written: ${basename(dumpPath)} (${mb(manifest.dump.bytes)}, ` +
+        `${Object.keys(manifest.rowCounts).length} tables, ${Math.round((Date.now() - started) / 1000)} s).`,
+    );
+  } catch (error) {
+    console.error(
+      `HealthLog: ERROR — the database dump before migrating failed: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    return 1;
+  }
+  try {
+    await applyRetention(dir, { dailyKeep: resolveDailyKeep(env) });
+  } catch (error) {
+    warn(
+      `could not remove old dumps in ${dir} (${error instanceof Error ? error.message : String(error)}).`,
+    );
+  }
+  return 0;
+}
+
+async function main() {
+  const kind = process.argv[2];
+  if (kind !== "pre-migration") {
+    console.error("db-dump: usage: node scripts/db-dump.mjs pre-migration");
+    process.exit(2);
+  }
+  process.exit(await preMigrationDump());
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+  await main();
+}

scripts/db-restore.mjs

@@ -0,0 +1,415 @@
+/**
+ * Restore a whole-database dump into an empty database.
+ *
+ *   docker compose run --rm db-restore list [directory]
+ *   docker compose run --rm db-restore list --from-bucket
+ *   docker compose run --rm db-restore latest [--from-bucket]
+ *   docker compose run --rm db-restore <file> [--ignore-key-mismatch]
+ *
+ * Runs in the app image with plain `node` (no `tsx`, no source checkout), as
+ * the `db-restore` compose service, so a new server needs only the compose
+ * file, the keys and the dump. `<file>` is a name in `DB_BACKUP_DIR` or a
+ * path inside the container: a `.dump` written on a server, or a `.dump.enc`
+ * downloaded from the off-host bucket (`database/` prefix), with its manifest
+ * (`.manifest.json` or `.manifest.json.enc`) beside it. With `--from-bucket`
+ * it fetches the pair from the bucket the `BACKUP_S3_*` variables name (the
+ * newest for `latest`), which is the restore after the host and its local
+ * dumps are gone.
+ *
+ * In this order, and it stops at the first thing that is wrong, before it
+ * writes anything:
+ *   1. an off-host copy is opened with BACKUP_ENCRYPTION_KEY (or a key in
+ *      BACKUP_ENCRYPTION_PREVIOUS_KEYS) into a temporary file;
+ *   2. the dump's SHA-256 and size must equal the manifest's;
+ *   3. the keys in this environment are compared with the manifest's
+ *      fingerprints; each one missing or different is named, and a mismatch
+ *      of the active encryption key refuses (`--ignore-key-mismatch` restores
+ *      anyway, for an operator who knows the key is lost);
+ *   4. the target database must hold no table at all. This never overwrites.
+ * Then `pg_restore` in one transaction, `ANALYZE`, and a count of every table
+ * against the manifest. A count that differs empties the database again and
+ * fails, so the app is never started on a restore that does not match.
+ *
+ * Exit 0 restored (or listed), 1 refused or failed, 2 usage.
+ */
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, dirname, isAbsolute, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import {
+  compareKeyFingerprints,
+  compareRowCounts,
+  connect,
+  countRows,
+  decryptOffhostFile,
+  emptyDatabase,
+  hashFile,
+  isGoodSet,
+  isWritableDirectory,
+  keyFingerprints,
+  listDumpSets,
+  listUserTables,
+  offhostKeysFromEnv,
+  offhostObjectKey,
+  parseDumpFileName,
+  parseManifest,
+  resolveBackupDir,
+  runTool,
+  statOrNull,
+  connectionFromUrl,
+  bucketConfigFromEnv,
+  openBucket,
+  pickBucketDump,
+  OFFHOST_DATABASE_PREFIX,
+} from "../src/lib/database-backup/dump-core.mjs";
+import { shippedMigrations } from "./pending-migrations.mjs";
+
+const out = (line = "") => console.log(line);
+const say = (line) => console.log(`db-restore: ${line}`);
+
+class Refusal extends Error {}
+
+function mb(bytes) {
+  return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}
+
+/**
+ * Whether this image knows the dump's newest migration. A dump from a newer
+ * release restored under an older image leaves the app with a schema its code
+ * does not expect, so the restore says which image to run.
+ */
+function imageKnowsMigration(name) {
+  if (!name) return true;
+  try {
+    return shippedMigrations(
+      join(process.cwd(), "prisma", "migrations"),
+    ).includes(name);
+  } catch {
+    return true;
+  }
+}
+
+function listCommand(dir) {
+  const sets = listDumpSets(dir);
+  if (sets.length === 0) {
+    say(`no dumps in ${dir}`);
+    return 0;
+  }
+  out(`Dumps in ${dir}, newest first:`);
+  for (const set of sets) {
+    const state = isGoodSet(set)
+      ? `${mb(set.dump.bytes)}`
+      : set.encryptedDump
+        ? "off-host copy (.enc)"
+        : set.dump
+          ? "no manifest, not restorable"
+          : "incomplete";
+    out(
+      `  ${set.base}.dump${set.encryptedDump && !set.dump ? ".enc" : ""}  ${set.kind}  ${set.timestamp.toISOString()}  ${state}`,
+    );
+  }
+  return 0;
+}
+
+/**
+ * The dump and manifest to restore: a local pair, or an encrypted pair
+ * opened into `scratch`.
+ */
+async function resolveInput(arg, dir, env) {
+  if (arg === "latest") {
+    const set = listDumpSets(dir).find(
+      (s) =>
+        isGoodSet(s) ||
+        (s.encryptedDump !== null &&
+          (s.encryptedManifest !== null || s.manifest !== null)),
+    );
+    if (!set) throw new Refusal(`no restorable dump in ${dir}`);
+    arg = set.dump ? set.dump.path : set.encryptedDump;
+  }
+  const path = isAbsolute(arg) ? arg : join(dir, arg);
+  if (!(await statOrNull(path))) throw new Refusal(`${path} does not exist`);
+  const name = basename(path);
+  const parsed = parseDumpFileName(name);
+  if (!parsed || parsed.role !== "dump" || parsed.variant === "partial") {
+    throw new Refusal(
+      `${name} is not a HealthLog database dump (healthlog-<time>-<kind>.dump or .dump.enc)`,
+    );
+  }
+  const folder = dirname(path);
+  const plainManifest = join(folder, `${parsed.base}.manifest.json`);
+  const encManifest = `${plainManifest}.enc`;
+
+  if (parsed.variant === "final") {
+    if (!(await statOrNull(plainManifest))) {
+      throw new Refusal(
+        `${basename(plainManifest)} is missing beside ${name}; a dump is restored only with its manifest`,
+      );
+    }
+    return {
+      dumpPath: path,
+      manifest: parseManifest(
+        JSON.parse(await readFile(plainManifest, "utf8")),
+      ),
+      scratch: null,
+    };
+  }
+
+  // An off-host copy. Open it next to the backups if there is room to write
+  // there, in the container's temp directory otherwise.
+  const keys = offhostKeysFromEnv(env);
+  if (keys.length === 0) {
+    throw new Refusal(
+      "this is an off-host copy and BACKUP_ENCRYPTION_KEY is not set; it cannot be opened without it",
+    );
+  }
+  const scratch = await mkdtemp(
+    join(isWritableDirectory(dir) ? dir : tmpdir(), ".restore-"),
+  );
+  try {
+    let manifest;
+    if (await statOrNull(encManifest)) {
+      const opened = join(scratch, `${parsed.base}.manifest.json`);
+      await decryptOffhostFile({
+        inPath: encManifest,
+        outPath: opened,
+        objectKey: offhostObjectKey(basename(encManifest)),
+        keys,
+      });
+      manifest = parseManifest(JSON.parse(await readFile(opened, "utf8")));
+    } else if (await statOrNull(plainManifest)) {
+      manifest = parseManifest(
+        JSON.parse(await readFile(plainManifest, "utf8")),
+      );
+    } else {
+      throw new Refusal(
+        `${basename(encManifest)} is missing beside ${name}; download it from the bucket too`,
+      );
+    }
+    say(`opening the off-host copy ${name}...`);
+    const dumpPath = join(scratch, `${parsed.base}.dump`);
+    await decryptOffhostFile({
+      inPath: path,
+      outPath: dumpPath,
+      objectKey: offhostObjectKey(name),
+      keys,
+    });
+    return { dumpPath, manifest, scratch };
+  } catch (error) {
+    await rm(scratch, { recursive: true, force: true });
+    throw error instanceof Refusal ? error : new Refusal(error.message);
+  }
+}
+
+/**
+ * Fetch a dump and its manifest from the off-host bucket (`BACKUP_S3_*`) into
+ * a scratch directory: the newest pair for `latest`, the named one otherwise.
+ * Answers the downloaded `.dump.enc` and the directory to remove afterwards.
+ */
+async function downloadFromBucket(arg, dir, env) {
+  const config = bucketConfigFromEnv(env);
+  if (!config) {
+    throw new Refusal(
+      "--from-bucket needs BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY and BACKUP_S3_SECRET_KEY",
+    );
+  }
+  const bucket = await openBucket(config);
+  const keys = await bucket.list(OFFHOST_DATABASE_PREFIX);
+  const pick = pickBucketDump(keys, arg === "latest" ? null : basename(arg));
+  if (!pick) {
+    throw new Refusal(
+      arg === "latest"
+        ? `no database dump with its manifest under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`
+        : `${basename(arg)} and its manifest are not both under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`,
+    );
+  }
+  const scratch = await mkdtemp(
+    join(isWritableDirectory(dir) ? dir : tmpdir(), ".download-"),
+  );
+  try {
+    say(
+      `downloading ${pick.dumpKey} and its manifest from ${config.bucket}...`,
+    );
+    const dumpPath = join(scratch, basename(pick.dumpKey));
+    await bucket.download(
+      pick.manifestKey,
+      join(scratch, basename(pick.manifestKey)),
+    );
+    await bucket.download(pick.dumpKey, dumpPath);
+    return { dumpPath, scratch };
+  } catch (error) {
+    await rm(scratch, { recursive: true, force: true });
+    throw new Refusal(
+      `could not download from the bucket: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
+async function listBucketCommand(env) {
+  const config = bucketConfigFromEnv(env);
+  if (!config)
+    throw new Refusal("--from-bucket needs the BACKUP_S3_* variables");
+  const keys = (await (await openBucket(config)).list(OFFHOST_DATABASE_PREFIX))
+    .filter((k) => k.endsWith(".dump.enc"))
+    .sort()
+    .reverse();
+  out(
+    `Dumps in bucket ${config.bucket} under ${OFFHOST_DATABASE_PREFIX}, newest first:`,
+  );
+  for (const key of keys) out(`  ${key}`);
+  if (keys.length === 0) say("none");
+  return 0;
+}
+
+async function restoreCommand(arg, flags, env) {
+  const dir = resolveBackupDir(env);
+  const databaseUrl = env.DATABASE_URL;
+  if (!databaseUrl) throw new Refusal("DATABASE_URL is not set");
+
+  let input = arg;
+  let download = null;
+  if (flags.has("--from-bucket")) {
+    download = await downloadFromBucket(arg, dir, env);
+    input = download.dumpPath;
+  }
+  let resolved;
+  try {
+    resolved = await resolveInput(input, dir, env);
+  } finally {
+    // The sealed copy is not needed once it is opened (or failed to open).
+    if (download) await rm(download.scratch, { recursive: true, force: true });
+  }
+  const { dumpPath, manifest, scratch } = resolved;
+  try {
+    say(
+      `dump ${manifest.file}: taken ${manifest.createdAt} by HealthLog ${manifest.appVersion ?? "(unknown version)"}, ` +
+        `newest migration ${manifest.latestMigration ?? "(none)"}, Postgres ${manifest.postgres.serverVersion}, ` +
+        `${Object.keys(manifest.rowCounts).length} tables`,
+    );
+    if (!imageKnowsMigration(manifest.latestMigration)) {
+      say(
+        `warning: this image does not know the dump's newest migration ${manifest.latestMigration}; ` +
+          `start the app with HealthLog ${manifest.appVersion ?? "the version that wrote it"} or newer (HEALTHLOG_IMAGE_REF), not this image`,
+      );
+    }
+    if (basename(dumpPath) !== manifest.file) {
+      throw new Refusal(
+        `the manifest describes ${manifest.file}, not ${basename(dumpPath)}`,
+      );
+    }
+
+    // 2. The file is the one the manifest describes.
+    const { sha256, bytes } = await hashFile(dumpPath);
+    if (sha256 !== manifest.dump.sha256 || bytes !== manifest.dump.bytes) {
+      throw new Refusal(
+        `the dump does not match its manifest (SHA-256 ${sha256.slice(0, 12)}…, ${bytes} bytes; ` +
+          `the manifest says ${manifest.dump.sha256.slice(0, 12)}…, ${manifest.dump.bytes} bytes). It is damaged or not this dump.`,
+      );
+    }
+    say(`checksum matches the manifest (${mb(bytes)})`);
+
+    // 3. The keys.
+    const keys = compareKeyFingerprints(manifest.keys, keyFingerprints(env));
+    if (keys.findings.length === 0) {
+      say("every key the dump was written with is configured here");
+    }
+    for (const finding of keys.findings) {
+      say(
+        `${finding.severity === "refuse" ? "KEY MISMATCH" : "warning"}: ${finding.key}: ${finding.detail}`,
+      );
+    }
+    if (keys.refuse && !flags.has("--ignore-key-mismatch")) {
+      throw new Refusal(
+        "refusing to restore: set the encryption keys this dump was written with (see above). " +
+          "Only if that key is lost for good, run again with --ignore-key-mismatch; everything encrypted will stay unreadable.",
+      );
+    }
+
+    // 4. Never over existing data.
+    const client = await connect(databaseUrl);
+    try {
+      const tables = await listUserTables(client);
+      if (tables.length > 0) {
+        throw new Refusal(
+          `the target database already holds ${tables.length} ${tables.length === 1 ? "table" : "tables"} (${tables.slice(0, 3).join(", ")}${tables.length > 3 ? ", …" : ""}). ` +
+            "A restore goes only into an empty database: start it on a new, empty database volume, and do not start the app before the restore.",
+        );
+      }
+
+      say("restoring (one transaction)...");
+      const started = Date.now();
+      await runTool(
+        "pg_restore",
+        [
+          "--no-owner",
+          "--no-privileges",
+          "--single-transaction",
+          "--exit-on-error",
+          "--no-password",
+          `--dbname=${connectionFromUrl(databaseUrl).pgEnv.PGDATABASE}`,
+          dumpPath,
+        ],
+        { env: connectionFromUrl(databaseUrl).pgEnv },
+      );
+      say(`restored in ${Math.round((Date.now() - started) / 1000)} s`);
+
+      await client.query("ANALYZE");
+      say("planner statistics gathered");
+
+      const problems = compareRowCounts(
+        manifest.rowCounts,
+        await countRows(client),
+      );
+      if (problems.length > 0) {
+        for (const line of problems.slice(0, 20)) say(`row count: ${line}`);
+        await emptyDatabase(client);
+        throw new Refusal(
+          `${problems.length} table(s) do not match the manifest; the database was emptied again. Do not start the app on it.`,
+        );
+      }
+      say(
+        `all ${Object.keys(manifest.rowCounts).length} tables hold the row counts the manifest names`,
+      );
+    } finally {
+      await client.end().catch(() => {});
+    }
+    out();
+    out(
+      `Restored ${manifest.file}. Start the app with the image this dump came from or a newer one ` +
+        `(HealthLog ${manifest.appVersion ?? "?"}, newest migration ${manifest.latestMigration ?? "none"}): docker compose up -d`,
+    );
+    return 0;
+  } finally {
+    if (scratch) await rm(scratch, { recursive: true, force: true });
+  }
+}
+
+export async function main(argv = process.argv.slice(2), env = process.env) {
+  const flags = new Set(argv.filter((a) => a.startsWith("--")));
+  const [command] = argv.filter((a) => !a.startsWith("--"));
+  if (!command) {
+    console.error(
+      "db-restore: usage: db-restore list [directory] | latest | <file> [--from-bucket] [--ignore-key-mismatch]",
+    );
+    return 2;
+  }
+  try {
+    if (command === "list" && flags.has("--from-bucket")) {
+      return await listBucketCommand(env);
+    }
+    if (command === "list") {
+      const [, other] = argv.filter((a) => !a.startsWith("--"));
+      return listCommand(other ?? resolveBackupDir(env));
+    }
+    return await restoreCommand(command, flags, env);
+  } catch (error) {
+    console.error(
+      `db-restore: ${error instanceof Refusal ? "" : "failed: "}${error instanceof Error ? error.message : String(error)}`,
+    );
+    return 1;
+  }
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+  process.exit(await main());
+}

scripts/dr-drill-seed.ts

@@ -0,0 +1,59 @@
+/**
+ * The accounts the disaster-recovery drill adds beside the demo seed: a
+ * Guardian with a password, and a managed profile that Guardian administers.
+ * Written through the app's own code (`createManagedProfile`,
+ * `hashPassword`), so the rows are the ones a person creating them in the app
+ * would leave.
+ *
+ *   DATABASE_URL=… pnpm dlx tsx scripts/dr-drill-seed.ts
+ *
+ * Prints one JSON line: `{ "guardianId": …, "profileId": … }`. The drill
+ * (`scripts/compose-dr-drill.sh`) checks after the restore that the Guardian
+ * still sees that profile. Idempotent: a second run answers the same ids.
+ */
+import { hashPassword } from "@/lib/auth/password";
+import { prisma } from "@/lib/db";
+import { createManagedProfile } from "@/lib/managed-profiles/create";
+
+const GUARDIAN = "drill-guardian";
+const PASSWORD = "drill-guardian-password-1";
+
+async function main(): Promise<void> {
+  let guardian = await prisma.user.findUnique({
+    where: { username: GUARDIAN },
+  });
+  guardian ??= await prisma.user.create({
+    data: {
+      username: GUARDIAN,
+      email: "drill-guardian@example.test",
+      passwordHash: await hashPassword(PASSWORD),
+      displayName: "Drill Guardian",
+      locale: "en",
+      timezone: "Europe/Berlin",
+      onboardingCompletedAt: new Date(),
+    },
+  });
+  const existing = await prisma.accountGrant.findFirst({
+    where: { granteeId: guardian.id, access: "MANAGE" },
+    select: { grantorId: true },
+  });
+  const profileId =
+    existing?.grantorId ??
+    (
+      await createManagedProfile({
+        creatorId: guardian.id,
+        displayName: "Drill Child",
+        dateOfBirth: new Date("2018-05-04T00:00:00Z"),
+        locale: "en",
+        timezone: "Europe/Berlin",
+      })
+    ).profile.id;
+  console.log(JSON.stringify({ guardianId: guardian.id, profileId }));
+}
+
+main()
+  .catch((error: unknown) => {
+    console.error(error);
+    process.exitCode = 1;
+  })
+  .finally(() => prisma.$disconnect());

scripts/env-manifest.json

@@ -360,6 +360,38 @@
         }
       ]
     },
+    {
+      "name": "Database dumps",
+      "description": "The nightly whole-database dump and the dump before migrations (docs/ops/disaster-recovery.md). Optional: each has a default, and the bundled compose file mounts the `backups` volume at the default directory.",
+      "required": false,
+      "variables": [
+        {
+          "name": "DB_BACKUP_ENABLED",
+          "purpose": "Set false to stop the nightly database dump. On when unset.",
+          "example": "true"
+        },
+        {
+          "name": "DB_BACKUP_DIR",
+          "purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. Without a writable directory the nightly dump only goes off-host and no dump is taken before migrations.",
+          "example": "/var/lib/healthlog/backups"
+        },
+        {
+          "name": "DB_BACKUP_SCHEDULE",
+          "purpose": "Five-field cron for the nightly dump, in Europe/Berlin time like every scheduled job. Default 30 1 * * * (01:30).",
+          "example": "30 1 * * *"
+        },
+        {
+          "name": "DB_BACKUP_KEEP",
+          "purpose": "How many nightly dumps stay in DB_BACKUP_DIR (default 7, at least 1). The last two dumps taken before a migration are kept besides.",
+          "example": "7"
+        },
+        {
+          "name": "DB_BACKUP_BEFORE_MIGRATE",
+          "purpose": "Set false to start without the dump the entrypoint takes before applying migrations. On when unset; a failed dump stops the start until it works or this is false.",
+          "example": "true"
+        }
+      ]
+    },
     {
       "name": "Operational windows and cookie flags",
       "description": "Values the product states back to a user or that decide whether a session cookie is accepted. Optional: each has a documented default. They are listed so `pnpm check-env` reports them, and so the compose whitelist and the example files can be held in step with this manifest.",

scripts/pending-migrations.mjs

@@ -1,12 +1,11 @@
 /**
  * Boot-time notice before `prisma migrate deploy`: how many migrations this
- * image is about to apply, and a reminder to have a backup first.
+ * image is about to apply, and that a dump is taken first
+ * (`scripts/db-dump.mjs`, which imports `pendingMigrations` from here).
  *
  * Run by `docker-entrypoint.sh` as `node scripts/pending-migrations.mjs`.
  * It only reports. A migration that changes data cannot be undone by pulling
- * the previous image, so an operator who reads the count in the log still has
- * a moment to stop the container and take a `pg_dump`; the entrypoint does
- * not wait for that.
+ * the previous image; the dump the entrypoint takes next is the way back.
  *
  * Best effort by design: any failure to count (no `_prisma_migrations` table
  * yet is not a failure, it means every migration is pending) prints one line
@@ -67,9 +66,9 @@ export function pendingNotice(count) {
   const noun = count === 1 ? "migration" : "migrations";
   return (
     `HealthLog: ${count} database ${noun} pending. ` +
-    "Back up the database before updating (pg_dump, see the Updates section " +
-    "of docs/self-hosting/getting-started.md); a migration is not undone by " +
-    "going back to the previous image."
+    "Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is " +
+    "written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); " +
+    "a migration is not undone by going back to the previous image."
   );
 }