From 1a4289545ee127bee631b56684cd090abec5ecfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 13:05:47 +0100 Subject: [PATCH] ci: rebuild a lost server from its dump in a compose drill scripts/compose-dr-drill.sh plays docs/ops/disaster-recovery.md against real containers: it builds the image from the checkout, starts the bundled compose stack with fixed throwaway keys, seeds the demo account and a Guardian with a managed profile (through the app's own code), writes a reading with an encrypted note through the API, and waits for the scheduled database-backup job to take the dump. It then keeps only the dump files, destroys every volume, starts a new stack, restores with `docker compose run --rm db-restore latest`, counts every table against the manifest apart from the restore, and boots the app: health, the demo login, the note read back, a document opened, the Guardian still seeing the profile. DRILL_MUTATION breaks one step on purpose (wrong-key, skip-table, non-empty-target), and the drill fails on each. The workflow runs it on pull requests that touch the compose file, the image, the entrypoint or the dump and restore code, nightly, and by hand. --- .github/workflows/compose-dr-drill.yml | 61 ++++++ scripts/compose-dr-drill.sh | 273 +++++++++++++++++++++++++ scripts/dr-drill-seed.ts | 59 ++++++ 3 files changed, 393 insertions(+) create mode 100644 .github/workflows/compose-dr-drill.yml create mode 100755 scripts/compose-dr-drill.sh create mode 100644 scripts/dr-drill-seed.ts diff --git a/.github/workflows/compose-dr-drill.yml b/.github/workflows/compose-dr-drill.yml new file mode 100644 index 000000000..7cab1dd9b --- /dev/null +++ b/.github/workflows/compose-dr-drill.yml @@ -0,0 +1,61 @@ +name: Disaster recovery drill + +# Rebuilds a lost server from the nightly database dump and the keys alone, +# against real containers: the app image built from this checkout, the +# bundled compose file, the demo seed plus a Guardian with a managed profile, +# the dump taken by the scheduled job, every volume destroyed, a new stack, +# `docker compose run --rm db-restore latest`, and the app booted on the +# result (health, login, an encrypted note, a document, the Guardian's +# profile, every table's row count against the manifest). +# `scripts/compose-dr-drill.sh` is the same drill, runnable locally; its +# header lists what it checks and the mutations that must turn it red. + +on: + pull_request: + branches: [main] + paths: + - docker-compose.yml + - Dockerfile + - docker-entrypoint.sh + - scripts/compose-dr-drill.sh + - scripts/dr-drill-seed.ts + - scripts/db-dump.mjs + - scripts/db-restore.mjs + - scripts/pending-migrations.mjs + - src/lib/database-backup/** + - src/lib/jobs/database-backup-queue.ts + - src/lib/jobs/reminder/backup-handlers.ts + - .github/workflows/compose-dr-drill.yml + schedule: + # Nightly, 02:41 UTC: off the hour, after the evening merges. + - cron: "41 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: compose-dr-drill-${{ github.ref }} + cancel-in-progress: true + +jobs: + drill: + name: Restore a lost server from its dump + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + # The seed scripts run from the checkout against the drill's database. + - run: pnpm install --frozen-lockfile + - run: pnpm db:generate + + - name: Drill + run: bash scripts/compose-dr-drill.sh diff --git a/scripts/compose-dr-drill.sh b/scripts/compose-dr-drill.sh new file mode 100755 index 000000000..2d1a0f400 --- /dev/null +++ b/scripts/compose-dr-drill.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +# Plays the documented disaster recovery end to end against real containers: +# a server is lost with everything on it, and a new one is rebuilt from the +# nightly database dump and the keys alone (docs/ops/disaster-recovery.md). +# +# 1. Build the app image from this checkout and start the bundled compose +# stack with fixed throwaway keys. +# 2. Seed real data: the demo seed (documents, encrypted notes, years of +# readings) and a Guardian with a managed profile, written by the app's +# own code; then, through the API, a reading with an encrypted note. +# 3. Let the scheduled `database-backup` job take the dump (its cron is set +# to every minute here) and wait for its ledger row. +# 4. Keep only the dump files, the way they would sit off the lost host, +# and DESTROY every volume of the stack. +# 5. Start a new stack, put the dump in its backups volume and run +# `docker compose run --rm db-restore latest` with the same keys. +# 6. Count every table against the manifest, independently of the restore. +# 7. Boot the app on the restored database and check: the health endpoint, +# the demo login, the encrypted note read back through the API, a +# document opened, and the Guardian still seeing the managed profile. +# +# DRILL_MUTATION breaks one step on purpose, and the drill must then fail +# (a check that cannot fail proves nothing): +# wrong-key restore with a different ENCRYPTION_KEY +# skip-table a dump whose archive lacks one table's rows (the +# manifest's checksum fixed up, its counts left true) +# non-empty-target the new app starts and migrates before the restore +# +# Needs Docker, openssl, pnpm and the repo's node_modules (for the seeds). Run it +# from anywhere in the repo: `bash scripts/compose-dr-drill.sh`. +# DRILL_IMAGE names an image to use instead of building one. +set -euo pipefail + +ROOT=$(git rev-parse --show-toplevel) +PROJECT=hl-dr-drill +APP_PORT=${DRILL_APP_PORT:-39301} +DB_PORT=${DRILL_DB_PORT:-55419} +IMAGE=${DRILL_IMAGE:-} +MUTATION=${DRILL_MUTATION:-} +OFFSITE=${PROJECT}-offsite +WORK=$(mktemp -d) +BASE_URL="http://127.0.0.1:$APP_PORT" + +case "$MUTATION" in + "" | wrong-key | skip-table | non-empty-target) ;; + *) echo "unknown DRILL_MUTATION: $MUTATION" >&2; exit 2 ;; +esac + +export PRISMA_HIDE_UPDATE_MESSAGE=1 +export POSTGRES_PASSWORD +POSTGRES_PASSWORD=$(openssl rand -hex 16) +# Fixed throwaway keys: the restore gets these and nothing else of the lost +# host. Two encryption keys, the second active, so a retired key is carried. +export ENCRYPTION_KEY="" +export ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"2222222222222222222222222222222222222222222222222222222222222222"}' +export ENCRYPTION_ACTIVE_KEY_ID=v2 +export API_TOKEN_HMAC_KEY=3333333333333333333333333333333333333333333333333333333333333333 +export SESSION_COOKIE_SECURE=false +export NEXT_PUBLIC_APP_URL=$BASE_URL APP_URL=$BASE_URL +export DB_BACKUP_SCHEDULE="* * * * *" + +step() { echo "== $*"; } +fail() { + echo "FAIL: $*" >&2 + docker logs "$PROJECT-app" 2>&1 | tail -40 >&2 || true + exit 1 +} + +if [ -z "$IMAGE" ]; then + IMAGE=healthlog-dr-drill:local + step "building the app image from this checkout" + docker build -q \ + --build-arg NEXT_PUBLIC_APP_VERSION="drill-$(git -C "$ROOT" rev-parse --short HEAD)" \ + --build-arg NEXT_PUBLIC_APP_BUILD_SHA="$(git -C "$ROOT" rev-parse --short HEAD)" \ + -t "$IMAGE" "$ROOT" >/dev/null +fi + +# The bundled compose file, with this image, its own names and ports, so the +# drill never meets another stack on the same machine. +cat >"$WORK/drill.yml" </dev/null 2>&1 || true + docker volume rm "$OFFSITE" >/dev/null 2>&1 || true +} +trap 'teardown; rm -rf "$WORK"' EXIT + +wait_healthy() { + local i=0 + until [ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/api/health")" = 200 ]; do + i=$((i + 1)) + [ "$i" -lt 150 ] || fail "the app did not report healthy within five minutes" + docker inspect -f '{{.State.Running}}' "$PROJECT-app" 2>/dev/null | grep -q true || + fail "the app container stopped" + sleep 2 + done +} + +# POST/GET as a signed-in account, with its own cookie jar. +login() { + local jar=$1 user=$2 password=$3 + local code + code=$(curl -s -o "$WORK/login.json" -w '%{http_code}' -c "$jar" \ + -H 'content-type: application/json' \ + -d "{\"email\":\"$user\",\"password\":\"$password\"}" "$BASE_URL/api/auth/login") + [ "$code" = 200 ] || fail "login as $user answered $code: $(cat "$WORK/login.json")" +} +api() { curl -s -b "$1" -w '\n%{http_code}' "${@:2}"; } +json() { node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const v=JSON.parse(s);console.log((${1})(v))})"; } + +# A shell in the app image as root, with two volumes, for moving the dump +# files the way an operator copies them; `cp -p` keeps the app user as owner. +as_root() { + local command=$1 mounts=() + shift + for volume in "$@"; do mounts+=(-v "$volume"); done + docker run --rm --user 0:0 --entrypoint sh "${mounts[@]}" "$IMAGE" -c "$command" +} + +teardown + +step "a running server with real data" +stack up -d app >"$WORK/up.log" 2>&1 || { tail -20 "$WORK/up.log" >&2; fail "the stack did not start"; } +wait_healthy +DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$DB_PORT/healthlog?schema=public" +export DATABASE_URL +(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/seed-demo.ts) >"$WORK/seed.log" 2>&1 || + { tail -20 "$WORK/seed.log" >&2; fail "the demo seed failed"; } +IDS=$(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/dr-drill-seed.ts | tail -1) +PROFILE_ID=$(echo "$IDS" | json 'v=>v.profileId') +unset DATABASE_URL + +login "$WORK/demo.jar" demo demo123demo123 +NOTE="drill note $(openssl rand -hex 6)" +OUT=$(api "$WORK/demo.jar" -H 'content-type: application/json' \ + -d "{\"type\":\"WEIGHT\",\"value\":81.4,\"measuredAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"notes\":\"$NOTE\"}" \ + "$BASE_URL/api/measurements") +[ "$(echo "$OUT" | tail -1)" = 201 ] || [ "$(echo "$OUT" | tail -1)" = 200 ] || + fail "creating a reading with a note answered: $OUT" +READING_ID=$(echo "$OUT" | sed '$d' | json 'v=>v.data.id') +[ "$(sql "SELECT count(*) FROM measurements WHERE id = '$READING_ID' AND position('$NOTE' in measurements::text) > 0")" = 0 ] || + fail "the note is stored in plain text" +DOCUMENTS=$(sql "SELECT count(*) FROM inbound_documents WHERE deleted_at IS NULL") +[ "$DOCUMENTS" -gt 0 ] || fail "the seed wrote no documents" +SEEDED_AT=$(sql "SELECT now()") + +step "the scheduled job takes the dump" +i=0 +until DUMP=$(sql "SELECT file_name FROM database_backup_runs WHERE status = 'succeeded' AND started_at > '$SEEDED_AT'::timestamptz ORDER BY started_at DESC LIMIT 1") && [ -n "$DUMP" ]; do + i=$((i + 1)) + [ "$i" -lt 120 ] || fail "no succeeded database-backup run within four minutes: $(sql "SELECT status, error FROM database_backup_runs ORDER BY started_at DESC LIMIT 3")" + sleep 2 +done +BASE=${DUMP%.dump} +echo " $DUMP" + +step "only the dump leaves the host, then every volume is destroyed" +docker volume create "$OFFSITE" >/dev/null +as_root "cp -p /from/$BASE.dump /from/$BASE.manifest.json /to/" \ + "${PROJECT}_backups:/from:ro" "$OFFSITE:/to" || + fail "could not copy the dump off the host" +stack down -v --remove-orphans >/dev/null 2>&1 +[ -z "$(docker volume ls -q --filter "name=^${PROJECT}_")" ] || fail "a volume of the old stack survived" + +if [ "$MUTATION" = skip-table ]; then + step "MUTATION: the dump loses the rows of public.measurements" + # A scratch Postgres restores the real dump and dumps it again without that + # table's rows; the files stay the app user's, read and written by root. + docker run --rm -v "$OFFSITE:/d" -e BASE="$BASE" --entrypoint sh postgres:18.6-alpine -ec ' + gosu postgres initdb -D /tmp/pg >/dev/null + gosu postgres pg_ctl -D /tmp/pg -o "-c listen_addresses=" -w start >/dev/null + gosu postgres createdb scratch + gosu postgres pg_restore --no-owner --no-privileges -d scratch <"/d/$BASE.dump" + gosu postgres pg_dump -Fc --exclude-table-data=public.measurements scratch >"/d/$BASE.dump" + sum=$(sha256sum "/d/$BASE.dump" | cut -d" " -f1) + size=$(wc -c <"/d/$BASE.dump") + sed -i "s/\"sha256\": \"[0-9a-f]*\"/\"sha256\": \"$sum\"/; s/\"bytes\": [0-9]*/\"bytes\": $size/" "/d/$BASE.manifest.json" + chown 1001:1001 "/d/$BASE.dump"' || fail "could not build the mutated dump" +fi + +step "a new server: the compose file, the keys and the dump" +if [ "$MUTATION" = wrong-key ]; then + echo " MUTATION: the restore gets a different active encryption key" + ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"4444444444444444444444444444444444444444444444444444444444444444"}' +fi +# Mounting the image's directory fills the new volume with the app user's +# ownership; then the two files go in. +stack run --rm --no-deps --entrypoint true db-restore >/dev/null 2>&1 +as_root "cp -p /from/* /to/" "$OFFSITE:/from:ro" "${PROJECT}_backups:/to" || + fail "could not put the dump on the new server" +if [ "$MUTATION" = non-empty-target ]; then + echo " MUTATION: the new app starts (and migrates) before the restore" + stack up -d app >/dev/null 2>&1 + wait_healthy +fi +stack up -d db >/dev/null 2>&1 +if ! stack run --rm db-restore latest 2>&1 | tee "$WORK/restore.log"; then + fail "the restore failed" +fi +grep -q "^Restored $DUMP" "$WORK/restore.log" || fail "the restore did not say it restored $DUMP" + +step "every table against the manifest, counted apart from the restore" +as_root "cat /d/$BASE.manifest.json" "$OFFSITE:/d:ro" >"$WORK/manifest.json" +node -e ' + const m = require(process.argv[1]); + const q = String.fromCharCode(39); + const lit = (t) => q + t.split(q).join(q + q) + q; + console.log(Object.keys(m.rowCounts).map((t) => `SELECT ${lit(t)} || chr(9) || count(*) FROM ${t}`).join(" UNION ALL ") + ";"); +' "$WORK/manifest.json" >"$WORK/count.sql" +docker exec -i "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atq <"$WORK/count.sql" >"$WORK/counts.tsv" +node -e ' + const fs = require("node:fs"); + const m = require(process.argv[1]); + const got = Object.fromEntries(fs.readFileSync(process.argv[2], "utf8").trim().split("\n").map((l) => l.split("\t")).map(([t, n]) => [t, Number(n)])); + const bad = Object.entries(m.rowCounts).filter(([t, n]) => got[t] !== n); + if (bad.length) { console.error(bad.map(([t, n]) => `${t}: manifest ${n}, database ${got[t]}`).join("\n")); process.exit(1); } + if (m.rowCounts["public.users"] < 3) { console.error("fewer than three accounts in the dump"); process.exit(1); } + const c = m.rowCounts; + console.log(` ${Object.keys(c).length} tables, ${Object.values(c).reduce((a, b) => a + b, 0)} rows: ` + + `${c["public.users"]} accounts, ${c["public.account_grants"]} grants, ${c["public.measurements"]} readings, ` + + `${c["public.inbound_documents"]} documents, ${c["pgboss.job"] ?? 0} queued jobs`); +' "$WORK/manifest.json" "$WORK/counts.tsv" || fail "the restored row counts differ from the manifest" + +step "the app on the restored database" +stack up -d app >/dev/null 2>&1 || fail "the app did not start on the restored database" +wait_healthy +[ "$(curl -s "$BASE_URL/api/health" | json 'v=>v.status ?? v.data?.status')" = ok ] || fail "health is not ok" + +login "$WORK/demo2.jar" demo demo123demo123 +OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/measurements/$READING_ID") +[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "reading the note back answered: $OUT" +[ "$(echo "$OUT" | sed '$d' | json 'v=>v.data.notes')" = "$NOTE" ] || fail "the encrypted note did not read back" +echo " the encrypted note reads back" + +OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/documents/inbound") +[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing documents answered: $OUT" +DOC_ID=$(echo "$OUT" | sed '$d' | json 'v=>(v.data.documents ?? v.data.items ?? v.data)[0].id') +CODE=$(curl -s -b "$WORK/demo2.jar" -o "$WORK/doc.bin" -w '%{http_code}' "$BASE_URL/api/documents/inbound/$DOC_ID/original") +[ "$CODE" = 200 ] || fail "opening document $DOC_ID answered $CODE" +[ "$(head -c 4 "$WORK/doc.bin")" = "%PDF" ] || fail "document $DOC_ID did not open to a PDF" +echo " a document opens ($(wc -c <"$WORK/doc.bin" | tr -d ' ') bytes)" + +login "$WORK/guardian.jar" drill-guardian drill-guardian-password-1 +OUT=$(api "$WORK/guardian.jar" "$BASE_URL/api/account/grants") +[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing the Guardian's grants answered: $OUT" +echo "$OUT" | sed '$d' | node -e " + let s=''; process.stdin.on('data', d => s += d).on('end', () => { + const received = JSON.parse(s).data.received ?? []; + const found = received.some((g) => JSON.stringify(g).includes('$PROFILE_ID')); + process.exit(found ? 0 : 1); + });" || fail "the Guardian no longer sees the managed profile" +echo " the Guardian still sees the managed profile" + +echo "OK: the server was rebuilt from the dump and the keys alone" diff --git a/scripts/dr-drill-seed.ts b/scripts/dr-drill-seed.ts new file mode 100644 index 000000000..24afdbed1 --- /dev/null +++ b/scripts/dr-drill-seed.ts @@ -0,0 +1,59 @@ +/** + * The accounts the disaster-recovery drill adds beside the demo seed: a + * Guardian with a password, and a managed profile that Guardian administers. + * Written through the app's own code (`createManagedProfile`, + * `hashPassword`), so the rows are the ones a person creating them in the app + * would leave. + * + * DATABASE_URL=… pnpm dlx tsx scripts/dr-drill-seed.ts + * + * Prints one JSON line: `{ "guardianId": …, "profileId": … }`. The drill + * (`scripts/compose-dr-drill.sh`) checks after the restore that the Guardian + * still sees that profile. Idempotent: a second run answers the same ids. + */ +import { hashPassword } from "@/lib/auth/password"; +import { prisma } from "@/lib/db"; +import { createManagedProfile } from "@/lib/managed-profiles/create"; + +const GUARDIAN = "drill-guardian"; +const PASSWORD = "drill-guardian-password-1"; + +async function main(): Promise { + let guardian = await prisma.user.findUnique({ + where: { username: GUARDIAN }, + }); + guardian ??= await prisma.user.create({ + data: { + username: GUARDIAN, + email: "drill-guardian@example.test", + passwordHash: await hashPassword(PASSWORD), + displayName: "Drill Guardian", + locale: "en", + timezone: "Europe/Berlin", + onboardingCompletedAt: new Date(), + }, + }); + const existing = await prisma.accountGrant.findFirst({ + where: { granteeId: guardian.id, access: "MANAGE" }, + select: { grantorId: true }, + }); + const profileId = + existing?.grantorId ?? + ( + await createManagedProfile({ + creatorId: guardian.id, + displayName: "Drill Child", + dateOfBirth: new Date("2018-05-04T00:00:00Z"), + locale: "en", + timezone: "Europe/Berlin", + }) + ).profile.id; + console.log(JSON.stringify({ guardianId: guardian.id, profileId })); +} + +main() + .catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect());