diff --git a/.env.example b/.env.example index 092422769..254f84bfb 100644 --- a/.env.example +++ b/.env.example @@ -277,6 +277,31 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation +# ----------------------------------------------------------------------------- +# Database dumps (optional, on by default) +# ----------------------------------------------------------------------------- +# The whole-database dump that rebuilds a lost server: nightly, and before +# every update that brings migrations. Written to DB_BACKUP_DIR with a +# manifest beside it (row counts, key fingerprints, versions); with the +# off-host bucket above set, each nightly dump is also uploaded under +# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not +# encrypted: they sit beside the database volume, in the same trust zone. +# Restore: `docker compose run --rm db-restore latest`. See +# docs/ops/disaster-recovery.md. +# +# Set "false" to stop the nightly dump. +# DB_BACKUP_ENABLED="true" +# Where dumps are written. The bundled compose file mounts the `backups` +# volume here; change it only together with that mount. +# DB_BACKUP_DIR="/var/lib/healthlog/backups" +# Five-field cron, in Europe/Berlin time like every scheduled job. +# DB_BACKUP_SCHEDULE="30 1 * * *" +# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept +# besides. +# DB_BACKUP_KEEP="7" +# Set "false" to start an update without the dump before its migrations. +# DB_BACKUP_BEFORE_MIGRATE="true" + # ----------------------------------------------------------------------------- # Stored backups in the database (optional) # ----------------------------------------------------------------------------- diff --git a/.env.production.example b/.env.production.example index f74d78f11..81cb84e3c 100644 --- a/.env.production.example +++ b/.env.production.example @@ -414,6 +414,31 @@ API_TOKEN_HMAC_KEY="" # key once the lifecycle rule has retired every object written under it. # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" +# ----------------------------------------------------------------------------- +# Database dumps (optional, on by default) +# ----------------------------------------------------------------------------- +# The whole-database dump that rebuilds a lost server: nightly, and before +# every update that brings migrations. Written to DB_BACKUP_DIR with a +# manifest beside it (row counts, key fingerprints, versions); with the +# off-host bucket above set, each nightly dump is also uploaded under +# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not +# encrypted: they sit beside the database volume, in the same trust zone. +# Restore: `docker compose run --rm db-restore latest`. See +# docs/ops/disaster-recovery.md. +# +# Set "false" to stop the nightly dump. +# DB_BACKUP_ENABLED="true" +# Where dumps are written. The bundled compose file mounts the `backups` +# volume here; change it only together with that mount. +# DB_BACKUP_DIR="/var/lib/healthlog/backups" +# Five-field cron, in Europe/Berlin time like every scheduled job. +# DB_BACKUP_SCHEDULE="30 1 * * *" +# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept +# besides. +# DB_BACKUP_KEEP="7" +# Set "false" to start an update without the dump before its migrations. +# DB_BACKUP_BEFORE_MIGRATE="true" + # ----------------------------------------------------------------------------- # Stored backups in the database (optional) # ----------------------------------------------------------------------------- diff --git a/.github/workflows/compose-dr-drill.yml b/.github/workflows/compose-dr-drill.yml new file mode 100644 index 000000000..f08e84449 --- /dev/null +++ b/.github/workflows/compose-dr-drill.yml @@ -0,0 +1,70 @@ +name: Disaster recovery drill + +# Rebuilds a lost server from the nightly database dump and the keys alone, +# against real containers: the app image built from this checkout, the +# bundled compose file with an S3-compatible bucket outside it, the demo seed +# plus a Guardian with a managed profile, the dump taken and uploaded by the +# scheduled job, every volume of the stack destroyed, a new stack, the +# restore, and the app booted on the result (health, login, an encrypted +# note, a document, the Guardian's profile, every table's row count against +# the manifest). Two sources: from the bucket (`db-restore latest +# --from-bucket`, the disaster path) and from dump files copied off the host +# (`db-restore latest`). +# `scripts/compose-dr-drill.sh` is the same drill, runnable locally; its +# header lists what it checks and the mutations that must turn it red. + +on: + pull_request: + branches: [main] + paths: + - docker-compose.yml + - Dockerfile + - docker-entrypoint.sh + - scripts/compose-dr-drill.sh + - scripts/dr-drill-seed.ts + - scripts/db-dump.mjs + - scripts/db-restore.mjs + - scripts/pending-migrations.mjs + - src/lib/database-backup/** + - src/lib/jobs/database-backup-queue.ts + - src/lib/jobs/reminder/backup-handlers.ts + - .github/workflows/compose-dr-drill.yml + schedule: + # Nightly, 02:41 UTC: off the hour, after the evening merges. + - cron: "41 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: compose-dr-drill-${{ github.ref }} + cancel-in-progress: true + +jobs: + drill: + name: Restore a lost server from its dump (${{ matrix.source }}) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + source: [bucket, local] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + # The seed scripts run from the checkout against the drill's database. + - run: pnpm install --frozen-lockfile + - run: pnpm db:generate + + - name: Drill + env: + DRILL_SOURCE: ${{ matrix.source }} + run: bash scripts/compose-dr-drill.sh diff --git a/Dockerfile b/Dockerfile index b09dba475..7c07c594d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,13 @@ FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb # security patches from the Alpine repo override it deliberately — # CVE-2026-14456 in openssl was the first instance the container scan # caught between base-image rebuilds). -RUN apk add --no-cache tzdata && apk upgrade --no-cache +# +# `postgresql18-client` carries `pg_dump` and `pg_restore` for the nightly +# database dump, the pre-migration dump in the entrypoint and the +# `db-restore` service (docs/ops/disaster-recovery.md). Major 18 matches the +# bundled database; a client dumps servers of its own major and older, so it +# also serves an install still on its own Postgres 16. +RUN apk add --no-cache tzdata postgresql18-client && apk upgrade --no-cache WORKDIR /app ENV NODE_ENV=production @@ -111,6 +117,13 @@ ENV NEXT_PUBLIC_APP_BUILD_SHA=$NEXT_PUBLIC_APP_BUILD_SHA RUN addgroup --system --gid 1001 nodejs && \ adduser --system --uid 1001 nextjs +# Where the database dumps go (`DB_BACKUP_DIR`). Owned by the app user, so a +# fresh named volume mounted here starts with that owner: Docker fills an +# empty named volume from the image directory it covers, ownership included. +RUN mkdir -p /var/lib/healthlog/backups && \ + chown nextjs:nodejs /var/lib/healthlog/backups && \ + chmod 700 /var/lib/healthlog/backups + # Copy built assets COPY --from=builder /app/public ./public # The server reads its message catalogs from here on first use, one locale at @@ -233,11 +246,16 @@ COPY --from=builder /app/src/generated ./src/generated # the runbooks and the entrypoint against this list. `pending-migrations.mjs` # and `migrate-deploy.mjs` are the entrypoint's own: the first logs the # pending migration count, the second runs `migrate deploy` with its -# lock-timeout retry. Root-owned on purpose: the app user can run them but not -# rewrite them. +# lock-timeout retry. `db-dump.mjs` takes the dump before migrating, and +# `db-restore.mjs` is the `db-restore` service; both share +# `src/lib/database-backup/dump-core.mjs` with the app. Root-owned on purpose: +# the app user can run them but not rewrite them. COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs +COPY --from=builder /app/scripts/db-dump.mjs ./scripts/db-dump.mjs +COPY --from=builder /app/scripts/db-restore.mjs ./scripts/db-restore.mjs +COPY --from=builder /app/src/lib/database-backup/dump-core.mjs ./src/lib/database-backup/dump-core.mjs COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts # Prove the reset CLI's imports resolve in this tree: with every import found @@ -247,6 +265,14 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:" RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \ node --input-type=module -e "await import('pg')" && \ node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')" +# The dump and restore scripts load, the client tools they run are the +# major the bundled database runs, and the S3 client `db-restore --from-bucket` +# loads resolves from the shared module's place in the tree. +RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \ + node --input-type=module -e "import { createRequire } from 'node:module'; createRequire('/app/src/lib/database-backup/dump-core.mjs').resolve('@aws-sdk/client-s3')" && \ + node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \ + pg_dump --version | grep -q "(PostgreSQL) 18\." && \ + pg_restore --version | grep -q "(PostgreSQL) 18\." # Install the migration CLI, its config dependency, and a pinned launcher for # the maintenance scripts already shipped in the standalone tree. The Prisma diff --git a/docker-compose.yml b/docker-compose.yml index cdfbfeea6..19ce9acec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,9 +14,11 @@ # and login silently fails. This is the single most common self-host snag. # Leave it unset (or true) only when a TLS reverse proxy serves HTTPS. # -# Persistent state lives in exactly one named volume (`postgres-data`, -# Postgres' data). The app container is stateless, so a backup of the database -# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to +# Persistent state lives in the named volume `postgres-data` (Postgres' data). +# The app container is stateless, so a backup of the database is a backup of +# everything. `backups` holds the database dumps the app takes every night and +# before every migration (docs/ops/disaster-recovery.md); keep a copy of them +# off this host, or set the off-host bucket below. `pgdata` is where Postgres 16 kept the data up to # v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once # and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked # into the image at build time or mounted read-only from the host and @@ -147,6 +149,20 @@ x-healthlog-env: &healthlog_env # serves anyway. A last resort for a refusal you are certain is false: it # disables a safety check. See docs/self-hosting/getting-started.md. ENCRYPTION_KEY_CHECK: "${ENCRYPTION_KEY_CHECK:-}" + # The whole-database dump (docs/ops/disaster-recovery.md). Nightly at + # DB_BACKUP_SCHEDULE (five-field cron in Europe/Berlin time, default + # "30 1 * * *") into DB_BACKUP_DIR (default /var/lib/healthlog/backups, the + # `backups` volume below), keeping DB_BACKUP_KEEP daily dumps (default 7) + # and the last two taken before a migration. With the off-host bucket set, + # each nightly dump is also uploaded under `database/`, sealed with + # BACKUP_ENCRYPTION_KEY. DB_BACKUP_ENABLED=false stops the nightly dump; + # DB_BACKUP_BEFORE_MIGRATE=false skips the dump before migrations. Change + # DB_BACKUP_DIR only together with the volume mount. + DB_BACKUP_ENABLED: "${DB_BACKUP_ENABLED:-}" + DB_BACKUP_DIR: "${DB_BACKUP_DIR:-}" + DB_BACKUP_SCHEDULE: "${DB_BACKUP_SCHEDULE:-}" + DB_BACKUP_KEEP: "${DB_BACKUP_KEEP:-}" + DB_BACKUP_BEFORE_MIGRATE: "${DB_BACKUP_BEFORE_MIGRATE:-}" BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}" BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}" BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}" @@ -475,6 +491,11 @@ services: - "3000:3000" environment: <<: *healthlog_env + # The database dumps (DB_BACKUP_DIR). Without this mount the app still + # copies its nightly dump off-host when the bucket is set, but keeps none + # here and takes none before a migration; the boot log says so. + volumes: + - backups:/var/lib/healthlog/backups # Bring your own GeoLite2 databases. The published image only carries # MMDBs if the build had a MaxMind licence key, so on `:latest` the # offline tier is usually absent and every login IP goes to the online @@ -484,9 +505,9 @@ services: # matches GEOLITE2_DIR's default (/opt/geolite2), so this works as-is # once uncommented. Mounting at a different container path also works, # just also set GEOLITE2_DIR in .env to match it. - # Steps: docs/self-hosting/geolite2.md. + # Steps: docs/self-hosting/geolite2.md. The line goes under `volumes:` + # above, beside the backups line: # - # volumes: # - ./geolite2:/opt/geolite2:ro depends_on: db: @@ -563,6 +584,9 @@ services: # environment: # <<: *healthlog_env # HEALTHLOG_PROCESS_TYPE: "worker" + # # The worker takes the nightly database dump, so it needs the volume. + # volumes: + # - backups:/var/lib/healthlog/backups # depends_on: # db: # condition: service_healthy @@ -776,6 +800,42 @@ services: mv "$$MOVING" "$$MARK" log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md" + # Restores a database dump into an EMPTY database (docs/ops/disaster-recovery.md). + # Never started by `docker compose up`; run it by name: + # + # docker compose run --rm db-restore list + # docker compose run --rm db-restore latest + # docker compose run --rm db-restore healthlog-