diff --git a/.env.example b/.env.example index 092422769..254f84bfb 100644 --- a/.env.example +++ b/.env.example @@ -277,6 +277,31 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation +# ----------------------------------------------------------------------------- +# Database dumps (optional, on by default) +# ----------------------------------------------------------------------------- +# The whole-database dump that rebuilds a lost server: nightly, and before +# every update that brings migrations. Written to DB_BACKUP_DIR with a +# manifest beside it (row counts, key fingerprints, versions); with the +# off-host bucket above set, each nightly dump is also uploaded under +# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not +# encrypted: they sit beside the database volume, in the same trust zone. +# Restore: `docker compose run --rm db-restore latest`. See +# docs/ops/disaster-recovery.md. +# +# Set "false" to stop the nightly dump. +# DB_BACKUP_ENABLED="true" +# Where dumps are written. The bundled compose file mounts the `backups` +# volume here; change it only together with that mount. +# DB_BACKUP_DIR="/var/lib/healthlog/backups" +# Five-field cron, in Europe/Berlin time like every scheduled job. +# DB_BACKUP_SCHEDULE="30 1 * * *" +# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept +# besides. +# DB_BACKUP_KEEP="7" +# Set "false" to start an update without the dump before its migrations. +# DB_BACKUP_BEFORE_MIGRATE="true" + # ----------------------------------------------------------------------------- # Stored backups in the database (optional) # ----------------------------------------------------------------------------- diff --git a/.env.production.example b/.env.production.example index ecf31e293..2495f5d22 100644 --- a/.env.production.example +++ b/.env.production.example @@ -414,6 +414,31 @@ API_TOKEN_HMAC_KEY="" # key once the lifecycle rule has retired every object written under it. # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" +# ----------------------------------------------------------------------------- +# Database dumps (optional, on by default) +# ----------------------------------------------------------------------------- +# The whole-database dump that rebuilds a lost server: nightly, and before +# every update that brings migrations. Written to DB_BACKUP_DIR with a +# manifest beside it (row counts, key fingerprints, versions); with the +# off-host bucket above set, each nightly dump is also uploaded under +# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not +# encrypted: they sit beside the database volume, in the same trust zone. +# Restore: `docker compose run --rm db-restore latest`. See +# docs/ops/disaster-recovery.md. +# +# Set "false" to stop the nightly dump. +# DB_BACKUP_ENABLED="true" +# Where dumps are written. The bundled compose file mounts the `backups` +# volume here; change it only together with that mount. +# DB_BACKUP_DIR="/var/lib/healthlog/backups" +# Five-field cron, in Europe/Berlin time like every scheduled job. +# DB_BACKUP_SCHEDULE="30 1 * * *" +# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept +# besides. +# DB_BACKUP_KEEP="7" +# Set "false" to start an update without the dump before its migrations. +# DB_BACKUP_BEFORE_MIGRATE="true" + # ----------------------------------------------------------------------------- # Stored backups in the database (optional) # ----------------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile index b09dba475..033b08005 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,13 @@ FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb # security patches from the Alpine repo override it deliberately — # CVE-2026-14456 in openssl was the first instance the container scan # caught between base-image rebuilds). -RUN apk add --no-cache tzdata && apk upgrade --no-cache +# +# `postgresql18-client` carries `pg_dump` and `pg_restore` for the nightly +# database dump, the pre-migration dump in the entrypoint and the +# `db-restore` service (docs/ops/disaster-recovery.md). Major 18 matches the +# bundled database; a client dumps servers of its own major and older, so it +# also serves an install still on its own Postgres 16. +RUN apk add --no-cache tzdata postgresql18-client && apk upgrade --no-cache WORKDIR /app ENV NODE_ENV=production @@ -111,6 +117,13 @@ ENV NEXT_PUBLIC_APP_BUILD_SHA=$NEXT_PUBLIC_APP_BUILD_SHA RUN addgroup --system --gid 1001 nodejs && \ adduser --system --uid 1001 nextjs +# Where the database dumps go (`DB_BACKUP_DIR`). Owned by the app user, so a +# fresh named volume mounted here starts with that owner: Docker fills an +# empty named volume from the image directory it covers, ownership included. +RUN mkdir -p /var/lib/healthlog/backups && \ + chown nextjs:nodejs /var/lib/healthlog/backups && \ + chmod 700 /var/lib/healthlog/backups + # Copy built assets COPY --from=builder /app/public ./public # The server reads its message catalogs from here on first use, one locale at @@ -233,11 +246,16 @@ COPY --from=builder /app/src/generated ./src/generated # the runbooks and the entrypoint against this list. `pending-migrations.mjs` # and `migrate-deploy.mjs` are the entrypoint's own: the first logs the # pending migration count, the second runs `migrate deploy` with its -# lock-timeout retry. Root-owned on purpose: the app user can run them but not -# rewrite them. +# lock-timeout retry. `db-dump.mjs` takes the dump before migrating, and +# `db-restore.mjs` is the `db-restore` service; both share +# `src/lib/database-backup/dump-core.mjs` with the app. Root-owned on purpose: +# the app user can run them but not rewrite them. COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs +COPY --from=builder /app/scripts/db-dump.mjs ./scripts/db-dump.mjs +COPY --from=builder /app/scripts/db-restore.mjs ./scripts/db-restore.mjs +COPY --from=builder /app/src/lib/database-backup/dump-core.mjs ./src/lib/database-backup/dump-core.mjs COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts # Prove the reset CLI's imports resolve in this tree: with every import found @@ -247,6 +265,12 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:" RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \ node --input-type=module -e "await import('pg')" && \ node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')" +# The dump and restore scripts load, and the client tools they run are the +# major the bundled database runs. +RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \ + node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \ + pg_dump --version | grep -q "(PostgreSQL) 18\." && \ + pg_restore --version | grep -q "(PostgreSQL) 18\." # Install the migration CLI, its config dependency, and a pinned launcher for # the maintenance scripts already shipped in the standalone tree. The Prisma diff --git a/docker-compose.yml b/docker-compose.yml index cdfbfeea6..676522118 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,9 +14,11 @@ # and login silently fails. This is the single most common self-host snag. # Leave it unset (or true) only when a TLS reverse proxy serves HTTPS. # -# Persistent state lives in exactly one named volume (`postgres-data`, -# Postgres' data). The app container is stateless, so a backup of the database -# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to +# Persistent state lives in the named volume `postgres-data` (Postgres' data). +# The app container is stateless, so a backup of the database is a backup of +# everything. `backups` holds the database dumps the app takes every night and +# before every migration (docs/ops/disaster-recovery.md); keep a copy of them +# off this host, or set the off-host bucket below. `pgdata` is where Postgres 16 kept the data up to # v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once # and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked # into the image at build time or mounted read-only from the host and @@ -147,6 +149,20 @@ x-healthlog-env: &healthlog_env # serves anyway. A last resort for a refusal you are certain is false: it # disables a safety check. See docs/self-hosting/getting-started.md. ENCRYPTION_KEY_CHECK: "${ENCRYPTION_KEY_CHECK:-}" + # The whole-database dump (docs/ops/disaster-recovery.md). Nightly at + # DB_BACKUP_SCHEDULE (five-field cron in Europe/Berlin time, default + # "30 1 * * *") into DB_BACKUP_DIR (default /var/lib/healthlog/backups, the + # `backups` volume below), keeping DB_BACKUP_KEEP daily dumps (default 7) + # and the last two taken before a migration. With the off-host bucket set, + # each nightly dump is also uploaded under `database/`, sealed with + # BACKUP_ENCRYPTION_KEY. DB_BACKUP_ENABLED=false stops the nightly dump; + # DB_BACKUP_BEFORE_MIGRATE=false skips the dump before migrations. Change + # DB_BACKUP_DIR only together with the volume mount. + DB_BACKUP_ENABLED: "${DB_BACKUP_ENABLED:-}" + DB_BACKUP_DIR: "${DB_BACKUP_DIR:-}" + DB_BACKUP_SCHEDULE: "${DB_BACKUP_SCHEDULE:-}" + DB_BACKUP_KEEP: "${DB_BACKUP_KEEP:-}" + DB_BACKUP_BEFORE_MIGRATE: "${DB_BACKUP_BEFORE_MIGRATE:-}" BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}" BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}" BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}" @@ -475,6 +491,11 @@ services: - "3000:3000" environment: <<: *healthlog_env + # The database dumps (DB_BACKUP_DIR). Without this mount the app still + # copies its nightly dump off-host when the bucket is set, but keeps none + # here and takes none before a migration; the boot log says so. + volumes: + - backups:/var/lib/healthlog/backups # Bring your own GeoLite2 databases. The published image only carries # MMDBs if the build had a MaxMind licence key, so on `:latest` the # offline tier is usually absent and every login IP goes to the online @@ -484,9 +505,9 @@ services: # matches GEOLITE2_DIR's default (/opt/geolite2), so this works as-is # once uncommented. Mounting at a different container path also works, # just also set GEOLITE2_DIR in .env to match it. - # Steps: docs/self-hosting/geolite2.md. + # Steps: docs/self-hosting/geolite2.md. The line goes under `volumes:` + # above, beside the backups line: # - # volumes: # - ./geolite2:/opt/geolite2:ro depends_on: db: @@ -563,6 +584,9 @@ services: # environment: # <<: *healthlog_env # HEALTHLOG_PROCESS_TYPE: "worker" + # # The worker takes the nightly database dump, so it needs the volume. + # volumes: + # - backups:/var/lib/healthlog/backups # depends_on: # db: # condition: service_healthy @@ -776,6 +800,39 @@ services: mv "$$MOVING" "$$MARK" log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md" + # Restores a database dump into an EMPTY database (docs/ops/disaster-recovery.md). + # Never started by `docker compose up`; run it by name: + # + # docker compose run --rm db-restore list + # docker compose run --rm db-restore latest + # docker compose run --rm db-restore healthlog-