mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 13:01:00 +00:00
Merge remote-tracking branch 'origin/release/v1.46.0' into fix/v146-security
This commit is contained in:
commit
e79ea08b13
55 files changed
+5523
-91
No files matched your search
@@ -277,6 +277,31 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32"
|
||||
# BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY
|
||||
# BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Database dumps (optional, on by default)
|
||||
# -----------------------------------------------------------------------------
|
||||
# The whole-database dump that rebuilds a lost server: nightly, and before
|
||||
# every update that brings migrations. Written to DB_BACKUP_DIR with a
|
||||
# manifest beside it (row counts, key fingerprints, versions); with the
|
||||
# off-host bucket above set, each nightly dump is also uploaded under
|
||||
# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not
|
||||
# encrypted: they sit beside the database volume, in the same trust zone.
|
||||
# Restore: `docker compose run --rm db-restore latest`. See
|
||||
# docs/ops/disaster-recovery.md.
|
||||
#
|
||||
# Set "false" to stop the nightly dump.
|
||||
# DB_BACKUP_ENABLED="true"
|
||||
# Where dumps are written. The bundled compose file mounts the `backups`
|
||||
# volume here; change it only together with that mount.
|
||||
# DB_BACKUP_DIR="/var/lib/healthlog/backups"
|
||||
# Five-field cron, in Europe/Berlin time like every scheduled job.
|
||||
# DB_BACKUP_SCHEDULE="30 1 * * *"
|
||||
# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept
|
||||
# besides.
|
||||
# DB_BACKUP_KEEP="7"
|
||||
# Set "false" to start an update without the dump before its migrations.
|
||||
# DB_BACKUP_BEFORE_MIGRATE="true"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stored backups in the database (optional)
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -414,6 +414,31 @@ API_TOKEN_HMAC_KEY="<openssl rand -hex 32>"
|
||||
# key once the lifecycle rule has retired every object written under it.
|
||||
# BACKUP_ENCRYPTION_PREVIOUS_KEYS=""
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Database dumps (optional, on by default)
|
||||
# -----------------------------------------------------------------------------
|
||||
# The whole-database dump that rebuilds a lost server: nightly, and before
|
||||
# every update that brings migrations. Written to DB_BACKUP_DIR with a
|
||||
# manifest beside it (row counts, key fingerprints, versions); with the
|
||||
# off-host bucket above set, each nightly dump is also uploaded under
|
||||
# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not
|
||||
# encrypted: they sit beside the database volume, in the same trust zone.
|
||||
# Restore: `docker compose run --rm db-restore latest`. See
|
||||
# docs/ops/disaster-recovery.md.
|
||||
#
|
||||
# Set "false" to stop the nightly dump.
|
||||
# DB_BACKUP_ENABLED="true"
|
||||
# Where dumps are written. The bundled compose file mounts the `backups`
|
||||
# volume here; change it only together with that mount.
|
||||
# DB_BACKUP_DIR="/var/lib/healthlog/backups"
|
||||
# Five-field cron, in Europe/Berlin time like every scheduled job.
|
||||
# DB_BACKUP_SCHEDULE="30 1 * * *"
|
||||
# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept
|
||||
# besides.
|
||||
# DB_BACKUP_KEEP="7"
|
||||
# Set "false" to start an update without the dump before its migrations.
|
||||
# DB_BACKUP_BEFORE_MIGRATE="true"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stored backups in the database (optional)
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: Disaster recovery drill
|
||||
|
||||
# Rebuilds a lost server from the nightly database dump and the keys alone,
|
||||
# against real containers: the app image built from this checkout, the
|
||||
# bundled compose file with an S3-compatible bucket outside it, the demo seed
|
||||
# plus a Guardian with a managed profile, the dump taken and uploaded by the
|
||||
# scheduled job, every volume of the stack destroyed, a new stack, the
|
||||
# restore, and the app booted on the result (health, login, an encrypted
|
||||
# note, a document, the Guardian's profile, every table's row count against
|
||||
# the manifest). Two sources: from the bucket (`db-restore latest
|
||||
# --from-bucket`, the disaster path) and from dump files copied off the host
|
||||
# (`db-restore latest`).
|
||||
# `scripts/compose-dr-drill.sh` is the same drill, runnable locally; its
|
||||
# header lists what it checks and the mutations that must turn it red.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- docker-compose.yml
|
||||
- Dockerfile
|
||||
- docker-entrypoint.sh
|
||||
- scripts/compose-dr-drill.sh
|
||||
- scripts/dr-drill-seed.ts
|
||||
- scripts/db-dump.mjs
|
||||
- scripts/db-restore.mjs
|
||||
- scripts/pending-migrations.mjs
|
||||
- src/lib/database-backup/**
|
||||
- src/lib/jobs/database-backup-queue.ts
|
||||
- src/lib/jobs/reminder/backup-handlers.ts
|
||||
- .github/workflows/compose-dr-drill.yml
|
||||
schedule:
|
||||
# Nightly, 02:41 UTC: off the hour, after the evening merges.
|
||||
- cron: "41 2 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: compose-dr-drill-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
drill:
|
||||
name: Restore a lost server from its dump (${{ matrix.source }})
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
source: [bucket, local]
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
|
||||
# The seed scripts run from the checkout against the drill's database.
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm db:generate
|
||||
|
||||
- name: Drill
|
||||
env:
|
||||
DRILL_SOURCE: ${{ matrix.source }}
|
||||
run: bash scripts/compose-dr-drill.sh
|
||||
+29
-3
@@ -77,7 +77,13 @@ FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb
|
||||
# security patches from the Alpine repo override it deliberately —
|
||||
# CVE-2026-14456 in openssl was the first instance the container scan
|
||||
# caught between base-image rebuilds).
|
||||
RUN apk add --no-cache tzdata && apk upgrade --no-cache
|
||||
#
|
||||
# `postgresql18-client` carries `pg_dump` and `pg_restore` for the nightly
|
||||
# database dump, the pre-migration dump in the entrypoint and the
|
||||
# `db-restore` service (docs/ops/disaster-recovery.md). Major 18 matches the
|
||||
# bundled database; a client dumps servers of its own major and older, so it
|
||||
# also serves an install still on its own Postgres 16.
|
||||
RUN apk add --no-cache tzdata postgresql18-client && apk upgrade --no-cache
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
@@ -111,6 +117,13 @@ ENV NEXT_PUBLIC_APP_BUILD_SHA=$NEXT_PUBLIC_APP_BUILD_SHA
|
||||
RUN addgroup --system --gid 1001 nodejs && \
|
||||
adduser --system --uid 1001 nextjs
|
||||
|
||||
# Where the database dumps go (`DB_BACKUP_DIR`). Owned by the app user, so a
|
||||
# fresh named volume mounted here starts with that owner: Docker fills an
|
||||
# empty named volume from the image directory it covers, ownership included.
|
||||
RUN mkdir -p /var/lib/healthlog/backups && \
|
||||
chown nextjs:nodejs /var/lib/healthlog/backups && \
|
||||
chmod 700 /var/lib/healthlog/backups
|
||||
|
||||
# Copy built assets
|
||||
COPY --from=builder /app/public ./public
|
||||
# The server reads its message catalogs from here on first use, one locale at
|
||||
@@ -233,11 +246,16 @@ COPY --from=builder /app/src/generated ./src/generated
|
||||
# the runbooks and the entrypoint against this list. `pending-migrations.mjs`
|
||||
# and `migrate-deploy.mjs` are the entrypoint's own: the first logs the
|
||||
# pending migration count, the second runs `migrate deploy` with its
|
||||
# lock-timeout retry. Root-owned on purpose: the app user can run them but not
|
||||
# rewrite them.
|
||||
# lock-timeout retry. `db-dump.mjs` takes the dump before migrating, and
|
||||
# `db-restore.mjs` is the `db-restore` service; both share
|
||||
# `src/lib/database-backup/dump-core.mjs` with the app. Root-owned on purpose:
|
||||
# the app user can run them but not rewrite them.
|
||||
COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs
|
||||
COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs
|
||||
COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs
|
||||
COPY --from=builder /app/scripts/db-dump.mjs ./scripts/db-dump.mjs
|
||||
COPY --from=builder /app/scripts/db-restore.mjs ./scripts/db-restore.mjs
|
||||
COPY --from=builder /app/src/lib/database-backup/dump-core.mjs ./src/lib/database-backup/dump-core.mjs
|
||||
COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs
|
||||
COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts
|
||||
# Prove the reset CLI's imports resolve in this tree: with every import found
|
||||
@@ -247,6 +265,14 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:"
|
||||
RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \
|
||||
node --input-type=module -e "await import('pg')" && \
|
||||
node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')"
|
||||
# The dump and restore scripts load, the client tools they run are the
|
||||
# major the bundled database runs, and the S3 client `db-restore --from-bucket`
|
||||
# loads resolves from the shared module's place in the tree.
|
||||
RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \
|
||||
node --input-type=module -e "import { createRequire } from 'node:module'; createRequire('/app/src/lib/database-backup/dump-core.mjs').resolve('@aws-sdk/client-s3')" && \
|
||||
node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \
|
||||
pg_dump --version | grep -q "(PostgreSQL) 18\." && \
|
||||
pg_restore --version | grep -q "(PostgreSQL) 18\."
|
||||
|
||||
# Install the migration CLI, its config dependency, and a pinned launcher for
|
||||
# the maintenance scripts already shipped in the standalone tree. The Prisma
|
||||
|
||||
+65
-5
@@ -14,9 +14,11 @@
|
||||
# and login silently fails. This is the single most common self-host snag.
|
||||
# Leave it unset (or true) only when a TLS reverse proxy serves HTTPS.
|
||||
#
|
||||
# Persistent state lives in exactly one named volume (`postgres-data`,
|
||||
# Postgres' data). The app container is stateless, so a backup of the database
|
||||
# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to
|
||||
# Persistent state lives in the named volume `postgres-data` (Postgres' data).
|
||||
# The app container is stateless, so a backup of the database is a backup of
|
||||
# everything. `backups` holds the database dumps the app takes every night and
|
||||
# before every migration (docs/ops/disaster-recovery.md); keep a copy of them
|
||||
# off this host, or set the off-host bucket below. `pgdata` is where Postgres 16 kept the data up to
|
||||
# v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once
|
||||
# and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked
|
||||
# into the image at build time or mounted read-only from the host and
|
||||
@@ -147,6 +149,20 @@ x-healthlog-env: &healthlog_env
|
||||
# serves anyway. A last resort for a refusal you are certain is false: it
|
||||
# disables a safety check. See docs/self-hosting/getting-started.md.
|
||||
ENCRYPTION_KEY_CHECK: "${ENCRYPTION_KEY_CHECK:-}"
|
||||
# The whole-database dump (docs/ops/disaster-recovery.md). Nightly at
|
||||
# DB_BACKUP_SCHEDULE (five-field cron in Europe/Berlin time, default
|
||||
# "30 1 * * *") into DB_BACKUP_DIR (default /var/lib/healthlog/backups, the
|
||||
# `backups` volume below), keeping DB_BACKUP_KEEP daily dumps (default 7)
|
||||
# and the last two taken before a migration. With the off-host bucket set,
|
||||
# each nightly dump is also uploaded under `database/`, sealed with
|
||||
# BACKUP_ENCRYPTION_KEY. DB_BACKUP_ENABLED=false stops the nightly dump;
|
||||
# DB_BACKUP_BEFORE_MIGRATE=false skips the dump before migrations. Change
|
||||
# DB_BACKUP_DIR only together with the volume mount.
|
||||
DB_BACKUP_ENABLED: "${DB_BACKUP_ENABLED:-}"
|
||||
DB_BACKUP_DIR: "${DB_BACKUP_DIR:-}"
|
||||
DB_BACKUP_SCHEDULE: "${DB_BACKUP_SCHEDULE:-}"
|
||||
DB_BACKUP_KEEP: "${DB_BACKUP_KEEP:-}"
|
||||
DB_BACKUP_BEFORE_MIGRATE: "${DB_BACKUP_BEFORE_MIGRATE:-}"
|
||||
BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}"
|
||||
BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}"
|
||||
BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}"
|
||||
@@ -475,6 +491,11 @@ services:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
<<: *healthlog_env
|
||||
# The database dumps (DB_BACKUP_DIR). Without this mount the app still
|
||||
# copies its nightly dump off-host when the bucket is set, but keeps none
|
||||
# here and takes none before a migration; the boot log says so.
|
||||
volumes:
|
||||
- backups:/var/lib/healthlog/backups
|
||||
# Bring your own GeoLite2 databases. The published image only carries
|
||||
# MMDBs if the build had a MaxMind licence key, so on `:latest` the
|
||||
# offline tier is usually absent and every login IP goes to the online
|
||||
@@ -484,9 +505,9 @@ services:
|
||||
# matches GEOLITE2_DIR's default (/opt/geolite2), so this works as-is
|
||||
# once uncommented. Mounting at a different container path also works,
|
||||
# just also set GEOLITE2_DIR in .env to match it.
|
||||
# Steps: docs/self-hosting/geolite2.md.
|
||||
# Steps: docs/self-hosting/geolite2.md. The line goes under `volumes:`
|
||||
# above, beside the backups line:
|
||||
#
|
||||
# volumes:
|
||||
# - ./geolite2:/opt/geolite2:ro
|
||||
depends_on:
|
||||
db:
|
||||
@@ -563,6 +584,9 @@ services:
|
||||
# environment:
|
||||
# <<: *healthlog_env
|
||||
# HEALTHLOG_PROCESS_TYPE: "worker"
|
||||
# # The worker takes the nightly database dump, so it needs the volume.
|
||||
# volumes:
|
||||
# - backups:/var/lib/healthlog/backups
|
||||
# depends_on:
|
||||
# db:
|
||||
# condition: service_healthy
|
||||
@@ -776,6 +800,42 @@ services:
|
||||
mv "$$MOVING" "$$MARK"
|
||||
log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md"
|
||||
|
||||
# Restores a database dump into an EMPTY database (docs/ops/disaster-recovery.md).
|
||||
# Never started by `docker compose up`; run it by name:
|
||||
#
|
||||
# docker compose run --rm db-restore list
|
||||
# docker compose run --rm db-restore latest
|
||||
# docker compose run --rm db-restore healthlog-<time>-daily.dump
|
||||
# docker compose run --rm db-restore latest --from-bucket
|
||||
#
|
||||
# It checks the dump against its manifest and the keys in .env against the
|
||||
# keys the dump was written with, refuses a database that already holds
|
||||
# tables, restores in one transaction and compares every table's row count
|
||||
# with the manifest. Start the app only after it says "Restored".
|
||||
# `--from-bucket` downloads the dump and its manifest from the off-host bucket
|
||||
# (`database/`, the BACKUP_S3_* variables) and opens them with
|
||||
# BACKUP_ENCRYPTION_KEY: the restore after the host and its local dumps are
|
||||
# gone. Files you downloaded yourself restore from a directory you mount:
|
||||
# `docker compose run --rm -v ./restore:/restore:ro db-restore
|
||||
# /restore/healthlog-<time>-daily.dump.enc`.
|
||||
db-restore:
|
||||
image: ghcr.io/mbombeck/healthlog${HEALTHLOG_IMAGE_REF:-:latest}
|
||||
profiles: ["restore"]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
environment:
|
||||
<<: *healthlog_env
|
||||
volumes:
|
||||
- backups:/var/lib/healthlog/backups
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
entrypoint: ["node", "scripts/db-restore.mjs"]
|
||||
command: ["list"]
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
pgdata:
|
||||
backups:
|
||||
@@ -88,6 +88,15 @@ echo "HealthLog: Database is reachable."
|
||||
# Say how many migrations this image is about to apply, with a reminder to
|
||||
# back up first. Reporting only: it never stops the boot.
|
||||
node scripts/pending-migrations.mjs || true
|
||||
# When migrations are pending, dump the database into DB_BACKUP_DIR first, so
|
||||
# the state before them can be restored (docs/ops/disaster-recovery.md). No
|
||||
# writable directory: a warning, and the boot goes on. A writable directory
|
||||
# and a failed dump: the boot stops here, before anything is migrated.
|
||||
if ! node scripts/db-dump.mjs pre-migration; then
|
||||
echo "HealthLog: ERROR — not migrating without the dump. Free disk space or fix the error above and start again." >&2
|
||||
echo "HealthLog: to migrate without a dump this once, set DB_BACKUP_BEFORE_MIGRATE=false." >&2
|
||||
exit 1
|
||||
fi
|
||||
# `migrate-deploy.mjs` runs `prisma migrate deploy` and retries only a
|
||||
# migration that timed out on a lock and is marked safe to re-run; any other
|
||||
# failure stops the boot with Prisma's exit code (docs/ops/deploy.md).
|
||||
|
||||
@@ -5,6 +5,11 @@ user's JSON dump, encrypted with AES-256-GCM under a SEPARATE key
|
||||
(`BACKUP_ENCRYPTION_KEY`), to any S3-compatible bucket — Cloudflare R2,
|
||||
AWS S3, Backblaze B2, MinIO, etc.
|
||||
|
||||
These per-account copies roll one account back on the same server. To
|
||||
rebuild a server, use the nightly database dump, which the same job
|
||||
configuration also uploads (`database/healthlog-<time>-daily.dump.enc`, in the
|
||||
envelope below): [`disaster-recovery.md`](disaster-recovery.md).
|
||||
|
||||
The backup runs at **02:30 Europe/Berlin** every day from the worker
|
||||
container (queue `data-backup-offhost`). Object key layout:
|
||||
|
||||
@@ -72,6 +77,14 @@ only those:
|
||||
was deleted is removed by the upload itself.
|
||||
- **The connection test** deletes its own `_healthcheck/<ts>.bin`.
|
||||
|
||||
The database dumps are not purged per account. A dump is the whole database
|
||||
at one moment, so an account deleted since stays in the dumps taken before
|
||||
its deletion until they age out: after `DB_BACKUP_KEEP` nights on the server,
|
||||
and in the bucket after the lifecycle rule for `database/`. That is backup
|
||||
retention, and the period to state to the people using your instance; the
|
||||
backups page shows both numbers and warns when no rule covers `database/`
|
||||
([`disaster-recovery.md`](disaster-recovery.md)).
|
||||
|
||||
The admin wipe of every account deletes nothing in the bucket. It is you
|
||||
clearing your own host, the bucket is yours, and its copies are the way back
|
||||
from a wipe pressed by mistake. If you are handing the host on, empty the
|
||||
|
||||
+24
-11
@@ -13,11 +13,13 @@ Migrations land **before** the new app container takes traffic. The
|
||||
canonical command sequence on a single-instance host:
|
||||
|
||||
```sh
|
||||
# 0. Dump the database first. A migration is not undone by going back to
|
||||
# the previous image; the dump is the way back. See "Updates" in
|
||||
# docs/self-hosting/getting-started.md.
|
||||
docker compose exec -T db pg_dump -U healthlog healthlog \
|
||||
| gzip > healthlog-before-update-$(date +%F).sql.gz
|
||||
# 0. Nothing to do by hand for the way back: when the new image brings
|
||||
# migrations, its entrypoint first writes
|
||||
# healthlog-<time>-pre-migration.dump into DB_BACKUP_DIR (the `backups`
|
||||
# volume) and keeps the last two. A migration is not undone by going back
|
||||
# to the previous image; that dump is the way back
|
||||
# (docs/ops/disaster-recovery.md, "Going back to before a migration").
|
||||
# Copy it off the host if the host itself is what you are worried about.
|
||||
|
||||
# 1. Pull the new image. If you pin a version with HEALTHLOG_IMAGE_REF,
|
||||
# set it to the new one first.
|
||||
@@ -33,6 +35,14 @@ docker compose run --rm app sh -c 'exit 0'
|
||||
docker compose up -d app
|
||||
```
|
||||
|
||||
Step 2 takes the pre-migration dump too, in the one-off container, before it
|
||||
migrates. When the backups directory is writable and that dump fails (a full
|
||||
disk, most often), the entrypoint stops before migrating and says so; free
|
||||
the space and run the step again, or set `DB_BACKUP_BEFORE_MIGRATE=false` for
|
||||
one start without the dump. Without a writable directory (an install that
|
||||
runs the image without the `backups` volume) the log carries a warning, no
|
||||
dump is taken, and the migrations run.
|
||||
|
||||
Use `docker compose run` rather than a plain `docker run` for step 2. A
|
||||
plain `docker run` starts outside the stack's network, where `db` does not
|
||||
resolve, and the entrypoint gives up after thirty seconds with "Database
|
||||
@@ -230,12 +240,15 @@ roll back:
|
||||
dev --create-only --name vX_Y_Z_rollback_<col>`), edit the SQL,
|
||||
commit, and re-deploy. Never `migrate resolve --rolled-back` against
|
||||
production unless you've taken a fresh `pg_dump` first.
|
||||
4. Rolling back from v1.40.2 or later to v1.40.1 or earlier: an account
|
||||
that saved the Coach lookback "Last year" reads its whole Coach
|
||||
preference blob as defaults on the older version, metric exclusions
|
||||
included, until the setting is saved again. Ask affected accounts to
|
||||
pick another lookback before the rollback, or re-save their exclusions
|
||||
after it.
|
||||
|
||||
When the data itself has to go back to before the update, restore the
|
||||
pre-migration dump with the previous image; the steps are in
|
||||
[`disaster-recovery.md`](disaster-recovery.md#going-back-to-before-a-migration). 4. Rolling back from v1.40.2 or later to v1.40.1 or earlier: an account
|
||||
that saved the Coach lookback "Last year" reads its whole Coach
|
||||
preference blob as defaults on the older version, metric exclusions
|
||||
included, until the setting is saved again. Ask affected accounts to
|
||||
pick another lookback before the rollback, or re-save their exclusions
|
||||
after it.
|
||||
|
||||
## Catalog templates
|
||||
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
# Disaster recovery
|
||||
|
||||
The copy that rebuilds a lost server is the database dump. This page says
|
||||
what is dumped and where it goes, which keys a restore needs, how to restore
|
||||
onto a new server, and how the procedure is tested.
|
||||
|
||||
## What is backed up, and where
|
||||
|
||||
| Copy | When | Where | Kept |
|
||||
| ----------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------- |
|
||||
| Nightly dump | 01:30 Europe/Berlin, the zone every schedule runs in (`DB_BACKUP_SCHEDULE`) | `DB_BACKUP_DIR`, the `backups` volume of the bundled compose file | the newest 7 (`DB_BACKUP_KEEP`) |
|
||||
| Pre-migration dump | on start, when the new image brings migrations, before them | the same directory | the newest 2 |
|
||||
| Off-host copy of a dump | right after each nightly dump, if the bucket is configured | your bucket, under `database/`, sealed with `BACKUP_ENCRYPTION_KEY` | your bucket's lifecycle rule |
|
||||
|
||||
A dump is `pg_dump --format=custom` of the whole database: every account,
|
||||
sign-ins, passkeys, sharing grants and managed profiles, documents, the job
|
||||
queue and the instance settings. It is two files:
|
||||
|
||||
- `healthlog-<UTC time>-<daily|pre-migration>.dump`, the archive;
|
||||
- `healthlog-<UTC time>-<daily|pre-migration>.manifest.json` beside it: the
|
||||
app version and build, the newest applied migration, the Postgres version,
|
||||
the archive's size and SHA-256, the row count of every table, and the
|
||||
fingerprints (never the values) of the keys the database needs.
|
||||
|
||||
A dump counts as good only when both files exist under their final names. The
|
||||
archive is written as `.partial`, read back with `pg_restore --list`, and
|
||||
renamed after its manifest; a run that fails leaves nothing that looks good.
|
||||
The admin backups page shows the newest good dump, where it is, and the
|
||||
newest failed run. The start log says on every boot where dumps go, or that
|
||||
none can be kept.
|
||||
|
||||
Local dumps are not encrypted. They sit beside the database volume, in the
|
||||
same trust zone: whoever can read one volume can read the other. The columns
|
||||
HealthLog encrypts stay encrypted inside the dump, under `ENCRYPTION_KEYS`.
|
||||
Keep a copy of the dumps off the host; the off-host bucket does that for you,
|
||||
and it seals each copy with `BACKUP_ENCRYPTION_KEY` in the envelope the
|
||||
per-account copies use.
|
||||
|
||||
A dump holds every account that existed when it was taken, including one
|
||||
deleted since. Deleted accounts leave the dumps as they age out: after
|
||||
`DB_BACKUP_KEEP` nights on the server, and after your lifecycle rule in the
|
||||
bucket. Give that rule a prefix that covers `database/` too; the admin
|
||||
backups page reads the bucket's rules and says when none does. This is the
|
||||
retention period to state where you tell people that deleting their account
|
||||
deletes their data.
|
||||
|
||||
Without a writable `DB_BACKUP_DIR` (an install that runs the image without the
|
||||
`backups` volume), the nightly dump is uploaded off-host and nothing is kept
|
||||
on the server, and no dump is taken before migrations; with no bucket either,
|
||||
no dump is taken at all. The start log and the admin page say so.
|
||||
|
||||
The per-account copies (the weekly in-database copy, the nightly per-account
|
||||
objects) roll one account back on the same server. They cannot rebuild a
|
||||
server; see [`backup-restore.md`](backup-restore.md).
|
||||
|
||||
## The keys a restore needs
|
||||
|
||||
| Key | Without it |
|
||||
| ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ENCRYPTION_KEYS` + `ENCRYPTION_ACTIVE_KEY_ID`, or `ENCRYPTION_KEY` | Nothing encrypted can be read. The restore refuses when the active key is missing or different. Keep every retired id still in the map. |
|
||||
| `API_TOKEN_HMAC_KEY` | No data is lost, but every session, app token and share link stops working; people sign in again and re-create tokens. |
|
||||
| `BACKUP_ENCRYPTION_KEY` (and `BACKUP_ENCRYPTION_PREVIOUS_KEYS`) | Off-host copies do not open. |
|
||||
| `POSTGRES_PASSWORD` | Not needed: a new server can use a new one. |
|
||||
|
||||
The restore compares the keys in its environment with the fingerprints in the
|
||||
manifest before it writes anything, and names each key that is missing or
|
||||
different.
|
||||
|
||||
## Restore onto a new server
|
||||
|
||||
1. Install Docker and take the `docker-compose.yml` of the version that wrote
|
||||
the dump or a newer one. The manifest's `appVersion` names it; to run that
|
||||
exact version set `HEALTHLOG_IMAGE_REF=:v<version>` in `.env`.
|
||||
2. Write `.env` with the keys from your records (see above) and a new
|
||||
`POSTGRES_PASSWORD`.
|
||||
3. Do not run `docker compose up` yet. The app would create an empty schema,
|
||||
and the restore only writes into a database with no tables. Start only the
|
||||
database:
|
||||
|
||||
```bash
|
||||
docker compose up -d db
|
||||
```
|
||||
|
||||
4. Restore. When the host is lost, its local dumps went with it, and the
|
||||
copy is the one in your bucket: put the `BACKUP_S3_*` variables and
|
||||
`BACKUP_ENCRYPTION_KEY` (with any `BACKUP_ENCRYPTION_PREVIOUS_KEYS`) in
|
||||
`.env` as on the old server, then
|
||||
|
||||
```bash
|
||||
docker compose run --rm db-restore list --from-bucket
|
||||
docker compose run --rm db-restore latest --from-bucket
|
||||
```
|
||||
|
||||
`latest --from-bucket` downloads the newest `database/…dump.enc` whose
|
||||
manifest is in the bucket too, opens both with `BACKUP_ENCRYPTION_KEY`
|
||||
and restores. Name a dump instead of `latest` to take an older one.
|
||||
|
||||
With the files in hand instead (copied off the host, or downloaded from
|
||||
the bucket with your own tools: both `<name>.dump.enc` and
|
||||
`<name>.manifest.json.enc`, or a `.dump` with its `.manifest.json`), put
|
||||
them in a directory such as `./restore`, make them readable for the
|
||||
container (`chmod a+r restore/*`) and name the dump:
|
||||
|
||||
```bash
|
||||
docker compose run --rm -v "$PWD/restore:/restore:ro" db-restore list /restore
|
||||
docker compose run --rm -v "$PWD/restore:/restore:ro" db-restore \
|
||||
/restore/healthlog-20261024T233000Z-daily.dump
|
||||
```
|
||||
|
||||
Either way it checks the archive against the manifest's SHA-256, the keys
|
||||
against the manifest's fingerprints, and that the database is empty. Then
|
||||
it restores in one transaction, gathers planner statistics and counts
|
||||
every table against the manifest. A count that differs empties the
|
||||
database again and fails. Wait for the line that starts with `Restored`.
|
||||
|
||||
5. Start the stack and check it:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
curl -s http://localhost:3000/api/health
|
||||
```
|
||||
|
||||
Sign in. The app's own key check runs at start and refuses to serve on a
|
||||
key that does not open the data.
|
||||
|
||||
When the dumps are still in the server's `backups` volume (the database
|
||||
volume was lost, the backups volume was not), step 4 is
|
||||
`docker compose run --rm db-restore latest`.
|
||||
|
||||
`--ignore-key-mismatch` restores although the active encryption key differs.
|
||||
Use it only when that key is lost for good: everything encrypted stays
|
||||
unreadable.
|
||||
|
||||
## Going back to before a migration
|
||||
|
||||
The entrypoint dumps the database before it applies migrations. To return to
|
||||
that state (the previous image, the data as it was before the update):
|
||||
|
||||
```bash
|
||||
docker compose stop app
|
||||
# Keep the current state too, in case it is wanted later.
|
||||
docker compose exec -T db pg_dump -U healthlog -Fc healthlog > healthlog-current.dump
|
||||
docker compose exec db dropdb -U healthlog healthlog
|
||||
docker compose exec db createdb -U healthlog healthlog
|
||||
docker compose run --rm db-restore list
|
||||
docker compose run --rm db-restore healthlog-<time>-pre-migration.dump
|
||||
# Set HEALTHLOG_IMAGE_REF in .env to the version you ran before the update.
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
The dump before a migration is taken while the new image starts. If it fails
|
||||
on a writable directory (a full disk, most often), the start stops before
|
||||
anything is migrated and the log names the switch: `DB_BACKUP_BEFORE_MIGRATE=false`
|
||||
starts without it, once.
|
||||
|
||||
## How this is tested
|
||||
|
||||
`scripts/compose-dr-drill.sh` plays this page against real containers: it
|
||||
builds the image, seeds several accounts (the demo account with documents
|
||||
and encrypted notes, a Guardian with a managed profile), lets the scheduled
|
||||
job take the dump and upload it to an S3-compatible bucket, destroys every
|
||||
volume of the stack (the local dumps included), and restores on a new stack
|
||||
given only the bucket and the keys
|
||||
(`docker compose run --rm db-restore latest --from-bucket`). A second run
|
||||
restores from the dump files instead (`DRILL_SOURCE=local`). Both then count
|
||||
every table against the manifest and boot the app: health, sign-in, an
|
||||
encrypted note read back, a document opened, the Guardian still seeing the
|
||||
profile. It runs in CI nightly and on every change to the
|
||||
compose file, the image or the dump and restore code
|
||||
(`.github/workflows/compose-dr-drill.yml`), and it is checked to fail on a
|
||||
wrong encryption key, a wrong off-host key, a dump that lost a table's rows
|
||||
and a target that is not empty.
|
||||
|
||||
Test your own copies too: once, and after any change to how the server is
|
||||
run, restore last night's dump on a spare machine with the steps above.
|
||||
|
||||
## Settings
|
||||
|
||||
| Variable | Default | What it does |
|
||||
| -------------------------- | ---------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `DB_BACKUP_ENABLED` | on | `false` stops the nightly dump. |
|
||||
| `DB_BACKUP_DIR` | `/var/lib/healthlog/backups` | Where dumps are written and read. Change it only together with the volume mount. |
|
||||
| `DB_BACKUP_SCHEDULE` | `30 1 * * *` | Five-field cron, Europe/Berlin time like every HealthLog schedule. |
|
||||
| `DB_BACKUP_KEEP` | `7` | Nightly dumps kept (at least 1). Two pre-migration dumps are kept besides. |
|
||||
| `DB_BACKUP_BEFORE_MIGRATE` | on | `false` starts an update without the dump before its migrations. |
|
||||
@@ -346,6 +346,15 @@ written under that key, so remove the setting once the key is confirmed.
|
||||
Your database is the only stateful piece — back it up, and back up the
|
||||
key that decrypts it.
|
||||
|
||||
**Database dumps the app takes itself.** Every night, and before an update
|
||||
applies migrations, the app writes a `pg_dump` of the whole database with a
|
||||
manifest into the `backups` volume and keeps the newest few. With the
|
||||
off-host bucket configured, each nightly dump is also uploaded, encrypted.
|
||||
`docker compose run --rm db-restore latest` restores one into an empty
|
||||
database. What is kept where, the keys a restore needs, and restoring onto a
|
||||
new server: [`docs/ops/disaster-recovery.md`](../ops/disaster-recovery.md).
|
||||
Keep a copy of those dumps off this host.
|
||||
|
||||
**Local snapshot (the common single-box case).** Dump the `db` service
|
||||
with `pg_dump`:
|
||||
|
||||
@@ -379,8 +388,11 @@ encryption key safely (keep the old key until zero rows remain on it) see
|
||||
A new image can bring database migrations, and `docker-entrypoint.sh`
|
||||
applies them on the first start of that image. Most are additive, but some
|
||||
rewrite data, and going back to the previous image does not undo them. The
|
||||
only way back from a migration that went wrong is the dump you took before
|
||||
it. So take one before every pull:
|
||||
way back from a migration that went wrong is a dump taken before it. The
|
||||
entrypoint takes one itself (`healthlog-<time>-pre-migration.dump` in the
|
||||
`backups` volume, see
|
||||
[`docs/ops/disaster-recovery.md`](../ops/disaster-recovery.md)); a dump of
|
||||
your own before every pull, kept off the host, is the second copy:
|
||||
|
||||
```bash
|
||||
# 1. Dump the database (same command as under "Backup and restore").
|
||||
@@ -405,14 +417,18 @@ the `image:` line, a new major (16 to 18) is a dump and restore, described in
|
||||
The start log says what is about to happen before the migrations run:
|
||||
|
||||
```text
|
||||
HealthLog: 2 database migrations pending. Back up the database before updating (pg_dump, see the Updates section of docs/self-hosting/getting-started.md); a migration is not undone by going back to the previous image.
|
||||
HealthLog: 2 database migrations pending. Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); a migration is not undone by going back to the previous image.
|
||||
HealthLog: taking a database dump before 2 pending migration(s) into /var/lib/healthlog/backups...
|
||||
HealthLog: pre-migration dump written: healthlog-20261025T081502Z-pre-migration.dump (212.4 MB, 231 tables, 38 s).
|
||||
HealthLog: Running database migrations...
|
||||
```
|
||||
|
||||
or `HealthLog: Database schema is up to date.` when the image brings none.
|
||||
The line is a notice and does not pause the start: the migrations run right
|
||||
after it. So the dump belongs before step 2, not after you see the line; by
|
||||
then the database is already being migrated.
|
||||
The migrations run right after the dump. A dump that fails on a writable
|
||||
directory stops the start before anything is migrated, and the log names the
|
||||
switch that skips it once (`DB_BACKUP_BEFORE_MIGRATE=false`). Your own dump
|
||||
belongs before step 2, not after you see these lines; by then the database is
|
||||
already being migrated.
|
||||
|
||||
If an update to v1.43 stops at migration `0409_drop_plaintext_location`,
|
||||
nothing was changed; the recovery is in
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Name eines Stimmungs-Tags oder einer Gruppe mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne Namen behalten)",
|
||||
"restoreSkippedVisitCiphertext": "Text zu Arztbesuch oder Praxis mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne ihn behalten)",
|
||||
"restoreSkippedVaccinationCiphertext": "Impf-Notiz mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Impfung ohne sie behalten)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Ort eines Umweltzeitraums oder -tags mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (verschlüsselt behalten, diese Tage werden nicht neu abgerufen)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Ort eines Umweltzeitraums oder -tags mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (verschlüsselt behalten, diese Tage werden nicht neu abgerufen)",
|
||||
"database": {
|
||||
"title": "Datenbank-Dump",
|
||||
"lastGood": "Letzter gültiger Dump {when}, {size}, {where}.",
|
||||
"whereBoth": "auf diesem Server und extern gespeichert",
|
||||
"whereLocal": "auf diesem Server gespeichert",
|
||||
"whereOffhost": "nur extern gespeichert",
|
||||
"none": "Noch kein Datenbank-Dump. Der erste läuft heute Nacht.",
|
||||
"failedTitle": "Der letzte Datenbank-Dump ist fehlgeschlagen",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "Datenbank-Dumps sind abgeschaltet (DB_BACKUP_ENABLED=false). Sichere die Datenbank auf anderem Weg.",
|
||||
"noDirectoryOffhost": "Dieser Server kann nicht nach {dir} schreiben. Dumps gehen deshalb nur extern, hier wird keiner aufbewahrt. Binde das Backup-Volume ein.",
|
||||
"noDirectory": "Dieser Server kann nicht nach {dir} schreiben und es ist kein externer Speicher eingerichtet. Deshalb entsteht kein Datenbank-Dump. Binde das Backup-Volume ein.",
|
||||
"retentionLocal": "Gelöschte Konten bleiben in den Dumps, bis diese aussortiert werden: Auf diesem Server bleiben die neuesten {nights}.",
|
||||
"retentionOffhost": "Externe Kopien entfernt die Lebenszyklus-Regel des Buckets nach {days} Tagen.",
|
||||
"retentionOffhostMissing": "Keine Lebenszyklus-Regel im Bucket gilt für database/. Dumps dort und die gelöschten Konten darin werden deshalb nie entfernt. Lege eine Regel für dieses Präfix an."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Gefahrenzone",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Mood tag or group name sealed with a key this server does not hold (kept without its name)",
|
||||
"restoreSkippedVisitCiphertext": "Visit or practitioner text sealed with a key this server does not hold (kept without it)",
|
||||
"restoreSkippedVaccinationCiphertext": "Vaccination note sealed with a key this server does not hold (dose kept without it)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Location of an environment period or day sealed with a key this server does not hold (kept sealed, those days are not refreshed)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Location of an environment period or day sealed with a key this server does not hold (kept sealed, those days are not refreshed)",
|
||||
"database": {
|
||||
"title": "Database dump",
|
||||
"lastGood": "Last good dump {when}, {size}, {where}.",
|
||||
"whereBoth": "kept on this server and off-host",
|
||||
"whereLocal": "kept on this server",
|
||||
"whereOffhost": "off-host only",
|
||||
"none": "No database dump yet. The first one runs tonight.",
|
||||
"failedTitle": "The last database dump failed",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "Database dumps are switched off (DB_BACKUP_ENABLED=false). Back the database up another way.",
|
||||
"noDirectoryOffhost": "This server cannot write to {dir}, so dumps go off-host only and none is kept here. Mount the backups volume.",
|
||||
"noDirectory": "This server cannot write to {dir} and no off-host bucket is set, so no database dump can be taken. Mount the backups volume.",
|
||||
"retentionLocal": "Deleted accounts stay in the dumps until those age out: the newest {nights} are kept on this server.",
|
||||
"retentionOffhost": "Off-host copies are removed after {days} days by the bucket's lifecycle rule.",
|
||||
"retentionOffhostMissing": "No lifecycle rule in the bucket covers database/, so dumps there, and the deleted accounts inside them, are never removed. Add a rule for that prefix."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Danger zone",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Nombre de etiqueta o grupo de ánimo cifrado con una clave que este servidor no tiene (se conserva sin el nombre)",
|
||||
"restoreSkippedVisitCiphertext": "Texto de visita o profesional cifrado con una clave que este servidor no tiene (se conserva sin él)",
|
||||
"restoreSkippedVaccinationCiphertext": "Nota de vacunación cifrada con una clave que este servidor no tiene (la dosis se conserva sin ella)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Ubicación de un periodo o día ambiental cifrada con una clave que este servidor no tiene (se conserva cifrada, esos días no se actualizan)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Ubicación de un periodo o día ambiental cifrada con una clave que este servidor no tiene (se conserva cifrada, esos días no se actualizan)",
|
||||
"database": {
|
||||
"title": "Volcado de la base de datos",
|
||||
"lastGood": "Último volcado correcto {when}, {size}, {where}.",
|
||||
"whereBoth": "guardado en este servidor y fuera de él",
|
||||
"whereLocal": "guardado en este servidor",
|
||||
"whereOffhost": "solo fuera del servidor",
|
||||
"none": "Aún no hay ningún volcado de la base de datos. El primero se hace esta noche.",
|
||||
"failedTitle": "El último volcado de la base de datos falló",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "Los volcados de la base de datos están desactivados (DB_BACKUP_ENABLED=false). Haz la copia de la base de datos de otra forma.",
|
||||
"noDirectoryOffhost": "Este servidor no puede escribir en {dir}, así que los volcados solo se guardan fuera y aquí no queda ninguno. Monta el volumen de copias.",
|
||||
"noDirectory": "Este servidor no puede escribir en {dir} y no hay almacenamiento externo configurado, así que no se puede hacer ningún volcado. Monta el volumen de copias.",
|
||||
"retentionLocal": "Las cuentas eliminadas siguen en los volcados hasta que estos caducan: en este servidor se guardan los {nights} más recientes.",
|
||||
"retentionOffhost": "La regla de ciclo de vida del bucket elimina las copias externas tras {days} días.",
|
||||
"retentionOffhostMissing": "Ninguna regla de ciclo de vida del bucket cubre database/, así que los volcados de ahí, y las cuentas eliminadas que contienen, nunca se borran. Añade una regla para ese prefijo."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Zona de peligro",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Nom d’étiquette ou de groupe d’humeur chiffré avec une clé que ce serveur ne possède pas (conservé sans son nom)",
|
||||
"restoreSkippedVisitCiphertext": "Texte de consultation ou de praticien chiffré avec une clé que ce serveur ne possède pas (conservé sans)",
|
||||
"restoreSkippedVaccinationCiphertext": "Note de vaccination chiffrée avec une clé que ce serveur ne possède pas (dose conservée sans)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Lieu d'une période ou d'un jour environnemental chiffré avec une clé que ce serveur ne possède pas (conservé chiffré, ces jours ne sont pas actualisés)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Lieu d'une période ou d'un jour environnemental chiffré avec une clé que ce serveur ne possède pas (conservé chiffré, ces jours ne sont pas actualisés)",
|
||||
"database": {
|
||||
"title": "Dump de la base de données",
|
||||
"lastGood": "Dernier dump valide {when}, {size}, {where}.",
|
||||
"whereBoth": "conservé sur ce serveur et hors site",
|
||||
"whereLocal": "conservé sur ce serveur",
|
||||
"whereOffhost": "hors site uniquement",
|
||||
"none": "Aucun dump de la base de données pour l'instant. Le premier a lieu cette nuit.",
|
||||
"failedTitle": "Le dernier dump de la base de données a échoué",
|
||||
"failedDescription": "{when} : {reason}",
|
||||
"disabled": "Les dumps de la base de données sont désactivés (DB_BACKUP_ENABLED=false). Sauvegardez la base de données autrement.",
|
||||
"noDirectoryOffhost": "Ce serveur ne peut pas écrire dans {dir} : les dumps partent uniquement hors site et aucun n'est conservé ici. Montez le volume de sauvegarde.",
|
||||
"noDirectory": "Ce serveur ne peut pas écrire dans {dir} et aucun stockage hors site n'est configuré : aucun dump ne peut être fait. Montez le volume de sauvegarde.",
|
||||
"retentionLocal": "Les comptes supprimés restent dans les dumps jusqu'à leur expiration : ce serveur conserve les {nights} plus récents.",
|
||||
"retentionOffhost": "La règle de cycle de vie du bucket supprime les copies hors site après {days} jours.",
|
||||
"retentionOffhostMissing": "Aucune règle de cycle de vie du bucket ne couvre database/ : les dumps qui s'y trouvent, et les comptes supprimés qu'ils contiennent, ne sont jamais effacés. Ajoutez une règle pour ce préfixe."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Zone de danger",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Nome di tag o gruppo dell’umore cifrato con una chiave che questo server non possiede (mantenuto senza nome)",
|
||||
"restoreSkippedVisitCiphertext": "Testo di visita o professionista cifrato con una chiave che questo server non possiede (mantenuto senza)",
|
||||
"restoreSkippedVaccinationCiphertext": "Nota di vaccinazione cifrata con una chiave che questo server non possiede (dose mantenuta senza)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Luogo di un periodo o giorno ambientale cifrato con una chiave che questo server non possiede (mantenuto cifrato, quei giorni non vengono aggiornati)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Luogo di un periodo o giorno ambientale cifrato con una chiave che questo server non possiede (mantenuto cifrato, quei giorni non vengono aggiornati)",
|
||||
"database": {
|
||||
"title": "Dump del database",
|
||||
"lastGood": "Ultimo dump valido {when}, {size}, {where}.",
|
||||
"whereBoth": "conservato su questo server e fuori sede",
|
||||
"whereLocal": "conservato su questo server",
|
||||
"whereOffhost": "solo fuori sede",
|
||||
"none": "Ancora nessun dump del database. Il primo viene eseguito stanotte.",
|
||||
"failedTitle": "L'ultimo dump del database non è riuscito",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "I dump del database sono disattivati (DB_BACKUP_ENABLED=false). Esegui il backup del database in un altro modo.",
|
||||
"noDirectoryOffhost": "Questo server non può scrivere in {dir}, quindi i dump vanno solo fuori sede e qui non ne resta nessuno. Monta il volume dei backup.",
|
||||
"noDirectory": "Questo server non può scrivere in {dir} e non è configurato alcuno storage esterno, quindi non si può fare alcun dump. Monta il volume dei backup.",
|
||||
"retentionLocal": "Gli account eliminati restano nei dump finché questi non scadono: su questo server restano i {nights} più recenti.",
|
||||
"retentionOffhost": "La regola del ciclo di vita del bucket elimina le copie esterne dopo {days} giorni.",
|
||||
"retentionOffhostMissing": "Nessuna regola del ciclo di vita del bucket copre database/, quindi i dump lì, e gli account eliminati che contengono, non vengono mai rimossi. Aggiungi una regola per quel prefisso."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Zona pericolosa",
|
||||
|
||||
+17
-1
@@ -8285,7 +8285,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "이 서버에 없는 키로 암호화된 기분 태그 또는 그룹 이름(이름 없이 유지됨)",
|
||||
"restoreSkippedVisitCiphertext": "이 서버에 없는 키로 암호화된 진료 또는 의료진 텍스트(해당 내용 없이 유지됨)",
|
||||
"restoreSkippedVaccinationCiphertext": "이 서버에 없는 키로 암호화된 예방접종 메모(접종 기록은 메모 없이 유지됨)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "이 서버에 없는 키로 암호화된 환경 기간 또는 날짜의 위치(암호화된 채 유지되며 해당 날짜는 새로 가져오지 않음)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "이 서버에 없는 키로 암호화된 환경 기간 또는 날짜의 위치(암호화된 채 유지되며 해당 날짜는 새로 가져오지 않음)",
|
||||
"database": {
|
||||
"title": "데이터베이스 덤프",
|
||||
"lastGood": "마지막 정상 덤프 {when}, {size}, {where}.",
|
||||
"whereBoth": "이 서버와 외부 저장소에 보관됨",
|
||||
"whereLocal": "이 서버에 보관됨",
|
||||
"whereOffhost": "외부 저장소에만 있음",
|
||||
"none": "아직 데이터베이스 덤프가 없습니다. 첫 덤프는 오늘 밤에 실행됩니다.",
|
||||
"failedTitle": "마지막 데이터베이스 덤프가 실패했습니다",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "데이터베이스 덤프가 꺼져 있습니다(DB_BACKUP_ENABLED=false). 다른 방법으로 데이터베이스를 백업하세요.",
|
||||
"noDirectoryOffhost": "이 서버는 {dir}에 쓸 수 없어 덤프가 외부 저장소로만 가고 여기에는 보관되지 않습니다. 백업 볼륨을 마운트하세요.",
|
||||
"noDirectory": "이 서버는 {dir}에 쓸 수 없고 외부 저장소도 설정되지 않아 데이터베이스 덤프를 만들 수 없습니다. 백업 볼륨을 마운트하세요.",
|
||||
"retentionLocal": "삭제된 계정은 덤프가 만료될 때까지 덤프에 남습니다. 이 서버에는 최신 덤프 {nights}개가 보관됩니다.",
|
||||
"retentionOffhost": "버킷의 수명 주기 규칙이 외부 사본을 {days}일 후에 삭제합니다.",
|
||||
"retentionOffhostMissing": "버킷의 어떤 수명 주기 규칙도 database/를 다루지 않아 그곳의 덤프와 그 안의 삭제된 계정이 절대 삭제되지 않습니다. 이 접두사에 대한 규칙을 추가하세요."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "위험 구역",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Stemmingslabel of groepsnaam versleuteld met een sleutel die deze server niet heeft (behouden zonder naam)",
|
||||
"restoreSkippedVisitCiphertext": "Tekst van bezoek of zorgverlener versleuteld met een sleutel die deze server niet heeft (behouden zonder)",
|
||||
"restoreSkippedVaccinationCiphertext": "Vaccinatienotitie versleuteld met een sleutel die deze server niet heeft (dosis behouden zonder)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Locatie van een omgevingsperiode of -dag versleuteld met een sleutel die deze server niet heeft (versleuteld behouden, die dagen worden niet bijgewerkt)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Locatie van een omgevingsperiode of -dag versleuteld met een sleutel die deze server niet heeft (versleuteld behouden, die dagen worden niet bijgewerkt)",
|
||||
"database": {
|
||||
"title": "Databasedump",
|
||||
"lastGood": "Laatste geldige dump {when}, {size}, {where}.",
|
||||
"whereBoth": "bewaard op deze server en extern",
|
||||
"whereLocal": "bewaard op deze server",
|
||||
"whereOffhost": "alleen extern",
|
||||
"none": "Nog geen databasedump. De eerste wordt vannacht gemaakt.",
|
||||
"failedTitle": "De laatste databasedump is mislukt",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "Databasedumps staan uit (DB_BACKUP_ENABLED=false). Maak op een andere manier een back-up van de database.",
|
||||
"noDirectoryOffhost": "Deze server kan niet schrijven naar {dir}, dus dumps gaan alleen extern en hier wordt er geen bewaard. Koppel het back-upvolume.",
|
||||
"noDirectory": "Deze server kan niet schrijven naar {dir} en er is geen externe opslag ingesteld, dus er kan geen databasedump worden gemaakt. Koppel het back-upvolume.",
|
||||
"retentionLocal": "Verwijderde accounts blijven in de dumps tot die verlopen: op deze server blijven de nieuwste {nights} bewaard.",
|
||||
"retentionOffhost": "De levenscyclusregel van de bucket verwijdert externe kopieën na {days} dagen.",
|
||||
"retentionOffhostMissing": "Geen levenscyclusregel in de bucket geldt voor database/, dus dumps daar, en de verwijderde accounts erin, worden nooit verwijderd. Voeg een regel voor dat voorvoegsel toe."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Gevarenzone",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Nazwa tagu lub grupy nastroju zaszyfrowana kluczem, którego ten serwer nie ma (zachowana bez nazwy)",
|
||||
"restoreSkippedVisitCiphertext": "Tekst wizyty lub lekarza zaszyfrowany kluczem, którego ten serwer nie ma (zachowany bez niego)",
|
||||
"restoreSkippedVaccinationCiphertext": "Notatka szczepienia zaszyfrowana kluczem, którego ten serwer nie ma (dawka zachowana bez niej)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Lokalizacja okresu lub dnia środowiskowego zaszyfrowana kluczem, którego ten serwer nie ma (zachowana w postaci zaszyfrowanej, te dni nie są odświeżane)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Lokalizacja okresu lub dnia środowiskowego zaszyfrowana kluczem, którego ten serwer nie ma (zachowana w postaci zaszyfrowanej, te dni nie są odświeżane)",
|
||||
"database": {
|
||||
"title": "Zrzut bazy danych",
|
||||
"lastGood": "Ostatni poprawny zrzut {when}, {size}, {where}.",
|
||||
"whereBoth": "przechowywany na tym serwerze i poza nim",
|
||||
"whereLocal": "przechowywany na tym serwerze",
|
||||
"whereOffhost": "tylko poza serwerem",
|
||||
"none": "Nie ma jeszcze zrzutu bazy danych. Pierwszy powstanie tej nocy.",
|
||||
"failedTitle": "Ostatni zrzut bazy danych się nie powiódł",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "Zrzuty bazy danych są wyłączone (DB_BACKUP_ENABLED=false). Zrób kopię bazy danych w inny sposób.",
|
||||
"noDirectoryOffhost": "Ten serwer nie może zapisywać w {dir}, więc zrzuty trafiają tylko poza serwer i żaden nie zostaje tutaj. Podłącz wolumin kopii.",
|
||||
"noDirectory": "Ten serwer nie może zapisywać w {dir} i nie ustawiono magazynu zewnętrznego, więc nie da się wykonać zrzutu bazy. Podłącz wolumin kopii.",
|
||||
"retentionLocal": "Usunięte konta pozostają w zrzutach, dopóki te nie wygasną: na tym serwerze zostaje {nights} najnowszych.",
|
||||
"retentionOffhost": "Reguła cyklu życia bucketu usuwa kopie zewnętrzne po {days} dniach.",
|
||||
"retentionOffhostMissing": "Żadna reguła cyklu życia bucketu nie obejmuje database/, więc zrzuty tam, wraz z usuniętymi kontami w nich, nigdy nie są usuwane. Dodaj regułę dla tego prefiksu."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Strefa zagrożenia",
|
||||
|
||||
+17
-1
@@ -8310,7 +8310,23 @@
|
||||
"restoreSkippedMoodLabelCiphertext": "Etiqueta ou nome de grupo de humor selado com uma chave que este servidor não possui (mantido sem o nome)",
|
||||
"restoreSkippedVisitCiphertext": "Texto de consulta ou profissional selado com uma chave que este servidor não possui (mantido sem ele)",
|
||||
"restoreSkippedVaccinationCiphertext": "Nota de vacinação selada com uma chave que este servidor não possui (dose mantida sem ela)",
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Local de um período ou dia de ambiente selado com uma chave que este servidor não possui (mantido selado, esses dias não são atualizados)"
|
||||
"restoreSkippedEnvironmentLocationCiphertext": "Local de um período ou dia de ambiente selado com uma chave que este servidor não possui (mantido selado, esses dias não são atualizados)",
|
||||
"database": {
|
||||
"title": "Dump do banco de dados",
|
||||
"lastGood": "Último dump válido {when}, {size}, {where}.",
|
||||
"whereBoth": "guardado neste servidor e fora dele",
|
||||
"whereLocal": "guardado neste servidor",
|
||||
"whereOffhost": "somente fora do servidor",
|
||||
"none": "Ainda não há dump do banco de dados. O primeiro é feito esta noite.",
|
||||
"failedTitle": "O último dump do banco de dados falhou",
|
||||
"failedDescription": "{when}: {reason}",
|
||||
"disabled": "Os dumps do banco de dados estão desligados (DB_BACKUP_ENABLED=false). Faça o backup do banco de outra forma.",
|
||||
"noDirectoryOffhost": "Este servidor não consegue gravar em {dir}, então os dumps vão só para fora e nenhum fica aqui. Monte o volume de backups.",
|
||||
"noDirectory": "Este servidor não consegue gravar em {dir} e não há armazenamento externo configurado, então nenhum dump pode ser feito. Monte o volume de backups.",
|
||||
"retentionLocal": "Contas excluídas continuam nos dumps até que eles expirem: este servidor guarda os {nights} mais recentes.",
|
||||
"retentionOffhost": "A regra de ciclo de vida do bucket remove as cópias externas após {days} dias.",
|
||||
"retentionOffhostMissing": "Nenhuma regra de ciclo de vida do bucket cobre database/, então os dumps ali, e as contas excluídas dentro deles, nunca são removidos. Adicione uma regra para esse prefixo."
|
||||
}
|
||||
},
|
||||
"danger-zone": {
|
||||
"title": "Zona de perigo",
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
-- The nightly whole-database dump records each run: whether a dump was
|
||||
-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not.
|
||||
-- Instance metadata with no account; the admin backups card and the backup
|
||||
-- alerts read the newest rows.
|
||||
--
|
||||
-- The table is new, so nothing waits on its lock in practice; the wait is
|
||||
-- bounded in a DO block anyway, and every statement is safe to run again, so
|
||||
-- the boot retries the file on a lock timeout.
|
||||
--
|
||||
-- @retry-on-lock-timeout
|
||||
|
||||
DO $migration$
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
CREATE TABLE IF NOT EXISTS "database_backup_runs" (
|
||||
"id" TEXT NOT NULL,
|
||||
"started_at" TIMESTAMP(3) NOT NULL,
|
||||
"finished_at" TIMESTAMP(3),
|
||||
"status" TEXT NOT NULL,
|
||||
"file_name" TEXT,
|
||||
"dump_bytes" BIGINT,
|
||||
"local_result" TEXT NOT NULL,
|
||||
"offhost_result" TEXT NOT NULL,
|
||||
"error" TEXT,
|
||||
CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx"
|
||||
ON "database_backup_runs"("started_at");
|
||||
END
|
||||
$migration$;
|
||||
@@ -5929,6 +5929,35 @@ model OffhostPurgeRequest {
|
||||
@@map("offhost_purge_requests")
|
||||
}
|
||||
|
||||
/// One run of the nightly whole-database dump (the `database-backup` job):
|
||||
/// whether a dump was written on this server and copied off-host, and why not
|
||||
/// when it was not. Instance metadata, not an account's data. Written by
|
||||
/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the
|
||||
/// database row of the admin backups card and the backup alerts. A row left
|
||||
/// `running` by a process that died is read as a failed run once the job's
|
||||
/// expiry has passed.
|
||||
model DatabaseBackupRun {
|
||||
id String @id @default(cuid())
|
||||
startedAt DateTime @map("started_at")
|
||||
finishedAt DateTime? @map("finished_at")
|
||||
/// `running`, `succeeded`, `failed` or `skipped`.
|
||||
status String
|
||||
/// The dump's file name, once one was written and checked.
|
||||
fileName String? @map("file_name")
|
||||
/// Size of the dump in bytes.
|
||||
dumpBytes BigInt? @map("dump_bytes")
|
||||
/// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable
|
||||
/// directory, the dump went off-host only), or `failed`.
|
||||
localResult String @map("local_result")
|
||||
/// `uploaded`, `not_configured` or `failed`.
|
||||
offhostResult String @map("offhost_result")
|
||||
/// Why the run failed or was skipped, trimmed. Never key material.
|
||||
error String?
|
||||
|
||||
@@index([startedAt])
|
||||
@@map("database_backup_runs")
|
||||
}
|
||||
|
||||
// ─── Apple Health export.zip import (v1.4.34) ───────────────
|
||||
//
|
||||
// Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* The restore's refusals, which all come before it writes anything.
|
||||
*
|
||||
* The database is a stand-in here: `connect` answers a client whose only use
|
||||
* is to be asked which tables exist, and `runTool` (which would run
|
||||
* `pg_restore`) is a spy that must never be called on a refusal. The real
|
||||
* restore into a real empty database, and its refusal of a real database with
|
||||
* tables, are in `tests/integration/database-backup.test.ts` and in the
|
||||
* compose drill (`scripts/compose-dr-drill.sh`).
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const tablesInTarget = vi.fn(async (): Promise<string[]> => []);
|
||||
const runTool = vi.fn(async () => "");
|
||||
const end = vi.fn(async () => {});
|
||||
|
||||
vi.mock("../../src/lib/database-backup/dump-core.mjs", async (original) => ({
|
||||
...(await original<Record<string, unknown>>()),
|
||||
connect: async () => ({ query: async () => ({ rows: [] }), end }),
|
||||
listUserTables: () => tablesInTarget(),
|
||||
runTool: (...args: unknown[]) => runTool(...(args as [])),
|
||||
}));
|
||||
|
||||
import { buildManifest } from "../../src/lib/database-backup/dump-core.mjs";
|
||||
import { main } from "../db-restore.mjs";
|
||||
|
||||
const KEY_A = "a".repeat(64);
|
||||
const KEY_B = "b".repeat(64);
|
||||
const BASE = "healthlog-20261025T013000Z-daily";
|
||||
|
||||
let dir: string;
|
||||
let errors: string[];
|
||||
let lines: string[];
|
||||
|
||||
function env(extra: Record<string, string> = {}) {
|
||||
return {
|
||||
DATABASE_URL: "postgresql://healthlog:pw@db:5432/healthlog?schema=public",
|
||||
DB_BACKUP_DIR: dir,
|
||||
ENCRYPTION_KEY: KEY_A,
|
||||
API_TOKEN_HMAC_KEY: "hmac",
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
/** A dump file and a manifest that describes it, written with KEY_A. */
|
||||
function writeDump(body = Buffer.from("PGDMP not really an archive")) {
|
||||
writeFileSync(join(dir, `${BASE}.dump`), body);
|
||||
const manifest = buildManifest({
|
||||
kind: "daily",
|
||||
file: `${BASE}.dump`,
|
||||
createdAt: new Date("2026-10-25T01:30:00Z"),
|
||||
appVersion: "1.46.0",
|
||||
buildSha: null,
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
server: { version: "18.6", versionNum: 180006 },
|
||||
pgDump: null,
|
||||
archive: {
|
||||
bytes: body.length,
|
||||
sha256: createHash("sha256").update(body).digest("hex"),
|
||||
entries: 1,
|
||||
},
|
||||
rowCounts: { "public.users": 2 },
|
||||
env: { ENCRYPTION_KEY: KEY_A, API_TOKEN_HMAC_KEY: "hmac" },
|
||||
});
|
||||
writeFileSync(
|
||||
join(dir, `${BASE}.manifest.json`),
|
||||
JSON.stringify(manifest, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hl-db-restore-"));
|
||||
errors = [];
|
||||
lines = [];
|
||||
vi.spyOn(console, "error").mockImplementation((line: string) => {
|
||||
errors.push(String(line));
|
||||
});
|
||||
vi.spyOn(console, "log").mockImplementation((line: string) => {
|
||||
lines.push(String(line));
|
||||
});
|
||||
tablesInTarget.mockResolvedValue([]);
|
||||
runTool.mockClear();
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("db-restore refuses before it writes", () => {
|
||||
it("refuses a target database that already holds tables", async () => {
|
||||
writeDump();
|
||||
tablesInTarget.mockResolvedValue(["public.users", "public.sessions"]);
|
||||
expect(await main(["latest"], env())).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(
|
||||
/already holds 2 tables .*only into an empty database/,
|
||||
);
|
||||
expect(runTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses when the active encryption key is not the dump's, and names it", async () => {
|
||||
writeDump();
|
||||
expect(await main(["latest"], env({ ENCRYPTION_KEY: KEY_B }))).toBe(1);
|
||||
expect(lines.join("\n")).toMatch(/KEY MISMATCH: ENCRYPTION_KEYS\[v1\]/);
|
||||
expect(errors.join("\n")).toMatch(/refusing to restore/);
|
||||
expect(runTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("goes past the key check with --ignore-key-mismatch, and still refuses a full database", async () => {
|
||||
writeDump();
|
||||
tablesInTarget.mockResolvedValue(["public.users"]);
|
||||
expect(
|
||||
await main(
|
||||
["latest", "--ignore-key-mismatch"],
|
||||
env({ ENCRYPTION_KEY: KEY_B }),
|
||||
),
|
||||
).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/already holds 1 table /);
|
||||
expect(runTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses a dump that does not match its manifest", async () => {
|
||||
writeDump();
|
||||
writeFileSync(join(dir, `${BASE}.dump`), "PGDMP altered");
|
||||
expect(await main(["latest"], env())).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/does not match its manifest/);
|
||||
expect(runTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses a dump without its manifest, and a file it did not write", async () => {
|
||||
writeFileSync(join(dir, `${BASE}.dump`), "x");
|
||||
expect(await main([`${BASE}.dump`], env())).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/manifest.json is missing/);
|
||||
writeFileSync(join(dir, "other.dump"), "x");
|
||||
expect(await main(["other.dump"], env())).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/not a HealthLog database dump/);
|
||||
expect(runTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("says when there is nothing to restore, and how it is used", async () => {
|
||||
expect(await main(["latest"], env())).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/no restorable dump/);
|
||||
expect(await main([], env())).toBe(2);
|
||||
});
|
||||
|
||||
it("refuses --from-bucket without a configured bucket, before it connects", async () => {
|
||||
expect(await main(["latest", "--from-bucket"], env())).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/--from-bucket needs BACKUP_S3_ENDPOINT/);
|
||||
expect(runTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("lists what is there", async () => {
|
||||
writeDump();
|
||||
expect(await main(["list"], env())).toBe(0);
|
||||
expect(lines.join("\n")).toContain(`${BASE}.dump daily`);
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,9 @@ ROOT=$(git rev-parse --show-toplevel)
|
||||
# The last release whose compose file ran Postgres 16.
|
||||
OLD_REF=${OLD_REF:-v1.45.0}
|
||||
PROJECT=hl-db-upgrade-check
|
||||
PORT=${CHECK_DB_PORT:-55416}
|
||||
# A free port unless CHECK_DB_PORT names one, so the check never collides with
|
||||
# another stack on the same machine.
|
||||
PORT=${CHECK_DB_PORT:-$(node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})")}
|
||||
WORK=$(mktemp -d)
|
||||
|
||||
export PRISMA_HIDE_UPDATE_MESSAGE=1
|
||||
@@ -63,11 +65,40 @@ run_upgrade() {
|
||||
}
|
||||
upgrade_log() { docker logs healthlog-db-upgrade 2>&1 | grep '^db-upgrade:' || true; }
|
||||
|
||||
# The container can report healthy before its published port forwards (Docker
|
||||
# Desktop and OrbStack set the forward up a moment later), and a migrate
|
||||
# started in that gap failed with "Can't reach database server". So wait
|
||||
# until the port answers, for at most thirty seconds, and retry the migrate
|
||||
# within the same bound.
|
||||
wait_for_port() {
|
||||
local i=0
|
||||
until node -e "
|
||||
const s = require('node:net').connect(Number(process.argv[1]), '127.0.0.1');
|
||||
s.on('connect', () => { s.end(); process.exit(0); });
|
||||
s.on('error', () => process.exit(1));
|
||||
" "$PORT" 2>/dev/null; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -lt 30 ] || return 1
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
migrate_old() {
|
||||
local started=$SECONDS
|
||||
until DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
|
||||
node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do
|
||||
if [ $((SECONDS - started)) -ge 30 ] || ! grep -q "P1001" "$WORK/migrate.log"; then
|
||||
cat "$WORK/migrate.log" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
down_all
|
||||
step "a v1.45 database on Postgres 16"
|
||||
old up -d --wait db >/dev/null 2>&1
|
||||
DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
|
||||
node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >/dev/null
|
||||
old up -d --wait db >"$WORK/up.log" 2>&1 || { cat "$WORK/up.log" >&2; fail "the v1.45 database did not start"; }
|
||||
wait_for_port || fail "port $PORT did not answer within thirty seconds"
|
||||
migrate_old || fail "the migrations did not apply to the v1.45 database"
|
||||
old exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -q <<'SQL'
|
||||
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
||||
CREATE SCHEMA check_probe;
|
||||
|
||||
Executable
+346
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env bash
|
||||
# Plays the documented disaster recovery end to end against real containers:
|
||||
# a server is lost with everything on it, and a new one is rebuilt from the
|
||||
# nightly database dump and the keys alone (docs/ops/disaster-recovery.md).
|
||||
#
|
||||
# 1. Build the app image from this checkout and start the bundled compose
|
||||
# stack with fixed throwaway keys, and an S3-compatible bucket outside
|
||||
# it (Versity S3 Gateway, pinned) as the off-host target.
|
||||
# 2. Seed real data: the demo seed (documents, encrypted notes, years of
|
||||
# readings) and a Guardian with a managed profile, written by the app's
|
||||
# own code; then, through the API, a reading with an encrypted note.
|
||||
# 3. Let the scheduled `database-backup` job take the dump (its cron is set
|
||||
# to every minute here) and wait for its ledger row: kept on the server
|
||||
# and uploaded to the bucket.
|
||||
# 4. DESTROY every volume of the stack, the local dumps included. The
|
||||
# bucket lives outside the stack and survives, as an off-host copy does.
|
||||
# 5. Start a new stack and restore with the keys alone:
|
||||
# DRILL_SOURCE=bucket (the default, the disaster path)
|
||||
# `docker compose run --rm db-restore latest --from-bucket`;
|
||||
# DRILL_SOURCE=local (the dump files had been copied off the host)
|
||||
# the files go into the new backups volume, then
|
||||
# `docker compose run --rm db-restore latest`.
|
||||
# 6. Count every table against the manifest, independently of the restore.
|
||||
# 7. Boot the app on the restored database and check: the health endpoint,
|
||||
# the demo login, the encrypted note read back through the API, a
|
||||
# document opened, and the Guardian still seeing the managed profile.
|
||||
#
|
||||
# DRILL_MUTATION breaks one step on purpose, and the drill must then fail
|
||||
# (a check that cannot fail proves nothing):
|
||||
# wrong-key restore with a different active encryption key
|
||||
# wrong-backup-key restore from the bucket with a different
|
||||
# BACKUP_ENCRYPTION_KEY (DRILL_SOURCE=bucket)
|
||||
# skip-table a dump whose archive lacks one table's rows, the
|
||||
# manifest's checksum fixed up and its counts left true
|
||||
# (DRILL_SOURCE=local)
|
||||
# non-empty-target the new app starts and migrates before the restore
|
||||
#
|
||||
# Needs Docker, openssl, pnpm and the repo's node_modules (for the seeds). Run
|
||||
# it from anywhere in the repo: `bash scripts/compose-dr-drill.sh`.
|
||||
# DRILL_IMAGE names an image to use instead of building one. The ports are
|
||||
# free ones picked at start unless DRILL_APP_PORT, DRILL_DB_PORT or
|
||||
# DRILL_BUCKET_PORT name them.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(git rev-parse --show-toplevel)
|
||||
PROJECT=hl-dr-drill
|
||||
free_port() {
|
||||
node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})"
|
||||
}
|
||||
APP_PORT=${DRILL_APP_PORT:-$(free_port)}
|
||||
DB_PORT=${DRILL_DB_PORT:-$(free_port)}
|
||||
BUCKET_PORT=${DRILL_BUCKET_PORT:-$(free_port)}
|
||||
IMAGE=${DRILL_IMAGE:-}
|
||||
MUTATION=${DRILL_MUTATION:-}
|
||||
SOURCE=${DRILL_SOURCE:-bucket}
|
||||
OFFSITE=$PROJECT-offsite
|
||||
BUCKET=$PROJECT-bucket
|
||||
BUCKET_IMAGE=versity/versitygw:v1.8.0@sha256:30292fc2eeacc67a36993b01f7a7a5e3361a19cced0e80c1d71cfa2a4b0a2499
|
||||
WORK=$(mktemp -d)
|
||||
BASE_URL="http://127.0.0.1:$APP_PORT"
|
||||
|
||||
case "$SOURCE/$MUTATION" in
|
||||
bucket/ | local/ | */wrong-key | */non-empty-target | bucket/wrong-backup-key | local/skip-table) ;;
|
||||
*)
|
||||
echo "DRILL_SOURCE=$SOURCE with DRILL_MUTATION=$MUTATION is not a drill" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
export PRISMA_HIDE_UPDATE_MESSAGE=1
|
||||
export POSTGRES_PASSWORD
|
||||
POSTGRES_PASSWORD=$(openssl rand -hex 16)
|
||||
# Fixed throwaway keys: the restore gets these and nothing else of the lost
|
||||
# host. Two encryption keys, the second active, so a retired key is carried.
|
||||
export ENCRYPTION_KEY=""
|
||||
export ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"2222222222222222222222222222222222222222222222222222222222222222"}'
|
||||
export ENCRYPTION_ACTIVE_KEY_ID=v2
|
||||
export API_TOKEN_HMAC_KEY=3333333333333333333333333333333333333333333333333333333333333333
|
||||
export SESSION_COOKIE_SECURE=false
|
||||
export NEXT_PUBLIC_APP_URL=$BASE_URL APP_URL=$BASE_URL
|
||||
export DB_BACKUP_SCHEDULE="* * * * *"
|
||||
# The off-host bucket, reached from the stack by its container name on a
|
||||
# network of its own that outlives the stack, as a real bucket does.
|
||||
export BACKUP_S3_ENDPOINT=http://$BUCKET:7070
|
||||
export BACKUP_S3_BUCKET=healthlog-backups
|
||||
export BACKUP_S3_ACCESS_KEY=drill-access
|
||||
export BACKUP_S3_SECRET_KEY=drill-secret-0123456789
|
||||
export BACKUP_S3_REGION=us-east-1
|
||||
export BACKUP_ENCRYPTION_KEY=5555555555555555555555555555555555555555555555555555555555555555
|
||||
|
||||
step() { echo "== $*"; }
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
docker logs "$PROJECT-app" 2>&1 | tail -40 >&2 || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ -z "$IMAGE" ]; then
|
||||
IMAGE=healthlog-dr-drill:local
|
||||
step "building the app image from this checkout"
|
||||
docker build -q \
|
||||
--build-arg NEXT_PUBLIC_APP_VERSION="drill-$(git -C "$ROOT" rev-parse --short HEAD)" \
|
||||
--build-arg NEXT_PUBLIC_APP_BUILD_SHA="$(git -C "$ROOT" rev-parse --short HEAD)" \
|
||||
-t "$IMAGE" "$ROOT" >/dev/null
|
||||
fi
|
||||
|
||||
# The bundled compose file, with this image, its own names and ports, so the
|
||||
# drill never meets another stack on the same machine, and the app and the
|
||||
# restore on the bucket's network.
|
||||
cat >"$WORK/drill.yml" <<YAML
|
||||
services:
|
||||
app:
|
||||
image: $IMAGE
|
||||
pull_policy: never
|
||||
container_name: $PROJECT-app
|
||||
ports: !override
|
||||
- "127.0.0.1:$APP_PORT:3000"
|
||||
networks: [default, bucket]
|
||||
db:
|
||||
container_name: $PROJECT-db
|
||||
ports:
|
||||
- "127.0.0.1:$DB_PORT:5432"
|
||||
db-upgrade:
|
||||
container_name: $PROJECT-db-upgrade
|
||||
db-restore:
|
||||
image: $IMAGE
|
||||
pull_policy: never
|
||||
networks: [default, bucket]
|
||||
networks:
|
||||
bucket:
|
||||
external: true
|
||||
name: $BUCKET
|
||||
YAML
|
||||
|
||||
stack() { docker compose -p "$PROJECT" -f "$ROOT/docker-compose.yml" -f "$WORK/drill.yml" "$@"; }
|
||||
sql() { docker exec "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$1"; }
|
||||
teardown() {
|
||||
stack down -v --remove-orphans >/dev/null 2>&1 || true
|
||||
docker volume rm "$OFFSITE" >/dev/null 2>&1 || true
|
||||
docker rm -f "$BUCKET" >/dev/null 2>&1 || true
|
||||
docker volume rm "$BUCKET-data" >/dev/null 2>&1 || true
|
||||
docker network rm "$BUCKET" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap 'teardown; rm -rf "$WORK"' EXIT
|
||||
|
||||
wait_healthy() {
|
||||
local i=0
|
||||
until [ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/api/health")" = 200 ]; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -lt 150 ] || fail "the app did not report healthy within five minutes"
|
||||
docker inspect -f '{{.State.Running}}' "$PROJECT-app" 2>/dev/null | grep -q true ||
|
||||
fail "the app container stopped"
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
# POST/GET as a signed-in account, with its own cookie jar.
|
||||
login() {
|
||||
local jar=$1 user=$2 password=$3
|
||||
local code
|
||||
code=$(curl -s -o "$WORK/login.json" -w '%{http_code}' -c "$jar" \
|
||||
-H 'content-type: application/json' \
|
||||
-d "{\"email\":\"$user\",\"password\":\"$password\"}" "$BASE_URL/api/auth/login")
|
||||
[ "$code" = 200 ] || fail "login as $user answered $code: $(cat "$WORK/login.json")"
|
||||
}
|
||||
api() { curl -s -b "$1" -w '\n%{http_code}' "${@:2}"; }
|
||||
json() { node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const v=JSON.parse(s);console.log((${1})(v))})"; }
|
||||
|
||||
# A shell in the app image as root, with volumes, for moving the dump files
|
||||
# the way an operator copies them; `cp -p` keeps the app user as owner.
|
||||
as_root() {
|
||||
local command=$1 mounts=()
|
||||
shift
|
||||
for volume in "$@"; do mounts+=(-v "$volume"); done
|
||||
docker run --rm --user 0:0 --entrypoint sh "${mounts[@]}" "$IMAGE" -c "$command"
|
||||
}
|
||||
|
||||
teardown
|
||||
|
||||
step "an off-host bucket, outside the stack"
|
||||
docker network create "$BUCKET" >/dev/null
|
||||
docker run -d --name "$BUCKET" --network "$BUCKET" -p "127.0.0.1:$BUCKET_PORT:7070" \
|
||||
-v "$BUCKET-data:/data" --entrypoint sh "$BUCKET_IMAGE" -c \
|
||||
"mkdir -p /data/$BACKUP_S3_BUCKET && exec versitygw --access $BACKUP_S3_ACCESS_KEY --secret $BACKUP_S3_SECRET_KEY posix /data" \
|
||||
>/dev/null || fail "the bucket did not start"
|
||||
|
||||
step "a running server with real data"
|
||||
stack up -d app >"$WORK/up.log" 2>&1 || { tail -20 "$WORK/up.log" >&2; fail "the stack did not start"; }
|
||||
wait_healthy
|
||||
DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$DB_PORT/healthlog?schema=public"
|
||||
export DATABASE_URL
|
||||
(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/seed-demo.ts) >"$WORK/seed.log" 2>&1 ||
|
||||
{ tail -20 "$WORK/seed.log" >&2; fail "the demo seed failed"; }
|
||||
IDS=$(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/dr-drill-seed.ts | tail -1)
|
||||
PROFILE_ID=$(echo "$IDS" | json 'v=>v.profileId')
|
||||
unset DATABASE_URL
|
||||
|
||||
login "$WORK/demo.jar" demo demo123demo123
|
||||
NOTE="drill note $(openssl rand -hex 6)"
|
||||
OUT=$(api "$WORK/demo.jar" -H 'content-type: application/json' \
|
||||
-d "{\"type\":\"WEIGHT\",\"value\":81.4,\"measuredAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"notes\":\"$NOTE\"}" \
|
||||
"$BASE_URL/api/measurements")
|
||||
[ "$(echo "$OUT" | tail -1)" = 201 ] || [ "$(echo "$OUT" | tail -1)" = 200 ] ||
|
||||
fail "creating a reading with a note answered: $OUT"
|
||||
READING_ID=$(echo "$OUT" | sed '$d' | json 'v=>v.data.id')
|
||||
[ "$(sql "SELECT count(*) FROM measurements WHERE id = '$READING_ID' AND position('$NOTE' in measurements::text) > 0")" = 0 ] ||
|
||||
fail "the note is stored in plain text"
|
||||
DOCUMENTS=$(sql "SELECT count(*) FROM inbound_documents WHERE deleted_at IS NULL")
|
||||
[ "$DOCUMENTS" -gt 0 ] || fail "the seed wrote no documents"
|
||||
SEEDED_AT=$(sql "SELECT now()")
|
||||
|
||||
step "the scheduled job takes the dump and uploads it"
|
||||
i=0
|
||||
until DUMP=$(sql "SELECT file_name FROM database_backup_runs WHERE status = 'succeeded' AND local_result = 'written' AND offhost_result = 'uploaded' AND started_at > '$SEEDED_AT'::timestamptz ORDER BY started_at DESC LIMIT 1") && [ -n "$DUMP" ]; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -lt 120 ] || fail "no succeeded and uploaded database-backup run within four minutes: $(sql "SELECT status, local_result, offhost_result, error FROM database_backup_runs ORDER BY started_at DESC LIMIT 3")"
|
||||
sleep 2
|
||||
done
|
||||
BASE=${DUMP%.dump}
|
||||
echo " $DUMP"
|
||||
|
||||
if [ "$SOURCE" = local ]; then
|
||||
step "the dump files are copied off the host"
|
||||
docker volume create "$OFFSITE" >/dev/null
|
||||
as_root "cp -p /from/$BASE.dump /from/$BASE.manifest.json /to/" \
|
||||
"${PROJECT}_backups:/from:ro" "$OFFSITE:/to" ||
|
||||
fail "could not copy the dump off the host"
|
||||
fi
|
||||
|
||||
step "the host is lost: every volume of the stack is destroyed"
|
||||
stack down -v --remove-orphans >/dev/null 2>&1
|
||||
[ -z "$(docker volume ls -q --filter "name=^${PROJECT}_")" ] || fail "a volume of the old stack survived"
|
||||
|
||||
if [ "$MUTATION" = skip-table ]; then
|
||||
step "MUTATION: the dump loses the rows of public.measurements"
|
||||
# A scratch Postgres restores the real dump and dumps it again without that
|
||||
# table's rows; the files stay the app user's, read and written by root.
|
||||
docker run --rm -v "$OFFSITE:/d" -e BASE="$BASE" --entrypoint sh postgres:18.6-alpine -ec '
|
||||
gosu postgres initdb -D /tmp/pg >/dev/null
|
||||
gosu postgres pg_ctl -D /tmp/pg -o "-c listen_addresses=" -w start >/dev/null
|
||||
gosu postgres createdb scratch
|
||||
gosu postgres pg_restore --no-owner --no-privileges -d scratch <"/d/$BASE.dump"
|
||||
gosu postgres pg_dump -Fc --exclude-table-data=public.measurements scratch >"/d/$BASE.dump"
|
||||
sum=$(sha256sum "/d/$BASE.dump" | cut -d" " -f1)
|
||||
size=$(wc -c <"/d/$BASE.dump")
|
||||
sed -i "s/\"sha256\": \"[0-9a-f]*\"/\"sha256\": \"$sum\"/; s/\"bytes\": [0-9]*/\"bytes\": $size/" "/d/$BASE.manifest.json"
|
||||
chown 1001:1001 "/d/$BASE.dump"' || fail "could not build the mutated dump"
|
||||
fi
|
||||
|
||||
step "a new server: the compose file, the keys and the $SOURCE copy"
|
||||
if [ "$MUTATION" = wrong-key ]; then
|
||||
echo " MUTATION: the restore gets a different active encryption key"
|
||||
ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"4444444444444444444444444444444444444444444444444444444444444444"}'
|
||||
fi
|
||||
if [ "$MUTATION" = wrong-backup-key ]; then
|
||||
echo " MUTATION: the restore gets a different BACKUP_ENCRYPTION_KEY"
|
||||
BACKUP_ENCRYPTION_KEY=6666666666666666666666666666666666666666666666666666666666666666
|
||||
fi
|
||||
if [ "$SOURCE" = local ]; then
|
||||
# Mounting the image's directory fills the new volume with the app user's
|
||||
# ownership; then the two files go in.
|
||||
stack run --rm --no-deps --entrypoint true db-restore >/dev/null 2>&1
|
||||
as_root "cp -p /from/* /to/" "$OFFSITE:/from:ro" "${PROJECT}_backups:/to" ||
|
||||
fail "could not put the dump on the new server"
|
||||
RESTORE=(latest)
|
||||
else
|
||||
RESTORE=(latest --from-bucket)
|
||||
fi
|
||||
if [ "$MUTATION" = non-empty-target ]; then
|
||||
echo " MUTATION: the new app starts (and migrates) before the restore"
|
||||
stack up -d app >/dev/null 2>&1
|
||||
wait_healthy
|
||||
fi
|
||||
stack up -d db >/dev/null 2>&1
|
||||
if ! stack run --rm db-restore "${RESTORE[@]}" 2>&1 | tee "$WORK/restore.log"; then
|
||||
fail "the restore failed"
|
||||
fi
|
||||
grep -q "^Restored $DUMP" "$WORK/restore.log" || fail "the restore did not say it restored $DUMP"
|
||||
|
||||
step "every table against the manifest, counted apart from the restore"
|
||||
if [ "$SOURCE" = local ]; then
|
||||
as_root "cat /d/$BASE.manifest.json" "$OFFSITE:/d:ro" >"$WORK/manifest.json"
|
||||
else
|
||||
# From the bucket, opened with the restore's own reader.
|
||||
(cd "$ROOT" && node --input-type=module -e '
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
const core = await import(join(process.cwd(), "src/lib/database-backup/dump-core.mjs"));
|
||||
const [port, base, work] = process.argv.slice(1);
|
||||
const bucket = await core.openBucket({ ...core.bucketConfigFromEnv(process.env), endpoint: `http://127.0.0.1:${port}` });
|
||||
const name = `${base}.manifest.json.enc`;
|
||||
await bucket.download(core.offhostObjectKey(name), join(work, name));
|
||||
await core.decryptOffhostFile({ inPath: join(work, name), outPath: join(work, "manifest.json"), objectKey: core.offhostObjectKey(name), keys: core.offhostKeysFromEnv(process.env) });
|
||||
' "$BUCKET_PORT" "$BASE" "$WORK") || fail "could not read the manifest from the bucket"
|
||||
fi
|
||||
node -e '
|
||||
const m = require(process.argv[1]);
|
||||
const q = String.fromCharCode(39);
|
||||
const lit = (t) => q + t.split(q).join(q + q) + q;
|
||||
console.log(Object.keys(m.rowCounts).map((t) => `SELECT ${lit(t)} || chr(9) || count(*) FROM ${t}`).join(" UNION ALL ") + ";");
|
||||
' "$WORK/manifest.json" >"$WORK/count.sql"
|
||||
docker exec -i "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atq <"$WORK/count.sql" >"$WORK/counts.tsv"
|
||||
node -e '
|
||||
const fs = require("node:fs");
|
||||
const m = require(process.argv[1]);
|
||||
const got = Object.fromEntries(fs.readFileSync(process.argv[2], "utf8").trim().split("\n").map((l) => l.split("\t")).map(([t, n]) => [t, Number(n)]));
|
||||
const bad = Object.entries(m.rowCounts).filter(([t, n]) => got[t] !== n);
|
||||
if (bad.length) { console.error(bad.map(([t, n]) => `${t}: manifest ${n}, database ${got[t]}`).join("\n")); process.exit(1); }
|
||||
if (m.rowCounts["public.users"] < 3) { console.error("fewer than three accounts in the dump"); process.exit(1); }
|
||||
const c = m.rowCounts;
|
||||
console.log(` ${Object.keys(c).length} tables, ${Object.values(c).reduce((a, b) => a + b, 0)} rows: ` +
|
||||
`${c["public.users"]} accounts, ${c["public.account_grants"]} grants, ${c["public.measurements"]} readings, ` +
|
||||
`${c["public.inbound_documents"]} documents, ${c["pgboss.job"] ?? 0} queued jobs`);
|
||||
' "$WORK/manifest.json" "$WORK/counts.tsv" || fail "the restored row counts differ from the manifest"
|
||||
|
||||
step "the app on the restored database"
|
||||
stack up -d app >/dev/null 2>&1 || fail "the app did not start on the restored database"
|
||||
wait_healthy
|
||||
[ "$(curl -s "$BASE_URL/api/health" | json 'v=>v.status ?? v.data?.status')" = ok ] || fail "health is not ok"
|
||||
|
||||
login "$WORK/demo2.jar" demo demo123demo123
|
||||
OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/measurements/$READING_ID")
|
||||
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "reading the note back answered: $OUT"
|
||||
[ "$(echo "$OUT" | sed '$d' | json 'v=>v.data.notes')" = "$NOTE" ] || fail "the encrypted note did not read back"
|
||||
echo " the encrypted note reads back"
|
||||
|
||||
OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/documents/inbound")
|
||||
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing documents answered: $OUT"
|
||||
DOC_ID=$(echo "$OUT" | sed '$d' | json 'v=>(v.data.documents ?? v.data.items ?? v.data)[0].id')
|
||||
CODE=$(curl -s -b "$WORK/demo2.jar" -o "$WORK/doc.bin" -w '%{http_code}' "$BASE_URL/api/documents/inbound/$DOC_ID/original")
|
||||
[ "$CODE" = 200 ] || fail "opening document $DOC_ID answered $CODE"
|
||||
[ "$(head -c 4 "$WORK/doc.bin")" = "%PDF" ] || fail "document $DOC_ID did not open to a PDF"
|
||||
echo " a document opens ($(wc -c <"$WORK/doc.bin" | tr -d ' ') bytes)"
|
||||
|
||||
login "$WORK/guardian.jar" drill-guardian drill-guardian-password-1
|
||||
OUT=$(api "$WORK/guardian.jar" "$BASE_URL/api/account/grants")
|
||||
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing the Guardian's grants answered: $OUT"
|
||||
echo "$OUT" | sed '$d' | node -e "
|
||||
let s=''; process.stdin.on('data', d => s += d).on('end', () => {
|
||||
const received = JSON.parse(s).data.received ?? [];
|
||||
const found = received.some((g) => JSON.stringify(g).includes('$PROFILE_ID'));
|
||||
process.exit(found ? 0 : 1);
|
||||
});" || fail "the Guardian no longer sees the managed profile"
|
||||
echo " the Guardian still sees the managed profile"
|
||||
|
||||
echo "OK: the server was rebuilt from the $SOURCE copy of its dump and the keys alone"
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* The pre-migration dump: a whole-database dump the entrypoint takes before
|
||||
* `prisma migrate deploy` applies anything.
|
||||
*
|
||||
* node scripts/db-dump.mjs pre-migration
|
||||
*
|
||||
* A migration is not undone by going back to the previous image, and the
|
||||
* bundled compose file pulls `:latest` and migrates on every start. So when
|
||||
* this image brings migrations the database has not applied, it first writes
|
||||
* `healthlog-<timestamp>-pre-migration.dump` with its manifest into
|
||||
* `DB_BACKUP_DIR` (`src/lib/database-backup/dump-core.mjs`), and keeps the
|
||||
* newest two.
|
||||
*
|
||||
* Exit 0: the dump was written, or there was nothing to do (no pending
|
||||
* migration, a new database, `DB_BACKUP_BEFORE_MIGRATE=false`), or there is no
|
||||
* writable directory to put it in, which is said as a warning and is not a
|
||||
* reason to keep the app down. Exit 1: the directory is writable and the dump
|
||||
* failed. The entrypoint then stops the boot and names the switch that skips
|
||||
* the dump, because migrating without the way back the operator was promised
|
||||
* is the one thing this step exists to prevent.
|
||||
*
|
||||
* Fast on purpose: one snapshot, `pg_dump`, a count per table for the
|
||||
* manifest, `pg_restore --list` to prove the file reads. Nothing else.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { basename, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
connect,
|
||||
createDatabaseDump,
|
||||
isWritableDirectory,
|
||||
latestMigration,
|
||||
listUserTables,
|
||||
connectionFromUrl,
|
||||
preMigrationDumpEnabled,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
} from "../src/lib/database-backup/dump-core.mjs";
|
||||
import { pendingMigrations } from "./pending-migrations.mjs";
|
||||
|
||||
const log = (line) => console.log(`HealthLog: ${line}`);
|
||||
const warn = (line) => console.error(`HealthLog: WARNING — ${line}`);
|
||||
|
||||
/** The image's version: the build argument, else package.json beside it. */
|
||||
function imageVersion(env) {
|
||||
const fromEnv = (env.NEXT_PUBLIC_APP_VERSION ?? "").trim();
|
||||
if (fromEnv) return fromEnv;
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8"))
|
||||
.version;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Megabytes as an operator reads them. */
|
||||
function mb(bytes) {
|
||||
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
export async function preMigrationDump(env = process.env) {
|
||||
if (!preMigrationDumpEnabled(env)) {
|
||||
log("DB_BACKUP_BEFORE_MIGRATE=false, no dump before migrating.");
|
||||
return 0;
|
||||
}
|
||||
const databaseUrl = env.DATABASE_URL;
|
||||
if (!databaseUrl) return 0;
|
||||
|
||||
let pending;
|
||||
try {
|
||||
pending = await pendingMigrations({
|
||||
connectionString: databaseUrl,
|
||||
migrationsDir: join(process.cwd(), "prisma", "migrations"),
|
||||
});
|
||||
} catch (error) {
|
||||
warn(
|
||||
`could not count pending migrations (${error instanceof Error ? error.message : String(error)}); no dump before migrating.`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
if (pending.length === 0) return 0;
|
||||
|
||||
// A database no migration has touched holds nothing to lose.
|
||||
const client = await connect(databaseUrl);
|
||||
let applied;
|
||||
let tables;
|
||||
try {
|
||||
applied = await latestMigration(
|
||||
client,
|
||||
connectionFromUrl(databaseUrl).schema,
|
||||
);
|
||||
tables = (await listUserTables(client)).length;
|
||||
} finally {
|
||||
await client.end().catch(() => {});
|
||||
}
|
||||
if (applied === null && tables === 0) {
|
||||
log("new database, no dump before the first migrations.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
const dir = resolveBackupDir(env);
|
||||
if (!isWritableDirectory(dir)) {
|
||||
warn(
|
||||
`${dir} is not writable, so no dump is taken before the ${pending.length} pending migration(s). ` +
|
||||
"Mount the backups volume (docker-compose.yml) to keep a way back; continuing.",
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
log(
|
||||
`taking a database dump before ${pending.length} pending migration(s) into ${dir}...`,
|
||||
);
|
||||
const started = Date.now();
|
||||
try {
|
||||
const { dumpPath, manifest } = await createDatabaseDump({
|
||||
databaseUrl,
|
||||
dir,
|
||||
kind: "pre-migration",
|
||||
env,
|
||||
appVersion: imageVersion(env),
|
||||
buildSha: (env.NEXT_PUBLIC_APP_BUILD_SHA ?? "").trim() || null,
|
||||
});
|
||||
log(
|
||||
`pre-migration dump written: ${basename(dumpPath)} (${mb(manifest.dump.bytes)}, ` +
|
||||
`${Object.keys(manifest.rowCounts).length} tables, ${Math.round((Date.now() - started) / 1000)} s).`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`HealthLog: ERROR — the database dump before migrating failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
try {
|
||||
await applyRetention(dir, { dailyKeep: resolveDailyKeep(env) });
|
||||
} catch (error) {
|
||||
warn(
|
||||
`could not remove old dumps in ${dir} (${error instanceof Error ? error.message : String(error)}).`,
|
||||
);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const kind = process.argv[2];
|
||||
if (kind !== "pre-migration") {
|
||||
console.error("db-dump: usage: node scripts/db-dump.mjs pre-migration");
|
||||
process.exit(2);
|
||||
}
|
||||
process.exit(await preMigrationDump());
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
/**
|
||||
* Restore a whole-database dump into an empty database.
|
||||
*
|
||||
* docker compose run --rm db-restore list [directory]
|
||||
* docker compose run --rm db-restore list --from-bucket
|
||||
* docker compose run --rm db-restore latest [--from-bucket]
|
||||
* docker compose run --rm db-restore <file> [--ignore-key-mismatch]
|
||||
*
|
||||
* Runs in the app image with plain `node` (no `tsx`, no source checkout), as
|
||||
* the `db-restore` compose service, so a new server needs only the compose
|
||||
* file, the keys and the dump. `<file>` is a name in `DB_BACKUP_DIR` or a
|
||||
* path inside the container: a `.dump` written on a server, or a `.dump.enc`
|
||||
* downloaded from the off-host bucket (`database/` prefix), with its manifest
|
||||
* (`.manifest.json` or `.manifest.json.enc`) beside it. With `--from-bucket`
|
||||
* it fetches the pair from the bucket the `BACKUP_S3_*` variables name (the
|
||||
* newest for `latest`), which is the restore after the host and its local
|
||||
* dumps are gone.
|
||||
*
|
||||
* In this order, and it stops at the first thing that is wrong, before it
|
||||
* writes anything:
|
||||
* 1. an off-host copy is opened with BACKUP_ENCRYPTION_KEY (or a key in
|
||||
* BACKUP_ENCRYPTION_PREVIOUS_KEYS) into a temporary file;
|
||||
* 2. the dump's SHA-256 and size must equal the manifest's;
|
||||
* 3. the keys in this environment are compared with the manifest's
|
||||
* fingerprints; each one missing or different is named, and a mismatch
|
||||
* of the active encryption key refuses (`--ignore-key-mismatch` restores
|
||||
* anyway, for an operator who knows the key is lost);
|
||||
* 4. the target database must hold no table at all. This never overwrites.
|
||||
* Then `pg_restore` in one transaction, `ANALYZE`, and a count of every table
|
||||
* against the manifest. A count that differs empties the database again and
|
||||
* fails, so the app is never started on a restore that does not match.
|
||||
*
|
||||
* Exit 0 restored (or listed), 1 refused or failed, 2 usage.
|
||||
*/
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname, isAbsolute, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import {
|
||||
compareKeyFingerprints,
|
||||
compareRowCounts,
|
||||
connect,
|
||||
countRows,
|
||||
decryptOffhostFile,
|
||||
emptyDatabase,
|
||||
hashFile,
|
||||
isGoodSet,
|
||||
isWritableDirectory,
|
||||
keyFingerprints,
|
||||
listDumpSets,
|
||||
listUserTables,
|
||||
offhostKeysFromEnv,
|
||||
offhostObjectKey,
|
||||
parseDumpFileName,
|
||||
parseManifest,
|
||||
resolveBackupDir,
|
||||
runTool,
|
||||
statOrNull,
|
||||
connectionFromUrl,
|
||||
bucketConfigFromEnv,
|
||||
openBucket,
|
||||
pickBucketDump,
|
||||
OFFHOST_DATABASE_PREFIX,
|
||||
} from "../src/lib/database-backup/dump-core.mjs";
|
||||
import { shippedMigrations } from "./pending-migrations.mjs";
|
||||
|
||||
const out = (line = "") => console.log(line);
|
||||
const say = (line) => console.log(`db-restore: ${line}`);
|
||||
|
||||
class Refusal extends Error {}
|
||||
|
||||
function mb(bytes) {
|
||||
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this image knows the dump's newest migration. A dump from a newer
|
||||
* release restored under an older image leaves the app with a schema its code
|
||||
* does not expect, so the restore says which image to run.
|
||||
*/
|
||||
function imageKnowsMigration(name) {
|
||||
if (!name) return true;
|
||||
try {
|
||||
return shippedMigrations(
|
||||
join(process.cwd(), "prisma", "migrations"),
|
||||
).includes(name);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
function listCommand(dir) {
|
||||
const sets = listDumpSets(dir);
|
||||
if (sets.length === 0) {
|
||||
say(`no dumps in ${dir}`);
|
||||
return 0;
|
||||
}
|
||||
out(`Dumps in ${dir}, newest first:`);
|
||||
for (const set of sets) {
|
||||
const state = isGoodSet(set)
|
||||
? `${mb(set.dump.bytes)}`
|
||||
: set.encryptedDump
|
||||
? "off-host copy (.enc)"
|
||||
: set.dump
|
||||
? "no manifest, not restorable"
|
||||
: "incomplete";
|
||||
out(
|
||||
` ${set.base}.dump${set.encryptedDump && !set.dump ? ".enc" : ""} ${set.kind} ${set.timestamp.toISOString()} ${state}`,
|
||||
);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* The dump and manifest to restore: a local pair, or an encrypted pair
|
||||
* opened into `scratch`.
|
||||
*/
|
||||
async function resolveInput(arg, dir, env) {
|
||||
if (arg === "latest") {
|
||||
const set = listDumpSets(dir).find(
|
||||
(s) =>
|
||||
isGoodSet(s) ||
|
||||
(s.encryptedDump !== null &&
|
||||
(s.encryptedManifest !== null || s.manifest !== null)),
|
||||
);
|
||||
if (!set) throw new Refusal(`no restorable dump in ${dir}`);
|
||||
arg = set.dump ? set.dump.path : set.encryptedDump;
|
||||
}
|
||||
const path = isAbsolute(arg) ? arg : join(dir, arg);
|
||||
if (!(await statOrNull(path))) throw new Refusal(`${path} does not exist`);
|
||||
const name = basename(path);
|
||||
const parsed = parseDumpFileName(name);
|
||||
if (!parsed || parsed.role !== "dump" || parsed.variant === "partial") {
|
||||
throw new Refusal(
|
||||
`${name} is not a HealthLog database dump (healthlog-<time>-<kind>.dump or .dump.enc)`,
|
||||
);
|
||||
}
|
||||
const folder = dirname(path);
|
||||
const plainManifest = join(folder, `${parsed.base}.manifest.json`);
|
||||
const encManifest = `${plainManifest}.enc`;
|
||||
|
||||
if (parsed.variant === "final") {
|
||||
if (!(await statOrNull(plainManifest))) {
|
||||
throw new Refusal(
|
||||
`${basename(plainManifest)} is missing beside ${name}; a dump is restored only with its manifest`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
dumpPath: path,
|
||||
manifest: parseManifest(
|
||||
JSON.parse(await readFile(plainManifest, "utf8")),
|
||||
),
|
||||
scratch: null,
|
||||
};
|
||||
}
|
||||
|
||||
// An off-host copy. Open it next to the backups if there is room to write
|
||||
// there, in the container's temp directory otherwise.
|
||||
const keys = offhostKeysFromEnv(env);
|
||||
if (keys.length === 0) {
|
||||
throw new Refusal(
|
||||
"this is an off-host copy and BACKUP_ENCRYPTION_KEY is not set; it cannot be opened without it",
|
||||
);
|
||||
}
|
||||
const scratch = await mkdtemp(
|
||||
join(isWritableDirectory(dir) ? dir : tmpdir(), ".restore-"),
|
||||
);
|
||||
try {
|
||||
let manifest;
|
||||
if (await statOrNull(encManifest)) {
|
||||
const opened = join(scratch, `${parsed.base}.manifest.json`);
|
||||
await decryptOffhostFile({
|
||||
inPath: encManifest,
|
||||
outPath: opened,
|
||||
objectKey: offhostObjectKey(basename(encManifest)),
|
||||
keys,
|
||||
});
|
||||
manifest = parseManifest(JSON.parse(await readFile(opened, "utf8")));
|
||||
} else if (await statOrNull(plainManifest)) {
|
||||
manifest = parseManifest(
|
||||
JSON.parse(await readFile(plainManifest, "utf8")),
|
||||
);
|
||||
} else {
|
||||
throw new Refusal(
|
||||
`${basename(encManifest)} is missing beside ${name}; download it from the bucket too`,
|
||||
);
|
||||
}
|
||||
say(`opening the off-host copy ${name}...`);
|
||||
const dumpPath = join(scratch, `${parsed.base}.dump`);
|
||||
await decryptOffhostFile({
|
||||
inPath: path,
|
||||
outPath: dumpPath,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys,
|
||||
});
|
||||
return { dumpPath, manifest, scratch };
|
||||
} catch (error) {
|
||||
await rm(scratch, { recursive: true, force: true });
|
||||
throw error instanceof Refusal ? error : new Refusal(error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a dump and its manifest from the off-host bucket (`BACKUP_S3_*`) into
|
||||
* a scratch directory: the newest pair for `latest`, the named one otherwise.
|
||||
* Answers the downloaded `.dump.enc` and the directory to remove afterwards.
|
||||
*/
|
||||
async function downloadFromBucket(arg, dir, env) {
|
||||
const config = bucketConfigFromEnv(env);
|
||||
if (!config) {
|
||||
throw new Refusal(
|
||||
"--from-bucket needs BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY and BACKUP_S3_SECRET_KEY",
|
||||
);
|
||||
}
|
||||
const bucket = await openBucket(config);
|
||||
const keys = await bucket.list(OFFHOST_DATABASE_PREFIX);
|
||||
const pick = pickBucketDump(keys, arg === "latest" ? null : basename(arg));
|
||||
if (!pick) {
|
||||
throw new Refusal(
|
||||
arg === "latest"
|
||||
? `no database dump with its manifest under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`
|
||||
: `${basename(arg)} and its manifest are not both under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`,
|
||||
);
|
||||
}
|
||||
const scratch = await mkdtemp(
|
||||
join(isWritableDirectory(dir) ? dir : tmpdir(), ".download-"),
|
||||
);
|
||||
try {
|
||||
say(
|
||||
`downloading ${pick.dumpKey} and its manifest from ${config.bucket}...`,
|
||||
);
|
||||
const dumpPath = join(scratch, basename(pick.dumpKey));
|
||||
await bucket.download(
|
||||
pick.manifestKey,
|
||||
join(scratch, basename(pick.manifestKey)),
|
||||
);
|
||||
await bucket.download(pick.dumpKey, dumpPath);
|
||||
return { dumpPath, scratch };
|
||||
} catch (error) {
|
||||
await rm(scratch, { recursive: true, force: true });
|
||||
throw new Refusal(
|
||||
`could not download from the bucket: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function listBucketCommand(env) {
|
||||
const config = bucketConfigFromEnv(env);
|
||||
if (!config)
|
||||
throw new Refusal("--from-bucket needs the BACKUP_S3_* variables");
|
||||
const keys = (await (await openBucket(config)).list(OFFHOST_DATABASE_PREFIX))
|
||||
.filter((k) => k.endsWith(".dump.enc"))
|
||||
.sort()
|
||||
.reverse();
|
||||
out(
|
||||
`Dumps in bucket ${config.bucket} under ${OFFHOST_DATABASE_PREFIX}, newest first:`,
|
||||
);
|
||||
for (const key of keys) out(` ${key}`);
|
||||
if (keys.length === 0) say("none");
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function restoreCommand(arg, flags, env) {
|
||||
const dir = resolveBackupDir(env);
|
||||
const databaseUrl = env.DATABASE_URL;
|
||||
if (!databaseUrl) throw new Refusal("DATABASE_URL is not set");
|
||||
|
||||
let input = arg;
|
||||
let download = null;
|
||||
if (flags.has("--from-bucket")) {
|
||||
download = await downloadFromBucket(arg, dir, env);
|
||||
input = download.dumpPath;
|
||||
}
|
||||
let resolved;
|
||||
try {
|
||||
resolved = await resolveInput(input, dir, env);
|
||||
} finally {
|
||||
// The sealed copy is not needed once it is opened (or failed to open).
|
||||
if (download) await rm(download.scratch, { recursive: true, force: true });
|
||||
}
|
||||
const { dumpPath, manifest, scratch } = resolved;
|
||||
try {
|
||||
say(
|
||||
`dump ${manifest.file}: taken ${manifest.createdAt} by HealthLog ${manifest.appVersion ?? "(unknown version)"}, ` +
|
||||
`newest migration ${manifest.latestMigration ?? "(none)"}, Postgres ${manifest.postgres.serverVersion}, ` +
|
||||
`${Object.keys(manifest.rowCounts).length} tables`,
|
||||
);
|
||||
if (!imageKnowsMigration(manifest.latestMigration)) {
|
||||
say(
|
||||
`warning: this image does not know the dump's newest migration ${manifest.latestMigration}; ` +
|
||||
`start the app with HealthLog ${manifest.appVersion ?? "the version that wrote it"} or newer (HEALTHLOG_IMAGE_REF), not this image`,
|
||||
);
|
||||
}
|
||||
if (basename(dumpPath) !== manifest.file) {
|
||||
throw new Refusal(
|
||||
`the manifest describes ${manifest.file}, not ${basename(dumpPath)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// 2. The file is the one the manifest describes.
|
||||
const { sha256, bytes } = await hashFile(dumpPath);
|
||||
if (sha256 !== manifest.dump.sha256 || bytes !== manifest.dump.bytes) {
|
||||
throw new Refusal(
|
||||
`the dump does not match its manifest (SHA-256 ${sha256.slice(0, 12)}…, ${bytes} bytes; ` +
|
||||
`the manifest says ${manifest.dump.sha256.slice(0, 12)}…, ${manifest.dump.bytes} bytes). It is damaged or not this dump.`,
|
||||
);
|
||||
}
|
||||
say(`checksum matches the manifest (${mb(bytes)})`);
|
||||
|
||||
// 3. The keys.
|
||||
const keys = compareKeyFingerprints(manifest.keys, keyFingerprints(env));
|
||||
if (keys.findings.length === 0) {
|
||||
say("every key the dump was written with is configured here");
|
||||
}
|
||||
for (const finding of keys.findings) {
|
||||
say(
|
||||
`${finding.severity === "refuse" ? "KEY MISMATCH" : "warning"}: ${finding.key}: ${finding.detail}`,
|
||||
);
|
||||
}
|
||||
if (keys.refuse && !flags.has("--ignore-key-mismatch")) {
|
||||
throw new Refusal(
|
||||
"refusing to restore: set the encryption keys this dump was written with (see above). " +
|
||||
"Only if that key is lost for good, run again with --ignore-key-mismatch; everything encrypted will stay unreadable.",
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Never over existing data.
|
||||
const client = await connect(databaseUrl);
|
||||
try {
|
||||
const tables = await listUserTables(client);
|
||||
if (tables.length > 0) {
|
||||
throw new Refusal(
|
||||
`the target database already holds ${tables.length} ${tables.length === 1 ? "table" : "tables"} (${tables.slice(0, 3).join(", ")}${tables.length > 3 ? ", …" : ""}). ` +
|
||||
"A restore goes only into an empty database: start it on a new, empty database volume, and do not start the app before the restore.",
|
||||
);
|
||||
}
|
||||
|
||||
say("restoring (one transaction)...");
|
||||
const started = Date.now();
|
||||
await runTool(
|
||||
"pg_restore",
|
||||
[
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
"--single-transaction",
|
||||
"--exit-on-error",
|
||||
"--no-password",
|
||||
`--dbname=${connectionFromUrl(databaseUrl).pgEnv.PGDATABASE}`,
|
||||
dumpPath,
|
||||
],
|
||||
{ env: connectionFromUrl(databaseUrl).pgEnv },
|
||||
);
|
||||
say(`restored in ${Math.round((Date.now() - started) / 1000)} s`);
|
||||
|
||||
await client.query("ANALYZE");
|
||||
say("planner statistics gathered");
|
||||
|
||||
const problems = compareRowCounts(
|
||||
manifest.rowCounts,
|
||||
await countRows(client),
|
||||
);
|
||||
if (problems.length > 0) {
|
||||
for (const line of problems.slice(0, 20)) say(`row count: ${line}`);
|
||||
await emptyDatabase(client);
|
||||
throw new Refusal(
|
||||
`${problems.length} table(s) do not match the manifest; the database was emptied again. Do not start the app on it.`,
|
||||
);
|
||||
}
|
||||
say(
|
||||
`all ${Object.keys(manifest.rowCounts).length} tables hold the row counts the manifest names`,
|
||||
);
|
||||
} finally {
|
||||
await client.end().catch(() => {});
|
||||
}
|
||||
out();
|
||||
out(
|
||||
`Restored ${manifest.file}. Start the app with the image this dump came from or a newer one ` +
|
||||
`(HealthLog ${manifest.appVersion ?? "?"}, newest migration ${manifest.latestMigration ?? "none"}): docker compose up -d`,
|
||||
);
|
||||
return 0;
|
||||
} finally {
|
||||
if (scratch) await rm(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), env = process.env) {
|
||||
const flags = new Set(argv.filter((a) => a.startsWith("--")));
|
||||
const [command] = argv.filter((a) => !a.startsWith("--"));
|
||||
if (!command) {
|
||||
console.error(
|
||||
"db-restore: usage: db-restore list [directory] | latest | <file> [--from-bucket] [--ignore-key-mismatch]",
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
try {
|
||||
if (command === "list" && flags.has("--from-bucket")) {
|
||||
return await listBucketCommand(env);
|
||||
}
|
||||
if (command === "list") {
|
||||
const [, other] = argv.filter((a) => !a.startsWith("--"));
|
||||
return listCommand(other ?? resolveBackupDir(env));
|
||||
}
|
||||
return await restoreCommand(command, flags, env);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`db-restore: ${error instanceof Refusal ? "" : "failed: "}${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
process.exit(await main());
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* The accounts the disaster-recovery drill adds beside the demo seed: a
|
||||
* Guardian with a password, and a managed profile that Guardian administers.
|
||||
* Written through the app's own code (`createManagedProfile`,
|
||||
* `hashPassword`), so the rows are the ones a person creating them in the app
|
||||
* would leave.
|
||||
*
|
||||
* DATABASE_URL=… pnpm dlx tsx scripts/dr-drill-seed.ts
|
||||
*
|
||||
* Prints one JSON line: `{ "guardianId": …, "profileId": … }`. The drill
|
||||
* (`scripts/compose-dr-drill.sh`) checks after the restore that the Guardian
|
||||
* still sees that profile. Idempotent: a second run answers the same ids.
|
||||
*/
|
||||
import { hashPassword } from "@/lib/auth/password";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { createManagedProfile } from "@/lib/managed-profiles/create";
|
||||
|
||||
const GUARDIAN = "drill-guardian";
|
||||
const PASSWORD = "drill-guardian-password-1";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
let guardian = await prisma.user.findUnique({
|
||||
where: { username: GUARDIAN },
|
||||
});
|
||||
guardian ??= await prisma.user.create({
|
||||
data: {
|
||||
username: GUARDIAN,
|
||||
email: "drill-guardian@example.test",
|
||||
passwordHash: await hashPassword(PASSWORD),
|
||||
displayName: "Drill Guardian",
|
||||
locale: "en",
|
||||
timezone: "Europe/Berlin",
|
||||
onboardingCompletedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const existing = await prisma.accountGrant.findFirst({
|
||||
where: { granteeId: guardian.id, access: "MANAGE" },
|
||||
select: { grantorId: true },
|
||||
});
|
||||
const profileId =
|
||||
existing?.grantorId ??
|
||||
(
|
||||
await createManagedProfile({
|
||||
creatorId: guardian.id,
|
||||
displayName: "Drill Child",
|
||||
dateOfBirth: new Date("2018-05-04T00:00:00Z"),
|
||||
locale: "en",
|
||||
timezone: "Europe/Berlin",
|
||||
})
|
||||
).profile.id;
|
||||
console.log(JSON.stringify({ guardianId: guardian.id, profileId }));
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error: unknown) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(() => prisma.$disconnect());
|
||||
@@ -360,6 +360,38 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Database dumps",
|
||||
"description": "The nightly whole-database dump and the dump before migrations (docs/ops/disaster-recovery.md). Optional: each has a default, and the bundled compose file mounts the `backups` volume at the default directory.",
|
||||
"required": false,
|
||||
"variables": [
|
||||
{
|
||||
"name": "DB_BACKUP_ENABLED",
|
||||
"purpose": "Set false to stop the nightly database dump. On when unset.",
|
||||
"example": "true"
|
||||
},
|
||||
{
|
||||
"name": "DB_BACKUP_DIR",
|
||||
"purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. Without a writable directory the nightly dump only goes off-host and no dump is taken before migrations.",
|
||||
"example": "/var/lib/healthlog/backups"
|
||||
},
|
||||
{
|
||||
"name": "DB_BACKUP_SCHEDULE",
|
||||
"purpose": "Five-field cron for the nightly dump, in Europe/Berlin time like every scheduled job. Default 30 1 * * * (01:30).",
|
||||
"example": "30 1 * * *"
|
||||
},
|
||||
{
|
||||
"name": "DB_BACKUP_KEEP",
|
||||
"purpose": "How many nightly dumps stay in DB_BACKUP_DIR (default 7, at least 1). The last two dumps taken before a migration are kept besides.",
|
||||
"example": "7"
|
||||
},
|
||||
{
|
||||
"name": "DB_BACKUP_BEFORE_MIGRATE",
|
||||
"purpose": "Set false to start without the dump the entrypoint takes before applying migrations. On when unset; a failed dump stops the start until it works or this is false.",
|
||||
"example": "true"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Operational windows and cookie flags",
|
||||
"description": "Values the product states back to a user or that decide whether a session cookie is accepted. Optional: each has a documented default. They are listed so `pnpm check-env` reports them, and so the compose whitelist and the example files can be held in step with this manifest.",
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
/**
|
||||
* Boot-time notice before `prisma migrate deploy`: how many migrations this
|
||||
* image is about to apply, and a reminder to have a backup first.
|
||||
* image is about to apply, and that a dump is taken first
|
||||
* (`scripts/db-dump.mjs`, which imports `pendingMigrations` from here).
|
||||
*
|
||||
* Run by `docker-entrypoint.sh` as `node scripts/pending-migrations.mjs`.
|
||||
* It only reports. A migration that changes data cannot be undone by pulling
|
||||
* the previous image, so an operator who reads the count in the log still has
|
||||
* a moment to stop the container and take a `pg_dump`; the entrypoint does
|
||||
* not wait for that.
|
||||
* the previous image; the dump the entrypoint takes next is the way back.
|
||||
*
|
||||
* Best effort by design: any failure to count (no `_prisma_migrations` table
|
||||
* yet is not a failure, it means every migration is pending) prints one line
|
||||
@@ -67,9 +66,9 @@ export function pendingNotice(count) {
|
||||
const noun = count === 1 ? "migration" : "migrations";
|
||||
return (
|
||||
`HealthLog: ${count} database ${noun} pending. ` +
|
||||
"Back up the database before updating (pg_dump, see the Updates section " +
|
||||
"of docs/self-hosting/getting-started.md); a migration is not undone by " +
|
||||
"going back to the previous image."
|
||||
"Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is " +
|
||||
"written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); " +
|
||||
"a migration is not undone by going back to the previous image."
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,9 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
|
||||
it("keeps the new data where the Postgres 18 image expects it", () => {
|
||||
// Mounted at .../data, as 16 had it, the 18 image refuses to start.
|
||||
expect(db.volumes).toEqual(["postgres-data:/var/lib/postgresql"]);
|
||||
// `backups` holds the app's database dumps, not database files.
|
||||
expect(Object.keys(compose.volumes).sort()).toEqual([
|
||||
"backups",
|
||||
"pgdata",
|
||||
"postgres-data",
|
||||
]);
|
||||
|
||||
@@ -54,9 +54,29 @@ vi.mock("@/lib/jobs/offhost-backup", () => ({
|
||||
probeOffhostLifecycle: async () => ({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
}),
|
||||
}));
|
||||
|
||||
const databaseHealth = {
|
||||
enabled: true,
|
||||
directory: "/var/lib/healthlog/backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
at: "2026-09-21T23:31:40.000Z",
|
||||
fileName: "healthlog-20260921T233000Z-daily.dump",
|
||||
sizeBytes: 52_428_800,
|
||||
local: true,
|
||||
offhost: true,
|
||||
},
|
||||
lastFailure: null,
|
||||
};
|
||||
vi.mock("@/lib/database-backup/health", () => ({
|
||||
getDatabaseBackupHealth: async () => databaseHealth,
|
||||
}));
|
||||
|
||||
import { GET } from "../route";
|
||||
import type { BackupsList } from "@/types/backups";
|
||||
|
||||
@@ -166,7 +186,11 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
// No bucket to ask on a host that is not configured.
|
||||
lifecycle: { state: "unknown", expirationDays: null },
|
||||
lifecycle: {
|
||||
state: "unknown",
|
||||
expirationDays: null,
|
||||
database: { state: "unknown", expirationDays: null },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -229,6 +253,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
expect(body.offhost.lifecycle).toEqual({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
});
|
||||
expect(body.offhost.pendingDeletions).toEqual({
|
||||
count: 2,
|
||||
@@ -285,3 +310,10 @@ describe("GET /api/admin/backups — a run in progress and one a process died un
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/admin/backups — the database dump", () => {
|
||||
it("carries the dump's health beside the account copies", async () => {
|
||||
const body = await read();
|
||||
expect(body.database).toEqual(databaseHealth);
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,10 @@
|
||||
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
||||
* timestamp either way.
|
||||
*
|
||||
* The database row comes first in what an operator needs and last in this
|
||||
* file: the nightly whole-database dump (`getDatabaseBackupHealth`), the
|
||||
* one copy that rebuilds a lost server, read from its own ledger.
|
||||
*
|
||||
* And alongside both, the off-host leg: one row per account saying when this
|
||||
* host last put that account's encrypted copy in the operator's bucket and how
|
||||
* stale that is against the nightly schedule. It comes from the ledger the
|
||||
@@ -31,11 +35,13 @@ import {
|
||||
} from "@/lib/jobs/job-failures";
|
||||
import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue";
|
||||
import { LIFECYCLE_UNKNOWN } from "@/lib/jobs/offhost-lifecycle";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
@@ -190,16 +196,15 @@ export const GET = apiHandler(async () => {
|
||||
readBackupPassActivity(prisma, DATA_BACKUP_QUEUE, weeklyRunningSince),
|
||||
readBackupPassActivity(prisma, OFFHOST_BACKUP_QUEUE, offhostRunningSince),
|
||||
]);
|
||||
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
|
||||
const [purgeCount, oldestPurge, lifecycle, database] = await Promise.all([
|
||||
prisma.offhostPurgeRequest.count({ where: stillInBucket }),
|
||||
prisma.offhostPurgeRequest.findFirst({
|
||||
where: stillInBucket,
|
||||
orderBy: { requestedAt: "asc" },
|
||||
select: { requestedAt: true, lastFailure: true },
|
||||
}),
|
||||
configured
|
||||
? probeOffhostLifecycle()
|
||||
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
|
||||
configured ? probeOffhostLifecycle() : Promise.resolve(LIFECYCLE_UNKNOWN),
|
||||
getDatabaseBackupHealth(now),
|
||||
]);
|
||||
|
||||
const payload: BackupsList = {
|
||||
@@ -229,6 +234,7 @@ export const GET = apiHandler(async () => {
|
||||
},
|
||||
lifecycle,
|
||||
},
|
||||
database,
|
||||
};
|
||||
|
||||
return apiSuccess(payload);
|
||||
|
||||
@@ -26,7 +26,11 @@ import { OffhostBackupsSection } from "../offhost-backups-section";
|
||||
const QUIET = {
|
||||
activity: { runningSince: null, interrupted: [] },
|
||||
pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null },
|
||||
lifecycle: { state: "configured" as const, expirationDays: 30 },
|
||||
lifecycle: {
|
||||
state: "configured" as const,
|
||||
expirationDays: 30,
|
||||
database: { state: "configured" as const, expirationDays: 30 },
|
||||
},
|
||||
};
|
||||
|
||||
function render(data: BackupsList["offhost"] | null): string {
|
||||
@@ -133,7 +137,11 @@ describe("<OffhostBackupsSection>", () => {
|
||||
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFailure: "AccessDenied",
|
||||
},
|
||||
lifecycle: { state: "missing", expirationDays: null },
|
||||
lifecycle: {
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
},
|
||||
});
|
||||
expect(html).toContain('data-lifecycle-state="missing"');
|
||||
expect(html).toContain("no lifecycle rule");
|
||||
|
||||
@@ -28,6 +28,7 @@ import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
|
||||
import { RestoreJobStatus, visibleRestoreJobs } from "./backup-restore-status";
|
||||
import { BackupRowsCards, BackupRowsTable } from "./backups/backup-rows";
|
||||
import { DatabaseDumpRow } from "./backups/database-dump-row";
|
||||
import { BackupUploadCard } from "./backups/backup-upload-card";
|
||||
import {
|
||||
RestoreRefusalNotice,
|
||||
@@ -110,6 +111,14 @@ export function BackupsSection() {
|
||||
}
|
||||
/>
|
||||
|
||||
{data?.database ? (
|
||||
<DatabaseDumpRow
|
||||
health={data.database}
|
||||
bucketLifecycle={
|
||||
data.offhost.configured ? data.offhost.lifecycle.database : null
|
||||
}
|
||||
/>
|
||||
) : null}
|
||||
{data?.schedule ? (
|
||||
<ScheduleHealthNotice schedule={data.schedule} />
|
||||
) : null}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* The database row on the backups card says where the newest good dump is,
|
||||
* how long a deleted account stays in the dumps, and each thing that keeps a
|
||||
* dump from being taken or from ever ageing out of the bucket.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
|
||||
|
||||
import { DatabaseDumpRow } from "../database-dump-row";
|
||||
|
||||
const HEALTHY: DatabaseBackupHealth = {
|
||||
enabled: true,
|
||||
directory: "/var/lib/healthlog/backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
at: "2026-10-24T23:31:00.000Z",
|
||||
fileName: "healthlog-20261024T233000Z-daily.dump",
|
||||
sizeBytes: 52_428_800,
|
||||
local: true,
|
||||
offhost: true,
|
||||
},
|
||||
lastFailure: null,
|
||||
};
|
||||
|
||||
function render(
|
||||
health: DatabaseBackupHealth,
|
||||
bucketLifecycle: LifecycleVerdict | null = {
|
||||
state: "configured",
|
||||
expirationDays: 30,
|
||||
},
|
||||
): string {
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<DatabaseDumpRow health={health} bucketLifecycle={bucketLifecycle} />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("<DatabaseDumpRow>", () => {
|
||||
it("names the newest good dump, where it is, and how long dumps keep deleted accounts", () => {
|
||||
const html = render(HEALTHY);
|
||||
expect(html).toContain("kept on this server and off-host");
|
||||
expect(html).toContain("the newest 7 are kept on this server");
|
||||
expect(html).toContain("removed after 30 days");
|
||||
expect(html).not.toContain('data-slot="database-dump-failed"');
|
||||
expect(html).not.toContain('data-slot="database-dump-setup"');
|
||||
expect(html).not.toContain('data-slot="database-dump-no-lifecycle"');
|
||||
});
|
||||
|
||||
it("warns when no bucket rule ever removes the dumps", () => {
|
||||
const html = render(HEALTHY, { state: "missing", expirationDays: null });
|
||||
expect(html).toContain('data-slot="database-dump-no-lifecycle"');
|
||||
expect(html).toContain("never removed");
|
||||
});
|
||||
|
||||
it("shows a failure newer than the last good dump, and not an older one", () => {
|
||||
const failed = render({
|
||||
...HEALTHY,
|
||||
lastFailure: {
|
||||
at: "2026-10-25T23:31:00.000Z",
|
||||
error: "pg_dump exited 1: No space left on device",
|
||||
},
|
||||
});
|
||||
expect(failed).toContain('data-slot="database-dump-failed"');
|
||||
expect(failed).toContain("No space left on device");
|
||||
const recovered = render({
|
||||
...HEALTHY,
|
||||
lastFailure: { at: "2026-10-20T23:31:00.000Z", error: "old" },
|
||||
});
|
||||
expect(recovered).not.toContain('data-slot="database-dump-failed"');
|
||||
});
|
||||
|
||||
it("says when dumps go off-host only, or cannot be taken at all", () => {
|
||||
expect(render({ ...HEALTHY, localDirWritable: false }, null)).toContain(
|
||||
"dumps go off-host only",
|
||||
);
|
||||
expect(
|
||||
render(
|
||||
{ ...HEALTHY, localDirWritable: false, offhostConfigured: false },
|
||||
null,
|
||||
),
|
||||
).toContain("no database dump can be taken");
|
||||
expect(render({ ...HEALTHY, lastGood: null }, null)).toContain(
|
||||
"No database dump yet",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* The nightly whole-database dump, as one row on the backups card.
|
||||
*
|
||||
* The account copies below it restore one account on this server; the dump is
|
||||
* the copy that rebuilds the server. So the row leads the card and says three
|
||||
* things: when the newest good dump was taken and where it is, whether the
|
||||
* newest run failed, and whether this server can keep a dump at all.
|
||||
*/
|
||||
import { AlertTriangle, HardDrive } from "lucide-react";
|
||||
|
||||
import { SettingsInfoTile } from "@/components/settings/_info-tile";
|
||||
import { formatBytes } from "@/lib/format/bytes";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
|
||||
|
||||
function whereKept(
|
||||
good: NonNullable<DatabaseBackupHealth["lastGood"]>,
|
||||
t: ReturnType<typeof useTranslations>["t"],
|
||||
) {
|
||||
if (good.local && good.offhost) {
|
||||
return t("admin.section.backups.database.whereBoth");
|
||||
}
|
||||
if (good.offhost) return t("admin.section.backups.database.whereOffhost");
|
||||
return t("admin.section.backups.database.whereLocal");
|
||||
}
|
||||
|
||||
export function DatabaseDumpRow({
|
||||
health,
|
||||
bucketLifecycle,
|
||||
}: {
|
||||
health: DatabaseBackupHealth;
|
||||
/** The bucket's rule for `database/`; null when nothing goes off-host. */
|
||||
bucketLifecycle: LifecycleVerdict | null;
|
||||
}) {
|
||||
const { t, locale } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
const good = health.lastGood;
|
||||
const failure = health.lastFailure;
|
||||
const failedSinceGood =
|
||||
failure !== null && (good === null || failure.at > good.at);
|
||||
|
||||
let setupProblem: string | null = null;
|
||||
if (!health.enabled) {
|
||||
setupProblem = t("admin.section.backups.database.disabled");
|
||||
} else if (!health.localDirWritable) {
|
||||
setupProblem = health.offhostConfigured
|
||||
? t("admin.section.backups.database.noDirectoryOffhost", {
|
||||
dir: health.directory,
|
||||
})
|
||||
: t("admin.section.backups.database.noDirectory", {
|
||||
dir: health.directory,
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-2" data-slot="database-dump">
|
||||
<div className="flex items-start gap-2">
|
||||
<HardDrive
|
||||
className="text-muted-foreground mt-0.5 h-4 w-4 shrink-0"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<div>
|
||||
<p className="text-sm font-medium">
|
||||
{t("admin.section.backups.database.title")}
|
||||
</p>
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="database-dump-last-good"
|
||||
>
|
||||
{good
|
||||
? t("admin.section.backups.database.lastGood", {
|
||||
when: fmt.dateTime(good.at),
|
||||
size: formatBytes(good.sizeBytes ?? 0, locale),
|
||||
where: whereKept(good, t),
|
||||
})
|
||||
: t("admin.section.backups.database.none")}
|
||||
</p>
|
||||
{/* How long a deleted account stays in the dumps: the operator's
|
||||
side of the deletion promise (GDPR retention of backups). */}
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="database-dump-retention"
|
||||
>
|
||||
{t("admin.section.backups.database.retentionLocal", {
|
||||
nights: health.keepNightly,
|
||||
})}
|
||||
{bucketLifecycle?.state === "configured"
|
||||
? ` ${t("admin.section.backups.database.retentionOffhost", {
|
||||
days: bucketLifecycle.expirationDays ?? 0,
|
||||
})}`
|
||||
: null}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{failedSinceGood && failure ? (
|
||||
<div
|
||||
role="alert"
|
||||
data-slot="database-dump-failed"
|
||||
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertTriangle
|
||||
className="mt-0.5 h-4 w-4 shrink-0"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<div>
|
||||
<p className="font-medium">
|
||||
{t("admin.section.backups.database.failedTitle")}
|
||||
</p>
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.database.failedDescription", {
|
||||
when: fmt.dateTime(failure.at),
|
||||
reason: failure.error,
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{bucketLifecycle?.state === "missing" ? (
|
||||
<SettingsInfoTile
|
||||
icon={AlertTriangle}
|
||||
tone="warning"
|
||||
role="alert"
|
||||
data-slot="database-dump-no-lifecycle"
|
||||
>
|
||||
{t("admin.section.backups.database.retentionOffhostMissing")}
|
||||
</SettingsInfoTile>
|
||||
) : null}
|
||||
|
||||
{setupProblem ? (
|
||||
<SettingsInfoTile
|
||||
icon={AlertTriangle}
|
||||
tone="warning"
|
||||
role="alert"
|
||||
data-slot="database-dump-setup"
|
||||
>
|
||||
{setupProblem}
|
||||
</SettingsInfoTile>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => {
|
||||
expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("collectReadiness — the database dump", () => {
|
||||
const offhost = {
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.test",
|
||||
BACKUP_S3_BUCKET: "b",
|
||||
BACKUP_S3_ACCESS_KEY: "a",
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
BACKUP_ENCRYPTION_KEY: HEX32,
|
||||
};
|
||||
const dumpLine = (
|
||||
env: Record<string, string | undefined>,
|
||||
writable: boolean,
|
||||
) =>
|
||||
collectReadiness(env, () => writable).lines.find(
|
||||
(l) => l.label === "Database dump",
|
||||
);
|
||||
|
||||
it("names the schedule, the directory and the copies kept", () => {
|
||||
const line = dumpLine(baseEnv(), true);
|
||||
expect(line?.status).toBe("ok");
|
||||
expect(line?.detail).toBe(
|
||||
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
|
||||
);
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
|
||||
"copied off-host",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns when nothing can be kept on the server", () => {
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("goes off-host only"),
|
||||
});
|
||||
expect(dumpLine(baseEnv(), false)).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("no database dump can be taken"),
|
||||
});
|
||||
});
|
||||
|
||||
it("warns when it is switched off, and names a schedule it ignored", () => {
|
||||
expect(
|
||||
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
|
||||
).toMatchObject({ status: "warn" });
|
||||
const report = collectReadiness(
|
||||
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
|
||||
() => true,
|
||||
);
|
||||
expect(
|
||||
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
|
||||
).toBe("not a five-field cron; using 30 1 * * *");
|
||||
expect(report.hasBlocker).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -19,6 +19,13 @@ import {
|
||||
LEGACY_ANY_HOST_WARNING,
|
||||
legacyAnyHostConfigured,
|
||||
} from "@/lib/ai/local-host-allowlist";
|
||||
import {
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export interface ReadinessLine {
|
||||
/** Short label, e.g. "ENCRYPTION_KEY". */
|
||||
@@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean {
|
||||
return keys.some((k) => (env[k] ?? "").trim().length > 0);
|
||||
}
|
||||
|
||||
/** Inspect the environment and build the readiness report. Pure + side-effect free. */
|
||||
export function collectReadiness(env: Env = process.env): ReadinessReport {
|
||||
/**
|
||||
* The nightly database dump's line. The dump is what rebuilds a lost server,
|
||||
* so a host where it cannot be kept anywhere is said out loud at every boot,
|
||||
* and so is one where it only goes off-host.
|
||||
*/
|
||||
export function databaseBackupReadiness(
|
||||
env: Env,
|
||||
writable: (dir: string) => boolean,
|
||||
): ReadinessLine[] {
|
||||
const label = "Database dump";
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
return [
|
||||
{
|
||||
label,
|
||||
status: "warn",
|
||||
detail:
|
||||
"DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way",
|
||||
},
|
||||
];
|
||||
}
|
||||
const lines: ReadinessLine[] = [];
|
||||
const schedule = resolveBackupSchedule(env);
|
||||
if (!schedule.valid) {
|
||||
lines.push({
|
||||
label: "DB_BACKUP_SCHEDULE",
|
||||
status: "warn",
|
||||
detail: `not a five-field cron; using ${schedule.cron}`,
|
||||
});
|
||||
}
|
||||
const dir = resolveBackupDir(env);
|
||||
const offhost = allSet(env, [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
]);
|
||||
if (writable(dir)) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "ok",
|
||||
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
|
||||
});
|
||||
} else if (offhost) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
} else {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the environment and build the readiness report. Pure apart from
|
||||
* `writable`, the probe of the dump directory, which tests replace.
|
||||
*/
|
||||
export function collectReadiness(
|
||||
env: Env = process.env,
|
||||
writable: (dir: string) => boolean = isWritableDirectory,
|
||||
): ReadinessReport {
|
||||
const lines: ReadinessLine[] = [];
|
||||
|
||||
// --- Core secrets (boot blockers) ---------------------------------------
|
||||
@@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport {
|
||||
});
|
||||
}
|
||||
|
||||
lines.push(...databaseBackupReadiness(env, writable));
|
||||
|
||||
// --- Optional subsystems (informational) --------------------------------
|
||||
const optional: Array<[string, boolean]> = [
|
||||
["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])],
|
||||
|
||||
@@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
|
||||
HostMetric: "instance-wide host telemetry, not attributable to an account",
|
||||
OffhostBackupKeyUse:
|
||||
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
|
||||
DatabaseBackupRun:
|
||||
"the instance's ledger of nightly whole-database dumps; describes the server, not any account",
|
||||
OffhostPurgeRequest:
|
||||
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
|
||||
CycleSymptomCategory:
|
||||
|
||||
@@ -0,0 +1,689 @@
|
||||
/**
|
||||
* The rules of the whole-database dump that need no database: names,
|
||||
* retention, the manifest, key fingerprints, row-count comparison, the
|
||||
* connection the client tools get, and the off-host envelope read back by the
|
||||
* restore's own reader.
|
||||
*
|
||||
* The fingerprints and the envelope are checked against the app's own code
|
||||
* (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy
|
||||
* of the rule: the restore script cannot import TypeScript, so these are the
|
||||
* tests that keep its plain-JS reading of both from drifting.
|
||||
*/
|
||||
import {
|
||||
mkdtempSync,
|
||||
createReadStream,
|
||||
rmSync,
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
bucketConfigFromEnv,
|
||||
pickBucketDump,
|
||||
buildManifest,
|
||||
compareKeyFingerprints,
|
||||
compareRowCounts,
|
||||
connectionFromUrl,
|
||||
decodeKeyMaterial,
|
||||
decryptOffhostFile,
|
||||
dumpBaseName,
|
||||
formatDumpTimestamp,
|
||||
keyFingerprints,
|
||||
listDumpSets,
|
||||
offhostObjectKey,
|
||||
parseDumpFileName,
|
||||
parseManifest,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
selectForDeletion,
|
||||
type DumpSet,
|
||||
} from "../dump-core.mjs";
|
||||
import type { S3Like } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
/** Every byte of a stream, in order. */
|
||||
async function collect(stream: AsyncIterable<unknown>): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
const HEX_A = "a".repeat(64);
|
||||
const HEX_B = "b".repeat(64);
|
||||
const HEX_C = "c".repeat(64);
|
||||
const HEX_D = "d".repeat(64);
|
||||
|
||||
const dirs: string[] = [];
|
||||
function tempDir(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0))
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("dump names", () => {
|
||||
it("names a dump by its UTC instant and kind", () => {
|
||||
const at = new Date("2026-10-25T01:30:07.123Z");
|
||||
expect(formatDumpTimestamp(at)).toBe("20261025T013007Z");
|
||||
expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily");
|
||||
expect(dumpBaseName("pre-migration", at)).toBe(
|
||||
"healthlog-20261025T013007Z-pre-migration",
|
||||
);
|
||||
});
|
||||
|
||||
it("parses every file it writes, and back to the same instant", () => {
|
||||
const base = "healthlog-20261025T013007Z-daily";
|
||||
const cases: Array<[string, string, string]> = [
|
||||
[`${base}.dump`, "dump", "final"],
|
||||
[`${base}.manifest.json`, "manifest", "final"],
|
||||
[`${base}.dump.partial`, "dump", "partial"],
|
||||
[`${base}.manifest.json.partial`, "manifest", "partial"],
|
||||
[`${base}.dump.enc`, "dump", "encrypted"],
|
||||
[`${base}.manifest.json.enc`, "manifest", "encrypted"],
|
||||
];
|
||||
for (const [name, role, variant] of cases) {
|
||||
const parsed = parseDumpFileName(name);
|
||||
expect(parsed, name).not.toBeNull();
|
||||
expect(parsed?.base).toBe(base);
|
||||
expect(parsed?.kind).toBe("daily");
|
||||
expect(parsed?.role).toBe(role);
|
||||
expect(parsed?.variant).toBe(variant);
|
||||
expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z");
|
||||
}
|
||||
expect(
|
||||
parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind,
|
||||
).toBe("pre-migration");
|
||||
});
|
||||
|
||||
it("refuses every name it did not write", () => {
|
||||
for (const name of [
|
||||
"healthlog.dump",
|
||||
"backup-20261025T013007Z-daily.dump",
|
||||
"healthlog-20261025T013007Z-weekly.dump",
|
||||
"healthlog-20261025-daily.dump",
|
||||
"healthlog-20261025T013007Z-daily.sql",
|
||||
"healthlog-20261025T013007Z-daily.dump.bak",
|
||||
"healthlog-20261399T013007Z-daily.dump",
|
||||
"notes.txt",
|
||||
]) {
|
||||
expect(parseDumpFileName(name), name).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("configuration", () => {
|
||||
it("keeps at least one daily dump and ignores what is not a number", () => {
|
||||
expect(resolveDailyKeep({})).toBe(7);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7);
|
||||
});
|
||||
|
||||
it("takes a five-field cron and says when it ignored one", () => {
|
||||
expect(resolveBackupSchedule({})).toEqual({
|
||||
cron: "30 1 * * *",
|
||||
valid: true,
|
||||
configured: false,
|
||||
});
|
||||
expect(
|
||||
resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }),
|
||||
).toEqual({
|
||||
cron: "0 2 * * 1-5",
|
||||
valid: true,
|
||||
configured: true,
|
||||
});
|
||||
for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) {
|
||||
expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({
|
||||
cron: "30 1 * * *",
|
||||
valid: false,
|
||||
configured: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/** A dump set as `listDumpSets` reports it, for the pure retention rule. */
|
||||
function set(
|
||||
kind: "daily" | "pre-migration",
|
||||
iso: string,
|
||||
parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {},
|
||||
now = Date.parse("2026-10-25T12:00:00Z"),
|
||||
): DumpSet {
|
||||
const base = dumpBaseName(kind, new Date(iso));
|
||||
return {
|
||||
base,
|
||||
kind,
|
||||
timestamp: new Date(iso),
|
||||
dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 },
|
||||
manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`,
|
||||
encryptedDump: null,
|
||||
encryptedManifest: null,
|
||||
leftovers:
|
||||
parts.partialAgeMs === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
path: `/b/${base}.dump.partial`,
|
||||
mtimeMs: now - parts.partialAgeMs,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("retention", () => {
|
||||
const now = Date.parse("2026-10-25T12:00:00Z");
|
||||
|
||||
it("keeps the newest N daily dumps and removes the rest, both files each", () => {
|
||||
const sets = [1, 2, 3, 4, 5].map((d) =>
|
||||
set("daily", `2026-10-2${d}T01:30:00Z`),
|
||||
);
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 3, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed.sort()).toEqual(
|
||||
[
|
||||
"/b/healthlog-20261021T013000Z-daily.dump",
|
||||
"/b/healthlog-20261021T013000Z-daily.manifest.json",
|
||||
"/b/healthlog-20261022T013000Z-daily.dump",
|
||||
"/b/healthlog-20261022T013000Z-daily.manifest.json",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("never removes the newest good dump, whatever keep says", () => {
|
||||
const sets = [
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
set("daily", "2026-10-23T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 0, "pre-migration": 0 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump");
|
||||
expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump");
|
||||
});
|
||||
|
||||
it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => {
|
||||
const sets = [
|
||||
set("daily", "2026-10-25T01:30:00Z", {
|
||||
dump: false,
|
||||
partialAgeMs: 60_000,
|
||||
}),
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 1, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps two pre-migration dumps apart from the daily count", () => {
|
||||
const sets = [
|
||||
set("pre-migration", "2026-10-20T08:00:00Z"),
|
||||
set("pre-migration", "2026-10-10T08:00:00Z"),
|
||||
set("pre-migration", "2026-10-01T08:00:00Z"),
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 1, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([
|
||||
"/b/healthlog-20261001T080000Z-pre-migration.dump",
|
||||
"/b/healthlog-20261001T080000Z-pre-migration.manifest.json",
|
||||
]);
|
||||
});
|
||||
|
||||
it("removes a .partial only once it is older than twelve hours", () => {
|
||||
const fresh = set("daily", "2026-10-25T01:30:00Z", {
|
||||
dump: false,
|
||||
manifest: false,
|
||||
partialAgeMs: 60 * 60_000,
|
||||
});
|
||||
const stale = set("daily", "2026-10-23T01:30:00Z", {
|
||||
dump: false,
|
||||
manifest: false,
|
||||
partialAgeMs: 13 * 60 * 60_000,
|
||||
});
|
||||
const doomed = selectForDeletion([fresh, stale], {
|
||||
keep: { daily: 7, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([
|
||||
"/b/healthlog-20261023T013000Z-daily.dump.partial",
|
||||
]);
|
||||
});
|
||||
|
||||
it("works on a real directory and leaves files it did not write alone", async () => {
|
||||
const dir = tempDir();
|
||||
const names: string[] = [];
|
||||
for (let day = 15; day <= 24; day++) {
|
||||
const base = `healthlog-202610${day}T013000Z-daily`;
|
||||
names.push(`${base}.dump`, `${base}.manifest.json`);
|
||||
}
|
||||
// Two that are not ours, and an off-host copy an operator put here.
|
||||
names.push(
|
||||
"my-own-dump.dump",
|
||||
"healthlog-20261001T013000Z-daily.dump.bak",
|
||||
"healthlog-20261002T013000Z-daily.dump.enc",
|
||||
);
|
||||
for (const name of names) writeFileSync(join(dir, name), "x");
|
||||
// A crash's leftovers: one old, one from a run that may still be going.
|
||||
const oldPartial = join(
|
||||
dir,
|
||||
"healthlog-20261003T013000Z-daily.dump.partial",
|
||||
);
|
||||
writeFileSync(oldPartial, "x");
|
||||
const day = 24 * 60 * 60;
|
||||
utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day);
|
||||
writeFileSync(
|
||||
join(dir, "healthlog-20261025T013000Z-daily.dump.partial"),
|
||||
"x",
|
||||
);
|
||||
|
||||
const removed = await applyRetention(dir, { dailyKeep: 7 });
|
||||
expect(removed).toHaveLength(7);
|
||||
const left = (await readdir(dir)).sort();
|
||||
expect(left).toContain("my-own-dump.dump");
|
||||
expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak");
|
||||
expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc");
|
||||
expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial");
|
||||
expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial");
|
||||
expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual(
|
||||
[18, 19, 20, 21, 22, 23, 24].map(
|
||||
(d) => `healthlog-202610${d}T013000Z-daily.dump`,
|
||||
),
|
||||
);
|
||||
expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily");
|
||||
});
|
||||
});
|
||||
|
||||
describe("key fingerprints", () => {
|
||||
it("are the fingerprints the app's own key code gives, for the map and the single key", async () => {
|
||||
vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B }));
|
||||
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2");
|
||||
vi.stubEnv("ENCRYPTION_KEY", "");
|
||||
const crypto = await import("@/lib/crypto");
|
||||
crypto._resetCryptoCacheForTests();
|
||||
const prints = keyFingerprints(process.env);
|
||||
expect(prints.encryption.activeKeyId).toBe("v2");
|
||||
expect(prints.encryption.keys).toEqual([
|
||||
{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") },
|
||||
{ id: "v2", fingerprint: crypto.getKeyFingerprint("v2") },
|
||||
]);
|
||||
|
||||
vi.stubEnv("ENCRYPTION_KEYS", "");
|
||||
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
|
||||
const base64 = Buffer.from(HEX_C, "hex").toString("base64");
|
||||
vi.stubEnv("ENCRYPTION_KEY", base64);
|
||||
crypto._resetCryptoCacheForTests();
|
||||
const single = keyFingerprints(process.env);
|
||||
expect(single.encryption).toEqual({
|
||||
activeKeyId: "v1",
|
||||
keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }],
|
||||
});
|
||||
crypto._resetCryptoCacheForTests();
|
||||
});
|
||||
|
||||
it("names the off-host key the way an off-host object records it", async () => {
|
||||
const { offhostKeyId } = await import("@/lib/jobs/offhost-backup");
|
||||
const prints = keyFingerprints({
|
||||
ENCRYPTION_KEY: HEX_A,
|
||||
BACKUP_ENCRYPTION_KEY: HEX_D,
|
||||
BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`,
|
||||
API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789",
|
||||
});
|
||||
expect(prints.backupEncryptionKey).toBe(
|
||||
offhostKeyId(Buffer.from(HEX_D, "hex")),
|
||||
);
|
||||
expect(prints.backupEncryptionPreviousKeys).toEqual([
|
||||
offhostKeyId(Buffer.from(HEX_C, "hex")),
|
||||
]);
|
||||
expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/);
|
||||
});
|
||||
|
||||
it("never carries key material", () => {
|
||||
const env = {
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }),
|
||||
BACKUP_ENCRYPTION_KEY: HEX_B,
|
||||
API_TOKEN_HMAC_KEY: HEX_C,
|
||||
};
|
||||
const text = JSON.stringify(keyFingerprints(env));
|
||||
for (const secret of [HEX_A, HEX_B, HEX_C]) {
|
||||
expect(text).not.toContain(secret.slice(0, 16));
|
||||
}
|
||||
});
|
||||
|
||||
it("decodes keys the way the app does, and nothing that is not a key", () => {
|
||||
expect(decodeKeyMaterial(HEX_A)?.length).toBe(32);
|
||||
expect(
|
||||
decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length,
|
||||
).toBe(32);
|
||||
expect(decodeKeyMaterial("")).toBeNull();
|
||||
expect(decodeKeyMaterial("not a key!")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("comparing a dump's keys with the environment's", () => {
|
||||
const written = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
|
||||
it("finds nothing when every key is there", () => {
|
||||
const same = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
expect(compareKeyFingerprints(written, same)).toEqual({
|
||||
findings: [],
|
||||
refuse: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses when the active key is different, and says which", () => {
|
||||
const wrong = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
const result = compareKeyFingerprints(written, wrong);
|
||||
expect(result.refuse).toBe(true);
|
||||
expect(result.findings).toHaveLength(1);
|
||||
expect(result.findings[0]).toMatchObject({
|
||||
key: "ENCRYPTION_KEYS[v2]",
|
||||
state: "different",
|
||||
severity: "refuse",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses when the active key is missing, warns for a retired one", () => {
|
||||
const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A });
|
||||
const result = compareKeyFingerprints(written, onlyOld);
|
||||
expect(result.refuse).toBe(true);
|
||||
const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f]));
|
||||
expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "refuse",
|
||||
});
|
||||
expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined();
|
||||
expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "warn",
|
||||
});
|
||||
expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "warn",
|
||||
});
|
||||
|
||||
const noRetired = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }),
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-two",
|
||||
});
|
||||
const second = compareKeyFingerprints(written, noRetired);
|
||||
expect(second.refuse).toBe(false);
|
||||
expect(second.findings.map((f) => [f.key, f.severity])).toEqual([
|
||||
["ENCRYPTION_KEYS[v1]", "warn"],
|
||||
["API_TOKEN_HMAC_KEY", "warn"],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the manifest", () => {
|
||||
const manifest = buildManifest({
|
||||
kind: "daily",
|
||||
file: "healthlog-20261025T013000Z-daily.dump",
|
||||
createdAt: new Date("2026-10-25T01:30:00Z"),
|
||||
appVersion: "1.46.0",
|
||||
buildSha: "abc1234",
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
server: { version: "18.6", versionNum: 180006 },
|
||||
pgDump: "pg_dump (PostgreSQL) 18.6",
|
||||
archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
|
||||
rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 },
|
||||
env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" },
|
||||
});
|
||||
|
||||
it("carries what a restore and an operator need", () => {
|
||||
expect(manifest).toMatchObject({
|
||||
format: "healthlog-database-dump",
|
||||
formatVersion: 1,
|
||||
kind: "daily",
|
||||
file: "healthlog-20261025T013000Z-daily.dump",
|
||||
createdAt: "2026-10-25T01:30:00.000Z",
|
||||
appVersion: "1.46.0",
|
||||
buildSha: "abc1234",
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
postgres: { serverVersion: "18.6", serverVersionNum: 180006 },
|
||||
dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
|
||||
});
|
||||
expect(Object.keys(manifest.rowCounts)).toEqual([
|
||||
"pgboss.job",
|
||||
"public.a",
|
||||
"public.users",
|
||||
]);
|
||||
expect(manifest.keys.encryption.keys[0].id).toBe("v1");
|
||||
});
|
||||
|
||||
it("reads back, and refuses anything else", () => {
|
||||
expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual(
|
||||
manifest,
|
||||
);
|
||||
expect(() => parseManifest({ format: "something-else" })).toThrow(
|
||||
/not a HealthLog/,
|
||||
);
|
||||
expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow();
|
||||
expect(() => parseManifest(null)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("row counts after a restore", () => {
|
||||
it("names every table that is missing, extra or different", () => {
|
||||
expect(
|
||||
compareRowCounts(
|
||||
{ "public.a": 1, "public.b": 2 },
|
||||
{ "public.a": 1, "public.b": 2 },
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
compareRowCounts(
|
||||
{ "public.a": 1, "public.b": 2, "public.c": 5 },
|
||||
{ "public.a": 1, "public.b": 0, "public.d": 1 },
|
||||
),
|
||||
).toEqual([
|
||||
"public.b: 2 rows in the dump, 0 restored",
|
||||
"public.c: in the dump with 5 rows, missing now",
|
||||
"public.d: restored but not in the dump's manifest",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the connection the client tools get", () => {
|
||||
it("drops Prisma's parameters and keeps the password out of the arguments", () => {
|
||||
const c = connectionFromUrl(
|
||||
"postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require",
|
||||
);
|
||||
expect(c.pgEnv).toEqual({
|
||||
PGHOST: "db",
|
||||
PGPORT: "6543",
|
||||
PGUSER: "health@log",
|
||||
PGPASSWORD: "p@ss/w",
|
||||
PGDATABASE: "hl",
|
||||
PGSSLMODE: "require",
|
||||
});
|
||||
expect(c.schema).toBe("app");
|
||||
const cs = String(c.clientConfig.connectionString);
|
||||
expect(cs).not.toContain("connection_limit");
|
||||
expect(cs).not.toContain("schema=");
|
||||
expect(cs).toContain("sslmode=require");
|
||||
});
|
||||
});
|
||||
|
||||
describe("an off-host copy read back by the restore", () => {
|
||||
/** The uploader the job uses, into memory. */
|
||||
async function upload(objectKey: string, key: Buffer, body: Buffer) {
|
||||
const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup");
|
||||
const stored = new Map<string, Buffer>();
|
||||
const s3: S3Like = {
|
||||
putStream: async (k: string, stream: Readable) => {
|
||||
stored.set(k, await collect(stream));
|
||||
},
|
||||
putObject: async () => {},
|
||||
getObject: async () => Buffer.alloc(0),
|
||||
headObject: async () => false,
|
||||
listObjects: async () => [],
|
||||
deleteObject: async () => {},
|
||||
};
|
||||
await uploadEncryptedBackup(s3, objectKey, key, async (write) => {
|
||||
// Several pieces, as the job streams the file.
|
||||
for (let i = 0; i < body.length; i += 1000) {
|
||||
await write(body.subarray(i, i + 1000));
|
||||
}
|
||||
});
|
||||
return stored.get(objectKey) as Buffer;
|
||||
}
|
||||
|
||||
it("opens what the job's uploader sealed, byte for byte", async () => {
|
||||
const dir = tempDir();
|
||||
const key = Buffer.from(HEX_D, "hex");
|
||||
const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251));
|
||||
const name = "healthlog-20261025T013000Z-daily.dump.enc";
|
||||
const sealed = await upload(offhostObjectKey(name), key, body);
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
|
||||
await decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: join(dir, "opened.dump"),
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [Buffer.from(HEX_A, "hex"), key],
|
||||
});
|
||||
expect(
|
||||
(await collect(createReadStream(join(dir, "opened.dump")))).equals(body),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => {
|
||||
const dir = tempDir();
|
||||
const key = Buffer.from(HEX_D, "hex");
|
||||
const body = Buffer.from("PGDMP fake archive bytes".repeat(100));
|
||||
const name = "healthlog-20261025T013000Z-daily.dump.enc";
|
||||
const sealed = await upload(offhostObjectKey(name), key, body);
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
const out = join(dir, "opened.dump");
|
||||
|
||||
// Renamed: the object key is bound as associated data.
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(
|
||||
"healthlog-20261024T013000Z-daily.dump.enc",
|
||||
),
|
||||
keys: [key],
|
||||
}),
|
||||
).rejects.toThrow(/does not open/);
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
|
||||
// Altered.
|
||||
const altered = Buffer.from(sealed);
|
||||
altered[altered.length - 40] ^= 1;
|
||||
writeFileSync(join(dir, name), altered);
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [key],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
|
||||
// A key it was not written under.
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [Buffer.from(HEX_A, "hex")],
|
||||
}),
|
||||
).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/);
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("finding a dump in the bucket", () => {
|
||||
const keys = [
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261023T013000Z-daily.manifest.json.enc",
|
||||
"database/healthlog-20261024T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
|
||||
// The newest dump whose manifest never arrived, and things that are
|
||||
// not dumps at all.
|
||||
"database/healthlog-20261025T013000Z-daily.dump.enc",
|
||||
"2026-10-25/user-abc.json.enc",
|
||||
"database/notes.txt",
|
||||
];
|
||||
|
||||
it("takes the newest dump whose manifest is there too", () => {
|
||||
expect(pickBucketDump(keys)).toEqual({
|
||||
dumpKey: "database/healthlog-20261024T013000Z-daily.dump.enc",
|
||||
manifestKey:
|
||||
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
|
||||
});
|
||||
});
|
||||
|
||||
it("finds a named one with or without its suffix, and nothing else", () => {
|
||||
for (const name of [
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
"healthlog-20261023T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
]) {
|
||||
expect(pickBucketDump(keys, name)?.dumpKey).toBe(
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
);
|
||||
}
|
||||
expect(
|
||||
pickBucketDump(keys, "healthlog-20261025T013000Z-daily.dump"),
|
||||
).toBeNull();
|
||||
expect(pickBucketDump([])).toBeNull();
|
||||
});
|
||||
|
||||
it("reads the bucket from the four required variables", () => {
|
||||
const env = {
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.test",
|
||||
BACKUP_S3_BUCKET: "b",
|
||||
BACKUP_S3_ACCESS_KEY: "a",
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
};
|
||||
expect(bucketConfigFromEnv(env)).toEqual({
|
||||
endpoint: "https://s3.example.test",
|
||||
bucket: "b",
|
||||
accessKey: "a",
|
||||
secretKey: "s",
|
||||
region: "auto",
|
||||
});
|
||||
expect(
|
||||
bucketConfigFromEnv({ ...env, BACKUP_S3_SECRET_KEY: "" }),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* Whether the nightly whole-database dump is happening, as facts a page or an
|
||||
* alert can act on.
|
||||
*
|
||||
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
|
||||
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
|
||||
* off-host bucket is configured. A ledger row that says a copy went off-host
|
||||
* outranks this process's environment, for the reason the off-host card gives
|
||||
* (a split install can hand the bucket variables to the worker only).
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
import {
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
export interface DatabaseBackupHealth {
|
||||
/** `DB_BACKUP_ENABLED` is not off. */
|
||||
enabled: boolean;
|
||||
/** The directory dumps are kept in (`DB_BACKUP_DIR`). */
|
||||
directory: string;
|
||||
/**
|
||||
* Nightly dumps kept in that directory (`DB_BACKUP_KEEP`). Also how long a
|
||||
* deleted account stays in the dumps on this server, which the backups
|
||||
* page says.
|
||||
*/
|
||||
keepNightly: number;
|
||||
/** This process can write there. False means dumps only go off-host. */
|
||||
localDirWritable: boolean;
|
||||
/** The off-host bucket is configured, or a dump has been uploaded to it. */
|
||||
offhostConfigured: boolean;
|
||||
/** The newest dump that was kept on the server or reached the bucket. */
|
||||
lastGood: {
|
||||
at: string;
|
||||
fileName: string;
|
||||
sizeBytes: number | null;
|
||||
local: boolean;
|
||||
offhost: boolean;
|
||||
} | null;
|
||||
/** The newest run that failed, or that a dying process left unfinished. */
|
||||
lastFailure: { at: string; error: string } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A run still `running` this long after it started was cut off by its
|
||||
* process: the job's expiry, past which pg-boss has failed it anyway.
|
||||
*/
|
||||
const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
|
||||
|
||||
export async function getDatabaseBackupHealth(
|
||||
now: Date = new Date(),
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<DatabaseBackupHealth> {
|
||||
const directory = resolveBackupDir(env);
|
||||
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
|
||||
[
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
fileName: { not: null },
|
||||
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
|
||||
},
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: {
|
||||
startedAt: true,
|
||||
finishedAt: true,
|
||||
fileName: true,
|
||||
dumpBytes: true,
|
||||
localResult: true,
|
||||
offhostResult: true,
|
||||
},
|
||||
}),
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { status: "failed" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true, error: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
status: "running",
|
||||
startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) },
|
||||
},
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
|
||||
],
|
||||
);
|
||||
|
||||
const failures = [
|
||||
lastFailed && {
|
||||
at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(),
|
||||
error: lastFailed.error ?? "failed without a reason",
|
||||
},
|
||||
lastAbandoned && {
|
||||
at: lastAbandoned.startedAt.toISOString(),
|
||||
error: "the run did not finish (the process stopped under it)",
|
||||
},
|
||||
]
|
||||
.filter((f): f is { at: string; error: string } => Boolean(f))
|
||||
.sort((a, b) => b.at.localeCompare(a.at));
|
||||
|
||||
return {
|
||||
enabled: databaseBackupEnabled(env),
|
||||
directory,
|
||||
keepNightly: resolveDailyKeep(env),
|
||||
localDirWritable: isWritableDirectory(directory),
|
||||
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
|
||||
lastGood: lastGood
|
||||
? {
|
||||
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
|
||||
fileName: lastGood.fileName ?? "",
|
||||
sizeBytes:
|
||||
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
|
||||
local: lastGood.localResult === "written",
|
||||
offhost: lastGood.offhostResult === "uploaded",
|
||||
}
|
||||
: null,
|
||||
lastFailure: failures[0] ?? null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* One run of the nightly whole-database dump.
|
||||
*
|
||||
* The dump is the artifact that rebuilds a lost server: every account, every
|
||||
* login, every grant and managed profile, the job queue and the instance
|
||||
* settings, in one `pg_dump --format=custom` file with a manifest beside it
|
||||
* (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the
|
||||
* bundled compose file) and, when the off-host bucket is configured, uploaded
|
||||
* under `database/` in the same envelope the per-account copies use, sealed
|
||||
* with `BACKUP_ENCRYPTION_KEY`.
|
||||
*
|
||||
* Without a writable directory (an install that runs the image without the
|
||||
* volume) the dump is written to a temporary directory, uploaded and removed:
|
||||
* the off-host copy still happens, nothing is kept on the server, and the
|
||||
* ledger and the boot summary say so. With neither, the run is skipped and
|
||||
* recorded as such.
|
||||
*
|
||||
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
|
||||
* what happened on each leg. `getDatabaseBackupHealth` reads them.
|
||||
*/
|
||||
import { createReadStream } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join } from "node:path";
|
||||
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import {
|
||||
getS3Client,
|
||||
loadOffhostConfig,
|
||||
uploadEncryptedBackup,
|
||||
type S3Like,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import packageJson from "../../../package.json";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
createDatabaseDump,
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
offhostObjectKey,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
|
||||
const LEDGER_RETENTION_DAYS = 400;
|
||||
|
||||
export type DatabaseBackupLocalResult =
|
||||
"written" | "no_directory" | "failed" | "skipped";
|
||||
export type DatabaseBackupOffhostResult =
|
||||
"uploaded" | "not_configured" | "failed" | "skipped";
|
||||
|
||||
export interface DatabaseBackupReport {
|
||||
status: "succeeded" | "failed" | "skipped";
|
||||
fileName: string | null;
|
||||
dumpBytes: number | null;
|
||||
local: DatabaseBackupLocalResult;
|
||||
offhost: DatabaseBackupOffhostResult;
|
||||
/** Dumps and manifests retention removed. */
|
||||
removed: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface RunDatabaseBackupOptions {
|
||||
now?: Date;
|
||||
/** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */
|
||||
deadline?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** A bucket double for tests; the configured bucket otherwise. */
|
||||
s3?: S3Like;
|
||||
}
|
||||
|
||||
/** The running build's version, as `/api/version` answers it. */
|
||||
function appVersion(env: NodeJS.ProcessEnv): string {
|
||||
return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version;
|
||||
}
|
||||
|
||||
/** An error as one line for the ledger and the admin page. */
|
||||
function reason(err: unknown): string {
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
return text.replace(/\s+/g, " ").trim().slice(0, 500);
|
||||
}
|
||||
|
||||
/** Upload one local file as an off-host object, streamed. */
|
||||
async function uploadFile(
|
||||
s3: S3Like,
|
||||
key: Buffer,
|
||||
path: string,
|
||||
): Promise<number> {
|
||||
return uploadEncryptedBackup(
|
||||
s3,
|
||||
offhostObjectKey(`${basename(path)}.enc`),
|
||||
key,
|
||||
async (write) => {
|
||||
for await (const chunk of createReadStream(path)) {
|
||||
await write(chunk as Buffer);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take tonight's dump, keep it, copy it off-host, prune the old ones, and
|
||||
* record what happened. Never throws: a failure is the report's `status`.
|
||||
*/
|
||||
export async function runDatabaseBackup(
|
||||
prisma: PrismaClient,
|
||||
options: RunDatabaseBackupOptions = {},
|
||||
): Promise<DatabaseBackupReport> {
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? new Date();
|
||||
const dir = resolveBackupDir(env);
|
||||
const localWritable = isWritableDirectory(dir);
|
||||
const offhostConfig = (() => {
|
||||
try {
|
||||
return loadOffhostConfig();
|
||||
} catch {
|
||||
// A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the
|
||||
// same value with their own message; here it reads as an upload error.
|
||||
return "invalid" as const;
|
||||
}
|
||||
})();
|
||||
|
||||
// Each leg reads "failed" until it has succeeded, so a run the process died
|
||||
// under says so once its expiry has passed.
|
||||
const report: DatabaseBackupReport = {
|
||||
status: "failed",
|
||||
fileName: null,
|
||||
dumpBytes: null,
|
||||
local: localWritable ? "failed" : "no_directory",
|
||||
offhost: offhostConfig ? "failed" : "not_configured",
|
||||
removed: 0,
|
||||
error: null,
|
||||
};
|
||||
const run = await prisma.databaseBackupRun.create({
|
||||
data: {
|
||||
startedAt: now,
|
||||
status: "running",
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
let scratch: string | null = null;
|
||||
try {
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
report.status = "skipped";
|
||||
report.local = "skipped";
|
||||
report.offhost = "skipped";
|
||||
report.error = "DB_BACKUP_ENABLED is false";
|
||||
} else if (!localWritable && !offhostConfig) {
|
||||
report.status = "skipped";
|
||||
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
|
||||
} else {
|
||||
const target = localWritable
|
||||
? dir
|
||||
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
|
||||
const dump = await createDatabaseDump({
|
||||
databaseUrl: env.DATABASE_URL ?? "",
|
||||
dir: target,
|
||||
kind: "daily",
|
||||
now,
|
||||
env,
|
||||
appVersion: appVersion(env),
|
||||
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
|
||||
deadline: options.deadline,
|
||||
});
|
||||
if (localWritable) report.local = "written";
|
||||
report.fileName = basename(dump.dumpPath);
|
||||
report.dumpBytes = dump.manifest.dump.bytes;
|
||||
|
||||
let uploadError: string | null = null;
|
||||
if (offhostConfig === "invalid") {
|
||||
uploadError =
|
||||
"BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64";
|
||||
} else if (offhostConfig) {
|
||||
try {
|
||||
const s3 = options.s3 ?? (await getS3Client(offhostConfig));
|
||||
// The manifest goes up first: a dump object in the bucket always
|
||||
// has the manifest a restore checks it against.
|
||||
await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath);
|
||||
await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath);
|
||||
report.offhost = "uploaded";
|
||||
} catch (err) {
|
||||
uploadError = `off-host upload failed: ${reason(err)}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (localWritable) {
|
||||
const removed = await applyRetention(dir, {
|
||||
dailyKeep: resolveDailyKeep(env),
|
||||
now: now.getTime(),
|
||||
});
|
||||
report.removed = removed.length;
|
||||
}
|
||||
|
||||
// Good when the dump exists somewhere it survives: on the volume, or in
|
||||
// the bucket. An upload that failed beside a kept dump still fails the
|
||||
// run, because the operator asked for both.
|
||||
report.status = uploadError ? "failed" : "succeeded";
|
||||
report.error = uploadError;
|
||||
}
|
||||
} catch (err) {
|
||||
report.status = "failed";
|
||||
report.error = reason(err);
|
||||
} finally {
|
||||
if (scratch) await rm(scratch, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
await prisma.databaseBackupRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
finishedAt: new Date(),
|
||||
status: report.status,
|
||||
fileName: report.fileName,
|
||||
dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes),
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
error: report.error,
|
||||
},
|
||||
});
|
||||
await prisma.databaseBackupRun.deleteMany({
|
||||
where: {
|
||||
startedAt: {
|
||||
lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "database.backup.run" },
|
||||
meta: {
|
||||
status: report.status,
|
||||
local: report.local,
|
||||
offhost: report.offhost,
|
||||
dump_bytes: report.dumpBytes ?? 0,
|
||||
removed: report.removed,
|
||||
},
|
||||
});
|
||||
return report;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Which lifecycle rules retire the database dumps under `database/`, apart
|
||||
* from the account copies: a rule scoped to another prefix, or to a tag, does
|
||||
* not touch them.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { LifecycleRule } from "@aws-sdk/client-s3";
|
||||
|
||||
import { lifecycleFromRules, ruleAppliesTo } from "../offhost-lifecycle";
|
||||
|
||||
const DUMP = "database/healthlog-20261025T013000Z-daily.dump.enc";
|
||||
|
||||
function rule(days: number, extra: Partial<LifecycleRule> = {}): LifecycleRule {
|
||||
return { Status: "Enabled", Expiration: { Days: days }, ...extra };
|
||||
}
|
||||
|
||||
describe("lifecycle rules and the database dumps", () => {
|
||||
it("counts a rule for the whole bucket, or for database/, for both", () => {
|
||||
expect(lifecycleFromRules([rule(30)])).toEqual({
|
||||
state: "configured",
|
||||
expirationDays: 30,
|
||||
database: { state: "configured", expirationDays: 30 },
|
||||
});
|
||||
expect(
|
||||
lifecycleFromRules([
|
||||
rule(30, { Filter: { Prefix: "2026-" } }),
|
||||
rule(14, { Filter: { Prefix: "database/" } }),
|
||||
]).database,
|
||||
).toEqual({ state: "configured", expirationDays: 14 });
|
||||
});
|
||||
|
||||
it("says database/ is not covered when the only rule is scoped elsewhere", () => {
|
||||
const result = lifecycleFromRules([
|
||||
rule(30, { Filter: { Prefix: "2026-" } }),
|
||||
]);
|
||||
expect(result.state).toBe("configured");
|
||||
expect(result.database).toEqual({ state: "missing", expirationDays: null });
|
||||
});
|
||||
|
||||
it("reads the legacy prefix, the And filter, tags and disabled rules", () => {
|
||||
expect(ruleAppliesTo(rule(5, { Prefix: "database/" }), DUMP)).toBe(true);
|
||||
expect(ruleAppliesTo(rule(5, { Prefix: "other/" }), DUMP)).toBe(false);
|
||||
expect(
|
||||
ruleAppliesTo(
|
||||
rule(5, {
|
||||
Filter: { And: { Prefix: "database/", ObjectSizeGreaterThan: 1 } },
|
||||
}),
|
||||
DUMP,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
ruleAppliesTo(
|
||||
rule(5, { Filter: { Tag: { Key: "k", Value: "v" } } }),
|
||||
DUMP,
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
lifecycleFromRules([rule(7, { Status: "Disabled" })]).database.state,
|
||||
).toBe("missing");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* The queue facts of the nightly whole-database dump: its name, its schedule
|
||||
* and its send options. Apart from the job so the worker registry can read
|
||||
* them without loading it (`job-registry-graph.test.ts`).
|
||||
*/
|
||||
import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy";
|
||||
import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export const DATABASE_BACKUP_QUEUE = "database-backup";
|
||||
|
||||
/**
|
||||
* `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the
|
||||
* zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30,
|
||||
* an hour before the off-host account copies so the two uploads do not meet.
|
||||
* A value that is not a five-field cron keeps the default, and the boot
|
||||
* readiness summary says so.
|
||||
*/
|
||||
export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron;
|
||||
|
||||
/**
|
||||
* Four hours. `pg_dump` of a large instance takes minutes, the upload of the
|
||||
* dump about as long as the dump; the window is there to be impossible to hit
|
||||
* for a working run. The handler kills `pg_dump` at three quarters of it
|
||||
* (`jobDeadline`), so a hung dump ends as a failed run with a reason rather
|
||||
* than an expired job.
|
||||
*/
|
||||
export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60;
|
||||
|
||||
/**
|
||||
* What the nightly schedule sends with: the expiry, the backups' heartbeat,
|
||||
* and one retry. A dump that failed for a passing reason (a lock it waited
|
||||
* out, a bucket that answered 503) gets a second attempt; one that fails
|
||||
* again waits for the next night, and the failure stays on the admin page.
|
||||
*/
|
||||
export const DATABASE_BACKUP_SEND_OPTIONS = {
|
||||
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
|
||||
heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS,
|
||||
retryLimit: 1,
|
||||
retryDelay: 15 * 60,
|
||||
} as const;
|
||||
@@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"continued",
|
||||
"conversation_titles_migrated",
|
||||
"created",
|
||||
// The nightly whole-database dump: where it went and how big.
|
||||
"database_backup_bytes",
|
||||
"database_backup_local",
|
||||
"database_backup_offhost",
|
||||
"database_backup_removed",
|
||||
"database_backup_status",
|
||||
"daily_rows_retired",
|
||||
"days_consolidated",
|
||||
"days_rebuilt",
|
||||
|
||||
@@ -44,6 +44,12 @@ import {
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { envValue } from "@/lib/env";
|
||||
import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue";
|
||||
import {
|
||||
LIFECYCLE_MISSING,
|
||||
LIFECYCLE_UNKNOWN,
|
||||
lifecycleFromRules,
|
||||
type OffhostLifecycle,
|
||||
} from "./offhost-lifecycle";
|
||||
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
const IV_LENGTH = 12;
|
||||
@@ -494,9 +500,9 @@ function createEnvelopeStream(
|
||||
return { stream, bytes: () => written };
|
||||
}
|
||||
|
||||
/** Produces the backup JSON in pieces. Every piece is written in order. */
|
||||
/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */
|
||||
export type BackupJsonProducer = (
|
||||
write: (chunk: string) => Promise<void>,
|
||||
write: (chunk: string | Buffer) => Promise<void>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export interface UploadBackupOptions {
|
||||
@@ -572,9 +578,9 @@ export async function uploadEncryptedBackup(
|
||||
},
|
||||
);
|
||||
|
||||
const write = async (chunk: string): Promise<void> => {
|
||||
const write = async (chunk: string | Buffer): Promise<void> => {
|
||||
if (failure) throw failure;
|
||||
if (gzip.write(chunk, "utf8")) return;
|
||||
if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = (): void => {
|
||||
gzip.off("error", onError);
|
||||
@@ -634,18 +640,7 @@ export interface S3Like {
|
||||
getLifecycle?(): Promise<OffhostLifecycle>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the bucket retires old copies by itself.
|
||||
*
|
||||
* `configured` names the shortest expiry of an enabled rule; `missing` means
|
||||
* the bucket answered that it has no rule; `unknown` means it would not say
|
||||
* (a credential without `s3:GetLifecycleConfiguration`, a target that does not
|
||||
* implement the call, a network failure) and says nothing either way.
|
||||
*/
|
||||
export interface OffhostLifecycle {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
}
|
||||
export type { OffhostLifecycle } from "./offhost-lifecycle";
|
||||
|
||||
export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
// Dynamic import so unit tests + dev environments without the SDK don't fail.
|
||||
@@ -765,13 +760,7 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(5_000) },
|
||||
);
|
||||
const days = (out.Rules ?? [])
|
||||
.filter((rule) => rule.Status === "Enabled")
|
||||
.map((rule) => rule.Expiration?.Days)
|
||||
.filter((d): d is number => typeof d === "number" && d > 0);
|
||||
return days.length > 0
|
||||
? { state: "configured", expirationDays: Math.min(...days) }
|
||||
: { state: "missing", expirationDays: null };
|
||||
return lifecycleFromRules(out.Rules ?? []);
|
||||
} catch (err) {
|
||||
const name = (err as { name?: string; Code?: string }).name;
|
||||
const code = (err as { Code?: string }).Code;
|
||||
@@ -779,9 +768,9 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
name === "NoSuchLifecycleConfiguration" ||
|
||||
code === "NoSuchLifecycleConfiguration"
|
||||
) {
|
||||
return { state: "missing", expirationDays: null };
|
||||
return LIFECYCLE_MISSING;
|
||||
}
|
||||
return { state: "unknown", expirationDays: null };
|
||||
return LIFECYCLE_UNKNOWN;
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -806,15 +795,13 @@ export async function probeOffhostLifecycle(
|
||||
return lifecycleCache.value;
|
||||
}
|
||||
const cfg = loadOffhostConfigSafe();
|
||||
if (!cfg) return { state: "unknown", expirationDays: null };
|
||||
if (!cfg) return LIFECYCLE_UNKNOWN;
|
||||
let value: OffhostLifecycle;
|
||||
try {
|
||||
const s3 = s3Override ?? (await getS3Client(cfg));
|
||||
value = s3.getLifecycle
|
||||
? await s3.getLifecycle()
|
||||
: { state: "unknown", expirationDays: null };
|
||||
value = s3.getLifecycle ? await s3.getLifecycle() : LIFECYCLE_UNKNOWN;
|
||||
} catch {
|
||||
value = { state: "unknown", expirationDays: null };
|
||||
value = LIFECYCLE_UNKNOWN;
|
||||
}
|
||||
if (!s3Override) lifecycleCache = { at: now, value };
|
||||
return value;
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Whether the operator's bucket retires old copies by itself, read from its
|
||||
* lifecycle rules (`GetBucketLifecycleConfiguration`).
|
||||
*
|
||||
* Two answers, because the bucket holds two kinds of object with different
|
||||
* keys: the per-account copies (`<date>/user-<id>.json.enc`) and the database
|
||||
* dumps (`database/…`). A rule scoped to a prefix covers only the keys under
|
||||
* it, so a bucket whose one rule expires `2026-` keys ages the account copies
|
||||
* out and keeps every database dump forever, and a database dump keeps every
|
||||
* deleted account's data for as long as it exists. The account answer keeps
|
||||
* its earlier reading (the shortest expiry of any enabled rule); the database
|
||||
* answer counts only the rules that apply to a key under `database/`.
|
||||
*
|
||||
* `configured` names the shortest expiry; `missing` means the bucket answered
|
||||
* that no rule applies; `unknown` means it would not say (a credential
|
||||
* without `s3:GetLifecycleConfiguration`, a target that does not implement the
|
||||
* call, a network failure) and says nothing either way.
|
||||
*/
|
||||
import type { LifecycleRule } from "@aws-sdk/client-s3";
|
||||
|
||||
export interface LifecycleVerdict {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
}
|
||||
|
||||
export interface OffhostLifecycle extends LifecycleVerdict {
|
||||
/** The same question for the database dumps under `database/`. */
|
||||
database: LifecycleVerdict;
|
||||
}
|
||||
|
||||
export const LIFECYCLE_UNKNOWN: OffhostLifecycle = {
|
||||
state: "unknown",
|
||||
expirationDays: null,
|
||||
database: { state: "unknown", expirationDays: null },
|
||||
};
|
||||
|
||||
export const LIFECYCLE_MISSING: OffhostLifecycle = {
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
};
|
||||
|
||||
/** A key a database dump is stored under, to test the rules against. */
|
||||
const DATABASE_SAMPLE_KEY =
|
||||
"database/healthlog-20260101T013000Z-daily.dump.enc";
|
||||
|
||||
/**
|
||||
* Whether a rule's filter takes in `key`. Our objects carry no tags, so a
|
||||
* rule that filters on a tag never applies to them; a size bound is not a
|
||||
* reason to call a rule inapplicable, so it is read as applying.
|
||||
*/
|
||||
export function ruleAppliesTo(rule: LifecycleRule, key: string): boolean {
|
||||
const filter = rule.Filter;
|
||||
if (filter?.Tag || filter?.And?.Tags?.length) return false;
|
||||
const prefix = filter?.Prefix ?? filter?.And?.Prefix ?? rule.Prefix ?? "";
|
||||
return key.startsWith(prefix);
|
||||
}
|
||||
|
||||
function verdict(rules: readonly LifecycleRule[]): LifecycleVerdict {
|
||||
const days = rules
|
||||
.filter((rule) => rule.Status === "Enabled")
|
||||
.map((rule) => rule.Expiration?.Days)
|
||||
.filter((d): d is number => typeof d === "number" && d > 0);
|
||||
return days.length > 0
|
||||
? { state: "configured", expirationDays: Math.min(...days) }
|
||||
: { state: "missing", expirationDays: null };
|
||||
}
|
||||
|
||||
/** Both answers from the rules a bucket returned. */
|
||||
export function lifecycleFromRules(
|
||||
rules: readonly LifecycleRule[],
|
||||
): OffhostLifecycle {
|
||||
return {
|
||||
...verdict(rules),
|
||||
database: verdict(
|
||||
rules.filter((rule) => ruleAppliesTo(rule, DATABASE_SAMPLE_KEY)),
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-
|
||||
import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill";
|
||||
import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run";
|
||||
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
|
||||
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
|
||||
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue";
|
||||
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
|
||||
import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue";
|
||||
@@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
|
||||
exclusive: "lockedPass",
|
||||
why: "Serialises every account's whole record.",
|
||||
},
|
||||
"database-backup": {
|
||||
runtime: "long",
|
||||
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
|
||||
expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"],
|
||||
stop: {
|
||||
file: "lib/jobs/reminder/backup-handlers.ts",
|
||||
fn: "handleDatabaseBackup",
|
||||
},
|
||||
exclusive: "lockedPass",
|
||||
why: "Runs pg_dump over the whole database and uploads the dump off-host.",
|
||||
},
|
||||
"data-backup-offhost": {
|
||||
runtime: "long",
|
||||
expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* The housekeeping queues: backups (weekly in-database, nightly off-host),
|
||||
* The housekeeping queues: backups (the nightly whole-database dump, weekly
|
||||
* in-database account copies, nightly off-host),
|
||||
* the restore drill and restores, every ledger and retention cleanup, the
|
||||
* medication maintenance passes, the imports (Apple Health, Health Connect,
|
||||
* medication intake), the converging encryption backfills, the document vault
|
||||
@@ -47,6 +48,11 @@ import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
import {
|
||||
DATABASE_BACKUP_CRON,
|
||||
DATABASE_BACKUP_QUEUE,
|
||||
DATABASE_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/database-backup-queue";
|
||||
import {
|
||||
DAWARICH_SYNC_CRON,
|
||||
DAWARICH_SYNC_QUEUE,
|
||||
@@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [
|
||||
lane: true,
|
||||
load: async () => (await backups()).handleDataBackup,
|
||||
}),
|
||||
// The whole-database dump: the copy that rebuilds a lost server. Locked, so
|
||||
// a retry that starts while a run is still dumping waits its turn.
|
||||
defineJob({
|
||||
queue: DATABASE_BACKUP_QUEUE,
|
||||
cron: DATABASE_BACKUP_CRON,
|
||||
scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS,
|
||||
work: { localConcurrency: 1 },
|
||||
lock: () => WHOLE_PASS,
|
||||
load: async () => (await backups()).handleDatabaseBackup,
|
||||
}),
|
||||
prune(
|
||||
RATE_LIMIT_CLEANUP_QUEUE,
|
||||
RATE_LIMIT_CLEANUP_CRON,
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* schedules, and boss.work registrations.
|
||||
*/
|
||||
import { type Job, type JobWithMetadata } from "pg-boss";
|
||||
import { jobBudget } from "@/lib/jobs/job-budget";
|
||||
import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import {
|
||||
BackupBlobTooLargeError,
|
||||
@@ -31,6 +31,37 @@ import {
|
||||
} from "@/lib/jobs/backup-pass-attempts";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
|
||||
/**
|
||||
* The nightly whole-database dump (`database-backup`).
|
||||
*
|
||||
* One `pg_dump` in a child process, so it is not in the heavy lane: the heap
|
||||
* holds a part of the upload at a time, not the database. `pg_dump` is stopped
|
||||
* at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs
|
||||
* on a lock ends as a failed run with its reason rather than an expired job.
|
||||
* A skipped run (switched off, nowhere to put it) completes; the admin page
|
||||
* and the boot summary carry why.
|
||||
*/
|
||||
export async function handleDatabaseBackup(
|
||||
jobs: Job<Record<string, never>>[],
|
||||
): Promise<JobOutcome> {
|
||||
return withBackgroundEvent("job.database_backup", async () => {
|
||||
const { runDatabaseBackup } = await import("@/lib/database-backup/run");
|
||||
const report = await runDatabaseBackup(getWorkerPrisma(), {
|
||||
deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined,
|
||||
});
|
||||
const did = {
|
||||
database_backup_status: report.status,
|
||||
database_backup_local: report.local,
|
||||
database_backup_offhost: report.offhost,
|
||||
database_backup_bytes: report.dumpBytes ?? 0,
|
||||
database_backup_removed: report.removed,
|
||||
};
|
||||
return report.status === "failed"
|
||||
? jobFailed("database dump failed", report.error ?? undefined, did)
|
||||
: jobDone(did);
|
||||
});
|
||||
}
|
||||
|
||||
export interface DataBackupPayload {
|
||||
triggeredAt: string;
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ const subsystemSurface = {
|
||||
"audit-log-cleanup": { audience: "system" },
|
||||
"step-up-elevation-cleanup": { audience: "system" },
|
||||
"data-backup-offhost": { audience: "system" },
|
||||
"database-backup": { audience: "system" },
|
||||
"data-restore-drill": { audience: "system" },
|
||||
"offhost-backup-purge": { audience: "system" },
|
||||
"backup-restore": { audience: "system" },
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
* both sides a route-handler-independent shared type home; the route
|
||||
* keeps owning the HTTP contract, the component keeps owning the UI.
|
||||
*/
|
||||
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import type { OffhostLifecycle } from "@/lib/jobs/offhost-lifecycle";
|
||||
import type { BackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
|
||||
@@ -76,10 +78,7 @@ export interface OffhostBackupOverview {
|
||||
* (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown`
|
||||
* when the credential may not read it.
|
||||
*/
|
||||
lifecycle: {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
};
|
||||
lifecycle: OffhostLifecycle;
|
||||
}
|
||||
|
||||
export interface BackupsList {
|
||||
@@ -101,6 +100,12 @@ export interface BackupsList {
|
||||
* the operator's bucket, and those two fail independently.
|
||||
*/
|
||||
offhost: OffhostBackupOverview;
|
||||
/**
|
||||
* The nightly whole-database dump, the copy that rebuilds a lost server:
|
||||
* the newest good dump (kept here, off-host, or both), the newest failed
|
||||
* run, and whether this server can keep dumps at all.
|
||||
*/
|
||||
database: DatabaseBackupHealth;
|
||||
/**
|
||||
* Soft retention hint — the worker is configured for weekly backups
|
||||
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"registeredQueueCount": 122,
|
||||
"registeredQueueCount": 123,
|
||||
"registeredQueues": [
|
||||
"achievement-unlock-sweep",
|
||||
"apple-health-import",
|
||||
@@ -23,6 +23,7 @@
|
||||
"data-backup",
|
||||
"data-backup-offhost",
|
||||
"data-restore-drill",
|
||||
"database-backup",
|
||||
"dawarich-sync",
|
||||
"dense-intraday-hourly-rebuild",
|
||||
"dense-intraday-retention",
|
||||
@@ -455,6 +456,21 @@
|
||||
"retry_delay_max": null,
|
||||
"retry_limit": 2
|
||||
},
|
||||
{
|
||||
"dead_letter": null,
|
||||
"deletion_seconds": 604800,
|
||||
"expire_seconds": 900,
|
||||
"heartbeat_seconds": null,
|
||||
"name": "database-backup",
|
||||
"notify": false,
|
||||
"partition": false,
|
||||
"policy": "standard",
|
||||
"retention_seconds": 1209600,
|
||||
"retry_backoff": false,
|
||||
"retry_delay": 0,
|
||||
"retry_delay_max": null,
|
||||
"retry_limit": 2
|
||||
},
|
||||
{
|
||||
"dead_letter": null,
|
||||
"deletion_seconds": 604800,
|
||||
@@ -2137,6 +2153,21 @@
|
||||
},
|
||||
"timezone": "Europe/Berlin"
|
||||
},
|
||||
{
|
||||
"cron": "30 1 * * *",
|
||||
"data": {},
|
||||
"key": "",
|
||||
"kind": "cron",
|
||||
"name": "database-backup",
|
||||
"options": {
|
||||
"expireInSeconds": 14400,
|
||||
"heartbeatSeconds": 300,
|
||||
"retryDelay": 900,
|
||||
"retryLimit": 1,
|
||||
"tz": "Europe/Berlin"
|
||||
},
|
||||
"timezone": "Europe/Berlin"
|
||||
},
|
||||
{
|
||||
"cron": "20 3 * * *",
|
||||
"data": {},
|
||||
@@ -3022,6 +3053,13 @@
|
||||
"pollingIntervalSeconds": 30
|
||||
}
|
||||
},
|
||||
{
|
||||
"queue": "database-backup",
|
||||
"options": {
|
||||
"localConcurrency": 1,
|
||||
"pollingIntervalSeconds": 30
|
||||
}
|
||||
},
|
||||
{
|
||||
"queue": "dawarich-sync",
|
||||
"options": {
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
/**
|
||||
* The nightly whole-database dump against a real Postgres: the job writes a
|
||||
* dump and its manifest, records the run, copies both off-host in the
|
||||
* envelope the restore opens, keeps only what retention allows; and the
|
||||
* restore puts the dump into an empty database and refuses one with tables.
|
||||
*
|
||||
* `pg_dump` and `pg_restore` are the test container's own. A workstation may
|
||||
* have none, or one of another major than the server (which `pg_dump`
|
||||
* refuses), so the test puts two small forwarders on PATH that run the tool
|
||||
* inside the database container with `docker exec`. Everything else, the
|
||||
* snapshot, the counts, the manifest, the ledger and the restore's checks,
|
||||
* runs exactly as in the image.
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import { runDatabaseBackup } from "@/lib/database-backup/run";
|
||||
import {
|
||||
connect,
|
||||
countRows,
|
||||
decryptOffhostFile,
|
||||
offhostObjectKey,
|
||||
readManifestFile,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
import { main as restoreMain } from "../../scripts/db-restore.mjs";
|
||||
import { shippedMigrations } from "../../scripts/pending-migrations.mjs";
|
||||
|
||||
const OFFHOST_KEY = "e".repeat(64);
|
||||
const work = mkdtempSync(join(tmpdir(), "hl-db-backup-"));
|
||||
const backups = join(work, "backups");
|
||||
const shims = join(work, "bin");
|
||||
const originalPath = process.env.PATH;
|
||||
|
||||
/** The id of the container that serves DATABASE_URL. */
|
||||
function containerId(): string {
|
||||
const port = new URL(process.env.DATABASE_URL ?? "").port;
|
||||
const rows = execFileSync(
|
||||
"docker",
|
||||
["ps", "--format", "{{.ID}} {{.Ports}}"],
|
||||
{
|
||||
encoding: "utf8",
|
||||
},
|
||||
).split("\n");
|
||||
const row = rows.find((r) => r.includes(`:${port}->5432/tcp`));
|
||||
if (!row) throw new Error(`no container publishes port ${port}`);
|
||||
return row.split(" ")[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* `pg_dump` / `pg_restore` that run inside the database container. The
|
||||
* connection comes from the PG* variables the core sets; host and port become
|
||||
* the container's own. A file argument that exists on this machine is sent on
|
||||
* standard input, which both tools read when no file is named.
|
||||
*/
|
||||
function writeShims(container: string): void {
|
||||
for (const tool of ["pg_dump", "pg_restore"]) {
|
||||
const path = join(shims, tool);
|
||||
writeFileSync(
|
||||
path,
|
||||
`#!/usr/bin/env node
|
||||
const { spawn } = require("node:child_process");
|
||||
const { createReadStream, existsSync } = require("node:fs");
|
||||
const args = process.argv.slice(2);
|
||||
let input = null;
|
||||
if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) {
|
||||
input = args.pop();
|
||||
}
|
||||
const env = [];
|
||||
for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) {
|
||||
if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]);
|
||||
}
|
||||
env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432");
|
||||
const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], {
|
||||
stdio: ["pipe", "inherit", "inherit"],
|
||||
});
|
||||
(input ? createReadStream(input) : process.stdin).pipe(child.stdin);
|
||||
child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0)));
|
||||
`,
|
||||
);
|
||||
chmodSync(path, 0o755);
|
||||
}
|
||||
}
|
||||
|
||||
/** A bucket in memory, with the one call the job's uploader uses. */
|
||||
function memoryBucket() {
|
||||
const objects = new Map<string, Buffer>();
|
||||
return {
|
||||
objects,
|
||||
putStream: async (key: string, body: Readable) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const c of body) chunks.push(c as Buffer);
|
||||
objects.set(key, Buffer.concat(chunks));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function backupEnv(extra: Record<string, string> = {}): NodeJS.ProcessEnv {
|
||||
return { ...process.env, DB_BACKUP_DIR: backups, ...extra };
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(shims, { recursive: true });
|
||||
mkdirSync(backups, { recursive: true });
|
||||
writeShims(containerId());
|
||||
process.env.PATH = `${shims}:${originalPath}`;
|
||||
for (const name of ["dump-a", "dump-b", "dump-c"]) {
|
||||
await prisma.user.create({
|
||||
data: { username: `${name}-${Date.now()}` },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
process.env.PATH = originalPath;
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("the nightly database dump", () => {
|
||||
let dumpFile = "";
|
||||
|
||||
it("writes a checked dump and manifest, records the run and copies both off-host", async () => {
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
|
||||
vi.stubEnv("BACKUP_S3_BUCKET", "b");
|
||||
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
|
||||
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
|
||||
const bucket = memoryBucket();
|
||||
const now = new Date("2026-10-20T01:30:00Z");
|
||||
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now,
|
||||
env: backupEnv(),
|
||||
s3: bucket as never,
|
||||
});
|
||||
expect(report).toMatchObject({
|
||||
status: "succeeded",
|
||||
local: "written",
|
||||
offhost: "uploaded",
|
||||
error: null,
|
||||
fileName: "healthlog-20261020T013000Z-daily.dump",
|
||||
});
|
||||
dumpFile = join(backups, report.fileName ?? "");
|
||||
|
||||
// Both files, under their final names, and nothing half-written.
|
||||
expect(readdirSync(backups).sort()).toEqual([
|
||||
"healthlog-20261020T013000Z-daily.dump",
|
||||
"healthlog-20261020T013000Z-daily.manifest.json",
|
||||
]);
|
||||
expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe(
|
||||
"PGDMP",
|
||||
);
|
||||
|
||||
const manifest = await readManifestFile(
|
||||
join(backups, "healthlog-20261020T013000Z-daily.manifest.json"),
|
||||
);
|
||||
expect(manifest.kind).toBe("daily");
|
||||
expect(manifest.dump.bytes).toBe(report.dumpBytes);
|
||||
expect(manifest.latestMigration).toBe(
|
||||
shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1),
|
||||
);
|
||||
expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000);
|
||||
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
|
||||
expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100);
|
||||
expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0);
|
||||
expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/);
|
||||
|
||||
// The ledger row and what the admin page reads from it.
|
||||
const runs = await prisma.databaseBackupRun.findMany({
|
||||
where: { startedAt: now },
|
||||
});
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]).toMatchObject({
|
||||
status: "succeeded",
|
||||
fileName: "healthlog-20261020T013000Z-daily.dump",
|
||||
localResult: "written",
|
||||
offhostResult: "uploaded",
|
||||
error: null,
|
||||
});
|
||||
expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes);
|
||||
const health = await getDatabaseBackupHealth(
|
||||
new Date("2026-10-20T02:00:00Z"),
|
||||
backupEnv(),
|
||||
);
|
||||
expect(health).toMatchObject({
|
||||
enabled: true,
|
||||
localDirWritable: true,
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
fileName: "healthlog-20261020T013000Z-daily.dump",
|
||||
sizeBytes: manifest.dump.bytes,
|
||||
local: true,
|
||||
offhost: true,
|
||||
},
|
||||
});
|
||||
|
||||
// The off-host pair opens, with the restore's own reader, to the very
|
||||
// bytes on disk.
|
||||
expect([...bucket.objects.keys()].sort()).toEqual([
|
||||
"database/healthlog-20261020T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261020T013000Z-daily.manifest.json.enc",
|
||||
]);
|
||||
const opened = join(work, "opened");
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(opened);
|
||||
for (const name of [
|
||||
"healthlog-20261020T013000Z-daily.dump",
|
||||
"healthlog-20261020T013000Z-daily.manifest.json",
|
||||
]) {
|
||||
writeFileSync(
|
||||
join(opened, `${name}.enc`),
|
||||
bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer,
|
||||
);
|
||||
await decryptOffhostFile({
|
||||
inPath: join(opened, `${name}.enc`),
|
||||
outPath: join(opened, name),
|
||||
objectKey: offhostObjectKey(`${name}.enc`),
|
||||
keys: [Buffer.from(OFFHOST_KEY, "hex")],
|
||||
});
|
||||
expect(
|
||||
readFileSync(join(opened, name)).equals(
|
||||
readFileSync(join(backups, name)),
|
||||
),
|
||||
).toBe(true);
|
||||
}
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => {
|
||||
for (const day of ["21", "22", "23"]) {
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now: new Date(`2026-10-${day}T01:30:00Z`),
|
||||
env: backupEnv({ DB_BACKUP_KEEP: "2" }),
|
||||
});
|
||||
expect(report.status).toBe("succeeded");
|
||||
expect(report.offhost).toBe("not_configured");
|
||||
}
|
||||
expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([
|
||||
"healthlog-20261022T013000Z-daily.dump",
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
]);
|
||||
});
|
||||
|
||||
it("records a failed dump, leaves nothing that looks good, and the page says so", async () => {
|
||||
const before = readdirSync(backups).sort();
|
||||
const url = new URL(process.env.DATABASE_URL ?? "");
|
||||
url.password = "not-the-password";
|
||||
const now = new Date("2026-10-24T01:30:00Z");
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now,
|
||||
env: backupEnv({ DATABASE_URL: url.toString() }),
|
||||
});
|
||||
expect(report.status).toBe("failed");
|
||||
expect(report.local).toBe("failed");
|
||||
expect(report.error).toMatch(/password authentication failed/);
|
||||
expect(readdirSync(backups).sort()).toEqual(before);
|
||||
const health = await getDatabaseBackupHealth(
|
||||
new Date("2026-10-24T02:00:00Z"),
|
||||
backupEnv(),
|
||||
);
|
||||
expect(health.lastFailure?.error).toMatch(/password authentication/);
|
||||
expect(health.lastGood?.fileName).toBe(
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
);
|
||||
});
|
||||
|
||||
it("skips, and says why, with nowhere to put a dump", async () => {
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now: new Date("2026-10-24T03:00:00Z"),
|
||||
env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }),
|
||||
});
|
||||
expect(report).toMatchObject({
|
||||
status: "skipped",
|
||||
local: "no_directory",
|
||||
offhost: "not_configured",
|
||||
});
|
||||
expect(report.error).toMatch(/no writable backup directory/);
|
||||
const off = await runDatabaseBackup(prisma, {
|
||||
now: new Date("2026-10-24T04:00:00Z"),
|
||||
env: backupEnv({ DB_BACKUP_ENABLED: "false" }),
|
||||
});
|
||||
expect(off).toMatchObject({
|
||||
status: "skipped",
|
||||
local: "skipped",
|
||||
offhost: "skipped",
|
||||
});
|
||||
});
|
||||
|
||||
describe("restoring it", () => {
|
||||
const target = `restore_${process.pid}_${Date.now()}`;
|
||||
let targetUrl = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
const admin = await connect(process.env.DATABASE_URL ?? "");
|
||||
await admin.query(`CREATE DATABASE ${target}`);
|
||||
await admin.end();
|
||||
const url = new URL(process.env.DATABASE_URL ?? "");
|
||||
url.pathname = `/${target}`;
|
||||
targetUrl = url.toString();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const admin = await connect(process.env.DATABASE_URL ?? "");
|
||||
await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`);
|
||||
await admin.end();
|
||||
});
|
||||
|
||||
it("refuses the database the dump came from, which has tables", async () => {
|
||||
const errors: string[] = [];
|
||||
const spy = vi
|
||||
.spyOn(console, "error")
|
||||
.mockImplementation((l: string) => void errors.push(String(l)));
|
||||
const code = await restoreMain(["latest"], backupEnv());
|
||||
spy.mockRestore();
|
||||
expect(code).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/only into an empty database/);
|
||||
});
|
||||
|
||||
it("restores the newest dump into an empty database with every row count", async () => {
|
||||
const lines: string[] = [];
|
||||
const spy = vi
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation((l: string) => void lines.push(String(l)));
|
||||
const code = await restoreMain(
|
||||
["latest"],
|
||||
backupEnv({ DATABASE_URL: targetUrl }),
|
||||
);
|
||||
spy.mockRestore();
|
||||
expect(code, lines.join("\n")).toBe(0);
|
||||
expect(lines.join("\n")).toMatch(
|
||||
/all \d+ tables hold the row counts the manifest names/,
|
||||
);
|
||||
|
||||
const manifest = await readManifestFile(
|
||||
join(backups, "healthlog-20261023T013000Z-daily.manifest.json"),
|
||||
);
|
||||
const client = await connect(targetUrl);
|
||||
try {
|
||||
expect(await countRows(client)).toEqual(manifest.rowCounts);
|
||||
const { rows } = await client.query(
|
||||
"SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'",
|
||||
);
|
||||
expect(rows[0].n).toBeGreaterThanOrEqual(3);
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
// Restoring again into the now full database is refused.
|
||||
const again = await restoreMain(
|
||||
["latest"],
|
||||
backupEnv({ DATABASE_URL: targetUrl }),
|
||||
);
|
||||
expect(again).toBe(1);
|
||||
expect(existsSync(dumpFile)).toBe(false); // the first dump aged out
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the dump before migrating", () => {
|
||||
const preDir = join(work, "pre");
|
||||
const appDir = join(work, "app");
|
||||
const failing = join(work, "failing-bin");
|
||||
let cwd = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
const { cpSync, mkdirSync } = await import("node:fs");
|
||||
// An image that ships one migration the database has not applied.
|
||||
mkdirSync(join(appDir, "prisma"), { recursive: true });
|
||||
cpSync(
|
||||
join(process.cwd(), "prisma", "migrations"),
|
||||
join(appDir, "prisma", "migrations"),
|
||||
{ recursive: true },
|
||||
);
|
||||
mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet"));
|
||||
writeFileSync(
|
||||
join(
|
||||
appDir,
|
||||
"prisma",
|
||||
"migrations",
|
||||
"9999_not_applied_yet",
|
||||
"migration.sql",
|
||||
),
|
||||
"",
|
||||
);
|
||||
mkdirSync(preDir, { recursive: true });
|
||||
mkdirSync(failing, { recursive: true });
|
||||
writeFileSync(
|
||||
join(failing, "pg_dump"),
|
||||
"#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(join(failing, "pg_dump"), 0o755);
|
||||
cwd = process.cwd();
|
||||
process.chdir(appDir);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
process.chdir(cwd);
|
||||
process.env.PATH = `${shims}:${originalPath}`;
|
||||
});
|
||||
|
||||
async function preMigration(env: Record<string, string>) {
|
||||
const { preMigrationDump } = await import("../../scripts/db-dump.mjs");
|
||||
const errors: string[] = [];
|
||||
const spyErr = vi
|
||||
.spyOn(console, "error")
|
||||
.mockImplementation((l: string) => void errors.push(String(l)));
|
||||
const spyLog = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
try {
|
||||
const code = await preMigrationDump({ ...process.env, ...env });
|
||||
return { code, errors: errors.join("\n") };
|
||||
} finally {
|
||||
spyErr.mockRestore();
|
||||
spyLog.mockRestore();
|
||||
}
|
||||
}
|
||||
|
||||
it("writes a checked dump with its manifest, and keeps the last two", async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (i > 0) await new Promise((r) => setTimeout(r, 1100));
|
||||
const { code } = await preMigration({ DB_BACKUP_DIR: preDir });
|
||||
expect(code).toBe(0);
|
||||
}
|
||||
const names = readdirSync(preDir).sort();
|
||||
expect(names).toHaveLength(4);
|
||||
expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength(
|
||||
2,
|
||||
);
|
||||
const manifest = await readManifestFile(
|
||||
join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""),
|
||||
);
|
||||
expect(manifest.kind).toBe("pre-migration");
|
||||
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
|
||||
});
|
||||
|
||||
it("does nothing when switched off", async () => {
|
||||
const before = readdirSync(preDir);
|
||||
const { code } = await preMigration({
|
||||
DB_BACKUP_DIR: preDir,
|
||||
DB_BACKUP_BEFORE_MIGRATE: "false",
|
||||
});
|
||||
expect(code).toBe(0);
|
||||
expect(readdirSync(preDir)).toEqual(before);
|
||||
});
|
||||
|
||||
it("warns and lets the boot go on without a writable directory", async () => {
|
||||
const { code, errors } = await preMigration({
|
||||
DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here",
|
||||
});
|
||||
expect(code).toBe(0);
|
||||
expect(errors).toMatch(
|
||||
/is not writable, so no dump is taken before the 1 pending migration/,
|
||||
);
|
||||
});
|
||||
|
||||
it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => {
|
||||
// A second of its own, so the name cannot meet the last dump's.
|
||||
await new Promise((r) => setTimeout(r, 1100));
|
||||
const before = readdirSync(preDir).sort();
|
||||
process.env.PATH = `${failing}:${originalPath}`;
|
||||
const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir });
|
||||
process.env.PATH = `${shims}:${originalPath}`;
|
||||
expect(code).toBe(1);
|
||||
expect(errors).toMatch(
|
||||
/the database dump before migrating failed: .*No space left/,
|
||||
);
|
||||
expect(readdirSync(preDir).sort()).toEqual(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe("two dumps in the same second", () => {
|
||||
it("refuses the second and leaves the first whole", async () => {
|
||||
const { createDatabaseDump } =
|
||||
await import("@/lib/database-backup/dump-core.mjs");
|
||||
const dir = join(work, "same-second");
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(dir);
|
||||
const now = new Date("2026-10-26T01:30:00Z");
|
||||
const options = {
|
||||
databaseUrl: process.env.DATABASE_URL ?? "",
|
||||
dir,
|
||||
kind: "daily" as const,
|
||||
now,
|
||||
};
|
||||
await createDatabaseDump(options);
|
||||
const before = readdirSync(dir)
|
||||
.sort()
|
||||
.map((n) => [n, readFileSync(join(dir, n)).length]);
|
||||
await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/);
|
||||
expect(
|
||||
readdirSync(dir)
|
||||
.sort()
|
||||
.map((n) => [n, readFileSync(join(dir, n)).length]),
|
||||
).toEqual(before);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user