Merge remote-tracking branch 'origin/release/v1.46.0' into fix/v146-security

This commit is contained in:
Marc-André Bombeck committed 2026-10-25 14:07:01 +01:00
commit e79ea08b13
55 files changed
+5523 -91

No files matched your search

+25
View File
@@ -277,6 +277,31 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32"
# BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY
# BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation
# -----------------------------------------------------------------------------
# Database dumps (optional, on by default)
# -----------------------------------------------------------------------------
# The whole-database dump that rebuilds a lost server: nightly, and before
# every update that brings migrations. Written to DB_BACKUP_DIR with a
# manifest beside it (row counts, key fingerprints, versions); with the
# off-host bucket above set, each nightly dump is also uploaded under
# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not
# encrypted: they sit beside the database volume, in the same trust zone.
# Restore: `docker compose run --rm db-restore latest`. See
# docs/ops/disaster-recovery.md.
#
# Set "false" to stop the nightly dump.
# DB_BACKUP_ENABLED="true"
# Where dumps are written. The bundled compose file mounts the `backups`
# volume here; change it only together with that mount.
# DB_BACKUP_DIR="/var/lib/healthlog/backups"
# Five-field cron, in Europe/Berlin time like every scheduled job.
# DB_BACKUP_SCHEDULE="30 1 * * *"
# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept
# besides.
# DB_BACKUP_KEEP="7"
# Set "false" to start an update without the dump before its migrations.
# DB_BACKUP_BEFORE_MIGRATE="true"
# -----------------------------------------------------------------------------
# Stored backups in the database (optional)
# -----------------------------------------------------------------------------
+25
View File
@@ -414,6 +414,31 @@ API_TOKEN_HMAC_KEY="<openssl rand -hex 32>"
# key once the lifecycle rule has retired every object written under it.
# BACKUP_ENCRYPTION_PREVIOUS_KEYS=""
# -----------------------------------------------------------------------------
# Database dumps (optional, on by default)
# -----------------------------------------------------------------------------
# The whole-database dump that rebuilds a lost server: nightly, and before
# every update that brings migrations. Written to DB_BACKUP_DIR with a
# manifest beside it (row counts, key fingerprints, versions); with the
# off-host bucket above set, each nightly dump is also uploaded under
# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not
# encrypted: they sit beside the database volume, in the same trust zone.
# Restore: `docker compose run --rm db-restore latest`. See
# docs/ops/disaster-recovery.md.
#
# Set "false" to stop the nightly dump.
# DB_BACKUP_ENABLED="true"
# Where dumps are written. The bundled compose file mounts the `backups`
# volume here; change it only together with that mount.
# DB_BACKUP_DIR="/var/lib/healthlog/backups"
# Five-field cron, in Europe/Berlin time like every scheduled job.
# DB_BACKUP_SCHEDULE="30 1 * * *"
# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept
# besides.
# DB_BACKUP_KEEP="7"
# Set "false" to start an update without the dump before its migrations.
# DB_BACKUP_BEFORE_MIGRATE="true"
# -----------------------------------------------------------------------------
# Stored backups in the database (optional)
# -----------------------------------------------------------------------------
+70
View File
@@ -0,0 +1,70 @@
name: Disaster recovery drill
# Rebuilds a lost server from the nightly database dump and the keys alone,
# against real containers: the app image built from this checkout, the
# bundled compose file with an S3-compatible bucket outside it, the demo seed
# plus a Guardian with a managed profile, the dump taken and uploaded by the
# scheduled job, every volume of the stack destroyed, a new stack, the
# restore, and the app booted on the result (health, login, an encrypted
# note, a document, the Guardian's profile, every table's row count against
# the manifest). Two sources: from the bucket (`db-restore latest
# --from-bucket`, the disaster path) and from dump files copied off the host
# (`db-restore latest`).
# `scripts/compose-dr-drill.sh` is the same drill, runnable locally; its
# header lists what it checks and the mutations that must turn it red.
on:
pull_request:
branches: [main]
paths:
- docker-compose.yml
- Dockerfile
- docker-entrypoint.sh
- scripts/compose-dr-drill.sh
- scripts/dr-drill-seed.ts
- scripts/db-dump.mjs
- scripts/db-restore.mjs
- scripts/pending-migrations.mjs
- src/lib/database-backup/**
- src/lib/jobs/database-backup-queue.ts
- src/lib/jobs/reminder/backup-handlers.ts
- .github/workflows/compose-dr-drill.yml
schedule:
# Nightly, 02:41 UTC: off the hour, after the evening merges.
- cron: "41 2 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: compose-dr-drill-${{ github.ref }}
cancel-in-progress: true
jobs:
drill:
name: Restore a lost server from its dump (${{ matrix.source }})
runs-on: ubuntu-24.04
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
source: [bucket, local]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
cache: pnpm
# The seed scripts run from the checkout against the drill's database.
- run: pnpm install --frozen-lockfile
- run: pnpm db:generate
- name: Drill
env:
DRILL_SOURCE: ${{ matrix.source }}
run: bash scripts/compose-dr-drill.sh
+29 -3
View File
@@ -77,7 +77,13 @@ FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb
# security patches from the Alpine repo override it deliberately —
# CVE-2026-14456 in openssl was the first instance the container scan
# caught between base-image rebuilds).
RUN apk add --no-cache tzdata && apk upgrade --no-cache
#
# `postgresql18-client` carries `pg_dump` and `pg_restore` for the nightly
# database dump, the pre-migration dump in the entrypoint and the
# `db-restore` service (docs/ops/disaster-recovery.md). Major 18 matches the
# bundled database; a client dumps servers of its own major and older, so it
# also serves an install still on its own Postgres 16.
RUN apk add --no-cache tzdata postgresql18-client && apk upgrade --no-cache
WORKDIR /app
ENV NODE_ENV=production
@@ -111,6 +117,13 @@ ENV NEXT_PUBLIC_APP_BUILD_SHA=$NEXT_PUBLIC_APP_BUILD_SHA
RUN addgroup --system --gid 1001 nodejs && \
adduser --system --uid 1001 nextjs
# Where the database dumps go (`DB_BACKUP_DIR`). Owned by the app user, so a
# fresh named volume mounted here starts with that owner: Docker fills an
# empty named volume from the image directory it covers, ownership included.
RUN mkdir -p /var/lib/healthlog/backups && \
chown nextjs:nodejs /var/lib/healthlog/backups && \
chmod 700 /var/lib/healthlog/backups
# Copy built assets
COPY --from=builder /app/public ./public
# The server reads its message catalogs from here on first use, one locale at
@@ -233,11 +246,16 @@ COPY --from=builder /app/src/generated ./src/generated
# the runbooks and the entrypoint against this list. `pending-migrations.mjs`
# and `migrate-deploy.mjs` are the entrypoint's own: the first logs the
# pending migration count, the second runs `migrate deploy` with its
# lock-timeout retry. Root-owned on purpose: the app user can run them but not
# rewrite them.
# lock-timeout retry. `db-dump.mjs` takes the dump before migrating, and
# `db-restore.mjs` is the `db-restore` service; both share
# `src/lib/database-backup/dump-core.mjs` with the app. Root-owned on purpose:
# the app user can run them but not rewrite them.
COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs
COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs
COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs
COPY --from=builder /app/scripts/db-dump.mjs ./scripts/db-dump.mjs
COPY --from=builder /app/scripts/db-restore.mjs ./scripts/db-restore.mjs
COPY --from=builder /app/src/lib/database-backup/dump-core.mjs ./src/lib/database-backup/dump-core.mjs
COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs
COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts
# Prove the reset CLI's imports resolve in this tree: with every import found
@@ -247,6 +265,14 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:"
RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \
node --input-type=module -e "await import('pg')" && \
node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')"
# The dump and restore scripts load, the client tools they run are the
# major the bundled database runs, and the S3 client `db-restore --from-bucket`
# loads resolves from the shared module's place in the tree.
RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \
node --input-type=module -e "import { createRequire } from 'node:module'; createRequire('/app/src/lib/database-backup/dump-core.mjs').resolve('@aws-sdk/client-s3')" && \
node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \
pg_dump --version | grep -q "(PostgreSQL) 18\." && \
pg_restore --version | grep -q "(PostgreSQL) 18\."
# Install the migration CLI, its config dependency, and a pinned launcher for
# the maintenance scripts already shipped in the standalone tree. The Prisma
+65 -5
View File
@@ -14,9 +14,11 @@
# and login silently fails. This is the single most common self-host snag.
# Leave it unset (or true) only when a TLS reverse proxy serves HTTPS.
#
# Persistent state lives in exactly one named volume (`postgres-data`,
# Postgres' data). The app container is stateless, so a backup of the database
# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to
# Persistent state lives in the named volume `postgres-data` (Postgres' data).
# The app container is stateless, so a backup of the database is a backup of
# everything. `backups` holds the database dumps the app takes every night and
# before every migration (docs/ops/disaster-recovery.md); keep a copy of them
# off this host, or set the off-host bucket below. `pgdata` is where Postgres 16 kept the data up to
# v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once
# and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked
# into the image at build time or mounted read-only from the host and
@@ -147,6 +149,20 @@ x-healthlog-env: &healthlog_env
# serves anyway. A last resort for a refusal you are certain is false: it
# disables a safety check. See docs/self-hosting/getting-started.md.
ENCRYPTION_KEY_CHECK: "${ENCRYPTION_KEY_CHECK:-}"
# The whole-database dump (docs/ops/disaster-recovery.md). Nightly at
# DB_BACKUP_SCHEDULE (five-field cron in Europe/Berlin time, default
# "30 1 * * *") into DB_BACKUP_DIR (default /var/lib/healthlog/backups, the
# `backups` volume below), keeping DB_BACKUP_KEEP daily dumps (default 7)
# and the last two taken before a migration. With the off-host bucket set,
# each nightly dump is also uploaded under `database/`, sealed with
# BACKUP_ENCRYPTION_KEY. DB_BACKUP_ENABLED=false stops the nightly dump;
# DB_BACKUP_BEFORE_MIGRATE=false skips the dump before migrations. Change
# DB_BACKUP_DIR only together with the volume mount.
DB_BACKUP_ENABLED: "${DB_BACKUP_ENABLED:-}"
DB_BACKUP_DIR: "${DB_BACKUP_DIR:-}"
DB_BACKUP_SCHEDULE: "${DB_BACKUP_SCHEDULE:-}"
DB_BACKUP_KEEP: "${DB_BACKUP_KEEP:-}"
DB_BACKUP_BEFORE_MIGRATE: "${DB_BACKUP_BEFORE_MIGRATE:-}"
BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}"
BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}"
BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}"
@@ -475,6 +491,11 @@ services:
- "3000:3000"
environment:
<<: *healthlog_env
# The database dumps (DB_BACKUP_DIR). Without this mount the app still
# copies its nightly dump off-host when the bucket is set, but keeps none
# here and takes none before a migration; the boot log says so.
volumes:
- backups:/var/lib/healthlog/backups
# Bring your own GeoLite2 databases. The published image only carries
# MMDBs if the build had a MaxMind licence key, so on `:latest` the
# offline tier is usually absent and every login IP goes to the online
@@ -484,9 +505,9 @@ services:
# matches GEOLITE2_DIR's default (/opt/geolite2), so this works as-is
# once uncommented. Mounting at a different container path also works,
# just also set GEOLITE2_DIR in .env to match it.
# Steps: docs/self-hosting/geolite2.md.
# Steps: docs/self-hosting/geolite2.md. The line goes under `volumes:`
# above, beside the backups line:
#
# volumes:
# - ./geolite2:/opt/geolite2:ro
depends_on:
db:
@@ -563,6 +584,9 @@ services:
# environment:
# <<: *healthlog_env
# HEALTHLOG_PROCESS_TYPE: "worker"
# # The worker takes the nightly database dump, so it needs the volume.
# volumes:
# - backups:/var/lib/healthlog/backups
# depends_on:
# db:
# condition: service_healthy
@@ -776,6 +800,42 @@ services:
mv "$$MOVING" "$$MARK"
log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md"
# Restores a database dump into an EMPTY database (docs/ops/disaster-recovery.md).
# Never started by `docker compose up`; run it by name:
#
# docker compose run --rm db-restore list
# docker compose run --rm db-restore latest
# docker compose run --rm db-restore healthlog-<time>-daily.dump
# docker compose run --rm db-restore latest --from-bucket
#
# It checks the dump against its manifest and the keys in .env against the
# keys the dump was written with, refuses a database that already holds
# tables, restores in one transaction and compares every table's row count
# with the manifest. Start the app only after it says "Restored".
# `--from-bucket` downloads the dump and its manifest from the off-host bucket
# (`database/`, the BACKUP_S3_* variables) and opens them with
# BACKUP_ENCRYPTION_KEY: the restore after the host and its local dumps are
# gone. Files you downloaded yourself restore from a directory you mount:
# `docker compose run --rm -v ./restore:/restore:ro db-restore
# /restore/healthlog-<time>-daily.dump.enc`.
db-restore:
image: ghcr.io/mbombeck/healthlog${HEALTHLOG_IMAGE_REF:-:latest}
profiles: ["restore"]
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
environment:
<<: *healthlog_env
volumes:
- backups:/var/lib/healthlog/backups
depends_on:
db:
condition: service_healthy
entrypoint: ["node", "scripts/db-restore.mjs"]
command: ["list"]
volumes:
postgres-data:
pgdata:
backups:
+9
View File
@@ -88,6 +88,15 @@ echo "HealthLog: Database is reachable."
# Say how many migrations this image is about to apply, with a reminder to
# back up first. Reporting only: it never stops the boot.
node scripts/pending-migrations.mjs || true
# When migrations are pending, dump the database into DB_BACKUP_DIR first, so
# the state before them can be restored (docs/ops/disaster-recovery.md). No
# writable directory: a warning, and the boot goes on. A writable directory
# and a failed dump: the boot stops here, before anything is migrated.
if ! node scripts/db-dump.mjs pre-migration; then
echo "HealthLog: ERROR — not migrating without the dump. Free disk space or fix the error above and start again." >&2
echo "HealthLog: to migrate without a dump this once, set DB_BACKUP_BEFORE_MIGRATE=false." >&2
exit 1
fi
# `migrate-deploy.mjs` runs `prisma migrate deploy` and retries only a
# migration that timed out on a lock and is marked safe to re-run; any other
# failure stops the boot with Prisma's exit code (docs/ops/deploy.md).
+13
View File
@@ -5,6 +5,11 @@ user's JSON dump, encrypted with AES-256-GCM under a SEPARATE key
(`BACKUP_ENCRYPTION_KEY`), to any S3-compatible bucket — Cloudflare R2,
AWS S3, Backblaze B2, MinIO, etc.
These per-account copies roll one account back on the same server. To
rebuild a server, use the nightly database dump, which the same job
configuration also uploads (`database/healthlog-<time>-daily.dump.enc`, in the
envelope below): [`disaster-recovery.md`](disaster-recovery.md).
The backup runs at **02:30 Europe/Berlin** every day from the worker
container (queue `data-backup-offhost`). Object key layout:
@@ -72,6 +77,14 @@ only those:
was deleted is removed by the upload itself.
- **The connection test** deletes its own `_healthcheck/<ts>.bin`.
The database dumps are not purged per account. A dump is the whole database
at one moment, so an account deleted since stays in the dumps taken before
its deletion until they age out: after `DB_BACKUP_KEEP` nights on the server,
and in the bucket after the lifecycle rule for `database/`. That is backup
retention, and the period to state to the people using your instance; the
backups page shows both numbers and warns when no rule covers `database/`
([`disaster-recovery.md`](disaster-recovery.md)).
The admin wipe of every account deletes nothing in the bucket. It is you
clearing your own host, the bucket is yours, and its copies are the way back
from a wipe pressed by mistake. If you are handing the host on, empty the
+24 -11
View File
@@ -13,11 +13,13 @@ Migrations land **before** the new app container takes traffic. The
canonical command sequence on a single-instance host:
```sh
# 0. Dump the database first. A migration is not undone by going back to
# the previous image; the dump is the way back. See "Updates" in
# docs/self-hosting/getting-started.md.
docker compose exec -T db pg_dump -U healthlog healthlog \
| gzip > healthlog-before-update-$(date +%F).sql.gz
# 0. Nothing to do by hand for the way back: when the new image brings
# migrations, its entrypoint first writes
# healthlog-<time>-pre-migration.dump into DB_BACKUP_DIR (the `backups`
# volume) and keeps the last two. A migration is not undone by going back
# to the previous image; that dump is the way back
# (docs/ops/disaster-recovery.md, "Going back to before a migration").
# Copy it off the host if the host itself is what you are worried about.
# 1. Pull the new image. If you pin a version with HEALTHLOG_IMAGE_REF,
# set it to the new one first.
@@ -33,6 +35,14 @@ docker compose run --rm app sh -c 'exit 0'
docker compose up -d app
```
Step 2 takes the pre-migration dump too, in the one-off container, before it
migrates. When the backups directory is writable and that dump fails (a full
disk, most often), the entrypoint stops before migrating and says so; free
the space and run the step again, or set `DB_BACKUP_BEFORE_MIGRATE=false` for
one start without the dump. Without a writable directory (an install that
runs the image without the `backups` volume) the log carries a warning, no
dump is taken, and the migrations run.
Use `docker compose run` rather than a plain `docker run` for step 2. A
plain `docker run` starts outside the stack's network, where `db` does not
resolve, and the entrypoint gives up after thirty seconds with "Database
@@ -230,12 +240,15 @@ roll back:
dev --create-only --name vX_Y_Z_rollback_<col>`), edit the SQL,
commit, and re-deploy. Never `migrate resolve --rolled-back` against
production unless you've taken a fresh `pg_dump` first.
4. Rolling back from v1.40.2 or later to v1.40.1 or earlier: an account
that saved the Coach lookback "Last year" reads its whole Coach
preference blob as defaults on the older version, metric exclusions
included, until the setting is saved again. Ask affected accounts to
pick another lookback before the rollback, or re-save their exclusions
after it.
When the data itself has to go back to before the update, restore the
pre-migration dump with the previous image; the steps are in
[`disaster-recovery.md`](disaster-recovery.md#going-back-to-before-a-migration). 4. Rolling back from v1.40.2 or later to v1.40.1 or earlier: an account
that saved the Coach lookback "Last year" reads its whole Coach
preference blob as defaults on the older version, metric exclusions
included, until the setting is saved again. Ask affected accounts to
pick another lookback before the rollback, or re-save their exclusions
after it.
## Catalog templates
+185
View File
@@ -0,0 +1,185 @@
# Disaster recovery
The copy that rebuilds a lost server is the database dump. This page says
what is dumped and where it goes, which keys a restore needs, how to restore
onto a new server, and how the procedure is tested.
## What is backed up, and where
| Copy | When | Where | Kept |
| ----------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------- |
| Nightly dump | 01:30 Europe/Berlin, the zone every schedule runs in (`DB_BACKUP_SCHEDULE`) | `DB_BACKUP_DIR`, the `backups` volume of the bundled compose file | the newest 7 (`DB_BACKUP_KEEP`) |
| Pre-migration dump | on start, when the new image brings migrations, before them | the same directory | the newest 2 |
| Off-host copy of a dump | right after each nightly dump, if the bucket is configured | your bucket, under `database/`, sealed with `BACKUP_ENCRYPTION_KEY` | your bucket's lifecycle rule |
A dump is `pg_dump --format=custom` of the whole database: every account,
sign-ins, passkeys, sharing grants and managed profiles, documents, the job
queue and the instance settings. It is two files:
- `healthlog-<UTC time>-<daily|pre-migration>.dump`, the archive;
- `healthlog-<UTC time>-<daily|pre-migration>.manifest.json` beside it: the
app version and build, the newest applied migration, the Postgres version,
the archive's size and SHA-256, the row count of every table, and the
fingerprints (never the values) of the keys the database needs.
A dump counts as good only when both files exist under their final names. The
archive is written as `.partial`, read back with `pg_restore --list`, and
renamed after its manifest; a run that fails leaves nothing that looks good.
The admin backups page shows the newest good dump, where it is, and the
newest failed run. The start log says on every boot where dumps go, or that
none can be kept.
Local dumps are not encrypted. They sit beside the database volume, in the
same trust zone: whoever can read one volume can read the other. The columns
HealthLog encrypts stay encrypted inside the dump, under `ENCRYPTION_KEYS`.
Keep a copy of the dumps off the host; the off-host bucket does that for you,
and it seals each copy with `BACKUP_ENCRYPTION_KEY` in the envelope the
per-account copies use.
A dump holds every account that existed when it was taken, including one
deleted since. Deleted accounts leave the dumps as they age out: after
`DB_BACKUP_KEEP` nights on the server, and after your lifecycle rule in the
bucket. Give that rule a prefix that covers `database/` too; the admin
backups page reads the bucket's rules and says when none does. This is the
retention period to state where you tell people that deleting their account
deletes their data.
Without a writable `DB_BACKUP_DIR` (an install that runs the image without the
`backups` volume), the nightly dump is uploaded off-host and nothing is kept
on the server, and no dump is taken before migrations; with no bucket either,
no dump is taken at all. The start log and the admin page say so.
The per-account copies (the weekly in-database copy, the nightly per-account
objects) roll one account back on the same server. They cannot rebuild a
server; see [`backup-restore.md`](backup-restore.md).
## The keys a restore needs
| Key | Without it |
| ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
| `ENCRYPTION_KEYS` + `ENCRYPTION_ACTIVE_KEY_ID`, or `ENCRYPTION_KEY` | Nothing encrypted can be read. The restore refuses when the active key is missing or different. Keep every retired id still in the map. |
| `API_TOKEN_HMAC_KEY` | No data is lost, but every session, app token and share link stops working; people sign in again and re-create tokens. |
| `BACKUP_ENCRYPTION_KEY` (and `BACKUP_ENCRYPTION_PREVIOUS_KEYS`) | Off-host copies do not open. |
| `POSTGRES_PASSWORD` | Not needed: a new server can use a new one. |
The restore compares the keys in its environment with the fingerprints in the
manifest before it writes anything, and names each key that is missing or
different.
## Restore onto a new server
1. Install Docker and take the `docker-compose.yml` of the version that wrote
the dump or a newer one. The manifest's `appVersion` names it; to run that
exact version set `HEALTHLOG_IMAGE_REF=:v<version>` in `.env`.
2. Write `.env` with the keys from your records (see above) and a new
`POSTGRES_PASSWORD`.
3. Do not run `docker compose up` yet. The app would create an empty schema,
and the restore only writes into a database with no tables. Start only the
database:
```bash
docker compose up -d db
```
4. Restore. When the host is lost, its local dumps went with it, and the
copy is the one in your bucket: put the `BACKUP_S3_*` variables and
`BACKUP_ENCRYPTION_KEY` (with any `BACKUP_ENCRYPTION_PREVIOUS_KEYS`) in
`.env` as on the old server, then
```bash
docker compose run --rm db-restore list --from-bucket
docker compose run --rm db-restore latest --from-bucket
```
`latest --from-bucket` downloads the newest `database/…dump.enc` whose
manifest is in the bucket too, opens both with `BACKUP_ENCRYPTION_KEY`
and restores. Name a dump instead of `latest` to take an older one.
With the files in hand instead (copied off the host, or downloaded from
the bucket with your own tools: both `<name>.dump.enc` and
`<name>.manifest.json.enc`, or a `.dump` with its `.manifest.json`), put
them in a directory such as `./restore`, make them readable for the
container (`chmod a+r restore/*`) and name the dump:
```bash
docker compose run --rm -v "$PWD/restore:/restore:ro" db-restore list /restore
docker compose run --rm -v "$PWD/restore:/restore:ro" db-restore \
/restore/healthlog-20261024T233000Z-daily.dump
```
Either way it checks the archive against the manifest's SHA-256, the keys
against the manifest's fingerprints, and that the database is empty. Then
it restores in one transaction, gathers planner statistics and counts
every table against the manifest. A count that differs empties the
database again and fails. Wait for the line that starts with `Restored`.
5. Start the stack and check it:
```bash
docker compose up -d
curl -s http://localhost:3000/api/health
```
Sign in. The app's own key check runs at start and refuses to serve on a
key that does not open the data.
When the dumps are still in the server's `backups` volume (the database
volume was lost, the backups volume was not), step 4 is
`docker compose run --rm db-restore latest`.
`--ignore-key-mismatch` restores although the active encryption key differs.
Use it only when that key is lost for good: everything encrypted stays
unreadable.
## Going back to before a migration
The entrypoint dumps the database before it applies migrations. To return to
that state (the previous image, the data as it was before the update):
```bash
docker compose stop app
# Keep the current state too, in case it is wanted later.
docker compose exec -T db pg_dump -U healthlog -Fc healthlog > healthlog-current.dump
docker compose exec db dropdb -U healthlog healthlog
docker compose exec db createdb -U healthlog healthlog
docker compose run --rm db-restore list
docker compose run --rm db-restore healthlog-<time>-pre-migration.dump
# Set HEALTHLOG_IMAGE_REF in .env to the version you ran before the update.
docker compose up -d
```
The dump before a migration is taken while the new image starts. If it fails
on a writable directory (a full disk, most often), the start stops before
anything is migrated and the log names the switch: `DB_BACKUP_BEFORE_MIGRATE=false`
starts without it, once.
## How this is tested
`scripts/compose-dr-drill.sh` plays this page against real containers: it
builds the image, seeds several accounts (the demo account with documents
and encrypted notes, a Guardian with a managed profile), lets the scheduled
job take the dump and upload it to an S3-compatible bucket, destroys every
volume of the stack (the local dumps included), and restores on a new stack
given only the bucket and the keys
(`docker compose run --rm db-restore latest --from-bucket`). A second run
restores from the dump files instead (`DRILL_SOURCE=local`). Both then count
every table against the manifest and boot the app: health, sign-in, an
encrypted note read back, a document opened, the Guardian still seeing the
profile. It runs in CI nightly and on every change to the
compose file, the image or the dump and restore code
(`.github/workflows/compose-dr-drill.yml`), and it is checked to fail on a
wrong encryption key, a wrong off-host key, a dump that lost a table's rows
and a target that is not empty.
Test your own copies too: once, and after any change to how the server is
run, restore last night's dump on a spare machine with the steps above.
## Settings
| Variable | Default | What it does |
| -------------------------- | ---------------------------- | -------------------------------------------------------------------------------- |
| `DB_BACKUP_ENABLED` | on | `false` stops the nightly dump. |
| `DB_BACKUP_DIR` | `/var/lib/healthlog/backups` | Where dumps are written and read. Change it only together with the volume mount. |
| `DB_BACKUP_SCHEDULE` | `30 1 * * *` | Five-field cron, Europe/Berlin time like every HealthLog schedule. |
| `DB_BACKUP_KEEP` | `7` | Nightly dumps kept (at least 1). Two pre-migration dumps are kept besides. |
| `DB_BACKUP_BEFORE_MIGRATE` | on | `false` starts an update without the dump before its migrations. |
+22 -6
View File
@@ -346,6 +346,15 @@ written under that key, so remove the setting once the key is confirmed.
Your database is the only stateful piece — back it up, and back up the
key that decrypts it.
**Database dumps the app takes itself.** Every night, and before an update
applies migrations, the app writes a `pg_dump` of the whole database with a
manifest into the `backups` volume and keeps the newest few. With the
off-host bucket configured, each nightly dump is also uploaded, encrypted.
`docker compose run --rm db-restore latest` restores one into an empty
database. What is kept where, the keys a restore needs, and restoring onto a
new server: [`docs/ops/disaster-recovery.md`](../ops/disaster-recovery.md).
Keep a copy of those dumps off this host.
**Local snapshot (the common single-box case).** Dump the `db` service
with `pg_dump`:
@@ -379,8 +388,11 @@ encryption key safely (keep the old key until zero rows remain on it) see
A new image can bring database migrations, and `docker-entrypoint.sh`
applies them on the first start of that image. Most are additive, but some
rewrite data, and going back to the previous image does not undo them. The
only way back from a migration that went wrong is the dump you took before
it. So take one before every pull:
way back from a migration that went wrong is a dump taken before it. The
entrypoint takes one itself (`healthlog-<time>-pre-migration.dump` in the
`backups` volume, see
[`docs/ops/disaster-recovery.md`](../ops/disaster-recovery.md)); a dump of
your own before every pull, kept off the host, is the second copy:
```bash
# 1. Dump the database (same command as under "Backup and restore").
@@ -405,14 +417,18 @@ the `image:` line, a new major (16 to 18) is a dump and restore, described in
The start log says what is about to happen before the migrations run:
```text
HealthLog: 2 database migrations pending. Back up the database before updating (pg_dump, see the Updates section of docs/self-hosting/getting-started.md); a migration is not undone by going back to the previous image.
HealthLog: 2 database migrations pending. Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); a migration is not undone by going back to the previous image.
HealthLog: taking a database dump before 2 pending migration(s) into /var/lib/healthlog/backups...
HealthLog: pre-migration dump written: healthlog-20261025T081502Z-pre-migration.dump (212.4 MB, 231 tables, 38 s).
HealthLog: Running database migrations...
```
or `HealthLog: Database schema is up to date.` when the image brings none.
The line is a notice and does not pause the start: the migrations run right
after it. So the dump belongs before step 2, not after you see the line; by
then the database is already being migrated.
The migrations run right after the dump. A dump that fails on a writable
directory stops the start before anything is migrated, and the log names the
switch that skips it once (`DB_BACKUP_BEFORE_MIGRATE=false`). Your own dump
belongs before step 2, not after you see these lines; by then the database is
already being migrated.
If an update to v1.43 stops at migration `0409_drop_plaintext_location`,
nothing was changed; the recovery is in
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Name eines Stimmungs-Tags oder einer Gruppe mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne Namen behalten)",
"restoreSkippedVisitCiphertext": "Text zu Arztbesuch oder Praxis mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne ihn behalten)",
"restoreSkippedVaccinationCiphertext": "Impf-Notiz mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Impfung ohne sie behalten)",
"restoreSkippedEnvironmentLocationCiphertext": "Ort eines Umweltzeitraums oder -tags mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (verschlüsselt behalten, diese Tage werden nicht neu abgerufen)"
"restoreSkippedEnvironmentLocationCiphertext": "Ort eines Umweltzeitraums oder -tags mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (verschlüsselt behalten, diese Tage werden nicht neu abgerufen)",
"database": {
"title": "Datenbank-Dump",
"lastGood": "Letzter gültiger Dump {when}, {size}, {where}.",
"whereBoth": "auf diesem Server und extern gespeichert",
"whereLocal": "auf diesem Server gespeichert",
"whereOffhost": "nur extern gespeichert",
"none": "Noch kein Datenbank-Dump. Der erste läuft heute Nacht.",
"failedTitle": "Der letzte Datenbank-Dump ist fehlgeschlagen",
"failedDescription": "{when}: {reason}",
"disabled": "Datenbank-Dumps sind abgeschaltet (DB_BACKUP_ENABLED=false). Sichere die Datenbank auf anderem Weg.",
"noDirectoryOffhost": "Dieser Server kann nicht nach {dir} schreiben. Dumps gehen deshalb nur extern, hier wird keiner aufbewahrt. Binde das Backup-Volume ein.",
"noDirectory": "Dieser Server kann nicht nach {dir} schreiben und es ist kein externer Speicher eingerichtet. Deshalb entsteht kein Datenbank-Dump. Binde das Backup-Volume ein.",
"retentionLocal": "Gelöschte Konten bleiben in den Dumps, bis diese aussortiert werden: Auf diesem Server bleiben die neuesten {nights}.",
"retentionOffhost": "Externe Kopien entfernt die Lebenszyklus-Regel des Buckets nach {days} Tagen.",
"retentionOffhostMissing": "Keine Lebenszyklus-Regel im Bucket gilt für database/. Dumps dort und die gelöschten Konten darin werden deshalb nie entfernt. Lege eine Regel für dieses Präfix an."
}
},
"danger-zone": {
"title": "Gefahrenzone",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Mood tag or group name sealed with a key this server does not hold (kept without its name)",
"restoreSkippedVisitCiphertext": "Visit or practitioner text sealed with a key this server does not hold (kept without it)",
"restoreSkippedVaccinationCiphertext": "Vaccination note sealed with a key this server does not hold (dose kept without it)",
"restoreSkippedEnvironmentLocationCiphertext": "Location of an environment period or day sealed with a key this server does not hold (kept sealed, those days are not refreshed)"
"restoreSkippedEnvironmentLocationCiphertext": "Location of an environment period or day sealed with a key this server does not hold (kept sealed, those days are not refreshed)",
"database": {
"title": "Database dump",
"lastGood": "Last good dump {when}, {size}, {where}.",
"whereBoth": "kept on this server and off-host",
"whereLocal": "kept on this server",
"whereOffhost": "off-host only",
"none": "No database dump yet. The first one runs tonight.",
"failedTitle": "The last database dump failed",
"failedDescription": "{when}: {reason}",
"disabled": "Database dumps are switched off (DB_BACKUP_ENABLED=false). Back the database up another way.",
"noDirectoryOffhost": "This server cannot write to {dir}, so dumps go off-host only and none is kept here. Mount the backups volume.",
"noDirectory": "This server cannot write to {dir} and no off-host bucket is set, so no database dump can be taken. Mount the backups volume.",
"retentionLocal": "Deleted accounts stay in the dumps until those age out: the newest {nights} are kept on this server.",
"retentionOffhost": "Off-host copies are removed after {days} days by the bucket's lifecycle rule.",
"retentionOffhostMissing": "No lifecycle rule in the bucket covers database/, so dumps there, and the deleted accounts inside them, are never removed. Add a rule for that prefix."
}
},
"danger-zone": {
"title": "Danger zone",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Nombre de etiqueta o grupo de ánimo cifrado con una clave que este servidor no tiene (se conserva sin el nombre)",
"restoreSkippedVisitCiphertext": "Texto de visita o profesional cifrado con una clave que este servidor no tiene (se conserva sin él)",
"restoreSkippedVaccinationCiphertext": "Nota de vacunación cifrada con una clave que este servidor no tiene (la dosis se conserva sin ella)",
"restoreSkippedEnvironmentLocationCiphertext": "Ubicación de un periodo o día ambiental cifrada con una clave que este servidor no tiene (se conserva cifrada, esos días no se actualizan)"
"restoreSkippedEnvironmentLocationCiphertext": "Ubicación de un periodo o día ambiental cifrada con una clave que este servidor no tiene (se conserva cifrada, esos días no se actualizan)",
"database": {
"title": "Volcado de la base de datos",
"lastGood": "Último volcado correcto {when}, {size}, {where}.",
"whereBoth": "guardado en este servidor y fuera de él",
"whereLocal": "guardado en este servidor",
"whereOffhost": "solo fuera del servidor",
"none": "Aún no hay ningún volcado de la base de datos. El primero se hace esta noche.",
"failedTitle": "El último volcado de la base de datos falló",
"failedDescription": "{when}: {reason}",
"disabled": "Los volcados de la base de datos están desactivados (DB_BACKUP_ENABLED=false). Haz la copia de la base de datos de otra forma.",
"noDirectoryOffhost": "Este servidor no puede escribir en {dir}, así que los volcados solo se guardan fuera y aquí no queda ninguno. Monta el volumen de copias.",
"noDirectory": "Este servidor no puede escribir en {dir} y no hay almacenamiento externo configurado, así que no se puede hacer ningún volcado. Monta el volumen de copias.",
"retentionLocal": "Las cuentas eliminadas siguen en los volcados hasta que estos caducan: en este servidor se guardan los {nights} más recientes.",
"retentionOffhost": "La regla de ciclo de vida del bucket elimina las copias externas tras {days} días.",
"retentionOffhostMissing": "Ninguna regla de ciclo de vida del bucket cubre database/, así que los volcados de ahí, y las cuentas eliminadas que contienen, nunca se borran. Añade una regla para ese prefijo."
}
},
"danger-zone": {
"title": "Zona de peligro",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Nom d’étiquette ou de groupe d’humeur chiffré avec une clé que ce serveur ne possède pas (conservé sans son nom)",
"restoreSkippedVisitCiphertext": "Texte de consultation ou de praticien chiffré avec une clé que ce serveur ne possède pas (conservé sans)",
"restoreSkippedVaccinationCiphertext": "Note de vaccination chiffrée avec une clé que ce serveur ne possède pas (dose conservée sans)",
"restoreSkippedEnvironmentLocationCiphertext": "Lieu d'une période ou d'un jour environnemental chiffré avec une clé que ce serveur ne possède pas (conservé chiffré, ces jours ne sont pas actualisés)"
"restoreSkippedEnvironmentLocationCiphertext": "Lieu d'une période ou d'un jour environnemental chiffré avec une clé que ce serveur ne possède pas (conservé chiffré, ces jours ne sont pas actualisés)",
"database": {
"title": "Dump de la base de données",
"lastGood": "Dernier dump valide {when}, {size}, {where}.",
"whereBoth": "conservé sur ce serveur et hors site",
"whereLocal": "conservé sur ce serveur",
"whereOffhost": "hors site uniquement",
"none": "Aucun dump de la base de données pour l'instant. Le premier a lieu cette nuit.",
"failedTitle": "Le dernier dump de la base de données a échoué",
"failedDescription": "{when} : {reason}",
"disabled": "Les dumps de la base de données sont désactivés (DB_BACKUP_ENABLED=false). Sauvegardez la base de données autrement.",
"noDirectoryOffhost": "Ce serveur ne peut pas écrire dans {dir} : les dumps partent uniquement hors site et aucun n'est conservé ici. Montez le volume de sauvegarde.",
"noDirectory": "Ce serveur ne peut pas écrire dans {dir} et aucun stockage hors site n'est configuré : aucun dump ne peut être fait. Montez le volume de sauvegarde.",
"retentionLocal": "Les comptes supprimés restent dans les dumps jusqu'à leur expiration : ce serveur conserve les {nights} plus récents.",
"retentionOffhost": "La règle de cycle de vie du bucket supprime les copies hors site après {days} jours.",
"retentionOffhostMissing": "Aucune règle de cycle de vie du bucket ne couvre database/ : les dumps qui s'y trouvent, et les comptes supprimés qu'ils contiennent, ne sont jamais effacés. Ajoutez une règle pour ce préfixe."
}
},
"danger-zone": {
"title": "Zone de danger",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Nome di tag o gruppo dell’umore cifrato con una chiave che questo server non possiede (mantenuto senza nome)",
"restoreSkippedVisitCiphertext": "Testo di visita o professionista cifrato con una chiave che questo server non possiede (mantenuto senza)",
"restoreSkippedVaccinationCiphertext": "Nota di vaccinazione cifrata con una chiave che questo server non possiede (dose mantenuta senza)",
"restoreSkippedEnvironmentLocationCiphertext": "Luogo di un periodo o giorno ambientale cifrato con una chiave che questo server non possiede (mantenuto cifrato, quei giorni non vengono aggiornati)"
"restoreSkippedEnvironmentLocationCiphertext": "Luogo di un periodo o giorno ambientale cifrato con una chiave che questo server non possiede (mantenuto cifrato, quei giorni non vengono aggiornati)",
"database": {
"title": "Dump del database",
"lastGood": "Ultimo dump valido {when}, {size}, {where}.",
"whereBoth": "conservato su questo server e fuori sede",
"whereLocal": "conservato su questo server",
"whereOffhost": "solo fuori sede",
"none": "Ancora nessun dump del database. Il primo viene eseguito stanotte.",
"failedTitle": "L'ultimo dump del database non è riuscito",
"failedDescription": "{when}: {reason}",
"disabled": "I dump del database sono disattivati (DB_BACKUP_ENABLED=false). Esegui il backup del database in un altro modo.",
"noDirectoryOffhost": "Questo server non può scrivere in {dir}, quindi i dump vanno solo fuori sede e qui non ne resta nessuno. Monta il volume dei backup.",
"noDirectory": "Questo server non può scrivere in {dir} e non è configurato alcuno storage esterno, quindi non si può fare alcun dump. Monta il volume dei backup.",
"retentionLocal": "Gli account eliminati restano nei dump finché questi non scadono: su questo server restano i {nights} più recenti.",
"retentionOffhost": "La regola del ciclo di vita del bucket elimina le copie esterne dopo {days} giorni.",
"retentionOffhostMissing": "Nessuna regola del ciclo di vita del bucket copre database/, quindi i dump lì, e gli account eliminati che contengono, non vengono mai rimossi. Aggiungi una regola per quel prefisso."
}
},
"danger-zone": {
"title": "Zona pericolosa",
+17 -1
View File
@@ -8285,7 +8285,23 @@
"restoreSkippedMoodLabelCiphertext": "이 서버에 없는 키로 암호화된 기분 태그 또는 그룹 이름(이름 없이 유지됨)",
"restoreSkippedVisitCiphertext": "이 서버에 없는 키로 암호화된 진료 또는 의료진 텍스트(해당 내용 없이 유지됨)",
"restoreSkippedVaccinationCiphertext": "이 서버에 없는 키로 암호화된 예방접종 메모(접종 기록은 메모 없이 유지됨)",
"restoreSkippedEnvironmentLocationCiphertext": "이 서버에 없는 키로 암호화된 환경 기간 또는 날짜의 위치(암호화된 채 유지되며 해당 날짜는 새로 가져오지 않음)"
"restoreSkippedEnvironmentLocationCiphertext": "이 서버에 없는 키로 암호화된 환경 기간 또는 날짜의 위치(암호화된 채 유지되며 해당 날짜는 새로 가져오지 않음)",
"database": {
"title": "데이터베이스 덤프",
"lastGood": "마지막 정상 덤프 {when}, {size}, {where}.",
"whereBoth": "이 서버와 외부 저장소에 보관됨",
"whereLocal": "이 서버에 보관됨",
"whereOffhost": "외부 저장소에만 있음",
"none": "아직 데이터베이스 덤프가 없습니다. 첫 덤프는 오늘 밤에 실행됩니다.",
"failedTitle": "마지막 데이터베이스 덤프가 실패했습니다",
"failedDescription": "{when}: {reason}",
"disabled": "데이터베이스 덤프가 꺼져 있습니다(DB_BACKUP_ENABLED=false). 다른 방법으로 데이터베이스를 백업하세요.",
"noDirectoryOffhost": "이 서버는 {dir}에 쓸 수 없어 덤프가 외부 저장소로만 가고 여기에는 보관되지 않습니다. 백업 볼륨을 마운트하세요.",
"noDirectory": "이 서버는 {dir}에 쓸 수 없고 외부 저장소도 설정되지 않아 데이터베이스 덤프를 만들 수 없습니다. 백업 볼륨을 마운트하세요.",
"retentionLocal": "삭제된 계정은 덤프가 만료될 때까지 덤프에 남습니다. 이 서버에는 최신 덤프 {nights}개가 보관됩니다.",
"retentionOffhost": "버킷의 수명 주기 규칙이 외부 사본을 {days}일 후에 삭제합니다.",
"retentionOffhostMissing": "버킷의 어떤 수명 주기 규칙도 database/를 다루지 않아 그곳의 덤프와 그 안의 삭제된 계정이 절대 삭제되지 않습니다. 이 접두사에 대한 규칙을 추가하세요."
}
},
"danger-zone": {
"title": "위험 구역",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Stemmingslabel of groepsnaam versleuteld met een sleutel die deze server niet heeft (behouden zonder naam)",
"restoreSkippedVisitCiphertext": "Tekst van bezoek of zorgverlener versleuteld met een sleutel die deze server niet heeft (behouden zonder)",
"restoreSkippedVaccinationCiphertext": "Vaccinatienotitie versleuteld met een sleutel die deze server niet heeft (dosis behouden zonder)",
"restoreSkippedEnvironmentLocationCiphertext": "Locatie van een omgevingsperiode of -dag versleuteld met een sleutel die deze server niet heeft (versleuteld behouden, die dagen worden niet bijgewerkt)"
"restoreSkippedEnvironmentLocationCiphertext": "Locatie van een omgevingsperiode of -dag versleuteld met een sleutel die deze server niet heeft (versleuteld behouden, die dagen worden niet bijgewerkt)",
"database": {
"title": "Databasedump",
"lastGood": "Laatste geldige dump {when}, {size}, {where}.",
"whereBoth": "bewaard op deze server en extern",
"whereLocal": "bewaard op deze server",
"whereOffhost": "alleen extern",
"none": "Nog geen databasedump. De eerste wordt vannacht gemaakt.",
"failedTitle": "De laatste databasedump is mislukt",
"failedDescription": "{when}: {reason}",
"disabled": "Databasedumps staan uit (DB_BACKUP_ENABLED=false). Maak op een andere manier een back-up van de database.",
"noDirectoryOffhost": "Deze server kan niet schrijven naar {dir}, dus dumps gaan alleen extern en hier wordt er geen bewaard. Koppel het back-upvolume.",
"noDirectory": "Deze server kan niet schrijven naar {dir} en er is geen externe opslag ingesteld, dus er kan geen databasedump worden gemaakt. Koppel het back-upvolume.",
"retentionLocal": "Verwijderde accounts blijven in de dumps tot die verlopen: op deze server blijven de nieuwste {nights} bewaard.",
"retentionOffhost": "De levenscyclusregel van de bucket verwijdert externe kopieën na {days} dagen.",
"retentionOffhostMissing": "Geen levenscyclusregel in de bucket geldt voor database/, dus dumps daar, en de verwijderde accounts erin, worden nooit verwijderd. Voeg een regel voor dat voorvoegsel toe."
}
},
"danger-zone": {
"title": "Gevarenzone",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Nazwa tagu lub grupy nastroju zaszyfrowana kluczem, którego ten serwer nie ma (zachowana bez nazwy)",
"restoreSkippedVisitCiphertext": "Tekst wizyty lub lekarza zaszyfrowany kluczem, którego ten serwer nie ma (zachowany bez niego)",
"restoreSkippedVaccinationCiphertext": "Notatka szczepienia zaszyfrowana kluczem, którego ten serwer nie ma (dawka zachowana bez niej)",
"restoreSkippedEnvironmentLocationCiphertext": "Lokalizacja okresu lub dnia środowiskowego zaszyfrowana kluczem, którego ten serwer nie ma (zachowana w postaci zaszyfrowanej, te dni nie są odświeżane)"
"restoreSkippedEnvironmentLocationCiphertext": "Lokalizacja okresu lub dnia środowiskowego zaszyfrowana kluczem, którego ten serwer nie ma (zachowana w postaci zaszyfrowanej, te dni nie są odświeżane)",
"database": {
"title": "Zrzut bazy danych",
"lastGood": "Ostatni poprawny zrzut {when}, {size}, {where}.",
"whereBoth": "przechowywany na tym serwerze i poza nim",
"whereLocal": "przechowywany na tym serwerze",
"whereOffhost": "tylko poza serwerem",
"none": "Nie ma jeszcze zrzutu bazy danych. Pierwszy powstanie tej nocy.",
"failedTitle": "Ostatni zrzut bazy danych się nie powiódł",
"failedDescription": "{when}: {reason}",
"disabled": "Zrzuty bazy danych są wyłączone (DB_BACKUP_ENABLED=false). Zrób kopię bazy danych w inny sposób.",
"noDirectoryOffhost": "Ten serwer nie może zapisywać w {dir}, więc zrzuty trafiają tylko poza serwer i żaden nie zostaje tutaj. Podłącz wolumin kopii.",
"noDirectory": "Ten serwer nie może zapisywać w {dir} i nie ustawiono magazynu zewnętrznego, więc nie da się wykonać zrzutu bazy. Podłącz wolumin kopii.",
"retentionLocal": "Usunięte konta pozostają w zrzutach, dopóki te nie wygasną: na tym serwerze zostaje {nights} najnowszych.",
"retentionOffhost": "Reguła cyklu życia bucketu usuwa kopie zewnętrzne po {days} dniach.",
"retentionOffhostMissing": "Żadna reguła cyklu życia bucketu nie obejmuje database/, więc zrzuty tam, wraz z usuniętymi kontami w nich, nigdy nie są usuwane. Dodaj regułę dla tego prefiksu."
}
},
"danger-zone": {
"title": "Strefa zagrożenia",
+17 -1
View File
@@ -8310,7 +8310,23 @@
"restoreSkippedMoodLabelCiphertext": "Etiqueta ou nome de grupo de humor selado com uma chave que este servidor não possui (mantido sem o nome)",
"restoreSkippedVisitCiphertext": "Texto de consulta ou profissional selado com uma chave que este servidor não possui (mantido sem ele)",
"restoreSkippedVaccinationCiphertext": "Nota de vacinação selada com uma chave que este servidor não possui (dose mantida sem ela)",
"restoreSkippedEnvironmentLocationCiphertext": "Local de um período ou dia de ambiente selado com uma chave que este servidor não possui (mantido selado, esses dias não são atualizados)"
"restoreSkippedEnvironmentLocationCiphertext": "Local de um período ou dia de ambiente selado com uma chave que este servidor não possui (mantido selado, esses dias não são atualizados)",
"database": {
"title": "Dump do banco de dados",
"lastGood": "Último dump válido {when}, {size}, {where}.",
"whereBoth": "guardado neste servidor e fora dele",
"whereLocal": "guardado neste servidor",
"whereOffhost": "somente fora do servidor",
"none": "Ainda não há dump do banco de dados. O primeiro é feito esta noite.",
"failedTitle": "O último dump do banco de dados falhou",
"failedDescription": "{when}: {reason}",
"disabled": "Os dumps do banco de dados estão desligados (DB_BACKUP_ENABLED=false). Faça o backup do banco de outra forma.",
"noDirectoryOffhost": "Este servidor não consegue gravar em {dir}, então os dumps vão só para fora e nenhum fica aqui. Monte o volume de backups.",
"noDirectory": "Este servidor não consegue gravar em {dir} e não há armazenamento externo configurado, então nenhum dump pode ser feito. Monte o volume de backups.",
"retentionLocal": "Contas excluídas continuam nos dumps até que eles expirem: este servidor guarda os {nights} mais recentes.",
"retentionOffhost": "A regra de ciclo de vida do bucket remove as cópias externas após {days} dias.",
"retentionOffhostMissing": "Nenhuma regra de ciclo de vida do bucket cobre database/, então os dumps ali, e as contas excluídas dentro deles, nunca são removidos. Adicione uma regra para esse prefixo."
}
},
"danger-zone": {
"title": "Zona de perigo",
@@ -0,0 +1,30 @@
-- The nightly whole-database dump records each run: whether a dump was
-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not.
-- Instance metadata with no account; the admin backups card and the backup
-- alerts read the newest rows.
--
-- The table is new, so nothing waits on its lock in practice; the wait is
-- bounded in a DO block anyway, and every statement is safe to run again, so
-- the boot retries the file on a lock timeout.
--
-- @retry-on-lock-timeout
DO $migration$
BEGIN
PERFORM set_config('lock_timeout', '5s', true);
CREATE TABLE IF NOT EXISTS "database_backup_runs" (
"id" TEXT NOT NULL,
"started_at" TIMESTAMP(3) NOT NULL,
"finished_at" TIMESTAMP(3),
"status" TEXT NOT NULL,
"file_name" TEXT,
"dump_bytes" BIGINT,
"local_result" TEXT NOT NULL,
"offhost_result" TEXT NOT NULL,
"error" TEXT,
CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id")
);
CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx"
ON "database_backup_runs"("started_at");
END
$migration$;
+29
View File
@@ -5929,6 +5929,35 @@ model OffhostPurgeRequest {
@@map("offhost_purge_requests")
}
/// One run of the nightly whole-database dump (the `database-backup` job):
/// whether a dump was written on this server and copied off-host, and why not
/// when it was not. Instance metadata, not an account's data. Written by
/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the
/// database row of the admin backups card and the backup alerts. A row left
/// `running` by a process that died is read as a failed run once the job's
/// expiry has passed.
model DatabaseBackupRun {
id String @id @default(cuid())
startedAt DateTime @map("started_at")
finishedAt DateTime? @map("finished_at")
/// `running`, `succeeded`, `failed` or `skipped`.
status String
/// The dump's file name, once one was written and checked.
fileName String? @map("file_name")
/// Size of the dump in bytes.
dumpBytes BigInt? @map("dump_bytes")
/// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable
/// directory, the dump went off-host only), or `failed`.
localResult String @map("local_result")
/// `uploaded`, `not_configured` or `failed`.
offhostResult String @map("offhost_result")
/// Why the run failed or was skipped, trimmed. Never key material.
error String?
@@index([startedAt])
@@map("database_backup_runs")
}
// ─── Apple Health export.zip import (v1.4.34) ───────────────
//
// Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the
+161
View File
@@ -0,0 +1,161 @@
/**
* The restore's refusals, which all come before it writes anything.
*
* The database is a stand-in here: `connect` answers a client whose only use
* is to be asked which tables exist, and `runTool` (which would run
* `pg_restore`) is a spy that must never be called on a refusal. The real
* restore into a real empty database, and its refusal of a real database with
* tables, are in `tests/integration/database-backup.test.ts` and in the
* compose drill (`scripts/compose-dr-drill.sh`).
*/
import { createHash } from "node:crypto";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const tablesInTarget = vi.fn(async (): Promise<string[]> => []);
const runTool = vi.fn(async () => "");
const end = vi.fn(async () => {});
vi.mock("../../src/lib/database-backup/dump-core.mjs", async (original) => ({
...(await original<Record<string, unknown>>()),
connect: async () => ({ query: async () => ({ rows: [] }), end }),
listUserTables: () => tablesInTarget(),
runTool: (...args: unknown[]) => runTool(...(args as [])),
}));
import { buildManifest } from "../../src/lib/database-backup/dump-core.mjs";
import { main } from "../db-restore.mjs";
const KEY_A = "a".repeat(64);
const KEY_B = "b".repeat(64);
const BASE = "healthlog-20261025T013000Z-daily";
let dir: string;
let errors: string[];
let lines: string[];
function env(extra: Record<string, string> = {}) {
return {
DATABASE_URL: "postgresql://healthlog:pw@db:5432/healthlog?schema=public",
DB_BACKUP_DIR: dir,
ENCRYPTION_KEY: KEY_A,
API_TOKEN_HMAC_KEY: "hmac",
...extra,
};
}
/** A dump file and a manifest that describes it, written with KEY_A. */
function writeDump(body = Buffer.from("PGDMP not really an archive")) {
writeFileSync(join(dir, `${BASE}.dump`), body);
const manifest = buildManifest({
kind: "daily",
file: `${BASE}.dump`,
createdAt: new Date("2026-10-25T01:30:00Z"),
appVersion: "1.46.0",
buildSha: null,
latestMigration: "0486_database_backup_runs",
server: { version: "18.6", versionNum: 180006 },
pgDump: null,
archive: {
bytes: body.length,
sha256: createHash("sha256").update(body).digest("hex"),
entries: 1,
},
rowCounts: { "public.users": 2 },
env: { ENCRYPTION_KEY: KEY_A, API_TOKEN_HMAC_KEY: "hmac" },
});
writeFileSync(
join(dir, `${BASE}.manifest.json`),
JSON.stringify(manifest, null, 2),
);
}
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), "hl-db-restore-"));
errors = [];
lines = [];
vi.spyOn(console, "error").mockImplementation((line: string) => {
errors.push(String(line));
});
vi.spyOn(console, "log").mockImplementation((line: string) => {
lines.push(String(line));
});
tablesInTarget.mockResolvedValue([]);
runTool.mockClear();
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
describe("db-restore refuses before it writes", () => {
it("refuses a target database that already holds tables", async () => {
writeDump();
tablesInTarget.mockResolvedValue(["public.users", "public.sessions"]);
expect(await main(["latest"], env())).toBe(1);
expect(errors.join("\n")).toMatch(
/already holds 2 tables .*only into an empty database/,
);
expect(runTool).not.toHaveBeenCalled();
});
it("refuses when the active encryption key is not the dump's, and names it", async () => {
writeDump();
expect(await main(["latest"], env({ ENCRYPTION_KEY: KEY_B }))).toBe(1);
expect(lines.join("\n")).toMatch(/KEY MISMATCH: ENCRYPTION_KEYS\[v1\]/);
expect(errors.join("\n")).toMatch(/refusing to restore/);
expect(runTool).not.toHaveBeenCalled();
});
it("goes past the key check with --ignore-key-mismatch, and still refuses a full database", async () => {
writeDump();
tablesInTarget.mockResolvedValue(["public.users"]);
expect(
await main(
["latest", "--ignore-key-mismatch"],
env({ ENCRYPTION_KEY: KEY_B }),
),
).toBe(1);
expect(errors.join("\n")).toMatch(/already holds 1 table /);
expect(runTool).not.toHaveBeenCalled();
});
it("refuses a dump that does not match its manifest", async () => {
writeDump();
writeFileSync(join(dir, `${BASE}.dump`), "PGDMP altered");
expect(await main(["latest"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/does not match its manifest/);
expect(runTool).not.toHaveBeenCalled();
});
it("refuses a dump without its manifest, and a file it did not write", async () => {
writeFileSync(join(dir, `${BASE}.dump`), "x");
expect(await main([`${BASE}.dump`], env())).toBe(1);
expect(errors.join("\n")).toMatch(/manifest.json is missing/);
writeFileSync(join(dir, "other.dump"), "x");
expect(await main(["other.dump"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/not a HealthLog database dump/);
expect(runTool).not.toHaveBeenCalled();
});
it("says when there is nothing to restore, and how it is used", async () => {
expect(await main(["latest"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/no restorable dump/);
expect(await main([], env())).toBe(2);
});
it("refuses --from-bucket without a configured bucket, before it connects", async () => {
expect(await main(["latest", "--from-bucket"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/--from-bucket needs BACKUP_S3_ENDPOINT/);
expect(runTool).not.toHaveBeenCalled();
});
it("lists what is there", async () => {
writeDump();
expect(await main(["list"], env())).toBe(0);
expect(lines.join("\n")).toContain(`${BASE}.dump daily`);
});
});
+35 -4
View File
@@ -21,7 +21,9 @@ ROOT=$(git rev-parse --show-toplevel)
# The last release whose compose file ran Postgres 16.
OLD_REF=${OLD_REF:-v1.45.0}
PROJECT=hl-db-upgrade-check
PORT=${CHECK_DB_PORT:-55416}
# A free port unless CHECK_DB_PORT names one, so the check never collides with
# another stack on the same machine.
PORT=${CHECK_DB_PORT:-$(node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})")}
WORK=$(mktemp -d)
export PRISMA_HIDE_UPDATE_MESSAGE=1
@@ -63,11 +65,40 @@ run_upgrade() {
}
upgrade_log() { docker logs healthlog-db-upgrade 2>&1 | grep '^db-upgrade:' || true; }
# The container can report healthy before its published port forwards (Docker
# Desktop and OrbStack set the forward up a moment later), and a migrate
# started in that gap failed with "Can't reach database server". So wait
# until the port answers, for at most thirty seconds, and retry the migrate
# within the same bound.
wait_for_port() {
local i=0
until node -e "
const s = require('node:net').connect(Number(process.argv[1]), '127.0.0.1');
s.on('connect', () => { s.end(); process.exit(0); });
s.on('error', () => process.exit(1));
" "$PORT" 2>/dev/null; do
i=$((i + 1))
[ "$i" -lt 30 ] || return 1
sleep 1
done
}
migrate_old() {
local started=$SECONDS
until DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do
if [ $((SECONDS - started)) -ge 30 ] || ! grep -q "P1001" "$WORK/migrate.log"; then
cat "$WORK/migrate.log" >&2
return 1
fi
sleep 1
done
}
down_all
step "a v1.45 database on Postgres 16"
old up -d --wait db >/dev/null 2>&1
DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >/dev/null
old up -d --wait db >"$WORK/up.log" 2>&1 || { cat "$WORK/up.log" >&2; fail "the v1.45 database did not start"; }
wait_for_port || fail "port $PORT did not answer within thirty seconds"
migrate_old || fail "the migrations did not apply to the v1.45 database"
old exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -q <<'SQL'
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
CREATE SCHEMA check_probe;
+346
View File
@@ -0,0 +1,346 @@
#!/usr/bin/env bash
# Plays the documented disaster recovery end to end against real containers:
# a server is lost with everything on it, and a new one is rebuilt from the
# nightly database dump and the keys alone (docs/ops/disaster-recovery.md).
#
# 1. Build the app image from this checkout and start the bundled compose
# stack with fixed throwaway keys, and an S3-compatible bucket outside
# it (Versity S3 Gateway, pinned) as the off-host target.
# 2. Seed real data: the demo seed (documents, encrypted notes, years of
# readings) and a Guardian with a managed profile, written by the app's
# own code; then, through the API, a reading with an encrypted note.
# 3. Let the scheduled `database-backup` job take the dump (its cron is set
# to every minute here) and wait for its ledger row: kept on the server
# and uploaded to the bucket.
# 4. DESTROY every volume of the stack, the local dumps included. The
# bucket lives outside the stack and survives, as an off-host copy does.
# 5. Start a new stack and restore with the keys alone:
# DRILL_SOURCE=bucket (the default, the disaster path)
# `docker compose run --rm db-restore latest --from-bucket`;
# DRILL_SOURCE=local (the dump files had been copied off the host)
# the files go into the new backups volume, then
# `docker compose run --rm db-restore latest`.
# 6. Count every table against the manifest, independently of the restore.
# 7. Boot the app on the restored database and check: the health endpoint,
# the demo login, the encrypted note read back through the API, a
# document opened, and the Guardian still seeing the managed profile.
#
# DRILL_MUTATION breaks one step on purpose, and the drill must then fail
# (a check that cannot fail proves nothing):
# wrong-key restore with a different active encryption key
# wrong-backup-key restore from the bucket with a different
# BACKUP_ENCRYPTION_KEY (DRILL_SOURCE=bucket)
# skip-table a dump whose archive lacks one table's rows, the
# manifest's checksum fixed up and its counts left true
# (DRILL_SOURCE=local)
# non-empty-target the new app starts and migrates before the restore
#
# Needs Docker, openssl, pnpm and the repo's node_modules (for the seeds). Run
# it from anywhere in the repo: `bash scripts/compose-dr-drill.sh`.
# DRILL_IMAGE names an image to use instead of building one. The ports are
# free ones picked at start unless DRILL_APP_PORT, DRILL_DB_PORT or
# DRILL_BUCKET_PORT name them.
set -euo pipefail
ROOT=$(git rev-parse --show-toplevel)
PROJECT=hl-dr-drill
free_port() {
node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})"
}
APP_PORT=${DRILL_APP_PORT:-$(free_port)}
DB_PORT=${DRILL_DB_PORT:-$(free_port)}
BUCKET_PORT=${DRILL_BUCKET_PORT:-$(free_port)}
IMAGE=${DRILL_IMAGE:-}
MUTATION=${DRILL_MUTATION:-}
SOURCE=${DRILL_SOURCE:-bucket}
OFFSITE=$PROJECT-offsite
BUCKET=$PROJECT-bucket
BUCKET_IMAGE=versity/versitygw:v1.8.0@sha256:30292fc2eeacc67a36993b01f7a7a5e3361a19cced0e80c1d71cfa2a4b0a2499
WORK=$(mktemp -d)
BASE_URL="http://127.0.0.1:$APP_PORT"
case "$SOURCE/$MUTATION" in
bucket/ | local/ | */wrong-key | */non-empty-target | bucket/wrong-backup-key | local/skip-table) ;;
*)
echo "DRILL_SOURCE=$SOURCE with DRILL_MUTATION=$MUTATION is not a drill" >&2
exit 2
;;
esac
export PRISMA_HIDE_UPDATE_MESSAGE=1
export POSTGRES_PASSWORD
POSTGRES_PASSWORD=$(openssl rand -hex 16)
# Fixed throwaway keys: the restore gets these and nothing else of the lost
# host. Two encryption keys, the second active, so a retired key is carried.
export ENCRYPTION_KEY=""
export ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"2222222222222222222222222222222222222222222222222222222222222222"}'
export ENCRYPTION_ACTIVE_KEY_ID=v2
export API_TOKEN_HMAC_KEY=3333333333333333333333333333333333333333333333333333333333333333
export SESSION_COOKIE_SECURE=false
export NEXT_PUBLIC_APP_URL=$BASE_URL APP_URL=$BASE_URL
export DB_BACKUP_SCHEDULE="* * * * *"
# The off-host bucket, reached from the stack by its container name on a
# network of its own that outlives the stack, as a real bucket does.
export BACKUP_S3_ENDPOINT=http://$BUCKET:7070
export BACKUP_S3_BUCKET=healthlog-backups
export BACKUP_S3_ACCESS_KEY=drill-access
export BACKUP_S3_SECRET_KEY=drill-secret-0123456789
export BACKUP_S3_REGION=us-east-1
export BACKUP_ENCRYPTION_KEY=5555555555555555555555555555555555555555555555555555555555555555
step() { echo "== $*"; }
fail() {
echo "FAIL: $*" >&2
docker logs "$PROJECT-app" 2>&1 | tail -40 >&2 || true
exit 1
}
if [ -z "$IMAGE" ]; then
IMAGE=healthlog-dr-drill:local
step "building the app image from this checkout"
docker build -q \
--build-arg NEXT_PUBLIC_APP_VERSION="drill-$(git -C "$ROOT" rev-parse --short HEAD)" \
--build-arg NEXT_PUBLIC_APP_BUILD_SHA="$(git -C "$ROOT" rev-parse --short HEAD)" \
-t "$IMAGE" "$ROOT" >/dev/null
fi
# The bundled compose file, with this image, its own names and ports, so the
# drill never meets another stack on the same machine, and the app and the
# restore on the bucket's network.
cat >"$WORK/drill.yml" <<YAML
services:
app:
image: $IMAGE
pull_policy: never
container_name: $PROJECT-app
ports: !override
- "127.0.0.1:$APP_PORT:3000"
networks: [default, bucket]
db:
container_name: $PROJECT-db
ports:
- "127.0.0.1:$DB_PORT:5432"
db-upgrade:
container_name: $PROJECT-db-upgrade
db-restore:
image: $IMAGE
pull_policy: never
networks: [default, bucket]
networks:
bucket:
external: true
name: $BUCKET
YAML
stack() { docker compose -p "$PROJECT" -f "$ROOT/docker-compose.yml" -f "$WORK/drill.yml" "$@"; }
sql() { docker exec "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$1"; }
teardown() {
stack down -v --remove-orphans >/dev/null 2>&1 || true
docker volume rm "$OFFSITE" >/dev/null 2>&1 || true
docker rm -f "$BUCKET" >/dev/null 2>&1 || true
docker volume rm "$BUCKET-data" >/dev/null 2>&1 || true
docker network rm "$BUCKET" >/dev/null 2>&1 || true
}
trap 'teardown; rm -rf "$WORK"' EXIT
wait_healthy() {
local i=0
until [ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/api/health")" = 200 ]; do
i=$((i + 1))
[ "$i" -lt 150 ] || fail "the app did not report healthy within five minutes"
docker inspect -f '{{.State.Running}}' "$PROJECT-app" 2>/dev/null | grep -q true ||
fail "the app container stopped"
sleep 2
done
}
# POST/GET as a signed-in account, with its own cookie jar.
login() {
local jar=$1 user=$2 password=$3
local code
code=$(curl -s -o "$WORK/login.json" -w '%{http_code}' -c "$jar" \
-H 'content-type: application/json' \
-d "{\"email\":\"$user\",\"password\":\"$password\"}" "$BASE_URL/api/auth/login")
[ "$code" = 200 ] || fail "login as $user answered $code: $(cat "$WORK/login.json")"
}
api() { curl -s -b "$1" -w '\n%{http_code}' "${@:2}"; }
json() { node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const v=JSON.parse(s);console.log((${1})(v))})"; }
# A shell in the app image as root, with volumes, for moving the dump files
# the way an operator copies them; `cp -p` keeps the app user as owner.
as_root() {
local command=$1 mounts=()
shift
for volume in "$@"; do mounts+=(-v "$volume"); done
docker run --rm --user 0:0 --entrypoint sh "${mounts[@]}" "$IMAGE" -c "$command"
}
teardown
step "an off-host bucket, outside the stack"
docker network create "$BUCKET" >/dev/null
docker run -d --name "$BUCKET" --network "$BUCKET" -p "127.0.0.1:$BUCKET_PORT:7070" \
-v "$BUCKET-data:/data" --entrypoint sh "$BUCKET_IMAGE" -c \
"mkdir -p /data/$BACKUP_S3_BUCKET && exec versitygw --access $BACKUP_S3_ACCESS_KEY --secret $BACKUP_S3_SECRET_KEY posix /data" \
>/dev/null || fail "the bucket did not start"
step "a running server with real data"
stack up -d app >"$WORK/up.log" 2>&1 || { tail -20 "$WORK/up.log" >&2; fail "the stack did not start"; }
wait_healthy
DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$DB_PORT/healthlog?schema=public"
export DATABASE_URL
(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/seed-demo.ts) >"$WORK/seed.log" 2>&1 ||
{ tail -20 "$WORK/seed.log" >&2; fail "the demo seed failed"; }
IDS=$(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/dr-drill-seed.ts | tail -1)
PROFILE_ID=$(echo "$IDS" | json 'v=>v.profileId')
unset DATABASE_URL
login "$WORK/demo.jar" demo demo123demo123
NOTE="drill note $(openssl rand -hex 6)"
OUT=$(api "$WORK/demo.jar" -H 'content-type: application/json' \
-d "{\"type\":\"WEIGHT\",\"value\":81.4,\"measuredAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"notes\":\"$NOTE\"}" \
"$BASE_URL/api/measurements")
[ "$(echo "$OUT" | tail -1)" = 201 ] || [ "$(echo "$OUT" | tail -1)" = 200 ] ||
fail "creating a reading with a note answered: $OUT"
READING_ID=$(echo "$OUT" | sed '$d' | json 'v=>v.data.id')
[ "$(sql "SELECT count(*) FROM measurements WHERE id = '$READING_ID' AND position('$NOTE' in measurements::text) > 0")" = 0 ] ||
fail "the note is stored in plain text"
DOCUMENTS=$(sql "SELECT count(*) FROM inbound_documents WHERE deleted_at IS NULL")
[ "$DOCUMENTS" -gt 0 ] || fail "the seed wrote no documents"
SEEDED_AT=$(sql "SELECT now()")
step "the scheduled job takes the dump and uploads it"
i=0
until DUMP=$(sql "SELECT file_name FROM database_backup_runs WHERE status = 'succeeded' AND local_result = 'written' AND offhost_result = 'uploaded' AND started_at > '$SEEDED_AT'::timestamptz ORDER BY started_at DESC LIMIT 1") && [ -n "$DUMP" ]; do
i=$((i + 1))
[ "$i" -lt 120 ] || fail "no succeeded and uploaded database-backup run within four minutes: $(sql "SELECT status, local_result, offhost_result, error FROM database_backup_runs ORDER BY started_at DESC LIMIT 3")"
sleep 2
done
BASE=${DUMP%.dump}
echo " $DUMP"
if [ "$SOURCE" = local ]; then
step "the dump files are copied off the host"
docker volume create "$OFFSITE" >/dev/null
as_root "cp -p /from/$BASE.dump /from/$BASE.manifest.json /to/" \
"${PROJECT}_backups:/from:ro" "$OFFSITE:/to" ||
fail "could not copy the dump off the host"
fi
step "the host is lost: every volume of the stack is destroyed"
stack down -v --remove-orphans >/dev/null 2>&1
[ -z "$(docker volume ls -q --filter "name=^${PROJECT}_")" ] || fail "a volume of the old stack survived"
if [ "$MUTATION" = skip-table ]; then
step "MUTATION: the dump loses the rows of public.measurements"
# A scratch Postgres restores the real dump and dumps it again without that
# table's rows; the files stay the app user's, read and written by root.
docker run --rm -v "$OFFSITE:/d" -e BASE="$BASE" --entrypoint sh postgres:18.6-alpine -ec '
gosu postgres initdb -D /tmp/pg >/dev/null
gosu postgres pg_ctl -D /tmp/pg -o "-c listen_addresses=" -w start >/dev/null
gosu postgres createdb scratch
gosu postgres pg_restore --no-owner --no-privileges -d scratch <"/d/$BASE.dump"
gosu postgres pg_dump -Fc --exclude-table-data=public.measurements scratch >"/d/$BASE.dump"
sum=$(sha256sum "/d/$BASE.dump" | cut -d" " -f1)
size=$(wc -c <"/d/$BASE.dump")
sed -i "s/\"sha256\": \"[0-9a-f]*\"/\"sha256\": \"$sum\"/; s/\"bytes\": [0-9]*/\"bytes\": $size/" "/d/$BASE.manifest.json"
chown 1001:1001 "/d/$BASE.dump"' || fail "could not build the mutated dump"
fi
step "a new server: the compose file, the keys and the $SOURCE copy"
if [ "$MUTATION" = wrong-key ]; then
echo " MUTATION: the restore gets a different active encryption key"
ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"4444444444444444444444444444444444444444444444444444444444444444"}'
fi
if [ "$MUTATION" = wrong-backup-key ]; then
echo " MUTATION: the restore gets a different BACKUP_ENCRYPTION_KEY"
BACKUP_ENCRYPTION_KEY=6666666666666666666666666666666666666666666666666666666666666666
fi
if [ "$SOURCE" = local ]; then
# Mounting the image's directory fills the new volume with the app user's
# ownership; then the two files go in.
stack run --rm --no-deps --entrypoint true db-restore >/dev/null 2>&1
as_root "cp -p /from/* /to/" "$OFFSITE:/from:ro" "${PROJECT}_backups:/to" ||
fail "could not put the dump on the new server"
RESTORE=(latest)
else
RESTORE=(latest --from-bucket)
fi
if [ "$MUTATION" = non-empty-target ]; then
echo " MUTATION: the new app starts (and migrates) before the restore"
stack up -d app >/dev/null 2>&1
wait_healthy
fi
stack up -d db >/dev/null 2>&1
if ! stack run --rm db-restore "${RESTORE[@]}" 2>&1 | tee "$WORK/restore.log"; then
fail "the restore failed"
fi
grep -q "^Restored $DUMP" "$WORK/restore.log" || fail "the restore did not say it restored $DUMP"
step "every table against the manifest, counted apart from the restore"
if [ "$SOURCE" = local ]; then
as_root "cat /d/$BASE.manifest.json" "$OFFSITE:/d:ro" >"$WORK/manifest.json"
else
# From the bucket, opened with the restore's own reader.
(cd "$ROOT" && node --input-type=module -e '
import { writeFileSync } from "node:fs";
import { join } from "node:path";
const core = await import(join(process.cwd(), "src/lib/database-backup/dump-core.mjs"));
const [port, base, work] = process.argv.slice(1);
const bucket = await core.openBucket({ ...core.bucketConfigFromEnv(process.env), endpoint: `http://127.0.0.1:${port}` });
const name = `${base}.manifest.json.enc`;
await bucket.download(core.offhostObjectKey(name), join(work, name));
await core.decryptOffhostFile({ inPath: join(work, name), outPath: join(work, "manifest.json"), objectKey: core.offhostObjectKey(name), keys: core.offhostKeysFromEnv(process.env) });
' "$BUCKET_PORT" "$BASE" "$WORK") || fail "could not read the manifest from the bucket"
fi
node -e '
const m = require(process.argv[1]);
const q = String.fromCharCode(39);
const lit = (t) => q + t.split(q).join(q + q) + q;
console.log(Object.keys(m.rowCounts).map((t) => `SELECT ${lit(t)} || chr(9) || count(*) FROM ${t}`).join(" UNION ALL ") + ";");
' "$WORK/manifest.json" >"$WORK/count.sql"
docker exec -i "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atq <"$WORK/count.sql" >"$WORK/counts.tsv"
node -e '
const fs = require("node:fs");
const m = require(process.argv[1]);
const got = Object.fromEntries(fs.readFileSync(process.argv[2], "utf8").trim().split("\n").map((l) => l.split("\t")).map(([t, n]) => [t, Number(n)]));
const bad = Object.entries(m.rowCounts).filter(([t, n]) => got[t] !== n);
if (bad.length) { console.error(bad.map(([t, n]) => `${t}: manifest ${n}, database ${got[t]}`).join("\n")); process.exit(1); }
if (m.rowCounts["public.users"] < 3) { console.error("fewer than three accounts in the dump"); process.exit(1); }
const c = m.rowCounts;
console.log(` ${Object.keys(c).length} tables, ${Object.values(c).reduce((a, b) => a + b, 0)} rows: ` +
`${c["public.users"]} accounts, ${c["public.account_grants"]} grants, ${c["public.measurements"]} readings, ` +
`${c["public.inbound_documents"]} documents, ${c["pgboss.job"] ?? 0} queued jobs`);
' "$WORK/manifest.json" "$WORK/counts.tsv" || fail "the restored row counts differ from the manifest"
step "the app on the restored database"
stack up -d app >/dev/null 2>&1 || fail "the app did not start on the restored database"
wait_healthy
[ "$(curl -s "$BASE_URL/api/health" | json 'v=>v.status ?? v.data?.status')" = ok ] || fail "health is not ok"
login "$WORK/demo2.jar" demo demo123demo123
OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/measurements/$READING_ID")
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "reading the note back answered: $OUT"
[ "$(echo "$OUT" | sed '$d' | json 'v=>v.data.notes')" = "$NOTE" ] || fail "the encrypted note did not read back"
echo " the encrypted note reads back"
OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/documents/inbound")
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing documents answered: $OUT"
DOC_ID=$(echo "$OUT" | sed '$d' | json 'v=>(v.data.documents ?? v.data.items ?? v.data)[0].id')
CODE=$(curl -s -b "$WORK/demo2.jar" -o "$WORK/doc.bin" -w '%{http_code}' "$BASE_URL/api/documents/inbound/$DOC_ID/original")
[ "$CODE" = 200 ] || fail "opening document $DOC_ID answered $CODE"
[ "$(head -c 4 "$WORK/doc.bin")" = "%PDF" ] || fail "document $DOC_ID did not open to a PDF"
echo " a document opens ($(wc -c <"$WORK/doc.bin" | tr -d ' ') bytes)"
login "$WORK/guardian.jar" drill-guardian drill-guardian-password-1
OUT=$(api "$WORK/guardian.jar" "$BASE_URL/api/account/grants")
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing the Guardian's grants answered: $OUT"
echo "$OUT" | sed '$d' | node -e "
let s=''; process.stdin.on('data', d => s += d).on('end', () => {
const received = JSON.parse(s).data.received ?? [];
const found = received.some((g) => JSON.stringify(g).includes('$PROFILE_ID'));
process.exit(found ? 0 : 1);
});" || fail "the Guardian no longer sees the managed profile"
echo " the Guardian still sees the managed profile"
echo "OK: the server was rebuilt from the $SOURCE copy of its dump and the keys alone"
+156
View File
@@ -0,0 +1,156 @@
/**
* The pre-migration dump: a whole-database dump the entrypoint takes before
* `prisma migrate deploy` applies anything.
*
* node scripts/db-dump.mjs pre-migration
*
* A migration is not undone by going back to the previous image, and the
* bundled compose file pulls `:latest` and migrates on every start. So when
* this image brings migrations the database has not applied, it first writes
* `healthlog-<timestamp>-pre-migration.dump` with its manifest into
* `DB_BACKUP_DIR` (`src/lib/database-backup/dump-core.mjs`), and keeps the
* newest two.
*
* Exit 0: the dump was written, or there was nothing to do (no pending
* migration, a new database, `DB_BACKUP_BEFORE_MIGRATE=false`), or there is no
* writable directory to put it in, which is said as a warning and is not a
* reason to keep the app down. Exit 1: the directory is writable and the dump
* failed. The entrypoint then stops the boot and names the switch that skips
* the dump, because migrating without the way back the operator was promised
* is the one thing this step exists to prevent.
*
* Fast on purpose: one snapshot, `pg_dump`, a count per table for the
* manifest, `pg_restore --list` to prove the file reads. Nothing else.
*/
import { readFileSync } from "node:fs";
import { basename, join } from "node:path";
import { pathToFileURL } from "node:url";
import {
applyRetention,
connect,
createDatabaseDump,
isWritableDirectory,
latestMigration,
listUserTables,
connectionFromUrl,
preMigrationDumpEnabled,
resolveBackupDir,
resolveDailyKeep,
} from "../src/lib/database-backup/dump-core.mjs";
import { pendingMigrations } from "./pending-migrations.mjs";
const log = (line) => console.log(`HealthLog: ${line}`);
const warn = (line) => console.error(`HealthLog: WARNING — ${line}`);
/** The image's version: the build argument, else package.json beside it. */
function imageVersion(env) {
const fromEnv = (env.NEXT_PUBLIC_APP_VERSION ?? "").trim();
if (fromEnv) return fromEnv;
try {
return JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8"))
.version;
} catch {
return null;
}
}
/** Megabytes as an operator reads them. */
function mb(bytes) {
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
}
export async function preMigrationDump(env = process.env) {
if (!preMigrationDumpEnabled(env)) {
log("DB_BACKUP_BEFORE_MIGRATE=false, no dump before migrating.");
return 0;
}
const databaseUrl = env.DATABASE_URL;
if (!databaseUrl) return 0;
let pending;
try {
pending = await pendingMigrations({
connectionString: databaseUrl,
migrationsDir: join(process.cwd(), "prisma", "migrations"),
});
} catch (error) {
warn(
`could not count pending migrations (${error instanceof Error ? error.message : String(error)}); no dump before migrating.`,
);
return 0;
}
if (pending.length === 0) return 0;
// A database no migration has touched holds nothing to lose.
const client = await connect(databaseUrl);
let applied;
let tables;
try {
applied = await latestMigration(
client,
connectionFromUrl(databaseUrl).schema,
);
tables = (await listUserTables(client)).length;
} finally {
await client.end().catch(() => {});
}
if (applied === null && tables === 0) {
log("new database, no dump before the first migrations.");
return 0;
}
const dir = resolveBackupDir(env);
if (!isWritableDirectory(dir)) {
warn(
`${dir} is not writable, so no dump is taken before the ${pending.length} pending migration(s). ` +
"Mount the backups volume (docker-compose.yml) to keep a way back; continuing.",
);
return 0;
}
log(
`taking a database dump before ${pending.length} pending migration(s) into ${dir}...`,
);
const started = Date.now();
try {
const { dumpPath, manifest } = await createDatabaseDump({
databaseUrl,
dir,
kind: "pre-migration",
env,
appVersion: imageVersion(env),
buildSha: (env.NEXT_PUBLIC_APP_BUILD_SHA ?? "").trim() || null,
});
log(
`pre-migration dump written: ${basename(dumpPath)} (${mb(manifest.dump.bytes)}, ` +
`${Object.keys(manifest.rowCounts).length} tables, ${Math.round((Date.now() - started) / 1000)} s).`,
);
} catch (error) {
console.error(
`HealthLog: ERROR — the database dump before migrating failed: ${error instanceof Error ? error.message : String(error)}`,
);
return 1;
}
try {
await applyRetention(dir, { dailyKeep: resolveDailyKeep(env) });
} catch (error) {
warn(
`could not remove old dumps in ${dir} (${error instanceof Error ? error.message : String(error)}).`,
);
}
return 0;
}
async function main() {
const kind = process.argv[2];
if (kind !== "pre-migration") {
console.error("db-dump: usage: node scripts/db-dump.mjs pre-migration");
process.exit(2);
}
process.exit(await preMigrationDump());
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
await main();
}
+415
View File
@@ -0,0 +1,415 @@
/**
* Restore a whole-database dump into an empty database.
*
* docker compose run --rm db-restore list [directory]
* docker compose run --rm db-restore list --from-bucket
* docker compose run --rm db-restore latest [--from-bucket]
* docker compose run --rm db-restore <file> [--ignore-key-mismatch]
*
* Runs in the app image with plain `node` (no `tsx`, no source checkout), as
* the `db-restore` compose service, so a new server needs only the compose
* file, the keys and the dump. `<file>` is a name in `DB_BACKUP_DIR` or a
* path inside the container: a `.dump` written on a server, or a `.dump.enc`
* downloaded from the off-host bucket (`database/` prefix), with its manifest
* (`.manifest.json` or `.manifest.json.enc`) beside it. With `--from-bucket`
* it fetches the pair from the bucket the `BACKUP_S3_*` variables name (the
* newest for `latest`), which is the restore after the host and its local
* dumps are gone.
*
* In this order, and it stops at the first thing that is wrong, before it
* writes anything:
* 1. an off-host copy is opened with BACKUP_ENCRYPTION_KEY (or a key in
* BACKUP_ENCRYPTION_PREVIOUS_KEYS) into a temporary file;
* 2. the dump's SHA-256 and size must equal the manifest's;
* 3. the keys in this environment are compared with the manifest's
* fingerprints; each one missing or different is named, and a mismatch
* of the active encryption key refuses (`--ignore-key-mismatch` restores
* anyway, for an operator who knows the key is lost);
* 4. the target database must hold no table at all. This never overwrites.
* Then `pg_restore` in one transaction, `ANALYZE`, and a count of every table
* against the manifest. A count that differs empties the database again and
* fails, so the app is never started on a restore that does not match.
*
* Exit 0 restored (or listed), 1 refused or failed, 2 usage.
*/
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, isAbsolute, join } from "node:path";
import { pathToFileURL } from "node:url";
import {
compareKeyFingerprints,
compareRowCounts,
connect,
countRows,
decryptOffhostFile,
emptyDatabase,
hashFile,
isGoodSet,
isWritableDirectory,
keyFingerprints,
listDumpSets,
listUserTables,
offhostKeysFromEnv,
offhostObjectKey,
parseDumpFileName,
parseManifest,
resolveBackupDir,
runTool,
statOrNull,
connectionFromUrl,
bucketConfigFromEnv,
openBucket,
pickBucketDump,
OFFHOST_DATABASE_PREFIX,
} from "../src/lib/database-backup/dump-core.mjs";
import { shippedMigrations } from "./pending-migrations.mjs";
const out = (line = "") => console.log(line);
const say = (line) => console.log(`db-restore: ${line}`);
class Refusal extends Error {}
function mb(bytes) {
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
}
/**
* Whether this image knows the dump's newest migration. A dump from a newer
* release restored under an older image leaves the app with a schema its code
* does not expect, so the restore says which image to run.
*/
function imageKnowsMigration(name) {
if (!name) return true;
try {
return shippedMigrations(
join(process.cwd(), "prisma", "migrations"),
).includes(name);
} catch {
return true;
}
}
function listCommand(dir) {
const sets = listDumpSets(dir);
if (sets.length === 0) {
say(`no dumps in ${dir}`);
return 0;
}
out(`Dumps in ${dir}, newest first:`);
for (const set of sets) {
const state = isGoodSet(set)
? `${mb(set.dump.bytes)}`
: set.encryptedDump
? "off-host copy (.enc)"
: set.dump
? "no manifest, not restorable"
: "incomplete";
out(
` ${set.base}.dump${set.encryptedDump && !set.dump ? ".enc" : ""} ${set.kind} ${set.timestamp.toISOString()} ${state}`,
);
}
return 0;
}
/**
* The dump and manifest to restore: a local pair, or an encrypted pair
* opened into `scratch`.
*/
async function resolveInput(arg, dir, env) {
if (arg === "latest") {
const set = listDumpSets(dir).find(
(s) =>
isGoodSet(s) ||
(s.encryptedDump !== null &&
(s.encryptedManifest !== null || s.manifest !== null)),
);
if (!set) throw new Refusal(`no restorable dump in ${dir}`);
arg = set.dump ? set.dump.path : set.encryptedDump;
}
const path = isAbsolute(arg) ? arg : join(dir, arg);
if (!(await statOrNull(path))) throw new Refusal(`${path} does not exist`);
const name = basename(path);
const parsed = parseDumpFileName(name);
if (!parsed || parsed.role !== "dump" || parsed.variant === "partial") {
throw new Refusal(
`${name} is not a HealthLog database dump (healthlog-<time>-<kind>.dump or .dump.enc)`,
);
}
const folder = dirname(path);
const plainManifest = join(folder, `${parsed.base}.manifest.json`);
const encManifest = `${plainManifest}.enc`;
if (parsed.variant === "final") {
if (!(await statOrNull(plainManifest))) {
throw new Refusal(
`${basename(plainManifest)} is missing beside ${name}; a dump is restored only with its manifest`,
);
}
return {
dumpPath: path,
manifest: parseManifest(
JSON.parse(await readFile(plainManifest, "utf8")),
),
scratch: null,
};
}
// An off-host copy. Open it next to the backups if there is room to write
// there, in the container's temp directory otherwise.
const keys = offhostKeysFromEnv(env);
if (keys.length === 0) {
throw new Refusal(
"this is an off-host copy and BACKUP_ENCRYPTION_KEY is not set; it cannot be opened without it",
);
}
const scratch = await mkdtemp(
join(isWritableDirectory(dir) ? dir : tmpdir(), ".restore-"),
);
try {
let manifest;
if (await statOrNull(encManifest)) {
const opened = join(scratch, `${parsed.base}.manifest.json`);
await decryptOffhostFile({
inPath: encManifest,
outPath: opened,
objectKey: offhostObjectKey(basename(encManifest)),
keys,
});
manifest = parseManifest(JSON.parse(await readFile(opened, "utf8")));
} else if (await statOrNull(plainManifest)) {
manifest = parseManifest(
JSON.parse(await readFile(plainManifest, "utf8")),
);
} else {
throw new Refusal(
`${basename(encManifest)} is missing beside ${name}; download it from the bucket too`,
);
}
say(`opening the off-host copy ${name}...`);
const dumpPath = join(scratch, `${parsed.base}.dump`);
await decryptOffhostFile({
inPath: path,
outPath: dumpPath,
objectKey: offhostObjectKey(name),
keys,
});
return { dumpPath, manifest, scratch };
} catch (error) {
await rm(scratch, { recursive: true, force: true });
throw error instanceof Refusal ? error : new Refusal(error.message);
}
}
/**
* Fetch a dump and its manifest from the off-host bucket (`BACKUP_S3_*`) into
* a scratch directory: the newest pair for `latest`, the named one otherwise.
* Answers the downloaded `.dump.enc` and the directory to remove afterwards.
*/
async function downloadFromBucket(arg, dir, env) {
const config = bucketConfigFromEnv(env);
if (!config) {
throw new Refusal(
"--from-bucket needs BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY and BACKUP_S3_SECRET_KEY",
);
}
const bucket = await openBucket(config);
const keys = await bucket.list(OFFHOST_DATABASE_PREFIX);
const pick = pickBucketDump(keys, arg === "latest" ? null : basename(arg));
if (!pick) {
throw new Refusal(
arg === "latest"
? `no database dump with its manifest under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`
: `${basename(arg)} and its manifest are not both under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`,
);
}
const scratch = await mkdtemp(
join(isWritableDirectory(dir) ? dir : tmpdir(), ".download-"),
);
try {
say(
`downloading ${pick.dumpKey} and its manifest from ${config.bucket}...`,
);
const dumpPath = join(scratch, basename(pick.dumpKey));
await bucket.download(
pick.manifestKey,
join(scratch, basename(pick.manifestKey)),
);
await bucket.download(pick.dumpKey, dumpPath);
return { dumpPath, scratch };
} catch (error) {
await rm(scratch, { recursive: true, force: true });
throw new Refusal(
`could not download from the bucket: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
async function listBucketCommand(env) {
const config = bucketConfigFromEnv(env);
if (!config)
throw new Refusal("--from-bucket needs the BACKUP_S3_* variables");
const keys = (await (await openBucket(config)).list(OFFHOST_DATABASE_PREFIX))
.filter((k) => k.endsWith(".dump.enc"))
.sort()
.reverse();
out(
`Dumps in bucket ${config.bucket} under ${OFFHOST_DATABASE_PREFIX}, newest first:`,
);
for (const key of keys) out(` ${key}`);
if (keys.length === 0) say("none");
return 0;
}
async function restoreCommand(arg, flags, env) {
const dir = resolveBackupDir(env);
const databaseUrl = env.DATABASE_URL;
if (!databaseUrl) throw new Refusal("DATABASE_URL is not set");
let input = arg;
let download = null;
if (flags.has("--from-bucket")) {
download = await downloadFromBucket(arg, dir, env);
input = download.dumpPath;
}
let resolved;
try {
resolved = await resolveInput(input, dir, env);
} finally {
// The sealed copy is not needed once it is opened (or failed to open).
if (download) await rm(download.scratch, { recursive: true, force: true });
}
const { dumpPath, manifest, scratch } = resolved;
try {
say(
`dump ${manifest.file}: taken ${manifest.createdAt} by HealthLog ${manifest.appVersion ?? "(unknown version)"}, ` +
`newest migration ${manifest.latestMigration ?? "(none)"}, Postgres ${manifest.postgres.serverVersion}, ` +
`${Object.keys(manifest.rowCounts).length} tables`,
);
if (!imageKnowsMigration(manifest.latestMigration)) {
say(
`warning: this image does not know the dump's newest migration ${manifest.latestMigration}; ` +
`start the app with HealthLog ${manifest.appVersion ?? "the version that wrote it"} or newer (HEALTHLOG_IMAGE_REF), not this image`,
);
}
if (basename(dumpPath) !== manifest.file) {
throw new Refusal(
`the manifest describes ${manifest.file}, not ${basename(dumpPath)}`,
);
}
// 2. The file is the one the manifest describes.
const { sha256, bytes } = await hashFile(dumpPath);
if (sha256 !== manifest.dump.sha256 || bytes !== manifest.dump.bytes) {
throw new Refusal(
`the dump does not match its manifest (SHA-256 ${sha256.slice(0, 12)}…, ${bytes} bytes; ` +
`the manifest says ${manifest.dump.sha256.slice(0, 12)}…, ${manifest.dump.bytes} bytes). It is damaged or not this dump.`,
);
}
say(`checksum matches the manifest (${mb(bytes)})`);
// 3. The keys.
const keys = compareKeyFingerprints(manifest.keys, keyFingerprints(env));
if (keys.findings.length === 0) {
say("every key the dump was written with is configured here");
}
for (const finding of keys.findings) {
say(
`${finding.severity === "refuse" ? "KEY MISMATCH" : "warning"}: ${finding.key}: ${finding.detail}`,
);
}
if (keys.refuse && !flags.has("--ignore-key-mismatch")) {
throw new Refusal(
"refusing to restore: set the encryption keys this dump was written with (see above). " +
"Only if that key is lost for good, run again with --ignore-key-mismatch; everything encrypted will stay unreadable.",
);
}
// 4. Never over existing data.
const client = await connect(databaseUrl);
try {
const tables = await listUserTables(client);
if (tables.length > 0) {
throw new Refusal(
`the target database already holds ${tables.length} ${tables.length === 1 ? "table" : "tables"} (${tables.slice(0, 3).join(", ")}${tables.length > 3 ? ", …" : ""}). ` +
"A restore goes only into an empty database: start it on a new, empty database volume, and do not start the app before the restore.",
);
}
say("restoring (one transaction)...");
const started = Date.now();
await runTool(
"pg_restore",
[
"--no-owner",
"--no-privileges",
"--single-transaction",
"--exit-on-error",
"--no-password",
`--dbname=${connectionFromUrl(databaseUrl).pgEnv.PGDATABASE}`,
dumpPath,
],
{ env: connectionFromUrl(databaseUrl).pgEnv },
);
say(`restored in ${Math.round((Date.now() - started) / 1000)} s`);
await client.query("ANALYZE");
say("planner statistics gathered");
const problems = compareRowCounts(
manifest.rowCounts,
await countRows(client),
);
if (problems.length > 0) {
for (const line of problems.slice(0, 20)) say(`row count: ${line}`);
await emptyDatabase(client);
throw new Refusal(
`${problems.length} table(s) do not match the manifest; the database was emptied again. Do not start the app on it.`,
);
}
say(
`all ${Object.keys(manifest.rowCounts).length} tables hold the row counts the manifest names`,
);
} finally {
await client.end().catch(() => {});
}
out();
out(
`Restored ${manifest.file}. Start the app with the image this dump came from or a newer one ` +
`(HealthLog ${manifest.appVersion ?? "?"}, newest migration ${manifest.latestMigration ?? "none"}): docker compose up -d`,
);
return 0;
} finally {
if (scratch) await rm(scratch, { recursive: true, force: true });
}
}
export async function main(argv = process.argv.slice(2), env = process.env) {
const flags = new Set(argv.filter((a) => a.startsWith("--")));
const [command] = argv.filter((a) => !a.startsWith("--"));
if (!command) {
console.error(
"db-restore: usage: db-restore list [directory] | latest | <file> [--from-bucket] [--ignore-key-mismatch]",
);
return 2;
}
try {
if (command === "list" && flags.has("--from-bucket")) {
return await listBucketCommand(env);
}
if (command === "list") {
const [, other] = argv.filter((a) => !a.startsWith("--"));
return listCommand(other ?? resolveBackupDir(env));
}
return await restoreCommand(command, flags, env);
} catch (error) {
console.error(
`db-restore: ${error instanceof Refusal ? "" : "failed: "}${error instanceof Error ? error.message : String(error)}`,
);
return 1;
}
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
process.exit(await main());
}
+59
View File
@@ -0,0 +1,59 @@
/**
* The accounts the disaster-recovery drill adds beside the demo seed: a
* Guardian with a password, and a managed profile that Guardian administers.
* Written through the app's own code (`createManagedProfile`,
* `hashPassword`), so the rows are the ones a person creating them in the app
* would leave.
*
* DATABASE_URL=… pnpm dlx tsx scripts/dr-drill-seed.ts
*
* Prints one JSON line: `{ "guardianId": …, "profileId": … }`. The drill
* (`scripts/compose-dr-drill.sh`) checks after the restore that the Guardian
* still sees that profile. Idempotent: a second run answers the same ids.
*/
import { hashPassword } from "@/lib/auth/password";
import { prisma } from "@/lib/db";
import { createManagedProfile } from "@/lib/managed-profiles/create";
const GUARDIAN = "drill-guardian";
const PASSWORD = "drill-guardian-password-1";
async function main(): Promise<void> {
let guardian = await prisma.user.findUnique({
where: { username: GUARDIAN },
});
guardian ??= await prisma.user.create({
data: {
username: GUARDIAN,
email: "drill-guardian@example.test",
passwordHash: await hashPassword(PASSWORD),
displayName: "Drill Guardian",
locale: "en",
timezone: "Europe/Berlin",
onboardingCompletedAt: new Date(),
},
});
const existing = await prisma.accountGrant.findFirst({
where: { granteeId: guardian.id, access: "MANAGE" },
select: { grantorId: true },
});
const profileId =
existing?.grantorId ??
(
await createManagedProfile({
creatorId: guardian.id,
displayName: "Drill Child",
dateOfBirth: new Date("2018-05-04T00:00:00Z"),
locale: "en",
timezone: "Europe/Berlin",
})
).profile.id;
console.log(JSON.stringify({ guardianId: guardian.id, profileId }));
}
main()
.catch((error: unknown) => {
console.error(error);
process.exitCode = 1;
})
.finally(() => prisma.$disconnect());
+32
View File
@@ -360,6 +360,38 @@
}
]
},
{
"name": "Database dumps",
"description": "The nightly whole-database dump and the dump before migrations (docs/ops/disaster-recovery.md). Optional: each has a default, and the bundled compose file mounts the `backups` volume at the default directory.",
"required": false,
"variables": [
{
"name": "DB_BACKUP_ENABLED",
"purpose": "Set false to stop the nightly database dump. On when unset.",
"example": "true"
},
{
"name": "DB_BACKUP_DIR",
"purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. Without a writable directory the nightly dump only goes off-host and no dump is taken before migrations.",
"example": "/var/lib/healthlog/backups"
},
{
"name": "DB_BACKUP_SCHEDULE",
"purpose": "Five-field cron for the nightly dump, in Europe/Berlin time like every scheduled job. Default 30 1 * * * (01:30).",
"example": "30 1 * * *"
},
{
"name": "DB_BACKUP_KEEP",
"purpose": "How many nightly dumps stay in DB_BACKUP_DIR (default 7, at least 1). The last two dumps taken before a migration are kept besides.",
"example": "7"
},
{
"name": "DB_BACKUP_BEFORE_MIGRATE",
"purpose": "Set false to start without the dump the entrypoint takes before applying migrations. On when unset; a failed dump stops the start until it works or this is false.",
"example": "true"
}
]
},
{
"name": "Operational windows and cookie flags",
"description": "Values the product states back to a user or that decide whether a session cookie is accepted. Optional: each has a documented default. They are listed so `pnpm check-env` reports them, and so the compose whitelist and the example files can be held in step with this manifest.",
+6 -7
View File
@@ -1,12 +1,11 @@
/**
* Boot-time notice before `prisma migrate deploy`: how many migrations this
* image is about to apply, and a reminder to have a backup first.
* image is about to apply, and that a dump is taken first
* (`scripts/db-dump.mjs`, which imports `pendingMigrations` from here).
*
* Run by `docker-entrypoint.sh` as `node scripts/pending-migrations.mjs`.
* It only reports. A migration that changes data cannot be undone by pulling
* the previous image, so an operator who reads the count in the log still has
* a moment to stop the container and take a `pg_dump`; the entrypoint does
* not wait for that.
* the previous image; the dump the entrypoint takes next is the way back.
*
* Best effort by design: any failure to count (no `_prisma_migrations` table
* yet is not a failure, it means every migration is pending) prints one line
@@ -67,9 +66,9 @@ export function pendingNotice(count) {
const noun = count === 1 ? "migration" : "migrations";
return (
`HealthLog: ${count} database ${noun} pending. ` +
"Back up the database before updating (pg_dump, see the Updates section " +
"of docs/self-hosting/getting-started.md); a migration is not undone by " +
"going back to the previous image."
"Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is " +
"written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); " +
"a migration is not undone by going back to the previous image."
);
}
@@ -58,7 +58,9 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
it("keeps the new data where the Postgres 18 image expects it", () => {
// Mounted at .../data, as 16 had it, the 18 image refuses to start.
expect(db.volumes).toEqual(["postgres-data:/var/lib/postgresql"]);
// `backups` holds the app's database dumps, not database files.
expect(Object.keys(compose.volumes).sort()).toEqual([
"backups",
"pgdata",
"postgres-data",
]);
@@ -54,9 +54,29 @@ vi.mock("@/lib/jobs/offhost-backup", () => ({
probeOffhostLifecycle: async () => ({
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
}),
}));
const databaseHealth = {
enabled: true,
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
offhostConfigured: true,
lastGood: {
at: "2026-09-21T23:31:40.000Z",
fileName: "healthlog-20260921T233000Z-daily.dump",
sizeBytes: 52_428_800,
local: true,
offhost: true,
},
lastFailure: null,
};
vi.mock("@/lib/database-backup/health", () => ({
getDatabaseBackupHealth: async () => databaseHealth,
}));
import { GET } from "../route";
import type { BackupsList } from "@/types/backups";
@@ -166,7 +186,11 @@ describe("GET /api/admin/backups — off-host freshness", () => {
periodHours: 24,
rows: [],
// No bucket to ask on a host that is not configured.
lifecycle: { state: "unknown", expirationDays: null },
lifecycle: {
state: "unknown",
expirationDays: null,
database: { state: "unknown", expirationDays: null },
},
});
});
@@ -229,6 +253,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
expect(body.offhost.lifecycle).toEqual({
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
});
expect(body.offhost.pendingDeletions).toEqual({
count: 2,
@@ -285,3 +310,10 @@ describe("GET /api/admin/backups — a run in progress and one a process died un
});
});
});
describe("GET /api/admin/backups — the database dump", () => {
it("carries the dump's health beside the account copies", async () => {
const body = await read();
expect(body.database).toEqual(databaseHealth);
});
});
+10 -4
View File
@@ -14,6 +14,10 @@
* stopped six weeks ago — every row it lists has a perfectly ordinary
* timestamp either way.
*
* The database row comes first in what an operator needs and last in this
* file: the nightly whole-database dump (`getDatabaseBackupHealth`), the
* one copy that rebuilds a lost server, read from its own ledger.
*
* And alongside both, the off-host leg: one row per account saying when this
* host last put that account's encrypted copy in the operator's bucket and how
* stale that is against the nightly schedule. It comes from the ledger the
@@ -31,11 +35,13 @@ import {
} from "@/lib/jobs/job-failures";
import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
import {
offhostBackupConfigured,
probeOffhostLifecycle,
} from "@/lib/jobs/offhost-backup";
import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue";
import { LIFECYCLE_UNKNOWN } from "@/lib/jobs/offhost-lifecycle";
import {
classifyOffhostBackup,
OFFHOST_BACKUP_PERIOD_HOURS,
@@ -190,16 +196,15 @@ export const GET = apiHandler(async () => {
readBackupPassActivity(prisma, DATA_BACKUP_QUEUE, weeklyRunningSince),
readBackupPassActivity(prisma, OFFHOST_BACKUP_QUEUE, offhostRunningSince),
]);
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
const [purgeCount, oldestPurge, lifecycle, database] = await Promise.all([
prisma.offhostPurgeRequest.count({ where: stillInBucket }),
prisma.offhostPurgeRequest.findFirst({
where: stillInBucket,
orderBy: { requestedAt: "asc" },
select: { requestedAt: true, lastFailure: true },
}),
configured
? probeOffhostLifecycle()
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
configured ? probeOffhostLifecycle() : Promise.resolve(LIFECYCLE_UNKNOWN),
getDatabaseBackupHealth(now),
]);
const payload: BackupsList = {
@@ -229,6 +234,7 @@ export const GET = apiHandler(async () => {
},
lifecycle,
},
database,
};
return apiSuccess(payload);
@@ -26,7 +26,11 @@ import { OffhostBackupsSection } from "../offhost-backups-section";
const QUIET = {
activity: { runningSince: null, interrupted: [] },
pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null },
lifecycle: { state: "configured" as const, expirationDays: 30 },
lifecycle: {
state: "configured" as const,
expirationDays: 30,
database: { state: "configured" as const, expirationDays: 30 },
},
};
function render(data: BackupsList["offhost"] | null): string {
@@ -133,7 +137,11 @@ describe("<OffhostBackupsSection>", () => {
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
lastFailure: "AccessDenied",
},
lifecycle: { state: "missing", expirationDays: null },
lifecycle: {
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
},
});
expect(html).toContain('data-lifecycle-state="missing"');
expect(html).toContain("no lifecycle rule");
+9
View File
@@ -28,6 +28,7 @@ import type { BackupRow, BackupsList } from "@/types/backups";
import { RestoreJobStatus, visibleRestoreJobs } from "./backup-restore-status";
import { BackupRowsCards, BackupRowsTable } from "./backups/backup-rows";
import { DatabaseDumpRow } from "./backups/database-dump-row";
import { BackupUploadCard } from "./backups/backup-upload-card";
import {
RestoreRefusalNotice,
@@ -110,6 +111,14 @@ export function BackupsSection() {
}
/>
{data?.database ? (
<DatabaseDumpRow
health={data.database}
bucketLifecycle={
data.offhost.configured ? data.offhost.lifecycle.database : null
}
/>
) : null}
{data?.schedule ? (
<ScheduleHealthNotice schedule={data.schedule} />
) : null}
@@ -0,0 +1,93 @@
/**
* The database row on the backups card says where the newest good dump is,
* how long a deleted account stays in the dumps, and each thing that keeps a
* dump from being taken or from ever ageing out of the bucket.
*/
import { describe, expect, it } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
import { I18nProvider } from "@/lib/i18n/context";
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
import { DatabaseDumpRow } from "../database-dump-row";
const HEALTHY: DatabaseBackupHealth = {
enabled: true,
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
offhostConfigured: true,
lastGood: {
at: "2026-10-24T23:31:00.000Z",
fileName: "healthlog-20261024T233000Z-daily.dump",
sizeBytes: 52_428_800,
local: true,
offhost: true,
},
lastFailure: null,
};
function render(
health: DatabaseBackupHealth,
bucketLifecycle: LifecycleVerdict | null = {
state: "configured",
expirationDays: 30,
},
): string {
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<DatabaseDumpRow health={health} bucketLifecycle={bucketLifecycle} />
</I18nProvider>,
);
}
describe("<DatabaseDumpRow>", () => {
it("names the newest good dump, where it is, and how long dumps keep deleted accounts", () => {
const html = render(HEALTHY);
expect(html).toContain("kept on this server and off-host");
expect(html).toContain("the newest 7 are kept on this server");
expect(html).toContain("removed after 30 days");
expect(html).not.toContain('data-slot="database-dump-failed"');
expect(html).not.toContain('data-slot="database-dump-setup"');
expect(html).not.toContain('data-slot="database-dump-no-lifecycle"');
});
it("warns when no bucket rule ever removes the dumps", () => {
const html = render(HEALTHY, { state: "missing", expirationDays: null });
expect(html).toContain('data-slot="database-dump-no-lifecycle"');
expect(html).toContain("never removed");
});
it("shows a failure newer than the last good dump, and not an older one", () => {
const failed = render({
...HEALTHY,
lastFailure: {
at: "2026-10-25T23:31:00.000Z",
error: "pg_dump exited 1: No space left on device",
},
});
expect(failed).toContain('data-slot="database-dump-failed"');
expect(failed).toContain("No space left on device");
const recovered = render({
...HEALTHY,
lastFailure: { at: "2026-10-20T23:31:00.000Z", error: "old" },
});
expect(recovered).not.toContain('data-slot="database-dump-failed"');
});
it("says when dumps go off-host only, or cannot be taken at all", () => {
expect(render({ ...HEALTHY, localDirWritable: false }, null)).toContain(
"dumps go off-host only",
);
expect(
render(
{ ...HEALTHY, localDirWritable: false, offhostConfigured: false },
null,
),
).toContain("no database dump can be taken");
expect(render({ ...HEALTHY, lastGood: null }, null)).toContain(
"No database dump yet",
);
});
});
@@ -0,0 +1,148 @@
"use client";
/**
* The nightly whole-database dump, as one row on the backups card.
*
* The account copies below it restore one account on this server; the dump is
* the copy that rebuilds the server. So the row leads the card and says three
* things: when the newest good dump was taken and where it is, whether the
* newest run failed, and whether this server can keep a dump at all.
*/
import { AlertTriangle, HardDrive } from "lucide-react";
import { SettingsInfoTile } from "@/components/settings/_info-tile";
import { formatBytes } from "@/lib/format/bytes";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
function whereKept(
good: NonNullable<DatabaseBackupHealth["lastGood"]>,
t: ReturnType<typeof useTranslations>["t"],
) {
if (good.local && good.offhost) {
return t("admin.section.backups.database.whereBoth");
}
if (good.offhost) return t("admin.section.backups.database.whereOffhost");
return t("admin.section.backups.database.whereLocal");
}
export function DatabaseDumpRow({
health,
bucketLifecycle,
}: {
health: DatabaseBackupHealth;
/** The bucket's rule for `database/`; null when nothing goes off-host. */
bucketLifecycle: LifecycleVerdict | null;
}) {
const { t, locale } = useTranslations();
const fmt = useFormatters();
const good = health.lastGood;
const failure = health.lastFailure;
const failedSinceGood =
failure !== null && (good === null || failure.at > good.at);
let setupProblem: string | null = null;
if (!health.enabled) {
setupProblem = t("admin.section.backups.database.disabled");
} else if (!health.localDirWritable) {
setupProblem = health.offhostConfigured
? t("admin.section.backups.database.noDirectoryOffhost", {
dir: health.directory,
})
: t("admin.section.backups.database.noDirectory", {
dir: health.directory,
});
}
return (
<div className="space-y-2" data-slot="database-dump">
<div className="flex items-start gap-2">
<HardDrive
className="text-muted-foreground mt-0.5 h-4 w-4 shrink-0"
aria-hidden="true"
/>
<div>
<p className="text-sm font-medium">
{t("admin.section.backups.database.title")}
</p>
<p
className="text-muted-foreground text-xs"
data-slot="database-dump-last-good"
>
{good
? t("admin.section.backups.database.lastGood", {
when: fmt.dateTime(good.at),
size: formatBytes(good.sizeBytes ?? 0, locale),
where: whereKept(good, t),
})
: t("admin.section.backups.database.none")}
</p>
{/* How long a deleted account stays in the dumps: the operator's
side of the deletion promise (GDPR retention of backups). */}
<p
className="text-muted-foreground text-xs"
data-slot="database-dump-retention"
>
{t("admin.section.backups.database.retentionLocal", {
nights: health.keepNightly,
})}
{bucketLifecycle?.state === "configured"
? ` ${t("admin.section.backups.database.retentionOffhost", {
days: bucketLifecycle.expirationDays ?? 0,
})}`
: null}
</p>
</div>
</div>
{failedSinceGood && failure ? (
<div
role="alert"
data-slot="database-dump-failed"
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<AlertTriangle
className="mt-0.5 h-4 w-4 shrink-0"
aria-hidden="true"
/>
<div>
<p className="font-medium">
{t("admin.section.backups.database.failedTitle")}
</p>
<p className="text-xs">
{t("admin.section.backups.database.failedDescription", {
when: fmt.dateTime(failure.at),
reason: failure.error,
})}
</p>
</div>
</div>
</div>
) : null}
{bucketLifecycle?.state === "missing" ? (
<SettingsInfoTile
icon={AlertTriangle}
tone="warning"
role="alert"
data-slot="database-dump-no-lifecycle"
>
{t("admin.section.backups.database.retentionOffhostMissing")}
</SettingsInfoTile>
) : null}
{setupProblem ? (
<SettingsInfoTile
icon={AlertTriangle}
tone="warning"
role="alert"
data-slot="database-dump-setup"
>
{setupProblem}
</SettingsInfoTile>
) : null}
</div>
);
}
@@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => {
expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined();
});
});
describe("collectReadiness — the database dump", () => {
const offhost = {
BACKUP_S3_ENDPOINT: "https://s3.example.test",
BACKUP_S3_BUCKET: "b",
BACKUP_S3_ACCESS_KEY: "a",
BACKUP_S3_SECRET_KEY: "s",
BACKUP_ENCRYPTION_KEY: HEX32,
};
const dumpLine = (
env: Record<string, string | undefined>,
writable: boolean,
) =>
collectReadiness(env, () => writable).lines.find(
(l) => l.label === "Database dump",
);
it("names the schedule, the directory and the copies kept", () => {
const line = dumpLine(baseEnv(), true);
expect(line?.status).toBe("ok");
expect(line?.detail).toBe(
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
);
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
"copied off-host",
);
});
it("warns when nothing can be kept on the server", () => {
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
status: "warn",
detail: expect.stringContaining("goes off-host only"),
});
expect(dumpLine(baseEnv(), false)).toMatchObject({
status: "warn",
detail: expect.stringContaining("no database dump can be taken"),
});
});
it("warns when it is switched off, and names a schedule it ignored", () => {
expect(
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
).toMatchObject({ status: "warn" });
const report = collectReadiness(
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
() => true,
);
expect(
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
).toBe("not a five-field cron; using 30 1 * * *");
expect(report.hasBlocker).toBe(false);
});
});
+76 -2
View File
@@ -19,6 +19,13 @@ import {
LEGACY_ANY_HOST_WARNING,
legacyAnyHostConfigured,
} from "@/lib/ai/local-host-allowlist";
import {
databaseBackupEnabled,
isWritableDirectory,
resolveBackupDir,
resolveBackupSchedule,
resolveDailyKeep,
} from "@/lib/database-backup/dump-core.mjs";
export interface ReadinessLine {
/** Short label, e.g. "ENCRYPTION_KEY". */
@@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean {
return keys.some((k) => (env[k] ?? "").trim().length > 0);
}
/** Inspect the environment and build the readiness report. Pure + side-effect free. */
export function collectReadiness(env: Env = process.env): ReadinessReport {
/**
* The nightly database dump's line. The dump is what rebuilds a lost server,
* so a host where it cannot be kept anywhere is said out loud at every boot,
* and so is one where it only goes off-host.
*/
export function databaseBackupReadiness(
env: Env,
writable: (dir: string) => boolean,
): ReadinessLine[] {
const label = "Database dump";
if (!databaseBackupEnabled(env)) {
return [
{
label,
status: "warn",
detail:
"DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way",
},
];
}
const lines: ReadinessLine[] = [];
const schedule = resolveBackupSchedule(env);
if (!schedule.valid) {
lines.push({
label: "DB_BACKUP_SCHEDULE",
status: "warn",
detail: `not a five-field cron; using ${schedule.cron}`,
});
}
const dir = resolveBackupDir(env);
const offhost = allSet(env, [
"BACKUP_S3_ENDPOINT",
"BACKUP_S3_BUCKET",
"BACKUP_S3_ACCESS_KEY",
"BACKUP_S3_SECRET_KEY",
"BACKUP_ENCRYPTION_KEY",
]);
if (writable(dir)) {
lines.push({
label,
status: "ok",
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
});
} else if (offhost) {
lines.push({
label,
status: "warn",
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
});
} else {
lines.push({
label,
status: "warn",
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
});
}
return lines;
}
/**
* Inspect the environment and build the readiness report. Pure apart from
* `writable`, the probe of the dump directory, which tests replace.
*/
export function collectReadiness(
env: Env = process.env,
writable: (dir: string) => boolean = isWritableDirectory,
): ReadinessReport {
const lines: ReadinessLine[] = [];
// --- Core secrets (boot blockers) ---------------------------------------
@@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport {
});
}
lines.push(...databaseBackupReadiness(env, writable));
// --- Optional subsystems (informational) --------------------------------
const optional: Array<[string, boolean]> = [
["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])],
+2
View File
@@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
HostMetric: "instance-wide host telemetry, not attributable to an account",
OffhostBackupKeyUse:
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
DatabaseBackupRun:
"the instance's ledger of nightly whole-database dumps; describes the server, not any account",
OffhostPurgeRequest:
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
CycleSymptomCategory:
@@ -0,0 +1,689 @@
/**
* The rules of the whole-database dump that need no database: names,
* retention, the manifest, key fingerprints, row-count comparison, the
* connection the client tools get, and the off-host envelope read back by the
* restore's own reader.
*
* The fingerprints and the envelope are checked against the app's own code
* (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy
* of the rule: the restore script cannot import TypeScript, so these are the
* tests that keep its plain-JS reading of both from drifting.
*/
import {
mkdtempSync,
createReadStream,
rmSync,
utimesSync,
writeFileSync,
} from "node:fs";
import { readdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Readable } from "node:stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
applyRetention,
bucketConfigFromEnv,
pickBucketDump,
buildManifest,
compareKeyFingerprints,
compareRowCounts,
connectionFromUrl,
decodeKeyMaterial,
decryptOffhostFile,
dumpBaseName,
formatDumpTimestamp,
keyFingerprints,
listDumpSets,
offhostObjectKey,
parseDumpFileName,
parseManifest,
resolveBackupSchedule,
resolveDailyKeep,
selectForDeletion,
type DumpSet,
} from "../dump-core.mjs";
import type { S3Like } from "@/lib/jobs/offhost-backup";
/** Every byte of a stream, in order. */
async function collect(stream: AsyncIterable<unknown>): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array));
return Buffer.concat(chunks);
}
const HEX_A = "a".repeat(64);
const HEX_B = "b".repeat(64);
const HEX_C = "c".repeat(64);
const HEX_D = "d".repeat(64);
const dirs: string[] = [];
function tempDir(): string {
const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-"));
dirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of dirs.splice(0))
rmSync(dir, { recursive: true, force: true });
vi.unstubAllEnvs();
});
describe("dump names", () => {
it("names a dump by its UTC instant and kind", () => {
const at = new Date("2026-10-25T01:30:07.123Z");
expect(formatDumpTimestamp(at)).toBe("20261025T013007Z");
expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily");
expect(dumpBaseName("pre-migration", at)).toBe(
"healthlog-20261025T013007Z-pre-migration",
);
});
it("parses every file it writes, and back to the same instant", () => {
const base = "healthlog-20261025T013007Z-daily";
const cases: Array<[string, string, string]> = [
[`${base}.dump`, "dump", "final"],
[`${base}.manifest.json`, "manifest", "final"],
[`${base}.dump.partial`, "dump", "partial"],
[`${base}.manifest.json.partial`, "manifest", "partial"],
[`${base}.dump.enc`, "dump", "encrypted"],
[`${base}.manifest.json.enc`, "manifest", "encrypted"],
];
for (const [name, role, variant] of cases) {
const parsed = parseDumpFileName(name);
expect(parsed, name).not.toBeNull();
expect(parsed?.base).toBe(base);
expect(parsed?.kind).toBe("daily");
expect(parsed?.role).toBe(role);
expect(parsed?.variant).toBe(variant);
expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z");
}
expect(
parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind,
).toBe("pre-migration");
});
it("refuses every name it did not write", () => {
for (const name of [
"healthlog.dump",
"backup-20261025T013007Z-daily.dump",
"healthlog-20261025T013007Z-weekly.dump",
"healthlog-20261025-daily.dump",
"healthlog-20261025T013007Z-daily.sql",
"healthlog-20261025T013007Z-daily.dump.bak",
"healthlog-20261399T013007Z-daily.dump",
"notes.txt",
]) {
expect(parseDumpFileName(name), name).toBeNull();
}
});
});
describe("configuration", () => {
it("keeps at least one daily dump and ignores what is not a number", () => {
expect(resolveDailyKeep({})).toBe(7);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7);
});
it("takes a five-field cron and says when it ignored one", () => {
expect(resolveBackupSchedule({})).toEqual({
cron: "30 1 * * *",
valid: true,
configured: false,
});
expect(
resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }),
).toEqual({
cron: "0 2 * * 1-5",
valid: true,
configured: true,
});
for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) {
expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({
cron: "30 1 * * *",
valid: false,
configured: true,
});
}
});
});
/** A dump set as `listDumpSets` reports it, for the pure retention rule. */
function set(
kind: "daily" | "pre-migration",
iso: string,
parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {},
now = Date.parse("2026-10-25T12:00:00Z"),
): DumpSet {
const base = dumpBaseName(kind, new Date(iso));
return {
base,
kind,
timestamp: new Date(iso),
dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 },
manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`,
encryptedDump: null,
encryptedManifest: null,
leftovers:
parts.partialAgeMs === undefined
? []
: [
{
path: `/b/${base}.dump.partial`,
mtimeMs: now - parts.partialAgeMs,
},
],
};
}
describe("retention", () => {
const now = Date.parse("2026-10-25T12:00:00Z");
it("keeps the newest N daily dumps and removes the rest, both files each", () => {
const sets = [1, 2, 3, 4, 5].map((d) =>
set("daily", `2026-10-2${d}T01:30:00Z`),
);
const doomed = selectForDeletion(sets, {
keep: { daily: 3, "pre-migration": 2 },
now,
});
expect(doomed.sort()).toEqual(
[
"/b/healthlog-20261021T013000Z-daily.dump",
"/b/healthlog-20261021T013000Z-daily.manifest.json",
"/b/healthlog-20261022T013000Z-daily.dump",
"/b/healthlog-20261022T013000Z-daily.manifest.json",
].sort(),
);
});
it("never removes the newest good dump, whatever keep says", () => {
const sets = [
set("daily", "2026-10-24T01:30:00Z"),
set("daily", "2026-10-23T01:30:00Z"),
];
const doomed = selectForDeletion(sets, {
keep: { daily: 0, "pre-migration": 0 },
now,
});
expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump");
expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump");
});
it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => {
const sets = [
set("daily", "2026-10-25T01:30:00Z", {
dump: false,
partialAgeMs: 60_000,
}),
set("daily", "2026-10-24T01:30:00Z"),
];
const doomed = selectForDeletion(sets, {
keep: { daily: 1, "pre-migration": 2 },
now,
});
expect(doomed).toEqual([]);
});
it("keeps two pre-migration dumps apart from the daily count", () => {
const sets = [
set("pre-migration", "2026-10-20T08:00:00Z"),
set("pre-migration", "2026-10-10T08:00:00Z"),
set("pre-migration", "2026-10-01T08:00:00Z"),
set("daily", "2026-10-24T01:30:00Z"),
];
const doomed = selectForDeletion(sets, {
keep: { daily: 1, "pre-migration": 2 },
now,
});
expect(doomed).toEqual([
"/b/healthlog-20261001T080000Z-pre-migration.dump",
"/b/healthlog-20261001T080000Z-pre-migration.manifest.json",
]);
});
it("removes a .partial only once it is older than twelve hours", () => {
const fresh = set("daily", "2026-10-25T01:30:00Z", {
dump: false,
manifest: false,
partialAgeMs: 60 * 60_000,
});
const stale = set("daily", "2026-10-23T01:30:00Z", {
dump: false,
manifest: false,
partialAgeMs: 13 * 60 * 60_000,
});
const doomed = selectForDeletion([fresh, stale], {
keep: { daily: 7, "pre-migration": 2 },
now,
});
expect(doomed).toEqual([
"/b/healthlog-20261023T013000Z-daily.dump.partial",
]);
});
it("works on a real directory and leaves files it did not write alone", async () => {
const dir = tempDir();
const names: string[] = [];
for (let day = 15; day <= 24; day++) {
const base = `healthlog-202610${day}T013000Z-daily`;
names.push(`${base}.dump`, `${base}.manifest.json`);
}
// Two that are not ours, and an off-host copy an operator put here.
names.push(
"my-own-dump.dump",
"healthlog-20261001T013000Z-daily.dump.bak",
"healthlog-20261002T013000Z-daily.dump.enc",
);
for (const name of names) writeFileSync(join(dir, name), "x");
// A crash's leftovers: one old, one from a run that may still be going.
const oldPartial = join(
dir,
"healthlog-20261003T013000Z-daily.dump.partial",
);
writeFileSync(oldPartial, "x");
const day = 24 * 60 * 60;
utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day);
writeFileSync(
join(dir, "healthlog-20261025T013000Z-daily.dump.partial"),
"x",
);
const removed = await applyRetention(dir, { dailyKeep: 7 });
expect(removed).toHaveLength(7);
const left = (await readdir(dir)).sort();
expect(left).toContain("my-own-dump.dump");
expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak");
expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc");
expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial");
expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial");
expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual(
[18, 19, 20, 21, 22, 23, 24].map(
(d) => `healthlog-202610${d}T013000Z-daily.dump`,
),
);
expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily");
});
});
describe("key fingerprints", () => {
it("are the fingerprints the app's own key code gives, for the map and the single key", async () => {
vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B }));
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2");
vi.stubEnv("ENCRYPTION_KEY", "");
const crypto = await import("@/lib/crypto");
crypto._resetCryptoCacheForTests();
const prints = keyFingerprints(process.env);
expect(prints.encryption.activeKeyId).toBe("v2");
expect(prints.encryption.keys).toEqual([
{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") },
{ id: "v2", fingerprint: crypto.getKeyFingerprint("v2") },
]);
vi.stubEnv("ENCRYPTION_KEYS", "");
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
const base64 = Buffer.from(HEX_C, "hex").toString("base64");
vi.stubEnv("ENCRYPTION_KEY", base64);
crypto._resetCryptoCacheForTests();
const single = keyFingerprints(process.env);
expect(single.encryption).toEqual({
activeKeyId: "v1",
keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }],
});
crypto._resetCryptoCacheForTests();
});
it("names the off-host key the way an off-host object records it", async () => {
const { offhostKeyId } = await import("@/lib/jobs/offhost-backup");
const prints = keyFingerprints({
ENCRYPTION_KEY: HEX_A,
BACKUP_ENCRYPTION_KEY: HEX_D,
BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`,
API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789",
});
expect(prints.backupEncryptionKey).toBe(
offhostKeyId(Buffer.from(HEX_D, "hex")),
);
expect(prints.backupEncryptionPreviousKeys).toEqual([
offhostKeyId(Buffer.from(HEX_C, "hex")),
]);
expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/);
});
it("never carries key material", () => {
const env = {
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }),
BACKUP_ENCRYPTION_KEY: HEX_B,
API_TOKEN_HMAC_KEY: HEX_C,
};
const text = JSON.stringify(keyFingerprints(env));
for (const secret of [HEX_A, HEX_B, HEX_C]) {
expect(text).not.toContain(secret.slice(0, 16));
}
});
it("decodes keys the way the app does, and nothing that is not a key", () => {
expect(decodeKeyMaterial(HEX_A)?.length).toBe(32);
expect(
decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length,
).toBe(32);
expect(decodeKeyMaterial("")).toBeNull();
expect(decodeKeyMaterial("not a key!")).toBeNull();
});
});
describe("comparing a dump's keys with the environment's", () => {
const written = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }),
ENCRYPTION_ACTIVE_KEY_ID: "v2",
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-one",
});
it("finds nothing when every key is there", () => {
const same = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }),
ENCRYPTION_ACTIVE_KEY_ID: "v2",
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-one",
});
expect(compareKeyFingerprints(written, same)).toEqual({
findings: [],
refuse: false,
});
});
it("refuses when the active key is different, and says which", () => {
const wrong = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }),
ENCRYPTION_ACTIVE_KEY_ID: "v2",
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-one",
});
const result = compareKeyFingerprints(written, wrong);
expect(result.refuse).toBe(true);
expect(result.findings).toHaveLength(1);
expect(result.findings[0]).toMatchObject({
key: "ENCRYPTION_KEYS[v2]",
state: "different",
severity: "refuse",
});
});
it("refuses when the active key is missing, warns for a retired one", () => {
const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A });
const result = compareKeyFingerprints(written, onlyOld);
expect(result.refuse).toBe(true);
const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f]));
expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({
state: "missing",
severity: "refuse",
});
expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined();
expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({
state: "missing",
severity: "warn",
});
expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({
state: "missing",
severity: "warn",
});
const noRetired = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }),
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-two",
});
const second = compareKeyFingerprints(written, noRetired);
expect(second.refuse).toBe(false);
expect(second.findings.map((f) => [f.key, f.severity])).toEqual([
["ENCRYPTION_KEYS[v1]", "warn"],
["API_TOKEN_HMAC_KEY", "warn"],
]);
});
});
describe("the manifest", () => {
const manifest = buildManifest({
kind: "daily",
file: "healthlog-20261025T013000Z-daily.dump",
createdAt: new Date("2026-10-25T01:30:00Z"),
appVersion: "1.46.0",
buildSha: "abc1234",
latestMigration: "0486_database_backup_runs",
server: { version: "18.6", versionNum: 180006 },
pgDump: "pg_dump (PostgreSQL) 18.6",
archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 },
env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" },
});
it("carries what a restore and an operator need", () => {
expect(manifest).toMatchObject({
format: "healthlog-database-dump",
formatVersion: 1,
kind: "daily",
file: "healthlog-20261025T013000Z-daily.dump",
createdAt: "2026-10-25T01:30:00.000Z",
appVersion: "1.46.0",
buildSha: "abc1234",
latestMigration: "0486_database_backup_runs",
postgres: { serverVersion: "18.6", serverVersionNum: 180006 },
dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
});
expect(Object.keys(manifest.rowCounts)).toEqual([
"pgboss.job",
"public.a",
"public.users",
]);
expect(manifest.keys.encryption.keys[0].id).toBe("v1");
});
it("reads back, and refuses anything else", () => {
expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual(
manifest,
);
expect(() => parseManifest({ format: "something-else" })).toThrow(
/not a HealthLog/,
);
expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow();
expect(() => parseManifest(null)).toThrow();
});
});
describe("row counts after a restore", () => {
it("names every table that is missing, extra or different", () => {
expect(
compareRowCounts(
{ "public.a": 1, "public.b": 2 },
{ "public.a": 1, "public.b": 2 },
),
).toEqual([]);
expect(
compareRowCounts(
{ "public.a": 1, "public.b": 2, "public.c": 5 },
{ "public.a": 1, "public.b": 0, "public.d": 1 },
),
).toEqual([
"public.b: 2 rows in the dump, 0 restored",
"public.c: in the dump with 5 rows, missing now",
"public.d: restored but not in the dump's manifest",
]);
});
});
describe("the connection the client tools get", () => {
it("drops Prisma's parameters and keeps the password out of the arguments", () => {
const c = connectionFromUrl(
"postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require",
);
expect(c.pgEnv).toEqual({
PGHOST: "db",
PGPORT: "6543",
PGUSER: "health@log",
PGPASSWORD: "p@ss/w",
PGDATABASE: "hl",
PGSSLMODE: "require",
});
expect(c.schema).toBe("app");
const cs = String(c.clientConfig.connectionString);
expect(cs).not.toContain("connection_limit");
expect(cs).not.toContain("schema=");
expect(cs).toContain("sslmode=require");
});
});
describe("an off-host copy read back by the restore", () => {
/** The uploader the job uses, into memory. */
async function upload(objectKey: string, key: Buffer, body: Buffer) {
const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup");
const stored = new Map<string, Buffer>();
const s3: S3Like = {
putStream: async (k: string, stream: Readable) => {
stored.set(k, await collect(stream));
},
putObject: async () => {},
getObject: async () => Buffer.alloc(0),
headObject: async () => false,
listObjects: async () => [],
deleteObject: async () => {},
};
await uploadEncryptedBackup(s3, objectKey, key, async (write) => {
// Several pieces, as the job streams the file.
for (let i = 0; i < body.length; i += 1000) {
await write(body.subarray(i, i + 1000));
}
});
return stored.get(objectKey) as Buffer;
}
it("opens what the job's uploader sealed, byte for byte", async () => {
const dir = tempDir();
const key = Buffer.from(HEX_D, "hex");
const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251));
const name = "healthlog-20261025T013000Z-daily.dump.enc";
const sealed = await upload(offhostObjectKey(name), key, body);
writeFileSync(join(dir, name), sealed);
await decryptOffhostFile({
inPath: join(dir, name),
outPath: join(dir, "opened.dump"),
objectKey: offhostObjectKey(name),
keys: [Buffer.from(HEX_A, "hex"), key],
});
expect(
(await collect(createReadStream(join(dir, "opened.dump")))).equals(body),
).toBe(true);
});
it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => {
const dir = tempDir();
const key = Buffer.from(HEX_D, "hex");
const body = Buffer.from("PGDMP fake archive bytes".repeat(100));
const name = "healthlog-20261025T013000Z-daily.dump.enc";
const sealed = await upload(offhostObjectKey(name), key, body);
writeFileSync(join(dir, name), sealed);
const out = join(dir, "opened.dump");
// Renamed: the object key is bound as associated data.
await expect(
decryptOffhostFile({
inPath: join(dir, name),
outPath: out,
objectKey: offhostObjectKey(
"healthlog-20261024T013000Z-daily.dump.enc",
),
keys: [key],
}),
).rejects.toThrow(/does not open/);
expect(await readdir(dir)).toEqual([name]);
// Altered.
const altered = Buffer.from(sealed);
altered[altered.length - 40] ^= 1;
writeFileSync(join(dir, name), altered);
await expect(
decryptOffhostFile({
inPath: join(dir, name),
outPath: out,
objectKey: offhostObjectKey(name),
keys: [key],
}),
).rejects.toThrow();
expect(await readdir(dir)).toEqual([name]);
// A key it was not written under.
writeFileSync(join(dir, name), sealed);
await expect(
decryptOffhostFile({
inPath: join(dir, name),
outPath: out,
objectKey: offhostObjectKey(name),
keys: [Buffer.from(HEX_A, "hex")],
}),
).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/);
expect(await readdir(dir)).toEqual([name]);
});
});
describe("finding a dump in the bucket", () => {
const keys = [
"database/healthlog-20261023T013000Z-daily.dump.enc",
"database/healthlog-20261023T013000Z-daily.manifest.json.enc",
"database/healthlog-20261024T013000Z-daily.dump.enc",
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
// The newest dump whose manifest never arrived, and things that are
// not dumps at all.
"database/healthlog-20261025T013000Z-daily.dump.enc",
"2026-10-25/user-abc.json.enc",
"database/notes.txt",
];
it("takes the newest dump whose manifest is there too", () => {
expect(pickBucketDump(keys)).toEqual({
dumpKey: "database/healthlog-20261024T013000Z-daily.dump.enc",
manifestKey:
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
});
});
it("finds a named one with or without its suffix, and nothing else", () => {
for (const name of [
"healthlog-20261023T013000Z-daily.dump",
"healthlog-20261023T013000Z-daily.dump.enc",
"database/healthlog-20261023T013000Z-daily.dump.enc",
]) {
expect(pickBucketDump(keys, name)?.dumpKey).toBe(
"database/healthlog-20261023T013000Z-daily.dump.enc",
);
}
expect(
pickBucketDump(keys, "healthlog-20261025T013000Z-daily.dump"),
).toBeNull();
expect(pickBucketDump([])).toBeNull();
});
it("reads the bucket from the four required variables", () => {
const env = {
BACKUP_S3_ENDPOINT: "https://s3.example.test",
BACKUP_S3_BUCKET: "b",
BACKUP_S3_ACCESS_KEY: "a",
BACKUP_S3_SECRET_KEY: "s",
};
expect(bucketConfigFromEnv(env)).toEqual({
endpoint: "https://s3.example.test",
bucket: "b",
accessKey: "a",
secretKey: "s",
region: "auto",
});
expect(
bucketConfigFromEnv({ ...env, BACKUP_S3_SECRET_KEY: "" }),
).toBeNull();
});
});
File diff suppressed because it is too large. Load diff
+125
View File
@@ -0,0 +1,125 @@
/**
* Whether the nightly whole-database dump is happening, as facts a page or an
* alert can act on.
*
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
* off-host bucket is configured. A ledger row that says a copy went off-host
* outranks this process's environment, for the reason the off-host card gives
* (a split install can hand the bucket variables to the worker only).
*/
import { prisma } from "@/lib/db";
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
import {
databaseBackupEnabled,
isWritableDirectory,
resolveBackupDir,
resolveDailyKeep,
} from "./dump-core.mjs";
export interface DatabaseBackupHealth {
/** `DB_BACKUP_ENABLED` is not off. */
enabled: boolean;
/** The directory dumps are kept in (`DB_BACKUP_DIR`). */
directory: string;
/**
* Nightly dumps kept in that directory (`DB_BACKUP_KEEP`). Also how long a
* deleted account stays in the dumps on this server, which the backups
* page says.
*/
keepNightly: number;
/** This process can write there. False means dumps only go off-host. */
localDirWritable: boolean;
/** The off-host bucket is configured, or a dump has been uploaded to it. */
offhostConfigured: boolean;
/** The newest dump that was kept on the server or reached the bucket. */
lastGood: {
at: string;
fileName: string;
sizeBytes: number | null;
local: boolean;
offhost: boolean;
} | null;
/** The newest run that failed, or that a dying process left unfinished. */
lastFailure: { at: string; error: string } | null;
}
/**
* A run still `running` this long after it started was cut off by its
* process: the job's expiry, past which pg-boss has failed it anyway.
*/
const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
export async function getDatabaseBackupHealth(
now: Date = new Date(),
env: NodeJS.ProcessEnv = process.env,
): Promise<DatabaseBackupHealth> {
const directory = resolveBackupDir(env);
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
[
prisma.databaseBackupRun.findFirst({
where: {
fileName: { not: null },
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
},
orderBy: { startedAt: "desc" },
select: {
startedAt: true,
finishedAt: true,
fileName: true,
dumpBytes: true,
localResult: true,
offhostResult: true,
},
}),
prisma.databaseBackupRun.findFirst({
where: { status: "failed" },
orderBy: { startedAt: "desc" },
select: { startedAt: true, finishedAt: true, error: true },
}),
prisma.databaseBackupRun.findFirst({
where: {
status: "running",
startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) },
},
orderBy: { startedAt: "desc" },
select: { startedAt: true },
}),
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
],
);
const failures = [
lastFailed && {
at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(),
error: lastFailed.error ?? "failed without a reason",
},
lastAbandoned && {
at: lastAbandoned.startedAt.toISOString(),
error: "the run did not finish (the process stopped under it)",
},
]
.filter((f): f is { at: string; error: string } => Boolean(f))
.sort((a, b) => b.at.localeCompare(a.at));
return {
enabled: databaseBackupEnabled(env),
directory,
keepNightly: resolveDailyKeep(env),
localDirWritable: isWritableDirectory(directory),
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
lastGood: lastGood
? {
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
fileName: lastGood.fileName ?? "",
sizeBytes:
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
local: lastGood.localResult === "written",
offhost: lastGood.offhostResult === "uploaded",
}
: null,
lastFailure: failures[0] ?? null,
};
}
+243
View File
@@ -0,0 +1,243 @@
/**
* One run of the nightly whole-database dump.
*
* The dump is the artifact that rebuilds a lost server: every account, every
* login, every grant and managed profile, the job queue and the instance
* settings, in one `pg_dump --format=custom` file with a manifest beside it
* (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the
* bundled compose file) and, when the off-host bucket is configured, uploaded
* under `database/` in the same envelope the per-account copies use, sealed
* with `BACKUP_ENCRYPTION_KEY`.
*
* Without a writable directory (an install that runs the image without the
* volume) the dump is written to a temporary directory, uploaded and removed:
* the off-host copy still happens, nothing is kept on the server, and the
* ledger and the boot summary say so. With neither, the run is skipped and
* recorded as such.
*
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
* what happened on each leg. `getDatabaseBackupHealth` reads them.
*/
import { createReadStream } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import type { PrismaClient } from "@/generated/prisma/client";
import {
getS3Client,
loadOffhostConfig,
uploadEncryptedBackup,
type S3Like,
} from "@/lib/jobs/offhost-backup";
import { annotate } from "@/lib/logging/context";
import packageJson from "../../../package.json";
import {
applyRetention,
createDatabaseDump,
databaseBackupEnabled,
isWritableDirectory,
offhostObjectKey,
resolveBackupDir,
resolveDailyKeep,
} from "./dump-core.mjs";
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
const LEDGER_RETENTION_DAYS = 400;
export type DatabaseBackupLocalResult =
"written" | "no_directory" | "failed" | "skipped";
export type DatabaseBackupOffhostResult =
"uploaded" | "not_configured" | "failed" | "skipped";
export interface DatabaseBackupReport {
status: "succeeded" | "failed" | "skipped";
fileName: string | null;
dumpBytes: number | null;
local: DatabaseBackupLocalResult;
offhost: DatabaseBackupOffhostResult;
/** Dumps and manifests retention removed. */
removed: number;
error: string | null;
}
export interface RunDatabaseBackupOptions {
now?: Date;
/** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */
deadline?: number;
env?: NodeJS.ProcessEnv;
/** A bucket double for tests; the configured bucket otherwise. */
s3?: S3Like;
}
/** The running build's version, as `/api/version` answers it. */
function appVersion(env: NodeJS.ProcessEnv): string {
return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version;
}
/** An error as one line for the ledger and the admin page. */
function reason(err: unknown): string {
const text = err instanceof Error ? err.message : String(err);
return text.replace(/\s+/g, " ").trim().slice(0, 500);
}
/** Upload one local file as an off-host object, streamed. */
async function uploadFile(
s3: S3Like,
key: Buffer,
path: string,
): Promise<number> {
return uploadEncryptedBackup(
s3,
offhostObjectKey(`${basename(path)}.enc`),
key,
async (write) => {
for await (const chunk of createReadStream(path)) {
await write(chunk as Buffer);
}
},
);
}
/**
* Take tonight's dump, keep it, copy it off-host, prune the old ones, and
* record what happened. Never throws: a failure is the report's `status`.
*/
export async function runDatabaseBackup(
prisma: PrismaClient,
options: RunDatabaseBackupOptions = {},
): Promise<DatabaseBackupReport> {
const env = options.env ?? process.env;
const now = options.now ?? new Date();
const dir = resolveBackupDir(env);
const localWritable = isWritableDirectory(dir);
const offhostConfig = (() => {
try {
return loadOffhostConfig();
} catch {
// A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the
// same value with their own message; here it reads as an upload error.
return "invalid" as const;
}
})();
// Each leg reads "failed" until it has succeeded, so a run the process died
// under says so once its expiry has passed.
const report: DatabaseBackupReport = {
status: "failed",
fileName: null,
dumpBytes: null,
local: localWritable ? "failed" : "no_directory",
offhost: offhostConfig ? "failed" : "not_configured",
removed: 0,
error: null,
};
const run = await prisma.databaseBackupRun.create({
data: {
startedAt: now,
status: "running",
localResult: report.local,
offhostResult: report.offhost,
},
select: { id: true },
});
let scratch: string | null = null;
try {
if (!databaseBackupEnabled(env)) {
report.status = "skipped";
report.local = "skipped";
report.offhost = "skipped";
report.error = "DB_BACKUP_ENABLED is false";
} else if (!localWritable && !offhostConfig) {
report.status = "skipped";
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
} else {
const target = localWritable
? dir
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
const dump = await createDatabaseDump({
databaseUrl: env.DATABASE_URL ?? "",
dir: target,
kind: "daily",
now,
env,
appVersion: appVersion(env),
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
deadline: options.deadline,
});
if (localWritable) report.local = "written";
report.fileName = basename(dump.dumpPath);
report.dumpBytes = dump.manifest.dump.bytes;
let uploadError: string | null = null;
if (offhostConfig === "invalid") {
uploadError =
"BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64";
} else if (offhostConfig) {
try {
const s3 = options.s3 ?? (await getS3Client(offhostConfig));
// The manifest goes up first: a dump object in the bucket always
// has the manifest a restore checks it against.
await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath);
await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath);
report.offhost = "uploaded";
} catch (err) {
uploadError = `off-host upload failed: ${reason(err)}`;
}
}
if (localWritable) {
const removed = await applyRetention(dir, {
dailyKeep: resolveDailyKeep(env),
now: now.getTime(),
});
report.removed = removed.length;
}
// Good when the dump exists somewhere it survives: on the volume, or in
// the bucket. An upload that failed beside a kept dump still fails the
// run, because the operator asked for both.
report.status = uploadError ? "failed" : "succeeded";
report.error = uploadError;
}
} catch (err) {
report.status = "failed";
report.error = reason(err);
} finally {
if (scratch) await rm(scratch, { recursive: true, force: true });
}
await prisma.databaseBackupRun.update({
where: { id: run.id },
data: {
finishedAt: new Date(),
status: report.status,
fileName: report.fileName,
dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes),
localResult: report.local,
offhostResult: report.offhost,
error: report.error,
},
});
await prisma.databaseBackupRun.deleteMany({
where: {
startedAt: {
lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000),
},
},
});
annotate({
action: { name: "database.backup.run" },
meta: {
status: report.status,
local: report.local,
offhost: report.offhost,
dump_bytes: report.dumpBytes ?? 0,
removed: report.removed,
},
});
return report;
}
@@ -0,0 +1,61 @@
/**
* Which lifecycle rules retire the database dumps under `database/`, apart
* from the account copies: a rule scoped to another prefix, or to a tag, does
* not touch them.
*/
import { describe, expect, it } from "vitest";
import type { LifecycleRule } from "@aws-sdk/client-s3";
import { lifecycleFromRules, ruleAppliesTo } from "../offhost-lifecycle";
const DUMP = "database/healthlog-20261025T013000Z-daily.dump.enc";
function rule(days: number, extra: Partial<LifecycleRule> = {}): LifecycleRule {
return { Status: "Enabled", Expiration: { Days: days }, ...extra };
}
describe("lifecycle rules and the database dumps", () => {
it("counts a rule for the whole bucket, or for database/, for both", () => {
expect(lifecycleFromRules([rule(30)])).toEqual({
state: "configured",
expirationDays: 30,
database: { state: "configured", expirationDays: 30 },
});
expect(
lifecycleFromRules([
rule(30, { Filter: { Prefix: "2026-" } }),
rule(14, { Filter: { Prefix: "database/" } }),
]).database,
).toEqual({ state: "configured", expirationDays: 14 });
});
it("says database/ is not covered when the only rule is scoped elsewhere", () => {
const result = lifecycleFromRules([
rule(30, { Filter: { Prefix: "2026-" } }),
]);
expect(result.state).toBe("configured");
expect(result.database).toEqual({ state: "missing", expirationDays: null });
});
it("reads the legacy prefix, the And filter, tags and disabled rules", () => {
expect(ruleAppliesTo(rule(5, { Prefix: "database/" }), DUMP)).toBe(true);
expect(ruleAppliesTo(rule(5, { Prefix: "other/" }), DUMP)).toBe(false);
expect(
ruleAppliesTo(
rule(5, {
Filter: { And: { Prefix: "database/", ObjectSizeGreaterThan: 1 } },
}),
DUMP,
),
).toBe(true);
expect(
ruleAppliesTo(
rule(5, { Filter: { Tag: { Key: "k", Value: "v" } } }),
DUMP,
),
).toBe(false);
expect(
lifecycleFromRules([rule(7, { Status: "Disabled" })]).database.state,
).toBe("missing");
});
});
+40
View File
@@ -0,0 +1,40 @@
/**
* The queue facts of the nightly whole-database dump: its name, its schedule
* and its send options. Apart from the job so the worker registry can read
* them without loading it (`job-registry-graph.test.ts`).
*/
import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy";
import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs";
export const DATABASE_BACKUP_QUEUE = "database-backup";
/**
* `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the
* zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30,
* an hour before the off-host account copies so the two uploads do not meet.
* A value that is not a five-field cron keeps the default, and the boot
* readiness summary says so.
*/
export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron;
/**
* Four hours. `pg_dump` of a large instance takes minutes, the upload of the
* dump about as long as the dump; the window is there to be impossible to hit
* for a working run. The handler kills `pg_dump` at three quarters of it
* (`jobDeadline`), so a hung dump ends as a failed run with a reason rather
* than an expired job.
*/
export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60;
/**
* What the nightly schedule sends with: the expiry, the backups' heartbeat,
* and one retry. A dump that failed for a passing reason (a lock it waited
* out, a bucket that answered 503) gets a second attempt; one that fails
* again waits for the next night, and the failure stays on the admin page.
*/
export const DATABASE_BACKUP_SEND_OPTIONS = {
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS,
retryLimit: 1,
retryDelay: 15 * 60,
} as const;
+6
View File
@@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"continued",
"conversation_titles_migrated",
"created",
// The nightly whole-database dump: where it went and how big.
"database_backup_bytes",
"database_backup_local",
"database_backup_offhost",
"database_backup_removed",
"database_backup_status",
"daily_rows_retired",
"days_consolidated",
"days_rebuilt",
+17 -30
View File
@@ -44,6 +44,12 @@ import {
import { annotate, getEvent } from "@/lib/logging/context";
import { envValue } from "@/lib/env";
import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue";
import {
LIFECYCLE_MISSING,
LIFECYCLE_UNKNOWN,
lifecycleFromRules,
type OffhostLifecycle,
} from "./offhost-lifecycle";
const ALGORITHM = "aes-256-gcm";
const IV_LENGTH = 12;
@@ -494,9 +500,9 @@ function createEnvelopeStream(
return { stream, bytes: () => written };
}
/** Produces the backup JSON in pieces. Every piece is written in order. */
/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */
export type BackupJsonProducer = (
write: (chunk: string) => Promise<void>,
write: (chunk: string | Buffer) => Promise<void>,
) => Promise<unknown>;
export interface UploadBackupOptions {
@@ -572,9 +578,9 @@ export async function uploadEncryptedBackup(
},
);
const write = async (chunk: string): Promise<void> => {
const write = async (chunk: string | Buffer): Promise<void> => {
if (failure) throw failure;
if (gzip.write(chunk, "utf8")) return;
if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding
await new Promise<void>((resolve, reject) => {
const onDrain = (): void => {
gzip.off("error", onError);
@@ -634,18 +640,7 @@ export interface S3Like {
getLifecycle?(): Promise<OffhostLifecycle>;
}
/**
* Whether the bucket retires old copies by itself.
*
* `configured` names the shortest expiry of an enabled rule; `missing` means
* the bucket answered that it has no rule; `unknown` means it would not say
* (a credential without `s3:GetLifecycleConfiguration`, a target that does not
* implement the call, a network failure) and says nothing either way.
*/
export interface OffhostLifecycle {
state: "configured" | "missing" | "unknown";
expirationDays: number | null;
}
export type { OffhostLifecycle } from "./offhost-lifecycle";
export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
// Dynamic import so unit tests + dev environments without the SDK don't fail.
@@ -765,13 +760,7 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
}),
{ abortSignal: AbortSignal.timeout(5_000) },
);
const days = (out.Rules ?? [])
.filter((rule) => rule.Status === "Enabled")
.map((rule) => rule.Expiration?.Days)
.filter((d): d is number => typeof d === "number" && d > 0);
return days.length > 0
? { state: "configured", expirationDays: Math.min(...days) }
: { state: "missing", expirationDays: null };
return lifecycleFromRules(out.Rules ?? []);
} catch (err) {
const name = (err as { name?: string; Code?: string }).name;
const code = (err as { Code?: string }).Code;
@@ -779,9 +768,9 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
name === "NoSuchLifecycleConfiguration" ||
code === "NoSuchLifecycleConfiguration"
) {
return { state: "missing", expirationDays: null };
return LIFECYCLE_MISSING;
}
return { state: "unknown", expirationDays: null };
return LIFECYCLE_UNKNOWN;
}
},
};
@@ -806,15 +795,13 @@ export async function probeOffhostLifecycle(
return lifecycleCache.value;
}
const cfg = loadOffhostConfigSafe();
if (!cfg) return { state: "unknown", expirationDays: null };
if (!cfg) return LIFECYCLE_UNKNOWN;
let value: OffhostLifecycle;
try {
const s3 = s3Override ?? (await getS3Client(cfg));
value = s3.getLifecycle
? await s3.getLifecycle()
: { state: "unknown", expirationDays: null };
value = s3.getLifecycle ? await s3.getLifecycle() : LIFECYCLE_UNKNOWN;
} catch {
value = { state: "unknown", expirationDays: null };
value = LIFECYCLE_UNKNOWN;
}
if (!s3Override) lifecycleCache = { at: now, value };
return value;
+79
View File
@@ -0,0 +1,79 @@
/**
* Whether the operator's bucket retires old copies by itself, read from its
* lifecycle rules (`GetBucketLifecycleConfiguration`).
*
* Two answers, because the bucket holds two kinds of object with different
* keys: the per-account copies (`<date>/user-<id>.json.enc`) and the database
* dumps (`database/…`). A rule scoped to a prefix covers only the keys under
* it, so a bucket whose one rule expires `2026-` keys ages the account copies
* out and keeps every database dump forever, and a database dump keeps every
* deleted account's data for as long as it exists. The account answer keeps
* its earlier reading (the shortest expiry of any enabled rule); the database
* answer counts only the rules that apply to a key under `database/`.
*
* `configured` names the shortest expiry; `missing` means the bucket answered
* that no rule applies; `unknown` means it would not say (a credential
* without `s3:GetLifecycleConfiguration`, a target that does not implement the
* call, a network failure) and says nothing either way.
*/
import type { LifecycleRule } from "@aws-sdk/client-s3";
export interface LifecycleVerdict {
state: "configured" | "missing" | "unknown";
expirationDays: number | null;
}
export interface OffhostLifecycle extends LifecycleVerdict {
/** The same question for the database dumps under `database/`. */
database: LifecycleVerdict;
}
export const LIFECYCLE_UNKNOWN: OffhostLifecycle = {
state: "unknown",
expirationDays: null,
database: { state: "unknown", expirationDays: null },
};
export const LIFECYCLE_MISSING: OffhostLifecycle = {
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
};
/** A key a database dump is stored under, to test the rules against. */
const DATABASE_SAMPLE_KEY =
"database/healthlog-20260101T013000Z-daily.dump.enc";
/**
* Whether a rule's filter takes in `key`. Our objects carry no tags, so a
* rule that filters on a tag never applies to them; a size bound is not a
* reason to call a rule inapplicable, so it is read as applying.
*/
export function ruleAppliesTo(rule: LifecycleRule, key: string): boolean {
const filter = rule.Filter;
if (filter?.Tag || filter?.And?.Tags?.length) return false;
const prefix = filter?.Prefix ?? filter?.And?.Prefix ?? rule.Prefix ?? "";
return key.startsWith(prefix);
}
function verdict(rules: readonly LifecycleRule[]): LifecycleVerdict {
const days = rules
.filter((rule) => rule.Status === "Enabled")
.map((rule) => rule.Expiration?.Days)
.filter((d): d is number => typeof d === "number" && d > 0);
return days.length > 0
? { state: "configured", expirationDays: Math.min(...days) }
: { state: "missing", expirationDays: null };
}
/** Both answers from the rules a bucket returned. */
export function lifecycleFromRules(
rules: readonly LifecycleRule[],
): OffhostLifecycle {
return {
...verdict(rules),
database: verdict(
rules.filter((rule) => ruleAppliesTo(rule, DATABASE_SAMPLE_KEY)),
),
};
}
+12
View File
@@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-
import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill";
import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run";
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue";
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue";
@@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
exclusive: "lockedPass",
why: "Serialises every account's whole record.",
},
"database-backup": {
runtime: "long",
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"],
stop: {
file: "lib/jobs/reminder/backup-handlers.ts",
fn: "handleDatabaseBackup",
},
exclusive: "lockedPass",
why: "Runs pg_dump over the whole database and uploads the dump off-host.",
},
"data-backup-offhost": {
runtime: "long",
expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS,
+17 -1
View File
@@ -1,5 +1,6 @@
/**
* The housekeeping queues: backups (weekly in-database, nightly off-host),
* The housekeeping queues: backups (the nightly whole-database dump, weekly
* in-database account copies, nightly off-host),
* the restore drill and restores, every ledger and retention cleanup, the
* medication maintenance passes, the imports (Apple Health, Health Connect,
* medication intake), the converging encryption backfills, the document vault
@@ -47,6 +48,11 @@ import {
DATA_BACKUP_QUEUE,
DATA_BACKUP_SEND_OPTIONS,
} from "@/lib/jobs/data-backup-policy";
import {
DATABASE_BACKUP_CRON,
DATABASE_BACKUP_QUEUE,
DATABASE_BACKUP_SEND_OPTIONS,
} from "@/lib/jobs/database-backup-queue";
import {
DAWARICH_SYNC_CRON,
DAWARICH_SYNC_QUEUE,
@@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [
lane: true,
load: async () => (await backups()).handleDataBackup,
}),
// The whole-database dump: the copy that rebuilds a lost server. Locked, so
// a retry that starts while a run is still dumping waits its turn.
defineJob({
queue: DATABASE_BACKUP_QUEUE,
cron: DATABASE_BACKUP_CRON,
scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS,
work: { localConcurrency: 1 },
lock: () => WHOLE_PASS,
load: async () => (await backups()).handleDatabaseBackup,
}),
prune(
RATE_LIMIT_CLEANUP_QUEUE,
RATE_LIMIT_CLEANUP_CRON,
+32 -1
View File
@@ -5,7 +5,7 @@
* schedules, and boss.work registrations.
*/
import { type Job, type JobWithMetadata } from "pg-boss";
import { jobBudget } from "@/lib/jobs/job-budget";
import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget";
import { recordError } from "@/lib/jobs/worker-status";
import {
BackupBlobTooLargeError,
@@ -31,6 +31,37 @@ import {
} from "@/lib/jobs/backup-pass-attempts";
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
/**
* The nightly whole-database dump (`database-backup`).
*
* One `pg_dump` in a child process, so it is not in the heavy lane: the heap
* holds a part of the upload at a time, not the database. `pg_dump` is stopped
* at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs
* on a lock ends as a failed run with its reason rather than an expired job.
* A skipped run (switched off, nowhere to put it) completes; the admin page
* and the boot summary carry why.
*/
export async function handleDatabaseBackup(
jobs: Job<Record<string, never>>[],
): Promise<JobOutcome> {
return withBackgroundEvent("job.database_backup", async () => {
const { runDatabaseBackup } = await import("@/lib/database-backup/run");
const report = await runDatabaseBackup(getWorkerPrisma(), {
deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined,
});
const did = {
database_backup_status: report.status,
database_backup_local: report.local,
database_backup_offhost: report.offhost,
database_backup_bytes: report.dumpBytes ?? 0,
database_backup_removed: report.removed,
};
return report.status === "failed"
? jobFailed("database dump failed", report.error ?? undefined, did)
: jobDone(did);
});
}
export interface DataBackupPayload {
triggeredAt: string;
}
+1
View File
@@ -99,6 +99,7 @@ const subsystemSurface = {
"audit-log-cleanup": { audience: "system" },
"step-up-elevation-cleanup": { audience: "system" },
"data-backup-offhost": { audience: "system" },
"database-backup": { audience: "system" },
"data-restore-drill": { audience: "system" },
"offhost-backup-purge": { audience: "system" },
"backup-restore": { audience: "system" },
+9 -4
View File
@@ -9,6 +9,8 @@
* both sides a route-handler-independent shared type home; the route
* keeps owning the HTTP contract, the component keeps owning the UI.
*/
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { OffhostLifecycle } from "@/lib/jobs/offhost-lifecycle";
import type { BackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
@@ -76,10 +78,7 @@ export interface OffhostBackupOverview {
* (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown`
* when the credential may not read it.
*/
lifecycle: {
state: "configured" | "missing" | "unknown";
expirationDays: number | null;
};
lifecycle: OffhostLifecycle;
}
export interface BackupsList {
@@ -101,6 +100,12 @@ export interface BackupsList {
* the operator's bucket, and those two fail independently.
*/
offhost: OffhostBackupOverview;
/**
* The nightly whole-database dump, the copy that rebuilds a lost server:
* the newest good dump (kept here, off-host, or both), the newest failed
* run, and whether this server can keep dumps at all.
*/
database: DatabaseBackupHealth;
/**
* Soft retention hint — the worker is configured for weekly backups
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
@@ -1,5 +1,5 @@
{
"registeredQueueCount": 122,
"registeredQueueCount": 123,
"registeredQueues": [
"achievement-unlock-sweep",
"apple-health-import",
@@ -23,6 +23,7 @@
"data-backup",
"data-backup-offhost",
"data-restore-drill",
"database-backup",
"dawarich-sync",
"dense-intraday-hourly-rebuild",
"dense-intraday-retention",
@@ -455,6 +456,21 @@
"retry_delay_max": null,
"retry_limit": 2
},
{
"dead_letter": null,
"deletion_seconds": 604800,
"expire_seconds": 900,
"heartbeat_seconds": null,
"name": "database-backup",
"notify": false,
"partition": false,
"policy": "standard",
"retention_seconds": 1209600,
"retry_backoff": false,
"retry_delay": 0,
"retry_delay_max": null,
"retry_limit": 2
},
{
"dead_letter": null,
"deletion_seconds": 604800,
@@ -2137,6 +2153,21 @@
},
"timezone": "Europe/Berlin"
},
{
"cron": "30 1 * * *",
"data": {},
"key": "",
"kind": "cron",
"name": "database-backup",
"options": {
"expireInSeconds": 14400,
"heartbeatSeconds": 300,
"retryDelay": 900,
"retryLimit": 1,
"tz": "Europe/Berlin"
},
"timezone": "Europe/Berlin"
},
{
"cron": "20 3 * * *",
"data": {},
@@ -3022,6 +3053,13 @@
"pollingIntervalSeconds": 30
}
},
{
"queue": "database-backup",
"options": {
"localConcurrency": 1,
"pollingIntervalSeconds": 30
}
},
{
"queue": "dawarich-sync",
"options": {
+511
View File
@@ -0,0 +1,511 @@
/**
* The nightly whole-database dump against a real Postgres: the job writes a
* dump and its manifest, records the run, copies both off-host in the
* envelope the restore opens, keeps only what retention allows; and the
* restore puts the dump into an empty database and refuses one with tables.
*
* `pg_dump` and `pg_restore` are the test container's own. A workstation may
* have none, or one of another major than the server (which `pg_dump`
* refuses), so the test puts two small forwarders on PATH that run the tool
* inside the database container with `docker exec`. Everything else, the
* snapshot, the counts, the manifest, the ledger and the restore's checks,
* runs exactly as in the image.
*/
import { execFileSync } from "node:child_process";
import {
chmodSync,
existsSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Readable } from "node:stream";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { prisma } from "@/lib/db";
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
import { runDatabaseBackup } from "@/lib/database-backup/run";
import {
connect,
countRows,
decryptOffhostFile,
offhostObjectKey,
readManifestFile,
} from "@/lib/database-backup/dump-core.mjs";
import { main as restoreMain } from "../../scripts/db-restore.mjs";
import { shippedMigrations } from "../../scripts/pending-migrations.mjs";
const OFFHOST_KEY = "e".repeat(64);
const work = mkdtempSync(join(tmpdir(), "hl-db-backup-"));
const backups = join(work, "backups");
const shims = join(work, "bin");
const originalPath = process.env.PATH;
/** The id of the container that serves DATABASE_URL. */
function containerId(): string {
const port = new URL(process.env.DATABASE_URL ?? "").port;
const rows = execFileSync(
"docker",
["ps", "--format", "{{.ID}} {{.Ports}}"],
{
encoding: "utf8",
},
).split("\n");
const row = rows.find((r) => r.includes(`:${port}->5432/tcp`));
if (!row) throw new Error(`no container publishes port ${port}`);
return row.split(" ")[0];
}
/**
* `pg_dump` / `pg_restore` that run inside the database container. The
* connection comes from the PG* variables the core sets; host and port become
* the container's own. A file argument that exists on this machine is sent on
* standard input, which both tools read when no file is named.
*/
function writeShims(container: string): void {
for (const tool of ["pg_dump", "pg_restore"]) {
const path = join(shims, tool);
writeFileSync(
path,
`#!/usr/bin/env node
const { spawn } = require("node:child_process");
const { createReadStream, existsSync } = require("node:fs");
const args = process.argv.slice(2);
let input = null;
if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) {
input = args.pop();
}
const env = [];
for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) {
if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]);
}
env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432");
const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], {
stdio: ["pipe", "inherit", "inherit"],
});
(input ? createReadStream(input) : process.stdin).pipe(child.stdin);
child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0)));
`,
);
chmodSync(path, 0o755);
}
}
/** A bucket in memory, with the one call the job's uploader uses. */
function memoryBucket() {
const objects = new Map<string, Buffer>();
return {
objects,
putStream: async (key: string, body: Readable) => {
const chunks: Buffer[] = [];
for await (const c of body) chunks.push(c as Buffer);
objects.set(key, Buffer.concat(chunks));
},
};
}
function backupEnv(extra: Record<string, string> = {}): NodeJS.ProcessEnv {
return { ...process.env, DB_BACKUP_DIR: backups, ...extra };
}
beforeAll(async () => {
const { mkdirSync } = await import("node:fs");
mkdirSync(shims, { recursive: true });
mkdirSync(backups, { recursive: true });
writeShims(containerId());
process.env.PATH = `${shims}:${originalPath}`;
for (const name of ["dump-a", "dump-b", "dump-c"]) {
await prisma.user.create({
data: { username: `${name}-${Date.now()}` },
});
}
});
afterAll(() => {
process.env.PATH = originalPath;
vi.unstubAllEnvs();
rmSync(work, { recursive: true, force: true });
});
describe("the nightly database dump", () => {
let dumpFile = "";
it("writes a checked dump and manifest, records the run and copies both off-host", async () => {
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
vi.stubEnv("BACKUP_S3_BUCKET", "b");
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
const bucket = memoryBucket();
const now = new Date("2026-10-20T01:30:00Z");
const report = await runDatabaseBackup(prisma, {
now,
env: backupEnv(),
s3: bucket as never,
});
expect(report).toMatchObject({
status: "succeeded",
local: "written",
offhost: "uploaded",
error: null,
fileName: "healthlog-20261020T013000Z-daily.dump",
});
dumpFile = join(backups, report.fileName ?? "");
// Both files, under their final names, and nothing half-written.
expect(readdirSync(backups).sort()).toEqual([
"healthlog-20261020T013000Z-daily.dump",
"healthlog-20261020T013000Z-daily.manifest.json",
]);
expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe(
"PGDMP",
);
const manifest = await readManifestFile(
join(backups, "healthlog-20261020T013000Z-daily.manifest.json"),
);
expect(manifest.kind).toBe("daily");
expect(manifest.dump.bytes).toBe(report.dumpBytes);
expect(manifest.latestMigration).toBe(
shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1),
);
expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000);
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100);
expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0);
expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/);
// The ledger row and what the admin page reads from it.
const runs = await prisma.databaseBackupRun.findMany({
where: { startedAt: now },
});
expect(runs).toHaveLength(1);
expect(runs[0]).toMatchObject({
status: "succeeded",
fileName: "healthlog-20261020T013000Z-daily.dump",
localResult: "written",
offhostResult: "uploaded",
error: null,
});
expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes);
const health = await getDatabaseBackupHealth(
new Date("2026-10-20T02:00:00Z"),
backupEnv(),
);
expect(health).toMatchObject({
enabled: true,
localDirWritable: true,
offhostConfigured: true,
lastGood: {
fileName: "healthlog-20261020T013000Z-daily.dump",
sizeBytes: manifest.dump.bytes,
local: true,
offhost: true,
},
});
// The off-host pair opens, with the restore's own reader, to the very
// bytes on disk.
expect([...bucket.objects.keys()].sort()).toEqual([
"database/healthlog-20261020T013000Z-daily.dump.enc",
"database/healthlog-20261020T013000Z-daily.manifest.json.enc",
]);
const opened = join(work, "opened");
const { mkdirSync } = await import("node:fs");
mkdirSync(opened);
for (const name of [
"healthlog-20261020T013000Z-daily.dump",
"healthlog-20261020T013000Z-daily.manifest.json",
]) {
writeFileSync(
join(opened, `${name}.enc`),
bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer,
);
await decryptOffhostFile({
inPath: join(opened, `${name}.enc`),
outPath: join(opened, name),
objectKey: offhostObjectKey(`${name}.enc`),
keys: [Buffer.from(OFFHOST_KEY, "hex")],
});
expect(
readFileSync(join(opened, name)).equals(
readFileSync(join(backups, name)),
),
).toBe(true);
}
vi.unstubAllEnvs();
});
it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => {
for (const day of ["21", "22", "23"]) {
const report = await runDatabaseBackup(prisma, {
now: new Date(`2026-10-${day}T01:30:00Z`),
env: backupEnv({ DB_BACKUP_KEEP: "2" }),
});
expect(report.status).toBe("succeeded");
expect(report.offhost).toBe("not_configured");
}
expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([
"healthlog-20261022T013000Z-daily.dump",
"healthlog-20261023T013000Z-daily.dump",
]);
});
it("records a failed dump, leaves nothing that looks good, and the page says so", async () => {
const before = readdirSync(backups).sort();
const url = new URL(process.env.DATABASE_URL ?? "");
url.password = "not-the-password";
const now = new Date("2026-10-24T01:30:00Z");
const report = await runDatabaseBackup(prisma, {
now,
env: backupEnv({ DATABASE_URL: url.toString() }),
});
expect(report.status).toBe("failed");
expect(report.local).toBe("failed");
expect(report.error).toMatch(/password authentication failed/);
expect(readdirSync(backups).sort()).toEqual(before);
const health = await getDatabaseBackupHealth(
new Date("2026-10-24T02:00:00Z"),
backupEnv(),
);
expect(health.lastFailure?.error).toMatch(/password authentication/);
expect(health.lastGood?.fileName).toBe(
"healthlog-20261023T013000Z-daily.dump",
);
});
it("skips, and says why, with nowhere to put a dump", async () => {
const report = await runDatabaseBackup(prisma, {
now: new Date("2026-10-24T03:00:00Z"),
env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }),
});
expect(report).toMatchObject({
status: "skipped",
local: "no_directory",
offhost: "not_configured",
});
expect(report.error).toMatch(/no writable backup directory/);
const off = await runDatabaseBackup(prisma, {
now: new Date("2026-10-24T04:00:00Z"),
env: backupEnv({ DB_BACKUP_ENABLED: "false" }),
});
expect(off).toMatchObject({
status: "skipped",
local: "skipped",
offhost: "skipped",
});
});
describe("restoring it", () => {
const target = `restore_${process.pid}_${Date.now()}`;
let targetUrl = "";
beforeAll(async () => {
const admin = await connect(process.env.DATABASE_URL ?? "");
await admin.query(`CREATE DATABASE ${target}`);
await admin.end();
const url = new URL(process.env.DATABASE_URL ?? "");
url.pathname = `/${target}`;
targetUrl = url.toString();
});
afterAll(async () => {
const admin = await connect(process.env.DATABASE_URL ?? "");
await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`);
await admin.end();
});
it("refuses the database the dump came from, which has tables", async () => {
const errors: string[] = [];
const spy = vi
.spyOn(console, "error")
.mockImplementation((l: string) => void errors.push(String(l)));
const code = await restoreMain(["latest"], backupEnv());
spy.mockRestore();
expect(code).toBe(1);
expect(errors.join("\n")).toMatch(/only into an empty database/);
});
it("restores the newest dump into an empty database with every row count", async () => {
const lines: string[] = [];
const spy = vi
.spyOn(console, "log")
.mockImplementation((l: string) => void lines.push(String(l)));
const code = await restoreMain(
["latest"],
backupEnv({ DATABASE_URL: targetUrl }),
);
spy.mockRestore();
expect(code, lines.join("\n")).toBe(0);
expect(lines.join("\n")).toMatch(
/all \d+ tables hold the row counts the manifest names/,
);
const manifest = await readManifestFile(
join(backups, "healthlog-20261023T013000Z-daily.manifest.json"),
);
const client = await connect(targetUrl);
try {
expect(await countRows(client)).toEqual(manifest.rowCounts);
const { rows } = await client.query(
"SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'",
);
expect(rows[0].n).toBeGreaterThanOrEqual(3);
} finally {
await client.end();
}
// Restoring again into the now full database is refused.
const again = await restoreMain(
["latest"],
backupEnv({ DATABASE_URL: targetUrl }),
);
expect(again).toBe(1);
expect(existsSync(dumpFile)).toBe(false); // the first dump aged out
});
});
});
describe("the dump before migrating", () => {
const preDir = join(work, "pre");
const appDir = join(work, "app");
const failing = join(work, "failing-bin");
let cwd = "";
beforeAll(async () => {
const { cpSync, mkdirSync } = await import("node:fs");
// An image that ships one migration the database has not applied.
mkdirSync(join(appDir, "prisma"), { recursive: true });
cpSync(
join(process.cwd(), "prisma", "migrations"),
join(appDir, "prisma", "migrations"),
{ recursive: true },
);
mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet"));
writeFileSync(
join(
appDir,
"prisma",
"migrations",
"9999_not_applied_yet",
"migration.sql",
),
"",
);
mkdirSync(preDir, { recursive: true });
mkdirSync(failing, { recursive: true });
writeFileSync(
join(failing, "pg_dump"),
"#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n",
);
chmodSync(join(failing, "pg_dump"), 0o755);
cwd = process.cwd();
process.chdir(appDir);
});
afterAll(() => {
process.chdir(cwd);
process.env.PATH = `${shims}:${originalPath}`;
});
async function preMigration(env: Record<string, string>) {
const { preMigrationDump } = await import("../../scripts/db-dump.mjs");
const errors: string[] = [];
const spyErr = vi
.spyOn(console, "error")
.mockImplementation((l: string) => void errors.push(String(l)));
const spyLog = vi.spyOn(console, "log").mockImplementation(() => {});
try {
const code = await preMigrationDump({ ...process.env, ...env });
return { code, errors: errors.join("\n") };
} finally {
spyErr.mockRestore();
spyLog.mockRestore();
}
}
it("writes a checked dump with its manifest, and keeps the last two", async () => {
for (let i = 0; i < 3; i++) {
if (i > 0) await new Promise((r) => setTimeout(r, 1100));
const { code } = await preMigration({ DB_BACKUP_DIR: preDir });
expect(code).toBe(0);
}
const names = readdirSync(preDir).sort();
expect(names).toHaveLength(4);
expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength(
2,
);
const manifest = await readManifestFile(
join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""),
);
expect(manifest.kind).toBe("pre-migration");
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
});
it("does nothing when switched off", async () => {
const before = readdirSync(preDir);
const { code } = await preMigration({
DB_BACKUP_DIR: preDir,
DB_BACKUP_BEFORE_MIGRATE: "false",
});
expect(code).toBe(0);
expect(readdirSync(preDir)).toEqual(before);
});
it("warns and lets the boot go on without a writable directory", async () => {
const { code, errors } = await preMigration({
DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here",
});
expect(code).toBe(0);
expect(errors).toMatch(
/is not writable, so no dump is taken before the 1 pending migration/,
);
});
it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => {
// A second of its own, so the name cannot meet the last dump's.
await new Promise((r) => setTimeout(r, 1100));
const before = readdirSync(preDir).sort();
process.env.PATH = `${failing}:${originalPath}`;
const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir });
process.env.PATH = `${shims}:${originalPath}`;
expect(code).toBe(1);
expect(errors).toMatch(
/the database dump before migrating failed: .*No space left/,
);
expect(readdirSync(preDir).sort()).toEqual(before);
});
});
describe("two dumps in the same second", () => {
it("refuses the second and leaves the first whole", async () => {
const { createDatabaseDump } =
await import("@/lib/database-backup/dump-core.mjs");
const dir = join(work, "same-second");
const { mkdirSync } = await import("node:fs");
mkdirSync(dir);
const now = new Date("2026-10-26T01:30:00Z");
const options = {
databaseUrl: process.env.DATABASE_URL ?? "",
dir,
kind: "daily" as const,
now,
};
await createDatabaseDump(options);
const before = readdirSync(dir)
.sort()
.map((n) => [n, readFileSync(join(dir, n)).length]);
await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/);
expect(
readdirSync(dir)
.sort()
.map((n) => [n, readFileSync(join(dir, n)).length]),
).toEqual(before);
});
});