Merge the release into the deletion grace work

The translation files merge key by key (no key changed on both sides), the
backups list carries both the database dump and the pre-wipe copy window,
and the backup-restore doc keeps the shorter rewrite plus the dump in the
bucket and a lifecycle rule that covers database/.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 15:18:46 +01:00
commit 9db5e755e6
254 files changed
+16378 -3184

No files matched your search

+161
View File
@@ -0,0 +1,161 @@
/**
* The restore's refusals, which all come before it writes anything.
*
* The database is a stand-in here: `connect` answers a client whose only use
* is to be asked which tables exist, and `runTool` (which would run
* `pg_restore`) is a spy that must never be called on a refusal. The real
* restore into a real empty database, and its refusal of a real database with
* tables, are in `tests/integration/database-backup.test.ts` and in the
* compose drill (`scripts/compose-dr-drill.sh`).
*/
import { createHash } from "node:crypto";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const tablesInTarget = vi.fn(async (): Promise<string[]> => []);
const runTool = vi.fn(async () => "");
const end = vi.fn(async () => {});
vi.mock("../../src/lib/database-backup/dump-core.mjs", async (original) => ({
...(await original<Record<string, unknown>>()),
connect: async () => ({ query: async () => ({ rows: [] }), end }),
listUserTables: () => tablesInTarget(),
runTool: (...args: unknown[]) => runTool(...(args as [])),
}));
import { buildManifest } from "../../src/lib/database-backup/dump-core.mjs";
import { main } from "../db-restore.mjs";
const KEY_A = "a".repeat(64);
const KEY_B = "b".repeat(64);
const BASE = "healthlog-20261025T013000Z-daily";
let dir: string;
let errors: string[];
let lines: string[];
function env(extra: Record<string, string> = {}) {
return {
DATABASE_URL: "postgresql://healthlog:pw@db:5432/healthlog?schema=public",
DB_BACKUP_DIR: dir,
ENCRYPTION_KEY: KEY_A,
API_TOKEN_HMAC_KEY: "hmac",
...extra,
};
}
/** A dump file and a manifest that describes it, written with KEY_A. */
function writeDump(body = Buffer.from("PGDMP not really an archive")) {
writeFileSync(join(dir, `${BASE}.dump`), body);
const manifest = buildManifest({
kind: "daily",
file: `${BASE}.dump`,
createdAt: new Date("2026-10-25T01:30:00Z"),
appVersion: "1.46.0",
buildSha: null,
latestMigration: "0486_database_backup_runs",
server: { version: "18.6", versionNum: 180006 },
pgDump: null,
archive: {
bytes: body.length,
sha256: createHash("sha256").update(body).digest("hex"),
entries: 1,
},
rowCounts: { "public.users": 2 },
env: { ENCRYPTION_KEY: KEY_A, API_TOKEN_HMAC_KEY: "hmac" },
});
writeFileSync(
join(dir, `${BASE}.manifest.json`),
JSON.stringify(manifest, null, 2),
);
}
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), "hl-db-restore-"));
errors = [];
lines = [];
vi.spyOn(console, "error").mockImplementation((line: string) => {
errors.push(String(line));
});
vi.spyOn(console, "log").mockImplementation((line: string) => {
lines.push(String(line));
});
tablesInTarget.mockResolvedValue([]);
runTool.mockClear();
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
describe("db-restore refuses before it writes", () => {
it("refuses a target database that already holds tables", async () => {
writeDump();
tablesInTarget.mockResolvedValue(["public.users", "public.sessions"]);
expect(await main(["latest"], env())).toBe(1);
expect(errors.join("\n")).toMatch(
/already holds 2 tables .*only into an empty database/,
);
expect(runTool).not.toHaveBeenCalled();
});
it("refuses when the active encryption key is not the dump's, and names it", async () => {
writeDump();
expect(await main(["latest"], env({ ENCRYPTION_KEY: KEY_B }))).toBe(1);
expect(lines.join("\n")).toMatch(/KEY MISMATCH: ENCRYPTION_KEYS\[v1\]/);
expect(errors.join("\n")).toMatch(/refusing to restore/);
expect(runTool).not.toHaveBeenCalled();
});
it("goes past the key check with --ignore-key-mismatch, and still refuses a full database", async () => {
writeDump();
tablesInTarget.mockResolvedValue(["public.users"]);
expect(
await main(
["latest", "--ignore-key-mismatch"],
env({ ENCRYPTION_KEY: KEY_B }),
),
).toBe(1);
expect(errors.join("\n")).toMatch(/already holds 1 table /);
expect(runTool).not.toHaveBeenCalled();
});
it("refuses a dump that does not match its manifest", async () => {
writeDump();
writeFileSync(join(dir, `${BASE}.dump`), "PGDMP altered");
expect(await main(["latest"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/does not match its manifest/);
expect(runTool).not.toHaveBeenCalled();
});
it("refuses a dump without its manifest, and a file it did not write", async () => {
writeFileSync(join(dir, `${BASE}.dump`), "x");
expect(await main([`${BASE}.dump`], env())).toBe(1);
expect(errors.join("\n")).toMatch(/manifest.json is missing/);
writeFileSync(join(dir, "other.dump"), "x");
expect(await main(["other.dump"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/not a HealthLog database dump/);
expect(runTool).not.toHaveBeenCalled();
});
it("says when there is nothing to restore, and how it is used", async () => {
expect(await main(["latest"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/no restorable dump/);
expect(await main([], env())).toBe(2);
});
it("refuses --from-bucket without a configured bucket, before it connects", async () => {
expect(await main(["latest", "--from-bucket"], env())).toBe(1);
expect(errors.join("\n")).toMatch(/--from-bucket needs BACKUP_S3_ENDPOINT/);
expect(runTool).not.toHaveBeenCalled();
});
it("lists what is there", async () => {
writeDump();
expect(await main(["list"], env())).toBe(0);
expect(lines.join("\n")).toContain(`${BASE}.dump daily`);
});
});
+35 -4
View File
@@ -21,7 +21,9 @@ ROOT=$(git rev-parse --show-toplevel)
# The last release whose compose file ran Postgres 16.
OLD_REF=${OLD_REF:-v1.45.0}
PROJECT=hl-db-upgrade-check
PORT=${CHECK_DB_PORT:-55416}
# A free port unless CHECK_DB_PORT names one, so the check never collides with
# another stack on the same machine.
PORT=${CHECK_DB_PORT:-$(node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})")}
WORK=$(mktemp -d)
export PRISMA_HIDE_UPDATE_MESSAGE=1
@@ -63,11 +65,40 @@ run_upgrade() {
}
upgrade_log() { docker logs healthlog-db-upgrade 2>&1 | grep '^db-upgrade:' || true; }
# The container can report healthy before its published port forwards (Docker
# Desktop and OrbStack set the forward up a moment later), and a migrate
# started in that gap failed with "Can't reach database server". So wait
# until the port answers, for at most thirty seconds, and retry the migrate
# within the same bound.
wait_for_port() {
local i=0
until node -e "
const s = require('node:net').connect(Number(process.argv[1]), '127.0.0.1');
s.on('connect', () => { s.end(); process.exit(0); });
s.on('error', () => process.exit(1));
" "$PORT" 2>/dev/null; do
i=$((i + 1))
[ "$i" -lt 30 ] || return 1
sleep 1
done
}
migrate_old() {
local started=$SECONDS
until DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do
if [ $((SECONDS - started)) -ge 30 ] || ! grep -q "P1001" "$WORK/migrate.log"; then
cat "$WORK/migrate.log" >&2
return 1
fi
sleep 1
done
}
down_all
step "a v1.45 database on Postgres 16"
old up -d --wait db >/dev/null 2>&1
DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >/dev/null
old up -d --wait db >"$WORK/up.log" 2>&1 || { cat "$WORK/up.log" >&2; fail "the v1.45 database did not start"; }
wait_for_port || fail "port $PORT did not answer within thirty seconds"
migrate_old || fail "the migrations did not apply to the v1.45 database"
old exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -q <<'SQL'
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
CREATE SCHEMA check_probe;
+346
View File
@@ -0,0 +1,346 @@
#!/usr/bin/env bash
# Plays the documented disaster recovery end to end against real containers:
# a server is lost with everything on it, and a new one is rebuilt from the
# nightly database dump and the keys alone (docs/ops/disaster-recovery.md).
#
# 1. Build the app image from this checkout and start the bundled compose
# stack with fixed throwaway keys, and an S3-compatible bucket outside
# it (Versity S3 Gateway, pinned) as the off-host target.
# 2. Seed real data: the demo seed (documents, encrypted notes, years of
# readings) and a Guardian with a managed profile, written by the app's
# own code; then, through the API, a reading with an encrypted note.
# 3. Let the scheduled `database-backup` job take the dump (its cron is set
# to every minute here) and wait for its ledger row: kept on the server
# and uploaded to the bucket.
# 4. DESTROY every volume of the stack, the local dumps included. The
# bucket lives outside the stack and survives, as an off-host copy does.
# 5. Start a new stack and restore with the keys alone:
# DRILL_SOURCE=bucket (the default, the disaster path)
# `docker compose run --rm db-restore latest --from-bucket`;
# DRILL_SOURCE=local (the dump files had been copied off the host)
# the files go into the new backups volume, then
# `docker compose run --rm db-restore latest`.
# 6. Count every table against the manifest, independently of the restore.
# 7. Boot the app on the restored database and check: the health endpoint,
# the demo login, the encrypted note read back through the API, a
# document opened, and the Guardian still seeing the managed profile.
#
# DRILL_MUTATION breaks one step on purpose, and the drill must then fail
# (a check that cannot fail proves nothing):
# wrong-key restore with a different active encryption key
# wrong-backup-key restore from the bucket with a different
# BACKUP_ENCRYPTION_KEY (DRILL_SOURCE=bucket)
# skip-table a dump whose archive lacks one table's rows, the
# manifest's checksum fixed up and its counts left true
# (DRILL_SOURCE=local)
# non-empty-target the new app starts and migrates before the restore
#
# Needs Docker, openssl, pnpm and the repo's node_modules (for the seeds). Run
# it from anywhere in the repo: `bash scripts/compose-dr-drill.sh`.
# DRILL_IMAGE names an image to use instead of building one. The ports are
# free ones picked at start unless DRILL_APP_PORT, DRILL_DB_PORT or
# DRILL_BUCKET_PORT name them.
set -euo pipefail
ROOT=$(git rev-parse --show-toplevel)
PROJECT=hl-dr-drill
free_port() {
node -e "const s=require('node:net').createServer().listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})"
}
APP_PORT=${DRILL_APP_PORT:-$(free_port)}
DB_PORT=${DRILL_DB_PORT:-$(free_port)}
BUCKET_PORT=${DRILL_BUCKET_PORT:-$(free_port)}
IMAGE=${DRILL_IMAGE:-}
MUTATION=${DRILL_MUTATION:-}
SOURCE=${DRILL_SOURCE:-bucket}
OFFSITE=$PROJECT-offsite
BUCKET=$PROJECT-bucket
BUCKET_IMAGE=versity/versitygw:v1.8.0@sha256:30292fc2eeacc67a36993b01f7a7a5e3361a19cced0e80c1d71cfa2a4b0a2499
WORK=$(mktemp -d)
BASE_URL="http://127.0.0.1:$APP_PORT"
case "$SOURCE/$MUTATION" in
bucket/ | local/ | */wrong-key | */non-empty-target | bucket/wrong-backup-key | local/skip-table) ;;
*)
echo "DRILL_SOURCE=$SOURCE with DRILL_MUTATION=$MUTATION is not a drill" >&2
exit 2
;;
esac
export PRISMA_HIDE_UPDATE_MESSAGE=1
export POSTGRES_PASSWORD
POSTGRES_PASSWORD=$(openssl rand -hex 16)
# Fixed throwaway keys: the restore gets these and nothing else of the lost
# host. Two encryption keys, the second active, so a retired key is carried.
export ENCRYPTION_KEY=""
export ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"2222222222222222222222222222222222222222222222222222222222222222"}'
export ENCRYPTION_ACTIVE_KEY_ID=v2
export API_TOKEN_HMAC_KEY=3333333333333333333333333333333333333333333333333333333333333333
export SESSION_COOKIE_SECURE=false
export NEXT_PUBLIC_APP_URL=$BASE_URL APP_URL=$BASE_URL
export DB_BACKUP_SCHEDULE="* * * * *"
# The off-host bucket, reached from the stack by its container name on a
# network of its own that outlives the stack, as a real bucket does.
export BACKUP_S3_ENDPOINT=http://$BUCKET:7070
export BACKUP_S3_BUCKET=healthlog-backups
export BACKUP_S3_ACCESS_KEY=drill-access
export BACKUP_S3_SECRET_KEY=drill-secret-0123456789
export BACKUP_S3_REGION=us-east-1
export BACKUP_ENCRYPTION_KEY=5555555555555555555555555555555555555555555555555555555555555555
step() { echo "== $*"; }
fail() {
echo "FAIL: $*" >&2
docker logs "$PROJECT-app" 2>&1 | tail -40 >&2 || true
exit 1
}
if [ -z "$IMAGE" ]; then
IMAGE=healthlog-dr-drill:local
step "building the app image from this checkout"
docker build -q \
--build-arg NEXT_PUBLIC_APP_VERSION="drill-$(git -C "$ROOT" rev-parse --short HEAD)" \
--build-arg NEXT_PUBLIC_APP_BUILD_SHA="$(git -C "$ROOT" rev-parse --short HEAD)" \
-t "$IMAGE" "$ROOT" >/dev/null
fi
# The bundled compose file, with this image, its own names and ports, so the
# drill never meets another stack on the same machine, and the app and the
# restore on the bucket's network.
cat >"$WORK/drill.yml" <<YAML
services:
app:
image: $IMAGE
pull_policy: never
container_name: $PROJECT-app
ports: !override
- "127.0.0.1:$APP_PORT:3000"
networks: [default, bucket]
db:
container_name: $PROJECT-db
ports:
- "127.0.0.1:$DB_PORT:5432"
db-upgrade:
container_name: $PROJECT-db-upgrade
db-restore:
image: $IMAGE
pull_policy: never
networks: [default, bucket]
networks:
bucket:
external: true
name: $BUCKET
YAML
stack() { docker compose -p "$PROJECT" -f "$ROOT/docker-compose.yml" -f "$WORK/drill.yml" "$@"; }
sql() { docker exec "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$1"; }
teardown() {
stack down -v --remove-orphans >/dev/null 2>&1 || true
docker volume rm "$OFFSITE" >/dev/null 2>&1 || true
docker rm -f "$BUCKET" >/dev/null 2>&1 || true
docker volume rm "$BUCKET-data" >/dev/null 2>&1 || true
docker network rm "$BUCKET" >/dev/null 2>&1 || true
}
trap 'teardown; rm -rf "$WORK"' EXIT
wait_healthy() {
local i=0
until [ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/api/health")" = 200 ]; do
i=$((i + 1))
[ "$i" -lt 150 ] || fail "the app did not report healthy within five minutes"
docker inspect -f '{{.State.Running}}' "$PROJECT-app" 2>/dev/null | grep -q true ||
fail "the app container stopped"
sleep 2
done
}
# POST/GET as a signed-in account, with its own cookie jar.
login() {
local jar=$1 user=$2 password=$3
local code
code=$(curl -s -o "$WORK/login.json" -w '%{http_code}' -c "$jar" \
-H 'content-type: application/json' \
-d "{\"email\":\"$user\",\"password\":\"$password\"}" "$BASE_URL/api/auth/login")
[ "$code" = 200 ] || fail "login as $user answered $code: $(cat "$WORK/login.json")"
}
api() { curl -s -b "$1" -w '\n%{http_code}' "${@:2}"; }
json() { node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const v=JSON.parse(s);console.log((${1})(v))})"; }
# A shell in the app image as root, with volumes, for moving the dump files
# the way an operator copies them; `cp -p` keeps the app user as owner.
as_root() {
local command=$1 mounts=()
shift
for volume in "$@"; do mounts+=(-v "$volume"); done
docker run --rm --user 0:0 --entrypoint sh "${mounts[@]}" "$IMAGE" -c "$command"
}
teardown
step "an off-host bucket, outside the stack"
docker network create "$BUCKET" >/dev/null
docker run -d --name "$BUCKET" --network "$BUCKET" -p "127.0.0.1:$BUCKET_PORT:7070" \
-v "$BUCKET-data:/data" --entrypoint sh "$BUCKET_IMAGE" -c \
"mkdir -p /data/$BACKUP_S3_BUCKET && exec versitygw --access $BACKUP_S3_ACCESS_KEY --secret $BACKUP_S3_SECRET_KEY posix /data" \
>/dev/null || fail "the bucket did not start"
step "a running server with real data"
stack up -d app >"$WORK/up.log" 2>&1 || { tail -20 "$WORK/up.log" >&2; fail "the stack did not start"; }
wait_healthy
DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$DB_PORT/healthlog?schema=public"
export DATABASE_URL
(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/seed-demo.ts) >"$WORK/seed.log" 2>&1 ||
{ tail -20 "$WORK/seed.log" >&2; fail "the demo seed failed"; }
IDS=$(cd "$ROOT" && pnpm dlx tsx@4.23.1 scripts/dr-drill-seed.ts | tail -1)
PROFILE_ID=$(echo "$IDS" | json 'v=>v.profileId')
unset DATABASE_URL
login "$WORK/demo.jar" demo demo123demo123
NOTE="drill note $(openssl rand -hex 6)"
OUT=$(api "$WORK/demo.jar" -H 'content-type: application/json' \
-d "{\"type\":\"WEIGHT\",\"value\":81.4,\"measuredAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"notes\":\"$NOTE\"}" \
"$BASE_URL/api/measurements")
[ "$(echo "$OUT" | tail -1)" = 201 ] || [ "$(echo "$OUT" | tail -1)" = 200 ] ||
fail "creating a reading with a note answered: $OUT"
READING_ID=$(echo "$OUT" | sed '$d' | json 'v=>v.data.id')
[ "$(sql "SELECT count(*) FROM measurements WHERE id = '$READING_ID' AND position('$NOTE' in measurements::text) > 0")" = 0 ] ||
fail "the note is stored in plain text"
DOCUMENTS=$(sql "SELECT count(*) FROM inbound_documents WHERE deleted_at IS NULL")
[ "$DOCUMENTS" -gt 0 ] || fail "the seed wrote no documents"
SEEDED_AT=$(sql "SELECT now()")
step "the scheduled job takes the dump and uploads it"
i=0
until DUMP=$(sql "SELECT file_name FROM database_backup_runs WHERE status = 'succeeded' AND local_result = 'written' AND offhost_result = 'uploaded' AND started_at > '$SEEDED_AT'::timestamptz ORDER BY started_at DESC LIMIT 1") && [ -n "$DUMP" ]; do
i=$((i + 1))
[ "$i" -lt 120 ] || fail "no succeeded and uploaded database-backup run within four minutes: $(sql "SELECT status, local_result, offhost_result, error FROM database_backup_runs ORDER BY started_at DESC LIMIT 3")"
sleep 2
done
BASE=${DUMP%.dump}
echo " $DUMP"
if [ "$SOURCE" = local ]; then
step "the dump files are copied off the host"
docker volume create "$OFFSITE" >/dev/null
as_root "cp -p /from/$BASE.dump /from/$BASE.manifest.json /to/" \
"${PROJECT}_backups:/from:ro" "$OFFSITE:/to" ||
fail "could not copy the dump off the host"
fi
step "the host is lost: every volume of the stack is destroyed"
stack down -v --remove-orphans >/dev/null 2>&1
[ -z "$(docker volume ls -q --filter "name=^${PROJECT}_")" ] || fail "a volume of the old stack survived"
if [ "$MUTATION" = skip-table ]; then
step "MUTATION: the dump loses the rows of public.measurements"
# A scratch Postgres restores the real dump and dumps it again without that
# table's rows; the files stay the app user's, read and written by root.
docker run --rm -v "$OFFSITE:/d" -e BASE="$BASE" --entrypoint sh postgres:18.6-alpine -ec '
gosu postgres initdb -D /tmp/pg >/dev/null
gosu postgres pg_ctl -D /tmp/pg -o "-c listen_addresses=" -w start >/dev/null
gosu postgres createdb scratch
gosu postgres pg_restore --no-owner --no-privileges -d scratch <"/d/$BASE.dump"
gosu postgres pg_dump -Fc --exclude-table-data=public.measurements scratch >"/d/$BASE.dump"
sum=$(sha256sum "/d/$BASE.dump" | cut -d" " -f1)
size=$(wc -c <"/d/$BASE.dump")
sed -i "s/\"sha256\": \"[0-9a-f]*\"/\"sha256\": \"$sum\"/; s/\"bytes\": [0-9]*/\"bytes\": $size/" "/d/$BASE.manifest.json"
chown 1001:1001 "/d/$BASE.dump"' || fail "could not build the mutated dump"
fi
step "a new server: the compose file, the keys and the $SOURCE copy"
if [ "$MUTATION" = wrong-key ]; then
echo " MUTATION: the restore gets a different active encryption key"
ENCRYPTION_KEYS='{"v1":"1111111111111111111111111111111111111111111111111111111111111111","v2":"4444444444444444444444444444444444444444444444444444444444444444"}'
fi
if [ "$MUTATION" = wrong-backup-key ]; then
echo " MUTATION: the restore gets a different BACKUP_ENCRYPTION_KEY"
BACKUP_ENCRYPTION_KEY=6666666666666666666666666666666666666666666666666666666666666666
fi
if [ "$SOURCE" = local ]; then
# Mounting the image's directory fills the new volume with the app user's
# ownership; then the two files go in.
stack run --rm --no-deps --entrypoint true db-restore >/dev/null 2>&1
as_root "cp -p /from/* /to/" "$OFFSITE:/from:ro" "${PROJECT}_backups:/to" ||
fail "could not put the dump on the new server"
RESTORE=(latest)
else
RESTORE=(latest --from-bucket)
fi
if [ "$MUTATION" = non-empty-target ]; then
echo " MUTATION: the new app starts (and migrates) before the restore"
stack up -d app >/dev/null 2>&1
wait_healthy
fi
stack up -d db >/dev/null 2>&1
if ! stack run --rm db-restore "${RESTORE[@]}" 2>&1 | tee "$WORK/restore.log"; then
fail "the restore failed"
fi
grep -q "^Restored $DUMP" "$WORK/restore.log" || fail "the restore did not say it restored $DUMP"
step "every table against the manifest, counted apart from the restore"
if [ "$SOURCE" = local ]; then
as_root "cat /d/$BASE.manifest.json" "$OFFSITE:/d:ro" >"$WORK/manifest.json"
else
# From the bucket, opened with the restore's own reader.
(cd "$ROOT" && node --input-type=module -e '
import { writeFileSync } from "node:fs";
import { join } from "node:path";
const core = await import(join(process.cwd(), "src/lib/database-backup/dump-core.mjs"));
const [port, base, work] = process.argv.slice(1);
const bucket = await core.openBucket({ ...core.bucketConfigFromEnv(process.env), endpoint: `http://127.0.0.1:${port}` });
const name = `${base}.manifest.json.enc`;
await bucket.download(core.offhostObjectKey(name), join(work, name));
await core.decryptOffhostFile({ inPath: join(work, name), outPath: join(work, "manifest.json"), objectKey: core.offhostObjectKey(name), keys: core.offhostKeysFromEnv(process.env) });
' "$BUCKET_PORT" "$BASE" "$WORK") || fail "could not read the manifest from the bucket"
fi
node -e '
const m = require(process.argv[1]);
const q = String.fromCharCode(39);
const lit = (t) => q + t.split(q).join(q + q) + q;
console.log(Object.keys(m.rowCounts).map((t) => `SELECT ${lit(t)} || chr(9) || count(*) FROM ${t}`).join(" UNION ALL ") + ";");
' "$WORK/manifest.json" >"$WORK/count.sql"
docker exec -i "$PROJECT-db" psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atq <"$WORK/count.sql" >"$WORK/counts.tsv"
node -e '
const fs = require("node:fs");
const m = require(process.argv[1]);
const got = Object.fromEntries(fs.readFileSync(process.argv[2], "utf8").trim().split("\n").map((l) => l.split("\t")).map(([t, n]) => [t, Number(n)]));
const bad = Object.entries(m.rowCounts).filter(([t, n]) => got[t] !== n);
if (bad.length) { console.error(bad.map(([t, n]) => `${t}: manifest ${n}, database ${got[t]}`).join("\n")); process.exit(1); }
if (m.rowCounts["public.users"] < 3) { console.error("fewer than three accounts in the dump"); process.exit(1); }
const c = m.rowCounts;
console.log(` ${Object.keys(c).length} tables, ${Object.values(c).reduce((a, b) => a + b, 0)} rows: ` +
`${c["public.users"]} accounts, ${c["public.account_grants"]} grants, ${c["public.measurements"]} readings, ` +
`${c["public.inbound_documents"]} documents, ${c["pgboss.job"] ?? 0} queued jobs`);
' "$WORK/manifest.json" "$WORK/counts.tsv" || fail "the restored row counts differ from the manifest"
step "the app on the restored database"
stack up -d app >/dev/null 2>&1 || fail "the app did not start on the restored database"
wait_healthy
[ "$(curl -s "$BASE_URL/api/health" | json 'v=>v.status ?? v.data?.status')" = ok ] || fail "health is not ok"
login "$WORK/demo2.jar" demo demo123demo123
OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/measurements/$READING_ID")
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "reading the note back answered: $OUT"
[ "$(echo "$OUT" | sed '$d' | json 'v=>v.data.notes')" = "$NOTE" ] || fail "the encrypted note did not read back"
echo " the encrypted note reads back"
OUT=$(api "$WORK/demo2.jar" "$BASE_URL/api/documents/inbound")
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing documents answered: $OUT"
DOC_ID=$(echo "$OUT" | sed '$d' | json 'v=>(v.data.documents ?? v.data.items ?? v.data)[0].id')
CODE=$(curl -s -b "$WORK/demo2.jar" -o "$WORK/doc.bin" -w '%{http_code}' "$BASE_URL/api/documents/inbound/$DOC_ID/original")
[ "$CODE" = 200 ] || fail "opening document $DOC_ID answered $CODE"
[ "$(head -c 4 "$WORK/doc.bin")" = "%PDF" ] || fail "document $DOC_ID did not open to a PDF"
echo " a document opens ($(wc -c <"$WORK/doc.bin" | tr -d ' ') bytes)"
login "$WORK/guardian.jar" drill-guardian drill-guardian-password-1
OUT=$(api "$WORK/guardian.jar" "$BASE_URL/api/account/grants")
[ "$(echo "$OUT" | tail -1)" = 200 ] || fail "listing the Guardian's grants answered: $OUT"
echo "$OUT" | sed '$d' | node -e "
let s=''; process.stdin.on('data', d => s += d).on('end', () => {
const received = JSON.parse(s).data.received ?? [];
const found = received.some((g) => JSON.stringify(g).includes('$PROFILE_ID'));
process.exit(found ? 0 : 1);
});" || fail "the Guardian no longer sees the managed profile"
echo " the Guardian still sees the managed profile"
echo "OK: the server was rebuilt from the $SOURCE copy of its dump and the keys alone"
+156
View File
@@ -0,0 +1,156 @@
/**
* The pre-migration dump: a whole-database dump the entrypoint takes before
* `prisma migrate deploy` applies anything.
*
* node scripts/db-dump.mjs pre-migration
*
* A migration is not undone by going back to the previous image, and the
* bundled compose file pulls `:latest` and migrates on every start. So when
* this image brings migrations the database has not applied, it first writes
* `healthlog-<timestamp>-pre-migration.dump` with its manifest into
* `DB_BACKUP_DIR` (`src/lib/database-backup/dump-core.mjs`), and keeps the
* newest two.
*
* Exit 0: the dump was written, or there was nothing to do (no pending
* migration, a new database, `DB_BACKUP_BEFORE_MIGRATE=false`), or there is no
* writable directory to put it in, which is said as a warning and is not a
* reason to keep the app down. Exit 1: the directory is writable and the dump
* failed. The entrypoint then stops the boot and names the switch that skips
* the dump, because migrating without the way back the operator was promised
* is the one thing this step exists to prevent.
*
* Fast on purpose: one snapshot, `pg_dump`, a count per table for the
* manifest, `pg_restore --list` to prove the file reads. Nothing else.
*/
import { readFileSync } from "node:fs";
import { basename, join } from "node:path";
import { pathToFileURL } from "node:url";
import {
applyRetention,
connect,
createDatabaseDump,
isWritableDirectory,
latestMigration,
listUserTables,
connectionFromUrl,
preMigrationDumpEnabled,
resolveBackupDir,
resolveDailyKeep,
} from "../src/lib/database-backup/dump-core.mjs";
import { pendingMigrations } from "./pending-migrations.mjs";
const log = (line) => console.log(`HealthLog: ${line}`);
const warn = (line) => console.error(`HealthLog: WARNING — ${line}`);
/** The image's version: the build argument, else package.json beside it. */
function imageVersion(env) {
const fromEnv = (env.NEXT_PUBLIC_APP_VERSION ?? "").trim();
if (fromEnv) return fromEnv;
try {
return JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8"))
.version;
} catch {
return null;
}
}
/** Megabytes as an operator reads them. */
function mb(bytes) {
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
}
export async function preMigrationDump(env = process.env) {
if (!preMigrationDumpEnabled(env)) {
log("DB_BACKUP_BEFORE_MIGRATE=false, no dump before migrating.");
return 0;
}
const databaseUrl = env.DATABASE_URL;
if (!databaseUrl) return 0;
let pending;
try {
pending = await pendingMigrations({
connectionString: databaseUrl,
migrationsDir: join(process.cwd(), "prisma", "migrations"),
});
} catch (error) {
warn(
`could not count pending migrations (${error instanceof Error ? error.message : String(error)}); no dump before migrating.`,
);
return 0;
}
if (pending.length === 0) return 0;
// A database no migration has touched holds nothing to lose.
const client = await connect(databaseUrl);
let applied;
let tables;
try {
applied = await latestMigration(
client,
connectionFromUrl(databaseUrl).schema,
);
tables = (await listUserTables(client)).length;
} finally {
await client.end().catch(() => {});
}
if (applied === null && tables === 0) {
log("new database, no dump before the first migrations.");
return 0;
}
const dir = resolveBackupDir(env);
if (!isWritableDirectory(dir)) {
warn(
`${dir} is not writable, so no dump is taken before the ${pending.length} pending migration(s). ` +
"Mount the backups volume (docker-compose.yml) to keep a way back; continuing.",
);
return 0;
}
log(
`taking a database dump before ${pending.length} pending migration(s) into ${dir}...`,
);
const started = Date.now();
try {
const { dumpPath, manifest } = await createDatabaseDump({
databaseUrl,
dir,
kind: "pre-migration",
env,
appVersion: imageVersion(env),
buildSha: (env.NEXT_PUBLIC_APP_BUILD_SHA ?? "").trim() || null,
});
log(
`pre-migration dump written: ${basename(dumpPath)} (${mb(manifest.dump.bytes)}, ` +
`${Object.keys(manifest.rowCounts).length} tables, ${Math.round((Date.now() - started) / 1000)} s).`,
);
} catch (error) {
console.error(
`HealthLog: ERROR — the database dump before migrating failed: ${error instanceof Error ? error.message : String(error)}`,
);
return 1;
}
try {
await applyRetention(dir, { dailyKeep: resolveDailyKeep(env) });
} catch (error) {
warn(
`could not remove old dumps in ${dir} (${error instanceof Error ? error.message : String(error)}).`,
);
}
return 0;
}
async function main() {
const kind = process.argv[2];
if (kind !== "pre-migration") {
console.error("db-dump: usage: node scripts/db-dump.mjs pre-migration");
process.exit(2);
}
process.exit(await preMigrationDump());
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
await main();
}
+415
View File
@@ -0,0 +1,415 @@
/**
* Restore a whole-database dump into an empty database.
*
* docker compose run --rm db-restore list [directory]
* docker compose run --rm db-restore list --from-bucket
* docker compose run --rm db-restore latest [--from-bucket]
* docker compose run --rm db-restore <file> [--ignore-key-mismatch]
*
* Runs in the app image with plain `node` (no `tsx`, no source checkout), as
* the `db-restore` compose service, so a new server needs only the compose
* file, the keys and the dump. `<file>` is a name in `DB_BACKUP_DIR` or a
* path inside the container: a `.dump` written on a server, or a `.dump.enc`
* downloaded from the off-host bucket (`database/` prefix), with its manifest
* (`.manifest.json` or `.manifest.json.enc`) beside it. With `--from-bucket`
* it fetches the pair from the bucket the `BACKUP_S3_*` variables name (the
* newest for `latest`), which is the restore after the host and its local
* dumps are gone.
*
* In this order, and it stops at the first thing that is wrong, before it
* writes anything:
* 1. an off-host copy is opened with BACKUP_ENCRYPTION_KEY (or a key in
* BACKUP_ENCRYPTION_PREVIOUS_KEYS) into a temporary file;
* 2. the dump's SHA-256 and size must equal the manifest's;
* 3. the keys in this environment are compared with the manifest's
* fingerprints; each one missing or different is named, and a mismatch
* of the active encryption key refuses (`--ignore-key-mismatch` restores
* anyway, for an operator who knows the key is lost);
* 4. the target database must hold no table at all. This never overwrites.
* Then `pg_restore` in one transaction, `ANALYZE`, and a count of every table
* against the manifest. A count that differs empties the database again and
* fails, so the app is never started on a restore that does not match.
*
* Exit 0 restored (or listed), 1 refused or failed, 2 usage.
*/
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, isAbsolute, join } from "node:path";
import { pathToFileURL } from "node:url";
import {
compareKeyFingerprints,
compareRowCounts,
connect,
countRows,
decryptOffhostFile,
emptyDatabase,
hashFile,
isGoodSet,
isWritableDirectory,
keyFingerprints,
listDumpSets,
listUserTables,
offhostKeysFromEnv,
offhostObjectKey,
parseDumpFileName,
parseManifest,
resolveBackupDir,
runTool,
statOrNull,
connectionFromUrl,
bucketConfigFromEnv,
openBucket,
pickBucketDump,
OFFHOST_DATABASE_PREFIX,
} from "../src/lib/database-backup/dump-core.mjs";
import { shippedMigrations } from "./pending-migrations.mjs";
const out = (line = "") => console.log(line);
const say = (line) => console.log(`db-restore: ${line}`);
class Refusal extends Error {}
function mb(bytes) {
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
}
/**
* Whether this image knows the dump's newest migration. A dump from a newer
* release restored under an older image leaves the app with a schema its code
* does not expect, so the restore says which image to run.
*/
function imageKnowsMigration(name) {
if (!name) return true;
try {
return shippedMigrations(
join(process.cwd(), "prisma", "migrations"),
).includes(name);
} catch {
return true;
}
}
function listCommand(dir) {
const sets = listDumpSets(dir);
if (sets.length === 0) {
say(`no dumps in ${dir}`);
return 0;
}
out(`Dumps in ${dir}, newest first:`);
for (const set of sets) {
const state = isGoodSet(set)
? `${mb(set.dump.bytes)}`
: set.encryptedDump
? "off-host copy (.enc)"
: set.dump
? "no manifest, not restorable"
: "incomplete";
out(
` ${set.base}.dump${set.encryptedDump && !set.dump ? ".enc" : ""} ${set.kind} ${set.timestamp.toISOString()} ${state}`,
);
}
return 0;
}
/**
* The dump and manifest to restore: a local pair, or an encrypted pair
* opened into `scratch`.
*/
async function resolveInput(arg, dir, env) {
if (arg === "latest") {
const set = listDumpSets(dir).find(
(s) =>
isGoodSet(s) ||
(s.encryptedDump !== null &&
(s.encryptedManifest !== null || s.manifest !== null)),
);
if (!set) throw new Refusal(`no restorable dump in ${dir}`);
arg = set.dump ? set.dump.path : set.encryptedDump;
}
const path = isAbsolute(arg) ? arg : join(dir, arg);
if (!(await statOrNull(path))) throw new Refusal(`${path} does not exist`);
const name = basename(path);
const parsed = parseDumpFileName(name);
if (!parsed || parsed.role !== "dump" || parsed.variant === "partial") {
throw new Refusal(
`${name} is not a HealthLog database dump (healthlog-<time>-<kind>.dump or .dump.enc)`,
);
}
const folder = dirname(path);
const plainManifest = join(folder, `${parsed.base}.manifest.json`);
const encManifest = `${plainManifest}.enc`;
if (parsed.variant === "final") {
if (!(await statOrNull(plainManifest))) {
throw new Refusal(
`${basename(plainManifest)} is missing beside ${name}; a dump is restored only with its manifest`,
);
}
return {
dumpPath: path,
manifest: parseManifest(
JSON.parse(await readFile(plainManifest, "utf8")),
),
scratch: null,
};
}
// An off-host copy. Open it next to the backups if there is room to write
// there, in the container's temp directory otherwise.
const keys = offhostKeysFromEnv(env);
if (keys.length === 0) {
throw new Refusal(
"this is an off-host copy and BACKUP_ENCRYPTION_KEY is not set; it cannot be opened without it",
);
}
const scratch = await mkdtemp(
join(isWritableDirectory(dir) ? dir : tmpdir(), ".restore-"),
);
try {
let manifest;
if (await statOrNull(encManifest)) {
const opened = join(scratch, `${parsed.base}.manifest.json`);
await decryptOffhostFile({
inPath: encManifest,
outPath: opened,
objectKey: offhostObjectKey(basename(encManifest)),
keys,
});
manifest = parseManifest(JSON.parse(await readFile(opened, "utf8")));
} else if (await statOrNull(plainManifest)) {
manifest = parseManifest(
JSON.parse(await readFile(plainManifest, "utf8")),
);
} else {
throw new Refusal(
`${basename(encManifest)} is missing beside ${name}; download it from the bucket too`,
);
}
say(`opening the off-host copy ${name}...`);
const dumpPath = join(scratch, `${parsed.base}.dump`);
await decryptOffhostFile({
inPath: path,
outPath: dumpPath,
objectKey: offhostObjectKey(name),
keys,
});
return { dumpPath, manifest, scratch };
} catch (error) {
await rm(scratch, { recursive: true, force: true });
throw error instanceof Refusal ? error : new Refusal(error.message);
}
}
/**
* Fetch a dump and its manifest from the off-host bucket (`BACKUP_S3_*`) into
* a scratch directory: the newest pair for `latest`, the named one otherwise.
* Answers the downloaded `.dump.enc` and the directory to remove afterwards.
*/
async function downloadFromBucket(arg, dir, env) {
const config = bucketConfigFromEnv(env);
if (!config) {
throw new Refusal(
"--from-bucket needs BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY and BACKUP_S3_SECRET_KEY",
);
}
const bucket = await openBucket(config);
const keys = await bucket.list(OFFHOST_DATABASE_PREFIX);
const pick = pickBucketDump(keys, arg === "latest" ? null : basename(arg));
if (!pick) {
throw new Refusal(
arg === "latest"
? `no database dump with its manifest under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`
: `${basename(arg)} and its manifest are not both under ${OFFHOST_DATABASE_PREFIX} in bucket ${config.bucket}`,
);
}
const scratch = await mkdtemp(
join(isWritableDirectory(dir) ? dir : tmpdir(), ".download-"),
);
try {
say(
`downloading ${pick.dumpKey} and its manifest from ${config.bucket}...`,
);
const dumpPath = join(scratch, basename(pick.dumpKey));
await bucket.download(
pick.manifestKey,
join(scratch, basename(pick.manifestKey)),
);
await bucket.download(pick.dumpKey, dumpPath);
return { dumpPath, scratch };
} catch (error) {
await rm(scratch, { recursive: true, force: true });
throw new Refusal(
`could not download from the bucket: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
async function listBucketCommand(env) {
const config = bucketConfigFromEnv(env);
if (!config)
throw new Refusal("--from-bucket needs the BACKUP_S3_* variables");
const keys = (await (await openBucket(config)).list(OFFHOST_DATABASE_PREFIX))
.filter((k) => k.endsWith(".dump.enc"))
.sort()
.reverse();
out(
`Dumps in bucket ${config.bucket} under ${OFFHOST_DATABASE_PREFIX}, newest first:`,
);
for (const key of keys) out(` ${key}`);
if (keys.length === 0) say("none");
return 0;
}
async function restoreCommand(arg, flags, env) {
const dir = resolveBackupDir(env);
const databaseUrl = env.DATABASE_URL;
if (!databaseUrl) throw new Refusal("DATABASE_URL is not set");
let input = arg;
let download = null;
if (flags.has("--from-bucket")) {
download = await downloadFromBucket(arg, dir, env);
input = download.dumpPath;
}
let resolved;
try {
resolved = await resolveInput(input, dir, env);
} finally {
// The sealed copy is not needed once it is opened (or failed to open).
if (download) await rm(download.scratch, { recursive: true, force: true });
}
const { dumpPath, manifest, scratch } = resolved;
try {
say(
`dump ${manifest.file}: taken ${manifest.createdAt} by HealthLog ${manifest.appVersion ?? "(unknown version)"}, ` +
`newest migration ${manifest.latestMigration ?? "(none)"}, Postgres ${manifest.postgres.serverVersion}, ` +
`${Object.keys(manifest.rowCounts).length} tables`,
);
if (!imageKnowsMigration(manifest.latestMigration)) {
say(
`warning: this image does not know the dump's newest migration ${manifest.latestMigration}; ` +
`start the app with HealthLog ${manifest.appVersion ?? "the version that wrote it"} or newer (HEALTHLOG_IMAGE_REF), not this image`,
);
}
if (basename(dumpPath) !== manifest.file) {
throw new Refusal(
`the manifest describes ${manifest.file}, not ${basename(dumpPath)}`,
);
}
// 2. The file is the one the manifest describes.
const { sha256, bytes } = await hashFile(dumpPath);
if (sha256 !== manifest.dump.sha256 || bytes !== manifest.dump.bytes) {
throw new Refusal(
`the dump does not match its manifest (SHA-256 ${sha256.slice(0, 12)}…, ${bytes} bytes; ` +
`the manifest says ${manifest.dump.sha256.slice(0, 12)}…, ${manifest.dump.bytes} bytes). It is damaged or not this dump.`,
);
}
say(`checksum matches the manifest (${mb(bytes)})`);
// 3. The keys.
const keys = compareKeyFingerprints(manifest.keys, keyFingerprints(env));
if (keys.findings.length === 0) {
say("every key the dump was written with is configured here");
}
for (const finding of keys.findings) {
say(
`${finding.severity === "refuse" ? "KEY MISMATCH" : "warning"}: ${finding.key}: ${finding.detail}`,
);
}
if (keys.refuse && !flags.has("--ignore-key-mismatch")) {
throw new Refusal(
"refusing to restore: set the encryption keys this dump was written with (see above). " +
"Only if that key is lost for good, run again with --ignore-key-mismatch; everything encrypted will stay unreadable.",
);
}
// 4. Never over existing data.
const client = await connect(databaseUrl);
try {
const tables = await listUserTables(client);
if (tables.length > 0) {
throw new Refusal(
`the target database already holds ${tables.length} ${tables.length === 1 ? "table" : "tables"} (${tables.slice(0, 3).join(", ")}${tables.length > 3 ? ", …" : ""}). ` +
"A restore goes only into an empty database: start it on a new, empty database volume, and do not start the app before the restore.",
);
}
say("restoring (one transaction)...");
const started = Date.now();
await runTool(
"pg_restore",
[
"--no-owner",
"--no-privileges",
"--single-transaction",
"--exit-on-error",
"--no-password",
`--dbname=${connectionFromUrl(databaseUrl).pgEnv.PGDATABASE}`,
dumpPath,
],
{ env: connectionFromUrl(databaseUrl).pgEnv },
);
say(`restored in ${Math.round((Date.now() - started) / 1000)} s`);
await client.query("ANALYZE");
say("planner statistics gathered");
const problems = compareRowCounts(
manifest.rowCounts,
await countRows(client),
);
if (problems.length > 0) {
for (const line of problems.slice(0, 20)) say(`row count: ${line}`);
await emptyDatabase(client);
throw new Refusal(
`${problems.length} table(s) do not match the manifest; the database was emptied again. Do not start the app on it.`,
);
}
say(
`all ${Object.keys(manifest.rowCounts).length} tables hold the row counts the manifest names`,
);
} finally {
await client.end().catch(() => {});
}
out();
out(
`Restored ${manifest.file}. Start the app with the image this dump came from or a newer one ` +
`(HealthLog ${manifest.appVersion ?? "?"}, newest migration ${manifest.latestMigration ?? "none"}): docker compose up -d`,
);
return 0;
} finally {
if (scratch) await rm(scratch, { recursive: true, force: true });
}
}
export async function main(argv = process.argv.slice(2), env = process.env) {
const flags = new Set(argv.filter((a) => a.startsWith("--")));
const [command] = argv.filter((a) => !a.startsWith("--"));
if (!command) {
console.error(
"db-restore: usage: db-restore list [directory] | latest | <file> [--from-bucket] [--ignore-key-mismatch]",
);
return 2;
}
try {
if (command === "list" && flags.has("--from-bucket")) {
return await listBucketCommand(env);
}
if (command === "list") {
const [, other] = argv.filter((a) => !a.startsWith("--"));
return listCommand(other ?? resolveBackupDir(env));
}
return await restoreCommand(command, flags, env);
} catch (error) {
console.error(
`db-restore: ${error instanceof Refusal ? "" : "failed: "}${error instanceof Error ? error.message : String(error)}`,
);
return 1;
}
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
process.exit(await main());
}
+59
View File
@@ -0,0 +1,59 @@
/**
* The accounts the disaster-recovery drill adds beside the demo seed: a
* Guardian with a password, and a managed profile that Guardian administers.
* Written through the app's own code (`createManagedProfile`,
* `hashPassword`), so the rows are the ones a person creating them in the app
* would leave.
*
* DATABASE_URL=… pnpm dlx tsx scripts/dr-drill-seed.ts
*
* Prints one JSON line: `{ "guardianId": …, "profileId": … }`. The drill
* (`scripts/compose-dr-drill.sh`) checks after the restore that the Guardian
* still sees that profile. Idempotent: a second run answers the same ids.
*/
import { hashPassword } from "@/lib/auth/password";
import { prisma } from "@/lib/db";
import { createManagedProfile } from "@/lib/managed-profiles/create";
const GUARDIAN = "drill-guardian";
const PASSWORD = "drill-guardian-password-1";
async function main(): Promise<void> {
let guardian = await prisma.user.findUnique({
where: { username: GUARDIAN },
});
guardian ??= await prisma.user.create({
data: {
username: GUARDIAN,
email: "drill-guardian@example.test",
passwordHash: await hashPassword(PASSWORD),
displayName: "Drill Guardian",
locale: "en",
timezone: "Europe/Berlin",
onboardingCompletedAt: new Date(),
},
});
const existing = await prisma.accountGrant.findFirst({
where: { granteeId: guardian.id, access: "MANAGE" },
select: { grantorId: true },
});
const profileId =
existing?.grantorId ??
(
await createManagedProfile({
creatorId: guardian.id,
displayName: "Drill Child",
dateOfBirth: new Date("2018-05-04T00:00:00Z"),
locale: "en",
timezone: "Europe/Berlin",
})
).profile.id;
console.log(JSON.stringify({ guardianId: guardian.id, profileId }));
}
main()
.catch((error: unknown) => {
console.error(error);
process.exitCode = 1;
})
.finally(() => prisma.$disconnect());
+32
View File
@@ -360,6 +360,38 @@
}
]
},
{
"name": "Database dumps",
"description": "The nightly whole-database dump and the dump before migrations (docs/ops/disaster-recovery.md). Optional: each has a default, and the bundled compose file mounts the `backups` volume at the default directory.",
"required": false,
"variables": [
{
"name": "DB_BACKUP_ENABLED",
"purpose": "Set false to stop the nightly database dump. On when unset.",
"example": "true"
},
{
"name": "DB_BACKUP_DIR",
"purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. Without a writable directory the nightly dump only goes off-host and no dump is taken before migrations.",
"example": "/var/lib/healthlog/backups"
},
{
"name": "DB_BACKUP_SCHEDULE",
"purpose": "Five-field cron for the nightly dump, in Europe/Berlin time like every scheduled job. Default 30 1 * * * (01:30).",
"example": "30 1 * * *"
},
{
"name": "DB_BACKUP_KEEP",
"purpose": "How many nightly dumps stay in DB_BACKUP_DIR (default 7, at least 1). The last two dumps taken before a migration are kept besides.",
"example": "7"
},
{
"name": "DB_BACKUP_BEFORE_MIGRATE",
"purpose": "Set false to start without the dump the entrypoint takes before applying migrations. On when unset; a failed dump stops the start until it works or this is false.",
"example": "true"
}
]
},
{
"name": "Operational windows and cookie flags",
"description": "Values the product states back to a user or that decide whether a session cookie is accepted. Optional: each has a documented default. They are listed so `pnpm check-env` reports them, and so the compose whitelist and the example files can be held in step with this manifest.",
+6 -7
View File
@@ -1,12 +1,11 @@
/**
* Boot-time notice before `prisma migrate deploy`: how many migrations this
* image is about to apply, and a reminder to have a backup first.
* image is about to apply, and that a dump is taken first
* (`scripts/db-dump.mjs`, which imports `pendingMigrations` from here).
*
* Run by `docker-entrypoint.sh` as `node scripts/pending-migrations.mjs`.
* It only reports. A migration that changes data cannot be undone by pulling
* the previous image, so an operator who reads the count in the log still has
* a moment to stop the container and take a `pg_dump`; the entrypoint does
* not wait for that.
* the previous image; the dump the entrypoint takes next is the way back.
*
* Best effort by design: any failure to count (no `_prisma_migrations` table
* yet is not a failure, it means every migration is pending) prints one line
@@ -67,9 +66,9 @@ export function pendingNotice(count) {
const noun = count === 1 ? "migration" : "migrations";
return (
`HealthLog: ${count} database ${noun} pending. ` +
"Back up the database before updating (pg_dump, see the Updates section " +
"of docs/self-hosting/getting-started.md); a migration is not undone by " +
"going back to the previous image."
"Unless DB_BACKUP_BEFORE_MIGRATE=false, a pg_dump of the database is " +
"written to DB_BACKUP_DIR before they run (docs/ops/disaster-recovery.md); " +
"a migration is not undone by going back to the previous image."
);
}
+145 -60
View File
@@ -9,7 +9,9 @@
* - 1 admin user (demo/demo123demo123)
* - 90 days of measurements (weight, BP, pulse, resting HR, body fat,
* sleep, steps, fortnightly waist) — flat-or-improving trends,
* most-recent point from today
* most-recent point from today; blood pressure has one busy stretch two
* to three weeks back (about 144/91 for some ten days) that comes down
* after a ramipril dose increase
* - A full week of per-stage sleep nights (awake/REM/light(core)/deep),
* ~7.5 h each, including last night
* - 90 days of body composition (fat/lean/muscle mass, total body water,
@@ -42,12 +44,13 @@
* temperature, period flow, fertile-window mucus/OPK, per-day symptoms,
* and a cached forward prediction
* - Two practitioners, a knee arthroscopy with its operative report and
* discharge letter linked (small generated PDFs), its follow-up visit, and
* a check-up booked for tomorrow morning
* discharge letter linked (small generated PDFs), its follow-up visit, the
* blood-pressure visit that raised ramipril, and a check-up booked for
* tomorrow morning
* - An AI-configured Coach with five sample conversations spread from
* today back over a few weeks
* - App settings (registration disabled, English locale, the key-backup
* step confirmed for the key the demo runs with)
* - App settings (registration disabled, English locale) and the recovery
* kit confirmed for the keys the demo runs with
* - A multi-year history before the 90-day window (about four and a half
* years) for the timeline and the day view: blood pressure in bursts with
* real gaps, weekly weight, phone steps, watch resting heart rate / HRV /
@@ -87,8 +90,14 @@ import {
encrypt,
encryptBytes,
getActiveKeyId,
getConfiguredKeyIds,
getKeyFingerprint,
} from "../src/lib/crypto";
import {
applicationKitKeys,
backupKitKeys,
tokenKitKeys,
} from "../src/lib/recovery/kit-rules";
import { ENVIRONMENT_LOCATION_AAD } from "../src/lib/crypto/encrypted-columns";
import { withBriefingGeneratedAt } from "../src/lib/insights/briefing-generated-at";
import { dailyStatsExternalId } from "../src/lib/measurements/apple-health-mapping";
@@ -403,19 +412,45 @@ async function bulkInsert(
/**
* A busy stretch inside the 90-day window, in days before "now": blood
* pressure and pulse rise and settle again, and the mood dips with them.
* Without it every short-window series was flat noise around its target and a
* 7-day trend line over 30 days drew a straight line; with it the line has a
* shape to follow. It ends well before today, so the latest readings, the
* tiles and the stored texts still describe a settled, optimal band.
* pressure climbs over two days to about 144/91 and stays there for some ten
* days, pulse rises with it and the mood dips. At the family practice on
* `doseUp` ramipril goes from 5 to 10 mg, and over the following days the
* readings come back down to the settled band. Without it every short-window
* series was flat noise around its target, and on a blood-pressure axis that
* has to fit both systolic and diastolic, even a 15 mmHg swing read as a flat
* line. It ends a few days before today, so the latest readings, the tiles and
* the stored texts describe a settled, optimal band again.
*
* It is also no longer than this on purpose. The personal usual range is the
* median ± 3 MAD of the last 30 days: once the stretch fills about half of that
* window, the median lands on the stretch and the band reads something like
* 87-181 mmHg. Kept to this length the band stays around 107-132.
*/
const BUSY_STRETCH = { from: 27, to: 12 } as const;
const BUSY_STRETCH = { from: 21, peak: 19, doseUp: 11, settled: 6 } as const;
/** 0 outside the busy stretch, rising smoothly to 1 at its middle. */
/**
* Systolic, diastolic and pulse on top of the settled band at the stretch's
* height. The chart pads its axis and rounds it to even ticks: with systolic
* readings up to about 148 the 30-day axis runs 60 to 160 in steps of 20, one
* mmHg more and it jumps to 50 to 200 in steps of 50, which flattens the very
* rise the stretch is there to show. So the lift stops short of that.
*/
const BUSY_STRETCH_LIFT = { sys: 25, dia: 15, pulse: 6 } as const;
/** Ramipril dose after the stretch, in mg (from 5 mg before it). */
const RAMIPRIL_STRETCH_DOSE = 10;
/**
* 0 outside the busy stretch, 1 while it holds: a smooth climb from `from` to
* `peak`, level until the dose change on `doseUp`, a smooth fall to `settled`.
*/
function busyStretch(daysBefore: number): number {
const { from, to } = BUSY_STRETCH;
if (daysBefore > from || daysBefore < to) return 0;
return Math.sin((Math.PI * (from - daysBefore)) / (from - to)) ** 2;
const { from, peak, doseUp, settled } = BUSY_STRETCH;
const smooth = (t: number) => t * t * (3 - 2 * t);
if (daysBefore >= from || daysBefore <= settled) return 0;
if (daysBefore > peak) return smooth((from - daysBefore) / (from - peak));
if (daysBefore >= doseUp) return 1;
return smooth((daysBefore - settled) / (doseUp - settled));
}
/** Pollen season bump: 0 outside `[from, to]`, peaking at `peak` mid-season. */
@@ -577,12 +612,14 @@ export async function seed() {
// Weight: 86.5 → trending down to a stable ~82 (healthy BMI ~24.7 at 182cm)
const weights = randomWalk(86.5, 82.0, span, 0.6);
// Systolic BP: 128 → settled in the optimal band ~118 (clamped)
// Systolic BP: 128 → settled in the optimal band ~118 (clamped). The day-
// to-day noise is kept small, so the 7-day trend over the busy stretch
// below draws an arc rather than jitter.
const sysBP = randomWalk(
128,
118,
span,
3,
1.2,
VALUE_RANGES.BLOOD_PRESSURE_SYS,
0.1,
);
@@ -591,7 +628,7 @@ export async function seed() {
82,
76,
span,
2.5,
1,
VALUE_RANGES.BLOOD_PRESSURE_DIA,
0.1,
);
@@ -637,9 +674,21 @@ export async function seed() {
Math.max(range.min, Math.round((series[i] + by) * 10) / 10),
);
};
lift(sysBP, busy * 12 + weekend * 2, VALUE_RANGES.BLOOD_PRESSURE_SYS);
lift(diaBP, busy * 7 + weekend * 1, VALUE_RANGES.BLOOD_PRESSURE_DIA);
lift(pulse, busy * 6, VALUE_RANGES.PULSE);
// A little reading-to-reading scatter on top of the slow walk, as a
// cuff has; the 7-day mean averages it out.
lift(
sysBP,
busy * BUSY_STRETCH_LIFT.sys + weekend * 1 + (Math.random() - 0.5) * 3,
VALUE_RANGES.BLOOD_PRESSURE_SYS,
);
lift(
diaBP,
busy * BUSY_STRETCH_LIFT.dia +
weekend * 0.5 +
(Math.random() - 0.5) * 2,
VALUE_RANGES.BLOOD_PRESSURE_DIA,
);
lift(pulse, busy * BUSY_STRETCH_LIFT.pulse, VALUE_RANGES.PULSE);
}
for (const series of [weights, restingHr]) {
settleLatest(series);
@@ -647,7 +696,7 @@ export async function seed() {
// Today's reading settles on the days after the busy stretch only, so it
// describes the band the series came back to.
for (const series of [sysBP, diaBP, pulse]) {
settleLatest(series, BUSY_STRETCH.to - 1);
settleLatest(series, BUSY_STRETCH.settled);
}
// Each series carries the source that writes it on a real account, the
@@ -1406,19 +1455,21 @@ export async function seed() {
console.log("Creating medications...");
// Medication 1: Ramipril (blood pressure). Started about three years
// back at 2.5 mg and raised to 5 mg a year later; the multi-year history
// section below writes the dose changes, a short sick-day pause and the
// intakes before the 90-day window. `starts_on` is the stated start the
// timeline lane draws from.
// back at 2.5 mg, raised to 5 mg a year later, and to 10 mg during the
// busy stretch inside the 90-day window; the multi-year history section
// below writes the dose changes, a short sick-day pause and the intakes
// before the 90-day window. `starts_on` is the stated start the timeline
// lane draws from.
const med1Id = cuid();
await client.query(
`INSERT INTO medications (id, user_id, name, dose, active, notifications_enabled, starts_on, created_at, updated_at)
VALUES ($1, $2, 'Ramipril', '5mg', true, true, $3, $4, $4)`,
VALUES ($1, $2, 'Ramipril', $5, true, true, $3, $4, $4)`,
[
med1Id,
userId,
formatDate(daysAgo(RAMIPRIL_START_DAYS_AGO)),
daysAgo(RAMIPRIL_START_DAYS_AGO),
`${RAMIPRIL_STRETCH_DOSE}mg`,
],
);
const sched1Id = cuid();
@@ -2311,6 +2362,16 @@ export async function seed() {
reason: "Follow-up two weeks after the arthroscopy.",
outcome: "Healing well, full range of motion. Physiotherapy continues.",
});
// The visit that turned the busy stretch: ten days of high home readings,
// ramipril raised that morning.
await insertEncounter({
occurredAt: daysAgoAt(BUSY_STRETCH.doseUp, 8, 15),
status: "DONE",
kind: "ROUTINE",
practitionerId: familyPracticeId,
reason: "Home blood pressure high for about ten days.",
outcome: `Readings around 144/91 during a stressful stretch at work. Ramipril raised to ${RAMIPRIL_STRETCH_DOSE} mg, home readings again in two weeks.`,
});
await insertEncounter({
occurredAt: daysAgoAt(-1, 9, 30),
status: "PLANNED",
@@ -2633,10 +2694,12 @@ export async function seed() {
intakeRows,
);
// The ramipril dose history and its sick-day pause.
// The ramipril dose history and its sick-day pause. The last step falls
// inside the 90-day window, on the day the busy stretch turns.
for (const [n, value] of [
[RAMIPRIL_START_DAYS_AGO, 2.5],
[RAMIPRIL_DOSE_UP_DAYS_AGO, 5],
[BUSY_STRETCH.doseUp, RAMIPRIL_STRETCH_DOSE],
] as const) {
await client.query(
`INSERT INTO medication_dose_changes (id, medication_id, effective_from, dose_value, dose_unit, created_at)
@@ -3353,9 +3416,10 @@ export async function seed() {
content: "How has my blood pressure been trending lately?",
},
answer(
"Your blood pressure has been settling nicely. After a higher " +
"stretch about three weeks ago, the morning readings have come back " +
"down into the optimal band, and " +
"Your blood pressure has come back down nicely. During the stressful " +
"stretch two to three weeks ago your morning readings sat around " +
"144/91 for about ten days; since ramipril went up to 10 mg they " +
"have eased back into the optimal band within about a week, and " +
"your resting heart rate is sitting in a comfortable range too. " +
"Keeping up the steady sleep and daily movement looks like it is " +
"paying off — worth staying consistent with the morning measurements " +
@@ -3633,10 +3697,11 @@ export async function seed() {
keyFindings: [
{
tone: "good",
headline: "Blood pressure has settled into the optimal range.",
headline: "Blood pressure is back in the optimal range.",
detail:
"Your 30-day average is well inside the optimal band and lower " +
"than where the quarter began.",
"After a higher stretch two to three weeks ago and a ramipril " +
"dose increase, your readings have come back down below where " +
"the quarter began.",
delta: "↓ ~10 mmHg",
sourceWindow: "30d",
sourceMetric: "bp",
@@ -3674,7 +3739,7 @@ export async function seed() {
],
},
trendAnnotations: {
bp: "Your systolic is trending down into the optimal band — a pattern worth keeping.",
bp: "After a higher stretch and a ramipril dose increase, your systolic has come back down into the optimal band.",
weight:
"Weight has levelled off after an earlier decline — a stable, healthy place to hold.",
mood: "Mood has been on a quiet upward run across the quarter.",
@@ -3727,11 +3792,10 @@ export async function seed() {
{
scope: "blood-pressure",
text:
"Your blood pressure is averaging about 118/76 over the last two " +
"weeks, back down after a busier stretch before that. It sits " +
"roughly 10 mmHg systolic below where this quarter started, " +
"comfortably inside the optimal band. That is a real, earned " +
"shift, not day-to-day noise. Keeping the morning " +
"Your blood pressure is back around 118/76 this week. Two to three " +
"weeks ago it climbed to about 144/91 and held there for about ten " +
"days; after ramipril went up to 10 mg it came down over the " +
"following days and now sits inside the optimal band again. Keeping the morning " +
"readings on a steady schedule will make the trend even easier to " +
"follow.",
},
@@ -3846,8 +3910,9 @@ export async function seed() {
period: "month",
text:
"Over the past month the gains from earlier in the quarter have " +
"become your new normal. Blood pressure has held in the optimal " +
"band rather than just dipping into it, weight has stabilised at a " +
"become your new normal. Blood pressure had one higher stretch, " +
"came back down after the ramipril dose went up and now sits in the " +
"optimal band again, weight has stabilised at a " +
"healthy point, and resting heart rate has stayed low alongside " +
"consistent daily activity. Sleep lengthened and steadied, and " +
"your mood trend continued its quiet climb. The month reads less " +
@@ -3890,24 +3955,44 @@ export async function seed() {
// ── App Settings ─────────────────────────
console.log("Creating app settings...");
// The key-backup step is confirmed for the key the demo runs with, the
// same fields `POST /api/admin/encryption/key-backup/confirm` writes (key
// id + its public fingerprint, never the key). The demo account is an
// admin, so an unconfirmed step put a "Back up your encryption key"
// banner above the dashboard for every visitor, about a key no visitor
// can reach.
const activeKeyId = getActiveKeyId();
await client.query(
`INSERT INTO app_settings (id, registration_enabled, default_locale,
encryption_key_backup_confirmed_at, encryption_key_backup_confirmed_key_id,
encryption_key_backup_confirmed_fingerprint, encryption_key_backup_confirmed_by_user_id)
VALUES ('singleton', false, 'en', NOW(), $1, $2, $3)
ON CONFLICT (id) DO UPDATE SET registration_enabled = false, default_locale = 'en',
encryption_key_backup_confirmed_at = EXCLUDED.encryption_key_backup_confirmed_at,
encryption_key_backup_confirmed_key_id = EXCLUDED.encryption_key_backup_confirmed_key_id,
encryption_key_backup_confirmed_fingerprint = EXCLUDED.encryption_key_backup_confirmed_fingerprint,
encryption_key_backup_confirmed_by_user_id = EXCLUDED.encryption_key_backup_confirmed_by_user_id`,
[activeKeyId, getKeyFingerprint(activeKeyId), userId],
`INSERT INTO app_settings (id, registration_enabled, default_locale)
VALUES ('singleton', false, 'en')
ON CONFLICT (id) DO UPDATE SET registration_enabled = false, default_locale = 'en'`,
);
// The recovery kit is confirmed for the keys the demo runs with, the same
// entries `POST /api/admin/recovery-kit/confirm` writes (key ids and
// public fingerprints, never a key). The demo account is an admin, so an
// unconfirmed kit put a "Save your recovery kit" banner above the
// dashboard for every visitor, about keys no visitor can reach. The seed
// writes no backup copies, so no retired key is needed.
const kitEntries = [
...applicationKitKeys({
activeKeyId: getActiveKeyId(),
configuredKeyIds: getConfiguredKeyIds(),
fingerprintOf: getKeyFingerprint,
needs: {
stored: [],
unrecorded: { copies: 0, oldestAt: null },
offhost: [],
offhostExpirationDays: null,
retiredKeysStillNeeded: [],
},
now: new Date(),
}),
...backupKitKeys(
process.env.BACKUP_ENCRYPTION_KEY,
process.env.BACKUP_ENCRYPTION_PREVIOUS_KEYS,
),
...tokenKitKeys(process.env.API_TOKEN_HMAC_KEY),
].map(({ kind, id, fingerprint }) => ({ kind, id, fingerprint }));
await client.query(
`INSERT INTO recovery_kit_confirmations (id, confirmed_at, confirmed_by_user_id, entries)
VALUES ('singleton', NOW(), $1, $2::jsonb)
ON CONFLICT (id) DO UPDATE SET confirmed_at = EXCLUDED.confirmed_at,
confirmed_by_user_id = EXCLUDED.confirmed_by_user_id,
entries = EXCLUDED.entries`,
[userId, JSON.stringify(kitEntries)],
);
// ── Audit Log (some login entries) ──────
@@ -3962,7 +4047,7 @@ export async function seed() {
` Cycle: ${cycleLengths.length} cycles with BBT, flow, mucus/OPK, symptoms + a forecast`,
);
console.log(
` Visits: knee arthroscopy (2 linked PDFs) + follow-up, check-up booked for tomorrow`,
` Visits: knee arthroscopy (2 linked PDFs) + follow-up, a blood-pressure visit, check-up booked for tomorrow`,
);
console.log(` Coach: ${coachConversations.length} conversations`);
console.log(