diff --git a/.env.example b/.env.example index 092422769..c0efd6658 100644 --- a/.env.example +++ b/.env.example @@ -202,7 +202,10 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # # Set this to "1" ONLY when every public request to HealthLog terminates # at Cloudflare (Tunnel, Workers, classic proxy) before it reaches your -# origin. Deployments behind nginx, Caddy, Traefik, an AWS ALB, or any +# origin, and the origin accepts connections from Cloudflare alone (a +# Tunnel, or a firewall that admits only Cloudflare's address ranges): the +# header is believed from every connection, before any other check. +# Deployments behind nginx, Caddy, Traefik, an AWS ALB, or any # direct-to-Next.js exposure MUST leave this off. # # Security implications when the flag is mis-set: @@ -224,10 +227,19 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # Number of trusted reverse-proxy hops in front of the app. The client IP is # read that many entries from the right of the X-Forwarded-For chain. Set 0 # for a directly-exposed host; 1 (the default when unset) behind a single -# reverse proxy. X-Real-IP is read only at 1. A wrong value makes the app +# reverse proxy. X-Real-IP is read only at 1. Forwarding headers are read only +# from a connection whose own address is not public (loopback, private, CGNAT, +# unique-local, link-local, reserved or multicast; the full list is in +# docs/self-hosting/reverse-proxy.md). A wrong value makes the app # refuse the headers, log one warning, and share one rate-limit bucket across # all anonymous callers. # TRUST_PROXY_HOPS="1" +# The addresses the reverse proxy connects from (comma-separated CIDRs). Set, +# only those connections may report the client's address; unset, any private +# address may, which behind Docker Desktop, OrbStack or docker-proxy includes +# every caller of the published port. Same host: 127.0.0.1/32,::1/128; Docker +# network: the proxy's subnet. See docs/self-hosting/reverse-proxy.md. +# TRUSTED_PROXY_CIDRS="" # TRUST_CF_CONNECTING_IP="" @@ -277,6 +289,33 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation +# ----------------------------------------------------------------------------- +# Database dumps (optional, on by default) +# ----------------------------------------------------------------------------- +# The whole-database dump that rebuilds a lost server: nightly, and before +# every update that brings migrations. Written to DB_BACKUP_DIR with a +# manifest beside it (row counts, key fingerprints, versions); with the +# off-host bucket above set, each nightly dump is also uploaded under +# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not +# encrypted: they sit beside the database volume, in the same trust zone. +# Restore: `docker compose run --rm db-restore latest`. See +# docs/ops/disaster-recovery.md. +# +# Set "false" to stop the nightly dump. +# DB_BACKUP_ENABLED="true" +# Where dumps are written. The bundled compose file mounts the `backups` +# volume here; change it only together with that mount. In a container a +# directory that is not on a mount of its own sits in the container's layer +# and is lost at the next update, so no dump is kept there. +# DB_BACKUP_DIR="/var/lib/healthlog/backups" +# Five-field cron, in Europe/Berlin time like every scheduled job. +# DB_BACKUP_SCHEDULE="30 1 * * *" +# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept +# besides. +# DB_BACKUP_KEEP="7" +# Set "false" to start an update without the dump before its migrations. +# DB_BACKUP_BEFORE_MIGRATE="true" + # ----------------------------------------------------------------------------- # Stored backups in the database (optional) # ----------------------------------------------------------------------------- @@ -328,11 +367,21 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # Auto-deploy webhook (v1.4.15 — phase C2) # ----------------------------------------------------------------------------- # Coolify (or any CI) pings POST /api/internal/deploy-webhook after a deploy -# attempt with success/failure. We verify the request via this shared secret -# carried in the `X-Deploy-Webhook-Secret` header (timing-safe compare). -# Generate with `openssl rand -hex 32`. Configure the SAME value in Coolify's -# Notifications → Webhook custom-headers panel. See docs/audit/v1415-auto-deploy.md. +# attempt with success/failure. A request authenticates with this shared +# secret in the `X-Deploy-Webhook-Secret` header (timing-safe compare; what +# Coolify's stock sender does, with no replay protection), or by signing: +# `X-Deploy-Webhook-Signature: sha256=`, the HMAC-SHA256 under this +# secret of the `X-Deploy-Webhook-Timestamp` value, a dot and the raw body, +# the timestamp within 5 minutes. Generate with `openssl rand -hex 32`. +# Configure the SAME value in Coolify's Notifications → Webhook +# custom-headers panel. See docs/ops/deploy.md. # DEPLOY_WEBHOOK_SECRET="" +# "true" also requires a fresh X-Deploy-Webhook-Timestamp on a secret-header +# request; leave unset for Coolify's stock sender, which sends none. +# DEPLOY_WEBHOOK_REQUIRE_TIMESTAMP="" +# "true" accepts signed requests only (the setting that stops a replay); +# leave unset for Coolify's stock sender. +# DEPLOY_WEBHOOK_REQUIRE_SIGNATURE="" # ----------------------------------------------------------------------------- diff --git a/.env.production.example b/.env.production.example index ecf31e293..dc6d226c1 100644 --- a/.env.production.example +++ b/.env.production.example @@ -295,11 +295,17 @@ API_TOKEN_HMAC_KEY="" # Optional: deployment-dashboard URL (e.g. the Coolify UI) linked in # deploy-failure notifications. When unset the notification omits the link. # DEPLOY_LOGS_URL="" -# Optional replay hardening: require a fresh X-Deploy-Webhook-Timestamp -# header (unix seconds / ms or ISO-8601, +/- 5 minutes) on every webhook -# call. Leave unset for Coolify's stock sender (it attaches no -# timestamp); see docs/ops/deploy.md. +# Optional: require an X-Deploy-Webhook-Timestamp within +/- 5 minutes on a +# request that authenticates with the secret header. Keeps a stale capture +# out, not a fresh forgery by someone who saw the secret. Leave unset for +# Coolify's stock sender, which sends no timestamp. # DEPLOY_WEBHOOK_REQUIRE_TIMESTAMP="false" +# Optional replay protection: accept only signed webhook calls +# (X-Deploy-Webhook-Signature, an HMAC of the timestamp and body under the +# secret, with a timestamp within +/- 5 minutes); the secret header alone is +# refused. Leave unset for Coolify's stock sender, which can only send the +# secret header; see docs/ops/deploy.md. +# DEPLOY_WEBHOOK_REQUIRE_SIGNATURE="false" # ----------------------------------------------------------------------------- @@ -414,6 +420,33 @@ API_TOKEN_HMAC_KEY="" # key once the lifecycle rule has retired every object written under it. # BACKUP_ENCRYPTION_PREVIOUS_KEYS="" +# ----------------------------------------------------------------------------- +# Database dumps (optional, on by default) +# ----------------------------------------------------------------------------- +# The whole-database dump that rebuilds a lost server: nightly, and before +# every update that brings migrations. Written to DB_BACKUP_DIR with a +# manifest beside it (row counts, key fingerprints, versions); with the +# off-host bucket above set, each nightly dump is also uploaded under +# `database/`, sealed with BACKUP_ENCRYPTION_KEY. Local dumps are not +# encrypted: they sit beside the database volume, in the same trust zone. +# Restore: `docker compose run --rm db-restore latest`. See +# docs/ops/disaster-recovery.md. +# +# Set "false" to stop the nightly dump. +# DB_BACKUP_ENABLED="true" +# Where dumps are written. The bundled compose file mounts the `backups` +# volume here; change it only together with that mount. In a container a +# directory that is not on a mount of its own sits in the container's layer +# and is lost at the next update, so no dump is kept there. +# DB_BACKUP_DIR="/var/lib/healthlog/backups" +# Five-field cron, in Europe/Berlin time like every scheduled job. +# DB_BACKUP_SCHEDULE="30 1 * * *" +# Nightly dumps kept (at least 1). The last two pre-migration dumps are kept +# besides. +# DB_BACKUP_KEEP="7" +# Set "false" to start an update without the dump before its migrations. +# DB_BACKUP_BEFORE_MIGRATE="true" + # ----------------------------------------------------------------------------- # Stored backups in the database (optional) # ----------------------------------------------------------------------------- @@ -446,6 +479,16 @@ API_TOKEN_HMAC_KEY="" # 90 rather than repeat the default. # COACH_MESSAGE_RETENTION_DAYS="365" +# How long deleting an account, a managed profile or "all data" stays +# undoable, in days. Default 14, ceiling 30. The account is locked at once +# (signed out everywhere, sign-in refused, nothing runs for it) and an admin +# can undo the deletion from the users list until the window ends; the purge +# job erases it then, together with its stored and off-host copies. A wipe +# keeps a copy of the record an admin can restore for the same time. 0 erases +# immediately, as releases before 1.46 did. The Data & Privacy page states +# this number, so it has to reach the container. +# ACCOUNT_DELETION_GRACE_DAYS="14" + # How long the host CPU/memory samples behind the admin monitoring charts are # kept, in days. Default 7. Raise it for a longer trend; lower it on a host # where disk is the scarce resource. @@ -517,19 +560,43 @@ API_TOKEN_HMAC_KEY="" # ----------------------------------------------------------------------------- # Client-IP resolution -- optional (required behind Cloudflare) # ----------------------------------------------------------------------------- -# When the stack sits behind Cloudflare, the real visitor IP arrives in the -# `cf-connecting-ip` header; the X-Forwarded-For chain otherwise yields the -# Cloudflare edge IP, so the audit-log overview stores the CF datacentre -# address and the geo lookup resolves the wrong location. Set this to "1" -# ONLY when Cloudflare is in front -- a direct-exposed self-host must leave -# it unset, or an attacker could forge the header. Off by default. # Number of trusted reverse-proxy hops in front of the app; the client IP is # read that many entries from the right of the X-Forwarded-For chain. 0 for a -# directly-exposed host, 1 (the default when unset) behind a single reverse -# proxy; X-Real-IP is read only at 1. A wrong value makes the app refuse the -# headers, log one warning, and collapse anonymous callers into one -# rate-limit bucket. +# directly-exposed host (the connection's own address is used), 1 (the +# default when unset) behind a single reverse proxy; X-Real-IP is read only +# at 1. Forwarding headers are read only from a connection whose own address +# is not public: loopback, private, CGNAT, unique-local, link-local, reserved +# or multicast (the full list is in docs/self-hosting/reverse-proxy.md). A +# wrong value makes the app refuse the headers, log one warning, and collapse +# anonymous callers into one rate-limit bucket. # TRUST_PROXY_HOPS="1" +# +# The addresses your reverse proxy connects from, as comma-separated CIDRs +# (IPv4 or IPv6; a bare address is one host). When set, only those +# connections may report the client's address; every other connection is +# its own client. When unset, any private address may, which is the old +# behaviour and is NOT safe behind Docker Desktop, OrbStack or docker-proxy: +# there every caller of the published port arrives from the gateway, a +# private address, and could name its own client address. Set it whenever a +# reverse proxy is in front: +# proxy on the same host (proxying to 127.0.0.1:3000): 127.0.0.1/32,::1/128 +# proxy container on a Docker network: that network's subnet (docker +# network inspect ), e.g. 172.18.0.0/16 +# proxy on another machine with a public address: that address, /32 +# A proxy on a public address that is not listed is not believed: every +# request through it shares the proxy's address and one rate-limit bucket. +# The boot log names the rule in force in one [proxy-trust] line. +# TRUSTED_PROXY_CIDRS="127.0.0.1/32,::1/128" +# +# Behind Cloudflare the real visitor IP arrives in the `cf-connecting-ip` +# header; the X-Forwarded-For chain otherwise yields the Cloudflare edge IP, +# so the audit-log overview stores the CF datacentre address and the geo +# lookup resolves the wrong location. "1" believes that header from EVERY +# connection, before the checks above, so set it ONLY when the origin accepts +# connections from Cloudflare alone (a Cloudflare Tunnel, or a firewall that +# admits only Cloudflare's address ranges). An origin that also answers on +# its own public address lets any caller forge the address that rate limits +# and the audit log record. Off by default. # TRUST_CF_CONNECTING_IP="1" # Memory cap of the app container (read by docker-compose.yml, not by the app). diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2cb857975..985a0c309 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -79,6 +79,8 @@ updates: # The first run of this ecosystem proposed 16.15 -> 18.6, which would # have handed self-hosters a database their data directory cannot be # read by. Minor releases still arrive as PRs; a major is a deliberate - # project decision with a migration guide behind it. + # project decision with a migration guide behind it. The ignore also + # keeps `db-upgrade` on 16: it exists to read Postgres 16 data, and a + # bump to 18 would leave it unable to start on the volume it moves. - dependency-name: "postgres" update-types: ["version-update:semver-major"] diff --git a/.github/workflows/compose-db-upgrade.yml b/.github/workflows/compose-db-upgrade.yml new file mode 100644 index 000000000..edc86e0cb --- /dev/null +++ b/.github/workflows/compose-db-upgrade.yml @@ -0,0 +1,51 @@ +name: Compose database move + +# Plays the Postgres 16 -> 18 move in docker-compose.yml against real +# containers: a v1.45 database with every migration applied, then the current +# compose file's `db-upgrade` on its first start, a second start, a move cut +# short, an emptied new volume and a new install. The service runs once on an +# operator's host and has no second chance there, so every change to the +# compose file or to the check runs it here first. +# `scripts/compose-db-upgrade-check.sh` is the same check, runnable locally. + +on: + pull_request: + branches: [main] + paths: + - docker-compose.yml + - scripts/compose-db-upgrade-check.sh + - .github/workflows/compose-db-upgrade.yml + push: + branches: [main] + paths: + - docker-compose.yml + - scripts/compose-db-upgrade-check.sh + - .github/workflows/compose-db-upgrade.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + move: + name: Postgres 16 -> 18 move + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The check reads the v1.45 compose file from its tag. + - name: Fetch the v1.45.0 tag + run: git fetch --no-tags --depth=1 origin tag v1.45.0 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Move check + run: bash scripts/compose-db-upgrade-check.sh diff --git a/.github/workflows/compose-dr-drill.yml b/.github/workflows/compose-dr-drill.yml new file mode 100644 index 000000000..f08e84449 --- /dev/null +++ b/.github/workflows/compose-dr-drill.yml @@ -0,0 +1,70 @@ +name: Disaster recovery drill + +# Rebuilds a lost server from the nightly database dump and the keys alone, +# against real containers: the app image built from this checkout, the +# bundled compose file with an S3-compatible bucket outside it, the demo seed +# plus a Guardian with a managed profile, the dump taken and uploaded by the +# scheduled job, every volume of the stack destroyed, a new stack, the +# restore, and the app booted on the result (health, login, an encrypted +# note, a document, the Guardian's profile, every table's row count against +# the manifest). Two sources: from the bucket (`db-restore latest +# --from-bucket`, the disaster path) and from dump files copied off the host +# (`db-restore latest`). +# `scripts/compose-dr-drill.sh` is the same drill, runnable locally; its +# header lists what it checks and the mutations that must turn it red. + +on: + pull_request: + branches: [main] + paths: + - docker-compose.yml + - Dockerfile + - docker-entrypoint.sh + - scripts/compose-dr-drill.sh + - scripts/dr-drill-seed.ts + - scripts/db-dump.mjs + - scripts/db-restore.mjs + - scripts/pending-migrations.mjs + - src/lib/database-backup/** + - src/lib/jobs/database-backup-queue.ts + - src/lib/jobs/reminder/backup-handlers.ts + - .github/workflows/compose-dr-drill.yml + schedule: + # Nightly, 02:41 UTC: off the hour, after the evening merges. + - cron: "41 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: compose-dr-drill-${{ github.ref }} + cancel-in-progress: true + +jobs: + drill: + name: Restore a lost server from its dump (${{ matrix.source }}) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + source: [bucket, local] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + # The seed scripts run from the checkout against the drill's database. + - run: pnpm install --frozen-lockfile + - run: pnpm db:generate + + - name: Drill + env: + DRILL_SOURCE: ${{ matrix.source }} + run: bash scripts/compose-dr-drill.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3710d15fc..0640e5420 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -40,7 +40,7 @@ jobs: name: e2e shard ${{ matrix.shard }}/3 services: postgres: - image: postgres:16-alpine + image: postgres:18-alpine env: POSTGRES_USER: healthlog POSTGRES_PASSWORD: healthlog @@ -62,7 +62,7 @@ jobs: AUTH_RP_ORIGIN: http://localhost:3000 SESSION_COOKIE_SECURE: "false" NODE_ENV: production - # The WebKit spec runs in the `webkit` job below. + # The WebKit specs run in the `webkit` job below. E2E_SKIP_WEBKIT: "1" steps: @@ -146,8 +146,9 @@ jobs: if [ -z "$version" ]; then echo "playwright-core not found in pnpm-lock.yaml"; exit 1; fi echo "version=$version" >> "$GITHUB_OUTPUT" - # The WebKit spec (`bottom-sheet.spec.ts`, project `webkit-iphone`), in the - # same image as the shards. + # The WebKit specs, in the same image as the shards: the bottom sheet on an + # iPhone (`bottom-sheet.spec.ts`, project `webkit-iphone`) and the app shell + # on an iPad (`tablet-shell.spec.ts`, project `webkit-tablet`). webkit: runs-on: ubuntu-24.04 needs: playwright-version @@ -157,7 +158,7 @@ jobs: image: mcr.microsoft.com/playwright:v${{ needs.playwright-version.outputs.version }}-noble services: postgres: - image: postgres:16-alpine + image: postgres:18-alpine env: POSTGRES_USER: healthlog POSTGRES_PASSWORD: healthlog @@ -202,7 +203,7 @@ jobs: - run: NODE_OPTIONS=--max-old-space-size=8192 pnpm build - - run: pnpm exec playwright test --project=webkit-iphone + - run: pnpm exec playwright test --project=webkit-iphone --project=webkit-tablet env: E2E_BASE_URL: http://localhost:3000 diff --git a/.github/workflows/post-publish-verify.yml b/.github/workflows/post-publish-verify.yml index 4f4bdf70f..d22ef58f0 100644 --- a/.github/workflows/post-publish-verify.yml +++ b/.github/workflows/post-publish-verify.yml @@ -56,7 +56,7 @@ jobs: services: postgres: - image: postgres:16-alpine + image: postgres:18-alpine env: POSTGRES_USER: healthlog POSTGRES_PASSWORD: healthlog diff --git a/.github/workflows/test-nightly.yml b/.github/workflows/test-nightly.yml index 88c1e0a27..eddede148 100644 --- a/.github/workflows/test-nightly.yml +++ b/.github/workflows/test-nightly.yml @@ -84,7 +84,7 @@ jobs: fail-fast: false matrix: shard: [1, 2, 3, 4] - name: integration shard ${{ matrix.shard }}/4 (shuffled) + name: integration shard ${{ matrix.shard }}/4 (shuffled, Postgres 16) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -110,6 +110,10 @@ jobs: shell: bash env: HEALTHLOG_INTEGRATION_WORKERS: "3" + # Pull requests run the suite on Postgres 18, the compose default. + # Installs that bring their own database may still be on 16, so the + # nightly run keeps that major covered without doubling every PR. + INTEGRATION_PG_IMAGE: "postgres:16-alpine" # Four times each property's own case count; a property's timeout # grows by the same factor (`propertyTimeout`). FC_RUNS_FACTOR: "4" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7c022fb..e28d48b1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,435 @@ # Changelog +## [1.46.0] — 2026-10-25 + +### Added + +- **A database dump every night, and a restore that rebuilds a server.** + Until now every copy the app made was per account and could only roll an + account back on the same server; a lost server needed a dump you had to + make yourself. The app now takes a whole-database dump every night (01:30 + Europe/Berlin) and before every update that brings migrations, keeps the + last seven on the new `backups` volume, and also puts it in your off-host + bucket when one is set up. Each dump has a manifest beside it: app + version, migration, Postgres version, key fingerprints and the row count + of every table. `docker compose run --rm db-restore latest` (or + `latest --from-bucket` after losing the host) restores it into an empty + database only, checks the keys and the checksum first and every table's + row count after, and refuses anything else. A drill in CI does exactly + that every night and on every change to the backup path: it builds a + server, fills it, destroys every + volume, rebuilds it from the bucket and the keys alone, and checks that + people can sign in and read their notes and documents. +- **Recovery readiness, pushed to the admins.** One card on the admin + dashboard and the backups page says whether a server could be recovered + today: the nightly dump, the weekly account copies, the off-host copies, + the restore drill, whether off-host copies are set up, and the recovery + kit. When one of them goes wrong, every admin gets a notification, once, + and again daily while it stays broken, and once when it recovers. A + stopped backup used to show only if someone opened the page. +- **A recovery kit instead of "save the active key".** The kit lists every + key a copy still needs, retired ones included, with their fingerprints: + the encryption keys, the backup key and its predecessors, and the token + key. It becomes due again whenever that set changes, for example after a + key rotation. An upgraded server asks once, because the kit now also + lists the token key. +- **Fourteen days to change your mind.** Deleting an account or a managed + profile now locks it at once (every session, token, share link and + connection is revoked) and erases it after 14 days + (`ACCOUNT_DELETION_GRACE_DAYS`, 0 keeps the old immediate deletion). An + admin can undo it in the users list until then. "Delete all data" first + keeps a copy of the account for the same 14 days, and the copy restores + like any other. Nothing in the background touches an account while it + waits. +- **Apple Health write-back for measurement tokens.** A measurement token, + the narrow kind a scale or Home Assistant uses to add readings, now has a + "Write readings to Apple Health" switch, when you create it and in the + token list. With it on, the HealthLog app on your iPhone adds that + token's readings to Apple Health. The readings still show that they came + from an external device, and the token still cannot read or change + anything. The server decides which readings go to Apple Health and tells + the app, so switching it on later also covers what the token already + sent. Readings from before this release are matched to a token only + where the account has exactly one measurement token; the card says so + when older readings could not be matched. (#1272) +- **Recently deleted for documents.** While it holds anything, the + Documents page links to Recently deleted. It lists each deleted document + with its type, its date, where it came from (Paperless-ngx, for + example) and how many of its 30 days are left, and each one can be + restored or deleted for good right away. The import picker now lets you + tick a document you deleted: one still in Recently deleted comes back + with everything that belonged to it, one already removed is imported + again. Automatic imports, such as a Paperless workflow or the import + script, still never bring back a document you deleted. (discussion + #1275) +- **Questionnaires in every app language.** PHQ-9, GAD-7, WHO-5 and the + sleep questionnaire no longer fall back to English where no validated + translation exists. New validated versions: PHQ-9 in French, Dutch, + Korean and Brazilian Portuguese, GAD-7 in Dutch, Korean and Brazilian + Portuguese, WHO-5 in Dutch and Korean, and the sleep questionnaire in + Polish. Elsewhere HealthLog shows its own faithful translation with a + note on every step that the result is for orientation, and one tap + switches to the English original. Each check-in remembers which wording + it was answered in, and the doctor report marks a total from an + unvalidated translation with an asterisk. +- **Your own files import in full.** The JSON import reads every field + the full backup and the JSON export write: a night's sleep stages, range + readings, an ECG's verdict, the recording device, and a mood entry's + note, detailed ratings, time zone, tags, rated factors and day context. + A full backup also brings your own mood tags, factors and groups along. + Missing ones are created within the usual limits, a tag you already + have is never overwritten, and anything that cannot be matched is named + on the import card. +- **GPT-6 models.** gpt-6-sol, gpt-6-luna and gpt-6.1-sol get reasoning + control with a real "off", the request shape reasoning models expect, + and image input, through an OpenAI key and the ChatGPT sign-in alike. + The OpenAI presets offer gpt-6.1-sol and gpt-6-luna. +- **A queued restore says what it is waiting for.** A restore never runs + beside a backup or another large job. The admin console now says which + of the two it waits for and since when, instead of reading like a stuck + worker. + +### Changed + +- **Postgres 18.** The compose file now runs Postgres 18 instead of 16. An + install from 1.45 or earlier moves its database across by itself on the + first start: a short-lived `db-upgrade` step copies it into a new + `postgres-data` volume, compares the row count of every table, gathers + the planner statistics, and only then lets the app start. If anything + fails, the app does not start and the log says why; a move cut short is + run again from scratch on the next start. The old `pgdata` volume is + left as it was, so going back is the previous image and two lines of the + compose file. On a real 600 MB instance the move took eleven seconds. + HealthLog keeps running on Postgres 16 for installs that bring their own + database, and the tests run on both. (discussion #1282) +- **A quicker, lighter start.** On an account with ten years of data the + app answers its first request about a tenth sooner after a start, holds + about 55 MB less memory right after it, and the background worker is ready + after loading about a quarter of the code it used to need, with the rest + loaded in the background. Within a minute it has tried every job once and + reports any that cannot load, so a broken image shows up at deploy time + instead of when a job fails hours later. Pages and charts answer as fast + as in 1.45, with the same number of database reads; a ten-year chart takes + a few milliseconds longer because it now checks that a trend has enough + days behind it. +- **Chart axes that read cleanly.** Value ticks fall on round steps (60, + 80, 100 rather than 57, 83, 109), date ticks fall on a calendar grid + (days, Mondays, months, quarters, years) and keep their place as data + arrives, and the "your average" and target zone labels sit beside the + plot instead of over the readings or clipped to one letter. The dots + under a chart mark only the days the chart itself shows. +- **Tidier code underneath.** Large parts of the code were split up so + the next changes are smaller and safer to make: files over 800 lines + went from 123 to 84, functions over 200 lines from 411 to 346. Nothing + you use should look different. Signing in with a password or a passkey, + adding a weight, logging a mood, importing lab results, changing a + setting and restoring a full backup are now tested end to end against a + real server. +- **TrueNAS guide.** It covers the app in the TrueNAS Community train and + its backups, and the encryption key backup step points at the app's + Encryption Key field. + +### Fixed + +- **Slow local models get the time you allow them.** The response timeout + reached each model call, but not the work around it: a daily briefing is + up to five calls in a row and had one fixed bound of the timeout plus 30 + seconds, so a model needing several minutes per answer was cut off in + its second call. The briefing now gets your timeout for every call, the + Insights page keeps waiting as long as that allows, and the status cards + and the other AI background jobs follow the same setting instead of + fixed limits. (#1274) +- **The Coach no longer shows a model's reasoning as its answer.** With a + model that always reasons, served through an OpenAI-compatible gateway + or a local server, the Coach could show the thought instead of the + reply, cut off, and sometimes skip the tools it meant to call. The token + limit now keeps room for reasoning whatever the model is asked, + reasoning is taken out of the answer in every shape servers send it, + tool calls run whenever the reply carries them, and a model that still + produces nothing but reasoning gets its own message suggesting another + model. A model that does not reason keeps its own answer even when it + is cut off. (#1276) +- **A model that never answers can no longer run up requests.** One idle + hour against a model that reasoned into its token limit could cost well + over a thousand requests. One call now makes two requests at most. A + reply with reasoning and no answer is asked once more without + reasoning, and if that does not help either, that model is not asked + that way again for an hour. Background work leaves a provider that + keeps failing alone: after three failures in a row it waits five + minutes, from five on it pauses for half an hour, doubling while the + failures continue, up to a day. The Coach and "Re-run analysis" still + reach it and show its error, and saving your AI settings, the admin AI + settings or a fresh ChatGPT sign-in ends the pause. A status card whose + generation failed waits from five minutes up to two hours before the + next try, even when new data arrives. Tokens spent on replies without + an answer count toward the daily AI budget, "Index all documents" stops + after three failed calls in a row, and a gateway that refuses the + reasoning setting is noticed once instead of failing every call. + (#1279) +- **"Extract lab values" reads long documents.** On a stored document it + still sent the whole report in one call, which a long report outgrew, + and the answer came back as "Couldn't read the document. Try a clearer + copy." It now reads the way the lab scan does: a PDF page by page and + text in parts, up to 400 readings, with the same timeout and budget. If + the provider stops early, the readings already found are kept and the + review warns that some values may be missing, every time it is opened; + a read that stopped before the first value says so instead of blaming + the copy. An extraction the iPhone app waits for reads in at most three + parts and marks the document incomplete when that was not enough. + (#1257) +- **PDF previews, vision reads and search work together.** The server + loaded two versions of pdf.js, and whichever ran first broke the other: + after a local text read every preview failed and a vision read of a PDF + was refused, after a preview the local text read failed and the document + stayed out of search. The server now ships one pdf.js. Previews and + search entries the clash broke are repaired once on the first start + after the update. (#1277, PR #1278) +- **"End" on a medication works, and ends it now.** The button had failed + with an error on every press since it was added. It now ends the + running course at that moment: a dose taken earlier today still counts, + and a later one is no longer due, reminded or counted as missed. End is + offered only while a course is running, someone who manages your record + can use it too, undoing the end brings the later doses back, and a new + course can start from the next day. (#1280, PR #1281) +- **Memory stays flat under bursts of requests.** Every page the server + rendered stayed in memory for ten minutes after it was sent, about + 250 KB each, so a run of automated requests could push the container + towards its memory limit. A render is now released with its request. +- **Fast right after an import, not only once the database caught up.** + Postgres learns what a table holds in the background, some time after + rows arrive. Until it had, on a new install or right after a large + import, several reads chose a plan that grew with the square of the + data: the analytics page over a few thousand blood pressure readings + took seconds, and the long-range charts and all-time highs and lows of + a ten-year history could run past the database's time limit. Those + reads now take one pass over the readings whatever Postgres knows, with + the same results, and take a fraction of a second. +- **Trends need readings on enough days.** Three blood pressure readings + taken in one sitting, the only ones that week, produced a 7-day trend of + thousands of mmHg. A trend is now shown only when the readings fall on + at least three days spread over at least half the window (4 days of 7, + 15 of 30), wherever a trend appears. +- **iPad and tablets.** The app fills the window after a rotation and + under Safari's floating toolbars, without a strip of page colour at the + bottom, a cut-off last card or a page that scrolls under the app. In + landscape every sidebar destination is reachable, with a cue when more + are below. A vertical swipe over a row of tabs scrolls the page, and the + area over the home indicator matches the page in Edge, Chrome and + Firefox. +- **The app's own backup and export import again.** The JSON import + refused both, because they write an empty field as `null` or `""`. It + reads them as the app wrote them now, converts display units such as lb + or mmol/L, and skips deletion records. The portable backup also carries + a reading's glucose context, sleep stage, range, rhythm verdict and + device, which it used to drop. +- **Deleted imports stay deleted after a restore.** A backup restored onto + a fresh server let a Paperless workflow or the import script store again + every document you had deleted. The backup now carries which imports + were deleted. +- **ChatGPT sign-in works without a model override.** The ChatGPT backend + rejects every model from gpt-5.5 down, so an install without its own + model list ended in "all configured Codex slugs were rejected". The + default list now starts at gpt-5.6-terra. +- **Workout splits and GPX times.** Splits were missing for every synced + route, and an exported GPX track had no times. Both read the + stored route points correctly now. +- **A hung backup no longer blocks restores.** A backup that hung held the + slot for large jobs until the container restarted, so a restore waited + forever. A job that outlives its time limit now gives the slot up. +- **API answers match the published contract.** Creating or editing a + medication answers with the same fields as reading it, a dose answers + the full entry, medications and doses come back in the same order on + every request, readings opened by day carry their note, and the + contract now declares several answers the server already gave. +- Every page view saved your language to the account again and counted + against your write limit; it is saved once per tab now. +- A page reloaded right after you entered a reading shows that reading + instead of the dashboard from before. +- The daily briefing's chart notes and storyline were dropped from replies + that passed validation; they are kept now. +- A stored AI key that can no longer be decrypted counts as no key instead + of failing the briefing with a server error. +- After a Fitbit or Google Health history import, the dashboard and + analytics show the new data right away instead of after their cache ran + out. +- Averages and streaks take one clock per request, so a reading at the + edge of a window no longer moves in and out, and today's reading counts + for the streak whatever time zone the database runs in. +- A refused request by someone you share your record with shows in your + activity feed with its own wording. +- A stat's value and unit stay on one line beside the docked day view; + the neutral score label is readable in the dark theme; axis labels are + sized correctly when reduced motion is on. + +- **Large backups upload without filling the memory.** Uploading a copy + with many documents or ECG recordings held them whole while it was + checked, and a big vault could crash the server. Every reader of a + backup now streams the large parts; a 271 MB file needs about 2 MB. +- **Backup texts say what each copy can do.** The weekly copy is an + account rollback on this server, the personal export is for the person, + the JSON import reads measurements and mood within its stated limits, and + only the database dump rebuilds a server. Nothing claims more anymore. +- **Long Coach answers keep all their steps.** After a reload an answer + with more than 48 steps lost the rest. +- **The hint for reading scans on the device points to the right switch**, + Settings, Modules, Labs, in every language. +- **Help links open the right page** instead of going through redirects. +- **A directory setting below `/proc` no longer freezes the server.** Node's + way of creating a folder with its parents loops forever there on Linux, + so a `DB_BACKUP_DIR` or `GEOLITE2_DIR` pointing below it stopped the + process. Such a directory now reads as not writable. +- **The last hour of a long DST day keeps its doses.** On the day the + clocks go back, the list of today's doses that the apps offer from ended + an hour early, so a late evening dose was missing from it; on the day + they go forward it reached an hour into tomorrow. It now spans the real + local day, as the dashboard already did. +- **A dose logged for a two-day plan lands on its slot in the week the + clocks go back.** A plan with doses on two neighbouring weekdays, started + shortly before the change, could file a later dose as unscheduled, and + logging it again gave back the previous one. Whether a plan counts as + daily or weekly no longer depends on the stretch of days it is read + through. + +### Security + +- **A crafted PDF or image can no longer take the server down.** A small + PDF with one huge compressed image, or a JPEG with a padded header, made + the server reserve gigabytes and get killed, for every account on it. + Everything pdf.js does now runs in its own process with a memory and a + time limit, image sizes are read by the JPEG rules and an unreadable size + is refused, and a document over the limits is marked as such instead of + being tried again. +- **Rate limits count the real caller.** Forwarding headers are believed + only from a connection that comes from the server's own network or a + proxy on it; a caller reaching the port directly no longer picks its own + address, and the sign-in throttle no longer exempts an address an account + used before. +- **Writes from other sites are refused.** A request that changes something + and comes from another site, including a sibling subdomain, is refused + before it is read. The iPhone app and other token clients are not + affected. +- **The MCP consent page names where the access goes**, and warns when a + self-registered application sends it to a remote host. +- **Switching Labs off keeps lab values out of every AI feature.** The + Coach, the daily briefing, status notes and the MCP lab prompt read no + lab values while the module is off. Before, the Coach still sent them. +- Smaller fixes: share links show no documents while the documents module + is off, a narrow medication token can no longer probe medication names, + MCP tools fence medication and lab text like every other user text, + reading documents takes turns between accounts, so one account's heavy + files cannot hold up everyone else's, the session cookie cannot be + planted from a sibling subdomain over HTTPS, file-like paths under `/api` + go through the request checks, the deploy + webhook can be signed, the error relay to GlitchTip needs a signed-in + session, and database error details are redacted before they leave. +- Raised the minimum versions of body-parser and fast-uri, which the MCP + server runs on (CVE-2026-111181, high; CVE-2026-110180; + CVE-2026-110306), of ip-address, which took exponential time on a + crafted IPv6 zone identifier, and of a development-only copy of undici. + +### Upgrade notes + +- Migrations 0475 to 0485 (0475, 0476, 0477, 0478, 0479, 0480, 0482, 0483 + and 0485; 0481 and 0484 are not used). Most add a column under a + bounded lock wait and retry on a lock timeout. These do more: + - 0476 links existing readings to the measurement token that sent them + where the account has exactly one such token. It sets only that link, + so the readings do not count as changed. + - 0477 builds an index on `measurements` concurrently; writes carry on + during the build. + - 0478 clears the recorded preview failures and 0479 re-queues the PDFs + whose search entry failed, so the first start renders those previews + once more and indexes those PDFs once more. The indexing uses your AI + provider where one is set up and you have allowed it, as an upload + would. + - 0485 drops the unused preview-features column from `users`. + On a copy of a real instance all of them together took under a second. +- If a concurrent index build fails half-way, for example because the + container was stopped, mend the cause and mark the migration rolled + back; the next start drops the invalid index and builds it again. See + `docs/ops/deploy.md`. +- After a deploy, the worker reports any job it cannot load within a + minute; `docs/ops/deploy.md` says what to look for and when to roll + back. +- `/api/settings/preview-features` is retired and answers 410. It served + a switch for pages that never shipped, and no client reads it. A backup + from 1.44 or 1.45 that carries the field still restores. +- The Apple Health switch takes effect with an iPhone app build that + reads it; until then the app keeps mirroring Withings, imported and + hand-entered readings as before. +- If you set `CODEX_MODEL_FALLBACK_CHAIN` only to get past the rejected + ChatGPT models, you can remove it. +- Migrations 0486 to 0489 add the dump ledger and its key column, the + readiness and kit tables and the deletion columns. 0487 also drops the four old key-backup + columns from `app_settings` and carries an existing confirmation over; + as with 0485, do not run the previous image beside the new schema. +- **A new `backups` volume** holds the local dumps. The compose file + mounts it; a stack built without it keeps dumps off-host only and says + so. In the bucket, dumps go under `database//`, so two + servers sharing a bucket never mix them up; add `database/` to the + bucket's lifecycle rule, and the backups page warns when no rule covers + it. Dumps keep a deleted account until they + age out, so state that period as your backup retention. +- **New settings:** `DB_BACKUP_ENABLED`, `DB_BACKUP_DIR`, + `DB_BACKUP_SCHEDULE`, `DB_BACKUP_KEEP`, `DB_BACKUP_BEFORE_MIGRATE` and + `ACCOUNT_DELETION_GRACE_DAYS`, all on sensible defaults. If the dump + before a migration fails on a writable directory, the start stops and + names the switch to skip it. +- **Behind a reverse proxy,** forwarding headers are believed only from a + private or loopback address. If your proxy reaches HealthLog from a + public address, set `TRUSTED_PROXY_CIDRS` to that address; otherwise + every user shares one sign-in rate limit. `TRUSTED_PROXY_CIDRS` also + narrows the default when you know your proxy's network + (`docs/self-hosting/reverse-proxy.md`). Behind a proxy that rewrites the + Host header, make sure `APP_URL` is right, or writes from the browser + are refused as cross-site. +- The deploy webhook can now be signed (`X-Deploy-Webhook-Signature`); + `DEPLOY_WEBHOOK_REQUIRE_SIGNATURE=true` accepts signed requests only. + `DEPLOY_WEBHOOK_REQUIRE_TIMESTAMP` keeps its meaning, and the stock + Coolify setup needs no change. +- Over HTTPS the session cookie is now `__Host-healthlog_session`. The old + name is still read and swapped on the next request, so nobody is signed + out. +- The app image is about 6 MB larger: it now carries the Postgres 18 + client for the dumps. The supported memory floor stays at 1536 MB for + the app container. +- The first start with the new compose file moves the bundled database to + Postgres 18 (see Changed). Use `docker compose up -d` with the new + compose file; pulling only the new image keeps Postgres 16, which this + release still runs on. The move needs free disk space about the size of + the database. Rollback, the steps for a database you run yourself, and + how to remove the old volume later: `docs/ops/postgres-major-upgrade.md`. + +### Thanks + +Thanks to @haritowa for two reports that came with their fixes: the clash +between two pdf.js versions that broke previews and search (#1277), fixed +in PR #1278, and the "End" button that never worked (#1280), fixed in PR +#1281. Both pull requests are part of this release as written. + +Thanks to @Tio61 for the precise wire captures behind the Coach showing a +model's reasoning as its answer (#1276) and for the exact request counts +that exposed the runaway requests against a model that never answers +(#1279). + +Thanks to @gesus14 for noticing that the daily briefing gave up long +before the response timeout allowed (#1274). + +Thanks to @asmala for coming back to report that "Extract lab values" on +a stored document still failed on long reports (#1257). + +Thanks to @cnrd for the careful proposal that became the Apple Health +switch for measurement tokens (#1272). + +Thanks to @frubino for asking how to get a deleted document back, which +led to Recently deleted (discussion #1275). + +Thanks to @muhdusama for noticing that the MCP labs tool declared a +narrower answer than it gives, and for the fix with its tests (PR #1284). + +Thanks to @sttrebo for asking about Postgres 18, which turned into the +move itself, not just a guide for it (discussion #1282). + ## [1.45.0] — 2026-10-23 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22f7d22e5..78d3286d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,18 +151,27 @@ What a reviewer checks on a translation pull request: - Medical terms are the ones the language's clinical guidelines use, and the register matches the rest of the locale (`du` in German, `você` in Brazilian Portuguese). - Words the app or the Telegram bot matches literally stay as they are: slash commands, `taken `, and the unit symbols. - Machine translation is fine as a first draft, but someone who reads the language has gone through every string before the pull request is opened, and the description says so. -- Validated questionnaire items (the PHQ-9, GAD-7, WHO-5 and SCI items, their answer options and their stems under `mentalHealth.items`, `mentalHealth.options`, `mentalHealth.gad7Options`, `mentalHealth.who5Options`, `mentalHealth.sciOptions`, `mentalHealth.stems`, `mentalHealth.functional*`) are never translated freely, not even as a draft. A language gets them only from a published, licensed validated version, copied verbatim and named in `src/lib/mental-health/instruments.ts`; until then the app shows the English original and the bundle carries none of those keys, which the i18n tests enforce. +- No questionnaire text. The bundles hold the app's own copy around the questionnaires (titles, notes, buttons), never an item, an answer option or a stem; see the next section. To run the checks Fink shows on a local checkout: `npx @inlang/cli@3.4.4 check --project ./project.inlang`. It reads the files and changes nothing. +### Questionnaire wording + +The PHQ-9, GAD-7, WHO-5 and SCI are not translated through the bundles, and Fink never shows them. Their text lives in code, one file per instrument and language under `src/lib/mental-health/wording//.ts`, because changing a word in a validated questionnaire changes what it measures. Each file declares what it is: + +- `validated: true` with a `source`: a published validated translation, reproduced word for word (stem, items, answer options and, for the PHQ-9, the follow-up question, in the form of address the source uses). The source names who made it, the document it was read from with its link, and the terms that allow reproducing it in software. The file's header comment names anything that differs from the printed form, such as a corrected misprint. +- `validated: false`: no validated version exists that may be reproduced, so the file carries a translation written for HealthLog, faithful to the English and on the same scale. The app shows a note beside it that the result is orientation, offers the English original in one tap, and records on the result that it was answered in an unvalidated translation. The header comment names any validated version that was found and why it is not used (paywalled, terms that allow private study only). + +To propose a validated version for a language that has none, or a better source for one that does, open an issue or a pull request with the publication, a link to the PDF it can be read from, and its terms of use. A pull request replaces the language's file: the text copied exactly from the PDF, `validated: true`, and the `source` filled in. The tests in `src/lib/mental-health/__tests__/` check that every instrument has text in every language, that every validated entry names its source, and that each answer is bound to the score the instrument gives it; the spot checks in `instrument-wording.test.ts` are the place to pin a line that is easy to transcribe wrongly. A correction to HealthLog's own translation of an unvalidated entry is welcome too; it stays `validated: false`. + ### Adding a new locale If you want to introduce a locale that isn't shipped (e.g. `sv`, `cs`): 1. Open an issue first using the **Translation request** template. We'll confirm the locale is in scope and decide whether it joins the AI-initial-community-maintained tier or the maintained tier. -2. Copy `messages/en.json` to `messages/.json`, then delete the questionnaire wording the new language has no validated version of (the test output lists the keys). +2. Copy `messages/en.json` to `messages/.json`. Questionnaire text is not in it; the new language needs a wording file per instrument instead (see Questionnaire wording above), which step 4 makes impossible to forget. 3. Translate. AI assistance is welcome as a starting point but the PR description should note whether it's an AI-initial pass or a fully human-reviewed one — that information drives the banner state. -4. Add the locale to `locales` in `src/lib/i18n/config.ts` and to `locales` in `project.inlang/settings.json`, then run `pnpm typecheck`. Every per-locale table that still lacks the new locale fails to compile, and that list is the work: date and number formatting, the reply-language footers of the Coach and insights prompts, the outbound safety banks, the prose-grounding patterns and the no-key fallback copy. Include those in the same PR. +4. Add the locale to `locales` in `src/lib/i18n/config.ts` and to `locales` in `project.inlang/settings.json`, then run `pnpm typecheck`. Every per-locale table that still lacks the new locale fails to compile, and that list is the work: date and number formatting, the reply-language footers of the Coach and insights prompts, the outbound safety banks, the prose-grounding patterns, the no-key fallback copy and the questionnaire wording registry. Include those in the same PR. 5. Run `pnpm test` before submitting. The i18n guards hold the new bundle to the English keys and placeholders. ### Style guide for translators diff --git a/Dockerfile b/Dockerfile index db86834e2..9290eedd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,13 @@ FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb # security patches from the Alpine repo override it deliberately — # CVE-2026-14456 in openssl was the first instance the container scan # caught between base-image rebuilds). -RUN apk add --no-cache tzdata && apk upgrade --no-cache +# +# `postgresql18-client` carries `pg_dump` and `pg_restore` for the nightly +# database dump, the pre-migration dump in the entrypoint and the +# `db-restore` service (docs/ops/disaster-recovery.md). Major 18 matches the +# bundled database; a client dumps servers of its own major and older, so it +# also serves an install still on its own Postgres 16. +RUN apk add --no-cache tzdata postgresql18-client && apk upgrade --no-cache WORKDIR /app ENV NODE_ENV=production @@ -111,6 +117,15 @@ ENV NEXT_PUBLIC_APP_BUILD_SHA=$NEXT_PUBLIC_APP_BUILD_SHA RUN addgroup --system --gid 1001 nodejs && \ adduser --system --uid 1001 nextjs +# Where the database dumps go (`DB_BACKUP_DIR`). Owned by the app user, so a +# fresh named volume mounted here starts with that owner: Docker fills an +# empty named volume from the image directory it covers, ownership included. +# Without a mount the directory stays in the container's layer; the app reads +# its mount table, sees that, and keeps no dump here (dump-core.mjs). +RUN mkdir -p /var/lib/healthlog/backups && \ + chown nextjs:nodejs /var/lib/healthlog/backups && \ + chmod 700 /var/lib/healthlog/backups + # Copy built assets COPY --from=builder /app/public ./public # The server reads its message catalogs from here on first use, one locale at @@ -154,7 +169,7 @@ RUN node -e "require.resolve('pg-boss'); require.resolve('pg')" # before the platform package, so it decides which binary runs. # # Hoist the version the app pins, by name, exactly like pdfjs-dist below. The -# store holds a second, older canvas that pdf-parse's own pdfjs-dist pulls in, +# store once held a second, older canvas that a second pdfjs-dist pulled in, # and this step used to take whichever `find | head -1` returned first. It # returned the old binary, which then sat next to the new loader: rendering # still worked, but `loadImage` called an `Image.decode` the old binary does @@ -190,13 +205,15 @@ RUN set -e; \ # `import('pdfjs-dist/legacy/build/pdf.mjs')`. Node resolves that up to # /app/node_modules, so hoist the real pnpm-store copy to the top level too. # -# Hoist the version the app pins, by name. More than one pdfjs-dist reaches the -# store — `pdf-parse` carries its own, older copy for the text-extraction path — -# and this step used to take whatever `find | head -1` returned first. That is -# readdir order, so it could hoist the OLDER copy and the version named in -# package.json would never run in the image: the renderer resolving one version -# while every local check exercised another. Name the version and fail the build -# when it is absent, instead of degrading to whichever copy comes back first. +# Hoist the version the app pins, by name. More than one pdfjs-dist once reached +# the store (a text-extraction dependency carried its own, older copy), and this +# step used to take whatever `find | head -1` returned first. That is readdir +# order, so it could hoist the OLDER copy and the version named in package.json +# would never run in the image: the renderer resolving one version while every +# local check exercised another. Name the version and fail the build when it is +# absent, instead of degrading to whichever copy comes back first. Two copies in +# one process also break each other (#1277); the lockfile now resolves one, and +# `src/__tests__/single-pdfjs-guard.test.ts` keeps it that way. COPY --from=builder /app/.pdfjs-version /tmp/.pdfjs-version RUN set -e; \ PDFJS_VERSION="$(cat /tmp/.pdfjs-version)"; \ @@ -231,11 +248,16 @@ COPY --from=builder /app/src/generated ./src/generated # the runbooks and the entrypoint against this list. `pending-migrations.mjs` # and `migrate-deploy.mjs` are the entrypoint's own: the first logs the # pending migration count, the second runs `migrate deploy` with its -# lock-timeout retry. Root-owned on purpose: the app user can run them but not -# rewrite them. +# lock-timeout retry. `db-dump.mjs` takes the dump before migrating, and +# `db-restore.mjs` is the `db-restore` service; both share +# `src/lib/database-backup/dump-core.mjs` with the app. Root-owned on purpose: +# the app user can run them but not rewrite them. COPY --from=builder /app/scripts/reset-password.mjs ./scripts/reset-password.mjs COPY --from=builder /app/scripts/pending-migrations.mjs ./scripts/pending-migrations.mjs COPY --from=builder /app/scripts/migrate-deploy.mjs ./scripts/migrate-deploy.mjs +COPY --from=builder /app/scripts/db-dump.mjs ./scripts/db-dump.mjs +COPY --from=builder /app/scripts/db-restore.mjs ./scripts/db-restore.mjs +COPY --from=builder /app/src/lib/database-backup/dump-core.mjs ./src/lib/database-backup/dump-core.mjs COPY --from=builder /app/src/lib/auth/argon2-params.mjs ./src/lib/auth/argon2-params.mjs COPY --from=builder /app/scripts/repair-intake-anomalies.ts ./scripts/repair-intake-anomalies.ts # Prove the reset CLI's imports resolve in this tree: with every import found @@ -245,6 +267,14 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:" RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \ node --input-type=module -e "await import('pg')" && \ node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')" +# The dump and restore scripts load, the client tools they run are the +# major the bundled database runs, and the S3 client `db-restore --from-bucket` +# loads resolves from the shared module's place in the tree. +RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \ + node --input-type=module -e "import { createRequire } from 'node:module'; createRequire('/app/src/lib/database-backup/dump-core.mjs').resolve('@aws-sdk/client-s3')" && \ + node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \ + pg_dump --version | grep -q "(PostgreSQL) 18\." && \ + pg_restore --version | grep -q "(PostgreSQL) 18\." # Install the migration CLI, its config dependency, and a pinned launcher for # the maintenance scripts already shipped in the standalone tree. The Prisma diff --git a/README.md b/README.md index 299fa3065..4c880ab87 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Try the [live demo](https://demo.healthlog.dev) to see a working install — sig - **Clinical signals, opt-in and additive.** Validated self-checks for mental wellbeing and sleep (WHO-5, PHQ-9, GAD-7, Sleep Condition Indicator — official questionnaire wording, calm crisis-resource handling), grip strength, a 0–10 pain score, waist and body measurements, and a longevity lab panel — each added through one signal registry, so a new signal wires into charts, correlations, the coach, the FHIR map, and the API at once. - **Records, structured.** Allergies and family history as structured records that flow into the doctor report (reactions and notes are encrypted, the substance and condition names are not), plus lab values you key in or scan from a printed report — OCR extracts the stated values, units, and dates for you to review and confirm before anything is saved. HealthLog transcribes what a report says; it never interprets or diagnoses from it. A vaccination record keeps a lifetime immunization log, doses grouped under the disease they protect against, and prints in the doctor report. - **A document vault for the paper trail.** An opt-in module stores every letter, report, and scan encrypted at rest — PDFs and photos preview inline, Office and text files stay download-only, and nothing user-uploaded can ever execute in the app. One searchable timeline with type, condition, and year filters, drag-and-drop and paste-to-upload, bulk filing, undo-able delete with a 30-day grace, and per-file/per-account limits the operator tunes. -- **Context that explains the numbers.** An opt-in environmental module folds weather, daylight, and temperature in as influence signals (winter darkness against mood, heat against blood pressure) using a home location you set, so a pattern has somewhere to point. +- **Context that explains the numbers.** An environmental module folds weather, daylight, temperature, air quality, and pollen in as influence signals (winter darkness against mood, heat against blood pressure) using a home location you set, so a pattern has somewhere to point. It is on by default, and nothing leaves the host until someone sets that location. - **Your devices, one server.** Withings, WHOOP, Oura and Polar sync over OAuth2, Google Health connects Fitbit and Pixel Watch through your own Google Cloud client, Nightscout streams CGM glucose, an Apple Health `export.zip` folds your full history in, and the native iOS app streams HealthKit live. A per-metric source priority decides which reading is canonical when wearables overlap. - **Medication tracking that tells the truth.** Flexible schedules (weekly injections, weekday-only, intervals, PRN, cyclic), a configurable intake window per dose, and a traceable dose history where every slot reads taken, late, skipped, or missed — the same ledger the compliance rate is computed from, so the percentage can never disagree with the timeline. - **AI insights you own.** Daily briefing, health scores, correlations, and a conversational coach grounded in your own measurements and interpreted against cited guideline reference bands — where a value sits and what that means in plain words, never a diagnosis. The coach proposes concrete if-then plans you accept or decline, and suppresses proactive check-ins when there's nothing genuine to say. Bring your own OpenAI or Anthropic key, or point at a local endpoint (Ollama, LM Studio, vLLM) so AI requests stay on your network. @@ -85,11 +85,11 @@ Verify the running version: curl -s http://localhost:3000/api/version # returns version + buildSha + builtAt ``` -`SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/self-hosting/)). +`SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/getting-started/self-host-in-10-minutes/)). -Persistent state lives in exactly one named volume (`pgdata`, the Postgres data dir); the app container is stateless, so backing up the database backs up everything. +Persistent state lives in the database volume (`postgres-data`) and in `backups`, which holds the nightly database dumps; the app container is stateless. An install from v1.45 or earlier moves its Postgres 16 database from `pgdata` into the new volume on the first start ([details](docs/ops/postgres-major-upgrade.md)); recovering a lost server is in [disaster-recovery.md](docs/ops/disaster-recovery.md). -NAS and homelab users have ready-made paths: [Unraid Community Applications](docs/self-hosting/unraid.md) and [Portainer app templates](docs/self-hosting/portainer/README.md). +NAS and homelab users have ready-made paths: [Unraid Community Applications](docs/self-hosting/unraid.md), [Portainer app templates](docs/self-hosting/portainer/README.md) and [TrueNAS](docs/self-hosting/truenas.md) (the Community train app is pending catalog review). ## Self-hosting @@ -105,7 +105,7 @@ The App Store listing carries a small price, which exists to cover Apple's yearl ## Tech stack -Next.js 16 (App Router, React Server Components), TypeScript strict, PostgreSQL 16 with Prisma, pg-boss for jobs, Tailwind CSS 4 + shadcn/ui, Recharts, WebAuthn passkeys, Vitest + Playwright, multi-stage Alpine Docker image. The OpenAPI 3.1 contract for native clients is generated from the Zod schemas at [`docs/api/openapi.yaml`](docs/api/openapi.yaml). +Next.js 16 (App Router, React Server Components), TypeScript strict, PostgreSQL 18 with Prisma, pg-boss for jobs, Tailwind CSS 4 + shadcn/ui, Recharts, WebAuthn passkeys, Vitest + Playwright, multi-stage Alpine Docker image. The OpenAPI 3.1 contract for native clients is generated from the Zod schemas at [`docs/api/openapi.yaml`](docs/api/openapi.yaml). ## Documentation diff --git a/bundle-budget.json b/bundle-budget.json index 0553bb199..9825cf072 100644 --- a/bundle-budget.json +++ b/bundle-budget.json @@ -27,6 +27,7 @@ "/custom-metrics/[id]/page": 303, "/custom-metrics/[id]/values/page": 311, "/cycle/page": 412, + "/documents/deleted/page": 291, "/documents/page": 378, "/enroll-mfa/page": 305, "/illness/[id]/page": 447, diff --git a/docker-compose.yml b/docker-compose.yml index aa17c56b1..eb6dc7dcf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -# HealthLog — two-container stack (app + Postgres 16). +# HealthLog — two-container stack (app + Postgres 18). # # First run (clean host): # cp .env.example .env @@ -14,9 +14,13 @@ # and login silently fails. This is the single most common self-host snag. # Leave it unset (or true) only when a TLS reverse proxy serves HTTPS. # -# Persistent state lives in exactly one named volume (`pgdata`, Postgres' -# data dir). The app container is stateless, so a backup of the database is -# a backup of everything. The optional GeoLite2 databases are either baked +# Persistent state lives in the named volume `postgres-data` (Postgres' data). +# The app container is stateless, so a backup of the database is a backup of +# everything. `backups` holds the database dumps the app takes every night and +# before every migration (docs/ops/disaster-recovery.md); keep a copy of them +# off this host, or set the off-host bucket below. `pgdata` is where Postgres 16 kept the data up to +# v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once +# and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked # into the image at build time or mounted read-only from the host and # pointed at with GEOLITE2_DIR — either way they are replaceable, not state. @@ -88,8 +92,10 @@ x-healthlog-env: &healthlog_env TELEGRAM_WEBHOOK_SECRET: "${TELEGRAM_WEBHOOK_SECRET:-}" # v1.18.11 (W3) — client-IP resolution + geo lookup. Behind Cloudflare # the real visitor IP is in `cf-connecting-ip`; set - # TRUST_CF_CONNECTING_IP=1 ONLY when Cloudflare is in front (off by - # default — a direct-exposed host must not trust a forgeable header). + # TRUST_CF_CONNECTING_IP=1 ONLY when the origin accepts connections from + # Cloudflare alone (a Tunnel, or a firewall admitting only Cloudflare's + # ranges): the header is believed from every connection (off by default; + # see docs/self-hosting/reverse-proxy.md). # IP_GEO_LOOKUP_URL overrides the default ipwho.is provider (HTTPS by # default); IP_GEO_LOOKUP_DISABLED=1 stops all third-party IP egress. # IP_GEO_ALLOW_INSECURE=true permits a plain-HTTP provider URL (e.g. the @@ -102,7 +108,14 @@ x-healthlog-env: &healthlog_env # collapses every anonymous caller into one rate-limit bucket. Listed here # so an operator value in .env reaches the container (the `environment:` # block is a whitelist). + # TRUSTED_PROXY_CIDRS names the addresses the reverse proxy connects from + # (comma-separated CIDRs). Set, only those connections may report a client + # address; unset, any private address may, which behind Docker Desktop, + # OrbStack or docker-proxy includes every caller of the published port. + # Same host: 127.0.0.1/32,::1/128. Docker network: the proxy's subnet. + # See docs/self-hosting/reverse-proxy.md. TRUST_PROXY_HOPS: "${TRUST_PROXY_HOPS:-}" + TRUSTED_PROXY_CIDRS: "${TRUSTED_PROXY_CIDRS:-}" TRUST_CF_CONNECTING_IP: "${TRUST_CF_CONNECTING_IP:-}" IP_GEO_LOOKUP_URL: "${IP_GEO_LOOKUP_URL:-}" IP_GEO_LOOKUP_DISABLED: "${IP_GEO_LOOKUP_DISABLED:-}" @@ -145,6 +158,20 @@ x-healthlog-env: &healthlog_env # serves anyway. A last resort for a refusal you are certain is false: it # disables a safety check. See docs/self-hosting/getting-started.md. ENCRYPTION_KEY_CHECK: "${ENCRYPTION_KEY_CHECK:-}" + # The whole-database dump (docs/ops/disaster-recovery.md). Nightly at + # DB_BACKUP_SCHEDULE (five-field cron in Europe/Berlin time, default + # "30 1 * * *") into DB_BACKUP_DIR (default /var/lib/healthlog/backups, the + # `backups` volume below), keeping DB_BACKUP_KEEP daily dumps (default 7) + # and the last two taken before a migration. With the off-host bucket set, + # each nightly dump is also uploaded under `database/`, sealed with + # BACKUP_ENCRYPTION_KEY. DB_BACKUP_ENABLED=false stops the nightly dump; + # DB_BACKUP_BEFORE_MIGRATE=false skips the dump before migrations. Change + # DB_BACKUP_DIR only together with the volume mount. + DB_BACKUP_ENABLED: "${DB_BACKUP_ENABLED:-}" + DB_BACKUP_DIR: "${DB_BACKUP_DIR:-}" + DB_BACKUP_SCHEDULE: "${DB_BACKUP_SCHEDULE:-}" + DB_BACKUP_KEEP: "${DB_BACKUP_KEEP:-}" + DB_BACKUP_BEFORE_MIGRATE: "${DB_BACKUP_BEFORE_MIGRATE:-}" BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}" BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}" BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}" @@ -166,6 +193,7 @@ x-healthlog-env: &healthlog_env # the block exists: `environment:` is a whitelist, and a value in .env that # is not named here never arrives. AUDIT_LOG_RETENTION_DAYS: "${AUDIT_LOG_RETENTION_DAYS:-}" + ACCOUNT_DELETION_GRACE_DAYS: "${ACCOUNT_DELETION_GRACE_DAYS:-}" # How long Coach conversations are kept, in days (default 365, floor 30). # Like the audit window this number is PUBLISHED to the user on the Data & # Privacy page, so an instance running 90 has to be able to say 90. @@ -278,17 +306,23 @@ x-healthlog-env: &healthlog_env OIDC_SCOPES: "${OIDC_SCOPES:-}" OIDC_BUTTON_LABEL: "${OIDC_BUTTON_LABEL:-}" OIDC_ONLY: "${OIDC_ONLY:-}" - # Deploy-result webhook (optional). DEPLOY_WEBHOOK_SECRET authenticates - # POST /api/internal/deploy-webhook; DEPLOY_LOGS_URL is the - # deployment-dashboard URL (e.g. the Coolify UI) linked in - # deploy-failure notifications. Listed here so operator values in - # .env reach the container (the `environment:` block is a whitelist). + # Deploy-result webhook (optional). POST /api/internal/deploy-webhook + # authenticates either with DEPLOY_WEBHOOK_SECRET in the + # X-Deploy-Webhook-Secret header (what Coolify's stock sender can do; no + # replay protection) or with X-Deploy-Webhook-Signature: sha256=." under the secret>, the timestamp + # within 5 minutes. DEPLOY_WEBHOOK_REQUIRE_TIMESTAMP=true also requires a + # fresh X-Deploy-Webhook-Timestamp on a secret-header request; + # DEPLOY_WEBHOOK_REQUIRE_SIGNATURE=true accepts signed requests only, the + # one setting that stops a replay. DEPLOY_LOGS_URL is the + # deployment-dashboard URL (e.g. the Coolify UI) linked in deploy-failure + # notifications. Listed here so operator values in .env reach the + # container (the `environment:` block is a whitelist). See + # docs/ops/deploy.md. DEPLOY_WEBHOOK_SECRET: "${DEPLOY_WEBHOOK_SECRET:-}" DEPLOY_LOGS_URL: "${DEPLOY_LOGS_URL:-}" - # Optional replay hardening: when "true", the deploy webhook - # requires a fresh X-Deploy-Webhook-Timestamp header (±5 min). - # See docs/ops/deploy.md. DEPLOY_WEBHOOK_REQUIRE_TIMESTAMP: "${DEPLOY_WEBHOOK_REQUIRE_TIMESTAMP:-}" + DEPLOY_WEBHOOK_REQUIRE_SIGNATURE: "${DEPLOY_WEBHOOK_REQUIRE_SIGNATURE:-}" # TLS leaf SPKI-change alarm baseline (optional). The native client # pins the served TLS leaf; this is the known-good pin set the monitor # alarms against on a rotation. Comma-separated for the dual-pin @@ -473,6 +507,11 @@ services: - "3000:3000" environment: <<: *healthlog_env + # The database dumps (DB_BACKUP_DIR). Without this mount the app still + # copies its nightly dump off-host when the bucket is set, but keeps none + # here and takes none before a migration; the boot log says so. + volumes: + - backups:/var/lib/healthlog/backups # Bring your own GeoLite2 databases. The published image only carries # MMDBs if the build had a MaxMind licence key, so on `:latest` the # offline tier is usually absent and every login IP goes to the online @@ -482,13 +521,15 @@ services: # matches GEOLITE2_DIR's default (/opt/geolite2), so this works as-is # once uncommented. Mounting at a different container path also works, # just also set GEOLITE2_DIR in .env to match it. - # Steps: docs/self-hosting/geolite2.md. + # Steps: docs/self-hosting/geolite2.md. The line goes under `volumes:` + # above, beside the backups line: # - # volumes: # - ./geolite2:/opt/geolite2:ro depends_on: db: condition: service_healthy + db-upgrade: + condition: service_completed_successfully restart: unless-stopped # Bounded log rotation: the default json-file driver keeps every line # forever, which is exactly what fills the disk on the small NAS / @@ -559,9 +600,14 @@ services: # environment: # <<: *healthlog_env # HEALTHLOG_PROCESS_TYPE: "worker" + # # The worker takes the nightly database dump, so it needs the volume. + # volumes: + # - backups:/var/lib/healthlog/backups # depends_on: # db: # condition: service_healthy + # db-upgrade: + # condition: service_completed_successfully # restart: unless-stopped # logging: # driver: json-file @@ -572,12 +618,11 @@ services: # stop_grace_period: 45s db: - # Pinned to the concrete minor instead of the floating `16-alpine` + # Pinned to the concrete minor instead of the floating `18-alpine` # alias so a database update is always a deliberate, reviewable - # change (at pin time `16-alpine` resolved to this same digest, so - # existing deployments see no image change). Minor releases share - # the on-disk format — bumping this pin is safe without a dump / - # restore; a major-version bump (17+) is not. + # change. Minor releases share the on-disk format, so bumping this pin + # is safe without a dump / restore; a major version is not, which is + # what `db-upgrade` below and docs/ops/postgres-major-upgrade.md are for. # # The pin only works as long as something watches it: PostgreSQL # publishes minor releases with security and data-corruption fixes and @@ -587,7 +632,7 @@ services: # `.github/dependabot.yml` now raises a PR for each new minor; # `src/__tests__/database-pin-is-watched.test.ts` keeps both halves # (concrete pin, active watch) from being dropped. - image: postgres:16.15-alpine + image: postgres:18.6-alpine container_name: healthlog-db # Loads pg_stat_statements so slow queries can be found from measurements # instead of guesses. Loading the library only collects statistics; the @@ -617,8 +662,12 @@ services: POSTGRES_USER: healthlog POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" POSTGRES_DB: healthlog + # From 18 on, the image keeps its data under /var/lib/postgresql/18/docker + # and expects the volume one level up, so the next major can sit beside + # this one in the same volume. Mounting it at .../data, as 16 had it, + # makes the container refuse to start. volumes: - - pgdata:/var/lib/postgresql/data + - postgres-data:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U healthlog"] interval: 5s @@ -639,5 +688,199 @@ services: max-size: "10m" max-file: "3" + # Moves a Postgres 16 database from v1.45 or earlier into Postgres 18, once, + # on the first start after the update. A major version cannot read the data + # directory of an older one, so changing the image line alone would leave + # the database refusing to start. + # + # On every start it decides in a second and exits: + # - no `pgdata` data (a new install): nothing to do; + # - the database service runs Postgres 16 (after going back): nothing to + # do, the live 16 volume is left alone; + # - 16 was used again after an earlier move: refuses, so the app does not + # start on the move-day copy; + # - Postgres 18 already holds the database: nothing to do; + # - otherwise it starts Postgres 16 inside this container on the old + # volume, streams a dump into the new database in one transaction, + # compares the row count of every table, gathers planner statistics and + # marks the old volume as moved. `app` starts only after it succeeded. + # A move cut short (a restart, a full disk) leaves a "moving" marker, and + # the next start empties the new database and runs it again from scratch. + # + # The old volume is read, never changed, apart from that marker file, so + # going back is the previous image and volume lines. If anything fails, the + # new database stays empty, `app` does not start, and the log says why. + # It costs one small image pull; a later release removes the service. + db-upgrade: + image: postgres:16.15-alpine + container_name: healthlog-db-upgrade + # Runs once per `compose up` and exits. "no" also keeps hosts such as + # Coolify from reporting the exited container as a failed service. + restart: "no" + security_opt: + - no-new-privileges:true + environment: + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + volumes: + - pgdata:/var/lib/postgresql/data + # Read-only, only to check there is room for the copy before starting. + - postgres-data:/var/lib/postgresql/next:ro + depends_on: + db: + condition: service_healthy + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + # `$$` is how compose spells a literal `$`. + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + set -o pipefail + log() { echo "db-upgrade: $$*"; } + OLD=/var/lib/postgresql/data + MARK="$$OLD/HEALTHLOG_MOVED_TO_POSTGRES_18" + MOVING="$$OLD/HEALTHLOG_MOVING_TO_POSTGRES_18" + export PGPASSWORD="$$POSTGRES_PASSWORD" + new() { psql -h db -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$$1"; } + old() { psql -h /tmp -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$$1"; } + empty_new() { + for S in $$(new "SELECT quote_ident(nspname) FROM pg_namespace WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'"); do + new "DROP SCHEMA $$S CASCADE" + done + new "CREATE SCHEMA public" + } + if [ ! -s "$$OLD/PG_VERSION" ]; then + log "no Postgres 16 database to move, nothing to do" + exit 0 + fi + # The database service must be the new major. After going back to 16 the + # service runs on this very volume: touching it then would wipe the live data. + i=0 + until NEWVER=$$(new "SHOW server_version_num" 2>/dev/null); do + i=$$((i + 1)) + [ "$$i" -lt 60 ] || { log "the database service did not answer for a minute"; exit 1; } + sleep 1 + done + if [ "$$NEWVER" -lt 180000 ]; then + log "the database service runs Postgres $$((NEWVER / 10000)), not 18: nothing to move, the old volume is left alone" + exit 0 + fi + checkpoint() { pg_controldata "$$OLD" | awk -F': *' '/^Latest checkpoint location/ { print $$2 }'; } + USER_TABLES="FROM pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema')" + TABLES=$$(new "SELECT count(*) $$USER_TABLES") + if [ -e "$$MARK" ]; then + MOVED_ON=$$(sed -n 1p "$$MARK") + MOVED_CHECKPOINT=$$(sed -n 2p "$$MARK") + if [ -n "$$MOVED_CHECKPOINT" ] && [ "$$(checkpoint)" != "$$MOVED_CHECKPOINT" ]; then + log "the Postgres 16 database changed after it was moved on $$MOVED_ON: it was used again after going back to 16." + log "Postgres 18 still holds the state of that day, so it is not started on. To move the current 16 database," + log "see docs/ops/postgres-major-upgrade.md, \"Updating again after going back\"." + exit 1 + fi + if [ "$$TABLES" != 0 ]; then + log "the Postgres 16 database was moved on $$MOVED_ON, nothing to do" + exit 0 + fi + log "the Postgres 16 database was moved on $$MOVED_ON, but the Postgres 18 database is empty now." + log "Not importing the old state again by itself. See docs/ops/postgres-major-upgrade.md, \"Starting over\"." + exit 1 + fi + if [ -e "$$MOVING" ]; then + log "an earlier move did not finish; emptying the Postgres 18 database and starting it again" + empty_new + elif [ "$$TABLES" != 0 ]; then + log "the Postgres 18 database already holds data, leaving both as they are" + exit 0 + fi + VERSION=$$(cat "$$OLD/PG_VERSION") + if [ "$$VERSION" != 16 ]; then + log "the old data directory is Postgres $$VERSION, this step moves Postgres 16 only." + log "See docs/ops/postgres-major-upgrade.md to move it by hand." + exit 1 + fi + NEED=$$(du -sk "$$OLD" | cut -f1) + FREE=$$(df -Pk /var/lib/postgresql/next | awk 'NR == 2 { print $$4 }') + # Room for the copy, and for the dump the app takes before its first migration. + if [ "$$FREE" -le $$((NEED * 3 / 2)) ]; then + log "the move needs about $$((NEED * 3 / 2 / 1024)) MiB free for the new database and its first dump, the disk has $$((FREE / 1024)) MiB." + log "Nothing was changed. Free some space and start the stack again." + exit 1 + fi + trap 'CODE=$$?; [ "$$CODE" = 0 ] || log "the move stopped (exit $$CODE). The old volume is untouched; the next start begins it again."' EXIT + date -u +%Y-%m-%dT%H:%M:%SZ > "$$MOVING" + log "moving the Postgres 16 database ($$((NEED / 1024)) MiB on disk) into Postgres 18" + su-exec postgres postgres -D "$$OLD" -c listen_addresses= -c unix_socket_directories=/tmp & + PID=$$! + i=0 + until pg_isready -h /tmp -U healthlog -q; do + kill -0 "$$PID" 2>/dev/null || { log "the Postgres 16 server did not start, see the lines above"; exit 1; } + i=$$((i + 1)) + [ "$$i" -lt 600 ] || { log "the Postgres 16 server did not get ready in ten minutes"; exit 1; } + sleep 1 + done + pg_dump -h /tmp -U healthlog -Fc healthlog | + pg_restore -h db -U healthlog -d healthlog --no-owner --single-transaction --exit-on-error + COUNTED=0 + for T in $$(old "SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) $$USER_TABLES ORDER BY 1"); do + A=$$(old "SELECT count(*) FROM $$T") + B=$$(new "SELECT count(*) FROM $$T") + if [ "$$A" != "$$B" ]; then + log "table $$T has $$A rows in Postgres 16 but $$B in Postgres 18. Emptying the new database again." + log "Please report this with the lines above." + empty_new + exit 1 + fi + COUNTED=$$((COUNTED + 1)) + done + log "all $$COUNTED tables arrived with the same row counts" + vacuumdb -h db -U healthlog -d healthlog --analyze-in-stages --quiet + kill -INT "$$PID" + wait "$$PID" || true + # The marker keeps the old database's last checkpoint: a later start can tell + # whether 16 was used again after the move. + { date -u +%Y-%m-%dT%H:%M:%SZ; checkpoint; } > "$$MOVING" + mv "$$MOVING" "$$MARK" + log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md" + + # Restores a database dump into an EMPTY database (docs/ops/disaster-recovery.md). + # Never started by `docker compose up`; run it by name: + # + # docker compose run --rm db-restore list + # docker compose run --rm db-restore latest + # docker compose run --rm db-restore healthlog-