Merge remote-tracking branch 'origin/release/v1.46.0' into fix/demo-bp-story

.github/dependabot.yml

@@ -79,6 +79,8 @@ updates:
       # The first run of this ecosystem proposed 16.15 -> 18.6, which would
       # have handed self-hosters a database their data directory cannot be
       # read by. Minor releases still arrive as PRs; a major is a deliberate
-      # project decision with a migration guide behind it.
+      # project decision with a migration guide behind it. The ignore also
+      # keeps `db-upgrade` on 16: it exists to read Postgres 16 data, and a
+      # bump to 18 would leave it unable to start on the volume it moves.
       - dependency-name: "postgres"
         update-types: ["version-update:semver-major"]

.github/workflows/compose-db-upgrade.yml

@@ -0,0 +1,51 @@
+name: Compose database move
+
+# Plays the Postgres 16 -> 18 move in docker-compose.yml against real
+# containers: a v1.45 database with every migration applied, then the current
+# compose file's `db-upgrade` on its first start, a second start, a move cut
+# short, an emptied new volume and a new install. The service runs once on an
+# operator's host and has no second chance there, so every change to the
+# compose file or to the check runs it here first.
+# `scripts/compose-db-upgrade-check.sh` is the same check, runnable locally.
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - docker-compose.yml
+      - scripts/compose-db-upgrade-check.sh
+      - .github/workflows/compose-db-upgrade.yml
+  push:
+    branches: [main]
+    paths:
+      - docker-compose.yml
+      - scripts/compose-db-upgrade-check.sh
+      - .github/workflows/compose-db-upgrade.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  move:
+    name: Postgres 16 -> 18 move
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      # The check reads the v1.45 compose file from its tag.
+      - name: Fetch the v1.45.0 tag
+        run: git fetch --no-tags --depth=1 origin tag v1.45.0
+
+      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Move check
+        run: bash scripts/compose-db-upgrade-check.sh

.github/workflows/e2e.yml

@@ -40,7 +40,7 @@ jobs:
     name: e2e shard ${{ matrix.shard }}/3
     services:
       postgres:
-        image: postgres:16-alpine
+        image: postgres:18-alpine
         env:
           POSTGRES_USER: healthlog
           POSTGRES_PASSWORD: healthlog
@@ -158,7 +158,7 @@ jobs:
       image: mcr.microsoft.com/playwright:v${{ needs.playwright-version.outputs.version }}-noble
     services:
       postgres:
-        image: postgres:16-alpine
+        image: postgres:18-alpine
         env:
           POSTGRES_USER: healthlog
           POSTGRES_PASSWORD: healthlog

.github/workflows/post-publish-verify.yml

@@ -56,7 +56,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:16-alpine
+        image: postgres:18-alpine
         env:
           POSTGRES_USER: healthlog
           POSTGRES_PASSWORD: healthlog

.github/workflows/test-nightly.yml

@@ -84,7 +84,7 @@ jobs:
       fail-fast: false
       matrix:
         shard: [1, 2, 3, 4]
-    name: integration shard ${{ matrix.shard }}/4 (shuffled)
+    name: integration shard ${{ matrix.shard }}/4 (shuffled, Postgres 16)
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
@@ -110,6 +110,10 @@ jobs:
         shell: bash
         env:
           HEALTHLOG_INTEGRATION_WORKERS: "3"
+          # Pull requests run the suite on Postgres 18, the compose default.
+          # Installs that bring their own database may still be on 16, so the
+          # nightly run keeps that major covered without doubling every PR.
+          INTEGRATION_PG_IMAGE: "postgres:16-alpine"
           # Four times each property's own case count; a property's timeout
           # grows by the same factor (`propertyTimeout`).
           FC_RUNS_FACTOR: "4"

README.md

@@ -87,7 +87,7 @@ curl -s http://localhost:3000/api/version   # returns version + buildSha + built
 
 `SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/self-hosting/)).
 
-Persistent state lives in exactly one named volume (`pgdata`, the Postgres data dir); the app container is stateless, so backing up the database backs up everything.
+Persistent state lives in exactly one named volume (`postgres-data`, the Postgres data); the app container is stateless, so backing up the database backs up everything. An install from v1.45 or earlier moves its Postgres 16 database into it on the first start ([details](docs/ops/postgres-major-upgrade.md)).
 
 NAS and homelab users have ready-made paths: [Unraid Community Applications](docs/self-hosting/unraid.md), [Portainer app templates](docs/self-hosting/portainer/README.md) and [TrueNAS](docs/self-hosting/truenas.md) (the Community train app is pending catalog review).
 
@@ -105,7 +105,7 @@ The App Store listing carries a small price, which exists to cover Apple's yearl
 
 ## Tech stack
 
-Next.js 16 (App Router, React Server Components), TypeScript strict, PostgreSQL 16 with Prisma, pg-boss for jobs, Tailwind CSS 4 + shadcn/ui, Recharts, WebAuthn passkeys, Vitest + Playwright, multi-stage Alpine Docker image. The OpenAPI 3.1 contract for native clients is generated from the Zod schemas at [`docs/api/openapi.yaml`](docs/api/openapi.yaml).
+Next.js 16 (App Router, React Server Components), TypeScript strict, PostgreSQL 18 with Prisma, pg-boss for jobs, Tailwind CSS 4 + shadcn/ui, Recharts, WebAuthn passkeys, Vitest + Playwright, multi-stage Alpine Docker image. The OpenAPI 3.1 contract for native clients is generated from the Zod schemas at [`docs/api/openapi.yaml`](docs/api/openapi.yaml).
 
 ## Documentation
 

docker-compose.yml

@@ -1,4 +1,4 @@
-# HealthLog — two-container stack (app + Postgres 16).
+# HealthLog — two-container stack (app + Postgres 18).
 #
 # First run (clean host):
 #   cp .env.example .env
@@ -14,9 +14,11 @@
 # and login silently fails. This is the single most common self-host snag.
 # Leave it unset (or true) only when a TLS reverse proxy serves HTTPS.
 #
-# Persistent state lives in exactly one named volume (`pgdata`, Postgres'
-# data dir). The app container is stateless, so a backup of the database is
-# a backup of everything. The optional GeoLite2 databases are either baked
+# Persistent state lives in exactly one named volume (`postgres-data`,
+# Postgres' data). The app container is stateless, so a backup of the database
+# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to
+# v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once
+# and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked
 # into the image at build time or mounted read-only from the host and
 # pointed at with GEOLITE2_DIR — either way they are replaceable, not state.
 
@@ -489,6 +491,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      db-upgrade:
+        condition: service_completed_successfully
     restart: unless-stopped
     # Bounded log rotation: the default json-file driver keeps every line
     # forever, which is exactly what fills the disk on the small NAS /
@@ -562,6 +566,8 @@ services:
   #   depends_on:
   #     db:
   #       condition: service_healthy
+  #     db-upgrade:
+  #       condition: service_completed_successfully
   #   restart: unless-stopped
   #   logging:
   #     driver: json-file
@@ -572,13 +578,11 @@ services:
   #   stop_grace_period: 45s
 
   db:
-    # Pinned to the concrete minor instead of the floating `16-alpine`
+    # Pinned to the concrete minor instead of the floating `18-alpine`
     # alias so a database update is always a deliberate, reviewable
-    # change (at pin time `16-alpine` resolved to this same digest, so
-    # existing deployments see no image change). Minor releases share
-    # the on-disk format — bumping this pin is safe without a dump /
-    # restore; a major-version bump (17+) is not, see
-    # docs/ops/postgres-major-upgrade.md.
+    # change. Minor releases share the on-disk format, so bumping this pin
+    # is safe without a dump / restore; a major version is not, which is
+    # what `db-upgrade` below and docs/ops/postgres-major-upgrade.md are for.
     #
     # The pin only works as long as something watches it: PostgreSQL
     # publishes minor releases with security and data-corruption fixes and
@@ -588,7 +592,7 @@ services:
     # `.github/dependabot.yml` now raises a PR for each new minor;
     # `src/__tests__/database-pin-is-watched.test.ts` keeps both halves
     # (concrete pin, active watch) from being dropped.
-    image: postgres:16.15-alpine
+    image: postgres:18.6-alpine
     container_name: healthlog-db
     # Loads pg_stat_statements so slow queries can be found from measurements
     # instead of guesses. Loading the library only collects statistics; the
@@ -618,8 +622,12 @@ services:
       POSTGRES_USER: healthlog
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
       POSTGRES_DB: healthlog
+    # From 18 on, the image keeps its data under /var/lib/postgresql/18/docker
+    # and expects the volume one level up, so the next major can sit beside
+    # this one in the same volume. Mounting it at .../data, as 16 had it,
+    # makes the container refuse to start.
     volumes:
-      - pgdata:/var/lib/postgresql/data
+      - postgres-data:/var/lib/postgresql
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U healthlog"]
       interval: 5s
@@ -640,5 +648,134 @@ services:
         max-size: "10m"
         max-file: "3"
 
+  # Moves a Postgres 16 database from v1.45 or earlier into Postgres 18, once,
+  # on the first start after the update. A major version cannot read the data
+  # directory of an older one, so changing the image line alone would leave
+  # the database refusing to start.
+  #
+  # On every start it decides in a second and exits:
+  # - no `pgdata` data (a new install): nothing to do;
+  # - Postgres 18 already holds the database: nothing to do;
+  # - otherwise it starts Postgres 16 inside this container on the old
+  #   volume, streams a dump into the new database in one transaction,
+  #   compares the row count of every table, gathers planner statistics and
+  #   marks the old volume as moved. `app` starts only after it succeeded.
+  #   A move cut short (a restart, a full disk) leaves a "moving" marker, and
+  #   the next start empties the new database and runs it again from scratch.
+  #
+  # The old volume is read, never changed, apart from that marker file, so
+  # going back is the previous image and volume lines. If anything fails, the
+  # new database stays empty, `app` does not start, and the log says why.
+  # It costs one small image pull; a later release removes the service.
+  db-upgrade:
+    image: postgres:16.15-alpine
+    container_name: healthlog-db-upgrade
+    # Runs once per `compose up` and exits. "no" also keeps hosts such as
+    # Coolify from reporting the exited container as a failed service.
+    restart: "no"
+    security_opt:
+      - no-new-privileges:true
+    environment:
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+      # Read-only, only to check there is room for the copy before starting.
+      - postgres-data:/var/lib/postgresql/next:ro
+    depends_on:
+      db:
+        condition: service_healthy
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    # `$$` is how compose spells a literal `$`.
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        set -eu
+        set -o pipefail
+        log() { echo "db-upgrade: $$*"; }
+        OLD=/var/lib/postgresql/data
+        MARK="$$OLD/HEALTHLOG_MOVED_TO_POSTGRES_18"
+        MOVING="$$OLD/HEALTHLOG_MOVING_TO_POSTGRES_18"
+        export PGPASSWORD="$$POSTGRES_PASSWORD"
+        new() { psql -h db -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$$1"; }
+        old() { psql -h /tmp -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$$1"; }
+        empty_new() {
+          for S in $$(new "SELECT quote_ident(nspname) FROM pg_namespace WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'"); do
+            new "DROP SCHEMA $$S CASCADE"
+          done
+          new "CREATE SCHEMA public"
+        }
+        if [ ! -s "$$OLD/PG_VERSION" ]; then
+          log "no Postgres 16 database to move, nothing to do"
+          exit 0
+        fi
+        USER_TABLES="FROM pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema')"
+        TABLES=$$(new "SELECT count(*) $$USER_TABLES")
+        if [ -e "$$MARK" ]; then
+          if [ "$$TABLES" != 0 ]; then
+            log "the Postgres 16 database was moved on $$(cat "$$MARK"), nothing to do"
+            exit 0
+          fi
+          log "the Postgres 16 database was moved on $$(cat "$$MARK"), but the Postgres 18 database is empty now."
+          log "Not importing the old state again by itself. See docs/ops/postgres-major-upgrade.md, \"Starting over\"."
+          exit 1
+        fi
+        if [ -e "$$MOVING" ]; then
+          log "an earlier move did not finish; emptying the Postgres 18 database and starting it again"
+          empty_new
+        elif [ "$$TABLES" != 0 ]; then
+          log "the Postgres 18 database already holds data, leaving both as they are"
+          exit 0
+        fi
+        VERSION=$$(cat "$$OLD/PG_VERSION")
+        if [ "$$VERSION" != 16 ]; then
+          log "the old data directory is Postgres $$VERSION, this step moves Postgres 16 only."
+          log "See docs/ops/postgres-major-upgrade.md to move it by hand."
+          exit 1
+        fi
+        NEED=$$(du -sk "$$OLD" | cut -f1)
+        FREE=$$(df -Pk /var/lib/postgresql/next | awk 'NR == 2 { print $$4 }')
+        if [ "$$FREE" -le "$$NEED" ]; then
+          log "the move needs about $$((NEED / 1024)) MiB free for the new database, the disk has $$((FREE / 1024)) MiB."
+          log "Nothing was changed. Free some space and start the stack again."
+          exit 1
+        fi
+        trap 'CODE=$$?; [ "$$CODE" = 0 ] || log "the move stopped (exit $$CODE). The old volume is untouched; the next start begins it again."' EXIT
+        date -u +%Y-%m-%dT%H:%M:%SZ > "$$MOVING"
+        log "moving the Postgres 16 database ($$((NEED / 1024)) MiB on disk) into Postgres 18"
+        su-exec postgres postgres -D "$$OLD" -c listen_addresses= -c unix_socket_directories=/tmp &
+        PID=$$!
+        i=0
+        until pg_isready -h /tmp -U healthlog -q; do
+          kill -0 "$$PID" 2>/dev/null || { log "the Postgres 16 server did not start, see the lines above"; exit 1; }
+          i=$$((i + 1))
+          [ "$$i" -lt 600 ] || { log "the Postgres 16 server did not get ready in ten minutes"; exit 1; }
+          sleep 1
+        done
+        pg_dump -h /tmp -U healthlog -Fc healthlog |
+          pg_restore -h db -U healthlog -d healthlog --no-owner --single-transaction --exit-on-error
+        COUNTED=0
+        for T in $$(old "SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) $$USER_TABLES ORDER BY 1"); do
+          A=$$(old "SELECT count(*) FROM $$T")
+          B=$$(new "SELECT count(*) FROM $$T")
+          if [ "$$A" != "$$B" ]; then
+            log "table $$T has $$A rows in Postgres 16 but $$B in Postgres 18. Emptying the new database again."
+            log "Please report this with the lines above."
+            empty_new
+            exit 1
+          fi
+          COUNTED=$$((COUNTED + 1))
+        done
+        log "all $$COUNTED tables arrived with the same row counts"
+        vacuumdb -h db -U healthlog -d healthlog --analyze-in-stages --quiet
+        kill -INT "$$PID"
+        wait "$$PID" || true
+        mv "$$MOVING" "$$MARK"
+        log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md"
+
 volumes:
+  postgres-data:
   pgdata:

docs/contributing/benchmarks.md

@@ -14,7 +14,7 @@ Never point the seed or the benchmark at a database that holds real data. They w
 ```sh
 docker run -d --name hl-bench-db --shm-size=256m \
   -e POSTGRES_USER=bench -e POSTGRES_PASSWORD=bench -e POSTGRES_DB=healthlog_bench \
-  -p 55432:5432 postgres:16.15-alpine \
+  -p 55432:5432 postgres:18.6-alpine \
   postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=top
 ```
 

docs/diagrams/01-data-flow.svg

@@ -81,7 +81,7 @@
     <text x="600" y="360" text-anchor="middle" fill="#6272a4" font-size="11">pre-aggregated buckets</text>
 
     <rect x="510" y="392" width="180" height="56" rx="10" fill="#44475a" stroke="#ff79c6" stroke-width="1.5" />
-    <text x="600" y="417" text-anchor="middle" font-weight="600">PostgreSQL 16</text>
+    <text x="600" y="417" text-anchor="middle" font-weight="600">PostgreSQL 18</text>
     <text x="600" y="435" text-anchor="middle" fill="#bd93f9" font-size="11">encrypted at rest</text>
   </g>
 

docs/diagrams/03-self-hosting-topology.svg

@@ -1,6 +1,6 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 620" font-family="ui-sans-serif, -apple-system, Segoe UI, Roboto, sans-serif" role="img" aria-labelledby="title desc">
   <title id="title">HealthLog self-hosting topology</title>
-  <desc id="desc">Internet traffic terminates at a reverse proxy (Caddy, Traefik or Nginx) for TLS, then reaches the Next.js app on port 3000. The app shares a Postgres 16 database with the pg-boss worker process for reminders, insights, backups and the Apple Health import. GHCR ships the multi-arch image; Coolify watches the digest and pulls; off-host S3-compatible storage is optional for backups.</desc>
+  <desc id="desc">Internet traffic terminates at a reverse proxy (Caddy, Traefik or Nginx) for TLS, then reaches the Next.js app on port 3000. The app shares a Postgres 18 database with the pg-boss worker process for reminders, insights, backups and the Apple Health import. GHCR ships the multi-arch image; Coolify watches the digest and pulls; off-host S3-compatible storage is optional for backups.</desc>
 
   <rect x="0" y="0" width="1200" height="620" fill="#282a36" />
 
@@ -47,7 +47,7 @@
   <!-- Postgres (green) -->
   <g>
     <rect x="760" y="250" width="200" height="80" rx="10" fill="#44475a" stroke="#50fa7b" stroke-width="1.5" />
-    <text x="860" y="276" text-anchor="middle" fill="#f8f8f2" font-size="14" font-weight="600">PostgreSQL 16</text>
+    <text x="860" y="276" text-anchor="middle" fill="#f8f8f2" font-size="14" font-weight="600">PostgreSQL 18</text>
     <text x="860" y="296" text-anchor="middle" fill="#bd93f9" font-size="11">26 Prisma models</text>
     <text x="860" y="312" text-anchor="middle" fill="#6272a4" font-size="11">+ pg-boss queues</text>
   </g>

docs/diagrams/README.md

@@ -10,7 +10,7 @@ copy.
 | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `01-data-flow.svg`             | Sources (Withings, Apple Health export.zip, iPhone HealthKit, manual, Nightscout) → ingest endpoints → Measurement + MeasurementRollup → reads (rollup probe / live SQL fallback / cache) → surfaces (dashboard, insights, Coach, doctor PDF).        |
 | `02-coach-pipeline.svg`        | User question → snapshot builder → prompt assembler → provider chain (ChatGPT OAuth / OpenAI / Anthropic / local / admin-shared) → response parser → cited reply with mini-charts.                                                                    |
-| `03-self-hosting-topology.svg` | Internet → reverse proxy (TLS) → Next.js app (port 3000) ↔ pg-boss worker ↔ Postgres 16, plus GHCR image pull, optional Coolify autodeploy, optional S3 off-host backup.                                                                              |
+| `03-self-hosting-topology.svg` | Internet → reverse proxy (TLS) → Next.js app (port 3000) ↔ pg-boss worker ↔ Postgres 18, plus GHCR image pull, optional Coolify autodeploy, optional S3 off-host backup.                                                                              |
 | `04-source-priority.svg`       | Three sources for the same metric on the same day → resolver picks one canonical row per (cumulative vs point) ladder. Losing rows stay in the audit trail.                                                                                           |
 | `05-security-model.svg`        | Three concentric layers — auth perimeter (passkey + password fallback), session perimeter (HttpOnly cookie + Postgres sessions), encrypted core (AES-256-GCM versioned keys). Side rails: rate limiter, audit log, HMAC tokens, CSP/HSTS, SSRF guard. |
 

docs/ops/deploy.md

@@ -129,6 +129,26 @@ An INVALID index whose migration is already recorded applied is not touched at
 boot. The boot names it on stderr with `WARNING — index … is INVALID`; rebuild
 it with `REINDEX INDEX CONCURRENTLY "<index>";` when the database is quiet.
 
+### v1.46.0 notes
+
+- **Postgres 18.** The compose file moves the bundled database from
+  Postgres 16 to 18. The first `docker compose up -d` with the new compose
+  file runs `db-upgrade`, which copies the database into a new
+  `postgres-data` volume, checks every table's row count, and only then lets
+  the app start; the old `pgdata` volume stays as it was, as the way back.
+  Pulling only the new image with the old compose file keeps 16, which this
+  release still runs on. Details, rollback and the steps for a database you
+  run yourself: [postgres-major-upgrade.md](postgres-major-upgrade.md).
+- Migration `0485` drops `preview_features_json` from `users`, the same
+  pattern as `0430` in v1.44.0 below: do not run the previous image beside
+  the new schema. The sequence above stops the old container first; a
+  web/worker split stops both roles before the first new container starts.
+- The other migrations add columns and one index (built concurrently).
+  Three touch rows: `0476` links API-written measurements to their token on
+  accounts that have exactly one measurement token, and `0478`/`0479` give
+  PDFs whose preview or text read failed one more attempt. On a prod-sized
+  copy all of them together finished in under a second.
+
 ### v1.45.0 notes
 
 A single-container install (`HEALTHLOG_PROCESS_TYPE=all`, the shipped

docs/ops/postgres-major-upgrade.md

@@ -1,32 +1,97 @@
 # Moving the database to a newer Postgres major
 
-The compose file ships Postgres 16 and stays there for now, because a major
-version change is the one database update that cannot happen by itself on
-`docker compose pull`. HealthLog runs on Postgres 18 as well: the full
-integration suite runs against 18 (`INTEGRATION_PG_IMAGE=postgres:18-alpine
-pnpm test:integration`), and the TrueNAS app starts new installs on 18.
+From v1.46 the compose file runs Postgres 18. Installs from v1.45 or earlier
+ran 16, and a major version cannot open the data directory of an older one,
+so the compose file carries a one-time step that moves the database across
+on the first start after the update. Nothing needs doing by hand on a stack
+that runs the project's `docker-compose.yml`.
 
-## Why changing the image line is not enough
+Installs that bring their own Postgres (Unraid, an existing database server)
+are not touched: HealthLog runs on 16 and 18 alike, and the move there is
+yours to schedule with the steps under [By hand](#by-hand).
 
-Minor releases (16.14 to 16.15) share the on-disk format. Majors do not, and
-the official image refuses to start a data directory written by an older
-major. From 18 on, the image also keeps its data one level deeper, under
-`/var/lib/postgresql/18/docker`, and expects the volume mounted at
-`/var/lib/postgresql` instead of `/var/lib/postgresql/data`. With the old
-volume and only the image line changed, the `db` container exits at once with
-"there appears to be PostgreSQL data in: /var/lib/postgresql/data". Nothing is
-lost, but the stack stays down until the image line goes back to 16.
+## What happens on the first start
 
-So the move is a dump from 16 and a restore into a fresh 18 volume.
+`db-upgrade` is a short-lived container next to `db`. It starts after the new
+Postgres 18 database is up and before the app, and it exits in a second when
+there is nothing to move. When the old `pgdata` volume holds a Postgres 16
+database and the new one is empty, it:
 
-## Steps
+1. checks there is room on the disk for a second copy;
+2. starts Postgres 16 inside its own container on the old volume, reachable
+   only from inside that container;
+3. streams a dump into Postgres 18 in a single transaction;
+4. compares the row count of every table between the two;
+5. gathers the planner statistics, which a restore does not carry;
+6. stops Postgres 16 and leaves a marker file on the old volume.
 
-Take the encryption key's copy out of its safe place first: the dump is
-unreadable without it, and this is the moment you would find out.
+The app starts once that succeeded, applies the release's migrations and
+serves as before. On a typical personal record the whole move takes seconds;
+a database of a few gigabytes takes a few minutes. Watch it with
+`docker compose logs -f db-upgrade`.
+
+If any step fails, the new database stays empty, the app does not start, and
+the log says what happened. A move cut short by a restart or a full disk is
+run again from scratch on the next `docker compose up -d`.
+
+The old `pgdata` volume is read and never changed, apart from the marker
+file. That is the way back.
+
+## Going back
+
+Stop the stack, put the two lines of the `db` service back the way v1.45 had
+them, and start it with the v1.45 image:
+
+```yaml
+db:
+  image: postgres:16.15-alpine
+  volumes:
+    - pgdata:/var/lib/postgresql/data
+```
 
 ```bash
-# 1. Stop the app so nothing writes while the dump runs. If you run the
-#    worker role in its own container, stop it too: docker compose stop app app-worker
+docker compose down
+HEALTHLOG_IMAGE_REF=:v1.45.0 docker compose up -d
+```
+
+Anything written after the move lives only in the Postgres 18 volume.
+
+## Afterwards
+
+Keep the old volume until the new database has run for a while. When you are
+sure, remove it (`docker volume ls` shows its full name, prefixed with the
+compose project):
+
+```bash
+docker volume rm <project>_pgdata
+```
+
+`db-upgrade` then finds nothing to move. A later release drops the service
+and the `pgdata` line.
+
+## Starting over
+
+When the move has run and the Postgres 18 volume is empty again (deleted by
+hand, say), `db-upgrade` refuses to import the old state a second time, since
+that would quietly bring back the database as it was on the day of the move.
+To do exactly that on purpose, remove the marker and start again:
+
+```bash
+docker compose run --rm --entrypoint rm db-upgrade \
+  /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18
+docker compose up -d
+```
+
+## By hand
+
+For a Postgres you run yourself, or a compose file you have changed so much
+that the step does not fit, the move is a dump and a restore. Take the
+encryption key's copy out of its safe place first: the dump is unreadable
+without it, and this is the moment you would find out.
+
+```bash
+# 1. Stop the app so nothing writes while the dump runs (and app-worker, if
+#    you run the worker role in its own container).
 docker compose stop app
 
 # 2. Dump the database in Postgres' custom format.
@@ -37,56 +102,31 @@ docker compose exec -T db pg_dump -U healthlog -Fc healthlog \
 docker compose stop db
 ```
 
-4. In `docker-compose.yml`, change the `db` service:
-
-   ```yaml
-   image: postgres:18.6-alpine # or the current 18 minor
-   volumes:
-     - pgdata18:/var/lib/postgresql
-   ```
-
-   and add the new volume next to `pgdata` in the top-level `volumes:` block:
-
-   ```yaml
-   volumes:
-     pgdata:
-     pgdata18:
-   ```
+4. Point the `db` service at Postgres 18 and a fresh volume. From 18 on, the
+   image keeps its data under `/var/lib/postgresql/18/docker` and expects the
+   volume mounted one level up, at `/var/lib/postgresql`; mounted at
+   `.../data` it refuses to start with "there appears to be PostgreSQL data
+   in: /var/lib/postgresql/data".
 
 ```bash
-# 5. Start the new database and wait until it reports healthy.
+# 5. Start the new database and restore the dump into it.
 docker compose up -d db
-docker compose ps db
-
-# 6. Restore the dump into it.
 docker compose exec -T db pg_restore -U healthlog -d healthlog --no-owner \
   < healthlog-pg16-<date>.dump
 
-# 7. Gather the planner statistics. A restore brings the rows but not the
+# 6. Gather the planner statistics. A restore brings the rows but not the
 #    statistics, and until autovacuum gets round to every table the planner
 #    guesses at their sizes, which can turn a sub-second page into one that
 #    takes many seconds.
 docker compose exec -T db vacuumdb -U healthlog -d healthlog --analyze-in-stages
 
-# 8. Start the app (and app-worker, if you run it) and confirm it.
+# 7. Start the app and confirm it.
 docker compose up -d app
 curl -s http://localhost:3000/api/version
 ```
 
-The start log should say there are no pending migrations: the dump carried the
-migration history with it. Sign in and open a few pages before you go on.
-
-## Afterwards
-
-Keep the old `pgdata` volume and the dump until the new database has run for a
-few days. Going back means restoring the old `image:` and `volumes:` lines;
-the old volume still holds the database as it was before step 2. When you are
-sure, remove it (`docker volume ls` shows its full name, prefixed with the
-compose project):
-
-```bash
-docker volume rm <project>_pgdata
-```
+The start log should say there are no pending migrations beyond the release's
+own: the dump carried the migration history with it.
 
 If you had run `CREATE EXTENSION pg_stat_statements;` for
 [query statistics](query-statistics.md), the dump carries it and the compose

docs/self-hosting/getting-started.md

@@ -81,7 +81,9 @@ explicitly in `.env` (see the commented example there).
 
 ### Using an existing or shared Postgres
 
-HealthLog can live in a database on a Postgres instance you already run.
+HealthLog can live in a database on a Postgres instance you already run,
+version 16 or newer; the compose file ships 18, and every release is tested
+on both.
 It stores every time as UTC and pins each connection it opens to UTC
 (`TimeZone=UTC`, sent with the libpq `options` startup parameter), so the
 server's or the database's own `timezone` setting does not matter: an
@@ -105,9 +107,12 @@ ALTER DATABASE healthlog SET timezone TO 'UTC';
 docker compose up -d
 ```
 
-Compose starts two services:
+Compose starts two services and one short-lived step:
 
-- `db` — PostgreSQL 16 with a named volume for persistence.
+- `db` — PostgreSQL 18 with a named volume for persistence.
+- `db-upgrade` — moves a Postgres 16 database from v1.45 or earlier into
+  the new one, once, and exits; on a new install it has nothing to do. See
+  [Moving to a newer Postgres major](../ops/postgres-major-upgrade.md).
 - `app` — HealthLog itself, listening on `localhost:3000`. The image
   runs `prisma migrate deploy` on boot so the schema lands
   automatically; subsequent restarts no-op when the database is

docs/self-hosting/portainer/README.md

@@ -1,7 +1,7 @@
 # HealthLog on Portainer
 
 HealthLog ships a Portainer **app template** that deploys the app and its
-PostgreSQL 16 database as one stack from the project `docker-compose.yml`.
+PostgreSQL 18 database as one stack from the project `docker-compose.yml`.
 
 ## Add the template
 
@@ -39,7 +39,7 @@ curl -s http://<host>:3000/api/version
 
 ## Notes
 
-- Persistent state lives in one named volume (`pgdata`, the Postgres data dir);
+- Persistent state lives in one named volume (`postgres-data`, the Postgres data);
   the app container is stateless. Back up the database to back up everything.
 - Pin a release tag (`HEALTHLOG_IMAGE_REF=:vX.Y.Z`) for deliberate upgrades, or
   track `:latest`. Skim the

docs/self-hosting/scaling.md

@@ -174,7 +174,7 @@ sized for the web traffic alone leaves the worker's jobs queueing for
 the same connections.
 
 Plan total Postgres slots as **container_count × budget**.
-A stock Postgres 16 container ships with `max_connections = 100`, so
+A stock Postgres container ships with `max_connections = 100`, so
 the safe envelope is:
 
 | Web replicas | Worker replicas | Total containers | Pool slots used | Headroom under 100 |

docs/self-hosting/truenas.md

@@ -213,7 +213,7 @@ the menu next to Custom App, Install via YAML, using these values:
 | User         | runs as uid `1001` by default; no extra capabilities, `no-new-privileges` is fine        |
 | Health check | `GET /api/health`, `200` when ready                                                      |
 | Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot |
-| Database     | PostgreSQL 16 or 18, no extensions, in its own container with its own dataset            |
+| Database     | PostgreSQL 18 (16 works too), no extensions, in its own container with its own dataset   |
 | Volumes      | none for the app; the database gets its own dataset                                      |
 
 | Variable                | Required | Notes                                                                                    |

docs/self-hosting/unraid.md

@@ -69,10 +69,12 @@ curl -s http://tower:3000/api/version
 Use this if you prefer the per-app Unraid UI. You install PostgreSQL yourself,
 then add HealthLog pointed at it.
 
-### Step 1 — install PostgreSQL 16
+### Step 1 — install PostgreSQL
 
 1. In **Apps** (Community Applications), search for a **PostgreSQL** container
-   (for example jj9987's `postgresql`). Install version 16.
+   (for example jj9987's `postgresql`). Install version 18; an existing 16
+   works too. Moving a 16 to 18 later is a dump and restore, see
+   [Moving to a newer Postgres major](../ops/postgres-major-upgrade.md#by-hand).
 2. Set:
    - `POSTGRES_USER` = `healthlog`
    - `POSTGRES_PASSWORD` = your generated password

docs/self-hosting/unraid/healthlog.xml

@@ -2,7 +2,7 @@
 <!--
   HealthLog — Unraid Community Applications template.
 
-  CA templates describe ONE container. HealthLog needs PostgreSQL 16, so
+  CA templates describe ONE container. HealthLog needs PostgreSQL (18, or 16), so
   install a PostgreSQL container FIRST (CA has community PostgreSQL templates),
   create a `healthlog` database + user, then point DATABASE_URL below at it.
   Full walkthrough: docs/self-hosting/unraid.md.
@@ -26,7 +26,7 @@
     cycle, and medications on one encrypted timeline. AES-256-GCM at rest, no
     vendor cloud, no telemetry.
 
-    REQUIRES a separate PostgreSQL 16 container — install one first (CA has
+    REQUIRES a separate PostgreSQL 18 (or 16) container — install one first (CA has
     community PostgreSQL templates), create a database and user, then set
     DATABASE_URL below. Step-by-step guide:
     https://github.com/MBombeck/HealthLog/blob/main/docs/self-hosting/unraid.md

prisma/README.md

@@ -1,6 +1,6 @@
 # `prisma/`
 
-The database layer. HealthLog runs PostgreSQL 16 via Prisma 7. This directory holds the single schema file and the forward-only migration history; the generated client is emitted elsewhere.
+The database layer. HealthLog runs on PostgreSQL 18 (the compose default) or 16 via Prisma 7. This directory holds the single schema file and the forward-only migration history; the generated client is emitted elsewhere.
 
 ## Contents
 

scripts/compose-db-upgrade-check.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# Plays the Postgres 16 -> 18 move in docker-compose.yml end to end, against
+# real containers, the way an install from v1.45 meets it on its first update.
+#
+#   1. Start the database exactly as the v1.45 compose file had it, apply every
+#      migration and add rows in a second schema.
+#   2. Start the current compose file's `db-upgrade` and check the move: exit 0,
+#      Postgres 18, the same rows and the same migration history.
+#   3. Start it again: nothing to do.
+#   4. Pretend the move was cut short after the restore: the next start must
+#      empty the new database and run the move again.
+#   5. Empty the new volume: it must refuse to import the old state again,
+#      and move it once more after the marker is removed as the guide says.
+#   6. A new install with no old volume: nothing to do.
+#
+# Needs Docker and the repo's node_modules (for the migrations). Run it from
+# anywhere in the repo: `bash scripts/compose-db-upgrade-check.sh`.
+set -euo pipefail
+
+ROOT=$(git rev-parse --show-toplevel)
+# The last release whose compose file ran Postgres 16.
+OLD_REF=${OLD_REF:-v1.45.0}
+PROJECT=hl-db-upgrade-check
+PORT=${CHECK_DB_PORT:-55416}
+WORK=$(mktemp -d)
+
+export PRISMA_HIDE_UPDATE_MESSAGE=1
+export POSTGRES_PASSWORD
+POSTGRES_PASSWORD=$(openssl rand -hex 16)
+# The app service is never started here, but compose reads its environment.
+export ENCRYPTION_KEY API_TOKEN_HMAC_KEY
+ENCRYPTION_KEY=$(openssl rand -hex 32)
+API_TOKEN_HMAC_KEY=$(openssl rand -hex 32)
+
+git -C "$ROOT" show "$OLD_REF:docker-compose.yml" >"$WORK/old.yml"
+cat >"$WORK/old-port.yml" <<EOF
+services:
+  db:
+    ports:
+      - "127.0.0.1:$PORT:5432"
+EOF
+
+old() { docker compose -p "$PROJECT" -f "$WORK/old.yml" -f "$WORK/old-port.yml" "$@"; }
+new() { docker compose -p "$PROJECT" -f "$ROOT/docker-compose.yml" "$@"; }
+down_all() {
+  new down -v --remove-orphans >/dev/null 2>&1 || true
+  old down -v --remove-orphans >/dev/null 2>&1 || true
+}
+trap 'down_all; rm -rf "$WORK"' EXIT
+
+fail() {
+  echo "FAIL: $*" >&2
+  docker logs healthlog-db-upgrade 2>&1 | tail -40 >&2 || true
+  exit 1
+}
+step() { echo "== $*"; }
+sql() { docker exec healthlog-db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$1"; }
+# Starts db-upgrade (and the database it waits for), waits for it to exit and
+# prints its exit code.
+run_upgrade() {
+  new up -d db-upgrade >/dev/null 2>&1 || true
+  docker wait healthlog-db-upgrade
+}
+upgrade_log() { docker logs healthlog-db-upgrade 2>&1 | grep '^db-upgrade:' || true; }
+
+down_all
+step "a v1.45 database on Postgres 16"
+old up -d --wait db >/dev/null 2>&1
+DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
+  node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >/dev/null
+old exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -q <<'SQL'
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+CREATE SCHEMA check_probe;
+CREATE TABLE check_probe.rows AS
+  SELECT g AS id, md5(g::text) AS value FROM generate_series(1, 50000) AS g;
+SQL
+probe() {
+  "$@" psql -U healthlog -d healthlog -Atqc \
+    "SELECT md5(string_agg(value, ',' ORDER BY id)) FROM check_probe.rows"
+}
+migrations() {
+  "$@" psql -U healthlog -d healthlog -Atqc "SELECT count(*) FROM _prisma_migrations"
+}
+PROBE=$(probe old exec -T db)
+MIGRATIONS=$(migrations old exec -T db)
+[ "$MIGRATIONS" -gt 0 ] || fail "the migrations did not apply to the old database"
+old stop db >/dev/null 2>&1
+old rm -f db >/dev/null 2>&1
+
+step "first start of the current compose file moves it"
+CODE=$(run_upgrade)
+upgrade_log
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE on the move"
+upgrade_log | grep -q 'tables arrived with the same row counts' || fail "the move did not report its row-count check"
+VERSION=$(sql "SHOW server_version_num")
+[ "$VERSION" -ge 180000 ] || fail "the database runs $VERSION, expected Postgres 18"
+[ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the probe rows differ after the move"
+[ "$(migrations docker exec healthlog-db)" = "$MIGRATIONS" ] || fail "the migration history differs after the move"
+[ "$(sql "SELECT count(*) FROM pg_stats WHERE schemaname = 'check_probe'")" -gt 0 ] ||
+  fail "the moved tables have no planner statistics"
+
+step "second start: nothing to do"
+CODE=$(run_upgrade)
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE on a start after the move"
+upgrade_log | tail -1 | grep -q 'nothing to do' || fail "the second start did not say there was nothing to do"
+
+step "a move cut short after the restore runs again from scratch"
+new down >/dev/null 2>&1
+docker run --rm -v "${PROJECT}_pgdata:/old" postgres:16.15-alpine \
+  mv /old/HEALTHLOG_MOVED_TO_POSTGRES_18 /old/HEALTHLOG_MOVING_TO_POSTGRES_18
+CODE=$(run_upgrade)
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE when resuming a cut-short move"
+upgrade_log | grep -q 'did not finish' || fail "the resumed move did not say it starts again"
+[ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the probe rows differ after the resumed move"
+
+step "an emptied Postgres 18 volume is not refilled from the old state"
+new down >/dev/null 2>&1
+docker volume rm "${PROJECT}_postgres-data" >/dev/null
+CODE=$(run_upgrade)
+[ "$CODE" = 1 ] || fail "db-upgrade exited $CODE with the moved marker and an empty database, expected 1"
+upgrade_log | grep -q 'Not importing the old state again' || fail "the refusal did not explain itself"
+
+step "starting over as the guide says brings the moved state back"
+new run --rm --entrypoint rm db-upgrade /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18 >/dev/null 2>&1
+CODE=$(run_upgrade)
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE after the marker was removed"
+[ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the probe rows differ after starting over"
+
+step "a new install: nothing to do"
+new down -v >/dev/null 2>&1
+CODE=$(run_upgrade)
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE on a new install"
+upgrade_log | tail -1 | grep -q 'no Postgres 16 database to move' || fail "the new install did not say there was nothing to move"
+
+echo "OK: the Postgres 16 -> 18 move behaves in all six cases"

src/__tests__/compose-db-upgrade-guard.test.ts

@@ -0,0 +1,87 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { describe, expect, it } from "vitest";
+import { parse } from "yaml";
+
+/**
+ * The wiring around `db-upgrade` in docker-compose.yml, the one-time move of a
+ * Postgres 16 database from v1.45 or earlier into Postgres 18.
+ *
+ * What the move does is proven against real containers by
+ * `scripts/compose-db-upgrade-check.sh` (the "Compose database move"
+ * workflow). That check starts `db-upgrade` by name, so it cannot see the
+ * part that protects an operator's data on a normal `docker compose up`: the
+ * app must wait for the move to succeed, or it starts on an empty Postgres 18
+ * database and the instance looks wiped. This file freezes that and the
+ * volume layout the move depends on.
+ */
+
+type Service = {
+  image?: string;
+  restart?: string;
+  volumes?: string[];
+  depends_on?: Record<string, { condition?: string }>;
+  command?: string[];
+};
+
+const compose = parse(
+  readFileSync(resolve(__dirname, "../../docker-compose.yml"), "utf8"),
+) as { services: Record<string, Service>; volumes: Record<string, unknown> };
+const { app, db } = compose.services;
+const upgrade = compose.services["db-upgrade"];
+
+function major(image: string | undefined): number {
+  const match = image?.match(/^postgres:(\d+)\./);
+  if (!match) throw new Error(`not a pinned postgres image: ${image}`);
+  return Number(match[1]);
+}
+
+describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
+  it("starts the app only after the move succeeded", () => {
+    expect(app.depends_on?.["db-upgrade"]?.condition).toBe(
+      "service_completed_successfully",
+    );
+    expect(app.depends_on?.db?.condition).toBe("service_healthy");
+  });
+
+  it("moves into a running database", () => {
+    expect(upgrade.depends_on?.db?.condition).toBe("service_healthy");
+  });
+
+  it("reads the old data with the major that wrote it", () => {
+    expect(major(upgrade.image)).toBe(16);
+    expect(major(db.image)).toBe(18);
+    expect(upgrade.volumes).toContain("pgdata:/var/lib/postgresql/data");
+  });
+
+  it("keeps the new data where the Postgres 18 image expects it", () => {
+    // Mounted at .../data, as 16 had it, the 18 image refuses to start.
+    expect(db.volumes).toEqual(["postgres-data:/var/lib/postgresql"]);
+    expect(Object.keys(compose.volumes).sort()).toEqual([
+      "pgdata",
+      "postgres-data",
+    ]);
+  });
+
+  it("runs once per start instead of restarting", () => {
+    // A restart policy would rerun a failed move in a loop; "no" also keeps
+    // Coolify from counting the exited container as a failed service.
+    expect(upgrade.restart).toBe("no");
+  });
+
+  it("never fills a database that already holds data", () => {
+    const script = upgrade.command?.join("\n") ?? "";
+    // The checks run before the restore, in this order.
+    const order = [
+      "PG_VERSION",
+      "$$MARK",
+      "$$MOVING",
+      'elif [ "$$TABLES" != 0 ]',
+      "pg_restore",
+    ].map((needle) => script.indexOf(needle));
+    expect(order.every((index) => index >= 0)).toBe(true);
+    expect([...order].sort((a, b) => a - b)).toEqual(order);
+    expect(script).toContain("--single-transaction");
+  });
+});

src/__tests__/database-pin-is-watched.test.ts

@@ -31,9 +31,13 @@ describe("the bundled database pin", () => {
       ...read("docker-compose.yml").matchAll(/^\s*image:\s*(postgres:\S+)/gm),
     ].map((match) => match[1]);
 
-    expect(images.length).toBe(1);
-    // `postgres:16-alpine` or `postgres:16` would match a moving target.
-    expect(images[0]).toMatch(/^postgres:\d+\.\d+-alpine$/);
+    // The database itself and `db-upgrade`, which reads the Postgres 16 data
+    // of an install from v1.45 or earlier.
+    expect(images.length).toBe(2);
+    for (const image of images) {
+      // `postgres:18-alpine` or `postgres:18` would match a moving target.
+      expect(image).toMatch(/^postgres:\d+\.\d+-alpine$/);
+    }
   });
 
   it("lets a major version through only as a decision, never as an update PR", () => {

tests/integration/global-setup.ts

@@ -44,10 +44,11 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const PROJECT_ROOT = resolve(__dirname, "..", "..");
 
-// The compose file ships Postgres 16; INTEGRATION_PG_IMAGE runs the same
-// suite against another major (the TrueNAS app defaults to 18).
+// The compose file ships Postgres 18; INTEGRATION_PG_IMAGE runs the same
+// suite against another major. The nightly run uses it for Postgres 16, the
+// oldest major a self-host on its own database may still run.
 const POSTGRES_IMAGE =
-  process.env.INTEGRATION_PG_IMAGE?.trim() || "postgres:16-alpine";
+  process.env.INTEGRATION_PG_IMAGE?.trim() || "postgres:18-alpine";
 
 let started: StartedPostgreSqlContainer | null = null;