diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2cb857975..985a0c309 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -79,6 +79,8 @@ updates: # The first run of this ecosystem proposed 16.15 -> 18.6, which would # have handed self-hosters a database their data directory cannot be # read by. Minor releases still arrive as PRs; a major is a deliberate - # project decision with a migration guide behind it. + # project decision with a migration guide behind it. The ignore also + # keeps `db-upgrade` on 16: it exists to read Postgres 16 data, and a + # bump to 18 would leave it unable to start on the volume it moves. - dependency-name: "postgres" update-types: ["version-update:semver-major"] diff --git a/.github/workflows/compose-db-upgrade.yml b/.github/workflows/compose-db-upgrade.yml new file mode 100644 index 000000000..edc86e0cb --- /dev/null +++ b/.github/workflows/compose-db-upgrade.yml @@ -0,0 +1,51 @@ +name: Compose database move + +# Plays the Postgres 16 -> 18 move in docker-compose.yml against real +# containers: a v1.45 database with every migration applied, then the current +# compose file's `db-upgrade` on its first start, a second start, a move cut +# short, an emptied new volume and a new install. The service runs once on an +# operator's host and has no second chance there, so every change to the +# compose file or to the check runs it here first. +# `scripts/compose-db-upgrade-check.sh` is the same check, runnable locally. + +on: + pull_request: + branches: [main] + paths: + - docker-compose.yml + - scripts/compose-db-upgrade-check.sh + - .github/workflows/compose-db-upgrade.yml + push: + branches: [main] + paths: + - docker-compose.yml + - scripts/compose-db-upgrade-check.sh + - .github/workflows/compose-db-upgrade.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + move: + name: Postgres 16 -> 18 move + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The check reads the v1.45 compose file from its tag. + - name: Fetch the v1.45.0 tag + run: git fetch --no-tags --depth=1 origin tag v1.45.0 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Move check + run: bash scripts/compose-db-upgrade-check.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 165aca665..0640e5420 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -40,7 +40,7 @@ jobs: name: e2e shard ${{ matrix.shard }}/3 services: postgres: - image: postgres:16-alpine + image: postgres:18-alpine env: POSTGRES_USER: healthlog POSTGRES_PASSWORD: healthlog @@ -158,7 +158,7 @@ jobs: image: mcr.microsoft.com/playwright:v${{ needs.playwright-version.outputs.version }}-noble services: postgres: - image: postgres:16-alpine + image: postgres:18-alpine env: POSTGRES_USER: healthlog POSTGRES_PASSWORD: healthlog diff --git a/.github/workflows/post-publish-verify.yml b/.github/workflows/post-publish-verify.yml index 4f4bdf70f..d22ef58f0 100644 --- a/.github/workflows/post-publish-verify.yml +++ b/.github/workflows/post-publish-verify.yml @@ -56,7 +56,7 @@ jobs: services: postgres: - image: postgres:16-alpine + image: postgres:18-alpine env: POSTGRES_USER: healthlog POSTGRES_PASSWORD: healthlog diff --git a/.github/workflows/test-nightly.yml b/.github/workflows/test-nightly.yml index 88c1e0a27..eddede148 100644 --- a/.github/workflows/test-nightly.yml +++ b/.github/workflows/test-nightly.yml @@ -84,7 +84,7 @@ jobs: fail-fast: false matrix: shard: [1, 2, 3, 4] - name: integration shard ${{ matrix.shard }}/4 (shuffled) + name: integration shard ${{ matrix.shard }}/4 (shuffled, Postgres 16) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -110,6 +110,10 @@ jobs: shell: bash env: HEALTHLOG_INTEGRATION_WORKERS: "3" + # Pull requests run the suite on Postgres 18, the compose default. + # Installs that bring their own database may still be on 16, so the + # nightly run keeps that major covered without doubling every PR. + INTEGRATION_PG_IMAGE: "postgres:16-alpine" # Four times each property's own case count; a property's timeout # grows by the same factor (`propertyTimeout`). FC_RUNS_FACTOR: "4" diff --git a/README.md b/README.md index 214f00045..854eb8cc3 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ curl -s http://localhost:3000/api/version # returns version + buildSha + built `SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/self-hosting/)). -Persistent state lives in exactly one named volume (`pgdata`, the Postgres data dir); the app container is stateless, so backing up the database backs up everything. +Persistent state lives in exactly one named volume (`postgres-data`, the Postgres data); the app container is stateless, so backing up the database backs up everything. An install from v1.45 or earlier moves its Postgres 16 database into it on the first start ([details](docs/ops/postgres-major-upgrade.md)). NAS and homelab users have ready-made paths: [Unraid Community Applications](docs/self-hosting/unraid.md), [Portainer app templates](docs/self-hosting/portainer/README.md) and [TrueNAS](docs/self-hosting/truenas.md) (the Community train app is pending catalog review). @@ -105,7 +105,7 @@ The App Store listing carries a small price, which exists to cover Apple's yearl ## Tech stack -Next.js 16 (App Router, React Server Components), TypeScript strict, PostgreSQL 16 with Prisma, pg-boss for jobs, Tailwind CSS 4 + shadcn/ui, Recharts, WebAuthn passkeys, Vitest + Playwright, multi-stage Alpine Docker image. The OpenAPI 3.1 contract for native clients is generated from the Zod schemas at [`docs/api/openapi.yaml`](docs/api/openapi.yaml). +Next.js 16 (App Router, React Server Components), TypeScript strict, PostgreSQL 18 with Prisma, pg-boss for jobs, Tailwind CSS 4 + shadcn/ui, Recharts, WebAuthn passkeys, Vitest + Playwright, multi-stage Alpine Docker image. The OpenAPI 3.1 contract for native clients is generated from the Zod schemas at [`docs/api/openapi.yaml`](docs/api/openapi.yaml). ## Documentation diff --git a/docker-compose.yml b/docker-compose.yml index 01d9e0cd4..cdfbfeea6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -# HealthLog — two-container stack (app + Postgres 16). +# HealthLog — two-container stack (app + Postgres 18). # # First run (clean host): # cp .env.example .env @@ -14,9 +14,11 @@ # and login silently fails. This is the single most common self-host snag. # Leave it unset (or true) only when a TLS reverse proxy serves HTTPS. # -# Persistent state lives in exactly one named volume (`pgdata`, Postgres' -# data dir). The app container is stateless, so a backup of the database is -# a backup of everything. The optional GeoLite2 databases are either baked +# Persistent state lives in exactly one named volume (`postgres-data`, +# Postgres' data). The app container is stateless, so a backup of the database +# is a backup of everything. `pgdata` is where Postgres 16 kept the data up to +# v1.45; on an install that has it, `db-upgrade` moves it into Postgres 18 once +# and then leaves it alone as the way back (docs/ops/postgres-major-upgrade.md). The optional GeoLite2 databases are either baked # into the image at build time or mounted read-only from the host and # pointed at with GEOLITE2_DIR — either way they are replaceable, not state. @@ -489,6 +491,8 @@ services: depends_on: db: condition: service_healthy + db-upgrade: + condition: service_completed_successfully restart: unless-stopped # Bounded log rotation: the default json-file driver keeps every line # forever, which is exactly what fills the disk on the small NAS / @@ -562,6 +566,8 @@ services: # depends_on: # db: # condition: service_healthy + # db-upgrade: + # condition: service_completed_successfully # restart: unless-stopped # logging: # driver: json-file @@ -572,13 +578,11 @@ services: # stop_grace_period: 45s db: - # Pinned to the concrete minor instead of the floating `16-alpine` + # Pinned to the concrete minor instead of the floating `18-alpine` # alias so a database update is always a deliberate, reviewable - # change (at pin time `16-alpine` resolved to this same digest, so - # existing deployments see no image change). Minor releases share - # the on-disk format — bumping this pin is safe without a dump / - # restore; a major-version bump (17+) is not, see - # docs/ops/postgres-major-upgrade.md. + # change. Minor releases share the on-disk format, so bumping this pin + # is safe without a dump / restore; a major version is not, which is + # what `db-upgrade` below and docs/ops/postgres-major-upgrade.md are for. # # The pin only works as long as something watches it: PostgreSQL # publishes minor releases with security and data-corruption fixes and @@ -588,7 +592,7 @@ services: # `.github/dependabot.yml` now raises a PR for each new minor; # `src/__tests__/database-pin-is-watched.test.ts` keeps both halves # (concrete pin, active watch) from being dropped. - image: postgres:16.15-alpine + image: postgres:18.6-alpine container_name: healthlog-db # Loads pg_stat_statements so slow queries can be found from measurements # instead of guesses. Loading the library only collects statistics; the @@ -618,8 +622,12 @@ services: POSTGRES_USER: healthlog POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" POSTGRES_DB: healthlog + # From 18 on, the image keeps its data under /var/lib/postgresql/18/docker + # and expects the volume one level up, so the next major can sit beside + # this one in the same volume. Mounting it at .../data, as 16 had it, + # makes the container refuse to start. volumes: - - pgdata:/var/lib/postgresql/data + - postgres-data:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U healthlog"] interval: 5s @@ -640,5 +648,134 @@ services: max-size: "10m" max-file: "3" + # Moves a Postgres 16 database from v1.45 or earlier into Postgres 18, once, + # on the first start after the update. A major version cannot read the data + # directory of an older one, so changing the image line alone would leave + # the database refusing to start. + # + # On every start it decides in a second and exits: + # - no `pgdata` data (a new install): nothing to do; + # - Postgres 18 already holds the database: nothing to do; + # - otherwise it starts Postgres 16 inside this container on the old + # volume, streams a dump into the new database in one transaction, + # compares the row count of every table, gathers planner statistics and + # marks the old volume as moved. `app` starts only after it succeeded. + # A move cut short (a restart, a full disk) leaves a "moving" marker, and + # the next start empties the new database and runs it again from scratch. + # + # The old volume is read, never changed, apart from that marker file, so + # going back is the previous image and volume lines. If anything fails, the + # new database stays empty, `app` does not start, and the log says why. + # It costs one small image pull; a later release removes the service. + db-upgrade: + image: postgres:16.15-alpine + container_name: healthlog-db-upgrade + # Runs once per `compose up` and exits. "no" also keeps hosts such as + # Coolify from reporting the exited container as a failed service. + restart: "no" + security_opt: + - no-new-privileges:true + environment: + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + volumes: + - pgdata:/var/lib/postgresql/data + # Read-only, only to check there is room for the copy before starting. + - postgres-data:/var/lib/postgresql/next:ro + depends_on: + db: + condition: service_healthy + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + # `$$` is how compose spells a literal `$`. + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + set -o pipefail + log() { echo "db-upgrade: $$*"; } + OLD=/var/lib/postgresql/data + MARK="$$OLD/HEALTHLOG_MOVED_TO_POSTGRES_18" + MOVING="$$OLD/HEALTHLOG_MOVING_TO_POSTGRES_18" + export PGPASSWORD="$$POSTGRES_PASSWORD" + new() { psql -h db -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$$1"; } + old() { psql -h /tmp -U healthlog -d healthlog -v ON_ERROR_STOP=1 -Atqc "$$1"; } + empty_new() { + for S in $$(new "SELECT quote_ident(nspname) FROM pg_namespace WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'"); do + new "DROP SCHEMA $$S CASCADE" + done + new "CREATE SCHEMA public" + } + if [ ! -s "$$OLD/PG_VERSION" ]; then + log "no Postgres 16 database to move, nothing to do" + exit 0 + fi + USER_TABLES="FROM pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema')" + TABLES=$$(new "SELECT count(*) $$USER_TABLES") + if [ -e "$$MARK" ]; then + if [ "$$TABLES" != 0 ]; then + log "the Postgres 16 database was moved on $$(cat "$$MARK"), nothing to do" + exit 0 + fi + log "the Postgres 16 database was moved on $$(cat "$$MARK"), but the Postgres 18 database is empty now." + log "Not importing the old state again by itself. See docs/ops/postgres-major-upgrade.md, \"Starting over\"." + exit 1 + fi + if [ -e "$$MOVING" ]; then + log "an earlier move did not finish; emptying the Postgres 18 database and starting it again" + empty_new + elif [ "$$TABLES" != 0 ]; then + log "the Postgres 18 database already holds data, leaving both as they are" + exit 0 + fi + VERSION=$$(cat "$$OLD/PG_VERSION") + if [ "$$VERSION" != 16 ]; then + log "the old data directory is Postgres $$VERSION, this step moves Postgres 16 only." + log "See docs/ops/postgres-major-upgrade.md to move it by hand." + exit 1 + fi + NEED=$$(du -sk "$$OLD" | cut -f1) + FREE=$$(df -Pk /var/lib/postgresql/next | awk 'NR == 2 { print $$4 }') + if [ "$$FREE" -le "$$NEED" ]; then + log "the move needs about $$((NEED / 1024)) MiB free for the new database, the disk has $$((FREE / 1024)) MiB." + log "Nothing was changed. Free some space and start the stack again." + exit 1 + fi + trap 'CODE=$$?; [ "$$CODE" = 0 ] || log "the move stopped (exit $$CODE). The old volume is untouched; the next start begins it again."' EXIT + date -u +%Y-%m-%dT%H:%M:%SZ > "$$MOVING" + log "moving the Postgres 16 database ($$((NEED / 1024)) MiB on disk) into Postgres 18" + su-exec postgres postgres -D "$$OLD" -c listen_addresses= -c unix_socket_directories=/tmp & + PID=$$! + i=0 + until pg_isready -h /tmp -U healthlog -q; do + kill -0 "$$PID" 2>/dev/null || { log "the Postgres 16 server did not start, see the lines above"; exit 1; } + i=$$((i + 1)) + [ "$$i" -lt 600 ] || { log "the Postgres 16 server did not get ready in ten minutes"; exit 1; } + sleep 1 + done + pg_dump -h /tmp -U healthlog -Fc healthlog | + pg_restore -h db -U healthlog -d healthlog --no-owner --single-transaction --exit-on-error + COUNTED=0 + for T in $$(old "SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) $$USER_TABLES ORDER BY 1"); do + A=$$(old "SELECT count(*) FROM $$T") + B=$$(new "SELECT count(*) FROM $$T") + if [ "$$A" != "$$B" ]; then + log "table $$T has $$A rows in Postgres 16 but $$B in Postgres 18. Emptying the new database again." + log "Please report this with the lines above." + empty_new + exit 1 + fi + COUNTED=$$((COUNTED + 1)) + done + log "all $$COUNTED tables arrived with the same row counts" + vacuumdb -h db -U healthlog -d healthlog --analyze-in-stages --quiet + kill -INT "$$PID" + wait "$$PID" || true + mv "$$MOVING" "$$MARK" + log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md" + volumes: + postgres-data: pgdata: diff --git a/docs/contributing/benchmarks.md b/docs/contributing/benchmarks.md index aa10a1d73..9ceb4b7a0 100644 --- a/docs/contributing/benchmarks.md +++ b/docs/contributing/benchmarks.md @@ -14,7 +14,7 @@ Never point the seed or the benchmark at a database that holds real data. They w ```sh docker run -d --name hl-bench-db --shm-size=256m \ -e POSTGRES_USER=bench -e POSTGRES_PASSWORD=bench -e POSTGRES_DB=healthlog_bench \ - -p 55432:5432 postgres:16.15-alpine \ + -p 55432:5432 postgres:18.6-alpine \ postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=top ``` diff --git a/docs/diagrams/01-data-flow.svg b/docs/diagrams/01-data-flow.svg index 9bb7a272b..2a4b04956 100644 --- a/docs/diagrams/01-data-flow.svg +++ b/docs/diagrams/01-data-flow.svg @@ -81,7 +81,7 @@ pre-aggregated buckets - PostgreSQL 16 + PostgreSQL 18 encrypted at rest diff --git a/docs/diagrams/03-self-hosting-topology.svg b/docs/diagrams/03-self-hosting-topology.svg index 658a87b7b..0b860ce2b 100644 --- a/docs/diagrams/03-self-hosting-topology.svg +++ b/docs/diagrams/03-self-hosting-topology.svg @@ -1,6 +1,6 @@ HealthLog self-hosting topology - Internet traffic terminates at a reverse proxy (Caddy, Traefik or Nginx) for TLS, then reaches the Next.js app on port 3000. The app shares a Postgres 16 database with the pg-boss worker process for reminders, insights, backups and the Apple Health import. GHCR ships the multi-arch image; Coolify watches the digest and pulls; off-host S3-compatible storage is optional for backups. + Internet traffic terminates at a reverse proxy (Caddy, Traefik or Nginx) for TLS, then reaches the Next.js app on port 3000. The app shares a Postgres 18 database with the pg-boss worker process for reminders, insights, backups and the Apple Health import. GHCR ships the multi-arch image; Coolify watches the digest and pulls; off-host S3-compatible storage is optional for backups. @@ -47,7 +47,7 @@ - PostgreSQL 16 + PostgreSQL 18 26 Prisma models + pg-boss queues diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md index c462567c7..e690170b0 100644 --- a/docs/diagrams/README.md +++ b/docs/diagrams/README.md @@ -10,7 +10,7 @@ copy. | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `01-data-flow.svg` | Sources (Withings, Apple Health export.zip, iPhone HealthKit, manual, Nightscout) → ingest endpoints → Measurement + MeasurementRollup → reads (rollup probe / live SQL fallback / cache) → surfaces (dashboard, insights, Coach, doctor PDF). | | `02-coach-pipeline.svg` | User question → snapshot builder → prompt assembler → provider chain (ChatGPT OAuth / OpenAI / Anthropic / local / admin-shared) → response parser → cited reply with mini-charts. | -| `03-self-hosting-topology.svg` | Internet → reverse proxy (TLS) → Next.js app (port 3000) ↔ pg-boss worker ↔ Postgres 16, plus GHCR image pull, optional Coolify autodeploy, optional S3 off-host backup. | +| `03-self-hosting-topology.svg` | Internet → reverse proxy (TLS) → Next.js app (port 3000) ↔ pg-boss worker ↔ Postgres 18, plus GHCR image pull, optional Coolify autodeploy, optional S3 off-host backup. | | `04-source-priority.svg` | Three sources for the same metric on the same day → resolver picks one canonical row per (cumulative vs point) ladder. Losing rows stay in the audit trail. | | `05-security-model.svg` | Three concentric layers — auth perimeter (passkey + password fallback), session perimeter (HttpOnly cookie + Postgres sessions), encrypted core (AES-256-GCM versioned keys). Side rails: rate limiter, audit log, HMAC tokens, CSP/HSTS, SSRF guard. | diff --git a/docs/ops/deploy.md b/docs/ops/deploy.md index 032c55730..0a1f8ff7c 100644 --- a/docs/ops/deploy.md +++ b/docs/ops/deploy.md @@ -129,6 +129,26 @@ An INVALID index whose migration is already recorded applied is not touched at boot. The boot names it on stderr with `WARNING — index … is INVALID`; rebuild it with `REINDEX INDEX CONCURRENTLY "";` when the database is quiet. +### v1.46.0 notes + +- **Postgres 18.** The compose file moves the bundled database from + Postgres 16 to 18. The first `docker compose up -d` with the new compose + file runs `db-upgrade`, which copies the database into a new + `postgres-data` volume, checks every table's row count, and only then lets + the app start; the old `pgdata` volume stays as it was, as the way back. + Pulling only the new image with the old compose file keeps 16, which this + release still runs on. Details, rollback and the steps for a database you + run yourself: [postgres-major-upgrade.md](postgres-major-upgrade.md). +- Migration `0485` drops `preview_features_json` from `users`, the same + pattern as `0430` in v1.44.0 below: do not run the previous image beside + the new schema. The sequence above stops the old container first; a + web/worker split stops both roles before the first new container starts. +- The other migrations add columns and one index (built concurrently). + Three touch rows: `0476` links API-written measurements to their token on + accounts that have exactly one measurement token, and `0478`/`0479` give + PDFs whose preview or text read failed one more attempt. On a prod-sized + copy all of them together finished in under a second. + ### v1.45.0 notes A single-container install (`HEALTHLOG_PROCESS_TYPE=all`, the shipped diff --git a/docs/ops/postgres-major-upgrade.md b/docs/ops/postgres-major-upgrade.md index 744213f28..58af2d6c4 100644 --- a/docs/ops/postgres-major-upgrade.md +++ b/docs/ops/postgres-major-upgrade.md @@ -1,32 +1,97 @@ # Moving the database to a newer Postgres major -The compose file ships Postgres 16 and stays there for now, because a major -version change is the one database update that cannot happen by itself on -`docker compose pull`. HealthLog runs on Postgres 18 as well: the full -integration suite runs against 18 (`INTEGRATION_PG_IMAGE=postgres:18-alpine -pnpm test:integration`), and the TrueNAS app starts new installs on 18. +From v1.46 the compose file runs Postgres 18. Installs from v1.45 or earlier +ran 16, and a major version cannot open the data directory of an older one, +so the compose file carries a one-time step that moves the database across +on the first start after the update. Nothing needs doing by hand on a stack +that runs the project's `docker-compose.yml`. -## Why changing the image line is not enough +Installs that bring their own Postgres (Unraid, an existing database server) +are not touched: HealthLog runs on 16 and 18 alike, and the move there is +yours to schedule with the steps under [By hand](#by-hand). -Minor releases (16.14 to 16.15) share the on-disk format. Majors do not, and -the official image refuses to start a data directory written by an older -major. From 18 on, the image also keeps its data one level deeper, under -`/var/lib/postgresql/18/docker`, and expects the volume mounted at -`/var/lib/postgresql` instead of `/var/lib/postgresql/data`. With the old -volume and only the image line changed, the `db` container exits at once with -"there appears to be PostgreSQL data in: /var/lib/postgresql/data". Nothing is -lost, but the stack stays down until the image line goes back to 16. +## What happens on the first start -So the move is a dump from 16 and a restore into a fresh 18 volume. +`db-upgrade` is a short-lived container next to `db`. It starts after the new +Postgres 18 database is up and before the app, and it exits in a second when +there is nothing to move. When the old `pgdata` volume holds a Postgres 16 +database and the new one is empty, it: -## Steps +1. checks there is room on the disk for a second copy; +2. starts Postgres 16 inside its own container on the old volume, reachable + only from inside that container; +3. streams a dump into Postgres 18 in a single transaction; +4. compares the row count of every table between the two; +5. gathers the planner statistics, which a restore does not carry; +6. stops Postgres 16 and leaves a marker file on the old volume. -Take the encryption key's copy out of its safe place first: the dump is -unreadable without it, and this is the moment you would find out. +The app starts once that succeeded, applies the release's migrations and +serves as before. On a typical personal record the whole move takes seconds; +a database of a few gigabytes takes a few minutes. Watch it with +`docker compose logs -f db-upgrade`. + +If any step fails, the new database stays empty, the app does not start, and +the log says what happened. A move cut short by a restart or a full disk is +run again from scratch on the next `docker compose up -d`. + +The old `pgdata` volume is read and never changed, apart from the marker +file. That is the way back. + +## Going back + +Stop the stack, put the two lines of the `db` service back the way v1.45 had +them, and start it with the v1.45 image: + +```yaml +db: + image: postgres:16.15-alpine + volumes: + - pgdata:/var/lib/postgresql/data +``` ```bash -# 1. Stop the app so nothing writes while the dump runs. If you run the -# worker role in its own container, stop it too: docker compose stop app app-worker +docker compose down +HEALTHLOG_IMAGE_REF=:v1.45.0 docker compose up -d +``` + +Anything written after the move lives only in the Postgres 18 volume. + +## Afterwards + +Keep the old volume until the new database has run for a while. When you are +sure, remove it (`docker volume ls` shows its full name, prefixed with the +compose project): + +```bash +docker volume rm _pgdata +``` + +`db-upgrade` then finds nothing to move. A later release drops the service +and the `pgdata` line. + +## Starting over + +When the move has run and the Postgres 18 volume is empty again (deleted by +hand, say), `db-upgrade` refuses to import the old state a second time, since +that would quietly bring back the database as it was on the day of the move. +To do exactly that on purpose, remove the marker and start again: + +```bash +docker compose run --rm --entrypoint rm db-upgrade \ + /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18 +docker compose up -d +``` + +## By hand + +For a Postgres you run yourself, or a compose file you have changed so much +that the step does not fit, the move is a dump and a restore. Take the +encryption key's copy out of its safe place first: the dump is unreadable +without it, and this is the moment you would find out. + +```bash +# 1. Stop the app so nothing writes while the dump runs (and app-worker, if +# you run the worker role in its own container). docker compose stop app # 2. Dump the database in Postgres' custom format. @@ -37,56 +102,31 @@ docker compose exec -T db pg_dump -U healthlog -Fc healthlog \ docker compose stop db ``` -4. In `docker-compose.yml`, change the `db` service: - - ```yaml - image: postgres:18.6-alpine # or the current 18 minor - volumes: - - pgdata18:/var/lib/postgresql - ``` - - and add the new volume next to `pgdata` in the top-level `volumes:` block: - - ```yaml - volumes: - pgdata: - pgdata18: - ``` +4. Point the `db` service at Postgres 18 and a fresh volume. From 18 on, the + image keeps its data under `/var/lib/postgresql/18/docker` and expects the + volume mounted one level up, at `/var/lib/postgresql`; mounted at + `.../data` it refuses to start with "there appears to be PostgreSQL data + in: /var/lib/postgresql/data". ```bash -# 5. Start the new database and wait until it reports healthy. +# 5. Start the new database and restore the dump into it. docker compose up -d db -docker compose ps db - -# 6. Restore the dump into it. docker compose exec -T db pg_restore -U healthlog -d healthlog --no-owner \ < healthlog-pg16-.dump -# 7. Gather the planner statistics. A restore brings the rows but not the +# 6. Gather the planner statistics. A restore brings the rows but not the # statistics, and until autovacuum gets round to every table the planner # guesses at their sizes, which can turn a sub-second page into one that # takes many seconds. docker compose exec -T db vacuumdb -U healthlog -d healthlog --analyze-in-stages -# 8. Start the app (and app-worker, if you run it) and confirm it. +# 7. Start the app and confirm it. docker compose up -d app curl -s http://localhost:3000/api/version ``` -The start log should say there are no pending migrations: the dump carried the -migration history with it. Sign in and open a few pages before you go on. - -## Afterwards - -Keep the old `pgdata` volume and the dump until the new database has run for a -few days. Going back means restoring the old `image:` and `volumes:` lines; -the old volume still holds the database as it was before step 2. When you are -sure, remove it (`docker volume ls` shows its full name, prefixed with the -compose project): - -```bash -docker volume rm _pgdata -``` +The start log should say there are no pending migrations beyond the release's +own: the dump carried the migration history with it. If you had run `CREATE EXTENSION pg_stat_statements;` for [query statistics](query-statistics.md), the dump carries it and the compose diff --git a/docs/self-hosting/getting-started.md b/docs/self-hosting/getting-started.md index e9cd933a8..f4fb5d208 100644 --- a/docs/self-hosting/getting-started.md +++ b/docs/self-hosting/getting-started.md @@ -81,7 +81,9 @@ explicitly in `.env` (see the commented example there). ### Using an existing or shared Postgres -HealthLog can live in a database on a Postgres instance you already run. +HealthLog can live in a database on a Postgres instance you already run, +version 16 or newer; the compose file ships 18, and every release is tested +on both. It stores every time as UTC and pins each connection it opens to UTC (`TimeZone=UTC`, sent with the libpq `options` startup parameter), so the server's or the database's own `timezone` setting does not matter: an @@ -105,9 +107,12 @@ ALTER DATABASE healthlog SET timezone TO 'UTC'; docker compose up -d ``` -Compose starts two services: +Compose starts two services and one short-lived step: -- `db` — PostgreSQL 16 with a named volume for persistence. +- `db` — PostgreSQL 18 with a named volume for persistence. +- `db-upgrade` — moves a Postgres 16 database from v1.45 or earlier into + the new one, once, and exits; on a new install it has nothing to do. See + [Moving to a newer Postgres major](../ops/postgres-major-upgrade.md). - `app` — HealthLog itself, listening on `localhost:3000`. The image runs `prisma migrate deploy` on boot so the schema lands automatically; subsequent restarts no-op when the database is diff --git a/docs/self-hosting/portainer/README.md b/docs/self-hosting/portainer/README.md index b9da05db8..aa8dc2f9c 100644 --- a/docs/self-hosting/portainer/README.md +++ b/docs/self-hosting/portainer/README.md @@ -1,7 +1,7 @@ # HealthLog on Portainer HealthLog ships a Portainer **app template** that deploys the app and its -PostgreSQL 16 database as one stack from the project `docker-compose.yml`. +PostgreSQL 18 database as one stack from the project `docker-compose.yml`. ## Add the template @@ -39,7 +39,7 @@ curl -s http://:3000/api/version ## Notes -- Persistent state lives in one named volume (`pgdata`, the Postgres data dir); +- Persistent state lives in one named volume (`postgres-data`, the Postgres data); the app container is stateless. Back up the database to back up everything. - Pin a release tag (`HEALTHLOG_IMAGE_REF=:vX.Y.Z`) for deliberate upgrades, or track `:latest`. Skim the diff --git a/docs/self-hosting/scaling.md b/docs/self-hosting/scaling.md index 77759eeeb..c5f0b0071 100644 --- a/docs/self-hosting/scaling.md +++ b/docs/self-hosting/scaling.md @@ -174,7 +174,7 @@ sized for the web traffic alone leaves the worker's jobs queueing for the same connections. Plan total Postgres slots as **container_count × budget**. -A stock Postgres 16 container ships with `max_connections = 100`, so +A stock Postgres container ships with `max_connections = 100`, so the safe envelope is: | Web replicas | Worker replicas | Total containers | Pool slots used | Headroom under 100 | diff --git a/docs/self-hosting/truenas.md b/docs/self-hosting/truenas.md index 460867eb4..d312e4fb7 100644 --- a/docs/self-hosting/truenas.md +++ b/docs/self-hosting/truenas.md @@ -213,7 +213,7 @@ the menu next to Custom App, Install via YAML, using these values: | User | runs as uid `1001` by default; no extra capabilities, `no-new-privileges` is fine | | Health check | `GET /api/health`, `200` when ready | | Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot | -| Database | PostgreSQL 16 or 18, no extensions, in its own container with its own dataset | +| Database | PostgreSQL 18 (16 works too), no extensions, in its own container with its own dataset | | Volumes | none for the app; the database gets its own dataset | | Variable | Required | Notes | diff --git a/docs/self-hosting/unraid.md b/docs/self-hosting/unraid.md index 77abd18f7..87c0e91df 100644 --- a/docs/self-hosting/unraid.md +++ b/docs/self-hosting/unraid.md @@ -69,10 +69,12 @@ curl -s http://tower:3000/api/version Use this if you prefer the per-app Unraid UI. You install PostgreSQL yourself, then add HealthLog pointed at it. -### Step 1 — install PostgreSQL 16 +### Step 1 — install PostgreSQL 1. In **Apps** (Community Applications), search for a **PostgreSQL** container - (for example jj9987's `postgresql`). Install version 16. + (for example jj9987's `postgresql`). Install version 18; an existing 16 + works too. Moving a 16 to 18 later is a dump and restore, see + [Moving to a newer Postgres major](../ops/postgres-major-upgrade.md#by-hand). 2. Set: - `POSTGRES_USER` = `healthlog` - `POSTGRES_PASSWORD` = your generated password diff --git a/docs/self-hosting/unraid/healthlog.xml b/docs/self-hosting/unraid/healthlog.xml index c6e92f981..c68c70c58 100644 --- a/docs/self-hosting/unraid/healthlog.xml +++ b/docs/self-hosting/unraid/healthlog.xml @@ -2,7 +2,7 @@