diff --git a/docker-compose.yml b/docker-compose.yml index 292340e3a..ad87ed191 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -680,6 +680,10 @@ services: # # On every start it decides in a second and exits: # - no `pgdata` data (a new install): nothing to do; + # - the database service runs Postgres 16 (after going back): nothing to + # do, the live 16 volume is left alone; + # - 16 was used again after an earlier move: refuses, so the app does not + # start on the move-day copy; # - Postgres 18 already holds the database: nothing to do; # - otherwise it starts Postgres 16 inside this container on the old # volume, streams a dump into the new database in one transaction, @@ -737,14 +741,35 @@ services: log "no Postgres 16 database to move, nothing to do" exit 0 fi + # The database service must be the new major. After going back to 16 the + # service runs on this very volume: touching it then would wipe the live data. + i=0 + until NEWVER=$$(new "SHOW server_version_num" 2>/dev/null); do + i=$$((i + 1)) + [ "$$i" -lt 60 ] || { log "the database service did not answer for a minute"; exit 1; } + sleep 1 + done + if [ "$$NEWVER" -lt 180000 ]; then + log "the database service runs Postgres $$((NEWVER / 10000)), not 18: nothing to move, the old volume is left alone" + exit 0 + fi + checkpoint() { pg_controldata "$$OLD" | awk -F': *' '/^Latest checkpoint location/ { print $$2 }'; } USER_TABLES="FROM pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema')" TABLES=$$(new "SELECT count(*) $$USER_TABLES") if [ -e "$$MARK" ]; then + MOVED_ON=$$(sed -n 1p "$$MARK") + MOVED_CHECKPOINT=$$(sed -n 2p "$$MARK") + if [ -n "$$MOVED_CHECKPOINT" ] && [ "$$(checkpoint)" != "$$MOVED_CHECKPOINT" ]; then + log "the Postgres 16 database changed after it was moved on $$MOVED_ON: it was used again after going back to 16." + log "Postgres 18 still holds the state of that day, so it is not started on. To move the current 16 database," + log "see docs/ops/postgres-major-upgrade.md, \"Updating again after going back\"." + exit 1 + fi if [ "$$TABLES" != 0 ]; then - log "the Postgres 16 database was moved on $$(cat "$$MARK"), nothing to do" + log "the Postgres 16 database was moved on $$MOVED_ON, nothing to do" exit 0 fi - log "the Postgres 16 database was moved on $$(cat "$$MARK"), but the Postgres 18 database is empty now." + log "the Postgres 16 database was moved on $$MOVED_ON, but the Postgres 18 database is empty now." log "Not importing the old state again by itself. See docs/ops/postgres-major-upgrade.md, \"Starting over\"." exit 1 fi @@ -763,8 +788,9 @@ services: fi NEED=$$(du -sk "$$OLD" | cut -f1) FREE=$$(df -Pk /var/lib/postgresql/next | awk 'NR == 2 { print $$4 }') - if [ "$$FREE" -le "$$NEED" ]; then - log "the move needs about $$((NEED / 1024)) MiB free for the new database, the disk has $$((FREE / 1024)) MiB." + # Room for the copy, and for the dump the app takes before its first migration. + if [ "$$FREE" -le $$((NEED * 3 / 2)) ]; then + log "the move needs about $$((NEED * 3 / 2 / 1024)) MiB free for the new database and its first dump, the disk has $$((FREE / 1024)) MiB." log "Nothing was changed. Free some space and start the stack again." exit 1 fi @@ -798,6 +824,9 @@ services: vacuumdb -h db -U healthlog -d healthlog --analyze-in-stages --quiet kill -INT "$$PID" wait "$$PID" || true + # The marker keeps the old database's last checkpoint: a later start can tell + # whether 16 was used again after the move. + { date -u +%Y-%m-%dT%H:%M:%SZ; checkpoint; } > "$$MOVING" mv "$$MOVING" "$$MARK" log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md" diff --git a/docs/ops/postgres-major-upgrade.md b/docs/ops/postgres-major-upgrade.md index e856dc33f..670316b99 100644 --- a/docs/ops/postgres-major-upgrade.md +++ b/docs/ops/postgres-major-upgrade.md @@ -17,7 +17,9 @@ Postgres 18 database is up and before the app, and it exits in a second when there is nothing to move. When the old `pgdata` volume holds a Postgres 16 database and the new one is empty, it: -1. checks there is room on the disk for a second copy; +1. checks that the database service runs Postgres 18, and that there is room + on the disk for a second copy and for the dump the app takes before its + first migration; 2. starts Postgres 16 inside its own container on the old volume, reachable only from inside that container; 3. streams a dump into Postgres 18 in a single transaction; @@ -55,6 +57,29 @@ HEALTHLOG_IMAGE_REF=:v1.45.0 docker compose up -d ``` Anything written after the move lives only in the Postgres 18 volume. +`db-upgrade` can stay in the file: it sees that the database service runs +Postgres 16 and leaves that volume alone. This also holds after a move that +stopped half-way. + +## Updating again after going back + +If you used Postgres 16 again after going back, the Postgres 18 volume holds +the state of the day of the move, not what you wrote since. `db-upgrade` +notices that the old database changed after the move and refuses to start on +the old copy. To move the current 16 database, remove the old copy and the +marker, then start again: + +```bash +docker compose down +docker volume rm _postgres-data +docker compose run --rm --entrypoint rm db-upgrade \ + /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18 +docker compose up -d +``` + +`docker volume ls` shows the volume's full name. Removing it deletes what was +written on Postgres 18 before you went back; take a dump of it first if any +of that matters. ## Afterwards diff --git a/scripts/compose-db-upgrade-check.sh b/scripts/compose-db-upgrade-check.sh index ad49e8b39..b22f3dd02 100755 --- a/scripts/compose-db-upgrade-check.sh +++ b/scripts/compose-db-upgrade-check.sh @@ -11,7 +11,11 @@ # empty the new database and run the move again. # 5. Empty the new volume: it must refuse to import the old state again, # and move it once more after the marker is removed as the guide says. -# 6. A new install with no old volume: nothing to do. +# 6. Going back to 16 after a move that did not finish: db-upgrade leaves +# the live 16 database alone. +# 7. Using 16 again after a move, then updating: refused, not the old copy. +# 8. Updating again as the guide says: the current 16 database moves. +# 9. A new install with no old volume: nothing to do. # # Needs Docker and the repo's node_modules (for the migrations). Run it from # anywhere in the repo: `bash scripts/compose-db-upgrade-check.sh`. @@ -85,7 +89,7 @@ wait_for_port() { migrate_old() { local started=$SECONDS until DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \ - node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do + node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$WORK/old-prisma/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do if [ $((SECONDS - started)) -ge 30 ] || ! grep -q "P1001" "$WORK/migrate.log"; then cat "$WORK/migrate.log" >&2 return 1 @@ -94,6 +98,18 @@ migrate_old() { done } +# The v1.45 schema, migrated with v1.45's own migrations. +mkdir -p "$WORK/old-prisma" +git -C "$ROOT" archive "$OLD_REF" prisma | tar -x -C "$WORK/old-prisma" +# The current compose file with its database pointed back at Postgres 16, the +# way docs/ops/postgres-major-upgrade.md "Going back" describes it. +sed -e 's|image: postgres:18\.[0-9]*-alpine|image: postgres:16.15-alpine|' \ + -e 's|- postgres-data:/var/lib/postgresql$|- pgdata:/var/lib/postgresql/data|' \ + "$ROOT/docker-compose.yml" >"$WORK/back.yml" +grep -q 'image: postgres:16.15-alpine' "$WORK/back.yml" && grep -q -- '- pgdata:/var/lib/postgresql/data' "$WORK/back.yml" || + { echo "FAIL: could not derive the going-back compose file" >&2; exit 1; } +back() { docker compose -p "$PROJECT" -f "$WORK/back.yml" "$@"; } + down_all step "a v1.45 database on Postgres 16" old up -d --wait db >"$WORK/up.log" 2>&1 || { cat "$WORK/up.log" >&2; fail "the v1.45 database did not start"; } @@ -157,10 +173,42 @@ CODE=$(run_upgrade) [ "$CODE" = 0 ] || fail "db-upgrade exited $CODE after the marker was removed" [ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the probe rows differ after starting over" +step "going back to 16 after a move that did not finish leaves 16 alone" +new down >/dev/null 2>&1 +docker volume rm "${PROJECT}_postgres-data" >/dev/null +docker run --rm -v "${PROJECT}_pgdata:/old" postgres:16.15-alpine \ + mv /old/HEALTHLOG_MOVED_TO_POSTGRES_18 /old/HEALTHLOG_MOVING_TO_POSTGRES_18 +back up -d db-upgrade >/dev/null 2>&1 || true +CODE=$(docker wait healthlog-db-upgrade) +upgrade_log +[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE beside a Postgres 16 database service" +upgrade_log | grep -q 'not 18: nothing to move' || fail "db-upgrade did not recognise the Postgres 16 service" +[ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the live Postgres 16 data changed beside db-upgrade" + +step "using 16 again after a move, then updating, is refused instead of serving the old copy" +back down >/dev/null 2>&1 +CODE=$(run_upgrade) +[ "$CODE" = 0 ] || fail "the move after going back exited $CODE" +back up -d db >/dev/null 2>&1 +back exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -qc \ + "INSERT INTO check_probe.rows VALUES (50001, 'written on 16 after the move')" +back down >/dev/null 2>&1 +CODE=$(run_upgrade) +[ "$CODE" = 1 ] || fail "db-upgrade exited $CODE after 16 was used again, expected 1" +upgrade_log | grep -q 'changed after it was moved' || fail "the refusal did not say that 16 changed" + +step "updating again as the guide says moves the current 16 database" +new down >/dev/null 2>&1 +docker volume rm "${PROJECT}_postgres-data" >/dev/null +new run --rm --entrypoint rm db-upgrade /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18 >/dev/null 2>&1 +CODE=$(run_upgrade) +[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE when moving the current 16 database" +[ "$(sql "SELECT count(*) FROM check_probe.rows")" = 50001 ] || fail "the row written on 16 did not arrive" + step "a new install: nothing to do" new down -v >/dev/null 2>&1 CODE=$(run_upgrade) [ "$CODE" = 0 ] || fail "db-upgrade exited $CODE on a new install" upgrade_log | tail -1 | grep -q 'no Postgres 16 database to move' || fail "the new install did not say there was nothing to move" -echo "OK: the Postgres 16 -> 18 move behaves in all six cases" +echo "OK: the Postgres 16 -> 18 move behaves in all nine cases" diff --git a/src/__tests__/compose-db-upgrade-guard.test.ts b/src/__tests__/compose-db-upgrade-guard.test.ts index 1ab117a85..2109088d9 100644 --- a/src/__tests__/compose-db-upgrade-guard.test.ts +++ b/src/__tests__/compose-db-upgrade-guard.test.ts @@ -77,6 +77,8 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => { // The checks run before the restore, in this order. const order = [ "PG_VERSION", + // Never touch the old volume while the database service runs on it. + 'if [ "$$NEWVER" -lt 180000 ]', "$$MARK", "$$MOVING", 'elif [ "$$TABLES" != 0 ]',