fix(db): leave a Postgres 16 volume alone while the database runs on it

Going back to Postgres 16 after a move that stopped half-way left the
moving marker, and the next start emptied the live 16 database and
started a second server on its data directory. db-upgrade now waits for
the database service and does nothing unless it runs Postgres 18.

Using 16 again after a finished move and then updating started the app
on the move-day copy without a word. The marker now records the old
database's last checkpoint, and a later start refuses when it moved on;
the guide says how to move the current database instead. The disk check
leaves room for the first pre-migration dump, and the move check seeds
the old database with v1.45's own migrations.

docker-compose.yml

@@ -680,6 +680,10 @@ services:
   #
   # On every start it decides in a second and exits:
   # - no `pgdata` data (a new install): nothing to do;
+  # - the database service runs Postgres 16 (after going back): nothing to
+  #   do, the live 16 volume is left alone;
+  # - 16 was used again after an earlier move: refuses, so the app does not
+  #   start on the move-day copy;
   # - Postgres 18 already holds the database: nothing to do;
   # - otherwise it starts Postgres 16 inside this container on the old
   #   volume, streams a dump into the new database in one transaction,
@@ -737,14 +741,35 @@ services:
           log "no Postgres 16 database to move, nothing to do"
           exit 0
         fi
+        # The database service must be the new major. After going back to 16 the
+        # service runs on this very volume: touching it then would wipe the live data.
+        i=0
+        until NEWVER=$$(new "SHOW server_version_num" 2>/dev/null); do
+          i=$$((i + 1))
+          [ "$$i" -lt 60 ] || { log "the database service did not answer for a minute"; exit 1; }
+          sleep 1
+        done
+        if [ "$$NEWVER" -lt 180000 ]; then
+          log "the database service runs Postgres $$((NEWVER / 10000)), not 18: nothing to move, the old volume is left alone"
+          exit 0
+        fi
+        checkpoint() { pg_controldata "$$OLD" | awk -F': *' '/^Latest checkpoint location/ { print $$2 }'; }
         USER_TABLES="FROM pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema')"
         TABLES=$$(new "SELECT count(*) $$USER_TABLES")
         if [ -e "$$MARK" ]; then
+          MOVED_ON=$$(sed -n 1p "$$MARK")
+          MOVED_CHECKPOINT=$$(sed -n 2p "$$MARK")
+          if [ -n "$$MOVED_CHECKPOINT" ] && [ "$$(checkpoint)" != "$$MOVED_CHECKPOINT" ]; then
+            log "the Postgres 16 database changed after it was moved on $$MOVED_ON: it was used again after going back to 16."
+            log "Postgres 18 still holds the state of that day, so it is not started on. To move the current 16 database,"
+            log "see docs/ops/postgres-major-upgrade.md, \"Updating again after going back\"."
+            exit 1
+          fi
           if [ "$$TABLES" != 0 ]; then
-            log "the Postgres 16 database was moved on $$(cat "$$MARK"), nothing to do"
+            log "the Postgres 16 database was moved on $$MOVED_ON, nothing to do"
             exit 0
           fi
-          log "the Postgres 16 database was moved on $$(cat "$$MARK"), but the Postgres 18 database is empty now."
+          log "the Postgres 16 database was moved on $$MOVED_ON, but the Postgres 18 database is empty now."
           log "Not importing the old state again by itself. See docs/ops/postgres-major-upgrade.md, \"Starting over\"."
           exit 1
         fi
@@ -763,8 +788,9 @@ services:
         fi
         NEED=$$(du -sk "$$OLD" | cut -f1)
         FREE=$$(df -Pk /var/lib/postgresql/next | awk 'NR == 2 { print $$4 }')
-        if [ "$$FREE" -le "$$NEED" ]; then
-          log "the move needs about $$((NEED / 1024)) MiB free for the new database, the disk has $$((FREE / 1024)) MiB."
+        # Room for the copy, and for the dump the app takes before its first migration.
+        if [ "$$FREE" -le $$((NEED * 3 / 2)) ]; then
+          log "the move needs about $$((NEED * 3 / 2 / 1024)) MiB free for the new database and its first dump, the disk has $$((FREE / 1024)) MiB."
           log "Nothing was changed. Free some space and start the stack again."
           exit 1
         fi
@@ -798,6 +824,9 @@ services:
         vacuumdb -h db -U healthlog -d healthlog --analyze-in-stages --quiet
         kill -INT "$$PID"
         wait "$$PID" || true
+        # The marker keeps the old database's last checkpoint: a later start can tell
+        # whether 16 was used again after the move.
+        { date -u +%Y-%m-%dT%H:%M:%SZ; checkpoint; } > "$$MOVING"
         mv "$$MOVING" "$$MARK"
         log "done. The old volume stays as it was, as a way back; see docs/ops/postgres-major-upgrade.md"
 

docs/ops/postgres-major-upgrade.md

@@ -17,7 +17,9 @@ Postgres 18 database is up and before the app, and it exits in a second when
 there is nothing to move. When the old `pgdata` volume holds a Postgres 16
 database and the new one is empty, it:
 
-1. checks there is room on the disk for a second copy;
+1. checks that the database service runs Postgres 18, and that there is room
+   on the disk for a second copy and for the dump the app takes before its
+   first migration;
 2. starts Postgres 16 inside its own container on the old volume, reachable
    only from inside that container;
 3. streams a dump into Postgres 18 in a single transaction;
@@ -55,6 +57,29 @@ HEALTHLOG_IMAGE_REF=:v1.45.0 docker compose up -d
 ```
 
 Anything written after the move lives only in the Postgres 18 volume.
+`db-upgrade` can stay in the file: it sees that the database service runs
+Postgres 16 and leaves that volume alone. This also holds after a move that
+stopped half-way.
+
+## Updating again after going back
+
+If you used Postgres 16 again after going back, the Postgres 18 volume holds
+the state of the day of the move, not what you wrote since. `db-upgrade`
+notices that the old database changed after the move and refuses to start on
+the old copy. To move the current 16 database, remove the old copy and the
+marker, then start again:
+
+```bash
+docker compose down
+docker volume rm <project>_postgres-data
+docker compose run --rm --entrypoint rm db-upgrade \
+  /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18
+docker compose up -d
+```
+
+`docker volume ls` shows the volume's full name. Removing it deletes what was
+written on Postgres 18 before you went back; take a dump of it first if any
+of that matters.
 
 ## Afterwards
 

scripts/compose-db-upgrade-check.sh

@@ -11,7 +11,11 @@
 #      empty the new database and run the move again.
 #   5. Empty the new volume: it must refuse to import the old state again,
 #      and move it once more after the marker is removed as the guide says.
-#   6. A new install with no old volume: nothing to do.
+#   6. Going back to 16 after a move that did not finish: db-upgrade leaves
+#      the live 16 database alone.
+#   7. Using 16 again after a move, then updating: refused, not the old copy.
+#   8. Updating again as the guide says: the current 16 database moves.
+#   9. A new install with no old volume: nothing to do.
 #
 # Needs Docker and the repo's node_modules (for the migrations). Run it from
 # anywhere in the repo: `bash scripts/compose-db-upgrade-check.sh`.
@@ -85,7 +89,7 @@ wait_for_port() {
 migrate_old() {
   local started=$SECONDS
   until DATABASE_URL="postgresql://healthlog:$POSTGRES_PASSWORD@127.0.0.1:$PORT/healthlog?schema=public" \
-    node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$ROOT/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do
+    node "$ROOT/node_modules/prisma/build/index.js" migrate deploy --schema "$WORK/old-prisma/prisma/schema.prisma" >"$WORK/migrate.log" 2>&1; do
     if [ $((SECONDS - started)) -ge 30 ] || ! grep -q "P1001" "$WORK/migrate.log"; then
       cat "$WORK/migrate.log" >&2
       return 1
@@ -94,6 +98,18 @@ migrate_old() {
   done
 }
 
+# The v1.45 schema, migrated with v1.45's own migrations.
+mkdir -p "$WORK/old-prisma"
+git -C "$ROOT" archive "$OLD_REF" prisma | tar -x -C "$WORK/old-prisma"
+# The current compose file with its database pointed back at Postgres 16, the
+# way docs/ops/postgres-major-upgrade.md "Going back" describes it.
+sed -e 's|image: postgres:18\.[0-9]*-alpine|image: postgres:16.15-alpine|' \
+  -e 's|- postgres-data:/var/lib/postgresql$|- pgdata:/var/lib/postgresql/data|' \
+  "$ROOT/docker-compose.yml" >"$WORK/back.yml"
+grep -q 'image: postgres:16.15-alpine' "$WORK/back.yml" && grep -q -- '- pgdata:/var/lib/postgresql/data' "$WORK/back.yml" ||
+  { echo "FAIL: could not derive the going-back compose file" >&2; exit 1; }
+back() { docker compose -p "$PROJECT" -f "$WORK/back.yml" "$@"; }
+
 down_all
 step "a v1.45 database on Postgres 16"
 old up -d --wait db >"$WORK/up.log" 2>&1 || { cat "$WORK/up.log" >&2; fail "the v1.45 database did not start"; }
@@ -157,10 +173,42 @@ CODE=$(run_upgrade)
 [ "$CODE" = 0 ] || fail "db-upgrade exited $CODE after the marker was removed"
 [ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the probe rows differ after starting over"
 
+step "going back to 16 after a move that did not finish leaves 16 alone"
+new down >/dev/null 2>&1
+docker volume rm "${PROJECT}_postgres-data" >/dev/null
+docker run --rm -v "${PROJECT}_pgdata:/old" postgres:16.15-alpine \
+  mv /old/HEALTHLOG_MOVED_TO_POSTGRES_18 /old/HEALTHLOG_MOVING_TO_POSTGRES_18
+back up -d db-upgrade >/dev/null 2>&1 || true
+CODE=$(docker wait healthlog-db-upgrade)
+upgrade_log
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE beside a Postgres 16 database service"
+upgrade_log | grep -q 'not 18: nothing to move' || fail "db-upgrade did not recognise the Postgres 16 service"
+[ "$(probe docker exec healthlog-db)" = "$PROBE" ] || fail "the live Postgres 16 data changed beside db-upgrade"
+
+step "using 16 again after a move, then updating, is refused instead of serving the old copy"
+back down >/dev/null 2>&1
+CODE=$(run_upgrade)
+[ "$CODE" = 0 ] || fail "the move after going back exited $CODE"
+back up -d db >/dev/null 2>&1
+back exec -T db psql -U healthlog -d healthlog -v ON_ERROR_STOP=1 -qc \
+  "INSERT INTO check_probe.rows VALUES (50001, 'written on 16 after the move')"
+back down >/dev/null 2>&1
+CODE=$(run_upgrade)
+[ "$CODE" = 1 ] || fail "db-upgrade exited $CODE after 16 was used again, expected 1"
+upgrade_log | grep -q 'changed after it was moved' || fail "the refusal did not say that 16 changed"
+
+step "updating again as the guide says moves the current 16 database"
+new down >/dev/null 2>&1
+docker volume rm "${PROJECT}_postgres-data" >/dev/null
+new run --rm --entrypoint rm db-upgrade /var/lib/postgresql/data/HEALTHLOG_MOVED_TO_POSTGRES_18 >/dev/null 2>&1
+CODE=$(run_upgrade)
+[ "$CODE" = 0 ] || fail "db-upgrade exited $CODE when moving the current 16 database"
+[ "$(sql "SELECT count(*) FROM check_probe.rows")" = 50001 ] || fail "the row written on 16 did not arrive"
+
 step "a new install: nothing to do"
 new down -v >/dev/null 2>&1
 CODE=$(run_upgrade)
 [ "$CODE" = 0 ] || fail "db-upgrade exited $CODE on a new install"
 upgrade_log | tail -1 | grep -q 'no Postgres 16 database to move' || fail "the new install did not say there was nothing to move"
 
-echo "OK: the Postgres 16 -> 18 move behaves in all six cases"
+echo "OK: the Postgres 16 -> 18 move behaves in all nine cases"

src/__tests__/compose-db-upgrade-guard.test.ts

@@ -77,6 +77,8 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
     // The checks run before the restore, in this order.
     const order = [
       "PG_VERSION",
+      // Never touch the old volume while the database service runs on it.
+      'if [ "$$NEWVER" -lt 180000 ]',
       "$$MARK",
       "$$MOVING",
       'elif [ "$$TABLES" != 0 ]',