fix(backup): keep a dump locally only where it survives the container

The image creates DB_BACKUP_DIR owned by the app user, so an install that
runs it without the backups volume could write there, and every nightly and
pre-migration dump landed in the container's own layer. The next update
threw them away while the admin page, the boot line and the readiness card
called them kept on this server.

A dump directory now counts as kept only when it survives the container:
inside a container (Docker, Podman, Kubernetes, a container cgroup or an
overlay root) the directory must sit on a mount of its own that is not a
tmpfs, read from /proc/self/mountinfo with a device-number fallback;
outside one, a writable directory is a disk. Otherwise no dump is written
there: the nightly run uploads off-host only and records not_persistent,
the pre-migration dump is skipped with a warning, the boot line says the
directory is inside the container, the backups page stops promising local
dumps and ignores local-only ledger rows, and readiness fails with no
bucket and notes the bucket-only setup otherwise.

The Unraid template gains a path for the dumps, the Unraid and TrueNAS
pages say why it is needed, and the restore drill first runs the stack
without the volume and checks that nothing is kept or claimed.

scripts/compose-dr-drill.sh

@@ -6,6 +6,10 @@
 #   1. Build the app image from this checkout and start the bundled compose
 #      stack with fixed throwaway keys, and an S3-compatible bucket outside
 #      it (Versity S3 Gateway, pinned) as the off-host target.
+#      First without the `backups` volume: the run must keep nothing in the
+#      container's layer, say so in its ledger row and the start log, and
+#      still upload the dump. Then the stack is thrown away and started again
+#      as shipped.
 #   2. Seed real data: the demo seed (documents, encrypted notes, years of
 #      readings) and a Guardian with a managed profile, written by the app's
 #      own code; then, through the API, a reading with an encrypted note.
@@ -184,6 +188,33 @@ docker run -d --name "$BUCKET" --network "$BUCKET" -p "127.0.0.1:$BUCKET_PORT:70
   "mkdir -p /data/$BACKUP_S3_BUCKET && exec versitygw --access $BACKUP_S3_ACCESS_KEY --secret $BACKUP_S3_SECRET_KEY posix /data" \
   >/dev/null || fail "the bucket did not start"
 
+step "a server without the backups volume keeps no dump on itself"
+# The image creates the dump directory, so without the volume it is writable,
+# and it sits in the container's layer that the next update throws away. No
+# dump may be kept, or counted as kept, there; the bucket still gets one.
+cat >"$WORK/no-volume.yml" <<YAML
+services:
+  app:
+    volumes: !reset []
+YAML
+stack -f "$WORK/no-volume.yml" up -d app >"$WORK/up.log" 2>&1 ||
+  { tail -20 "$WORK/up.log" >&2; fail "the stack without the volume did not start"; }
+wait_healthy
+i=0
+until RUN=$(sql "SELECT status || ' ' || local_result || ' ' || offhost_result FROM database_backup_runs WHERE status <> 'running' ORDER BY started_at DESC LIMIT 1") && [ -n "$RUN" ]; do
+  i=$((i + 1))
+  [ "$i" -lt 120 ] || fail "no database-backup run within four minutes without the volume"
+  sleep 2
+done
+[ "$RUN" = "succeeded not_persistent uploaded" ] ||
+  fail "without the volume the run read '$RUN', not 'succeeded not_persistent uploaded'"
+LEFT=$(docker exec "$PROJECT-app" sh -c 'ls -A /var/lib/healthlog/backups')
+[ -z "$LEFT" ] || fail "a dump was left in the container's layer: $LEFT"
+docker logs "$PROJECT-app" 2>&1 | grep -q "is inside the container, not on a mounted volume" ||
+  fail "the start log does not say the dump directory is inside the container"
+echo "   $RUN, nothing in the container's layer"
+stack down -v --remove-orphans >/dev/null 2>&1
+
 step "a running server with real data"
 stack up -d app >"$WORK/up.log" 2>&1 || { tail -20 "$WORK/up.log" >&2; fail "the stack did not start"; }
 wait_healthy

scripts/db-dump.mjs

@@ -13,9 +13,11 @@
  *
  * Exit 0: the dump was written, or there was nothing to do (no pending
  * migration, a new database, `DB_BACKUP_BEFORE_MIGRATE=false`), or there is no
- * writable directory to put it in, which is said as a warning and is not a
- * reason to keep the app down. Exit 1: the directory is writable and the dump
- * failed. The entrypoint then stops the boot and names the switch that skips
+ * directory to put it in that survives the container (none writable, or only
+ * the image's own directory without the volume, which the update that brings
+ * these migrations is about to throw away), which is said as a warning and is
+ * not a reason to keep the app down. Exit 1: the directory is kept and the
+ * dump failed. The entrypoint then stops the boot and names the switch that skips
  * the dump, because migrating without the way back the operator was promised
  * is the one thing this step exists to prevent.
  *
@@ -28,9 +30,9 @@ import { pathToFileURL } from "node:url";
 
 import {
   applyRetention,
+  backupDirectoryStatus,
   connect,
   createDatabaseDump,
-  isWritableDirectory,
   latestMigration,
   listUserTables,
   connectionFromUrl,
@@ -60,7 +62,11 @@ function mb(bytes) {
   return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
 }
 
-export async function preMigrationDump(env = process.env) {
+/**
+ * @param {Record<string, string | undefined>} [env]
+ * @param {{ directoryStatus?: typeof backupDirectoryStatus }} [options]
+ */
+export async function preMigrationDump(env = process.env, options = {}) {
   if (!preMigrationDumpEnabled(env)) {
     log("DB_BACKUP_BEFORE_MIGRATE=false, no dump before migrating.");
     return 0;
@@ -101,9 +107,15 @@ export async function preMigrationDump(env = process.env) {
   }
 
   const dir = resolveBackupDir(env);
-  if (!isWritableDirectory(dir)) {
+  const status = (options.directoryStatus ?? backupDirectoryStatus)(dir);
+  if (!status.persistent) {
+    const why = !status.writable
+      ? "is not writable"
+      : status.reason === "memory"
+        ? "is in memory (tmpfs) and would not keep a dump past a restart"
+        : "is inside the container, not on a mounted volume, and would not keep a dump past this update";
     warn(
-      `${dir} is not writable, so no dump is taken before the ${pending.length} pending migration(s). ` +
+      `${dir} ${why}, so no dump is taken before the ${pending.length} pending migration(s). ` +
         "Mount the backups volume (docker-compose.yml) to keep a way back; continuing.",
     );
     return 0;

scripts/env-manifest.json

@@ -372,7 +372,7 @@
         },
         {
           "name": "DB_BACKUP_DIR",
-          "purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. Without a writable directory the nightly dump only goes off-host and no dump is taken before migrations.",
+          "purpose": "Directory the dumps are written to and restored from. Default /var/lib/healthlog/backups, the `backups` volume. In a container it must be a mount of its own: a directory in the container's layer is lost at the next update and counts as none. Without one the nightly dump only goes off-host and no dump is taken before migrations.",
           "example": "/var/lib/healthlog/backups"
         },
         {