fix(db-backup): keep instances apart in a shared bucket, and no decrypted dump behind

Off-host dumps were named by their time alone. Two instances sharing a bucket
(nothing forbids it, and it was safe for the account objects) overwrote each
other's dump at the same second of the same default schedule, and
latest --from-bucket restored whichever instance's dump was newest. Each
instance now uploads under database/<host of APP_URL>/, the object key (and so
the encryption's associated data) carries it, and the restore takes only this
instance's dumps: it refuses another's and names the instances it found,
unless --instance=<host> or the dump's full key says which one on purpose.

A restore killed outright (out of memory, docker kill) left its .restore- or
.download- directory, a whole database decrypted, in the backups directory,
where nothing removed it. A restore now removes them before it starts and on
SIGINT and SIGTERM, and the nightly retention sweeps any older than twelve
hours.

src/lib/database-backup/__tests__/dump-core.test.ts

@@ -52,6 +52,8 @@ import {
   resolveDailyKeep,
   selectForDeletion,
   verifyArchive,
+  instanceIdFromEnv,
+  listBucketDumps,
   type DirectoryProbe,
   type DumpSet,
   type Env,
@@ -784,6 +786,81 @@ describe("finding a dump in the bucket", () => {
   });
 });
 
+describe("dumps of more than one instance in one bucket", () => {
+  it("names the instance by the host of its public URL", () => {
+    expect(instanceIdFromEnv({ APP_URL: "https://Health.Example.org/" })).toBe(
+      "health.example.org",
+    );
+    expect(
+      instanceIdFromEnv({ NEXT_PUBLIC_APP_URL: "http://10.0.0.5:3000" }),
+    ).toBe("10.0.0.5-3000");
+    expect(instanceIdFromEnv({ APP_URL: "not a url" })).toBeNull();
+    expect(instanceIdFromEnv({})).toBeNull();
+    expect(offhostObjectKey("x.dump.enc", "health.example.org")).toBe(
+      "database/health.example.org/x.dump.enc",
+    );
+    expect(offhostObjectKey("x.dump.enc")).toBe("database/x.dump.enc");
+  });
+
+  it("takes the newest dump of this instance, never another's, unless named", () => {
+    const pair = (folder: string, stamp: string) => [
+      `database/${folder}healthlog-${stamp}-daily.dump.enc`,
+      `database/${folder}healthlog-${stamp}-daily.manifest.json.enc`,
+    ];
+    const keys = [
+      ...pair("prod.example/", "20261024T013000Z"),
+      // Newer, and another instance's.
+      ...pair("staging.example/", "20261025T013000Z"),
+      ...pair("", "20261023T013000Z"),
+    ];
+    expect(pickBucketDump(keys, null, "prod.example")?.dumpKey).toBe(
+      "database/prod.example/healthlog-20261024T013000Z-daily.dump.enc",
+    );
+    expect(pickBucketDump(keys, null, "nobody.example")).toBeNull();
+    // The dumps of an instance with no public URL, at the top.
+    expect(pickBucketDump(keys, null, null)?.dumpKey).toBe(
+      "database/healthlog-20261023T013000Z-daily.dump.enc",
+    );
+    // Named by its full key: on purpose.
+    expect(
+      pickBucketDump(
+        keys,
+        "database/staging.example/healthlog-20261025T013000Z-daily.dump.enc",
+        "prod.example",
+      )?.manifestKey,
+    ).toBe(
+      "database/staging.example/healthlog-20261025T013000Z-daily.manifest.json.enc",
+    );
+    expect(listBucketDumps(keys).map((d) => d.instance)).toEqual([
+      "staging.example",
+      "prod.example",
+      null,
+    ]);
+  });
+});
+
+describe("a restore's decrypted scratch", () => {
+  it("is swept by retention once it is older than a restore can run", async () => {
+    const dir = tempDir();
+    const { mkdirSync, existsSync } = await import("node:fs");
+    for (const name of [".restore-old", ".download-old", ".restore-new"]) {
+      mkdirSync(join(dir, name));
+      writeFileSync(join(dir, name, "x.dump"), "PGDMP");
+    }
+    const old = new Date(Date.now() - 13 * 60 * 60 * 1000);
+    utimesSync(join(dir, ".restore-old"), old, old);
+    utimesSync(join(dir, ".download-old"), old, old);
+    mkdirSync(join(dir, "not-ours"));
+    utimesSync(join(dir, "not-ours"), old, old);
+
+    await applyRetention(dir, { dailyKeep: 7 });
+    expect(existsSync(join(dir, ".restore-old"))).toBe(false);
+    expect(existsSync(join(dir, ".download-old"))).toBe(false);
+    expect(existsSync(join(dir, ".restore-new"))).toBe(true);
+    expect(existsSync(join(dir, "not-ours"))).toBe(true);
+  });
+});
+
 describe("whether the dump directory survives the container", () => {
   // /proc/self/mountinfo of the app container under the bundled compose
   // file, trimmed to the lines that matter, with and without the volume.

src/lib/database-backup/dump-core.mjs

@@ -1201,6 +1201,51 @@ export function selectForDeletion(sets, options) {
   return remove;
 }
 
+/**
+ * The directories a restore opens an off-host copy into (`.restore-…`) and
+ * downloads one into (`.download-…`). What is in them is a whole database,
+ * decrypted. A restore removes its own when it ends; one that was killed
+ * (out of memory, `docker kill`) cannot, so they are swept: by the next
+ * restore before it starts, and by the nightly retention once they are older
+ * than a restore could still be running.
+ */
+export const SCRATCH_DIR_PATTERN = /^\.(?:restore|download)-/;
+
+/**
+ * Remove the restore scratch directories in `dir` older than `olderThanMs`.
+ * Answers the paths removed. Never throws.
+ *
+ * @param {string} dir
+ * @param {{ olderThanMs: number; now?: number }} options
+ */
+export async function removeStaleScratch(dir, options) {
+  const now = options.now ?? Date.now();
+  /** @type {string[]} */
+  const removed = [];
+  let names = [];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return removed;
+  }
+  for (const name of names) {
+    if (!SCRATCH_DIR_PATTERN.test(name)) continue;
+    const path = join(dir, name);
+    try {
+      const info = statSync(path);
+      if (!info.isDirectory()) continue;
+      if (options.olderThanMs > 0 && now - info.mtimeMs < options.olderThanMs) {
+        continue;
+      }
+      await rm(path, { recursive: true, force: true });
+      removed.push(path);
+    } catch {
+      // Gone already, or not ours to remove.
+    }
+  }
+  return removed;
+}
+
 /**
  * Apply retention to `dir`. Answers the paths removed.
  *
@@ -1208,15 +1253,20 @@ export function selectForDeletion(sets, options) {
  * @param {{ dailyKeep?: number; now?: number }} [options]
  */
 export async function applyRetention(dir, options = {}) {
+  const now = options.now ?? Date.now();
   const doomed = selectForDeletion(listDumpSets(dir), {
     keep: {
       daily: options.dailyKeep ?? DEFAULT_DAILY_KEEP,
       "pre-migration": PRE_MIGRATION_KEEP,
     },
-    now: options.now ?? Date.now(),
+    now,
   });
   for (const path of doomed) await rm(path, { force: true });
-  return doomed;
+  const scratch = await removeStaleScratch(dir, {
+    olderThanMs: STALE_LEFTOVER_MS,
+    now,
+  });
+  return [...doomed, ...scratch];
 }
 
 /** @param {string} path */
@@ -1227,14 +1277,65 @@ export async function readManifestFile(path) {
 // ── Off-host copies ────────────────────────────────────────────────────────
 
 /**
- * The bucket key of an off-host database object. It is also the associated
- * data its encryption is bound to, so a copy cannot be passed off under
- * another name.
+ * Which instance wrote a dump into a shared bucket: the host of its public
+ * URL (`APP_URL`, else `NEXT_PUBLIC_APP_URL`), lowercased, with anything but
+ * letters, digits, dots and dashes as a dash. Null when neither is set.
+ *
+ * A bucket may hold the copies of more than one instance; nothing forbids it,
+ * and it was safe for the per-account objects. Database dumps were named by
+ * their time alone, so two instances dumping at the same second (the default
+ * schedule is the same everywhere) overwrote each other, and `latest
+ * --from-bucket` restored whichever instance's dump was newest.
+ *
+ * @param {Env} env
+ * @returns {string | null}
+ */
+export function instanceIdFromEnv(env = process.env) {
+  for (const name of ["APP_URL", "NEXT_PUBLIC_APP_URL"]) {
+    const raw = (env[name] ?? "").trim();
+    if (!raw) continue;
+    try {
+      const host = new URL(raw).host.toLowerCase().replace(/[^a-z0-9.-]/g, "-");
+      if (host) return host;
+    } catch {
+      // Not a URL; try the next one.
+    }
+  }
+  return null;
+}
+
+/**
+ * The bucket key of an off-host database object: under the instance's own
+ * folder (`database/<instance>/…`), or at the top of `database/` for an
+ * instance with no public URL set. It is also the associated data its
+ * encryption is bound to, so a copy cannot be passed off under another name,
+ * or as another instance's.
  *
  * @param {string} fileName e.g. `healthlog-…-daily.dump.enc`
+ * @param {string | null} [instanceId]
  */
-export function offhostObjectKey(fileName) {
-  return `${OFFHOST_DATABASE_PREFIX}${fileName}`;
+export function offhostObjectKey(fileName, instanceId = null) {
+  return instanceId
+    ? `${OFFHOST_DATABASE_PREFIX}${instanceId}/${fileName}`
+    : `${OFFHOST_DATABASE_PREFIX}${fileName}`;
+}
+
+/**
+ * A database object key read back: the instance folder it is in (null at the
+ * top of `database/`) and its file name, or null for anything else.
+ *
+ * @param {string} key
+ * @returns {{ instance: string | null; fileName: string } | null}
+ */
+export function parseOffhostObjectKey(key) {
+  if (!key.startsWith(OFFHOST_DATABASE_PREFIX)) return null;
+  const rest = key.slice(OFFHOST_DATABASE_PREFIX.length);
+  const parts = rest.split("/");
+  if (parts.length === 1) return { instance: null, fileName: parts[0] };
+  if (parts.length === 2 && parts[0]) {
+    return { instance: parts[0], fileName: parts[1] };
+  }
+  return null;
 }
 
 const ENVELOPE_MAGIC = "HLBK";
@@ -1364,45 +1465,76 @@ export function bucketConfigFromEnv(env = process.env) {
 }
 
 /**
- * The newest database dump in a bucket listing whose manifest is there too,
- * or the one `name` asks for. Pure. `name` may carry the `.enc` suffix or
- * not. Answers both object keys, or null.
+ * Every database dump in a bucket listing whose manifest is there too, with
+ * the instance that wrote it, newest first. Pure.
  *
  * @param {string[]} keys every key under the `database/` prefix
- * @param {string | null} name
- * @returns {{ dumpKey: string; manifestKey: string } | null}
+ * @returns {Array<{ instance: string | null; base: string; timestamp: number; dumpKey: string; manifestKey: string }>}
  */
-export function pickBucketDump(keys, name = null) {
+export function listBucketDumps(keys) {
   const present = new Set(keys);
-  const wanted = name
-    ? name.replace(/\.enc$/, "").replace(/^database\//, "")
-    : null;
-  const candidates = keys
-    .map((key) => ({
-      key,
-      parsed: parseDumpFileName(key.slice(OFFHOST_DATABASE_PREFIX.length)),
-    }))
+  return keys
+    .map((key) => {
+      const where = parseOffhostObjectKey(key);
+      const parsed = where ? parseDumpFileName(where.fileName) : null;
+      return where && parsed ? { where, parsed } : null;
+    })
     .filter(
-      ({ key, parsed }) =>
-        key.startsWith(OFFHOST_DATABASE_PREFIX) &&
-        parsed !== null &&
-        parsed.role === "dump" &&
-        parsed.variant === "encrypted" &&
-        (wanted === null || `${parsed.base}.dump` === wanted),
+      (c) =>
+        c !== null &&
+        c.parsed.role === "dump" &&
+        c.parsed.variant === "encrypted",
     )
-    .map(({ parsed }) => {
-      const p = /** @type {ParsedDumpName} */ (parsed);
+    .map((c) => {
+      const { where, parsed } =
+        /** @type {{ where: { instance: string | null }; parsed: ParsedDumpName }} */ (
+          c
+        );
       return {
-        timestamp: p.timestamp.getTime(),
-        dumpKey: offhostObjectKey(`${p.base}.dump.enc`),
-        manifestKey: offhostObjectKey(`${p.base}.manifest.json.enc`),
+        instance: where.instance,
+        base: parsed.base,
+        timestamp: parsed.timestamp.getTime(),
+        dumpKey: offhostObjectKey(`${parsed.base}.dump.enc`, where.instance),
+        manifestKey: offhostObjectKey(
+          `${parsed.base}.manifest.json.enc`,
+          where.instance,
+        ),
       };
     })
     .filter((c) => present.has(c.manifestKey))
     .sort((a, b) => b.timestamp - a.timestamp);
-  return candidates[0]
-    ? { dumpKey: candidates[0].dumpKey, manifestKey: candidates[0].manifestKey }
-    : null;
+}
+
+/**
+ * The newest database dump of `instance` in a bucket listing whose manifest
+ * is there too, or the one `name` asks for. Pure. `name` may carry the
+ * `.enc` suffix or not; a name with a folder
+ * (`database/<instance>/healthlog-….dump.enc` or `<instance>/…`) names that
+ * instance's dump, which is how an operator restores another instance's dump
+ * on purpose. Answers both object keys, or null.
+ *
+ * @param {string[]} keys every key under the `database/` prefix
+ * @param {string | null} name
+ * @param {string | null} [instance] the instance whose dumps count; null for
+ *   the dumps at the top of `database/`
+ * @returns {{ dumpKey: string; manifestKey: string } | null}
+ */
+export function pickBucketDump(keys, name = null, instance = null) {
+  let wanted = null;
+  let folder = instance;
+  if (name) {
+    const path = name
+      .replace(/\.enc$/, "")
+      .replace(new RegExp(`^${OFFHOST_DATABASE_PREFIX}`), "");
+    const slash = path.lastIndexOf("/");
+    if (slash >= 0) folder = path.slice(0, slash) || null;
+    wanted = path.slice(slash + 1);
+  }
+  const pick = listBucketDumps(keys).find(
+    (c) =>
+      c.instance === folder && (wanted === null || `${c.base}.dump` === wanted),
+  );
+  return pick ? { dumpKey: pick.dumpKey, manifestKey: pick.manifestKey } : null;
 }
 
 /**

src/lib/database-backup/run.ts

@@ -41,6 +41,7 @@ import {
   backupDirectoryStatus,
   createDatabaseDump,
   databaseBackupEnabled,
+  instanceIdFromEnv,
   offhostObjectKey,
   resolveBackupDir,
   resolveDailyKeep,
@@ -93,10 +94,11 @@ async function uploadFile(
   s3: S3Like,
   key: Buffer,
   path: string,
+  instanceId: string | null,
 ): Promise<number> {
   return uploadEncryptedBackup(
     s3,
-    offhostObjectKey(`${basename(path)}.enc`),
+    offhostObjectKey(`${basename(path)}.enc`, instanceId),
     key,
     async (write) => {
       for await (const chunk of createReadStream(path)) {
@@ -196,8 +198,21 @@ export async function runDatabaseBackup(
           const s3 = options.s3 ?? (await getS3Client(offhostConfig));
           // The manifest goes up first: a dump object in the bucket always
           // has the manifest a restore checks it against.
-          await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath);
-          await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath);
+          // Under this instance's own folder: a bucket may hold the copies
+          // of more than one instance (`instanceIdFromEnv`).
+          const instanceId = instanceIdFromEnv(env);
+          await uploadFile(
+            s3,
+            offhostConfig.encryptionKey,
+            dump.manifestPath,
+            instanceId,
+          );
+          await uploadFile(
+            s3,
+            offhostConfig.encryptionKey,
+            dump.dumpPath,
+            instanceId,
+          );
           report.offhost = "uploaded";
         } catch (err) {
           uploadError = `off-host upload failed: ${reason(err)}`;