fix(db-backup): keep instances apart in a shared bucket, and no decrypted dump behind

Off-host dumps were named by their time alone. Two instances sharing a bucket
(nothing forbids it, and it was safe for the account objects) overwrote each
other's dump at the same second of the same default schedule, and
latest --from-bucket restored whichever instance's dump was newest. Each
instance now uploads under database/<host of APP_URL>/, the object key (and so
the encryption's associated data) carries it, and the restore takes only this
instance's dumps: it refuses another's and names the instances it found,
unless --instance=<host> or the dump's full key says which one on purpose.

A restore killed outright (out of memory, docker kill) left its .restore- or
.download- directory, a whole database decrypted, in the backups directory,
where nothing removed it. A restore now removes them before it starts and on
SIGINT and SIGTERM, and the nightly retention sweeps any older than twelve
hours.

scripts/__tests__/db-restore.test.ts

@@ -18,12 +18,21 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 const tablesInTarget = vi.fn(async (): Promise<string[]> => []);
 const runTool = vi.fn(async () => "");
 const end = vi.fn(async () => {});
+/** What the stand-in bucket lists under `database/`; downloads are refused. */
+const bucketKeys = vi.fn(async (): Promise<string[]> => []);
+const downloads = vi.fn(async (key: string) => {
+  throw new Error(`no download in this test: ${key}`);
+});
 
 vi.mock("../../src/lib/database-backup/dump-core.mjs", async (original) => ({
   ...(await original<Record<string, unknown>>()),
   connect: async () => ({ query: async () => ({ rows: [] }), end }),
   listUserTables: () => tablesInTarget(),
   runTool: (...args: unknown[]) => runTool(...(args as [])),
+  openBucket: async () => ({
+    list: () => bucketKeys(),
+    download: (key: string) => downloads(key),
+  }),
 }));
 
 import { buildManifest } from "../../src/lib/database-backup/dump-core.mjs";
@@ -153,6 +162,50 @@ describe("db-restore refuses before it writes", () => {
     expect(runTool).not.toHaveBeenCalled();
   });
 
+  // Two instances may share a bucket. `latest --from-bucket` took the newest
+  // dump in it, whichever instance wrote it.
+  it("does not take another instance's dump from a shared bucket unless told", async () => {
+    const bucketEnv = env({
+      APP_URL: "https://health.example.org",
+      BACKUP_S3_ENDPOINT: "https://bucket.example",
+      BACKUP_S3_BUCKET: "shared",
+      BACKUP_S3_ACCESS_KEY: "a",
+      BACKUP_S3_SECRET_KEY: "s",
+      BACKUP_ENCRYPTION_KEY: KEY_B,
+    });
+    const other =
+      "database/staging.example.org/healthlog-20261025T013000Z-daily";
+    bucketKeys.mockResolvedValue([
+      `${other}.dump.enc`,
+      `${other}.manifest.json.enc`,
+    ]);
+    expect(await main(["latest", "--from-bucket"], bucketEnv)).toBe(1);
+    expect(errors.join("\n")).toMatch(
+      /no database dump of this instance \(health\.example\.org\).*holds dumps of: staging\.example\.org.*--instance=/,
+    );
+    expect(downloads).not.toHaveBeenCalled();
+
+    // Told: the other instance's newest dump is fetched.
+    await main(
+      ["latest", "--from-bucket", "--instance=staging.example.org"],
+      bucketEnv,
+    );
+    expect(downloads).toHaveBeenCalledWith(`${other}.manifest.json.enc`);
+  });
+
+  it("removes a decrypted copy a killed restore left behind before it starts", async () => {
+    const { mkdirSync, existsSync } = await import("node:fs");
+    mkdirSync(join(dir, ".restore-abc123"));
+    writeFileSync(join(dir, ".restore-abc123", `${BASE}.dump`), "PGDMP");
+    mkdirSync(join(dir, ".download-def456"));
+    writeDump();
+    tablesInTarget.mockResolvedValue(["public.users"]);
+    await main(["latest"], env());
+    expect(existsSync(join(dir, ".restore-abc123"))).toBe(false);
+    expect(existsSync(join(dir, ".download-def456"))).toBe(false);
+    expect(lines.join("\n")).toMatch(/removed \.restore-abc123/);
+  });
+
   it("lists what is there", async () => {
     writeDump();
     expect(await main(["list"], env())).toBe(0);