fix(recovery): the database dumps count in the keys the backups still need

The recovery kit, "Keys the backups still need" and safeToDropRetiredKeys
read only the stored copies and the off-host account objects. After a
rotation the page called the old key safe to drop while every dump taken
before it, the pre-migration dumps that are the way back from an update
among them, could only be restored with that key: db-restore refuses a dump
whose active key is missing.

A dump now counts for the key its rows were written under, the manifest's
active key: a dump on this server while it is kept (read from the manifests),
an off-host one until the bucket's rule for database/ expires it, or
indefinitely when that rule cannot be read. The nightly run records the key
on its ledger row (migration 0489, database_backup_runs.active_key_id). The
encryption view and the kit list the dumps per key, in every locale, and the
rotation and backup runbooks say so.

src/components/admin/__tests__/encryption-legacy-badge.test.tsx

@@ -44,6 +44,7 @@ function statusWith(configuredKeyCount: number) {
       stored: [],
       unrecorded: { copies: 0, oldestAt: null },
       offhost: [],
+      dumps: [],
       offhostExpirationDays: null,
       retiredKeysStillNeeded: [],
     },

src/components/admin/__tests__/recovery-kit-card.test.tsx

@@ -40,6 +40,8 @@ const need = (over: Partial<KitKey["need"]> = {}): KitKey["need"] => ({
   configured: true,
   storedCopies: 0,
   offhostUntil: null,
+  localDumps: 0,
+  offhostDumpsUntil: null,
   ...over,
 });
 

src/components/admin/encryption-section.tsx

@@ -88,6 +88,11 @@ interface EncryptionStatus {
       lastWrittenAt: string;
       neededUntil: string | null;
     }>;
+    dumps: Array<{
+      keyId: string;
+      local: number;
+      offhost: { lastWrittenAt: string; neededUntil: string | null } | null;
+    }>;
     offhostExpirationDays: number | null;
     retiredKeysStillNeeded: string[];
   };
@@ -306,9 +311,39 @@ export function EncryptionSection() {
                   })}
             </li>
           ))}
+          {s.backups.dumps.map((row) => (
+            <li key={`dump-${row.keyId}`}>
+              {[
+                row.local > 0
+                  ? t("admin.section.encryption.backupsDumpLocalLine", {
+                      key: row.keyId,
+                      count: row.local,
+                    })
+                  : null,
+                row.offhost
+                  ? row.offhost.neededUntil
+                    ? t("admin.section.encryption.backupsDumpOffhostLine", {
+                        key: row.keyId,
+                        when: fmt.dateTime(row.offhost.lastWrittenAt),
+                        until: fmt.dateTime(row.offhost.neededUntil),
+                      })
+                    : t(
+                        "admin.section.encryption.backupsDumpOffhostLineUnknown",
+                        {
+                          key: row.keyId,
+                          when: fmt.dateTime(row.offhost.lastWrittenAt),
+                        },
+                      )
+                  : null,
+              ]
+                .filter((line): line is string => line !== null)
+                .join(" ")}
+            </li>
+          ))}
           {s.backups.stored.length === 0 &&
           s.backups.unrecorded.copies === 0 &&
-          s.backups.offhost.length === 0 ? (
+          s.backups.offhost.length === 0 &&
+          s.backups.dumps.length === 0 ? (
             <li className="text-muted-foreground">
               {t("admin.section.encryption.backupsNone")}
             </li>

src/components/admin/recovery-kit-card.tsx

@@ -138,6 +138,20 @@ function KeyNeeds({ k }: { k: KitKey }) {
         }),
       );
     }
+    if (k.need.localDumps > 0) {
+      lines.push(
+        t("admin.recoveryKit.need.localDumps", { count: k.need.localDumps }),
+      );
+    }
+    if (k.need.offhostDumpsUntil === "unknown") {
+      lines.push(t("admin.recoveryKit.need.offhostDumpsUnknown"));
+    } else if (k.need.offhostDumpsUntil) {
+      lines.push(
+        t("admin.recoveryKit.need.offhostDumpsUntil", {
+          date: fmt.date(new Date(k.need.offhostDumpsUntil)),
+        }),
+      );
+    }
   }
   lines.push(t("admin.recoveryKit.source", { variable: k.source }));
   return <p className="text-muted-foreground text-xs">{lines.join(" ")}</p>;

src/lib/database-backup/run.ts

@@ -156,6 +156,7 @@ export async function runDatabaseBackup(
   });
 
   let scratch: string | null = null;
+  let activeKeyId: string | null = null;
   try {
     if (!databaseBackupEnabled(env)) {
       report.status = "skipped";
@@ -182,6 +183,7 @@ export async function runDatabaseBackup(
         deadline: options.deadline,
       });
       if (keepLocal) report.local = "written";
+      activeKeyId = dump.manifest.keys.encryption.activeKeyId;
       report.fileName = basename(dump.dumpPath);
       report.dumpBytes = dump.manifest.dump.bytes;
 
@@ -233,6 +235,7 @@ export async function runDatabaseBackup(
       localResult: report.local,
       offhostResult: report.offhost,
       error: report.error,
+      activeKeyId,
     },
   });
   await prisma.databaseBackupRun.deleteMany({

src/lib/recovery/__tests__/kit-rules.test.ts

@@ -31,6 +31,7 @@ const NO_NEEDS: BackupKeyNeeds = {
   stored: [],
   unrecorded: { copies: 0, oldestAt: null },
   offhost: [],
+  dumps: [],
   offhostExpirationDays: null,
   retiredKeysStillNeeded: [],
 };
@@ -76,6 +77,43 @@ describe("which keys the kit lists", () => {
     });
   });
 
+  // A dump is restored only with the key its rows were written under; with
+  // it out of the kit, a rotation read as finished while every dump from
+  // before it, the pre-migration ones among them, still needed the old key.
+  it("keeps a retired key while a database dump written under it exists", () => {
+    const local: BackupKeyNeeds = {
+      ...NO_NEEDS,
+      dumps: [{ keyId: "v1", local: 2, offhost: null }],
+    };
+    expect(neededKeyIds(local, NOW)).toEqual(new Set(["v1"]));
+    const [v1] = appKeys(["v2"], "v2", local);
+    expect(v1).toMatchObject({
+      id: "v1",
+      problem: "not_configured",
+      need: { localDumps: 2, offhostDumpsUntil: null },
+    });
+
+    const offhost = (neededUntil: string | null): BackupKeyNeeds => ({
+      ...NO_NEEDS,
+      dumps: [
+        {
+          keyId: "v1",
+          local: 0,
+          offhost: { lastWrittenAt: NOW.toISOString(), neededUntil },
+        },
+      ],
+    });
+    const later = new Date(NOW.getTime() + 3 * DAY).toISOString();
+    expect(neededKeyIds(offhost(later), NOW).has("v1")).toBe(true);
+    expect(appKeys(["v1", "v2"], "v2", offhost(later))[0].need).toMatchObject({
+      offhostDumpsUntil: later,
+    });
+    // Unknown retention keeps it; an expiry that has passed lets it go.
+    expect(neededKeyIds(offhost(null), NOW).has("v1")).toBe(true);
+    const earlier = new Date(NOW.getTime() - DAY).toISOString();
+    expect(neededKeyIds(offhost(earlier), NOW).has("v1")).toBe(false);
+  });
+
   it("keeps an off-host key while objects inside the retention need it", () => {
     const row = (neededUntil: string | null) => ({
       ...NO_NEEDS,

src/lib/recovery/backup-key-needs.ts

@@ -10,10 +10,25 @@
  * bucket the last night an object needing it went in, plus how long the
  * bucket's lifecycle rule keeps objects after that.
  *
+ * The database dumps count too. A dump is restored only with the key its rows
+ * were written under (its manifest's active key; `db-restore` refuses without
+ * it), so that key is needed for as long as the dump exists: a dump kept on
+ * this server until retention removes it, an off-host one until the bucket's
+ * rule for `database/` expires it. Without them, a rotation read as safe to
+ * finish a week later while every dump from before it, the pre-migration
+ * dumps that are the way back from an update among them, still needed the
+ * retired key.
+ *
  * Two readers ask it: the admin encryption view, which shows the table, and
  * the recovery kit, which lists every key a restore needs. One definition, so
  * the kit cannot drop a key the encryption view still calls needed.
  */
+import {
+  isGoodSet,
+  listDumpSets,
+  readManifestFile,
+  resolveBackupDir,
+} from "@/lib/database-backup/dump-core.mjs";
 import { prisma } from "@/lib/db";
 import {
   offhostBackupConfigured,
@@ -32,50 +47,109 @@ export interface BackupKeyNeeds {
     /** When the last object needing this key expires; null when unknown. */
     neededUntil: string | null;
   }>;
+  /**
+   * Per key id, the database dumps written under it: how many are kept on
+   * this server, and the newest one that went off-host with when the bucket
+   * expires it (`neededUntil` null when that cannot be read).
+   */
+  dumps: Array<{
+    keyId: string;
+    local: number;
+    offhost: { lastWrittenAt: string; neededUntil: string | null } | null;
+  }>;
   /** The bucket's expiry, when it has one the credential may read. */
   offhostExpirationDays: number | null;
   /** Retired key ids some backup still needs (never the active one). */
   retiredKeysStillNeeded: string[];
 }
 
+/**
+ * The active key of every good dump in `dir`, one entry per dump. A manifest
+ * that cannot be read names no key; `db-restore` refuses such a dump anyway.
+ */
+export async function localDumpKeyIds(dir: string): Promise<string[]> {
+  const keys: string[] = [];
+  for (const set of listDumpSets(dir).filter(isGoodSet)) {
+    try {
+      const manifest = await readManifestFile(set.manifest as string);
+      const id = manifest.keys.encryption.activeKeyId;
+      if (id) keys.push(id);
+    } catch {
+      // Unreadable: no key to name.
+    }
+  }
+  return keys;
+}
+
 export async function readBackupKeyNeeds(
   activeKeyId: string,
   now: Date,
+  options: { dumpDir?: string } = {},
 ): Promise<BackupKeyNeeds> {
-  const [stored, unrecorded, offhostRows] = await Promise.all([
-    prisma.$queryRaw<
-      Array<{ key_id: string; copies: number; oldest_at: Date }>
-    >`
+  const [stored, unrecorded, offhostRows, dumpUploads, localDumpKeys] =
+    await Promise.all([
+      prisma.$queryRaw<
+        Array<{ key_id: string; copies: number; oldest_at: Date }>
+      >`
       SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at
       FROM data_backups, unnest(inner_key_ids) AS k
       WHERE inner_key_ids_recorded
       GROUP BY k
       ORDER BY k
     `,
-    prisma.dataBackup.aggregate({
-      where: { innerKeyIdsRecorded: false },
-      _count: { _all: true },
-      _min: { createdAt: true },
-    }),
-    prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
-  ]);
+      prisma.dataBackup.aggregate({
+        where: { innerKeyIdsRecorded: false },
+        _count: { _all: true },
+        _min: { createdAt: true },
+      }),
+      prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
+      prisma.databaseBackupRun.groupBy({
+        by: ["activeKeyId"],
+        where: { offhostResult: "uploaded", activeKeyId: { not: null } },
+        _max: { startedAt: true },
+      }),
+      localDumpKeyIds(options.dumpDir ?? resolveBackupDir()),
+    ]);
   const lifecycle =
-    offhostRows.length > 0 && offhostBackupConfigured()
+    (offhostRows.length > 0 || dumpUploads.length > 0) &&
+    offhostBackupConfigured()
       ? await probeOffhostLifecycle()
       : null;
   const expirationDays =
     lifecycle?.state === "configured" ? lifecycle.expirationDays : null;
+  const dumpExpirationDays =
+    lifecycle?.database.state === "configured"
+      ? lifecycle.database.expirationDays
+      : null;
+  const expiresAfter = (writtenAt: Date, days: number | null) =>
+    days === null
+      ? null
+      : new Date(writtenAt.getTime() + (days + 1) * 86_400_000).toISOString();
+
+  const dumpKeyIds = new Set([
+    ...localDumpKeys,
+    ...dumpUploads.map((row) => row.activeKeyId as string),
+  ]);
+  const dumps = [...dumpKeyIds].sort().map((keyId) => {
+    const upload = dumpUploads.find((row) => row.activeKeyId === keyId);
+    const writtenAt = upload?._max.startedAt ?? null;
+    return {
+      keyId,
+      local: localDumpKeys.filter((id) => id === keyId).length,
+      offhost: writtenAt
+        ? {
+            lastWrittenAt: writtenAt.toISOString(),
+            neededUntil: expiresAfter(writtenAt, dumpExpirationDays),
+          }
+        : null,
+    };
+  });
 
   const offhost = offhostRows.map((row) => ({
     keyId: row.keyId,
     firstWrittenAt: row.firstWrittenAt.toISOString(),
     lastWrittenAt: row.lastWrittenAt.toISOString(),
-    neededUntil:
-      expirationDays === null
-        ? null
-        : new Date(
-            row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000,
-          ).toISOString(),
+    neededUntil: expiresAfter(row.lastWrittenAt, expirationDays),
   }));
 
   const needs: BackupKeyNeeds = {
@@ -89,6 +163,7 @@ export async function readBackupKeyNeeds(
       oldestAt: unrecorded._min.createdAt?.toISOString() ?? null,
     },
     offhost,
+    dumps,
     offhostExpirationDays: expirationDays,
     retiredKeysStillNeeded: [],
   };

src/lib/recovery/kit-rules.ts

@@ -82,6 +82,10 @@ export interface KitKeyNeed {
    * retention cannot be read, so they may need it indefinitely.
    */
   offhostUntil: string | null;
+  /** Database dumps on this server written under it. */
+  localDumps: number;
+  /** Off-host database dumps need it until this instant, or `"unknown"`. */
+  offhostDumpsUntil: string | null;
 }
 
 /** One key the kit lists, with its verdict. */
@@ -204,9 +208,18 @@ export function sameKit(
  */
 export function neededKeyIds(needs: BackupKeyNeeds, now: Date): Set<string> {
   const needed = new Set<string>();
+  const stillThere = (until: string | null) =>
+    until === null || Date.parse(until) > +now;
   for (const row of needs.stored) needed.add(row.keyId);
   for (const row of needs.offhost) {
-    if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) {
+    if (stillThere(row.neededUntil)) needed.add(row.keyId);
+  }
+  // A database dump needs the key its rows were written under while it
+  // exists: on this server until retention removes it, in the bucket until
+  // the rule for `database/` expires it.
+  for (const row of needs.dumps) {
+    if (row.local > 0) needed.add(row.keyId);
+    else if (row.offhost && stillThere(row.offhost.neededUntil)) {
       needed.add(row.keyId);
     }
   }
@@ -229,6 +242,7 @@ export function applicationKitKeys(args: {
   return [...ids].sort().map((id) => {
     const fingerprint = args.fingerprintOf(id);
     const offhost = args.needs.offhost.find((row) => row.keyId === id);
+    const dump = args.needs.dumps.find((row) => row.keyId === id);
     return {
       kind: "application" as const,
       id,
@@ -241,6 +255,13 @@ export function applicationKitKeys(args: {
           args.needs.stored.find((row) => row.keyId === id)?.copies ?? 0,
         offhostUntil:
           offhost && needed.has(id) ? (offhost.neededUntil ?? "unknown") : null,
+        localDumps: dump?.local ?? 0,
+        offhostDumpsUntil:
+          dump?.offhost &&
+          (dump.offhost.neededUntil === null ||
+            Date.parse(dump.offhost.neededUntil) > +args.now)
+            ? (dump.offhost.neededUntil ?? "unknown")
+            : null,
       },
       problem: fingerprint === null ? ("not_configured" as const) : null,
     };
@@ -248,7 +269,14 @@ export function applicationKitKeys(args: {
 }
 
 function noNeed(active: boolean): KitKeyNeed {
-  return { active, configured: true, storedCopies: 0, offhostUntil: null };
+  return {
+    active,
+    configured: true,
+    storedCopies: 0,
+    offhostUntil: null,
+    localDumps: 0,
+    offhostDumpsUntil: null,
+  };
 }
 
 /** Pure: the off-host keys, from the two variables as the operator set them. */