Merge remote-tracking branch 'origin/fix/rotation-registry' into integration/post-1385

# Conflicts:
#	CHANGELOG.md

src/lib/crypto/__tests__/encrypted-column-writers.test.ts

@@ -0,0 +1,461 @@
+/**
+ * Derive the ciphertext-bearing columns FROM THE CODE, then assert the
+ * key-rotation registry covers every one of them.
+ *
+ * Why this exists next to `encrypted-columns.test.ts`. That guard scans
+ * `schema.prisma` for the `*Encrypted` suffix plus a hand-kept list of
+ * historically-named exceptions. The suffix is a naming convention, not a
+ * fact about the value, and the moment a column holding ciphertext is called
+ * something else the guard goes green while the column is unrotatable.
+ * `DataBackup.data` — the whole-account backup blob — sat outside rotation
+ * that way: encrypted on write, invisible to a name scan, and therefore
+ * reported as "zero rows remaining" by a run that never looked at it. The
+ * runbook then told the operator that zero meant the old key was safe to
+ * drop, which would have destroyed every backup in the deployment.
+ *
+ * How the derivation works, in three steps:
+ *
+ *   1. Start from the two primitives in `src/lib/crypto.ts` (`encrypt`,
+ *      `encryptBytes`) and close over every function that RETURNS a value
+ *      derived from one — `packBackupBlob`, `encryptNote`, the per-domain
+ *      `encrypt*ToBytes` codecs, and so on. Taint propagates only through
+ *      pure expressions; anything awaited or read back from Prisma is a
+ *      round-trip, not a ciphertext this code produced.
+ *   2. Walk every `prisma|tx|db.<model>.<write>(...)` call, descend through
+ *      the Prisma envelope keys (`data` / `create` / `update`, and relation
+ *      writes resolved against the schema) into the payload objects, and
+ *      record `<Model>.<field>` for each field whose value is a producer
+ *      call or a variable derived from one.
+ *   3. Assert the result is a subset of the rotation registry.
+ *
+ * Known limit, stated so it is not mistaken for coverage: a payload assembled
+ * somewhere other than the call site (`data: updates` where `updates` is built
+ * field by field) resolves to nothing here. Those columns are covered by the
+ * name scan in the sibling guard; the two signals are deliberately
+ * independent, and a column has to escape BOTH to go unregistered. The scan
+ * asserts a non-zero, anchored match count, so a matcher that quietly stops
+ * matching fails instead of passing.
+ */
+import { describe, it, expect } from "vitest";
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { ENCRYPTED_COLUMNS, encryptedColumnKey } from "../encrypted-columns";
+
+const ROOT = join(__dirname, "../../../..");
+const SRC = join(ROOT, "src");
+const SCHEMA_PATH = join(ROOT, "prisma", "schema.prisma");
+
+/** Prisma write methods whose argument carries a column payload. */
+const WRITE_METHODS =
+  "create|createMany|createManyAndReturn|update|updateMany|upsert";
+/** Prisma envelope keys whose value is a payload object, not a column. */
+const ENVELOPE = new Set(["data", "create", "update"]);
+/** Relation-write envelopes nested inside a payload value. */
+const RELATION_ENVELOPE = new Set([
+  "create",
+  "update",
+  "upsert",
+  "createMany",
+  "connectOrCreate",
+]);
+
+/**
+ * Columns this scan derives that are deliberately NOT in the rotation
+ * registry. Empty today. An entry here is a written decision with a reason,
+ * never a way to quiet the guard.
+ */
+const NOT_ROTATED: ReadonlySet<string> = new Set<string>();
+
+function sourceFiles(dir: string, out: string[] = []): string[] {
+  for (const entry of readdirSync(dir)) {
+    const p = join(dir, entry);
+    if (statSync(p).isDirectory()) {
+      if (entry === "generated" || entry === "__tests__") continue;
+      sourceFiles(p, out);
+    } else if (/\.tsx?$/.test(entry) && !/\.test\.tsx?$/.test(entry)) {
+      out.push(p);
+    }
+  }
+  return out;
+}
+
+/** Strip comments so prose about `encrypt()` never counts as a call. */
+function stripComments(src: string): string {
+  return src
+    .replace(/\/\*[\s\S]*?\*\//g, "")
+    .replace(/(^|[^:"'`\\])\/\/.*$/gm, "$1");
+}
+
+const callRe = (names: string[]) =>
+  new RegExp(`(?<![.\\w])(?:${names.join("|")})\\s*\\(`);
+/**
+ * Two rules for the same identifier, and the difference matters.
+ *
+ * Propagation is loose: a codec builds its result through `Buffer.from(ct)` and
+ * `new Uint8Array(encoded.byteLength)`, so a member read still carries the
+ * taint forward or the chain snaps one hop before the return.
+ *
+ * The final check at a payload field is strict — the identifier must be used
+ * WHOLE. `enc.expiresAt` off a tainted credentials object is a timestamp, not
+ * ciphertext, and counting it would file plaintext columns as encrypted.
+ */
+const tokenReLoose = (names: string[]) =>
+  new RegExp(`(?<![.\\w])(?:${names.join("|")})(?![\\w])`);
+const tokenRe = (names: string[]) =>
+  new RegExp(`(?<![.\\w])(?:${names.join("|")})(?![\\w.])`);
+
+/** Local identifiers whose value transitively derives from a producer call. */
+function taintedIdentifiers(
+  body: string,
+  producers: string[],
+  loose: boolean,
+): Set<string> {
+  const assigns = [
+    ...body.matchAll(/(?:const|let|var)\s+(\w+)[^;=\n]*=\s*([^;]*)/g),
+  ];
+  const tainted = new Set<string>();
+  for (let pass = 0; pass < 8; pass++) {
+    const before = tainted.size;
+    const producerCall = callRe(producers);
+    const strictToken = tainted.size ? tokenRe([...tainted]) : null;
+    const looseToken = tainted.size ? tokenReLoose([...tainted]) : null;
+    for (const [, name, init] of assigns) {
+      if (tainted.has(name)) continue;
+      if (producerCall.test(init)) {
+        tainted.add(name);
+        continue;
+      }
+      // Propagate through pure expressions only: a value awaited or read back
+      // from the database is a round-trip, not ciphertext this code made.
+      if (/\bawait\b|\b(?:prisma|tx|db)\./.test(init)) continue;
+      // A member read normally carries no taint (`creds.expiresAt` is a
+      // timestamp). The one exception is a buffer being CONSTRUCTED around a
+      // ciphertext — `new Uint8Array(encoded.byteLength)` then `.set(encoded)`
+      // is how every Bytes codec here builds its result, and reading the
+      // length off the ciphertext is the only link the initialiser shows.
+      const allowMemberRead = loose && /\bnew\s+[A-Z]|Buffer\./.test(init);
+      const token = allowMemberRead ? looseToken : strictToken;
+      if (token?.test(init)) tainted.add(name);
+    }
+    if (tainted.size === before) break;
+  }
+  return tainted;
+}
+
+interface Fn {
+  name: string;
+  body: string;
+}
+
+function declaredFunctions(src: string): Fn[] {
+  const out: Fn[] = [];
+  const heads = [
+    ...src.matchAll(
+      /(?:^|\n)\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*[(<]/g,
+    ),
+  ];
+  for (let i = 0; i < heads.length; i++) {
+    const end = i + 1 < heads.length ? heads[i + 1].index! : src.length;
+    out.push({ name: heads[i][1], body: src.slice(heads[i].index!, end) });
+  }
+  for (const m of src.matchAll(
+    /(?:const|let)\s+(\w+)\s*(?::[^=\n]+)?=\s*(?:async\s*)?\([^)]*\)\s*(?::[^=\n]+)?=>/g,
+  )) {
+    // Bound the arrow at its own body. A fixed-size window instead would read
+    // the code AFTER the arrow as part of it, and a tiny id-resolving helper
+    // declared above a Prisma write would inherit that write's ciphertext.
+    const arrow = m.index! + m[0].length;
+    const brace = src.slice(arrow).search(/\S/) + arrow;
+    const end =
+      src[brace] === "{" ? balanced(src, brace) : endOfValue(src, brace);
+    out.push({ name: m[1], body: src.slice(m.index!, end + 1) });
+  }
+  return out;
+}
+
+/** Fixpoint: every function that returns something derived from `encrypt`. */
+function ciphertextProducers(sources: string[]): Set<string> {
+  const funcs = sources.flatMap(declaredFunctions);
+  const producers = new Set(["encrypt", "encryptBytes"]);
+  for (let pass = 0; pass < 8; pass++) {
+    const before = producers.size;
+    for (const fn of funcs) {
+      if (producers.has(fn.name)) continue;
+      const names = [...producers];
+      const tainted = taintedIdentifiers(fn.body, names, true);
+      const producerCall = callRe(names);
+      const taintedToken = tainted.size ? tokenReLoose([...tainted]) : null;
+      const returnsCiphertext = [
+        ...fn.body.matchAll(/\breturn\b([^;]*)/g),
+      ].some((m) => producerCall.test(m[1]) || taintedToken?.test(m[1]));
+      if (returnsCiphertext) producers.add(fn.name);
+    }
+    if (producers.size === before) break;
+  }
+  return producers;
+}
+
+// ── brace / paren walking ────────────────────────────────────────────────
+
+/** Index of the top-level comma (or closer) ending the value starting at `i`. */
+function endOfValue(s: string, i: number): number {
+  let depth = 0;
+  while (i < s.length) {
+    const c = s[i];
+    if (c === '"' || c === "'" || c === "`") {
+      const quote = c;
+      i++;
+      while (i < s.length && s[i] !== quote) {
+        if (s[i] === "\\") i++;
+        i++;
+      }
+      i++;
+      continue;
+    }
+    if ("({[".includes(c)) depth++;
+    else if (")}]".includes(c)) {
+      if (depth === 0) return i;
+      depth--;
+    } else if (c === "," && depth === 0) return i;
+    i++;
+  }
+  return i;
+}
+
+/** Index of the closer matching the opener at `i`. */
+function balanced(s: string, i: number): number {
+  const open = s[i];
+  const close = { "(": ")", "{": "}", "[": "]" }[open]!;
+  let depth = 0;
+  for (; i < s.length; i++) {
+    const c = s[i];
+    if (c === '"' || c === "'" || c === "`") {
+      const quote = c;
+      i++;
+      while (i < s.length && s[i] !== quote) {
+        if (s[i] === "\\") i++;
+        i++;
+      }
+      continue;
+    }
+    if (c === open) depth++;
+    else if (c === close && --depth === 0) return i;
+  }
+  return s.length - 1;
+}
+
+/** Top-level `key: value` pairs of an object literal (braces included). */
+function objectPairs(obj: string): Array<[string, string, number]> {
+  const out: Array<[string, string, number]> = [];
+  let i = 1;
+  while (i < obj.length - 1) {
+    const m = /^[\s,]*(\w+|"[^"]*"|'[^']*')\s*:/.exec(obj.slice(i));
+    if (!m) {
+      const end = endOfValue(obj, i);
+      if (end <= i) break;
+      i = end + 1;
+      continue;
+    }
+    const start = i + m[0].length;
+    const end = endOfValue(obj, start);
+    out.push([m[1].replace(/^["']|["']$/g, ""), obj.slice(start, end), start]);
+    i = end + 1;
+  }
+  return out;
+}
+
+/** Object literals inside a payload value: `{…}`, `[{…}]`, or `map(() => ({…}))`. */
+function payloadObjects(value: string): Array<[string, number]> {
+  const trimmed = value.trim();
+  if (trimmed.length === 0) return [];
+  const at = value.indexOf(trimmed[0]);
+  if (trimmed.startsWith("{")) {
+    return [[value.slice(at, balanced(value, at) + 1), at]];
+  }
+  if (trimmed.startsWith("[")) {
+    const arr = value.slice(at, balanced(value, at) + 1);
+    const out: Array<[string, number]> = [];
+    let i = 1;
+    while (i < arr.length - 1) {
+      while (i < arr.length && /[\s,]/.test(arr[i])) i++;
+      if (arr[i] === "{") {
+        const end = balanced(arr, i);
+        out.push([arr.slice(i, end + 1), at + i]);
+        i = end + 1;
+      } else {
+        const end = endOfValue(arr, i);
+        if (end <= i) break;
+        i = end + 1;
+      }
+    }
+    return out;
+  }
+  const out: Array<[string, number]> = [];
+  for (const m of value.matchAll(/=>\s*\(?\s*\{/g)) {
+    const brace = value.indexOf("{", m.index!);
+    out.push([value.slice(brace, balanced(value, brace) + 1), brace]);
+  }
+  return out;
+}
+
+// ── schema ───────────────────────────────────────────────────────────────
+
+interface Schema {
+  models: Set<string>;
+  fieldType: Map<string, string>;
+}
+
+function parseSchema(): Schema {
+  const src = readFileSync(SCHEMA_PATH, "utf8");
+  const models = new Set<string>();
+  const fieldType = new Map<string, string>();
+  let model: string | null = null;
+  for (const raw of src.split("\n")) {
+    const line = raw.trim();
+    const head = /^model\s+(\w+)\s*\{/.exec(line);
+    if (head) {
+      model = head[1];
+      models.add(model);
+      continue;
+    }
+    if (line === "}") {
+      model = null;
+      continue;
+    }
+    if (!model || line === "" || line.startsWith("//")) continue;
+    const field = /^(\w+)\s+([A-Za-z]\w*)(\?|\[\])?/.exec(line);
+    if (field) fieldType.set(`${model}.${field[1]}`, field[2]);
+  }
+  return { models, fieldType };
+}
+
+// ── the scan ─────────────────────────────────────────────────────────────
+
+function scanCiphertextWrites(): Map<string, Set<string>> {
+  const files = sourceFiles(SRC).map(
+    (f) => [f, stripComments(readFileSync(f, "utf8"))] as const,
+  );
+  const producers = [...ciphertextProducers(files.map(([, s]) => s))];
+  const schema = parseSchema();
+  const found = new Map<string, Set<string>>();
+
+  for (const [path, src] of files) {
+    // The rotation machinery re-encrypts what other code wrote; it is not a
+    // writer of new columns, and scanning it would fold its own generic
+    // `data: { [col.field]: … }` into the result.
+    if (path.includes("/lib/crypto/")) continue;
+    const tainted = taintedIdentifiers(src, producers, false);
+    const producerCall = callRe(producers);
+    const taintedToken = tainted.size ? tokenRe([...tainted]) : null;
+    const isCiphertext = (expr: string) =>
+      producerCall.test(expr) || Boolean(taintedToken?.test(expr));
+    const rel = path.slice(ROOT.length + 1);
+    const lineAt = (idx: number) => src.slice(0, idx).split("\n").length;
+
+    const record = (model: string, field: string, idx: number) => {
+      const key = `${model}.${field}`;
+      if (!found.has(key)) found.set(key, new Set());
+      found.get(key)!.add(`${rel}:${lineAt(idx)}`);
+    };
+
+    const descend = (
+      model: string,
+      value: string,
+      base: number,
+      depth = 0,
+    ): void => {
+      if (depth > 4) return;
+      for (const [obj, objOffset] of payloadObjects(value)) {
+        for (const [field, expr, exprOffset] of objectPairs(obj)) {
+          const at = base + objOffset + exprOffset;
+          const type = schema.fieldType.get(`${model}.${field}`);
+          if (type && schema.models.has(type)) {
+            // Relation field: the nested payload belongs to the child model.
+            descend(type, expr, at, depth + 1);
+            continue;
+          }
+          if (!type) {
+            if (RELATION_ENVELOPE.has(field))
+              descend(model, expr, at, depth + 1);
+            continue;
+          }
+          if (isCiphertext(expr)) record(model, field, at);
+        }
+      }
+    };
+
+    const writes = new RegExp(
+      `\\b(?:prisma|tx|db)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(?:${WRITE_METHODS})\\s*\\(`,
+      "g",
+    );
+    for (const m of src.matchAll(writes)) {
+      const model = m[1][0].toUpperCase() + m[1].slice(1);
+      if (!schema.models.has(model)) continue;
+      const paren = m.index! + m[0].length - 1;
+      const arg = src.slice(paren, balanced(src, paren) + 1);
+      const brace = arg.indexOf("{");
+      if (brace < 0) continue;
+      const obj = arg.slice(brace, balanced(arg, brace) + 1);
+      for (const [key, value, offset] of objectPairs(obj)) {
+        if (!ENVELOPE.has(key)) continue;
+        descend(model, value, paren + brace + offset);
+      }
+    }
+  }
+  return found;
+}
+
+describe("ciphertext columns derived from the code", () => {
+  const producers = ciphertextProducers(
+    sourceFiles(SRC).map((f) => stripComments(readFileSync(f, "utf8"))),
+  );
+  const written = scanCiphertextWrites();
+
+  it("closes over the encryption wrappers, not just the two primitives", () => {
+    // A closure that collapsed back to the seeds would make the write scan
+    // blind to every column written through a codec helper.
+    expect(producers.size).toBeGreaterThan(10);
+    for (const wrapper of [
+      "packBackupBlob",
+      "encryptNote",
+      "encryptToBytes",
+      "encryptCachedBody",
+    ]) {
+      expect([...producers]).toContain(wrapper);
+    }
+  });
+
+  it("finds ciphertext writes, including the non-suffixed columns", () => {
+    // Anchored non-zero match count. An empty or collapsed result set is the
+    // failure mode this whole file exists to make impossible, so name the
+    // columns whose only signal is the write and not the column's name.
+    expect(written.size).toBeGreaterThan(30);
+    for (const anchor of [
+      "DataBackup.data",
+      "IdempotencyKey.responseBody",
+      "NotificationChannel.config",
+      "PushSubscription.p256dh",
+      "IntegrationStatus.lastError",
+      "WithingsConnection.accessToken",
+    ]) {
+      expect(
+        [...written.keys()],
+        `${anchor} is written with ciphertext but the scan did not see it`,
+      ).toContain(anchor);
+    }
+  });
+
+  it("registers every column the code writes ciphertext into", () => {
+    const registered = new Set(ENCRYPTED_COLUMNS.map(encryptedColumnKey));
+    const unregistered = [...written.keys()]
+      .filter((key) => !registered.has(key) && !NOT_ROTATED.has(key))
+      .sort()
+      .map((key) => `${key} (written at ${[...written.get(key)!][0]})`);
+    expect(
+      unregistered,
+      "columns written with AES-256-GCM ciphertext but absent from the " +
+        "key-rotation registry — dropping a retired key makes these rows " +
+        "permanently undecryptable",
+    ).toEqual([]);
+  });
+});

src/lib/crypto/__tests__/encrypted-columns.test.ts

@@ -127,21 +127,29 @@ describe("encrypted-column registry", () => {
   });
 
   it("covers EVERY encrypted column in prisma/schema.prisma", () => {
-    const schemaEncrypted = parseSchemaColumns()
+    const schema = parseSchemaColumns();
+    const schemaEncrypted = schema
       .filter(isEncryptedColumn)
       .map((c) => `${c.model}.${c.field}`)
       .sort();
+    const allSchemaColumns = new Set(
+      schema.map((c) => `${c.model}.${c.field}`),
+    );
     const registered = ENCRYPTED_COLUMNS.map(encryptedColumnKey).sort();
 
     // Any schema column the registry forgot is a rotation gap (data-loss
     // risk on key drop). Any registry column missing from the schema is a
     // stale entry. Both fail here.
+    //
+    // Staleness is checked against the WHOLE schema, not against the name
+    // scan: the registry legitimately carries columns this scan cannot see —
+    // `DataBackup.data` holds ciphertext under a name the convention does not
+    // reach — and the sibling guard `encrypted-column-writers.test.ts` is
+    // what finds those, by tracing the writers instead of the names.
     const missingFromRegistry = schemaEncrypted.filter(
       (k) => !registered.includes(k),
     );
-    const staleInRegistry = registered.filter(
-      (k) => !schemaEncrypted.includes(k),
-    );
+    const staleInRegistry = registered.filter((k) => !allSchemaColumns.has(k));
     expect(
       missingFromRegistry,
       "encrypted schema columns NOT wired into the rotation registry",
@@ -152,6 +160,42 @@ describe("encrypted-column registry", () => {
     ).toEqual([]);
   });
 
+  it("declares the primary key for every model not keyed on `id`", () => {
+    // The rotation walk selects, orders by and addresses rows through this
+    // field. Get it wrong and Prisma rejects the select at runtime, which
+    // abandons the rest of the pass — a rotation that stops early looks the
+    // same from the outside as one that had nothing to do.
+    const src = readFileSync(SCHEMA_PATH, "utf8");
+    const primaryKey = new Map<string, string>();
+    let model: string | null = null;
+    for (const rawLine of src.split("\n")) {
+      const line = rawLine.trim();
+      const head = /^model\s+(\w+)\s*\{/.exec(line);
+      if (head) {
+        model = head[1];
+        continue;
+      }
+      if (line === "}") {
+        model = null;
+        continue;
+      }
+      if (!model) continue;
+      const field = /^(\w+)\s+[A-Za-z]\w*\??\s.*@id\b/.exec(line);
+      if (field) primaryKey.set(model, field[1]);
+    }
+
+    const wrong = (ENCRYPTED_COLUMNS as EncryptedColumn[])
+      .filter((c) => (c.pkField ?? "id") !== primaryKey.get(c.model))
+      .map(
+        (c) =>
+          `${encryptedColumnKey(c)} (schema key: ${primaryKey.get(c.model) ?? "none"})`,
+      );
+    expect(
+      wrong,
+      "registry entries whose pkField does not match the model's @id column",
+    ).toEqual([]);
+  });
+
   it("is fully referenced by the key-rotation script", () => {
     const script = readFileSync(ROTATION_SCRIPT_PATH, "utf8");
     // Field names are NOT unique across models (e.g. `noteEncrypted` lives on

src/lib/crypto/__tests__/encryption-corpus.test.ts

@@ -45,9 +45,17 @@ function makeClient(seed: Record<string, Row[]>): {
     if (client[key]) continue;
     const model = col.model;
     client[key] = {
-      findMany: async ({ select }) => {
+      // `take` / `cursor` are honoured so the batched-column walk terminates
+      // here the way it does against Postgres.
+      findMany: async ({ select, take, cursor, skip }) => {
         const fields = Object.keys(select);
-        return store[model].map((row) => {
+        let rows = store[model];
+        if (cursor) {
+          const at = rows.findIndex((r) => r.id === cursor.id);
+          rows = at < 0 ? [] : rows.slice(at + (skip ?? 0));
+        }
+        if (take !== undefined) rows = rows.slice(0, take);
+        return rows.map((row) => {
           const out: Record<string, unknown> = {};
           for (const f of fields) out[f] = row[f];
           return out;
@@ -58,6 +66,10 @@ function makeClient(seed: Record<string, Row[]>): {
         if (row) Object.assign(row, data);
         return row;
       },
+      delete: async ({ where }) => {
+        const at = store[model].findIndex((r) => r.id === where.id);
+        return at < 0 ? null : store[model].splice(at, 1)[0];
+      },
     };
   }
   return { client, store };

src/lib/crypto/encrypted-columns.ts

@@ -24,6 +24,17 @@
  *     reads/writes the value directly.
  *   - "bytes"  — the ciphertext is stored as a `Bytes` column; the script
  *     goes through a UTF-8 Buffer round-trip (matching the persistence layer).
+ *
+ * The `*Encrypted` suffix is NOT what makes a column belong here. It is a
+ * naming convention that happens to hold for most of them, and treating it as
+ * the definition is how `DataBackup.data` — the whole-account backup blob —
+ * went unregistered until v1.38.6: encrypted on write, invisible to
+ * a registry keyed on names, and therefore reported as "zero rows remaining"
+ * by a rotation that never looked at it. An operator following the runbook
+ * would then have dropped the old key and lost every backup. The guard test
+ * `encrypted-column-writers.test.ts` now derives the ciphertext-bearing set
+ * from the Prisma write payloads instead, so a column earns its place here by
+ * what the code stores in it, not by what it is called.
  */
 
 export type EncryptedColumnKind = "string" | "bytes";
@@ -43,6 +54,30 @@ export interface EncryptedColumn {
    * multi-megabyte blobs, so an unbounded `findMany` would balloon memory.
    */
   readonly codecField?: string;
+  /**
+   * Walk this column in bounded id-cursor batches instead of one `findMany`.
+   * Set it on any column whose rows are blobs rather than short strings — a
+   * whole-account backup compresses to megabytes, and pulling every row of
+   * them into one array is how the weekly backup pass died before
+   * `packBackupBlob` existed. `codecField` implies batching.
+   */
+  readonly batched?: boolean;
+  /**
+   * The column is a cache, not a record. Rotation still re-encrypts it, but a
+   * row it cannot read is DELETED rather than counted as an error: the value
+   * is reproducible, and leaving an undecryptable row behind would hand the
+   * next reader a body it cannot parse. Never set this on anything a user
+   * would notice the loss of.
+   */
+  readonly disposable?: boolean;
+  /**
+   * The model's primary-key field, when it is not `id`. The rotation walk
+   * selects it, orders by it and addresses each row through it, so a model
+   * keyed on something else (`MoodContext.moodEntryId`) crashes the whole run
+   * without this — Prisma rejects the unknown `id` in the select, and every
+   * column after it in the pass is left un-rotated.
+   */
+  readonly pkField?: string;
 }
 
 /**
@@ -226,7 +261,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
   // v1.38 — the one free-text note on a mood entry's day context. Same codec
   // as the entry note beside it; one column for the whole context rather than
   // one per section, so there is one rotation entry rather than four.
-  { model: "MoodContext", field: "notesEncrypted", kind: "bytes" },
+  {
+    model: "MoodContext",
+    field: "notesEncrypted",
+    kind: "bytes",
+    pkField: "moodEntryId",
+  },
 
   // ───── v1.25 medication free-text notes (Bytes columns) ─────
   // Side-effect log note, dose-change titration note, and inventory-item
@@ -318,6 +358,34 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
   // codec `DocumentContentIndex.textEncrypted` uses. A scanned medical preview
   // is PHI; rotation re-encrypts it on the standard Bytes walk.
   { model: "DocumentThumbnail", field: "thumbnailEncrypted", kind: "bytes" },
+
+  // ───── Whole-account backup blob (String, batched) ─────
+  // The weekly disaster-recovery snapshot and every manually uploaded pack.
+  // Named `data`, so the `*Encrypted` scan never saw it; the value is
+  // `packBackupBlob()` output, which is AES-256-GCM ciphertext like every
+  // other entry here. It is also the single worst column to miss: a backup
+  // that cannot be decrypted is the copy an operator reaches for precisely
+  // when nothing else works.
+  //
+  // Rotation re-encrypts the ciphertext WITHOUT touching the plaintext, so it
+  // is blind to the envelope inside: today a row is either the plain backup
+  // JSON or the `HLZ1:`-prefixed gzip form, and any further shape a future
+  // writer introduces rotates unchanged for the same reason. Batched, because
+  // a single row is megabytes.
+  { model: "DataBackup", field: "data", kind: "string", batched: true },
+
+  // ───── Idempotent-replay response cache (String, disposable) ─────
+  // The cached response body, encrypted because the PHI-returning creates echo
+  // their own decrypted DTO. Rows live 24 hours. It is registered so a key
+  // drop cannot leave a reader holding an undecryptable body, and marked
+  // disposable so an unreadable row is dropped instead of failing the run —
+  // the cache is an accelerator, and a miss only costs a re-run.
+  {
+    model: "IdempotencyKey",
+    field: "responseBody",
+    kind: "string",
+    disposable: true,
+  },
 ] as const;
 
 /** Stable `Model.field` key for a registry entry. */

src/lib/crypto/encryption-corpus.ts

@@ -47,9 +47,10 @@ import {
 export const LEGACY_BUCKET = "legacy";
 
 /**
- * Batch size for codec-dispatched blob columns (`codecField` present). The
- * document vault's rows are up to cap-sized ciphertexts, so the walk is
- * id-cursor paginated — at most this many blobs are in memory at once.
+ * Batch size for blob columns (`codecField` or `batched` set). The document
+ * vault's rows are up to cap-sized ciphertexts and a backup row is the whole
+ * account compressed, so the walk is id-cursor paginated — at most this many
+ * blobs are in memory at once.
  */
 export const BLOB_ROTATION_BATCH_SIZE = 25;
 
@@ -63,9 +64,11 @@ interface ColumnDelegate {
     skip?: number;
   }) => Promise<Array<Record<string, unknown>>>;
   update: (args: {
-    where: { id: string };
+    where: Record<string, string>;
     data: Record<string, unknown>;
   }) => Promise<unknown>;
+  /** Only ever called for a `disposable` column's unreadable rows. */
+  delete: (args: { where: Record<string, string> }) => Promise<unknown>;
 }
 
 /** The subset of the Prisma client we touch: one delegate per model. */
@@ -149,44 +152,75 @@ function reencryptBlob(value: Uint8Array, codec: string): Uint8Array {
 }
 
 /**
- * Walk a codec-dispatched blob column in bounded id-cursor batches, invoking
- * `onRow` per non-empty row. At most `BLOB_ROTATION_BATCH_SIZE` blobs are in
- * memory per step; an interrupted run resumes safely on re-invocation because
- * processing is idempotent (already-active rows are skipped by the callers).
+ * Walk a blob column in bounded id-cursor batches, invoking `onRow` per
+ * non-empty row. At most `BLOB_ROTATION_BATCH_SIZE` rows are in memory per
+ * step; an interrupted run resumes safely on re-invocation because processing
+ * is idempotent (already-active rows are skipped by the callers).
+ *
+ * `codec` is the row's own codec label for a codec-dispatched column and null
+ * for a plain `batched` one — a backup blob has one layout, just a large one.
  */
 async function walkBlobColumn(
   delegate: ColumnDelegate,
   col: EncryptedColumn,
   onRow: (row: {
     id: string;
-    value: Uint8Array;
-    codec: string;
+    value: unknown;
+    codec: string | null;
   }) => Promise<void> | void,
 ): Promise<void> {
-  const codecField = col.codecField!;
+  const codecField = col.codecField;
+  const pk = pkField(col);
   let cursor: string | null = null;
   for (;;) {
     const rows = await delegate.findMany({
-      select: { id: true, [col.field]: true, [codecField]: true },
-      orderBy: { id: "asc" },
+      select: {
+        [pk]: true,
+        [col.field]: true,
+        ...(codecField ? { [codecField]: true } : {}),
+      },
+      orderBy: { [pk]: "asc" },
       take: BLOB_ROTATION_BATCH_SIZE,
-      ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
+      ...(cursor ? { cursor: { [pk]: cursor }, skip: 1 } : {}),
     });
     if (rows.length === 0) break;
     for (const row of rows) {
-      const value = row[col.field] as Uint8Array | null;
-      if (!value || value.byteLength === 0) continue;
+      const value = row[col.field];
+      if (value == null) continue;
+      if (value instanceof Uint8Array && value.byteLength === 0) continue;
+      if (typeof value === "string" && value.length === 0) continue;
       await onRow({
-        id: row.id as string,
+        id: row[pk] as string,
         value,
-        codec: String(row[codecField] ?? ""),
+        codec: codecField ? String(row[codecField] ?? "") : null,
       });
     }
-    cursor = rows[rows.length - 1]!.id as string;
+    cursor = rows[rows.length - 1]![pk] as string;
     if (rows.length < BLOB_ROTATION_BATCH_SIZE) break;
   }
 }
 
+/** The model's primary-key field: `id` unless the registry says otherwise. */
+function pkField(col: EncryptedColumn): string {
+  return col.pkField ?? "id";
+}
+
+/** True when the column's rows must be walked in bounded batches. */
+function isBatched(col: EncryptedColumn): boolean {
+  return Boolean(col.codecField ?? col.batched);
+}
+
+/** The key id a walked row was written under, dispatching on its codec. */
+function walkedKeyId(
+  value: unknown,
+  codec: string | null,
+  kind: EncryptedColumn["kind"],
+): string | null {
+  if (codec !== null) return blobKeyId(value as Uint8Array, codec);
+  const ciphertext = toCiphertext(value, kind);
+  return ciphertext == null ? null : extractKeyId(ciphertext);
+}
+
 export interface ColumnScan {
   model: string;
   field: string;
@@ -208,16 +242,16 @@ export async function scanColumn(
   const byKeyId: Record<string, number> = {};
   let total = 0;
 
-  if (col.codecField) {
-    // Codec-dispatched blob column: bounded batches, per-row codec.
+  if (isBatched(col)) {
+    // Blob column: bounded batches, per-row codec where the column has one.
     await walkBlobColumn(delegate, col, ({ value, codec }) => {
       total += 1;
-      const id = blobKeyId(value, codec) ?? LEGACY_BUCKET;
+      const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET;
       byKeyId[id] = (byKeyId[id] ?? 0) + 1;
     });
   } else {
     const rows = await delegate.findMany({
-      select: { id: true, [col.field]: true },
+      select: { [pkField(col)]: true, [col.field]: true },
     });
     for (const row of rows) {
       const ciphertext = toCiphertext(row[col.field], col.kind);
@@ -283,6 +317,8 @@ export interface RotationResult {
   scanned: number;
   rotated: number;
   errors: number;
+  /** Unreadable rows deleted from a `disposable` column. */
+  dropped: number;
 }
 
 /** Re-encrypt one column's stale rows to the active key. */
@@ -297,51 +333,78 @@ export async function rotateColumn(
     scanned: 0,
     rotated: 0,
     errors: 0,
+    dropped: 0,
   };
 
-  if (col.codecField) {
-    // Codec-dispatched blob column: bounded id-cursor batches (never an
-    // unbounded blob findMany), re-encrypted under each row's OWN codec.
-    // Idempotent — rows already on the active key are skipped, so an
-    // interrupted run resumes cleanly on the next invocation.
+  /**
+   * A row that could not be re-encrypted. FAIL-CLOSED by default: count it and
+   * leave it untouched rather than dropping data. A `disposable` column takes
+   * the other branch — the value is a reproducible cache entry, and keeping an
+   * unreadable one only guarantees the next reader gets a body it cannot
+   * parse, so the row goes instead.
+   */
+  const onUnreadable = async (id: string): Promise<void> => {
+    if (!col.disposable) {
+      result.errors += 1;
+      return;
+    }
+    try {
+      await delegate.delete({ where: { [pkField(col)]: id } });
+      result.dropped += 1;
+    } catch {
+      result.errors += 1;
+    }
+  };
+
+  if (isBatched(col)) {
+    // Blob column: bounded id-cursor batches (never an unbounded findMany),
+    // re-encrypted under each row's OWN codec where it has one. Idempotent —
+    // rows already on the active key are skipped, so an interrupted run
+    // resumes cleanly on the next invocation.
     await walkBlobColumn(delegate, col, async ({ id, value, codec }) => {
       result.scanned += 1;
-      if (blobKeyId(value, codec) === getActiveKeyId()) return;
+      if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return;
       try {
+        // The plaintext is never inspected, only re-sealed, so every envelope
+        // a stored value can legitimately carry survives rotation untouched.
+        const next =
+          codec !== null
+            ? reencryptBlob(value as Uint8Array, codec)
+            : fromCiphertext(
+                encrypt(decrypt(toCiphertext(value, col.kind)!)),
+                col.kind,
+              );
         await delegate.update({
-          where: { id },
-          data: { [col.field]: reencryptBlob(value, codec) },
+          where: { [pkField(col)]: id },
+          data: { [col.field]: next },
         });
         result.rotated += 1;
       } catch {
-        // FAIL-CLOSED: unknown codec or a no-longer-configured key throws;
-        // count it and leave the row untouched rather than dropping data.
-        result.errors += 1;
+        await onUnreadable(id);
       }
     });
     return result;
   }
 
+  const pk = pkField(col);
   const rows = await delegate.findMany({
-    select: { id: true, [col.field]: true },
+    select: { [pk]: true, [col.field]: true },
   });
   result.scanned = rows.length;
   for (const row of rows) {
     const ciphertext = toCiphertext(row[col.field], col.kind);
     if (ciphertext == null || !shouldRotate(ciphertext)) continue;
-    const id = row.id as string;
+    const id = row[pk] as string;
     try {
       // ACTIVE-KEY-ONLY: encrypt() always writes the active key id.
       const reencrypted = encrypt(decrypt(ciphertext));
       await delegate.update({
-        where: { id },
+        where: { [pk]: id },
         data: { [col.field]: fromCiphertext(reencrypted, col.kind) },
       });
       result.rotated += 1;
     } catch {
-      // FAIL-CLOSED: a row under a no-longer-configured key throws on decrypt;
-      // count it and leave the row untouched rather than dropping data.
-      result.errors += 1;
+      await onUnreadable(id);
     }
   }
   return result;
@@ -353,6 +416,8 @@ export interface CorpusRotation {
   totalScanned: number;
   totalRotated: number;
   totalErrors: number;
+  /** Unreadable rows deleted from `disposable` columns. */
+  totalDropped: number;
 }
 
 /** Re-encrypt the whole corpus to the active key. Idempotent + active-key-only. */
@@ -367,10 +432,12 @@ export async function rotateCorpus(
   let totalScanned = 0;
   let totalRotated = 0;
   let totalErrors = 0;
+  let totalDropped = 0;
   for (const r of results) {
     totalScanned += r.scanned;
     totalRotated += r.rotated;
     totalErrors += r.errors;
+    totalDropped += r.dropped;
   }
   return {
     activeKeyId,
@@ -378,5 +445,6 @@ export async function rotateCorpus(
     totalScanned,
     totalRotated,
     totalErrors,
+    totalDropped,
   };
 }

src/lib/jobs/encryption-key-rotate.ts

@@ -48,6 +48,7 @@ export async function runEncryptionKeyRotation(): Promise<{
   totalScanned: number;
   totalRotated: number;
   totalErrors: number;
+  totalDropped: number;
 }> {
   const prisma = getWorkerPrisma();
   const out = await rotateCorpus(prisma as unknown as CorpusClient);
@@ -56,6 +57,7 @@ export async function runEncryptionKeyRotation(): Promise<{
     totalScanned: out.totalScanned,
     totalRotated: out.totalRotated,
     totalErrors: out.totalErrors,
+    totalDropped: out.totalDropped,
   };
 }
 
@@ -70,6 +72,7 @@ export async function handleEncryptionKeyRotate(
       evt.addMeta("rotate_scanned", result.totalScanned);
       evt.addMeta("rotate_rotated", result.totalRotated);
       evt.addMeta("rotate_errors", result.totalErrors);
+      evt.addMeta("rotate_dropped", result.totalDropped);
       await auditLog("admin.encryption.rotate.completed", {
         userId: requestedBy,
         details: {
@@ -77,6 +80,7 @@ export async function handleEncryptionKeyRotate(
           scanned: result.totalScanned,
           rotated: result.totalRotated,
           errors: result.totalErrors,
+          dropped: result.totalDropped,
         },
       });
       // Per-row errors are the fail-closed skip of a row under a key the
@@ -86,6 +90,7 @@ export async function handleEncryptionKeyRotate(
         rotate_scanned: result.totalScanned,
         rotate_rotated: result.totalRotated,
         rotate_errors: result.totalErrors,
+        rotate_dropped: result.totalDropped,
       });
     } catch (err) {
       evt.addWarning(`encryption-key-rotate failed: ${err}`);

src/lib/jobs/job-outcome.ts

@@ -173,6 +173,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "retried",
   "reviewed",
   "rollup_failed",
+  "rotate_dropped",
   "rotate_errors",
   "rotate_rotated",
   "rotate_scanned",