Merge branch 'fix/backup-chunked-storage' into release/v1.39.2

src/lib/crypto/__tests__/encryption-corpus.test.ts

@@ -52,7 +52,7 @@ function makeClient(seed: Record<string, Row[]>): {
         let rows = [...store[model]].sort((a, b) =>
           String(a.id).localeCompare(String(b.id)),
         );
-        const after = where?.id?.gt;
+        const after = (where?.id as { gt?: string } | undefined)?.gt;
         if (after !== undefined)
           rows = rows.filter((r) => String(r.id) > after);
         if (take !== undefined) rows = rows.slice(0, take);
@@ -256,4 +256,30 @@ describe("encryption-corpus scan + rotate", () => {
       expect(take!).toBeLessThanOrEqual(5_000);
     }
   });
+
+  it("treats a row deleted between the read and the write as gone, not as an error", async () => {
+    // A weekly backup replaces its pieces while a rotation walks them: the
+    // piece the walk read no longer exists when it writes it back.
+    const { client } = makeClient({
+      NotificationChannel: [
+        { id: "c1", config: activeUnder("v1", "a") },
+        { id: "c2", config: activeUnder("v1", "b") },
+      ],
+    });
+    const delegate = client.notificationChannel!;
+    const update = delegate.update;
+    delegate.update = async (args) => {
+      if (args.where.id === "c1") {
+        throw Object.assign(new Error("Record to update not found."), {
+          code: "P2025",
+        });
+      }
+      return update(args);
+    };
+    const rotation = await rotateCorpus(client);
+    const result = rotation.results.find(
+      (r) => r.model === "NotificationChannel",
+    )!;
+    expect(result).toMatchObject({ rotated: 1, errors: 0 });
+  });
 });

src/lib/crypto/encrypted-columns.ts

@@ -39,6 +39,13 @@
 
 export type EncryptedColumnKind = "string" | "bytes";
 
+/**
+ * The associated-data label a stored backup's pieces are sealed with. The same
+ * key seals other binary values (a stored document, for one); the label keeps
+ * a value sealed for another purpose from ever opening as a backup piece.
+ */
+export const BACKUP_CHUNK_AAD = "healthlog/data-backup-chunk/v1";
+
 export interface EncryptedColumn {
   /** Prisma model name (PascalCase, as declared in schema.prisma). */
   readonly model: string;
@@ -60,6 +67,19 @@ export interface EncryptedColumn {
    * layout. Like `codecField`, it implies batching.
    */
   readonly codec?: "binary2";
+  /**
+   * The associated-data label every value of the column is sealed with
+   * (`encryptBytes(value, aad)`). Rotation re-seals under the same label.
+   */
+  readonly aad?: string;
+  /**
+   * The column holds single-value backups, which rotation does not re-seal in
+   * place but converts into pieces in `DataBackupChunk` under the active key,
+   * reading the value a slice at a time. Re-sealing in place needs the whole
+   * value in memory several times over, and one value can be a hundred
+   * megabytes.
+   */
+  readonly convertsToPieces?: true;
   /**
    * Walk this column in bounded id-cursor batches instead of one `findMany`.
    * Set it on any column whose rows are blobs rather than short strings — a
@@ -387,9 +407,15 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
   //
   // Rotation re-encrypts the ciphertext WITHOUT touching the plaintext, so it
   // is blind to the envelope inside: a row is the plain backup JSON, the
-  // `HLZ1:`-prefixed gzip form, or the single `~hlgcm1.` stream v1.39.1 wrote
+  // `HLZ1:`-prefixed gzip form, or the single `~hlgcm1.` stream v1.38.6 to v1.39.1 wrote
   // (re-sealed as a stream). Batched, because a single row is megabytes.
-  { model: "DataBackup", field: "data", kind: "string", batched: true },
+  {
+    model: "DataBackup",
+    field: "data",
+    kind: "string",
+    batched: true,
+    convertsToPieces: true,
+  },
 
   // ───── Whole-account backup, in pieces (Bytes, binary2, batched) ─────
   // How every backup is stored from v1.39.2: ordered pieces of about a
@@ -402,6 +428,7 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
     field: "data",
     kind: "bytes",
     codec: "binary2",
+    aad: BACKUP_CHUNK_AAD,
   },
 
   // ───── Idempotent-replay response cache (String, disposable) ─────

src/lib/crypto/encryption-corpus.ts

@@ -39,8 +39,12 @@ import {
   getActiveKeyId,
   isStreamCiphertext,
   reencryptBytesToActive,
-  reencryptStreamToActive,
 } from "@/lib/crypto";
+import type { PrismaClient } from "@/generated/prisma/client";
+import {
+  convertSingleValueBackup,
+  singleValueBackupHead,
+} from "@/lib/export/store-backup-blob";
 import {
   ENCRYPTED_COLUMNS,
   type EncryptedColumn,
@@ -72,7 +76,7 @@ interface ColumnDelegate {
     select: Record<string, true>;
     orderBy?: Record<string, "asc" | "desc">;
     take?: number;
-    where?: Record<string, { gt: string }>;
+    where?: Record<string, unknown>;
   }) => Promise<Array<Record<string, unknown>>>;
   update: (args: {
     where: Record<string, string>;
@@ -139,10 +143,14 @@ function blobKeyId(value: Uint8Array, codec: string): string | null {
 }
 
 /** Re-encrypt one codec-dispatched blob under its OWN codec (never converts). */
-function reencryptBlob(value: Uint8Array, codec: string): Uint8Array {
+function reencryptBlob(
+  value: Uint8Array,
+  codec: string,
+  aad: string | undefined,
+): Uint8Array {
   const buf = Buffer.from(value);
   if (codec === "binary2") {
-    const rotated = reencryptBytesToActive(buf);
+    const rotated = reencryptBytesToActive(buf, aad);
     const next = new Uint8Array(new ArrayBuffer(rotated.byteLength));
     next.set(rotated);
     return next;
@@ -232,17 +240,44 @@ function walkedKeyId(
   if (codec !== null) return blobKeyId(value as Uint8Array, codec);
   const ciphertext = toCiphertext(value, kind);
   if (ciphertext == null) return null;
-  // The single-stream backup form v1.39.1 wrote carries its key id behind a
+  // The single-stream backup form v1.38.6 to v1.39.1 wrote carries its key id behind a
   // `~hlgcm1.` marker, where the string codec's parser does not look.
   if (isStreamCiphertext(ciphertext)) return extractStreamKeyId(ciphertext);
   return extractKeyId(ciphertext);
 }
 
-/** Re-seal a string-codec value under the active key, keeping its form. */
-function reencryptString(ciphertext: string): string {
-  return isStreamCiphertext(ciphertext)
-    ? reencryptStreamToActive(ciphertext)
-    : encrypt(decrypt(ciphertext));
+/**
+ * The ids of the rows of a `convertsToPieces` column that still hold a single
+ * value, a page of ids at a time. Only the id is read: the value can be a
+ * hundred megabytes.
+ */
+async function* singleValueIds(
+  delegate: ColumnDelegate,
+): AsyncGenerator<string> {
+  let cursor: string | null = null;
+  for (;;) {
+    const rows = await delegate.findMany({
+      select: { id: true },
+      orderBy: { id: "asc" },
+      take: ROW_ROTATION_BATCH_SIZE,
+      where: {
+        data: { not: null },
+        ...(cursor ? { id: { gt: cursor } } : {}),
+      },
+    });
+    for (const row of rows) yield row.id as string;
+    if (rows.length < ROW_ROTATION_BATCH_SIZE) return;
+    cursor = rows[rows.length - 1]!.id as string;
+  }
+}
+
+/** True for a Prisma "record to update not found" (P2025). */
+function isRowGone(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    (err as { code?: unknown }).code === "P2025"
+  );
 }
 
 export interface ColumnScan {
@@ -266,6 +301,29 @@ export async function scanColumn(
   const byKeyId: Record<string, number> = {};
   let total = 0;
 
+  if (col.convertsToPieces) {
+    // One value can be a hundred megabytes: read only the first characters,
+    // which carry the key id.
+    for await (const id of singleValueIds(delegate)) {
+      const head = await singleValueBackupHead(
+        client as unknown as PrismaClient,
+        id,
+      );
+      if (head === null) continue;
+      total += 1;
+      const key = walkedKeyId(head, null, col.kind) ?? LEGACY_BUCKET;
+      byKeyId[key] = (byKeyId[key] ?? 0) + 1;
+    }
+    return {
+      model: col.model,
+      field: col.field,
+      kind: col.kind,
+      total,
+      byKeyId,
+      legacy: byKeyId[LEGACY_BUCKET] ?? 0,
+    };
+  }
+
   // Bounded batches for every column, per-row codec where the column has one.
   await walkColumn(delegate, col, ({ value, codec }) => {
     total += 1;
@@ -367,6 +425,28 @@ export async function rotateColumn(
     }
   };
 
+  if (col.convertsToPieces) {
+    // Converted into pieces under the active key, one row at a time and a
+    // slice of the value at a time, whatever key it is under now: re-sealing
+    // it in place would hold the whole value several times over
+    // (`convertSingleValueBackup`). A row converted, replaced or deleted since
+    // it was listed is gone, not an error.
+    for await (const id of singleValueIds(delegate)) {
+      result.scanned += 1;
+      try {
+        const outcome = await convertSingleValueBackup(
+          client as unknown as PrismaClient,
+          id,
+        );
+        if (outcome === "converted") result.rotated += 1;
+        else result.scanned -= 1;
+      } catch {
+        result.errors += 1;
+      }
+    }
+    return result;
+  }
+
   // Bounded id-cursor batches for every column (never an unbounded
   // findMany), re-encrypted under each row's OWN codec where it has one.
   // Idempotent — rows already on the active key are skipped, so an
@@ -380,9 +460,9 @@ export async function rotateColumn(
       // a stored value can legitimately carry survives rotation untouched.
       const next =
         codec !== null
-          ? reencryptBlob(value as Uint8Array, codec)
+          ? reencryptBlob(value as Uint8Array, codec, col.aad)
           : fromCiphertext(
-              reencryptString(toCiphertext(value, col.kind)!),
+              encrypt(decrypt(toCiphertext(value, col.kind)!)),
               col.kind,
             );
       await delegate.update({
@@ -390,7 +470,10 @@ export async function rotateColumn(
         data: { [col.field]: next },
       });
       result.rotated += 1;
-    } catch {
+    } catch (err) {
+      // Deleted between the read and the write (a backup replacing its
+      // pieces, an account going): nothing is left to rotate.
+      if (isRowGone(err)) return;
       await onUnreadable(id);
     }
   });