fix(backup): store a backup in sealed pieces instead of one value

A stored backup was one value in data_backups.data. v1.39.1 wrote it a
piece at a time but still joined the pieces into that one value, and one
value has to pass through the app whole, so a copy was capped at a fifth of
the heap: 105 MB in the default 1 GB container. An account with 1.75
million readings is past that and could not be backed up at all (#1031).

A copy is now kept as ordered pieces of about a megabyte in the new
data_backup_chunks table (migration 0352). Each piece is sealed on its own
with AES-256-GCM, and its sealed header names the copy (a random stream id
kept on the row), its position and whether it is the last. A reader checks
every piece before it releases a byte, so a piece that is missing, moved,
altered, taken from another copy, or a copy cut short is refused with
backup.payload.undecryptable before a restore deletes anything, and the
second read during the restore checks each piece again.

The write is one transaction, so the previous copy stays whole and readable
until the new one commits. Pieces are written through the model API: a raw
query with a large parameter is kept alive by Prisma's query plan cache,
which left 160 MB of heap behind per hundred one-megabyte inserts.

Restore, preview, download, the restore job and the admin list read both
forms; copies written before this change keep their single value and still
restore. The heap-derived limit is gone. The one limit left is on storage,
BACKUP_MAX_STORED_MB (default 2048), and its message names the setting. Key
rotation covers the pieces, the wipe and account deletion take them with
their backup, and the backup plan lists the table as not carried.

Measured on an account of 2.6 million readings in a 1 GB container with web
and worker in one process: backup 87 s, 97 MiB in 96 pieces, preview 20 s,
download 25 s, restore 264 s with every reading back exactly, peak 567 MiB,
no restart.

.env.example

@@ -234,6 +234,17 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32"
 # BACKUP_RETENTION_DAYS="30"        # the number for your bucket lifecycle rule (see above)
 # BACKUP_ENCRYPTION_KEY=""          # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY
 
+# -----------------------------------------------------------------------------
+# Stored backups in the database (optional)
+# -----------------------------------------------------------------------------
+# The largest weekly or uploaded backup of one account the app keeps, in
+# megabytes of encrypted copy. Default 2048. It does not depend on the app's
+# memory: a copy is written and read in pieces of about a megabyte. The limit
+# only stops a runaway copy from filling the database volume, which holds the
+# old and the new copy side by side while a backup runs. An account with 2.6
+# million readings stores a copy of about 100 MB.
+# BACKUP_MAX_STORED_MB="2048"
+
 
 # -----------------------------------------------------------------------------
 # Audit-trail retention (optional)

.env.production.example

@@ -344,6 +344,17 @@ API_TOKEN_HMAC_KEY="<openssl rand -hex 32>"
 # silently. See docs/ops/backup-restore.md.
 # BACKUP_RETENTION_DAYS="30"
 
+# -----------------------------------------------------------------------------
+# Stored backups in the database (optional)
+# -----------------------------------------------------------------------------
+# The largest weekly or uploaded backup of one account the app keeps, in
+# megabytes of encrypted copy. Default 2048. It does not depend on the app's
+# memory: a copy is written and read in pieces of about a megabyte. The limit
+# only stops a runaway copy from filling the database volume, which holds the
+# old and the new copy side by side while a backup runs. An account with 2.6
+# million readings stores a copy of about 100 MB.
+# BACKUP_MAX_STORED_MB="2048"
+
 
 # -----------------------------------------------------------------------------
 # Audit-trail retention -- optional

docker-compose.yml

@@ -131,6 +131,9 @@ x-healthlog-env: &healthlog_env
   BACKUP_S3_SECRET_KEY: "${BACKUP_S3_SECRET_KEY:-}"
   BACKUP_S3_REGION: "${BACKUP_S3_REGION:-auto}"
   BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}"
+  # Largest stored backup of one account, in MB of encrypted copy (default
+  # 2048). A storage limit, not a memory one; see .env.production.example.
+  BACKUP_MAX_STORED_MB: "${BACKUP_MAX_STORED_MB:-}"
   # How long the audit trail is kept, in days. Default 365, floor 7 (a value
   # below the floor is ignored rather than obeyed, so a window set in seconds
   # by mistake cannot empty a fresh table). It is not only a retention knob:

docs/api/openapi.yaml

@@ -20580,8 +20580,10 @@ paths:
                 $ref: "#/components/schemas/ErrorEnvelope"
         "422":
           description: "The stored copy could not be opened: either the key that wrote it is no longer in `ENCRYPTION_KEYS` (a
-            rotation that dropped the legacy entry too early), or the stored bytes are not the ones that were written.
-            `meta.errorCode` = `backup.payload.undecryptable`. Nothing was changed."
+            rotation that dropped the legacy entry too early), or the stored bytes are not the ones that were written (a
+            stored piece missing, moved, altered, taken from another copy, or the copy cut short). Every piece is
+            checked before any of the copy is read. `meta.errorCode` = `backup.payload.undecryptable`. Nothing was
+            changed."
           content:
             application/json:
               schema:

prisma/migrations/0352_data_backup_chunks/migration.sql

@@ -0,0 +1,38 @@
+-- Stored backups in pieces.
+--
+-- A backup used to be one `text` value in `data_backups.data`. One value has
+-- to pass through the app as one value on its way in and out, so the largest
+-- backup a host could keep was tied to the app's memory: a fifth of its heap
+-- limit, 105 MB in the default 1 GB container, which an account with 1.75
+-- million readings outgrew (#1031). A copy is now written as ordered pieces of
+-- about a megabyte each, and nothing about its size depends on memory.
+--
+-- Additive for existing rows: a copy written before this migration keeps its
+-- single value and stays readable as it is. The next weekly run replaces the
+-- weekly copy in the new form; an uploaded copy keeps its old form until it is
+-- deleted.
+
+-- AlterTable
+ALTER TABLE "data_backups" ALTER COLUMN "data" DROP NOT NULL,
+ADD COLUMN     "chunk_count" INTEGER,
+ADD COLUMN     "chunk_stream_id" TEXT;
+
+-- CreateTable
+CREATE TABLE "data_backup_chunks" (
+    "id" TEXT NOT NULL,
+    "backup_id" TEXT NOT NULL,
+    "seq" INTEGER NOT NULL,
+    "data" BYTEA NOT NULL,
+
+    CONSTRAINT "data_backup_chunks_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "data_backup_chunks_backup_id_seq_key" ON "data_backup_chunks"("backup_id", "seq");
+
+-- AddForeignKey
+ALTER TABLE "data_backup_chunks" ADD CONSTRAINT "data_backup_chunks_backup_id_fkey" FOREIGN KEY ("backup_id") REFERENCES "data_backups"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Ciphertext does not compress, so storing it uncompressed saves the attempt;
+-- it still moves out of line like any large value.
+ALTER TABLE "data_backup_chunks" ALTER COLUMN "data" SET STORAGE EXTERNAL;

prisma/schema.prisma

@@ -4964,19 +4964,45 @@ model PushSubscription {
 // ─── Data Backups ────────────────────────────────────────────
 
 model DataBackup {
-  id        String   @id @default(cuid())
-  userId    String   @map("user_id")
-  type      String   @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL
-  data      String // JSON export blob
-  createdAt DateTime @default(now()) @map("created_at")
+  id            String   @id @default(cuid())
+  userId        String   @map("user_id")
+  type          String   @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_<ms>
+  /// The whole encrypted copy as one value: how every backup was stored up
+  /// to v1.39.1. NULL for a copy stored in pieces (`chunks`), which is how
+  /// every backup is written from v1.39.2 on. A row carries one or the other.
+  data          String?
+  /// How many pieces a chunked copy has, and the random id sealed into every
+  /// one of them (`src/lib/export/backup-chunks.ts`). The reader checks both
+  /// against the pieces before it releases a byte; NULL on a single-value copy.
+  chunkCount    Int?     @map("chunk_count")
+  chunkStreamId String?  @map("chunk_stream_id")
+  createdAt     DateTime @default(now()) @map("created_at")
 
-  user User @relation(fields: [userId], references: [id], onDelete: Cascade)
+  user   User              @relation(fields: [userId], references: [id], onDelete: Cascade)
+  chunks DataBackupChunk[]
 
   @@unique([userId, type], map: "data_backups_user_id_type_key")
   @@index([userId])
   @@map("data_backups")
 }
 
+/// One piece of a stored backup, in order. Each is sealed on its own with
+/// AES-256-GCM (the binary codec in `src/lib/crypto.ts`), and the sealed
+/// plaintext names the copy it belongs to, its position and whether it is
+/// the last, so a piece that is missing, moved, altered or borrowed from
+/// another copy is refused. Deleted with its backup.
+model DataBackupChunk {
+  id       String @id @default(cuid())
+  backupId String @map("backup_id")
+  seq      Int
+  data     Bytes
+
+  backup DataBackup @relation(fields: [backupId], references: [id], onDelete: Cascade)
+
+  @@unique([backupId, seq])
+  @@map("data_backup_chunks")
+}
+
 // ─── Backup restore jobs (v1.39.1) ──────────────────────────
 //
 // A restore of a stored backup runs on the `backup-restore` pg-boss queue, not

scripts/env-manifest.json

@@ -288,6 +288,11 @@
           "name": "TRUST_PROXY_HOPS",
           "purpose": "Trusted reverse-proxy hop count; the client IP is read that many entries from the right of X-Forwarded-For.",
           "example": "1"
+        },
+        {
+          "name": "BACKUP_MAX_STORED_MB",
+          "purpose": "Largest stored backup of one account the app keeps, in megabytes of encrypted copy (2048 by default). A storage limit, independent of the app's memory: a backup past it fails with a message naming this variable, and the previous copy stays.",
+          "example": "2048"
         }
       ]
     }

scripts/rotate-encryption-key.ts

@@ -855,14 +855,27 @@ async function main() {
   // operator that zero meant safe to drop the old key, and every stored backup
   // became undecryptable. Walked in bounded id-cursor batches because one row
   // is an entire compressed account, and re-sealed WITHOUT reading the
-  // plaintext, so both stored envelopes (the plain backup JSON and the
-  // `HLZ1:` gzip form) — and any later one — rotate unchanged.
+  // plaintext, so every single-value envelope (the plain backup JSON, the
+  // `HLZ1:` gzip form and the v1.39.1 `~hlgcm1.` stream) rotates unchanged.
   results.push(
     await rotateRegistryColumn("DataBackup", "data", {
       dataBackup: prisma.dataBackup,
     } as unknown as CorpusClient),
   );
 
+  // ───── Whole-account backup in pieces (Bytes, binary2, batched) ─────
+  // From v1.39.2 a backup is stored as sealed pieces in "data_backup_chunks",
+  // one `encryptBytes()` value each, and the "data" column above holds only
+  // copies written before that. Walked in id-cursor batches of a few pieces
+  // (each is about a megabyte) and re-sealed without reading the backup: the
+  // header that ties a piece to its copy and position is inside the
+  // ciphertext, so it comes through rotation unchanged.
+  results.push(
+    await rotateRegistryColumn("DataBackupChunk", "data", {
+      dataBackupChunk: prisma.dataBackupChunk,
+    } as unknown as CorpusClient),
+  );
+
   // ───── Idempotent-replay response cache (String, disposable) ─────
   // `IdempotencyKey.responseBody` is the cached response body, encrypted
   // because the PHI-returning creates echo their own decrypted DTO. Rotating

src/app/api/admin/backups/[id]/download/route.ts

@@ -24,8 +24,11 @@ import { auditLog } from "@/lib/auth/audit";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  openStoredBackup,
+  STORED_BACKUP_SELECT,
+} from "@/lib/export/stored-backup";
 import {
   readStreamedBackup,
   type BackupSource,
@@ -47,7 +50,12 @@ export const GET = apiHandler(
 
     const backup = await prisma.dataBackup.findUnique({
       where: { id },
-      include: { user: { select: { id: true, username: true } } },
+      select: {
+        ...STORED_BACKUP_SELECT,
+        type: true,
+        createdAt: true,
+        user: { select: { id: true, username: true } },
+      },
     });
 
     if (!backup) {
@@ -62,10 +70,11 @@ export const GET = apiHandler(
     }
 
     // Opened and streamed, never unpacked into one string: a large record's
-    // JSON is longer than any string V8 can hold (#1031).
+    // JSON is longer than any string V8 can hold (#1031). Opening checks
+    // every stored piece first.
     let source: BackupSource;
     try {
-      source = openBackupBlob(backup.data);
+      source = await openStoredBackup(prisma, backup);
     } catch (err) {
       // A rotated or dropped key, or a stored copy that is no longer the one
       // that was written. Both are bad stored input rather than a fault in

src/app/api/admin/backups/[id]/restore/route.ts

@@ -27,8 +27,12 @@ import { prisma } from "@/lib/db";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  openStoredBackup,
+  STORED_BACKUP_SELECT,
+  storedBackupIdentity,
+} from "@/lib/export/stored-backup";
 import { defaultUserIdResolver, withIdempotency } from "@/lib/idempotency";
 import {
   admitBackupRestore,
@@ -82,7 +86,7 @@ const handler = apiHandler(
 
     const backup = await prisma.dataBackup.findUnique({
       where: { id },
-      select: { id: true, userId: true, data: true },
+      select: STORED_BACKUP_SELECT,
     });
     if (!backup) {
       await auditLog("admin.backups.restore.denied", {
@@ -93,12 +97,13 @@ const handler = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    // Opening authenticates the whole ciphertext (and decompresses nothing),
-    // so a copy written under a key this host no longer holds is refused now,
-    // in the answer the operator is looking at, rather than minutes later in
-    // a job. Bad stored input, not a broken server: 422, nothing queued.
+    // Opening authenticates every stored piece (and decompresses nothing), so
+    // a copy written under a key this host no longer holds, or one whose
+    // pieces do not add up, is refused now, in the answer the operator is
+    // looking at, rather than minutes later in a job. Bad stored input, not a
+    // broken server: 422, nothing queued.
     try {
-      openBackupBlob(backup.data);
+      await openStoredBackup(prisma, backup);
     } catch (err) {
       await auditLog("admin.backups.restore.failed", {
         userId: admin.id,
@@ -118,7 +123,7 @@ const handler = apiHandler(
       userId: backup.userId,
       actorUserId: admin.id,
       backupId: backup.id,
-      backupDigest: backupDigest(backup.data),
+      backupDigest: backupDigest(storedBackupIdentity(backup)),
       restoreInstanceSettings,
     });
 

src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts

@@ -64,6 +64,8 @@ beforeEach(() => {
     id: "b1",
     userId: "u1",
     data: "ciphertext",
+    chunkCount: null,
+    chunkStreamId: null,
     createdAt: new Date("2026-08-01T00:00:00Z"),
     user: { id: "u1", username: "self-hoster" },
   });

src/app/api/admin/backups/[id]/summary/route.ts

@@ -19,8 +19,11 @@ import { apiError, apiSuccess } from "@/lib/api-response";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  openStoredBackup,
+  STORED_BACKUP_SELECT,
+} from "@/lib/export/stored-backup";
 import {
   readStreamedBackup,
   type BackupSource,
@@ -44,7 +47,11 @@ export const GET = apiHandler(
 
     const backup = await prisma.dataBackup.findUnique({
       where: { id },
-      include: { user: { select: { id: true, username: true } } },
+      select: {
+        ...STORED_BACKUP_SELECT,
+        createdAt: true,
+        user: { select: { id: true, username: true } },
+      },
     });
     if (!backup) {
       throw new HttpError(404, "Backup not found");
@@ -54,7 +61,7 @@ export const GET = apiHandler(
     // string V8 can hold (#1031).
     let source: BackupSource;
     try {
-      source = openBackupBlob(backup.data);
+      source = await openStoredBackup(prisma, backup);
     } catch {
       // Same refusal the restore and the download give, for the same reason:
       // a copy this instance cannot open is bad stored input, and the preview

src/app/api/admin/backups/route.ts

@@ -63,14 +63,22 @@ export const GET = apiHandler(async () => {
   // Metadata only, and the size read as `octet_length` in the database rather
   // than by pulling every blob into this process to measure it. The listing
   // needs a number, not the ciphertext, and on an instance with large records
-  // fetching them all was its own way of running out of memory. Base64 is
-  // ASCII, so the column's byte length is exactly what the wire reports.
+  // fetching them all was its own way of running out of memory. A copy is
+  // either one value in `data` (written before v1.39.2) or a run of pieces in
+  // `data_backup_chunks`, so the size is whichever of the two it has: what
+  // the copy occupies in the database.
   const rows = await prisma.$queryRaw<BackupMetaRow[]>`
     SELECT b.id, b.user_id, u.username, b.type,
-           octet_length(b.data) AS size_bytes,
+           COALESCE(octet_length(b.data), 0)
+             + COALESCE(c.bytes, 0) AS size_bytes,
            b.created_at
     FROM data_backups b
     JOIN users u ON u.id = b.user_id
+    LEFT JOIN (
+      SELECT backup_id, SUM(octet_length(data)) AS bytes
+      FROM data_backup_chunks
+      GROUP BY backup_id
+    ) c ON c.backup_id = b.id
     ORDER BY b.created_at DESC
   `;
 

src/app/api/admin/backups/upload/route.ts

@@ -220,7 +220,11 @@ export const POST = apiHandler(async (request: NextRequest) => {
   try {
     created = await storeBackupBlob(
       prisma,
-      () => ({ userId: owner!.id, type: uploadType }),
+      {
+        userId: admin.id,
+        type: uploadType,
+        ownerAfterRead: () => owner!.id,
+      },
       async (write) => {
         // One pass: every chunk is stored as it is checked, and a check that
         // fails throws, which rolls the store back.
@@ -336,9 +340,9 @@ export const POST = apiHandler(async (request: NextRequest) => {
       await denied(err.details.reason as string, err.details);
       return apiError(err.answer, err.status, err.meta);
     }
-    // The file was fine; its encrypted copy is larger than one stored backup
-    // may be on this host (a fifth of the heap limit). That is the operator's
-    // to act on, with the numbers in the message, and not a server fault for
+    // The file was fine; its encrypted copy is larger than the stored-copy
+    // limit (`BACKUP_MAX_STORED_MB`). That is the operator's to act on, with
+    // the numbers and the setting in the message, and not a server fault for
     // the error reporter. The store rolled back, so nothing was kept.
     if (err instanceof BackupBlobTooLargeError) {
       await denied("stored_copy_too_large", {

src/lib/crypto/__tests__/encrypted-column-writers.test.ts

@@ -67,14 +67,16 @@ const RELATION_ENVELOPE = new Set([
 const NOT_ROTATED: ReadonlySet<string> = new Set<string>();
 
 /**
- * Ciphertext written by raw SQL, which the Prisma-call walk below cannot see.
+ * Ciphertext written where the Prisma-call walk below cannot see it: by raw
+ * SQL, or through a write whose value arrives as a callback parameter.
  *
- * `DataBackup.data` is the case: the stored backup is produced as ciphertext
- * pieces (`packBackupBlobInto`) and assembled into the row by one SQL
- * statement, because holding it as one string in the process is what a large
- * record could not afford (#1031). Each entry names the file and the
- * statement, and the scan fails if the statement is no longer there, so a
- * moved or removed raw write cannot leave a stale entry that still passes.
+ * `DataBackupChunk.data` is the second case: the stored backup is sealed in
+ * pieces by `packBackupChunks`, which hands each sealed piece to a callback
+ * that creates the row (#1031). The value is ciphertext, but the walk only
+ * follows values it can trace back to an encryption call at the call site.
+ * Each entry names the file and the statement, and the scan fails if the
+ * statement is no longer there, so a moved or removed write cannot leave a
+ * stale entry that still passes.
  */
 const RAW_CIPHERTEXT_WRITES: ReadonlyArray<{
   key: string;
@@ -82,10 +84,10 @@ const RAW_CIPHERTEXT_WRITES: ReadonlyArray<{
   statement: RegExp;
 }> = [
   {
-    key: "DataBackup.data",
+    key: "DataBackupChunk.data",
     file: "src/lib/export/store-backup-blob.ts",
     statement:
-      /UPDATE\s+data_backups\s+SET\s+data\s*=\s*\(\s*SELECT\s+string_agg/,
+      /tx\.dataBackupChunk\.create\(\s*\{\s*data:\s*\{\s*backupId:\s*row\.id,\s*seq,\s*data:\s*new Uint8Array\(sealed\)/,
   },
 ];
 
@@ -471,7 +473,7 @@ describe("ciphertext columns derived from the code", () => {
     // columns whose only signal is the write and not the column's name.
     expect(written.size).toBeGreaterThan(30);
     for (const anchor of [
-      "DataBackup.data",
+      "DataBackupChunk.data",
       "IdempotencyKey.responseBody",
       "NotificationChannel.config",
       "PushSubscription.p256dh",

src/lib/crypto/encrypted-columns.ts

@@ -54,6 +54,12 @@ export interface EncryptedColumn {
    * multi-megabyte blobs, so an unbounded `findMany` would balloon memory.
    */
   readonly codecField?: string;
+  /**
+   * The codec every row of the column uses, for a column with one layout and
+   * no sibling column recording it. "binary2" is the binary `encryptBytes()`
+   * layout. Like `codecField`, it implies batching.
+   */
+  readonly codec?: "binary2";
   /**
    * Walk this column in bounded id-cursor batches instead of one `findMany`.
    * Set it on any column whose rows are blobs rather than short strings — a
@@ -377,12 +383,24 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
   // when nothing else works.
   //
   // Rotation re-encrypts the ciphertext WITHOUT touching the plaintext, so it
-  // is blind to the envelope inside: today a row is either the plain backup
-  // JSON or the `HLZ1:`-prefixed gzip form, and any further shape a future
-  // writer introduces rotates unchanged for the same reason. Batched, because
-  // a single row is megabytes.
+  // is blind to the envelope inside: a row is the plain backup JSON, the
+  // `HLZ1:`-prefixed gzip form, or the single `~hlgcm1.` stream v1.39.1 wrote
+  // (re-sealed as a stream). Batched, because a single row is megabytes.
   { model: "DataBackup", field: "data", kind: "string", batched: true },
 
+  // ───── Whole-account backup, in pieces (Bytes, binary2, batched) ─────
+  // How every backup is stored from v1.39.2: ordered pieces of about a
+  // megabyte, each sealed with `encryptBytes()` (`backup-chunks.ts`). The
+  // column above keeps only copies written before that. Rotation re-seals a
+  // piece without reading what it holds; the header that binds it to its copy
+  // and position is inside the ciphertext, so it survives unchanged.
+  {
+    model: "DataBackupChunk",
+    field: "data",
+    kind: "bytes",
+    codec: "binary2",
+  },
+
   // ───── Idempotent-replay response cache (String, disposable) ─────
   // The cached response body, encrypted because the PHI-returning creates echo
   // their own decrypted DTO. Rows live 24 hours. It is registered so a key

src/lib/crypto/encryption-corpus.ts

@@ -220,7 +220,7 @@ function pkField(col: EncryptedColumn): string {
 
 /** True when the column's rows are blobs, walked in the small batches. */
 function isBlobColumn(col: EncryptedColumn): boolean {
-  return Boolean(col.codecField ?? col.batched);
+  return Boolean(col.codecField ?? col.codec ?? col.batched);
 }
 
 /** The key id a walked row was written under, dispatching on its codec. */

src/lib/export/__tests__/backup-blob.test.ts

@@ -1,44 +1,71 @@
 /**
- * The stored-backup envelope: compress, then encrypt.
+ * The stored-backup envelope: compress, then encrypt, then store in pieces.
  *
- * The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to
- * Postgres. For an account with a few hundred thousand measurements that means
- * the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and
- * the copy the database driver makes of it all alive at once — well over a
- * gigabyte for a record whose JSON is a few hundred megabytes. The run died of
- * memory, not of slow SQL.
- *
- * Compressing before the cipher is what makes the tail of that pipeline cheap:
+ * Compressing before the cipher is what makes the tail of the pipeline cheap:
  * health JSON is extremely repetitive, so everything downstream of the gzip
- * shrinks by an order of magnitude. The envelope has to stay readable both
- * ways — rows written before this change are plain `encrypt(json)` and must
- * keep restoring.
+ * shrinks by an order of magnitude. Every form a copy was ever stored in has to
+ * stay readable: plain `encrypt(json)`, the `HLZ1:` gzip form, and the single
+ * `~hlgcm1.` stream v1.39.1 wrote. From v1.39.2 a copy is written as sealed
+ * pieces (`packBackupChunks`), and the one limit on its size is a storage
+ * setting, not a share of the heap (#1031).
  */
 process.env.ENCRYPTION_KEY ??=
   "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
 
 import { randomBytes } from "node:crypto";
-import v8 from "node:v8";
+import { gunzipSync } from "node:zlib";
 
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it } from "vitest";
 
 import { encrypt } from "@/lib/crypto";
+import {
+  legacyStreamedBlob,
+  legacyStreamedBlobFrom,
+} from "@/__tests__/helpers/legacy-backup-blob";
 import {
   BackupBlobTooLargeError,
-  defaultBackupBlobLimit,
+  defaultBackupStoreLimit,
   packBackupBlob,
-  packBackupBlobInto,
-  packBackupBlobStreaming,
+  packBackupChunks,
   unpackBackupBlob,
 } from "../backup-blob";
+import { BACKUP_CHUNK_BYTES, openBackupChunk } from "../backup-chunks";
 
-/** Pack a whole string through the streaming writer, in small pieces. */
-async function packStreamed(json: string, pieceSize = 4_096): Promise<string> {
-  return packBackupBlobStreaming(async (write) => {
-    for (let at = 0; at < json.length; at += pieceSize) {
-      await write(json.slice(at, at + pieceSize));
-    }
-  });
+const STREAM = "0123456789abcdef0123456789abcdef";
+
+/** Pack a whole string in small pieces; answer the sealed pieces. */
+async function packChunked(
+  json: string,
+  pieceSize = 4_096,
+  maxBytes?: number,
+): Promise<Buffer[]> {
+  const sealed: Buffer[] = [];
+  await packBackupChunks(
+    async (piece, seq) => {
+      expect(seq).toBe(sealed.length);
+      sealed.push(piece);
+    },
+    STREAM,
+    async (write) => {
+      for (let at = 0; at < json.length; at += pieceSize) {
+        await write(json.slice(at, at + pieceSize));
+      }
+    },
+    maxBytes === undefined ? {} : { maxBytes },
+  );
+  return sealed;
+}
+
+/** Open every piece in order and gunzip the result. */
+function readChunked(sealed: Buffer[]): string {
+  const gz = sealed.map((piece, seq) =>
+    openBackupChunk(piece, {
+      streamId: STREAM,
+      seq,
+      last: seq === sealed.length - 1,
+    }),
+  );
+  return gunzipSync(Buffer.concat(gz)).toString("utf8");
 }
 
 /** A record-shaped string: many rows, few distinct keys. */
@@ -82,183 +109,76 @@ describe("backup blob envelope", () => {
     expect(() => unpackBackupBlob(mangled)).toThrow();
   });
 
-  it("reads back a streamed blob byte for byte", async () => {
+  it("still reads the single stream v1.39.1 stored", () => {
     const json = sampleJson(2_000);
-    expect(unpackBackupBlob(await packStreamed(json))).toBe(json);
+    expect(unpackBackupBlob(legacyStreamedBlob(json))).toBe(json);
   });
 
-  it("is indifferent to where the producer's pieces fall", async () => {
-    // Base64 encodes three bytes at a time, so a writer that carries its
-    // sub-group remainder across chunks wrongly still round-trips at some
-    // piece sizes and corrupts at others. Cover every residue class.
-    // Small on purpose: a one-byte piece size means one write per character,
-    // and the point of the case is the residue class, not the volume.
-    const json = sampleJson(20);
-    for (const pieceSize of [1, 2, 3, 7, 64, 1_000, 1_001, json.length]) {
-      expect(unpackBackupBlob(await packStreamed(json, pieceSize))).toBe(json);
-    }
-  });
-
-  it("compresses the streamed form as hard as the buffered one", async () => {
-    const json = sampleJson(5_000);
-    expect((await packStreamed(json)).length).toBeLessThan(json.length / 4);
-  });
-
-  it("keeps the three stored shapes readable side by side", async () => {
-    // An operator's newest usable copy may predate either change, and the
+  it("keeps the three single-value shapes readable side by side", async () => {
+    // An operator's newest usable copy may predate the pieces, and the
     // restore has to work for exactly that person. Plain, compressed and
     // streamed all decode through the one entry point.
     const json = sampleJson(50);
     expect(unpackBackupBlob(encrypt(json))).toBe(json);
     expect(unpackBackupBlob(packBackupBlob(json))).toBe(json);
-    expect(unpackBackupBlob(await packStreamed(json))).toBe(json);
+    expect(
+      unpackBackupBlob(
+        await legacyStreamedBlobFrom(async (write) => {
+          await write(json);
+        }),
+      ),
+    ).toBe(json);
   });
 
-  it("cannot be confused for a legacy blob", async () => {
-    // The marker starts with a tilde, which is in neither the key-id charset
-    // nor the base64 alphabet, so no value the older writers can produce
-    // begins with it and the two families never have to be sniffed apart.
-    const streamed = await packStreamed(sampleJson(10));
-    expect(streamed.startsWith("~")).toBe(true);
-    expect(packBackupBlob(sampleJson(10)).startsWith("~")).toBe(false);
-    expect(encrypt("x").startsWith("~")).toBe(false);
-  });
-
-  it("refuses a streamed blob whose authentication tag was altered", async () => {
-    const streamed = await packStreamed(sampleJson(20));
+  it("refuses a v1.39.1 stream whose authentication tag was altered", () => {
+    const streamed = legacyStreamedBlob(sampleJson(20));
     // The tag is the last 16 bytes of the payload, so the tail of the base64.
     const mangled = `${streamed.slice(0, -6)}${streamed.slice(-6) === "AAAAAA" ? "BBBBBB" : "AAAAAA"}`;
     expect(() => unpackBackupBlob(mangled)).toThrow();
   });
-
-  it("propagates a producer failure instead of storing a truncated blob", async () => {
-    await expect(
-      packBackupBlobStreaming(async (write) => {
-        await write('{"measurements":[');
-        throw new Error("row source failed");
-      }),
-    ).rejects.toThrow("row source failed");
-  });
-
-  it("stops the account whose stored copy would not fit this process", async () => {
-    // The stored blob is the one copy the pipeline has to hold whole, so it is
-    // the one thing here that grows without bound as a record grows. 4 KB is
-    // an absurd limit; the point is that the writer counts what it produced
-    // and stops on that, not on how full the heap happened to be.
-    const err = await packBackupBlobStreaming(
-      async (write) => {
-        await write(sampleJson(5_000));
-      },
-      { maxBytes: 4_096 },
-    ).catch((e: unknown) => e);
-
-    expect(err).toBeInstanceOf(BackupBlobTooLargeError);
-    const failure = err as BackupBlobTooLargeError;
-    expect(failure.limitBytes).toBe(4_096);
-    expect(failure.bytes).toBeGreaterThan(4_096);
-    // The message has to describe what was measured. It says how much
-    // ciphertext this account produced and what it may occupy — not how full
-    // the process's heap was, which is what the check this replaced reported.
-    expect(failure.message).toContain("of encrypted backup for one account");
-    expect(failure.message).toContain("over the 4 KB");
-    // The check this replaced reported "aborted at N MB of heap", which was a
-    // reading of the whole process and said nothing about the account.
-    expect(failure.message).not.toContain("of heap");
-  });
-
-  it("does not stop a record that fits, however dirty the heap is", async () => {
-    // Collectable garbage past 419 MB: 80 % of the 524 MB V8 limit a 1 GB
-    // container gets, which is the exact budget that aborted all four accounts
-    // on the live instance — the smallest of them a 1.2 MB demo record.
-    const CONTAINER_BUDGET = Math.floor(524 * 1024 * 1024 * 0.8);
-    // Capped against this process's own limit, so a runner started with a
-    // smaller heap measures a backup rather than an out-of-memory abort.
-    const target = Math.min(
-      CONTAINER_BUDGET + 8 * 1024 * 1024,
-      Math.floor(v8.getHeapStatistics().heap_size_limit * 0.6),
-    );
-    let garbage: unknown[] = [];
-    for (
-      let round = 0;
-      round < 2_000 && process.memoryUsage().heapUsed < target;
-      round++
-    ) {
-      const chunk = new Array(30_000);
-      for (let at = 0; at < 30_000; at++) chunk[at] = { k: round * at };
-      garbage.push(chunk);
-    }
-    const dirty = process.memoryUsage().heapUsed;
-    garbage = [];
-    void garbage;
-
-    const blob = await packStreamed(sampleJson(200));
-    expect(unpackBackupBlob(blob)).toBe(sampleJson(200));
-    // Not a vacuous pass: the heap really was carrying the pile it aimed at
-    // — a 1 GB container's whole budget where the runner has room for it —
-    // when the backup ran.
-    expect(dirty).toBeGreaterThanOrEqual(target);
-    expect(dirty).toBeGreaterThan(64 * 1024 * 1024);
-  });
-
-  it("derives the default limit from the heap limit, not from heap usage", async () => {
-    const limit = defaultBackupBlobLimit();
-    const before = limit;
-    // Allocate and hold: usage moves, the limit must not.
-    const held: unknown[] = [];
-    for (let round = 0; round < 40; round++) {
-      const chunk = new Array(30_000);
-      for (let at = 0; at < 30_000; at++) chunk[at] = { k: round * at };
-      held.push(chunk);
-    }
-    expect(defaultBackupBlobLimit()).toBe(before);
-    expect(held).toHaveLength(40);
-    expect(limit).toBeGreaterThan(16 * 1024 * 1024);
-  });
 });
 
-/**
- * Issue #1031: the weekly and manual backup kept the stored copy as one string
- * before writing it, and at 1.25 million measurements that string and the
- * driver's copies of it added about 640 MB to the process. `packBackupBlobInto`
- * hands the stored copy on in bounded pieces instead, as it is produced.
- */
-describe("piecewise envelope", () => {
-  /** JSON that gzip cannot shrink much, so the stored copy is several MB. */
+describe("the copy in sealed pieces", () => {
+  const savedLimit = process.env.BACKUP_MAX_STORED_MB;
+  afterEach(() => {
+    if (savedLimit === undefined) delete process.env.BACKUP_MAX_STORED_MB;
+    else process.env.BACKUP_MAX_STORED_MB = savedLimit;
+  });
+
+  /** JSON that gzip cannot shrink much, so the copy spans several pieces. */
   function incompressibleJson(bytes: number): string {
     return JSON.stringify({
       noise: randomBytes(Math.ceil((bytes * 3) / 4)).toString("base64"),
     });
   }
 
-  it("delivers pieces that concatenate to a blob reading back the exact JSON", async () => {
-    const json = sampleJson(3_000);
-    const pieces: string[] = [];
-    await packBackupBlobInto(
-      (piece) => {
-        pieces.push(piece);
-      },
-      async (write) => {
-        for (let at = 0; at < json.length; at += 4_096) {
-          await write(json.slice(at, at + 4_096));
-        }
-      },
-    );
-    expect(unpackBackupBlob(pieces.join(""))).toBe(json);
+  it("reads back the exact JSON, whatever the producer's piece size", async () => {
+    const json = sampleJson(20);
+    for (const pieceSize of [1, 3, 64, 1_001, json.length]) {
+      expect(readChunked(await packChunked(json, pieceSize))).toBe(json);
+    }
   });
 
-  it("hands pieces on while the producer is still writing, each one bounded", async () => {
-    const json = incompressibleJson(12 * 1024 * 1024);
-    const pieces: number[] = [];
-    let producing = false;
-    let piecesWhileProducing = 0;
-    const chunks: string[] = [];
+  it("compresses: a record-shaped copy is far smaller than its JSON", async () => {
+    const json = sampleJson(5_000);
+    const stored = (await packChunked(json)).reduce(
+      (sum, piece) => sum + piece.byteLength,
+      0,
+    );
+    expect(stored).toBeLessThan(json.length / 4);
+  });
 
-    await packBackupBlobInto(
-      (piece) => {
-        pieces.push(piece.length);
-        chunks.push(piece);
-        if (producing) piecesWhileProducing += 1;
+  it("hands on bounded pieces while the producer is still writing", async () => {
+    const json = incompressibleJson(5 * BACKUP_CHUNK_BYTES);
+    let producing = false;
+    let whileProducing = 0;
+    const sealed: Buffer[] = [];
+    await packBackupChunks(
+      async (piece) => {
+        sealed.push(piece);
+        if (producing) whileProducing += 1;
       },
+      STREAM,
       async (write) => {
         producing = true;
         for (let at = 0; at < json.length; at += 64 * 1024) {
@@ -266,15 +186,66 @@ describe("piecewise envelope", () => {
         }
         producing = false;
       },
-      { maxBytes: Number.MAX_SAFE_INTEGER },
     );
+    expect(sealed.length).toBeGreaterThan(3);
+    expect(whileProducing).toBeGreaterThan(1);
+    // One piece's worth plus one gzip flush of slack, plus the seal.
+    for (const piece of sealed) {
+      expect(piece.byteLength).toBeLessThan(BACKUP_CHUNK_BYTES + 256 * 1024);
+    }
+    expect(readChunked(sealed)).toBe(json);
+  });
 
-    expect(pieces.length).toBeGreaterThan(2);
-    expect(piecesWhileProducing).toBeGreaterThan(1);
-    // The flush threshold is 4 MB; one gzip chunk's worth of slack on top.
-    for (const length of pieces) {
-      expect(length).toBeLessThan(4 * 1024 * 1024 + 256 * 1024);
+  it("always ends with a piece marked as the last, even an empty one", async () => {
+    const sealed = await packChunked("{}");
+    expect(sealed).toHaveLength(1);
+    expect(readChunked(sealed)).toBe("{}");
+  });
+
+  it("propagates a producer failure instead of sealing a truncated copy", async () => {
+    const sealed: Buffer[] = [];
+    await expect(
+      packBackupChunks(
+        async (piece) => {
+          sealed.push(piece);
+        },
+        STREAM,
+        async (write) => {
+          await write('{"measurements":[');
+          throw new Error("row source failed");
+        },
+      ),
+    ).rejects.toThrow("row source failed");
+    // Nothing was marked as the last piece of a complete copy.
+    expect(sealed).toHaveLength(0);
+  });
+
+  it("stops at the stored-copy limit with a message an operator can act on", async () => {
+    const err = await packChunked(sampleJson(5_000), 4_096, 4_096).catch(
+      (e: unknown) => e,
+    );
+    expect(err).toBeInstanceOf(BackupBlobTooLargeError);
+    const failure = err as BackupBlobTooLargeError;
+    expect(failure.limitBytes).toBe(4_096);
+    expect(failure.bytes).toBeGreaterThan(4_096);
+    expect(failure.message).toContain("of encrypted backup for one account");
+    expect(failure.message).toContain("BACKUP_MAX_STORED_MB");
+    expect(failure.message).toContain("previous copy is unchanged");
+    // Memory is not the answer any more, so the message must not send the
+    // operator after it.
+    expect(failure.message).not.toMatch(/heap|memory|max-old-space-size/i);
+  });
+
+  it("sets the default limit by storage, not by this process's heap", () => {
+    delete process.env.BACKUP_MAX_STORED_MB;
+    expect(defaultBackupStoreLimit()).toBe(2048 * 1024 * 1024);
+    process.env.BACKUP_MAX_STORED_MB = "300";
+    expect(defaultBackupStoreLimit()).toBe(300 * 1024 * 1024);
+    // A value that is not a positive number falls back to the default
+    // rather than to zero, which would refuse every backup.
+    for (const junk of ["", "0", "-5", "lots"]) {
+      process.env.BACKUP_MAX_STORED_MB = junk;
+      expect(defaultBackupStoreLimit()).toBe(2048 * 1024 * 1024);
     }
-    expect(unpackBackupBlob(chunks.join(""))).toBe(json);
   });
 });

src/lib/export/__tests__/backup-chunks.test.ts

@@ -0,0 +1,94 @@
+/**
+ * One sealed piece of a stored backup, and what it refuses.
+ *
+ * Every piece is authenticated on its own, and its sealed header names the
+ * copy, its position and whether it is the last. Each case below is one way a
+ * stored copy can stop being the copy that was written; each has to be refused
+ * before any of the piece's content is handed on.
+ */
+process.env.ENCRYPTION_KEY ??=
+  "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+import { Buffer } from "node:buffer";
+
+import { describe, expect, it } from "vitest";
+
+import {
+  BackupIntegrityError,
+  newChunkStreamId,
+  openBackupChunk,
+  sealBackupChunk,
+} from "../backup-chunks";
+
+const PAYLOAD = Buffer.from("a piece of a gzip stream");
+
+describe("sealed backup pieces", () => {
+  it("opens to exactly the payload it sealed", () => {
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 3, false, PAYLOAD);
+    expect(
+      openBackupChunk(sealed, { streamId: id, seq: 3, last: false }),
+    ).toEqual(PAYLOAD);
+  });
+
+  it("does not carry the payload in the clear", () => {
+    const sealed = sealBackupChunk(newChunkStreamId(), 0, true, PAYLOAD);
+    expect(sealed.includes(PAYLOAD)).toBe(false);
+  });
+
+  it("refuses a piece whose bytes were altered", () => {
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 0, true, PAYLOAD);
+    const altered = Buffer.from(sealed);
+    altered[altered.byteLength - 1]! ^= 0x01;
+    expect(() =>
+      openBackupChunk(altered, { streamId: id, seq: 0, last: true }),
+    ).toThrow(BackupIntegrityError);
+  });
+
+  it("refuses a piece stored at another position", () => {
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 1, false, PAYLOAD);
+    expect(() =>
+      openBackupChunk(sealed, { streamId: id, seq: 2, last: false }),
+    ).toThrow(/written as piece 1/);
+  });
+
+  it("refuses a piece from another copy", () => {
+    const sealed = sealBackupChunk(newChunkStreamId(), 0, true, PAYLOAD);
+    expect(() =>
+      openBackupChunk(sealed, {
+        streamId: newChunkStreamId(),
+        seq: 0,
+        last: true,
+      }),
+    ).toThrow(/different stored copy/);
+  });
+
+  it("refuses a copy cut short: the last listed piece was not written last", () => {
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 4, false, PAYLOAD);
+    expect(() =>
+      openBackupChunk(sealed, { streamId: id, seq: 4, last: true }),
+    ).toThrow(/more were written/);
+  });
+
+  it("refuses a last piece followed by more", () => {
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 4, true, PAYLOAD);
+    expect(() =>
+      openBackupChunk(sealed, { streamId: id, seq: 4, last: false }),
+    ).toThrow(/copy lists more/);
+  });
+
+  it("refuses a piece under a key this host does not hold", () => {
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 0, true, PAYLOAD);
+    // Re-label the key id in the binary header ([0x02][len][id…]).
+    const relabelled = Buffer.from(sealed);
+    relabelled[2] = "x".charCodeAt(0);
+    expect(() =>
+      openBackupChunk(relabelled, { streamId: id, seq: 0, last: true }),
+    ).toThrow(BackupIntegrityError);
+  });
+});

src/lib/export/__tests__/full-backup-stream-heap-gauge.test.ts

@@ -10,8 +10,8 @@
  * operator the record was too large and to buy memory.
  *
  * The bound that replaced it counts the bytes the backup itself produced, in
- * `packBackupBlobStreaming`, where the only copy that grows with a record
- * actually accumulates. This test is the tripwire for putting the gauge back.
+ * `packBackupChunks`, and it is a storage limit, not a memory one: the stored
+ * copy is written in sealed pieces and nothing in the process grows with it. This test is the tripwire for putting the gauge back.
  *
  * It is a grep with the limits of one: a heap reading spelled some other way —
  * a `v8.getHeapStatistics()` destructure, an import alias — would slip it. It

src/lib/export/__tests__/restore-backup-owner-mismatch.test.ts

@@ -61,7 +61,13 @@ import { auditLog } from "@/lib/auth/audit";
  */
 async function restore(): Promise<{ status: number }> {
   const outcome = await restoreBackup({
-    backup: { id: "b-1", userId: "user-A", data: "cipher" },
+    backup: {
+      id: "b-1",
+      userId: "user-A",
+      data: "cipher",
+      chunkCount: null,
+      chunkStreamId: null,
+    },
     actorUserId: "admin-1",
     ipAddress: null,
     restoreInstanceSettings: false,

src/lib/export/__tests__/restore-backup-round-trip.test.ts

@@ -471,7 +471,13 @@ async function roundTrip(
  */
 async function restore(): Promise<{ status: number }> {
   const outcome = await restoreBackup({
-    backup: { id: "b-1", userId: OWNER, data: "cipher" },
+    backup: {
+      id: "b-1",
+      userId: OWNER,
+      data: "cipher",
+      chunkCount: null,
+      chunkStreamId: null,
+    },
     actorUserId: "admin-1",
     ipAddress: null,
     restoreInstanceSettings: false,

src/lib/export/backup-blob.ts

@@ -1,52 +1,38 @@
 /**
- * The envelope a `DataBackup.data` row is stored in: compress, then encrypt.
+ * The envelope of a stored backup: compress, then encrypt.
  *
- * Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))`
- * straight to Postgres. For an account with a few hundred thousand
- * measurements that is four full copies of the record alive at the same
- * moment: the payload object graph, the JSON string, the base64 ciphertext at
- * 1.33× the JSON, and the copy the database driver makes of that parameter on
- * its way to the wire. Measured on a seeded 445 000-measurement account, the
- * JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap
- * it never reaches the insert, and on a bigger heap it only reaches it later.
- * The weekly pass was not slow; it could not finish.
+ * Why compress. A health record's JSON is extremely repetitive (the same
+ * twenty keys per row, timestamps sharing a prefix), so gzip takes it to about
+ * a tenth, and every copy after that shrinks with it. The backup also lives
+ * inside the database it would be needed to restore, so its size matters.
  *
- * Compressing first is what makes the whole tail of that pipeline cheap. A
- * health record's JSON is extremely repetitive — the same twenty keys per row,
- * timestamps sharing a prefix — so gzip takes that 242 MB to a few tens of
- * megabytes, and every copy after it shrinks with it. It is also the reason
- * the stored row stops being a liability of its own: the backup lives INSIDE
- * the database it would be needed to restore, so its size is not a cosmetic
- * concern.
+ * How it is written. The JSON goes into gzip a page at a time, and gzip's
+ * output is sealed in pieces of about a megabyte (`packBackupChunks`, the
+ * piece format in `backup-chunks.ts`), which `storeBackupBlob` puts into
+ * `data_backup_chunks`. Nothing in this process ever holds the whole JSON,
+ * the whole compressed copy or the whole ciphertext, so the size of a copy
+ * does not depend on the memory of the process that writes or reads it.
  *
- * Compressing first was not enough on its own. Even with a small stored blob,
- * `packBackupBlob` still needs the whole JSON as one argument, and building
- * that string is what exhausts the heap — measured under
- * `--max-old-space-size=450` on the seeded account: `FATAL ERROR: Reached heap
- * limit`. So the writer the weekly job actually uses is
- * `packBackupBlobStreaming`, which never sees a complete copy of the JSON, the
- * gzip output or the ciphertext: rows go in a page at a time, gzip and the
- * cipher consume them as they arrive, and only the base64 answer accumulates,
- * because the destination is a single `text` column and one value is what the
- * column takes.
- *
- * Every direction reads. Three shapes exist in the wild and all three restore:
- * the original `encrypt(json)`, the compressed `encrypt("HLZ1:" + gz)`, and
- * the streamed `~hlgcm1.…` form written from now on. An operator whose newest
- * usable copy predates any of this is exactly the person who needs it to work.
+ * Every earlier form still reads. Up to v1.39.1 a copy was one value in
+ * `data_backups.data`, in one of three shapes: the original `encrypt(json)`,
+ * the compressed `encrypt("HLZ1:" + gz)`, and the streamed `~hlgcm1.…` form.
+ * `openBackupBlob` below opens all three; an operator whose newest usable copy
+ * predates the pieces is exactly the person who needs it to work.
  */
 import { Buffer } from "node:buffer";
-import v8 from "node:v8";
 import { Readable } from "node:stream";
 import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
 
 import {
-  createStreamEncryptor,
   decrypt,
   decryptStream,
   encrypt,
   isStreamCiphertext,
 } from "@/lib/crypto";
+import {
+  sealBackupChunk,
+  BACKUP_CHUNK_BYTES,
+} from "@/lib/export/backup-chunks";
 
 /**
  * Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd.
@@ -55,29 +41,16 @@ import {
  */
 const GZIP_MARKER = "HLZ1:";
 
-/** Serialised backup JSON → the string stored in `DataBackup.data`. */
+/**
+ * Serialised backup JSON → the single-value form stored in `DataBackup.data`
+ * before v1.39.2. Nothing in the app writes it any more; the tests use it to
+ * stand in for a copy written by an earlier release.
+ */
 export function packBackupBlob(json: string): string {
   const compressed = gzipSync(json).toString("base64");
   return encrypt(`${GZIP_MARKER}${compressed}`);
 }
 
-/**
- * The share of this process's heap limit one stored backup may become.
- *
- * The blob is the single copy this pipeline cannot stream away:
- * `data_backups.data` is one `text` column, so the row has to be one value,
- * and the driver makes a second copy of it on its way to the wire. Two copies
- * of the answer plus whatever the process legitimately holds is what has to
- * fit at once, so a fifth of the heap limit is one account's share.
- *
- * It is a share of the LIMIT, not of what is currently used. How large a
- * single value this process can hold depends on how it was started; it does
- * not depend on how long it has been running or on how much garbage is
- * waiting to be collected. The check this replaced read the latter, and so
- * aborted a 1.2 MB record on a server that had merely been up for a week.
- */
-const BLOB_HEAP_SHARE = 0.2;
-
 /** Bytes as an operator reads them. Kilobytes below a megabyte. */
 function size(bytes: number): string {
   const mb = bytes / 1024 / 1024;
@@ -85,23 +58,39 @@ function size(bytes: number): string {
   return mb < 100 ? `${mb.toFixed(1)} MB` : `${Math.round(mb)} MB`;
 }
 
-/** Default cap for one stored blob, in bytes. */
-export function defaultBackupBlobLimit(): number {
-  return Math.floor(v8.getHeapStatistics().heap_size_limit * BLOB_HEAP_SHARE);
+/**
+ * The largest stored copy of one account, in megabytes, unless the operator
+ * sets `BACKUP_MAX_STORED_MB`.
+ *
+ * Not a memory bound. A copy is written and read a piece at a time
+ * (`backup-chunks.ts`), so its size does not depend on the process; the only
+ * thing it takes is room in the database, which holds the old and the new
+ * copy side by side until the new one commits. The limit is there so a
+ * runaway copy stops with a message instead of filling the database volume.
+ * 2 GB is far past any record seen so far: the copy of an account with 2.6
+ * million readings is about 100 MB.
+ */
+const DEFAULT_MAX_STORED_MB = 2048;
+
+/** The stored-copy limit in bytes: `BACKUP_MAX_STORED_MB`, or the default. */
+export function defaultBackupStoreLimit(): number {
+  const raw = process.env.BACKUP_MAX_STORED_MB?.trim();
+  const mb = raw ? Number(raw) : NaN;
+  const effective = Number.isFinite(mb) && mb > 0 ? mb : DEFAULT_MAX_STORED_MB;
+  return Math.floor(effective * 1024 * 1024);
 }
 
-/**
- * Thrown when one account's encrypted backup outgrows what this process can
- * hold as a single value.
- *
- * The message states what was counted — the ciphertext written so far — and
- * the limit it crossed, because that is the pair an operator can act on. It
- * is about the record, and it is true: raising the heap raises the limit with
- * it, since the limit is derived from the heap.
- */
 /** The stable code of the 413 an upload too large to store is answered with. */
 export const BACKUP_UPLOAD_TOO_LARGE_CODE = "backup.upload.too_large";
 
+/**
+ * Thrown when one account's encrypted copy passes the stored-copy limit.
+ *
+ * The message names what was counted, the limit it crossed, where the limit
+ * comes from and what to do about it, because that is what an operator reading
+ * a failed job needs. Nothing was stored: the write is one transaction, so the
+ * previous copy is still in place.
+ */
 export class BackupBlobTooLargeError extends Error {
   readonly bytes: number;
   readonly limitBytes: number;
@@ -109,12 +98,12 @@ export class BackupBlobTooLargeError extends Error {
   constructor(bytes: number, limitBytes: number) {
     super(
       `Backup stopped after ${size(bytes)} of encrypted backup for one ` +
-        `account, over the ${size(limitBytes)} a single stored copy may ` +
-        `occupy here (a fifth of this process's ` +
-        `${size(v8.getHeapStatistics().heap_size_limit)} heap limit). This ` +
-        `account's record is genuinely too large to store as one row on this ` +
-        `host; raise the container's memory or NODE_OPTIONS=` +
-        `--max-old-space-size and the limit rises with it.`,
+        `account, over the ${size(limitBytes)} limit for one stored copy ` +
+        `(BACKUP_MAX_STORED_MB, default ${DEFAULT_MAX_STORED_MB}). The ` +
+        `previous copy is unchanged. To store a copy this size, set ` +
+        `BACKUP_MAX_STORED_MB in .env to a larger number of megabytes and ` +
+        `recreate the app container; the database needs room for two copies ` +
+        `of this size while the new one is written.`,
     );
     this.name = "BackupBlobTooLargeError";
     this.bytes = bytes;
@@ -132,117 +121,80 @@ export type BackupJsonProducer = (
   write: (chunk: string | Buffer) => Promise<void>,
 ) => Promise<unknown>;
 
-export interface PackBackupBlobOptions {
+export interface PackBackupOptions {
   /**
-   * Largest stored blob this call may produce, in bytes. Defaults to
-   * `defaultBackupBlobLimit()`. Tests pass an explicit value; nothing else
+   * Largest stored copy this call may produce, in bytes. Defaults to
+   * `defaultBackupStoreLimit()`. Tests pass an explicit value; nothing else
    * should need to.
    */
   maxBytes?: number;
+  /**
+   * How much compressed backup one piece carries. Defaults to
+   * `BACKUP_CHUNK_BYTES`; tests lower it so a small record spans many pieces.
+   */
+  chunkBytes?: number;
 }
 
 /**
- * Serialised backup JSON, produced in pieces → the stored string.
- *
- * The pipeline is JSON piece → gzip → AES-256-GCM → base64, with nothing
- * buffered end to end but the base64 answer. Kept for callers that need the
- * answer as one value; the weekly and manual backup no longer do, they hand
- * the pieces to Postgres as they come (`packBackupBlobInto` below, via
- * `storeBackupBlob`).
+ * Receives each sealed piece of the copy, in order, with its position.
+ * Awaited, so a slow destination holds the producer back.
  */
-export async function packBackupBlobStreaming(
-  producer: BackupJsonProducer,
-  options: PackBackupBlobOptions = {},
-): Promise<string> {
-  const pieces: string[] = [];
-  await packBackupBlobInto(
-    (piece) => {
-      pieces.push(piece);
-    },
-    producer,
-    options,
-  );
-  return pieces.join("");
-}
-
-/** Receives the stored string a piece at a time, in order. Awaited. */
-export type BackupBlobSink = (piece: string) => void | Promise<void>;
+export type BackupChunkSink = (sealed: Buffer, seq: number) => Promise<void>;
 
 /**
- * How much base64 to gather before handing it to the sink. Large enough that
- * a sink writing to the database makes a few dozen round trips for a large
- * record rather than thousands, small enough to be irrelevant to the heap.
- */
-const SINK_FLUSH_BYTES = 4 * 1024 * 1024;
-
-/**
- * Serialised backup JSON, produced in pieces → the stored string, delivered
- * in pieces.
+ * Serialised backup JSON, produced in pieces → sealed pieces of the stored
+ * copy, handed to `sink` as they fill.
  *
- * The concatenation of every piece `sink` receives is exactly the string
- * `packBackupBlobStreaming` returns. `producer` decides how big its JSON
- * pieces are; the gzip stream applies backpressure through the promise this
- * hands back, so a fast producer cannot outrun the compressor, and a slow
- * sink holds the producer back the same way, so neither side piles up.
- *
- * `maxBytes` still counts the whole answer. Nothing here holds it any more,
- * but every reader of a stored copy (restore, download, summary) opens it as
- * one value, so the limit is what keeps a copy restorable on this host.
+ * The pipeline is JSON piece → gzip → a piece of about `BACKUP_CHUNK_BYTES`
+ * → AES-256-GCM (`sealBackupChunk`). Nothing is held but the piece being
+ * filled: the gzip stream applies backpressure through the promise `write`
+ * hands back, and a full piece is sealed and sunk before the producer may
+ * write again. The last piece is always sealed at the end, marked as the last,
+ * even when it carries nothing, because that mark is what tells a reader the
+ * copy is complete.
  */
-export async function packBackupBlobInto(
-  sink: BackupBlobSink,
+export async function packBackupChunks(
+  sink: BackupChunkSink,
+  streamId: string,
   producer: BackupJsonProducer,
-  options: PackBackupBlobOptions = {},
-): Promise<void> {
-  const limitBytes = options.maxBytes ?? defaultBackupBlobLimit();
-  const encryptor = createStreamEncryptor();
-  let pending: string[] = [encryptor.header];
-  let pendingBytes = encryptor.header.length;
-  let heldBytes = encryptor.header.length;
+  options: PackBackupOptions = {},
+): Promise<{ chunks: number; bytes: number }> {
+  const limitBytes = options.maxBytes ?? defaultBackupStoreLimit();
+  const chunkBytes = options.chunkBytes ?? BACKUP_CHUNK_BYTES;
   const gzip = createGzip();
+  let pending: Buffer[] = [];
+  let pendingBytes = 0;
+  let seq = 0;
+  let stored = 0;
 
-  let failure: unknown = null;
   gzip.on("data", (chunk: Buffer) => {
-    try {
-      const piece = encryptor.update(chunk);
-      if (piece === "") return;
-      // Base64 is ASCII, so a character counted here is a byte written here.
-      heldBytes += piece.length;
-      if (heldBytes > limitBytes) {
-        throw new BackupBlobTooLargeError(heldBytes, limitBytes);
-      }
-      pending.push(piece);
-      pendingBytes += piece.length;
-    } catch (err) {
-      failure ??= err;
-      gzip.destroy(err as Error);
-    }
+    pending.push(chunk);
+    pendingBytes += chunk.byteLength;
   });
-
   const finished = new Promise<void>((resolve, reject) => {
     gzip.on("end", resolve);
     gzip.on("error", reject);
   });
-  // A failure raised inside the `data` handler destroys the stream, which
-  // rejects this promise — and the producer's own `write` rethrows the same
-  // failure first, so nothing ever awaits it. Marking it handled keeps a
-  // failure that IS being reported from also surfacing as an unhandled
-  // rejection; `await finished` below still sees the rejection.
+  // The producer's own failure is what gets reported; this only keeps the
+  // same rejection from also surfacing as unhandled. `await finished` below
+  // still sees it.
   finished.catch(() => {});
 
-  const flush = async (force: boolean): Promise<void> => {
-    if (pendingBytes === 0 || (!force && pendingBytes < SINK_FLUSH_BYTES)) {
-      return;
-    }
-    const piece = pending.join("");
+  const seal = async (last: boolean): Promise<void> => {
+    const payload = pending.length === 1 ? pending[0]! : Buffer.concat(pending);
     pending = [];
     pendingBytes = 0;
-    await sink(piece);
+    const sealed = sealBackupChunk(streamId, seq, last, payload);
+    stored += sealed.byteLength;
+    if (stored > limitBytes) {
+      throw new BackupBlobTooLargeError(stored, limitBytes);
+    }
+    await sink(sealed, seq);
+    seq += 1;
   };
 
   const write = async (chunk: string | Buffer): Promise<void> => {
-    if (failure) throw failure;
-    await flush(false);
+    while (pendingBytes >= chunkBytes) await seal(false);
     if (gzip.write(chunk, "utf8")) return;
     await new Promise<void>((resolve, reject) => {
       const onDrain = () => {
@@ -266,12 +218,9 @@ export async function packBackupBlobInto(
   }
   gzip.end();
   await finished;
-  if (failure) throw failure;
-
-  const tail = encryptor.final();
-  pending.push(tail);
-  pendingBytes += tail.length;
-  await flush(true);
+  while (pendingBytes >= chunkBytes) await seal(false);
+  await seal(true);
+  return { chunks: seq, bytes: stored };
 }
 
 /**

src/lib/export/backup-chunks.ts

@@ -0,0 +1,162 @@
+/**
+ * The sealed form of one piece of a stored backup (`data_backup_chunks.data`).
+ *
+ * Why pieces. A stored backup used to be one value in one row, and one value
+ * has to pass through this process whole on its way in and out: the driver
+ * binds it as one parameter and returns it as one string. So the largest copy
+ * a host could keep was a share of the heap, 105 MB in the default 1 GB
+ * container, and an account with 1.75 million readings was past it (#1031).
+ * Stored as ordered pieces of about a megabyte, a copy of any size passes
+ * through a piece at a time.
+ *
+ * Why each piece is sealed on its own. The copy used to be one AES-256-GCM
+ * stream with the tag at the end, which authenticates the whole only once the
+ * whole has been read; checking it before releasing a byte meant holding every
+ * byte, which is the thing pieces are meant to avoid. Sealing every piece
+ * separately lets the reader check each one before it passes it on. On its own
+ * that would authenticate the pieces but not their order or their number, so
+ * the sealed plaintext of every piece starts with a header that says which
+ * copy it belongs to, where it goes and whether it is the last:
+ *
+ *   "HLBC" | version 0x01 | streamId (16 bytes) | seq (u32 BE) | flags (u8) | payload
+ *
+ * `streamId` is random per written copy, so a piece from an earlier copy or
+ * from another account's copy is refused; `seq` pins the position, so a piece
+ * moved or dropped is refused; the last-piece flag, set on exactly one piece,
+ * is what a reader needs to tell a complete copy from one cut short. That is
+ * the construction known as STREAM (Hoang, Reyhanitabar, Rogaway, Vizár 2015),
+ * with the position and the flag carried inside the authenticated plaintext
+ * rather than in the nonce. Every piece has its own random 96-bit IV.
+ *
+ * The header sits inside the ciphertext rather than beside it, deliberately:
+ * key rotation re-seals a piece without knowing what it holds
+ * (`reencryptBytesToActive`), and the binding survives that untouched. A
+ * binding held as associated data outside the ciphertext would need a rotation
+ * path of its own for this one column.
+ *
+ * The payload is gzip output. The pieces of one copy, opened in order and
+ * concatenated, are one gzip stream of the backup JSON.
+ */
+import { Buffer } from "node:buffer";
+import { randomBytes } from "node:crypto";
+
+import { decryptBytes, encryptBytes } from "@/lib/crypto";
+
+const MAGIC = Buffer.from("HLBC", "ascii");
+const VERSION = 0x01;
+const STREAM_ID_BYTES = 16;
+const FLAG_LAST = 0x01;
+const HEADER_BYTES = MAGIC.byteLength + 1 + STREAM_ID_BYTES + 4 + 1;
+
+/**
+ * How much compressed backup one piece carries. Large enough that a big copy
+ * is a few hundred rows rather than tens of thousands, small enough that a
+ * reader or the key rotation holding a batch of pieces is a few tens of
+ * megabytes whatever the account.
+ */
+export const BACKUP_CHUNK_BYTES = 1024 * 1024;
+
+/**
+ * A stored copy whose pieces do not add up to the copy that was written: a
+ * piece missing, moved, altered, taken from another copy, or the last one
+ * gone. Also what a piece that will not decrypt is reported as, since the
+ * reader cannot tell a dropped key from altered bytes and the operator's next
+ * step is the same.
+ */
+export class BackupIntegrityError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "BackupIntegrityError";
+  }
+}
+
+/** A fresh stream id for one written copy, as the hex the row stores. */
+export function newChunkStreamId(): string {
+  return randomBytes(STREAM_ID_BYTES).toString("hex");
+}
+
+function streamIdBytes(streamId: string): Buffer {
+  if (!/^[0-9a-f]{32}$/.test(streamId)) {
+    throw new BackupIntegrityError("The stored copy's stream id is malformed.");
+  }
+  return Buffer.from(streamId, "hex");
+}
+
+/** Seal one piece of the copy `streamId` at position `seq`. */
+export function sealBackupChunk(
+  streamId: string,
+  seq: number,
+  last: boolean,
+  payload: Buffer,
+): Buffer {
+  const header = Buffer.alloc(HEADER_BYTES);
+  let at = MAGIC.copy(header, 0);
+  header[at++] = VERSION;
+  at += streamIdBytes(streamId).copy(header, at);
+  header.writeUInt32BE(seq, at);
+  at += 4;
+  header[at] = last ? FLAG_LAST : 0;
+  return encryptBytes(Buffer.concat([header, payload]));
+}
+
+export interface ExpectedChunk {
+  streamId: string;
+  seq: number;
+  /** Whether this position is the copy's last, from the row's piece count. */
+  last: boolean;
+}
+
+/**
+ * Authenticate one sealed piece and check it is the piece expected at this
+ * position of this copy. Returns its payload; throws `BackupIntegrityError`
+ * for anything else, before any of the payload is released.
+ */
+export function openBackupChunk(
+  sealed: Uint8Array,
+  expected: ExpectedChunk,
+): Buffer {
+  let plain: Buffer;
+  try {
+    plain = decryptBytes(Buffer.from(sealed));
+  } catch (err) {
+    throw new BackupIntegrityError(
+      `Piece ${expected.seq} of the stored copy could not be decrypted: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+  if (
+    plain.byteLength < HEADER_BYTES ||
+    !plain.subarray(0, MAGIC.byteLength).equals(MAGIC) ||
+    plain[MAGIC.byteLength] !== VERSION
+  ) {
+    throw new BackupIntegrityError(
+      `Piece ${expected.seq} of the stored copy is not a backup piece.`,
+    );
+  }
+  let at = MAGIC.byteLength + 1;
+  const streamId = plain.subarray(at, at + STREAM_ID_BYTES);
+  at += STREAM_ID_BYTES;
+  const seq = plain.readUInt32BE(at);
+  at += 4;
+  const last = (plain[at]! & FLAG_LAST) !== 0;
+
+  if (!streamId.equals(streamIdBytes(expected.streamId))) {
+    throw new BackupIntegrityError(
+      `Piece ${expected.seq} belongs to a different stored copy.`,
+    );
+  }
+  if (seq !== expected.seq) {
+    throw new BackupIntegrityError(
+      `The piece stored at position ${expected.seq} was written as piece ${seq}.`,
+    );
+  }
+  if (last !== expected.last) {
+    throw new BackupIntegrityError(
+      last
+        ? `Piece ${seq} was written as the last, but the copy lists more.`
+        : `Piece ${seq} is the last the copy lists, but more were written.`,
+    );
+  }
+  return plain.subarray(HEADER_BYTES);
+}

src/lib/export/backup-plan.ts

@@ -666,6 +666,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
     "Replay protection for requests this host already answered. Restoring it would suppress a legitimate new request.",
   DataBackup:
     "The backup catalogue itself. A backup that contains the list of backups is a recursion with no reader.",
+  DataBackupChunk:
+    "The pieces a stored backup is kept in. They are that backup, not part of the account's record, and a backup that contained its own pieces would be the same recursion.",
   BackupRestoreJob:
     "The record of a restore this host ran from one of its own stored copies. It names a backup row that exists only in this database, so carrying it would describe a restore the receiving host never performed.",
   OffhostBackupState:

src/lib/export/full-backup-stream.ts

@@ -66,10 +66,11 @@ const PROGRESS_EVERY_ROWS = 50_000;
  * What this writer holds is bounded by its own shape and needs no gauge to
  * prove: the three tables that scale with a record are pulled a page at a
  * time, every other section is released as soon as its JSON is in the sink,
- * and at most `FLUSH_BYTES` of text is ever pending. The one copy that DOES
- * grow with the record is the sink's — the stored blob — so that is where the
- * bound belongs, and `packBackupBlobStreaming` enforces it there against bytes
- * it counted itself rather than against a heap the rest of the process shares.
+ * and at most `FLUSH_BYTES` of text is ever pending. The sink does not hold
+ * the record either: the stored copy goes to the database in sealed pieces
+ * (`packBackupChunks`), and the one limit on its size is a storage limit it
+ * counts against its own bytes, not against a heap the rest of the process
+ * shares.
  */
 
 /**

src/lib/export/restore-backup.ts

@@ -34,8 +34,11 @@ import { auditLog } from "@/lib/auth/audit";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  openStoredBackup,
+  type StoredBackupRef,
+} from "@/lib/export/stored-backup";
 import {
   insertMeasurementRows,
   type MeasurementInsertRow,
@@ -244,8 +247,8 @@ export type RestoreFailureCode =
   | "unexpected";
 
 export interface RestoreBackupInput {
-  /** The stored copy, as read from `data_backups`. */
-  backup: { id: string; userId: string; data: string };
+  /** The stored copy, as read from `data_backups` (`STORED_BACKUP_SELECT`). */
+  backup: StoredBackupRef;
   /** The admin who asked for the restore; the audit rows name them. */
   actorUserId: string;
   /** Where the request came from, when there was one. */
@@ -333,10 +336,11 @@ export async function restoreBackup(
 
   // Opened, not unpacked: the JSON of a large record is longer than any
   // string V8 can hold (#1031), so it is read as a stream below. Opening
-  // authenticates the whole ciphertext first, as unpacking did.
+  // authenticates every stored piece first, and checks they add up to the
+  // copy that was written, before anything is read or deleted.
   let source: BackupSource;
   try {
-    source = openBackupBlob(backup.data);
+    source = await openStoredBackup(prisma, backup);
   } catch (err) {
     await auditLog("admin.backups.restore.failed", {
       userId: input.actorUserId,
@@ -768,6 +772,7 @@ export async function restoreBackup(
           // their cumulative dailies too (Fitbit, Google Health) and have
           // never carried provenance — marking those would invent a state
           // they were never in. Ordinary point rows keep NULL honestly.
+          //
           // Only an ABSENT field is the old-backup case. A current backup
           // writes the field for every row, and an explicit null is the row's
           // real state: the nightly daily-mean fold writes Apple Health

src/lib/export/store-backup-blob.ts

@@ -1,35 +1,36 @@
 /**
- * Write one account's stored backup (`data_backups.data`) without ever
- * holding the stored copy in this process.
+ * Write one account's stored backup without ever holding the stored copy in
+ * this process.
  *
- * Why. `packBackupBlobStreaming` streams the JSON through gzip and the cipher,
- * but its answer is one string, because the column takes one value. Measured
- * on a seeded account of 1.25 million measurements: the answer is 64 MB of
- * base64, and between the pieces, the joined string and the copies the
- * database driver makes to bind it, the backup grew the process by about
- * 640 MB of resident memory (309 MB → 946 MB). The default compose stack runs
- * the web server and the worker in one container capped at 1 GB, so on a
- * record of that size the weekly or manual backup is the thing that takes the
- * container down; a killed worker leaves its job `active` until pg-boss
- * expires it two hours later as `job timed out`.
+ * The copy goes into `data_backup_chunks` as sealed pieces of about a
+ * megabyte (`packBackupChunks`, the piece format in `backup-chunks.ts`), so
+ * this process holds one piece at a time and the size of a copy has nothing to
+ * do with the memory of the process that writes it. Up to v1.39.1 the pieces
+ * were joined into one value in `data_backups.data`, which every reader then
+ * had to take whole; that capped a copy at a fifth of the heap, 105 MB in the
+ * default 1 GB container, and an account with 1.75 million readings was past
+ * it (#1031).
  *
- * What instead. The pieces go to Postgres as they come, a few megabytes at a
- * time, into a temporary table on the transaction's own connection, and one
- * statement assembles them into the row with `string_agg`. The process never
- * holds more than one piece; Postgres builds the value it was going to store
- * anyway.
+ * All of it is one transaction, so the account's previous copy stays in place,
+ * readable, until the new one is complete: a failure halfway through rolls
+ * back to it.
  *
- * All of it is one transaction, so the account's previous copy stays in place
- * until the new one is complete: a failure halfway through rolls back to it,
- * and the temporary table goes with the transaction either way.
+ * The pieces are written through the model API, never through a raw query.
+ * Prisma caches the plan of every raw query keyed by its parameter values, the
+ * last hundred of them, so a raw INSERT carrying a one-megabyte piece keeps
+ * that piece alive after the statement: measured, a hundred such inserts left
+ * 160 MB of heap behind that no collection frees, which is the memory bound
+ * this module exists to remove. A model create is planned once for its shape
+ * and keeps nothing.
  */
 import type { PrismaClient } from "@/generated/prisma/client";
 
 import {
-  packBackupBlobInto,
+  packBackupChunks,
   type BackupJsonProducer,
-  type PackBackupBlobOptions,
+  type PackBackupOptions,
 } from "@/lib/export/backup-blob";
+import { newChunkStreamId } from "@/lib/export/backup-chunks";
 
 /**
  * How long the storing transaction may stay open. The job around it expires
@@ -53,66 +54,66 @@ export interface StoreBackupBlobInput {
   userId: string;
   /** `DataBackup.type`: `WEEKLY_AUTO` for the scheduled and manual pass. */
   type: string;
+  /**
+   * The account the copy belongs to, when it is known only once the producer
+   * has finished: an uploaded file names its owner inside itself. The row is
+   * written under `userId` meanwhile (the uploading admin) and moved to this
+   * account before the transaction commits, so nothing outside it ever sees
+   * the row under the wrong owner.
+   */
+  ownerAfterRead?: () => string;
 }
 
 /**
- * Pack `producer`'s JSON into the stored envelope and upsert it as the
- * `(userId, type)` backup. Resolves to the row's id and the stored size.
- *
- * `input` may be a function, read once the producer has finished: an
- * uploaded file names its owner somewhere inside itself, and the upload
- * route only knows who that is once it has read the file through.
+ * Pack `producer`'s JSON into sealed pieces and store them as the
+ * `(userId, type)` backup, replacing that backup's previous copy. Resolves to
+ * the row's id, the stored size in bytes and the number of pieces.
  */
 export async function storeBackupBlob(
   prisma: PrismaClient,
-  input: StoreBackupBlobInput | (() => StoreBackupBlobInput),
+  input: StoreBackupBlobInput,
   producer: BackupJsonProducer,
-  options: PackBackupBlobOptions = {},
-): Promise<{ id: string; bytes: number }> {
+  options: PackBackupOptions = {},
+): Promise<{ id: string; bytes: number; chunks: number }> {
   return prisma.$transaction(
     async (tx) => {
       await tx.$executeRawUnsafe(
         `SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`,
       );
-      await tx.$executeRaw`
-        CREATE TEMP TABLE backup_blob_parts (
-          seq integer PRIMARY KEY,
-          piece text NOT NULL
-        ) ON COMMIT DROP
-      `;
+      // The row first, so the pieces have something to belong to. An existing
+      // copy is cleared here, inside the transaction: outside it the previous
+      // copy stays whole and readable until this one commits.
+      const row = await tx.dataBackup.upsert({
+        where: { userId_type: { userId: input.userId, type: input.type } },
+        update: { data: null },
+        create: { userId: input.userId, type: input.type, data: null },
+        select: { id: true },
+      });
+      await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
 
-      let seq = 0;
-      let bytes = 0;
-      await packBackupBlobInto(
-        async (piece) => {
-          await tx.$executeRaw`
-            INSERT INTO backup_blob_parts (seq, piece) VALUES (${seq}, ${piece})
-          `;
-          seq += 1;
-          bytes += piece.length;
+      const streamId = newChunkStreamId();
+      const { chunks, bytes } = await packBackupChunks(
+        async (sealed, seq) => {
+          await tx.dataBackupChunk.create({
+            data: { backupId: row.id, seq, data: new Uint8Array(sealed) },
+            select: { id: true },
+          });
         },
+        streamId,
         producer,
         options,
       );
 
-      const target = typeof input === "function" ? input() : input;
-      // The row first, through Prisma, so a new account's backup gets its id
-      // the same way every other row does; the data it briefly carries is
-      // never visible outside this transaction.
-      const row = await tx.dataBackup.upsert({
-        where: { userId_type: { userId: target.userId, type: target.type } },
-        update: { createdAt: new Date() },
-        create: { userId: target.userId, type: target.type, data: "" },
-        select: { id: true },
+      await tx.dataBackup.update({
+        where: { id: row.id },
+        data: {
+          ...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}),
+          chunkCount: chunks,
+          chunkStreamId: streamId,
+          createdAt: new Date(),
+        },
       });
-      await tx.$executeRaw`
-        UPDATE data_backups
-        SET data = (
-          SELECT string_agg(piece, '' ORDER BY seq) FROM backup_blob_parts
-        )
-        WHERE id = ${row.id}
-      `;
-      return { id: row.id, bytes };
+      return { id: row.id, bytes, chunks };
     },
     { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 },
   );

src/lib/export/stored-backup.ts

@@ -0,0 +1,137 @@
+/**
+ * Read a stored backup (`data_backups` + `data_backup_chunks`) back as its
+ * JSON, whichever form it was stored in.
+ *
+ * Two forms exist. From v1.39.2 a copy is a run of sealed pieces
+ * (`backup-chunks.ts`), read here one piece at a time, so a copy of any size
+ * passes through this process a megabyte at a time. Before that a copy was one
+ * value in `data_backups.data`; `openBackupBlob` still opens those as it
+ * always did. Nothing converts the old form: the weekly copy is replaced in
+ * the new form by the next weekly run, and an uploaded copy stays readable as
+ * it is until someone deletes it.
+ *
+ * Integrity before anything else. `openStoredBackup` walks every piece once
+ * before it hands out a source: each is authenticated, and each has to name
+ * this copy, its own position, and (for the last) that it is the last. A copy
+ * with a piece missing, moved, altered, borrowed from another copy or cut
+ * short is refused there, which is before any restore deletes a row. Every
+ * later read through the source checks each piece again as it goes, so a copy
+ * replaced while a restore was running fails that read too, and the restore's
+ * transaction rolls back rather than writing a mixture.
+ */
+import { Readable, pipeline } from "node:stream";
+import { createGunzip } from "node:zlib";
+
+import type { PrismaClient } from "@/generated/prisma/client";
+import { openBackupBlob } from "@/lib/export/backup-blob";
+import {
+  BackupIntegrityError,
+  openBackupChunk,
+} from "@/lib/export/backup-chunks";
+import type { BackupSource } from "@/lib/export/streamed-backup";
+
+/** The columns a reader needs to open a stored copy, in either form. */
+export const STORED_BACKUP_SELECT = {
+  id: true,
+  userId: true,
+  data: true,
+  chunkCount: true,
+  chunkStreamId: true,
+} as const;
+
+export interface StoredBackupRef {
+  id: string;
+  userId: string;
+  data: string | null;
+  chunkCount: number | null;
+  chunkStreamId: string | null;
+}
+
+type ChunkReader = Pick<PrismaClient, "dataBackupChunk">;
+
+/**
+ * A value that changes whenever the stored copy is replaced, for the restore
+ * job's check that the copy it was asked to restore is still the one there.
+ * For a chunked copy that is its stream id, which a new copy always changes
+ * and a key rotation never does; for a single-value copy it is the value.
+ */
+export function storedBackupIdentity(backup: StoredBackupRef): string {
+  if (backup.chunkStreamId != null) {
+    return `chunks:${backup.chunkStreamId}:${backup.chunkCount ?? ""}`;
+  }
+  return backup.data ?? "";
+}
+
+/** The pieces of one chunked copy, opened and checked, in order. */
+async function* openedChunks(
+  prisma: ChunkReader,
+  backupId: string,
+  streamId: string,
+  count: number,
+): AsyncGenerator<Buffer> {
+  for (let seq = 0; seq < count; seq++) {
+    const row = await prisma.dataBackupChunk.findUnique({
+      where: { backupId_seq: { backupId, seq } },
+      select: { data: true },
+    });
+    if (!row) {
+      throw new BackupIntegrityError(
+        `Piece ${seq} of ${count} of the stored copy is missing.`,
+      );
+    }
+    yield openBackupChunk(row.data, {
+      streamId,
+      seq,
+      last: seq === count - 1,
+    });
+  }
+}
+
+/**
+ * Open a stored copy as a source of its JSON bytes, openable as many times as
+ * the caller needs. Rejects when the copy cannot be read whole: a key that is
+ * no longer configured, or a copy whose pieces do not add up (see the module
+ * comment). Nothing of the JSON is released before that check has passed.
+ */
+export async function openStoredBackup(
+  prisma: ChunkReader,
+  backup: StoredBackupRef,
+): Promise<BackupSource> {
+  if (backup.chunkStreamId == null || backup.chunkCount == null) {
+    if (backup.data == null) {
+      throw new BackupIntegrityError("The stored copy has no content.");
+    }
+    return openBackupBlob(backup.data);
+  }
+  const { id, chunkStreamId: streamId, chunkCount: count } = backup;
+  if (count < 1) {
+    throw new BackupIntegrityError("The stored copy lists no pieces.");
+  }
+  // Pieces beyond the count would be ignored by the reads below, but they are
+  // not something this code ever writes, so their presence is refused too.
+  const stored = await prisma.dataBackupChunk.count({
+    where: { backupId: id },
+  });
+  if (stored !== count) {
+    throw new BackupIntegrityError(
+      `The stored copy lists ${count} pieces but ${stored} are stored.`,
+    );
+  }
+  // The whole copy, checked once before anyone reads a byte of it.
+  for await (const piece of openedChunks(prisma, id, streamId, count)) {
+    void piece;
+  }
+
+  return () => {
+    const gunzip = createGunzip();
+    // `pipeline`, not `pipe`: a piece that fails its check has to fail the
+    // reader, and `pipe` would leave the reader waiting for an end that never
+    // comes.
+    pipeline(
+      Readable.from(openedChunks(prisma, id, streamId, count)),
+      gunzip,
+      () => {},
+    );
+    return gunzip;
+  };
+}

src/lib/jobs/backup-restore.ts

@@ -62,6 +62,10 @@ import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
 import { withBackgroundEvent } from "@/lib/logging/background";
 import { annotate } from "@/lib/logging/context";
 import { isP2002 } from "@/lib/prisma-errors";
+import {
+  STORED_BACKUP_SELECT,
+  storedBackupIdentity,
+} from "@/lib/export/stored-backup";
 import {
   restoreBackup,
   RESTORE_SECTION_STEPS,
@@ -190,9 +194,12 @@ export function jobFactCode(code: BackupRestoreFailureCode): string {
   return code.replace(/\./g, "_");
 }
 
-/** SHA-256 of a stored copy, to tell whether it changed while a job waited. */
-export function backupDigest(data: string): string {
-  return createHash("sha256").update(data).digest("hex");
+/**
+ * SHA-256 of a stored copy's identity (`storedBackupIdentity`), to tell
+ * whether the copy was replaced while a job waited.
+ */
+export function backupDigest(identity: string): string {
+  return createHash("sha256").update(identity).digest("hex");
 }
 
 const viewSelect = {
@@ -557,7 +564,7 @@ export async function runBackupRestoreJob(
   try {
     const backup = await prisma.dataBackup.findUnique({
       where: { id: row.backupId },
-      select: { id: true, userId: true, data: true },
+      select: STORED_BACKUP_SELECT,
     });
     if (!backup || backup.userId !== row.userId) {
       await finish("failed", {
@@ -569,7 +576,7 @@ export async function runBackupRestoreJob(
       });
       return jobDone({ refused: "backup_not_found" });
     }
-    if (backupDigest(backup.data) !== row.backupDigest) {
+    if (backupDigest(storedBackupIdentity(backup)) !== row.backupDigest) {
       await finish("failed", {
         failure: {
           code: "backup_changed",

src/lib/jobs/reminder/backup-handlers.ts

@@ -146,13 +146,11 @@ export async function handleDataBackup(
         try {
           // Streamed, compressed, then encrypted (the record contains
           // sensitive health information), and stored a piece at a time.
-          // Nothing between the database rows and the stored row exists as a
-          // whole in this process: the payload goes into gzip a page at a
-          // time, the cipher consumes gzip's output as it comes, and the
-          // ciphertext goes back to Postgres in pieces that one statement
-          // assembles into the row (`storeBackupBlob`). The stored copy used
-          // to be built here as one string first, which on a large record
-          // added several hundred megabytes to a container capped at 1 GB.
+          // Nothing between the database rows and the stored copy exists as
+          // a whole in this process: the payload goes into gzip a page at a
+          // time, and gzip's output is sealed and stored in pieces of about a
+          // megabyte (`storeBackupBlob`). The copy is kept as those pieces,
+          // so its size does not depend on this process's memory (#1031).
           const { bytes: storedBytes } = await storeBackupBlob(
             prisma,
             { userId: user.id, type: "WEEKLY_AUTO" },
@@ -168,11 +166,9 @@ export async function handleDataBackup(
           // it rides out as a count so the weekly run is not retried for the
           // whole cohort.
           //
-          // A record whose stored copy does not fit this process lands here
-          // too, and that is the point of the size limit the envelope writer
-          // enforces: before it existed the same condition was an uncatchable
-          // V8 abort that restarted the instance, so one account's size was a
-          // denial of service on every other account on the host.
+          // A copy over the stored-copy limit (`BACKUP_MAX_STORED_MB`) lands
+          // here too, counted as oversized, with the setting to raise named
+          // in the message.
           usersFailed++;
           lastError = err;
           if (err instanceof BackupBlobTooLargeError) oversized++;
@@ -183,8 +179,8 @@ export async function handleDataBackup(
       // The stored size is the one number that says whether this pass is
       // heading back towards the wall it hit before: it tracks the record.
       evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes);
-      // How many accounts the envelope writer stopped rather than let the
-      // process die. Non-zero means this host needs more memory, not a retry.
+      // How many accounts' copies passed the stored-copy limit. Non-zero
+      // means raising `BACKUP_MAX_STORED_MB`, not a retry.
       evt.addMeta("data_backup_records_oversized", oversized);
       evt.setBackground({
         task_name: "job.data_backup",

src/lib/openapi/routes/admin-backups.ts

@@ -7,8 +7,9 @@
  * Most of `/api/admin/backups/*` is deliberately unpublished: it is a
  * cookie-only console, `requireAdmin()` refuses every Bearer caller, and a
  * contract nobody can call against is a contract nobody reads. These two are
- * here for the opposite reason. Both open the encrypted copy in
- * `data_backups.data`, and both therefore have a refusal an operator meets on
+ * here for the opposite reason. Both open the encrypted stored copy (pieces in
+ * `data_backup_chunks`, or one value in `data_backups.data` for a copy written
+ * before v1.39.2), and both therefore have a refusal an operator meets on
  * the day a key rotation went one step too far — the case
  * `docs/ops/encryption-key-rotation.md` warns about. That refusal is a promise
  * (422, `backup.payload.undecryptable`, nothing changed) rather than whatever
@@ -138,7 +139,7 @@ const restoreJob = z
  */
 const undecryptableResponse = {
   description:
-    "The stored copy could not be opened: either the key that wrote it is no longer in `ENCRYPTION_KEYS` (a rotation that dropped the legacy entry too early), or the stored bytes are not the ones that were written. `meta.errorCode` = `backup.payload.undecryptable`. Nothing was changed.",
+    "The stored copy could not be opened: either the key that wrote it is no longer in `ENCRYPTION_KEYS` (a rotation that dropped the legacy entry too early), or the stored bytes are not the ones that were written (a stored piece missing, moved, altered, taken from another copy, or the copy cut short). Every piece is checked before any of the copy is read. `meta.errorCode` = `backup.payload.undecryptable`. Nothing was changed.",
   content: { "application/json": { schema: errorEnvelope } },
 };
 

tests/integration/admin-backups-upload.test.ts

@@ -15,7 +15,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 process.env.ENCRYPTION_KEY ??=
   "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
 
-import { unpackBackupBlob } from "@/lib/export/backup-blob";
+import { readStoredBackup } from "./stored-backup-read";
 
 import { cookieJar, headerJar } from "./mock-next-headers";
 import { getPrismaClient, truncateAllTables } from "./setup";
@@ -170,7 +170,7 @@ describe("POST /api/admin/backups/upload", () => {
     // received. Through the envelope rather than through `decrypt` alone: an
     // uploaded file is stored exactly the way the weekly worker stores one, so
     // the restore path reads both without knowing where the row came from.
-    const decrypted = JSON.parse(unpackBackupBlob(row!.data));
+    const decrypted = JSON.parse(await readStoredBackup(prisma, row!.id));
     expect(decrypted.userId).toBe(admin.id);
     expect(decrypted.measurements).toHaveLength(1);
 

tests/integration/backup-chunked-storage.test.ts

@@ -0,0 +1,567 @@
+/**
+ * #1031 — a stored backup kept in pieces.
+ *
+ * v1.39.1 wrote a backup a piece at a time but still joined the pieces into
+ * one value in `data_backups.data`, and one value has to pass through the app
+ * whole. So a copy could be at most a fifth of the heap, 105 MB in a 1 GB
+ * container, and an account with 1.75 million readings could not be backed up
+ * at all. A copy is now a run of sealed pieces in `data_backup_chunks`, and
+ * this file pins what that has to keep true:
+ *
+ *   - the weekly copy restores exactly, and the size it lists is what it takes;
+ *   - a piece dropped, moved, altered or cut off the end is refused, and the
+ *     refusal comes before the restore deletes anything;
+ *   - a copy stored in one value by an earlier release still restores;
+ *   - a run that fails halfway leaves the previous copy whole and readable;
+ *   - replacing, wiping or deleting a copy takes its pieces with it.
+ *
+ * The pieces are made small here (`chunkBytes`) so a record that fits a test
+ * run still spans many of them.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+process.env.ENCRYPTION_KEY ??=
+  "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+import { cookieJar, headerJar } from "./mock-next-headers";
+import { getPrismaClient, truncateAllTables } from "./setup";
+import { readStoredBackup } from "./stored-backup-read";
+import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
+import { restoreBackup } from "@/lib/export/restore-backup";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { STORED_BACKUP_SELECT } from "@/lib/export/stored-backup";
+
+vi.mock("next/headers", async () => {
+  const { cookieJar, headerJar } = await import("./mock-next-headers");
+  return {
+    headers: vi.fn(async () => ({
+      get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
+    })),
+    cookies: vi.fn(async () => ({
+      get: (name: string) => {
+        const value = cookieJar.get(name);
+        return value ? { name, value } : undefined;
+      },
+      set: (name: string, value: string) => {
+        cookieJar.set(name, value);
+      },
+      delete: (name: string) => {
+        cookieJar.delete(name);
+      },
+    })),
+  };
+});
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+const COUNT = 12_000;
+/** Small pieces, so a test-sized record spans many of them. */
+const SMALL = { chunkBytes: 4 * 1024 };
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+  cookieJar.clear();
+  headerJar.clear();
+});
+
+async function seedAccount() {
+  const prisma = getPrismaClient();
+  const user = await prisma.user.create({
+    data: {
+      username: "chunked-owner",
+      email: "chunked-owner@example.test",
+      role: "ADMIN",
+    },
+  });
+  const session = await prisma.session.create({
+    data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) },
+  });
+  cookieJar.set("healthlog_session", session.id);
+  const start = Date.UTC(2024, 0, 1);
+  await prisma.measurement.createMany({
+    data: Array.from({ length: COUNT }, (_, i) => ({
+      id: `m-${String(i).padStart(5, "0")}`,
+      userId: user.id,
+      type: i % 2 === 0 ? ("PULSE" as const) : ("WEIGHT" as const),
+      value: 50 + (i % 40) + 0.5,
+      unit: i % 2 === 0 ? "bpm" : "kg",
+      source: "APPLE_HEALTH" as const,
+      measuredAt: new Date(start + i * 600_000),
+      externalId: `hk-${i}`,
+    })),
+  });
+  return user;
+}
+
+async function measurementsOf(userId: string) {
+  return getPrismaClient().measurement.findMany({
+    where: { userId },
+    orderBy: { id: "asc" },
+    omit: { createdAt: true, updatedAt: true },
+  });
+}
+
+/** The weekly job's own write, with small pieces. */
+async function weeklyCopy(
+  userId: string,
+  options: { chunkBytes: number; maxBytes?: number } = SMALL,
+) {
+  const prisma = getPrismaClient();
+  return storeBackupBlob(
+    prisma,
+    { userId, type: "WEEKLY_AUTO" },
+    (write) =>
+      streamFullBackupJson(prisma, userId, write, {
+        purpose: "disaster-recovery",
+      }),
+    options,
+  );
+}
+
+async function restoreRoute(id: string) {
+  const { POST } = await import("./restore-job-driver");
+  return POST(
+    new Request(`http://localhost/api/admin/backups/${id}/restore`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ confirm: "RESTORE" }),
+    }) as never,
+    { params: Promise.resolve({ id }) },
+  );
+}
+
+/** The restore itself, without the route's admission check in front of it. */
+async function restoreDirect(id: string, actorUserId: string) {
+  const backup = await getPrismaClient().dataBackup.findUniqueOrThrow({
+    where: { id },
+    select: STORED_BACKUP_SELECT,
+  });
+  return restoreBackup({
+    backup,
+    actorUserId,
+    ipAddress: null,
+    restoreInstanceSettings: false,
+  });
+}
+
+async function chunksOf(backupId: string) {
+  return getPrismaClient().dataBackupChunk.findMany({
+    where: { backupId },
+    orderBy: { seq: "asc" },
+  });
+}
+
+describe("a backup stored in pieces", () => {
+  it("restores every measurement exactly, from many pieces", async () => {
+    const user = await seedAccount();
+    const before = await measurementsOf(user.id);
+    const { id, chunks } = await weeklyCopy(user.id);
+    expect(chunks).toBeGreaterThan(5);
+
+    const prisma = getPrismaClient();
+    const row = await prisma.dataBackup.findUniqueOrThrow({ where: { id } });
+    expect(row.data).toBeNull();
+    expect(row.chunkCount).toBe(chunks);
+    expect(await chunksOf(id)).toHaveLength(chunks);
+
+    await prisma.measurement.deleteMany({ where: { userId: user.id } });
+    const res = await restoreRoute(id);
+    expect(res.status, JSON.stringify(await res.clone().json())).toBe(200);
+    expect(await measurementsOf(user.id)).toEqual(before);
+  });
+
+  it("previews and downloads the copy the job stored", async () => {
+    const user = await seedAccount();
+    const { id } = await weeklyCopy(user.id);
+    const params = { params: Promise.resolve({ id }) };
+
+    const { GET: summary } =
+      await import("@/app/api/admin/backups/[id]/summary/route");
+    const preview = await summary(new Request("http://x") as never, params);
+    expect(preview.status).toBe(200);
+    const { data } = (await preview.json()) as {
+      data: { summary: { measurements: number } };
+    };
+    expect(data.summary.measurements).toBe(COUNT);
+
+    const { GET: download } =
+      await import("@/app/api/admin/backups/[id]/download/route");
+    const file = await download(new Request("http://x") as never, params);
+    expect(file.status).toBe(200);
+    const text = await file.text();
+    expect(Number(file.headers.get("content-length"))).toBe(
+      Buffer.byteLength(text),
+    );
+    expect(text).toBe(await readStoredBackup(getPrismaClient(), id));
+    expect(JSON.parse(text).measurements).toHaveLength(COUNT);
+  });
+
+  it("lists the size the copy takes in the database, in either form", async () => {
+    const user = await seedAccount();
+    const prisma = getPrismaClient();
+    const { id, bytes } = await weeklyCopy(user.id);
+    const legacy = await prisma.dataBackup.create({
+      data: {
+        userId: user.id,
+        type: "MANUAL_UPLOAD_1",
+        data: await legacyStreamedBlobFrom((write) =>
+          streamFullBackupJson(prisma, user.id, write, {
+            purpose: "disaster-recovery",
+          }),
+        ),
+      },
+    });
+
+    const { GET } = await import("@/app/api/admin/backups/route");
+    const res = await GET();
+    const { data } = (await res.json()) as {
+      data: { rows: Array<{ id: string; sizeBytes: number }> };
+    };
+    const sizes = new Map(data.rows.map((b) => [b.id, b.sizeBytes]));
+    expect(sizes.get(id)).toBe(bytes);
+    expect(sizes.get(legacy.id)).toBe(legacy.data!.length);
+  });
+});
+
+describe("a stored copy whose pieces do not add up is refused before anything is deleted", () => {
+  type Tamper = (backupId: string) => Promise<void>;
+  const prisma = () => getPrismaClient();
+
+  const cases: Array<[string, Tamper]> = [
+    [
+      "a piece dropped from the middle",
+      async (backupId) => {
+        await prisma().dataBackupChunk.deleteMany({
+          where: { backupId, seq: 2 },
+        });
+      },
+    ],
+    [
+      "two pieces swapped",
+      async (backupId) => {
+        const [a, b] = [
+          await prisma().dataBackupChunk.findUniqueOrThrow({
+            where: { backupId_seq: { backupId, seq: 1 } },
+          }),
+          await prisma().dataBackupChunk.findUniqueOrThrow({
+            where: { backupId_seq: { backupId, seq: 2 } },
+          }),
+        ];
+        await prisma().dataBackupChunk.update({
+          where: { id: a.id },
+          data: { data: b.data },
+        });
+        await prisma().dataBackupChunk.update({
+          where: { id: b.id },
+          data: { data: a.data },
+        });
+      },
+    ],
+    [
+      "one byte of a piece altered",
+      async (backupId) => {
+        const piece = await prisma().dataBackupChunk.findUniqueOrThrow({
+          where: { backupId_seq: { backupId, seq: 3 } },
+        });
+        const altered = new Uint8Array(piece.data);
+        altered[altered.byteLength - 20]! ^= 0x01;
+        await prisma().dataBackupChunk.update({
+          where: { id: piece.id },
+          data: { data: altered },
+        });
+      },
+    ],
+    [
+      "the last piece cut off, and the count lowered to match",
+      async (backupId) => {
+        const row = await prisma().dataBackup.findUniqueOrThrow({
+          where: { id: backupId },
+        });
+        const last = row.chunkCount! - 1;
+        await prisma().dataBackupChunk.deleteMany({
+          where: { backupId, seq: last },
+        });
+        await prisma().dataBackup.update({
+          where: { id: backupId },
+          data: { chunkCount: last },
+        });
+      },
+    ],
+  ];
+
+  for (const [name, tamper] of cases) {
+    it(`${name}: the route refuses, the job refuses, nothing changes`, async () => {
+      const user = await seedAccount();
+      const { id } = await weeklyCopy(user.id);
+      await tamper(id);
+      const before = await measurementsOf(user.id);
+      expect(before).toHaveLength(COUNT);
+
+      const res = await restoreRoute(id);
+      expect(res.status).toBe(422);
+      const body = (await res.json()) as { meta?: { errorCode?: string } };
+      expect(body.meta?.errorCode).toBe("backup.payload.undecryptable");
+
+      // The job refuses on its own too, for a copy tampered with after the
+      // route admitted it.
+      const outcome = await restoreDirect(id, user.id);
+      expect(outcome.ok).toBe(false);
+      if (!outcome.ok)
+        expect(outcome.code).toBe("backup.payload.undecryptable");
+
+      expect(await measurementsOf(user.id)).toEqual(before);
+    });
+  }
+
+  it("a piece from another account's copy put in its place", async () => {
+    const user = await seedAccount();
+    const other = await getPrismaClient().user.create({
+      data: { username: "other", email: "other@example.test" },
+    });
+    await getPrismaClient().measurement.createMany({
+      data: Array.from({ length: COUNT }, (_, i) => ({
+        userId: other.id,
+        type: "PULSE" as const,
+        value: 70,
+        unit: "bpm",
+        source: "APPLE_HEALTH" as const,
+        measuredAt: new Date(Date.UTC(2025, 0, 1) + i * 60_000),
+      })),
+    });
+    const mine = await weeklyCopy(user.id);
+    const theirs = await weeklyCopy(other.id);
+    const foreign = await getPrismaClient().dataBackupChunk.findUniqueOrThrow({
+      where: { backupId_seq: { backupId: theirs.id, seq: 1 } },
+    });
+    await getPrismaClient().dataBackupChunk.update({
+      where: { backupId_seq: { backupId: mine.id, seq: 1 } },
+      data: { data: foreign.data },
+    });
+    const before = await measurementsOf(user.id);
+
+    expect((await restoreRoute(mine.id)).status).toBe(422);
+    const outcome = await restoreDirect(mine.id, user.id);
+    expect(outcome.ok).toBe(false);
+    expect(await measurementsOf(user.id)).toEqual(before);
+  });
+
+  it("the preview and the download refuse the same copy with the same answer", async () => {
+    const user = await seedAccount();
+    const { id } = await weeklyCopy(user.id);
+    await getPrismaClient().dataBackupChunk.deleteMany({
+      where: { backupId: id, seq: 1 },
+    });
+    const params = { params: Promise.resolve({ id }) };
+    const { GET: summary } =
+      await import("@/app/api/admin/backups/[id]/summary/route");
+    const { GET: download } =
+      await import("@/app/api/admin/backups/[id]/download/route");
+    for (const res of [
+      await summary(new Request("http://x") as never, params),
+      await download(new Request("http://x") as never, params),
+    ]) {
+      expect(res.status).toBe(422);
+      const body = (await res.json()) as { meta?: { errorCode?: string } };
+      expect(body.meta?.errorCode).toBe("backup.payload.undecryptable");
+    }
+  });
+});
+
+describe("an uploaded backup file", () => {
+  it("is stored in pieces under the account the file names, not the admin who sent it", async () => {
+    const admin = await seedAccount();
+    const prisma = getPrismaClient();
+    const owner = await prisma.user.create({
+      data: { username: "file-owner", email: "file-owner@example.test" },
+    });
+    await prisma.measurement.create({
+      data: {
+        userId: owner.id,
+        type: "WEIGHT",
+        value: 71.5,
+        unit: "kg",
+        source: "MANUAL",
+        measuredAt: new Date(Date.UTC(2025, 5, 1)),
+      },
+    });
+    const weekly = await weeklyCopy(owner.id);
+    const file = await readStoredBackup(prisma, weekly.id);
+
+    const { POST: upload } =
+      await import("@/app/api/admin/backups/upload/route");
+    const res = await upload(
+      new Request("http://localhost/api/admin/backups/upload", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: file,
+      }) as never,
+    );
+    expect(res.status).toBe(201);
+    const { data } = (await res.json()) as { data: { id: string } };
+    const row = await prisma.dataBackup.findUniqueOrThrow({
+      where: { id: data.id },
+    });
+    expect(row.userId).toBe(owner.id);
+    expect(row.userId).not.toBe(admin.id);
+    expect(row.data).toBeNull();
+    expect(await readStoredBackup(prisma, data.id)).toBe(file);
+  });
+});
+
+describe("a copy stored in one value by an earlier release", () => {
+  it("still restores exactly", async () => {
+    const user = await seedAccount();
+    const prisma = getPrismaClient();
+    const before = await measurementsOf(user.id);
+    const legacy = await prisma.dataBackup.create({
+      data: {
+        userId: user.id,
+        type: "WEEKLY_AUTO",
+        data: await legacyStreamedBlobFrom((write) =>
+          streamFullBackupJson(prisma, user.id, write, {
+            purpose: "disaster-recovery",
+          }),
+        ),
+      },
+    });
+    await prisma.measurement.deleteMany({ where: { userId: user.id } });
+
+    expect((await restoreRoute(legacy.id)).status).toBe(200);
+    expect(await measurementsOf(user.id)).toEqual(before);
+  });
+
+  it("is replaced in pieces by the next weekly run", async () => {
+    const user = await seedAccount();
+    const prisma = getPrismaClient();
+    const legacy = await prisma.dataBackup.create({
+      data: {
+        userId: user.id,
+        type: "WEEKLY_AUTO",
+        data: await legacyStreamedBlobFrom(async (write) => {
+          await write("{}");
+        }),
+      },
+    });
+    const { id, chunks } = await weeklyCopy(user.id);
+    expect(id).toBe(legacy.id);
+    const row = await prisma.dataBackup.findUniqueOrThrow({ where: { id } });
+    expect(row.data).toBeNull();
+    expect(row.chunkCount).toBe(chunks);
+  });
+});
+
+describe("a restore queued against a copy that is then replaced", () => {
+  it("is refused as backup_changed, and nothing is deleted", async () => {
+    const user = await seedAccount();
+    const { id } = await weeklyCopy(user.id);
+    const { acceptingBoss } = await import("./restore-job-driver");
+    const { POST: queueRestore } =
+      await import("@/app/api/admin/backups/[id]/restore/route");
+    const { getGlobalBoss, setGlobalBoss } =
+      await import("@/lib/jobs/boss-instance");
+    const { readBackupRestoreJob, runBackupRestoreJob } =
+      await import("@/lib/jobs/backup-restore");
+
+    const previous = getGlobalBoss();
+    setGlobalBoss(acceptingBoss);
+    let queued: Response;
+    try {
+      queued = await queueRestore(
+        new Request(`http://localhost/api/admin/backups/${id}/restore`, {
+          method: "POST",
+          headers: { "content-type": "application/json" },
+          body: JSON.stringify({ confirm: "RESTORE" }),
+        }) as never,
+        { params: Promise.resolve({ id }) },
+      );
+    } finally {
+      setGlobalBoss(previous);
+    }
+    expect(queued.status).toBe(202);
+    const { data } = (await queued.json()) as { data: { jobId: string } };
+
+    // The next weekly run writes a new copy into the same row.
+    await weeklyCopy(user.id);
+    const before = await measurementsOf(user.id);
+    await runBackupRestoreJob(data.jobId).catch(() => undefined);
+    const job = await readBackupRestoreJob(data.jobId);
+    expect(job?.failure?.code).toBe("backup_changed");
+    expect(await measurementsOf(user.id)).toEqual(before);
+  });
+});
+
+describe("writing and removing copies", () => {
+  it("keeps the previous copy whole when a run fails after writing pieces", async () => {
+    const user = await seedAccount();
+    const prisma = getPrismaClient();
+    const first = await weeklyCopy(user.id);
+    const before = await prisma.dataBackup.findUniqueOrThrow({
+      where: { id: first.id },
+    });
+    const text = await readStoredBackup(prisma, first.id);
+
+    let written = 0;
+    await expect(
+      storeBackupBlob(
+        prisma,
+        { userId: user.id, type: "WEEKLY_AUTO" },
+        async (write) => {
+          await streamFullBackupJson(
+            prisma,
+            user.id,
+            async (chunk) => {
+              await write(chunk);
+              written += chunk.length;
+              if (written > 200 * 1024) throw new Error("read failed halfway");
+            },
+            { purpose: "disaster-recovery" },
+          );
+        },
+        SMALL,
+      ),
+    ).rejects.toThrow("read failed halfway");
+
+    const after = await prisma.dataBackup.findUniqueOrThrow({
+      where: { id: first.id },
+    });
+    expect(after.chunkStreamId).toBe(before.chunkStreamId);
+    expect(after.chunkCount).toBe(before.chunkCount);
+    expect(after.createdAt).toEqual(before.createdAt);
+    expect(await chunksOf(first.id)).toHaveLength(first.chunks);
+    expect(await readStoredBackup(prisma, first.id)).toBe(text);
+  });
+
+  it("stops at the stored-copy limit and keeps the previous copy", async () => {
+    const user = await seedAccount();
+    const first = await weeklyCopy(user.id);
+    await expect(
+      weeklyCopy(user.id, { ...SMALL, maxBytes: 16 * 1024 }),
+    ).rejects.toThrow(/BACKUP_MAX_STORED_MB/);
+    expect(await chunksOf(first.id)).toHaveLength(first.chunks);
+  });
+
+  it("replacing a copy leaves only the new copy's pieces", async () => {
+    const user = await seedAccount();
+    const prisma = getPrismaClient();
+    const first = await weeklyCopy(user.id);
+    const second = await weeklyCopy(user.id);
+    expect(second.id).toBe(first.id);
+    expect(await prisma.dataBackupChunk.count()).toBe(second.chunks);
+  });
+
+  it("wiping the account's data and deleting the account take the pieces with them", async () => {
+    const user = await seedAccount();
+    const prisma = getPrismaClient();
+    await weeklyCopy(user.id);
+    expect(await prisma.dataBackupChunk.count()).toBeGreaterThan(0);
+    // The wipe deletes `DataBackup` by owner; the pieces follow by cascade.
+    await prisma.dataBackup.deleteMany({ where: { userId: user.id } });
+    expect(await prisma.dataBackupChunk.count()).toBe(0);
+
+    await weeklyCopy(user.id);
+    await prisma.user.delete({ where: { id: user.id } });
+    expect(await prisma.dataBackupChunk.count()).toBe(0);
+  });
+});

tests/integration/backup-round-trip.test.ts

@@ -66,7 +66,7 @@ import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
 import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker";
 import { decryptNoteFromBytes } from "@/lib/labs/store";
 import { decryptContextFromBytes } from "@/lib/labs/biomarker-store";
-import { packBackupBlobStreaming } from "@/lib/export/backup-blob";
+import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob";
 import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan";
 import { POST } from "./restore-job-driver";
@@ -1356,7 +1356,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
         // form, which is what every row written before any of this existed
         // looks like — an operator's newest usable copy may well be one of
         // those, so every arm has to reach the restore route.
-        data: await packBackupBlobStreaming(async (write) => {
+        data: await legacyStreamedBlobFrom(async (write) => {
           await write(streamedJson);
         }),
       },

tests/integration/backup-streaming-memory.test.ts

@@ -32,11 +32,9 @@ import vm from "node:vm";
 
 import { beforeAll, afterAll, describe, expect, it } from "vitest";
 
-import {
-  BackupBlobTooLargeError,
-  packBackupBlobStreaming,
-  unpackBackupBlob,
-} from "@/lib/export/backup-blob";
+import { BackupBlobTooLargeError } from "@/lib/export/backup-blob";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { readStoredBackup } from "./stored-backup-read";
 import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
 import { getPrismaClient, truncateAllTables } from "./setup";
@@ -233,26 +231,30 @@ describe("weekly backup under a memory budget", () => {
     let peakHeld = 0;
     let chunks = 0;
 
-    const blob = await packBackupBlobStreaming(async (write) => {
-      const counts = await streamFullBackupJson(
-        prisma,
-        OWNER_ID,
-        async (chunk) => {
-          await write(chunk);
-          // Sampled rather than continuous: a forced collection per chunk
-          // would dominate the runtime, and one in forty still lands inside
-          // every phase of the walk.
-          if (chunks++ % 40 === 0) {
-            peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
-          }
-        },
-        // The purpose the weekly job asks for: tombstones and ciphertext ride
-        // verbatim, which is the arm that has to fit in memory.
-        { purpose: "disaster-recovery" },
-      );
-      expect(counts.measurements).toBe(MEASUREMENT_ROWS);
-      expect(counts.moodEntries).toBe(MOOD_ROWS);
-    });
+    const { id: blob } = await storeBackupBlob(
+      prisma,
+      { userId: OWNER_ID, type: "WEEKLY_AUTO" },
+      async (write) => {
+        const counts = await streamFullBackupJson(
+          prisma,
+          OWNER_ID,
+          async (chunk) => {
+            await write(chunk);
+            // Sampled rather than continuous: a forced collection per chunk
+            // would dominate the runtime, and one in forty still lands inside
+            // every phase of the walk.
+            if (chunks++ % 40 === 0) {
+              peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
+            }
+          },
+          // The purpose the weekly job asks for: tombstones and ciphertext ride
+          // verbatim, which is the arm that has to fit in memory.
+          { purpose: "disaster-recovery" },
+        );
+        expect(counts.measurements).toBe(MEASUREMENT_ROWS);
+        expect(counts.moodEntries).toBe(MOOD_ROWS);
+      },
+    );
     peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
     reportPeak("large record", peakHeld, liveHeapBytes());
 
@@ -263,7 +265,7 @@ describe("weekly backup under a memory budget", () => {
     ).toBeLessThan(STREAM_BUDGET_BYTES);
 
     // A blob that writes but does not read is worse than none.
-    const restored = JSON.parse(unpackBackupBlob(blob)) as {
+    const restored = JSON.parse(await readStoredBackup(prisma, blob)) as {
       measurements: Array<{ deletedAt: string | null }>;
       moodEntries: unknown[];
     };
@@ -307,21 +309,25 @@ describe("weekly backup under a memory budget", () => {
     let peak = baseline;
 
     let jsonBytes = 0;
-    const blob = await packBackupBlobStreaming(async (write) => {
-      const counts = await streamFullBackupJson(
-        prisma,
-        TINY_OWNER_ID,
-        async (chunk) => {
-          jsonBytes += chunk.length;
-          peak = Math.max(peak, process.memoryUsage().heapUsed);
-          await write(chunk);
-        },
-        { purpose: "disaster-recovery" },
-      );
-      expect(counts.measurements).toBe(1);
-    });
+    const { id: blob } = await storeBackupBlob(
+      prisma,
+      { userId: TINY_OWNER_ID, type: "WEEKLY_AUTO" },
+      async (write) => {
+        const counts = await streamFullBackupJson(
+          prisma,
+          TINY_OWNER_ID,
+          async (chunk) => {
+            jsonBytes += chunk.length;
+            peak = Math.max(peak, process.memoryUsage().heapUsed);
+            await write(chunk);
+          },
+          { purpose: "disaster-recovery" },
+        );
+        expect(counts.measurements).toBe(1);
+      },
+    );
 
-    const restored = JSON.parse(unpackBackupBlob(blob)) as {
+    const restored = JSON.parse(await readStoredBackup(prisma, blob)) as {
       measurements: unknown[];
     };
     expect(restored.measurements).toHaveLength(1);
@@ -336,14 +342,14 @@ describe("weekly backup under a memory budget", () => {
     reportPeak("tiny record", peak - baseline, liveHeapBytes());
   }, 180_000);
 
-  it("fails as a job rather than as a process when the stored copy does not fit", async () => {
-    // The bound that remains is on the one copy the pipeline cannot stream
-    // away: the blob itself, counted in bytes it produced. An account whose
-    // backup outgrows what this process can hold as a single value is now a
-    // failed backup for that account instead of a restart for every account
-    // on the host.
+  it("fails as a job rather than as a process when the stored copy passes its limit", async () => {
+    // The one bound left is a storage limit on the stored copy, counted in
+    // bytes it produced. An account past it is a failed backup for that
+    // account, not a restart for every account on the host.
     await expect(
-      packBackupBlobStreaming(
+      storeBackupBlob(
+        prisma,
+        { userId: OWNER_ID, type: "WEEKLY_AUTO" },
         (write) =>
           streamFullBackupJson(prisma, OWNER_ID, write, {
             purpose: "disaster-recovery",

tests/integration/encryption-key-rotation-blobs.test.ts

@@ -40,8 +40,11 @@ import {
   type CorpusClient,
 } from "@/lib/crypto/encryption-corpus";
 import { packBackupBlob, unpackBackupBlob } from "@/lib/export/backup-blob";
+import { extractKeyIdFromBytes } from "@/lib/crypto";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
 import { legacyStreamedBlob } from "@/__tests__/helpers/legacy-backup-blob";
 import { getPrismaClient, truncateAllTables } from "./setup";
+import { readStoredBackup } from "./stored-backup-read";
 
 const TEST_USER_ID = "user-rotation-blobs";
 const BACKUP_JSON = JSON.stringify({
@@ -117,8 +120,8 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
 
     for (const id of [gzipped.id, plainEnvelope.id]) {
       const row = await prisma.dataBackup.findUniqueOrThrow({ where: { id } });
-      expect(extractKeyId(row.data)).toBe("v2");
-      expect(unpackBackupBlob(row.data)).toBe(BACKUP_JSON);
+      expect(extractKeyId(row.data!)).toBe("v2");
+      expect(unpackBackupBlob(row.data!)).toBe(BACKUP_JSON);
     }
 
     // Idempotent: a second pass finds nothing left to do.
@@ -151,7 +154,7 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
     const row = await prisma.dataBackup.findUniqueOrThrow({
       where: { id: backup.id },
     });
-    expect(unpackBackupBlob(row.data)).toBe(BACKUP_JSON);
+    expect(unpackBackupBlob(row.data!)).toBe(BACKUP_JSON);
   });
 
   it("rotates the single stream v1.39.1 stored, and it stays readable", async () => {
@@ -185,6 +188,50 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
     expect(unpackBackupBlob(after.data!)).toBe(BACKUP_JSON);
   });
 
+  it("rotates every piece of a chunked copy, which reads back once the old key is gone", async () => {
+    const prisma = getPrismaClient();
+    process.env.ENCRYPTION_ACTIVE_KEY_ID = "v1";
+    _resetCryptoCacheForTests();
+    const { id, chunks } = await storeBackupBlob(
+      prisma,
+      { userId: TEST_USER_ID, type: "WEEKLY_AUTO" },
+      async (write) => {
+        await write('{"rows":[');
+        for (let i = 0; i < 20_000; i++) {
+          await write(`${i ? "," : ""}{"i":${i},"r":${Math.random()}}`);
+        }
+        await write("]}");
+      },
+      { chunkBytes: 4 * 1024 },
+    );
+    expect(chunks).toBeGreaterThan(3);
+    const expected = await readStoredBackup(prisma, id);
+
+    process.env.ENCRYPTION_ACTIVE_KEY_ID = "v2";
+    _resetCryptoCacheForTests();
+    const client = {
+      dataBackupChunk: prisma.dataBackupChunk,
+    } as unknown as CorpusClient;
+    const result = await rotateColumn(
+      client,
+      column("DataBackupChunk", "data"),
+    );
+    expect(result).toMatchObject({
+      scanned: chunks,
+      rotated: chunks,
+      errors: 0,
+    });
+    const again = await rotateColumn(client, column("DataBackupChunk", "data"));
+    expect(again.rotated).toBe(0);
+
+    process.env.ENCRYPTION_KEYS = JSON.stringify({ v2: "2".repeat(64) });
+    _resetCryptoCacheForTests();
+    for (const piece of await prisma.dataBackupChunk.findMany()) {
+      expect(extractKeyIdFromBytes(Buffer.from(piece.data))).toBe("v2");
+    }
+    expect(await readStoredBackup(prisma, id)).toBe(expected);
+  });
+
   it("drops an unreadable idempotency row instead of failing the run", async () => {
     const prisma = getPrismaClient();
     const seed = async (key: string, body: string) =>
@@ -223,6 +270,7 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
   it("keeps both columns in the registry the script and the job read", () => {
     const keys = ENCRYPTED_COLUMNS.map(encryptedColumnKey);
     expect(keys).toContain("DataBackup.data");
+    expect(keys).toContain("DataBackupChunk.data");
     expect(keys).toContain("IdempotencyKey.responseBody");
   });
 });

tests/integration/large-account-jobs.test.ts

@@ -22,8 +22,8 @@ import { beforeEach, describe, expect, it } from "vitest";
 import { getPrismaClient, truncateAllTables } from "./setup";
 import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention";
 import { consolidateDailyMean } from "@/lib/measurements/consolidate-daily-mean";
-import { unpackBackupBlob } from "@/lib/export/backup-blob";
 import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { readStoredBackup } from "./stored-backup-read";
 
 const TZ = "Europe/Berlin";
 const RUN_A = "user-1031-stopped";
@@ -214,7 +214,7 @@ describe("storeBackupBlob (#1031)", () => {
 
   it("stores a copy that reads back as exactly the JSON produced", async () => {
     const prisma = getPrismaClient();
-    const { bytes } = await storeBackupBlob(
+    const { id, bytes, chunks } = await storeBackupBlob(
       prisma,
       { userId: RUN_A, type: "WEEKLY_AUTO" },
       producer(20_000),
@@ -223,8 +223,15 @@ describe("storeBackupBlob (#1031)", () => {
     const row = await prisma.dataBackup.findUniqueOrThrow({
       where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
     });
-    expect(row.data.length).toBe(bytes);
-    expect(unpackBackupBlob(row.data)).toBe(expectedJson(20_000));
+    expect(row.id).toBe(id);
+    expect(row.data).toBeNull();
+    expect(row.chunkCount).toBe(chunks);
+    const pieces = await prisma.dataBackupChunk.findMany({
+      where: { backupId: id },
+    });
+    expect(pieces).toHaveLength(chunks);
+    expect(pieces.reduce((sum, p) => sum + p.data.byteLength, 0)).toBe(bytes);
+    expect(await readStoredBackup(prisma, id)).toBe(expectedJson(20_000));
   });
 
   it("keeps the previous copy when a run fails halfway, and the next run still stores", async () => {
@@ -249,8 +256,9 @@ describe("storeBackupBlob (#1031)", () => {
     const after = await prisma.dataBackup.findUniqueOrThrow({
       where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
     });
-    expect(after.data).toBe(before.data);
+    expect(after.chunkStreamId).toBe(before.chunkStreamId);
     expect(after.createdAt.getTime()).toBe(before.createdAt.getTime());
+    expect(await readStoredBackup(prisma, after.id)).toBe(expectedJson(10));
 
     // The temporary table went with the failed transaction.
     await storeBackupBlob(
@@ -261,6 +269,6 @@ describe("storeBackupBlob (#1031)", () => {
     const replaced = await prisma.dataBackup.findUniqueOrThrow({
       where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
     });
-    expect(unpackBackupBlob(replaced.data)).toBe(expectedJson(30));
+    expect(await readStoredBackup(prisma, replaced.id)).toBe(expectedJson(30));
   });
 });

tests/integration/stored-backup-read.ts

@@ -0,0 +1,23 @@
+/**
+ * Read a stored backup back as its JSON text, whichever form it was stored
+ * in, through the same opener the restore, download and preview use.
+ */
+import type { PrismaClient } from "@/generated/prisma/client";
+import {
+  openStoredBackup,
+  STORED_BACKUP_SELECT,
+} from "@/lib/export/stored-backup";
+
+export async function readStoredBackup(
+  prisma: PrismaClient,
+  id: string,
+): Promise<string> {
+  const backup = await prisma.dataBackup.findUniqueOrThrow({
+    where: { id },
+    select: STORED_BACKUP_SELECT,
+  });
+  const source = await openStoredBackup(prisma, backup);
+  const parts: Buffer[] = [];
+  for await (const chunk of source()) parts.push(Buffer.from(chunk));
+  return Buffer.concat(parts).toString("utf8");
+}