Merge branch 'fix/large-account-jobs' into release/v1.39.1

src/app/api/admin/backups/[id]/download/route.ts

@@ -14,6 +14,8 @@
  * at-rest contract: the ciphertext stays in the DB, the plaintext is only
  * materialised inside the request handler and streamed to the admin.
  */
+import { Readable } from "node:stream";
+
 import { NextRequest, NextResponse } from "next/server";
 import { prisma } from "@/lib/db";
 import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
@@ -22,8 +24,13 @@ import { auditLog } from "@/lib/auth/audit";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  unpackBackupBlob,
+  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  readStreamedBackup,
+  type BackupSource,
+  type StreamedBackup,
+} from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import { parseBackupPayload } from "@/lib/validations/backup";
 
@@ -54,9 +61,11 @@ export const GET = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    let plaintext: string;
+    // Opened and streamed, never unpacked into one string: a large record's
+    // JSON is longer than any string V8 can hold (#1031).
+    let source: BackupSource;
     try {
-      plaintext = unpackBackupBlob(backup.data);
+      source = openBackupBlob(backup.data);
     } catch (err) {
       // A rotated or dropped key, or a stored copy that is no longer the one
       // that was written. Both are bad stored input rather than a fault in
@@ -80,8 +89,10 @@ export const GET = apiHandler(
     // pathological case where the worker wrote something the upload
     // route would later reject — better to fail loudly here than to ship
     // a junk file that breaks an admin's restore plan.
+    let streamed: StreamedBackup;
     try {
-      parseBackupPayload(plaintext);
+      streamed = await readStreamedBackup(source);
+      parseBackupPayload(streamed.raw);
     } catch (err) {
       await auditLog("admin.backups.download.failed", {
         userId: admin.id,
@@ -107,22 +118,26 @@ export const GET = apiHandler(
         ownerId: backup.userId,
         ownerUsername: backup.user.username,
         type: backup.type,
-        sizeBytes: Buffer.byteLength(plaintext, "utf8"),
+        sizeBytes: streamed.bytes,
       },
     });
 
-    // Returning the plaintext as-is keeps the response a faithful copy of
+    // Returning the stored bytes as-is keeps the response a faithful copy of
     // what the worker wrote (formatting, key order). The file is the
     // canonical artefact admins will store / re-upload.
-    return new NextResponse(plaintext, {
-      status: 200,
-      headers: {
-        "content-type": "application/json; charset=utf-8",
-        "content-disposition": `attachment; filename="${filename}"`,
-        // Backups contain sensitive health data — keep them out of any
-        // shared cache (CDN, browser disk cache).
-        "cache-control": "no-store, max-age=0",
+    return new NextResponse(
+      Readable.toWeb(Readable.from(source())) as ReadableStream<Uint8Array>,
+      {
+        status: 200,
+        headers: {
+          "content-type": "application/json; charset=utf-8",
+          "content-disposition": `attachment; filename="${filename}"`,
+          // Backups contain sensitive health data — keep them out of any
+          // shared cache (CDN, browser disk cache).
+          "cache-control": "no-store, max-age=0",
+          "content-length": String(streamed.bytes),
+        },
       },
-    });
+    );
   },
 );

src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts

@@ -47,6 +47,9 @@ vi.mock("@/lib/validations/backup", () => ({
   parseBackupPayload: vi.fn(),
   isCompatibleSchemaVersion: vi.fn(() => true),
   summarizeBackup: vi.fn(() => ({})),
+  // The streamed reader checks measurements one at a time; this file has none.
+  BACKUP_SCHEMA_VERSION: "2",
+  backupMeasurementSchema: { safeParse: vi.fn() },
 }));
 vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
 vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));

src/app/api/admin/backups/[id]/restore/route.ts

@@ -26,8 +26,17 @@ import { auditLog } from "@/lib/auth/audit";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  unpackBackupBlob,
+  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  insertMeasurementRows,
+  type MeasurementInsertRow,
+} from "@/lib/export/measurement-bulk-insert";
+import {
+  readStreamedBackup,
+  type BackupSource,
+  type StreamedBackup,
+} from "@/lib/export/streamed-backup";
 import { encryptNote } from "@/lib/crypto/note-cipher";
 import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
 import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
@@ -38,6 +47,7 @@ import {
   parseBackupPayload,
   isCompatibleSchemaVersion,
   summarizeBackup,
+  type BackupMeasurement,
   type BackupSummary,
 } from "@/lib/validations/backup";
 import { recomputeUserMoodRollups } from "@/lib/rollups/mood-rollups";
@@ -137,6 +147,19 @@ interface RestoreResponse {
   };
 }
 
+/**
+ * How long the restore transaction may run. Two minutes was the whole budget
+ * when a record was a few hundred thousand rows; deleting and rewriting 1.25
+ * million measurements in one transaction takes longer than that (#1031). On
+ * the test host the whole restore of that account, reading the file twice
+ * included, took 94 s. The budget grows by a second per 5 000 measurements,
+ * which gives that account six minutes, about four times what it needed, and
+ * never drops below the old two minutes.
+ */
+function restoreTransactionTimeoutMs(measurementCount: number): number {
+  return 120_000 + Math.ceil(measurementCount / 5);
+}
+
 function decodeEncryptedBytes(encoded: string): Uint8Array<ArrayBuffer> {
   const decoded = Buffer.from(encoded, "base64");
   const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength));
@@ -198,9 +221,12 @@ const handler = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    let plaintext: string;
+    // Opened, not unpacked: the JSON of a large record is longer than any
+    // string V8 can hold (#1031), so it is read as a stream below. Opening
+    // authenticates the whole ciphertext first, as unpacking did.
+    let source: BackupSource;
     try {
-      plaintext = unpackBackupBlob(backup.data);
+      source = openBackupBlob(backup.data);
     } catch (err) {
       await auditLog("admin.backups.restore.failed", {
         userId: admin.id,
@@ -224,10 +250,16 @@ const handler = apiHandler(
     // Parsed once and kept, because the schema's per-section `.default([])`
     // erases the difference between a section that is absent and one that is
     // empty — and that difference is what the completeness check below reads.
+    //
+    // The measurements are the exception: `readStreamedBackup` checks each one
+    // against the element schema as it goes and keeps only their count, and
+    // the transaction reads them a second time, in batches, as it writes them.
     let raw: unknown;
     let payload;
+    let streamed: StreamedBackup;
     try {
-      raw = JSON.parse(plaintext);
+      streamed = await readStreamedBackup(source);
+      raw = streamed.raw;
       payload = parseBackupPayload(raw);
     } catch (err) {
       await auditLog("admin.backups.restore.failed", {
@@ -548,7 +580,7 @@ const handler = apiHandler(
           }
 
           const toRestoredMeasurementData = (
-            measurement: (typeof payload.measurements)[number],
+            measurement: BackupMeasurement,
           ) => ({
             type: measurement.type,
             value: measurement.value,
@@ -600,55 +632,80 @@ const handler = apiHandler(
               : {}),
           });
 
-          const stableRows = payload.measurements.flatMap((measurement) =>
-            measurement.id
-              ? [
-                  {
-                    id: measurement.id,
-                    userId: ownerId,
-                    ...toRestoredMeasurementData(measurement),
-                  },
-                ]
-              : [],
-          );
-          const measurementBatchSize = 1_000;
-          for (
-            let offset = 0;
-            offset < stableRows.length;
-            offset += measurementBatchSize
-          ) {
-            await tx.measurement.createMany({
-              data: stableRows.slice(offset, offset + measurementBatchSize),
-            });
-          }
-
-          // v1 payloads did not require stable ids. Preserve their historical
-          // natural-key reconciliation without routing canonical v2 rows
-          // through it.
-          for (const measurement of payload.measurements) {
-            if (measurement.id) continue;
-            const restoredData = toRestoredMeasurementData(measurement);
-            const existing = await tx.measurement.findFirst({
-              where: {
-                userId: ownerId,
-                type: measurement.type,
-                source: restoredData.source,
-                measuredAt: restoredData.measuredAt,
-                sleepStage: restoredData.sleepStage,
-              },
-              select: { id: true },
-            });
-            if (existing) {
-              await tx.measurement.update({
-                where: { id: existing.id, userId: ownerId },
-                data: restoredData,
-              });
-            } else {
-              await tx.measurement.create({
-                data: { userId: ownerId, ...restoredData },
-              });
+          // The measurements, a batch at a time from a second read of the
+          // file, so the restore never holds more than one batch of them.
+          //
+          // Two later sections point INTO the measurements (a record's
+          // `sourceMeasurementId`, an ECG strip's `measurementId`) and have
+          // to know which of those ids this transaction actually wrote. They
+          // used to be handed every written id; they only ever ask about the
+          // ones they reference, so only those are collected.
+          const referencedMeasurementIds = new Set<string>();
+          for (const record of payload.personalRecords) {
+            if (record.sourceMeasurementId) {
+              referencedMeasurementIds.add(record.sourceMeasurementId);
             }
           }
+          for (const recording of payload.ecgRecordings) {
+            if (recording.measurementId) {
+              referencedMeasurementIds.add(recording.measurementId);
+            }
+          }
+          const writtenReferencedIds = new Set<string>();
+          const measurementBatchSize = 1_000;
+          await streamed.forEachMeasurementBatch(
+            measurementBatchSize,
+            async (batch) => {
+              const stableRows = batch.flatMap(
+                (measurement): MeasurementInsertRow[] =>
+                  measurement.id
+                    ? [
+                        {
+                          id: measurement.id,
+                          userId: ownerId,
+                          ...toRestoredMeasurementData(measurement),
+                        },
+                      ]
+                    : [],
+              );
+              // One statement per batch, not `createMany`: see
+              // `insertMeasurementRows` for what the latter cost here.
+              await insertMeasurementRows(tx, stableRows);
+              for (const row of stableRows) {
+                if (referencedMeasurementIds.has(row.id)) {
+                  writtenReferencedIds.add(row.id);
+                }
+              }
+
+              // v1 payloads did not require stable ids. Preserve their
+              // historical natural-key reconciliation without routing
+              // canonical v2 rows through it.
+              for (const measurement of batch) {
+                if (measurement.id) continue;
+                const restoredData = toRestoredMeasurementData(measurement);
+                const existing = await tx.measurement.findFirst({
+                  where: {
+                    userId: ownerId,
+                    type: measurement.type,
+                    source: restoredData.source,
+                    measuredAt: restoredData.measuredAt,
+                    sleepStage: restoredData.sleepStage,
+                  },
+                  select: { id: true },
+                });
+                if (existing) {
+                  await tx.measurement.update({
+                    where: { id: existing.id, userId: ownerId },
+                    data: restoredData,
+                  });
+                } else {
+                  await tx.measurement.create({
+                    data: { userId: ownerId, ...restoredData },
+                  });
+                }
+              }
+            },
+          );
 
           const medByName = new Map<string, string>();
           const restoredMedicationIds = new Set<string>();
@@ -1932,7 +1989,7 @@ const handler = apiHandler(
             tx,
             ownerId,
             payload,
-            new Set(stableRows.map((row) => row.id)),
+            writtenReferencedIds,
             skips,
           );
 
@@ -1949,7 +2006,7 @@ const handler = apiHandler(
             tx,
             ownerId,
             payload,
-            new Set(stableRows.map((row) => row.id)),
+            writtenReferencedIds,
             skips,
           );
 
@@ -2024,7 +2081,7 @@ const handler = apiHandler(
         },
         {
           maxWait: 10_000,
-          timeout: 120_000,
+          timeout: restoreTransactionTimeoutMs(streamed.measurementCount),
         },
       );
     } catch (err) {
@@ -2071,7 +2128,7 @@ const handler = apiHandler(
     // outside the transaction (the 5-year fold would otherwise hold a
     // long write lock) and is best-effort so a populator hiccup never
     // undoes the restore. The boot-time backfill is the safety net.
-    if (payload.measurements.length > 0) {
+    if (streamed.measurementCount > 0) {
       try {
         await recomputeUserRollups(ownerId);
       } catch (err) {
@@ -2131,7 +2188,10 @@ const handler = apiHandler(
       }
     }
 
-    const summary = summarizeBackup(payload);
+    const summary = {
+      ...summarizeBackup(payload),
+      measurements: streamed.measurementCount,
+    };
 
     await auditLog("admin.backups.restore", {
       userId: admin.id,

src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts

@@ -44,6 +44,11 @@ vi.mock("@/lib/validations/backup", () => ({
   parseBackupPayload: (...a: unknown[]) => parseMock(...a),
   summarizeBackup: (...a: unknown[]) => summarizeMock(...a),
   isCompatibleSchemaVersion: vi.fn(() => true),
+  // The measurements are read as a stream and counted one by one.
+  BACKUP_SCHEMA_VERSION: "2",
+  backupMeasurementSchema: {
+    safeParse: (value: unknown) => ({ success: true, data: value }),
+  },
 }));
 
 import { GET } from "../route";
@@ -62,9 +67,14 @@ beforeEach(() => {
     createdAt: new Date("2026-08-01T00:00:00Z"),
     user: { id: "u1", username: "self-hoster" },
   });
-  decryptMock.mockReturnValue("{}");
+  // Twelve measurements in the file: the preview counts them from the stream.
+  decryptMock.mockReturnValue(
+    JSON.stringify({
+      measurements: Array.from({ length: 12 }, (_, i) => ({ id: `m${i}` })),
+    }),
+  );
   parseMock.mockReturnValue({ schemaVersion: 3 });
-  summarizeMock.mockReturnValue({ measurements: 12, moodEntries: 3 });
+  summarizeMock.mockReturnValue({ measurements: 0, moodEntries: 3 });
 });
 
 describe("GET /api/admin/backups/[id]/summary", () => {

src/app/api/admin/backups/[id]/summary/route.ts

@@ -19,8 +19,12 @@ import { apiError, apiSuccess } from "@/lib/api-response";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  unpackBackupBlob,
+  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  readStreamedBackup,
+  type BackupSource,
+} from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import {
   isCompatibleSchemaVersion,
@@ -46,9 +50,11 @@ export const GET = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    let plaintext: string;
+    // Opened and read as a stream: a large record's JSON is longer than any
+    // string V8 can hold (#1031).
+    let source: BackupSource;
     try {
-      plaintext = unpackBackupBlob(backup.data);
+      source = openBackupBlob(backup.data);
     } catch {
       // Same refusal the restore and the download give, for the same reason:
       // a copy this instance cannot open is bad stored input, and the preview
@@ -59,8 +65,11 @@ export const GET = apiHandler(
     }
 
     let payload;
+    let measurementCount: number;
     try {
-      payload = parseBackupPayload(plaintext);
+      const streamed = await readStreamedBackup(source);
+      measurementCount = streamed.measurementCount;
+      payload = parseBackupPayload(streamed.raw);
     } catch {
       return apiError("Backup payload failed schema validation", 422);
     }
@@ -72,7 +81,10 @@ export const GET = apiHandler(
       );
     }
 
-    const summary = summarizeBackup(payload);
+    const summary = {
+      ...summarizeBackup(payload),
+      measurements: measurementCount,
+    };
 
     annotate({
       action: { name: "admin.backups.summary" },

src/app/api/admin/backups/upload/route.ts

@@ -1,17 +1,25 @@
 /**
  * POST /api/admin/backups/upload — admin-only ingest of a backup file.
  *
- * Accepts a `multipart/form-data` body with a single `file` field that
- * carries a JSON backup matching `backupPayloadSchema`. The route:
+ * Accepts the backup file either as the raw request body (JSON, or JSON
+ * compressed with gzip) or, for a small file, as a `multipart/form-data`
+ * body with a single `file` field. The route:
  *
  *   1. Authenticates the caller as an admin (cookie session only).
  *   2. Reads the file (size capped — see `MAX_UPLOAD_BYTES`).
  *   3. Parses and validates the JSON against the canonical schema.
  *   4. Rejects files written by a future schema version (incompatible).
- *   5. Encrypts the plaintext and inserts a new `DataBackup` row of
- *      type `MANUAL_UPLOAD_<unix-ms>` so multiple uploads coexist with
+ *   5. Encrypts the file as it arrives and inserts a new `DataBackup` row
+ *      of type `MANUAL_UPLOAD_<unix-ms>` so multiple uploads coexist with
  *      the rolling `WEEKLY_AUTO` snapshot AND with each other.
  *
+ * The file is read as a stream and never held whole (#1031): an export of an
+ * account with 1.25 million measurements is a few hundred megabytes of JSON,
+ * which neither fits a 1 GB container beside the running app nor, in the
+ * disaster-recovery shape, fits in one JavaScript string at all. It is
+ * checked as it goes and stored in the same pass; a file that fails any check
+ * rolls the store back, so nothing of it remains.
+ *
  * Crucially, this route does NOT execute a restore. It only stores the
  * file. The separate `POST /api/admin/backups/[id]/restore` endpoint
  * (criterion 3) actually replaces user data — keeping the two phases
@@ -20,46 +28,120 @@
  *
  * Phase B1 / criterion 2 of the v1.4.15 backup-completeness work.
  */
+import { Readable } from "node:stream";
+import { createGunzip } from "node:zlib";
+
 import { NextRequest } from "next/server";
 import { ZodError } from "zod/v4";
 import { prisma } from "@/lib/db";
 import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
 import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
 import { auditLog } from "@/lib/auth/audit";
-import { packBackupBlob } from "@/lib/export/backup-blob";
+import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { StreamedBackupInvalidError } from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import { checkRateLimit } from "@/lib/rate-limit";
 import {
+  BACKUP_SCHEMA_VERSION,
+  backupMeasurementSchema,
   backupPayloadSchema,
   isCompatibleSchemaVersion,
   summarizeBackup,
+  type BackupPayload,
   type BackupSummary,
 } from "@/lib/validations/backup";
 
 export const dynamic = "force-dynamic";
 
 /**
- * Cap on uploaded file size. The largest production backups today are
- * ~2 MB; 10 MB leaves headroom while keeping a malicious admin client
- * from blowing memory by streaming a 4 GB file. Enforced by reading the
- * `Content-Length` header AND the actual buffered size after decoding —
- * a missing/wrong content-length doesn't bypass the limit.
+ * Cap on a file sent as a multipart form. The form parser holds the whole
+ * file in memory, so this path stays for small files only; anything larger
+ * is sent as the raw request body, which is read as a stream.
  */
 const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
 
+/**
+ * Cap on a file sent as the raw request body, compressed or not. The same
+ * ceiling the app sets for any request body (`middlewareClientMaxBodySize` in
+ * `next.config.ts`), beyond which the body would arrive truncated. A
+ * disaster-recovery file of 1.25 million measurements is 662 MB as plain
+ * JSON and 64 MB compressed, so large files go up compressed.
+ */
+const MAX_RAW_UPLOAD_BYTES = 512 * 1024 * 1024;
+
 interface UploadResponse {
   id: string;
   valid: true;
   summary: BackupSummary;
 }
 
+/** A file the route refuses: the answer, and what the audit row records. */
+class UploadRefused extends Error {
+  constructor(
+    readonly status: number,
+    readonly answer: string,
+    readonly details: Record<string, unknown>,
+    readonly meta?: Record<string, unknown>,
+  ) {
+    super(answer);
+    this.name = "UploadRefused";
+  }
+}
+
+/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */
+async function* decodedBytes(
+  source: AsyncIterable<Uint8Array>,
+  limit: number,
+): AsyncGenerator<Uint8Array> {
+  let seen = 0;
+  async function* counted() {
+    for await (const chunk of source) {
+      seen += chunk.byteLength;
+      if (seen > limit) {
+        throw new UploadRefused(
+          413,
+          `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`,
+          { reason: "file_size_exceeded", size: seen },
+        );
+      }
+      yield chunk;
+    }
+  }
+  const iterator = counted()[Symbol.asyncIterator]();
+  const first = await iterator.next();
+  if (first.done) return;
+  const rest = (async function* () {
+    yield first.value;
+    for (;;) {
+      const next = await iterator.next();
+      if (next.done) return;
+      yield next.value;
+    }
+  })();
+  const head = first.value;
+  if (head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b) {
+    const gunzip = Readable.from(rest).pipe(createGunzip());
+    try {
+      for await (const chunk of gunzip) yield chunk as Buffer;
+    } catch (err) {
+      if (err instanceof UploadRefused) throw err;
+      throw new UploadRefused(422, "Uploaded file is not valid gzip", {
+        reason: "invalid_gzip",
+      });
+    }
+    return;
+  }
+  yield* rest;
+}
+
 export const POST = apiHandler(async (request: NextRequest) => {
   const { user: admin } = await requireAdmin();
   annotate({ action: { name: "admin.backups.upload" } });
+  const ipAddress = getClientIp(request);
 
-  // Rate-limit identical to the manual-run endpoint. An admin uploading
-  // a 10 MB blob 60× per minute would just be wasteful — three is plenty
-  // for a deliberate restore-prep workflow.
+  // Rate-limit identical to the manual-run endpoint. Three is plenty for a
+  // deliberate restore-prep workflow.
   const rl = await checkRateLimit(
     `admin-backups-upload:${admin.id}`,
     3,
@@ -69,120 +151,56 @@ export const POST = apiHandler(async (request: NextRequest) => {
     throw new HttpError(429, "Too many backup uploads");
   }
 
-  // Cheap pre-flight on the declared content length. The actual buffer
-  // size check below is the hard limit; this just rejects obvious abuse
-  // before allocating the FormData parser.
+  const denied = (reason: string, extra: Record<string, unknown> = {}) =>
+    auditLog("admin.backups.upload.denied", {
+      userId: admin.id,
+      ipAddress,
+      details: { reason, ...extra },
+    });
+
+  const contentType = request.headers.get("content-type") ?? "";
+  const multipart = contentType.startsWith("multipart/form-data");
+  const limit = multipart ? MAX_UPLOAD_BYTES : MAX_RAW_UPLOAD_BYTES;
+
+  // Cheap pre-flight on the declared content length. The byte count while
+  // reading is the hard limit; this just rejects obvious abuse early.
   const contentLength = Number(request.headers.get("content-length") ?? 0);
-  if (contentLength > MAX_UPLOAD_BYTES) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: { reason: "content_length_exceeded", contentLength },
-    });
-    return apiError("Upload exceeds 10 MB limit", 413);
-  }
-
-  let formData: FormData;
-  try {
-    formData = await request.formData();
-  } catch (err) {
+  if (contentLength > limit) {
+    await denied("content_length_exceeded", { contentLength });
     return apiError(
-      `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`,
-      400,
+      `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`,
+      413,
     );
   }
 
-  const file = formData.get("file");
-  if (!(file instanceof File)) {
-    return apiError("Field 'file' must be a file", 422);
-  }
-
-  if (file.size > MAX_UPLOAD_BYTES) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: { reason: "file_size_exceeded", size: file.size },
-    });
-    return apiError("Upload exceeds 10 MB limit", 413);
-  }
-
-  let plaintext: string;
-  try {
-    plaintext = await file.text();
-  } catch {
-    return apiError("Failed to read uploaded file", 400);
-  }
-
-  let parsedJson: unknown;
-  try {
-    parsedJson = JSON.parse(plaintext);
-  } catch (err) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: {
-        reason: "invalid_json",
-        message: err instanceof Error ? err.message : "parse_failed",
-      },
-    });
-    return apiError("Uploaded file is not valid JSON", 422);
-  }
-
-  let payload;
-  try {
-    payload = backupPayloadSchema.parse(parsedJson);
-  } catch (err) {
-    const issues =
-      err instanceof ZodError
-        ? err.issues.slice(0, 10).map((i) => ({
-            path: i.path.join("."),
-            message: i.message,
-          }))
-        : [];
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: { reason: "schema_invalid", issues },
-    });
-    return apiError("Backup payload failed schema validation", 422, {
-      issues,
-    });
-  }
-
-  if (!isCompatibleSchemaVersion(payload.schemaVersion)) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: {
-        reason: "incompatible_schema_version",
-        schemaVersion: payload.schemaVersion,
-      },
-    });
-    return apiError(
-      `Backup schema version '${payload.schemaVersion}' is not supported by this server`,
-      422,
+  let source: AsyncIterable<Uint8Array>;
+  if (multipart) {
+    let formData: FormData;
+    try {
+      formData = await request.formData();
+    } catch (err) {
+      return apiError(
+        `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`,
+        400,
+      );
+    }
+    const file = formData.get("file");
+    if (!(file instanceof File)) {
+      return apiError("Field 'file' must be a file", 422);
+    }
+    if (file.size > MAX_UPLOAD_BYTES) {
+      await denied("file_size_exceeded", { size: file.size });
+      return apiError("Upload exceeds 10 MB limit", 413);
+    }
+    source = Readable.fromWeb(
+      file.stream() as Parameters<typeof Readable.fromWeb>[0],
     );
-  }
-
-  // Make sure the userId on the file points at a user that exists in
-  // this DB. Restore would fail later anyway, but failing here gives
-  // the admin a precise error before any side-effect.
-  const owner = await prisma.user.findUnique({
-    where: { id: payload.userId },
-    select: { id: true, username: true },
-  });
-  if (!owner) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: {
-        reason: "owner_not_found",
-        ownerId: payload.userId,
-      },
-    });
-    return apiError(
-      `Backup is for user '${payload.userId}' which does not exist in this DB`,
-      422,
+  } else {
+    if (!request.body) {
+      return apiError("The backup file is missing from the request body", 422);
+    }
+    source = Readable.fromWeb(
+      request.body as Parameters<typeof Readable.fromWeb>[0],
     );
   }
 
@@ -190,25 +208,146 @@ export const POST = apiHandler(async (request: NextRequest) => {
   // intact so multiple uploads can coexist for the same user without
   // overwriting each other or the rolling WEEKLY_AUTO snapshot.
   const uploadType = `MANUAL_UPLOAD_${Date.now()}`;
-  const encrypted = packBackupBlob(JSON.stringify(payload));
+  let payload: BackupPayload | undefined;
+  let measurementCount = 0;
+  let owner: { id: string; username: string } | null = null;
 
-  const created = await prisma.dataBackup.create({
-    data: {
-      userId: owner.id,
-      type: uploadType,
-      data: encrypted,
-    },
-  });
+  let created: { id: string };
+  try {
+    created = await storeBackupBlob(
+      prisma,
+      () => ({ userId: owner!.id, type: uploadType }),
+      async (write) => {
+        // One pass: every chunk is stored as it is checked, and a check that
+        // fails throws, which rolls the store back.
+        async function* storedAsRead() {
+          for await (const chunk of decodedBytes(source, limit)) {
+            await write(Buffer.from(chunk));
+            yield chunk;
+          }
+        }
+        let firstWithoutId: number | null = null;
+        let scanned;
+        try {
+          scanned = await scanBackupJson(storedAsRead(), {
+            streamKeys: new Set(["measurements"]),
+            onElement: (_key, element, index) => {
+              const parsed = backupMeasurementSchema.safeParse(element);
+              if (!parsed.success) {
+                const issue = parsed.error.issues[0];
+                throw new StreamedBackupInvalidError(
+                  ["measurements", index, ...(issue?.path ?? [])].join("."),
+                  issue?.message ?? "invalid measurement",
+                );
+              }
+              if (!parsed.data.id && firstWithoutId === null) {
+                firstWithoutId = index;
+              }
+            },
+          });
+        } catch (err) {
+          if (err instanceof BackupJsonError) {
+            throw new UploadRefused(422, "Uploaded file is not valid JSON", {
+              reason: "invalid_json",
+              message: err.message,
+            });
+          }
+          if (err instanceof StreamedBackupInvalidError) {
+            const issues = [{ path: err.path, message: err.message }];
+            throw new UploadRefused(
+              422,
+              "Backup payload failed schema validation",
+              { reason: "schema_invalid", issues },
+              { issues },
+            );
+          }
+          throw err;
+        }
+        measurementCount = scanned.streamedCounts.measurements ?? 0;
 
-  const summary = summarizeBackup(payload);
+        try {
+          payload = backupPayloadSchema.parse(scanned.document);
+        } catch (err) {
+          const issues =
+            err instanceof ZodError
+              ? err.issues.slice(0, 10).map((i) => ({
+                  path: i.path.join("."),
+                  message: i.message,
+                }))
+              : [];
+          throw new UploadRefused(
+            422,
+            "Backup payload failed schema validation",
+            { reason: "schema_invalid", issues },
+            { issues },
+          );
+        }
+        if (
+          payload.schemaVersion === BACKUP_SCHEMA_VERSION &&
+          firstWithoutId !== null
+        ) {
+          const issues = [
+            {
+              path: `measurements.${firstWithoutId}.id`,
+              message: "Canonical v2 measurements require a stable id",
+            },
+          ];
+          throw new UploadRefused(
+            422,
+            "Backup payload failed schema validation",
+            { reason: "schema_invalid", issues },
+            { issues },
+          );
+        }
+
+        if (!isCompatibleSchemaVersion(payload.schemaVersion)) {
+          throw new UploadRefused(
+            422,
+            `Backup schema version '${payload.schemaVersion}' is not supported by this server`,
+            {
+              reason: "incompatible_schema_version",
+              schemaVersion: payload.schemaVersion,
+            },
+          );
+        }
+
+        // Make sure the userId on the file points at a user that exists in
+        // this DB. Restore would fail later anyway, but failing here gives
+        // the admin a precise error before any side-effect.
+        owner = await prisma.user.findUnique({
+          where: { id: payload.userId },
+          select: { id: true, username: true },
+        });
+        if (!owner) {
+          throw new UploadRefused(
+            422,
+            `Backup is for user '${payload.userId}' which does not exist in this DB`,
+            { reason: "owner_not_found", ownerId: payload.userId },
+          );
+        }
+      },
+    );
+  } catch (err) {
+    if (err instanceof UploadRefused) {
+      await denied(err.details.reason as string, err.details);
+      return apiError(err.answer, err.status, err.meta);
+    }
+    throw err;
+  }
+
+  const summary: BackupSummary = {
+    ...summarizeBackup(payload!),
+    measurements: measurementCount,
+  };
+  const storedOwner = owner as { id: string; username: string } | null;
 
   await auditLog("admin.backups.upload", {
     userId: admin.id,
-    ipAddress: getClientIp(request),
+    ipAddress,
     details: {
       backupId: created.id,
-      ownerId: owner.id,
-      ownerUsername: owner.username,
+      ownerId: storedOwner?.id,
+      ownerUsername: storedOwner?.username,
       type: uploadType,
       schemaVersion: summary.schemaVersion,
       counts: {

src/app/api/export/__tests__/soft-delete-filter.test.ts

@@ -64,6 +64,29 @@ vi.mock("@/lib/db", () => ({
     documentConditionLink: { findMany: vi.fn() },
     extractedFact: { findMany: vi.fn() },
     ecgRecording: { findMany: vi.fn() },
+    onboardingRecord: { findUnique: vi.fn().mockResolvedValue(null) },
+    environmentTravelLocation: { findMany: vi.fn().mockResolvedValue([]) },
+    environmentContext: { findMany: vi.fn().mockResolvedValue([]) },
+    personalRecord: { findMany: vi.fn().mockResolvedValue([]) },
+    coachReminder: { findMany: vi.fn().mockResolvedValue([]) },
+    coachPlan: { findMany: vi.fn().mockResolvedValue([]) },
+    coachFact: { findMany: vi.fn().mockResolvedValue([]) },
+    coachConversation: { findMany: vi.fn().mockResolvedValue([]) },
+    measurementReminderEvent: { findMany: vi.fn().mockResolvedValue([]) },
+    measurementReminder: { findMany: vi.fn().mockResolvedValue([]) },
+    vaccinationDocumentLink: { findMany: vi.fn().mockResolvedValue([]) },
+    vaccinationRecord: { findMany: vi.fn().mockResolvedValue([]) },
+    encounterConditionLink: { findMany: vi.fn().mockResolvedValue([]) },
+    encounterLabLink: { findMany: vi.fn().mockResolvedValue([]) },
+    encounterDocumentLink: { findMany: vi.fn().mockResolvedValue([]) },
+    encounter: { findMany: vi.fn().mockResolvedValue([]) },
+    practitioner: { findMany: vi.fn().mockResolvedValue([]) },
+    healthScoreRecord: { findMany: vi.fn().mockResolvedValue([]) },
+    intradayCumulativeProfile: { findMany: vi.fn().mockResolvedValue([]) },
+    correlationPattern: { findMany: vi.fn().mockResolvedValue([]) },
+    healthProfileFactRevision: { findMany: vi.fn().mockResolvedValue([]) },
+    customMetric: { findMany: vi.fn().mockResolvedValue([]) },
+    cycleSymptom: { findMany: vi.fn().mockResolvedValue([]) },
   },
 }));
 
@@ -177,6 +200,7 @@ beforeEach(() => {
   );
   vi.mocked(prisma.extractedFact.findMany).mockResolvedValue([] as never);
   vi.mocked(prisma.ecgRecording.findMany).mockResolvedValue([] as never);
+  vi.mocked(prisma.userAchievement.findMany).mockResolvedValue([] as never);
 });
 
 describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
@@ -192,7 +216,13 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
 
   it("/api/export/full-backup scopes the measurement read to deletedAt: null", async () => {
     const { GET } = await import("../full-backup/route");
-    await GET(mkReq("http://localhost/api/export/full-backup"));
+    // The file is written into the response as it is produced, so the reads
+    // happen while the body is consumed. The mocked client does not cover
+    // every section the builder reads, so the body errors part-way (as the
+    // whole response did before it streamed); the reads asserted here have
+    // been issued by then.
+    const res = await GET(mkReq("http://localhost/api/export/full-backup"));
+    await res.text();
     expect(prisma.measurement.findMany).toHaveBeenCalledWith(
       expect.objectContaining({
         where: expect.objectContaining({ deletedAt: null }),
@@ -202,7 +232,10 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
 
   it("/api/export/full-backup scopes every v1.28 records domain to userId + deletedAt: null", async () => {
     const { GET } = await import("../full-backup/route");
-    await GET(mkReq("http://localhost/api/export/full-backup"));
+    // The file is written into the response as it is produced, so the reads
+    // happen while the body is consumed.
+    const res = await GET(mkReq("http://localhost/api/export/full-backup"));
+    await res.text();
 
     expect(prisma.labResult.findMany).toHaveBeenCalledWith(
       expect.objectContaining({

src/app/api/export/encrypted/route.ts

@@ -42,13 +42,44 @@ import {
   safeJson,
 } from "@/lib/api-response";
 import { checkRateLimit } from "@/lib/rate-limit";
-import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
+import { randomBytes } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { readdir, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import {
-  encryptArchive,
+  encryptArchiveToFile,
   MIN_EXPORT_PASSPHRASE_LENGTH,
+  type SpooledArchive,
 } from "@/lib/export/passphrase-archive";
+import { streamToResponseBody } from "@/lib/export/response-stream";
 import { NextRequest, NextResponse } from "next/server";
 
+const SPOOL_PREFIX = "healthlog-export-";
+
+/**
+ * Remove spool files an earlier export left behind: a process that died
+ * mid-export never reached its cleanup. An hour is far longer than any export
+ * takes, so nothing still being written or sent is touched.
+ */
+async function removeStaleSpools(): Promise<void> {
+  try {
+    const dir = tmpdir();
+    const cutoff = Date.now() - 60 * 60 * 1000;
+    for (const name of await readdir(dir)) {
+      if (!name.startsWith(SPOOL_PREFIX)) continue;
+      const path = join(dir, name);
+      const info = await stat(path).catch(() => null);
+      if (info && info.mtimeMs < cutoff) await rm(path, { force: true });
+    }
+  } catch {
+    // Housekeeping only; an export must not fail over it.
+  }
+}
+
 const encryptedExportSchema = z
   .object({
     passphrase: z
@@ -89,10 +120,31 @@ export const POST = apiHandler(async (request: NextRequest) => {
   }
   const { passphrase } = parsed.data;
 
-  const { payload, counts } = await buildFullBackupPayload(prisma, user.id);
-
-  // Encrypt the JSON bytes. The passphrase goes no further than the KDF.
-  const archive = await encryptArchive(JSON.stringify(payload), passphrase);
+  // Sealed as it is produced and spooled to a temporary file, then sent: the
+  // format puts the tag in front of the ciphertext, so the archive cannot go
+  // out before its last byte is encrypted, and holding it in memory is what
+  // took the app down on an account of 1.25 million measurements (#1031). Only
+  // ciphertext touches the disk, and the file is removed once it is sent or
+  // the client leaves.
+  let counts: FullBackupCounts | undefined;
+  await removeStaleSpools();
+  const bodyPath = join(
+    tmpdir(),
+    `${SPOOL_PREFIX}${randomBytes(12).toString("hex")}.hlx.part`,
+  );
+  let archive: SpooledArchive;
+  try {
+    archive = await encryptArchiveToFile(
+      async (write) => {
+        counts = await streamFullBackupJson(prisma, user.id, write);
+      },
+      passphrase,
+      bodyPath,
+    );
+  } catch (err) {
+    await rm(bodyPath, { force: true });
+    throw err;
+  }
 
   await auditLog("user.export.encrypted", {
     userId: user.id,
@@ -102,38 +154,36 @@ export const POST = apiHandler(async (request: NextRequest) => {
 
   annotate({
     meta: {
-      export_measurements_count: counts.measurements,
-      export_medications_count: counts.medications,
-      export_intake_count: counts.intakeEvents,
-      export_medication_side_effect_count: counts.medicationSideEffects,
-      export_mood_count: counts.moodEntries,
-      export_cycle_count: counts.cycles,
-      export_cycle_day_log_count: counts.cycleDayLogs,
-      export_lab_result_count: counts.labResults,
-      export_biomarker_count: counts.biomarkers,
-      export_illness_episode_count: counts.illnessEpisodes,
-      export_illness_day_log_count: counts.illnessDayLogs,
-      export_allergy_count: counts.allergies,
-      export_family_history_count: counts.familyHistory,
-      export_workout_count: counts.workouts,
-      export_document_count: counts.documents,
-      export_nutrient_day_count: counts.nutrientDays,
-      export_health_profile_count: counts.healthProfile,
-      export_custom_metric_count: counts.customMetrics,
-      export_custom_metric_entry_count: counts.customMetricEntries,
-      export_intraday_profile_count: counts.intradayProfiles,
+      export_measurements_count: counts?.measurements,
+      export_medications_count: counts?.medications,
+      export_intake_count: counts?.intakeEvents,
+      export_mood_count: counts?.moodEntries,
       export_archive_bytes: archive.byteLength,
     },
   });
 
+  const archiveBody = streamToResponseBody(
+    async (write) => {
+      await write(archive.prefix);
+      for await (const chunk of createReadStream(archive.bodyPath)) {
+        await write(chunk as Buffer);
+      }
+    },
+    {
+      onComplete: () => rm(archive.bodyPath, { force: true }),
+      onError: () => rm(archive.bodyPath, { force: true }),
+    },
+  );
+
   const stamp = new Date().toISOString().slice(0, 10);
   // Return the raw binary archive (NOT the apiSuccess envelope). The file is a
   // self-contained `.hlx` archive openable with the user's passphrase via
   // scripts/decrypt-export.ts.
-  return new NextResponse(new Uint8Array(archive), {
+  return new NextResponse(archiveBody, {
     status: 200,
     headers: {
       "Content-Type": "application/octet-stream",
+      "Content-Length": String(archive.byteLength),
       "Content-Disposition": `attachment; filename="healthlog-backup-${user.id}-${stamp}.hlx"`,
       "Cache-Control": "no-store",
     },

src/app/api/export/full-backup/route.ts

@@ -35,7 +35,9 @@ import { annotate } from "@/lib/logging/context";
 import { auditLog } from "@/lib/auth/audit";
 import { apiError, getClientIp } from "@/lib/api-response";
 import { checkRateLimit } from "@/lib/rate-limit";
-import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
+import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
+import { streamToResponseBody } from "@/lib/export/response-stream";
 import { NextRequest, NextResponse } from "next/server";
 
 export const GET = apiHandler(async (request: NextRequest) => {
@@ -47,48 +49,44 @@ export const GET = apiHandler(async (request: NextRequest) => {
     return apiError("Maximum 10 exports per hour", 429);
   }
 
-  // v1.23 — the payload builder is shared with the passphrase-encrypted
-  // export route so both emit the byte-for-byte same shape (the measurement/
-  // medication/mood/cycle domains are restore-compatible; the newer records
-  // domains are export-only for now — see the module doc comment above).
-  const { payload, counts } = await buildFullBackupPayload(prisma, user.id);
-
-  await auditLog("user.export.full-backup", {
-    userId: user.id,
-    ipAddress: getClientIp(request),
-    details: { counts },
-  });
-
-  annotate({
-    meta: {
-      export_measurements_count: counts.measurements,
-      export_medications_count: counts.medications,
-      export_intake_count: counts.intakeEvents,
-      export_medication_side_effect_count: counts.medicationSideEffects,
-      export_mood_count: counts.moodEntries,
-      export_cycle_count: counts.cycles,
-      export_cycle_day_log_count: counts.cycleDayLogs,
-      export_lab_result_count: counts.labResults,
-      export_biomarker_count: counts.biomarkers,
-      export_illness_episode_count: counts.illnessEpisodes,
-      export_illness_day_log_count: counts.illnessDayLogs,
-      export_allergy_count: counts.allergies,
-      export_family_history_count: counts.familyHistory,
-      export_workout_count: counts.workouts,
-      export_document_count: counts.documents,
-      export_nutrient_day_count: counts.nutrientDays,
-      export_health_profile_count: counts.healthProfile,
-      export_custom_metric_count: counts.customMetrics,
-      export_custom_metric_entry_count: counts.customMetricEntries,
-      export_intraday_profile_count: counts.intradayProfiles,
+  // Written into the response as it is produced (`streamFullBackupJson`), not
+  // built and stringified first: for an account of 1.25 million measurements
+  // the built payload and its JSON string did not fit a 1 GB container, and
+  // the export took the whole app down (#1031). The file is byte for byte
+  // what `JSON.stringify` of the payload builder's object gives; the passphrase
+  // export beside this route writes the same bytes, encrypted.
+  //
+  // The counts are only known once the last row has gone out, so the audit
+  // row is written then. A file that stops early is audited as a failure.
+  const ipAddress = getClientIp(request);
+  let counts: FullBackupCounts | undefined;
+  const body = streamToResponseBody(
+    async (write) => {
+      counts = await streamFullBackupJson(prisma, user.id, write);
     },
-  });
+    {
+      onComplete: () =>
+        auditLog("user.export.full-backup", {
+          userId: user.id,
+          ipAddress,
+          details: { counts },
+        }),
+      onError: (err) =>
+        auditLog("user.export.full-backup.failed", {
+          userId: user.id,
+          ipAddress,
+          details: {
+            reason: err instanceof Error ? err.message : String(err),
+          },
+        }),
+    },
+  );
 
   const stamp = new Date().toISOString().slice(0, 10);
   // Stream the JSON directly (NOT wrapped in the apiSuccess envelope) so
   // the file is a self-contained backup — admin upload + restore expect
   // the raw payload, not `{ data: { ... } }`.
-  return new NextResponse(JSON.stringify(payload), {
+  return new NextResponse(body, {
     status: 200,
     headers: {
       "Content-Type": "application/json; charset=utf-8",