fix(backup): export, upload and restore a large account as streams

An account of 1.25 million measurements could not export its data or be
restored from any backup (#1031). Measured in a 1 GB container:

- The full export and the passphrase export built the whole payload and
  its JSON string in memory; the app ran out of heap and restarted.
- The weekly backup's JSON is 662 MB, longer than any string V8 can hold,
  so `unpackBackupBlob` threw on every read. Restore, download and the
  restore preview all answered "could not be decrypted".
- Upload accepted 10 MB, a fraction of a large export.

Every reader and writer of a backup file now streams it:

- `scanBackupJson` reads a backup's bytes once, keeps every section but
  the named bulk arrays, and hands their elements out one at a time.
  `readStreamedBackup` uses it twice: a first pass that validates each
  measurement against the element schema and keeps the rest of the file,
  and a second pass that hands the measurements to the writer in batches.
  Stored copies and off-host objects are authenticated whole on their
  compressed ciphertext before any plaintext is read; only the
  decompression is streamed.
- Restore writes the measurements with one `INSERT … FROM unnest` per
  batch. Prisma caches a query plan per `createMany` row count, and one
  1 000-row plan kept about 145 MB alive, which alone put the restore past
  the heap limit. The transaction budget grows with the measurement count.
  The awards and ECG sections are handed the ids they reference that were
  written, not every written id.
- The export routes write the file into the response as it is produced.
  The passphrase export spools its ciphertext to a temporary file because
  the HLX1 format puts the tag first; the format is unchanged, so existing
  archives open as before.
- Upload accepts the raw body, JSON or gzip, up to the 512 MB request
  ceiling, checks it as it arrives and stores it in the same pass; the admin
  page compresses the file in the browser before sending it. Multipart stays
  for small files.
- Download streams the stored JSON; the preview counts measurements from
  the stream. The monthly restore drill and `scripts/restore-backup.ts`
  read the off-host object the same way; the script can write `.json.gz`.

On the seeded account in a 1 GB container: export 25 s at 389 MB peak,
passphrase export 25 s, both uploads 19 s, restore of the weekly backup
94 s at 554 MB peak, and export, upload and restore round-trip to exactly
the same rows. The encrypted-column guard now also recognises the raw SQL
statement that writes `DataBackup.data`.

tests/integration/large-account-jobs.test.ts

@@ -214,7 +214,7 @@ describe("storeBackupBlob (#1031)", () => {
 
   it("stores a copy that reads back as exactly the JSON produced", async () => {
     const prisma = getPrismaClient();
-    const bytes = await storeBackupBlob(
+    const { bytes } = await storeBackupBlob(
       prisma,
       { userId: RUN_A, type: "WEEKLY_AUTO" },
       producer(20_000),

tests/integration/large-backup-round-trip.test.ts

@@ -0,0 +1,203 @@
+/**
+ * #1031 — an account too large to export or restore.
+ *
+ * For an account of 1.25 million measurements the full export took the app
+ * down, and its weekly backup could not be restored, downloaded or summarised
+ * at all: the backup's JSON (662 MB) is longer than any string V8 can hold,
+ * and every reader turned the file into one string first. Measured on a
+ * seeded account of that size in a 1 GB container, the streamed paths now
+ * export in 25 s, upload in under 20 s and restore in 94 s, and the restored
+ * rows match the file exactly.
+ *
+ * A million rows do not fit a test run. This file runs the same paths at a
+ * size that spans many pages and batches, and checks the rows come back
+ * exactly: the export streamed into the response, uploaded as gzip through
+ * the raw-body path, and restored through the streamed two-pass reader; and a
+ * weekly backup restored the same way, every column intact, with a personal
+ * record still pointing at the measurement it was found in.
+ */
+import { gzipSync } from "node:zlib";
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+process.env.ENCRYPTION_KEY ??=
+  "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+import { cookieJar, headerJar } from "./mock-next-headers";
+import { getPrismaClient, truncateAllTables } from "./setup";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { encryptNote } from "@/lib/crypto/note-cipher";
+
+vi.mock("next/headers", async () => {
+  const { cookieJar, headerJar } = await import("./mock-next-headers");
+  return {
+    headers: vi.fn(async () => ({
+      get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
+    })),
+    cookies: vi.fn(async () => ({
+      get: (name: string) => {
+        const value = cookieJar.get(name);
+        return value ? { name, value } : undefined;
+      },
+      set: (name: string, value: string) => {
+        cookieJar.set(name, value);
+      },
+      delete: (name: string) => {
+        cookieJar.delete(name);
+      },
+    })),
+  };
+});
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+const COUNT = 7_300; // more than one read page and several write batches
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+  cookieJar.clear();
+  headerJar.clear();
+});
+
+async function seedAccount() {
+  const prisma = getPrismaClient();
+  const user = await prisma.user.create({
+    data: {
+      username: "large-account",
+      email: "large-account@example.test",
+      role: "ADMIN",
+    },
+  });
+  const session = await prisma.session.create({
+    data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) },
+  });
+  cookieJar.set("healthlog_session", session.id);
+
+  const start = Date.UTC(2024, 0, 1);
+  await prisma.measurement.createMany({
+    data: Array.from({ length: COUNT }, (_, i) => ({
+      id: `m-${String(i).padStart(5, "0")}`,
+      userId: user.id,
+      type: i % 3 === 0 ? ("PULSE" as const) : ("SLEEP_DURATION" as const),
+      value: 50 + (i % 40) + 0.25,
+      valueMax: i % 3 === 0 ? 120 : null,
+      unit: i % 3 === 0 ? "bpm" : "minutes",
+      source: "APPLE_HEALTH" as const,
+      measuredAt: new Date(start + i * 300_000),
+      externalId: `hk-${i}`,
+      sleepStage: i % 3 === 0 ? null : ("CORE" as const),
+      deviceType: "watch",
+      syncVersion: 1 + (i % 4),
+      notesEncrypted: i === 17 ? encryptNote("after the long run") : null,
+      deletedAt: i % 97 === 0 ? new Date(start) : null,
+    })),
+  });
+  await prisma.personalRecord.create({
+    data: {
+      userId: user.id,
+      metricType: "PULSE",
+      direction: "MAX",
+      value: 89.25,
+      unit: "bpm",
+      achievedAt: new Date(start + 3 * 300_000),
+      sourceMeasurementId: "m-00003",
+    },
+  });
+  return user;
+}
+
+async function measurementsOf(userId: string) {
+  return getPrismaClient().measurement.findMany({
+    where: { userId },
+    orderBy: { id: "asc" },
+    omit: { createdAt: true, updatedAt: true },
+  });
+}
+
+async function restore(id: string) {
+  const { POST } = await import("@/app/api/admin/backups/[id]/restore/route");
+  const res = await POST(
+    new Request(`http://localhost/api/admin/backups/${id}/restore`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ confirm: "RESTORE" }),
+    }) as never,
+    { params: Promise.resolve({ id }) },
+  );
+  return res;
+}
+
+describe("large backups round-trip through the streamed paths (#1031)", () => {
+  it("export → gzip upload → restore gives back every live measurement", async () => {
+    const user = await seedAccount();
+    const live = (await measurementsOf(user.id)).filter((m) => !m.deletedAt);
+
+    const { GET } = await import("@/app/api/export/full-backup/route");
+    const exported = await GET(
+      new Request("http://localhost/api/export/full-backup") as never,
+    );
+    expect(exported.status).toBe(200);
+    const file = Buffer.from(await exported.arrayBuffer());
+    expect(JSON.parse(file.toString("utf8")).measurements).toHaveLength(
+      live.length,
+    );
+
+    const { POST: upload } =
+      await import("@/app/api/admin/backups/upload/route");
+    const uploaded = await upload(
+      new Request("http://localhost/api/admin/backups/upload", {
+        method: "POST",
+        headers: { "content-type": "application/gzip" },
+        body: gzipSync(file),
+      }) as never,
+    );
+    expect(uploaded.status).toBe(201);
+    const { data } = (await uploaded.json()) as {
+      data: { id: string; summary: { measurements: number } };
+    };
+    expect(data.summary.measurements).toBe(live.length);
+
+    expect((await restore(data.id)).status).toBe(200);
+    const restored = await measurementsOf(user.id);
+    // A portable file carries the fields a person reads, not the sync
+    // bookkeeping, so those are what must come back.
+    const readable = (rows: typeof restored) =>
+      rows.map((m) => ({
+        id: m.id,
+        type: m.type,
+        value: m.value,
+        unit: m.unit,
+        measuredAt: m.measuredAt.toISOString(),
+        source: m.source,
+      }));
+    expect(readable(restored)).toEqual(readable(live));
+  });
+
+  it("a weekly backup restores every column, and a record's pointer survives", async () => {
+    const user = await seedAccount();
+    const before = await measurementsOf(user.id);
+    const prisma = getPrismaClient();
+
+    const { id } = await storeBackupBlob(
+      prisma,
+      { userId: user.id, type: "WEEKLY_AUTO" },
+      (write) =>
+        streamFullBackupJson(prisma, user.id, write, {
+          purpose: "disaster-recovery",
+        }),
+    );
+
+    // Change the account so the restore has something to undo.
+    await prisma.measurement.deleteMany({ where: { userId: user.id } });
+
+    const res = await restore(id);
+    expect(res.status).toBe(200);
+    expect(await measurementsOf(user.id)).toEqual(before);
+    const record = await prisma.personalRecord.findFirstOrThrow({
+      where: { userId: user.id },
+    });
+    expect(record.sourceMeasurementId).toBe("m-00003");
+  });
+});

tests/integration/measurement-bulk-insert.test.ts

@@ -0,0 +1,139 @@
+/**
+ * `insertMeasurementRows` replaced `createMany` in the restore because
+ * Prisma's cached plan for a 1 000-row `createMany` kept about 145 MB alive
+ * (#1031). It writes raw SQL, so this pins that a row written through it
+ * reads back exactly as the same row written through Prisma, every column
+ * set, and every column null where it may be.
+ */
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+import {
+  insertMeasurementRows,
+  type MeasurementInsertRow,
+} from "@/lib/export/measurement-bulk-insert";
+
+const USER = "user-bulk-insert";
+
+beforeEach(async () => {
+  const prisma = getPrismaClient();
+  await truncateAllTables(prisma);
+  await prisma.user.create({
+    data: { id: USER, username: USER, email: `${USER}@example.test` },
+  });
+});
+
+function full(id: string, at: string): MeasurementInsertRow {
+  return {
+    id,
+    userId: USER,
+    type: "SLEEP_DURATION",
+    value: 47.25,
+    valueMin: 1.5,
+    valueMax: 99.75,
+    unit: "minutes",
+    source: "APPLE_HEALTH",
+    measuredAt: new Date(at),
+    notes: "legacy note",
+    notesEncrypted: new Uint8Array([0, 1, 2, 254, 255]),
+    externalId: `ext-${id}`,
+    externalSourceVersion: "17.4",
+    aggregationProvenance: "HEALTHKIT_STATISTICS",
+    glucoseContext: null,
+    sleepStage: "REM",
+    rhythmClassification: null,
+    deviceType: "watch",
+    syncVersion: 7,
+    deletedAt: new Date("2026-02-03T04:05:06.789Z"),
+    createdAt: new Date("2025-01-02T03:04:05.678Z"),
+    updatedAt: new Date("2025-06-07T08:09:10.111Z"),
+  };
+}
+
+function sparse(id: string, at: string): MeasurementInsertRow {
+  return {
+    id,
+    userId: USER,
+    type: "BLOOD_GLUCOSE",
+    value: 5.4,
+    valueMin: null,
+    valueMax: null,
+    unit: "mmol/L",
+    source: "MANUAL",
+    measuredAt: new Date(at),
+    notes: null,
+    notesEncrypted: null,
+    externalId: null,
+    externalSourceVersion: null,
+    aggregationProvenance: null,
+    glucoseContext: "FASTING",
+    sleepStage: null,
+    rhythmClassification: null,
+    deviceType: null,
+    syncVersion: 1,
+    deletedAt: null,
+  };
+}
+
+describe("insertMeasurementRows", () => {
+  it("writes every column exactly as Prisma create does", async () => {
+    const prisma = getPrismaClient();
+    const rows = [
+      [
+        full("a-bulk", "2026-01-01T10:00:00.123Z"),
+        full("a-prisma", "2026-01-02T10:00:00.123Z"),
+      ],
+      [
+        sparse("b-bulk", "2026-01-03T10:00:00.000Z"),
+        sparse("b-prisma", "2026-01-04T10:00:00.000Z"),
+      ],
+    ] as const;
+
+    for (const [viaBulk, viaPrisma] of rows) {
+      await prisma.$transaction((tx) => insertMeasurementRows(tx, [viaBulk]));
+      await prisma.measurement.create({ data: viaPrisma as never });
+    }
+
+    for (const [viaBulk, viaPrisma] of rows) {
+      const a = await prisma.measurement.findUniqueOrThrow({
+        where: { id: viaBulk.id },
+      });
+      const b = await prisma.measurement.findUniqueOrThrow({
+        where: { id: viaPrisma.id },
+      });
+      const strip = (row: typeof a) => ({
+        ...row,
+        id: undefined,
+        externalId: row.externalId?.replace(/-(bulk|prisma)$/, ""),
+        measuredAt: undefined,
+        // Filled with the current time when the row carries none.
+        ...(viaBulk.createdAt
+          ? {}
+          : { createdAt: undefined, updatedAt: undefined }),
+      });
+      expect(strip(a)).toEqual(strip(b));
+      expect(a.measuredAt.toISOString()).toBe(viaBulk.measuredAt.toISOString());
+    }
+    const sparseRow = await prisma.measurement.findUniqueOrThrow({
+      where: { id: "b-bulk" },
+    });
+    expect(Date.now() - sparseRow.createdAt.getTime()).toBeLessThan(60_000);
+  });
+
+  it("inserts a whole batch in one statement and reports the count", async () => {
+    const prisma = getPrismaClient();
+    const batch = Array.from({ length: 2_500 }, (_, i) =>
+      sparse(
+        `m-${i}`,
+        new Date(Date.UTC(2026, 0, 1) + i * 60_000).toISOString(),
+      ),
+    );
+    const inserted = await prisma.$transaction((tx) =>
+      insertMeasurementRows(tx, batch),
+    );
+    expect(inserted).toBe(2_500);
+    expect(await prisma.measurement.count({ where: { userId: USER } })).toBe(
+      2_500,
+    );
+  });
+});