Merge branch 'fix/large-account-jobs' into release/v1.39.1

docs/ops/encryption-key-rotation.md

@@ -60,8 +60,10 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`).
 
    Read three things off the output before going further:
 
-   - the per-column line, `scanned` / `rotated` / `errors` / `dropped`. Treat
-     `errors > 0` as a hard failure and re-run after fixing the cause. A
+   - the per-column line, `scanned` / `rotated` / `errors` / `dropped`.
+     `scanned` counts the rows that hold ciphertext in that column, not every
+     row of the table. Treat `errors > 0` as a hard failure and re-run after
+     fixing the cause. A
      `dropped` count is only ever a cache row the run could not read and
      deleted rather than leave unreadable.
    - `Columns walked: N/M registered`. The two numbers must match.

messages/de.json

@@ -7167,7 +7167,7 @@
         "uploadTitle": "Backup hochladen",
         "uploadDescription": "Eine Backup-Datei validieren und als manuellen Snapshot speichern. Das Wiederherstellen ist ein separater Schritt.",
         "uploadButton": "Datei hochladen",
-        "uploadHelp": "JSON-Datei passend zum aktuellen Backup-Schema. Max. 10 MB.",
+        "uploadHelp": "HealthLog-Backup als .json oder .json.gz. Die Datei wird vor dem Hochladen komprimiert; komprimiert darf sie bis zu 512 MB groß sein.",
         "uploadSuccess": "Backup hochgeladen ({count} Datensätze)",
         "uploadFailed": "Upload fehlgeschlagen",
         "restore": "Wiederherstellen",

messages/en.json

@@ -7167,7 +7167,7 @@
         "uploadTitle": "Upload backup",
         "uploadDescription": "Validate a backup file and store it as a manual snapshot. Restoring is a separate step.",
         "uploadButton": "Upload file",
-        "uploadHelp": "JSON file matching the current backup schema. Max 10 MB.",
+        "uploadHelp": "A HealthLog backup as .json or .json.gz. The file is compressed before upload; compressed, it may be up to 512 MB.",
         "uploadSuccess": "Backup uploaded ({count} records)",
         "uploadFailed": "Upload failed",
         "restore": "Restore",

messages/es.json

@@ -7167,7 +7167,7 @@
         "uploadTitle": "Subir copia de seguridad",
         "uploadDescription": "Valida un archivo de copia de seguridad y guárdalo como instantánea manual. La restauración es un paso aparte.",
         "uploadButton": "Subir archivo",
-        "uploadHelp": "Archivo JSON que coincida con el esquema de copia de seguridad actual. Máx. 10 MB.",
+        "uploadHelp": "Una copia de seguridad de HealthLog en .json o .json.gz. El archivo se comprime antes de subirlo; comprimido puede ocupar hasta 512 MB.",
         "uploadSuccess": "Copia de seguridad subida ({count} registros)",
         "uploadFailed": "La subida falló",
         "restore": "Restaurar",

messages/fr.json

@@ -7167,7 +7167,7 @@
         "uploadTitle": "Téléverser une sauvegarde",
         "uploadDescription": "Validez un fichier de sauvegarde et stockez-le comme instantané manuel. La restauration est une étape distincte.",
         "uploadButton": "Téléverser un fichier",
-        "uploadHelp": "Fichier JSON correspondant au schéma de sauvegarde actuel. Max 10 Mo.",
+        "uploadHelp": "Une sauvegarde HealthLog au format .json ou .json.gz. Le fichier est compressé avant l’envoi ; compressé, il peut atteindre 512 Mo.",
         "uploadSuccess": "Sauvegarde téléversée ({count} enregistrements)",
         "uploadFailed": "Échec du téléversement",
         "restore": "Restaurer",

messages/it.json

@@ -7167,7 +7167,7 @@
         "uploadTitle": "Carica backup",
         "uploadDescription": "Convalida un file di backup e archivialo come istantanea manuale. Il ripristino è un passaggio separato.",
         "uploadButton": "Carica file",
-        "uploadHelp": "File JSON conforme allo schema di backup attuale. Max 10 MB.",
+        "uploadHelp": "Un backup di HealthLog in formato .json o .json.gz. Il file viene compresso prima del caricamento; compresso può arrivare a 512 MB.",
         "uploadSuccess": "Backup caricato ({count} record)",
         "uploadFailed": "Caricamento non riuscito",
         "restore": "Ripristina",

messages/ko.json

@@ -7146,7 +7146,7 @@
         "uploadTitle": "백업 업로드",
         "uploadDescription": "백업 파일을 검사한 뒤 수동 스냅샷으로 저장해요. 복원은 별도 단계예요.",
         "uploadButton": "파일 업로드",
-        "uploadHelp": "현재 백업 스키마에 맞는 JSON 파일이에요. 최대 10 MB.",
+        "uploadHelp": "HealthLog 백업 파일(.json 또는 .json.gz)이에요. 업로드 전에 압축되며, 압축된 크기는 최대 512 MB까지 가능해요.",
         "uploadSuccess": "백업을 업로드했어요 (기록 {count}개)",
         "uploadFailed": "업로드하지 못했어요",
         "restore": "복원",

messages/pl.json

@@ -7167,7 +7167,7 @@
         "uploadTitle": "Prześlij kopię zapasową",
         "uploadDescription": "Sprawdź plik kopii zapasowej i zapisz go jako ręczną migawkę. Przywracanie to osobny krok.",
         "uploadButton": "Prześlij plik",
-        "uploadHelp": "Plik JSON zgodny z bieżącym schematem kopii zapasowej. Maks. 10 MB.",
+        "uploadHelp": "Kopia zapasowa HealthLog w formacie .json lub .json.gz. Plik jest kompresowany przed wysłaniem; po kompresji może mieć do 512 MB.",
         "uploadSuccess": "Przesłano kopię zapasową ({count} rekordów)",
         "uploadFailed": "Przesyłanie nie powiodło się",
         "restore": "Przywróć",

scripts/restore-backup.ts

@@ -15,12 +15,23 @@
  *   BACKUP_S3_REGION=auto         \
  *   BACKUP_ENCRYPTION_KEY=<hex64> \
  *   pnpm dlx tsx scripts/restore-backup.ts 2026-05-08/user-clx123.json.enc /tmp/restored.json
+ *
+ * An output name ending in `.gz` writes the JSON compressed, which is the
+ * form to hand to the admin upload for a large account: the JSON of 1.25
+ * million measurements is 662 MB, the compressed file about a tenth of it.
+ * The file is written as a stream in either case and never held whole.
  */
-import { writeFileSync } from "node:fs";
+import { createWriteStream } from "node:fs";
+import { rename } from "node:fs/promises";
+import { PassThrough, Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { createGzip } from "node:zlib";
+
+import { scanBackupJson } from "@/lib/export/backup-json-scan";
 import {
   loadOffhostConfig,
   getS3Client,
-  decryptBackup,
+  openBackupObject,
   OffhostBackupNotConfiguredError,
 } from "@/lib/jobs/offhost-backup";
 
@@ -98,28 +109,43 @@ async function main() {
     `Downloading s3://${cfg.bucket}/${key} from ${cfg.endpoint} (region=${cfg.region})`,
   );
   const ciphertext = await s3.getObject(key);
-  const plaintext = decryptBackup(ciphertext, cfg.encryptionKey);
+  const source = openBackupObject(ciphertext, cfg.encryptionKey);
 
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(plaintext);
-  } catch (err) {
-    throw new Error(
-      `Decrypted payload is not valid JSON: ${(err as Error).message}`,
-    );
-  }
-  const body = validateBackupShape(parsed);
+  // Checked first, in one streamed pass that keeps only the small sections,
+  // so nothing is written for an object that is not a backup.
+  const { document, streamedCounts } = await scanBackupJson(source(), {
+    streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]),
+  });
+  const body = validateBackupShape({
+    ...document,
+    measurements: [],
+    intakeEvents: [],
+    moodEntries: [],
+  });
   if (expectedUserId && body.userId !== expectedUserId) {
     throw new Error(
       `Refusing to write: backup.userId='${body.userId}' does not match --user-id='${expectedUserId}'`,
     );
   }
 
-  // The restored file is decrypted PHI in plaintext. Write it 0o600 so it is
-  // not world-readable on the operator's disk while it sits there.
-  writeFileSync(out, plaintext, { encoding: "utf8", mode: 0o600 });
+  // The restored file is decrypted PHI in plaintext. Written 0o600 so it is
+  // not world-readable on the operator's disk, to a temporary name first so
+  // an interrupted run never leaves a truncated file under the real one.
+  const partial = `${out}.part`;
+  let bytes = 0;
+  const counter = new PassThrough();
+  counter.on("data", (chunk: Buffer) => {
+    bytes += chunk.length;
+  });
+  await pipeline(
+    Readable.from(source()),
+    counter,
+    ...(out.endsWith(".gz") ? [createGzip()] : []),
+    createWriteStream(partial, { mode: 0o600 }),
+  );
+  await rename(partial, out);
   console.log(
-    `Restored userId=${body.userId} (${plaintext.length} bytes) -> ${out}`,
+    `Restored userId=${body.userId} (${bytes} bytes of JSON, ${streamedCounts.measurements ?? 0} measurements) -> ${out}`,
   );
 }
 

scripts/rotate-encryption-key.ts

@@ -108,6 +108,49 @@ function shouldRotate(value: string | null): boolean {
   return id !== getActiveKeyId();
 }
 
+/** The part of a Prisma delegate the per-column walks below need. */
+interface PagedDelegate<V> {
+  findMany: (args: {
+    select: Record<string, true>;
+    orderBy: { id: "asc" };
+    take: number;
+    cursor?: { id: string };
+    skip?: number;
+  }) => Promise<Array<Record<string, unknown>>>;
+  update: (args: {
+    where: { id: string };
+    data: Record<string, V>;
+  }) => Promise<unknown>;
+}
+
+/** Rows per page when walking a column. */
+const PAGE_SIZE = 5_000;
+
+/**
+ * Every row of one column, `id` and value only, a page at a time in id order.
+ * The walks used to read a column in one `findMany`, which for
+ * `Measurement.notesEncrypted` is every measurement of the instance: 1.25
+ * million rows on one large account (#1031).
+ */
+async function* pagedRows(
+  delegate: PagedDelegate<unknown>,
+  field: string,
+): AsyncGenerator<Record<string, unknown>> {
+  let cursor: string | null = null;
+  for (;;) {
+    const rows = await delegate.findMany({
+      select: { id: true, [field]: true },
+      orderBy: { id: "asc" },
+      take: PAGE_SIZE,
+      ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
+    });
+    if (rows.length === 0) return;
+    yield* rows;
+    if (rows.length < PAGE_SIZE) return;
+    cursor = rows[rows.length - 1]!.id as string;
+  }
+}
+
 /**
  * Rotate one `String` ciphertext column on a Prisma model. `delegate` is the
  * `prisma.<model>` accessor; `field` is the column. Reads only `id` + the
@@ -116,31 +159,28 @@ function shouldRotate(value: string | null): boolean {
 async function rotateStringColumn(
   table: string,
   field: string,
-  delegate: {
-    findMany: (args: {
-      select: Record<string, true>;
-    }) => Promise<Array<Record<string, unknown>>>;
-    update: (args: {
-      where: { id: string };
-      data: Record<string, string>;
-    }) => Promise<unknown>;
-  },
+  delegate: PagedDelegate<string>,
 ): Promise<RotationResult> {
-  const rows = await delegate.findMany({ select: { id: true, [field]: true } });
   const result: RotationResult = {
     table,
     field,
-    scanned: rows.length,
+    scanned: 0,
     rotated: 0,
     errors: 0,
     dropped: 0,
   };
-  for (const row of rows) {
+  for await (const row of pagedRows(
+    delegate as PagedDelegate<unknown>,
+    field,
+  )) {
     const v = row[field] as string | null;
+    // `scanned` counts rows that hold ciphertext, as the in-app rotation does.
+    if (!v) continue;
+    result.scanned++;
     if (!shouldRotate(v)) continue;
     const id = row.id as string;
     try {
-      const re = encrypt(decrypt(v as string));
+      const re = encrypt(decrypt(v));
       await delegate.update({ where: { id }, data: { [field]: re } });
       result.rotated++;
     } catch (err) {
@@ -159,28 +199,23 @@ async function rotateStringColumn(
 async function rotateBytesColumn(
   table: string,
   field: string,
-  delegate: {
-    findMany: (args: {
-      select: Record<string, true>;
-    }) => Promise<Array<Record<string, unknown>>>;
-    update: (args: {
-      where: { id: string };
-      data: Record<string, Uint8Array>;
-    }) => Promise<unknown>;
-  },
+  delegate: PagedDelegate<Uint8Array>,
 ): Promise<RotationResult> {
-  const rows = await delegate.findMany({ select: { id: true, [field]: true } });
   const result: RotationResult = {
     table,
     field,
-    scanned: rows.length,
+    scanned: 0,
     rotated: 0,
     errors: 0,
     dropped: 0,
   };
-  for (const row of rows) {
+  for await (const row of pagedRows(
+    delegate as PagedDelegate<unknown>,
+    field,
+  )) {
     const buf = row[field] as Uint8Array | null;
     if (!buf || buf.byteLength === 0) continue;
+    result.scanned++;
     const asString = Buffer.from(buf).toString("utf8");
     if (!shouldRotate(asString)) continue;
     const id = row.id as string;

scripts/seed-large-account.ts

@@ -0,0 +1,296 @@
+/**
+ * Seed one account shaped like a multi-year Apple Health `export.xml` import,
+ * for reproducing the maintenance jobs at the size a real heavy account
+ * reaches (issue #1031: about 1.26 million measurement rows).
+ *
+ *   DATABASE_URL=postgresql://… pnpm dlx tsx scripts/seed-large-account.ts \
+ *     [--user bench-large] [--years 4] [--pulse-seconds 165]
+ *
+ * Never point this at a database that holds real data: it inserts rows under
+ * the given user id and, with `--reset`, deletes that user first.
+ *
+ * The shape follows what the import and the iOS sync actually write:
+ *
+ *   - dense spot rows (heart rate every `--pulse-seconds`, HRV, SpO2) keyed by
+ *     a hashed sample id, the tier the dense-intraday retention folds;
+ *   - high-frequency mean rows (respiratory rate, gait metrics, audio
+ *     exposure), the tier the daily-mean consolidation folds;
+ *   - one `stats:` daily row per cumulative type and day (what the export
+ *     import writes for steps, energy, distance and flights);
+ *   - per-sample cumulative rows with opaque sample ids for the most recent
+ *     90 days (what the iOS sync writes, and what the cumulative drain folds);
+ *   - sleep-stage segments every night.
+ *
+ * Everything is generated inside Postgres with `generate_series`, so a million
+ * rows take seconds rather than a million round trips. `ANALYZE` runs at the
+ * end: a planner reading pre-seed statistics picks plans no production table
+ * of this size would get.
+ */
+import { prisma } from "@/lib/db";
+
+function arg(name: string, fallback: string): string {
+  const i = process.argv.indexOf(`--${name}`);
+  return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
+}
+
+const USER_ID = arg("user", "bench-large");
+const YEARS = Number(arg("years", "4"));
+const PULSE_SECONDS = Number(arg("pulse-seconds", "165"));
+const RESET = process.argv.includes("--reset");
+
+/** A spot series: `perDay` evenly spaced rows per day over the whole span. */
+interface SpotSeries {
+  type: string;
+  unit: string;
+  perDay: number;
+  base: number;
+  spread: number;
+  /** Days back from now the series covers. Defaults to the whole span. */
+  days?: number;
+  /** externalId prefix; `stats:` rows are generated separately. */
+  idPrefix: string;
+}
+
+async function insertSpot(s: SpotSeries, days: number): Promise<number> {
+  const step = Math.floor(86_400 / s.perDay);
+  const count = Math.floor((days * 86_400) / step);
+  // Jitter stays below the step so every measured_at is distinct per type
+  // (the (user, type, measured_at, source, sleep_stage) unique index).
+  const jitter = Math.max(1, Math.min(59, step - 1));
+  return prisma.$executeRawUnsafe(
+    `
+    INSERT INTO measurements
+      (id, user_id, type, value, unit, source, measured_at, external_id, created_at, updated_at)
+    SELECT
+      'seed-' || $1 || '-' || $2 || '-' || g,
+      $1,
+      $2::measurement_type,
+      round(($3 + $4 * sin(g / 37.0) + ($4 / 2) * ((g * 7919) % 100) / 100.0)::numeric, 2),
+      $5,
+      'APPLE_HEALTH',
+      date_trunc('minute', now()) - make_interval(secs => $6 * $7::bigint)
+        + make_interval(secs => g * $6 + (g * 31) % $8),
+      $9 || md5($2 || g::text),
+      now(), now()
+    FROM generate_series(0, $7::bigint - 1) AS g
+    `,
+    USER_ID,
+    s.type,
+    s.base,
+    s.spread,
+    s.unit,
+    step,
+    count,
+    jitter,
+    s.idPrefix,
+  );
+}
+
+async function main(): Promise<void> {
+  const days = Math.round(YEARS * 365);
+
+  if (RESET) {
+    await prisma.$executeRawUnsafe(`DELETE FROM users WHERE id = $1`, USER_ID);
+  }
+  await prisma.$executeRawUnsafe(
+    `INSERT INTO users (id, username, timezone, updated_at)
+     VALUES ($1, $1, 'Europe/Berlin', now())
+     ON CONFLICT (id) DO NOTHING`,
+    USER_ID,
+  );
+
+  const pulsePerDay = Math.floor(86_400 / PULSE_SECONDS);
+  const spot: SpotSeries[] = [
+    {
+      type: "PULSE",
+      unit: "bpm",
+      perDay: pulsePerDay,
+      base: 62,
+      spread: 25,
+      idPrefix: "hk-",
+    },
+    {
+      type: "HEART_RATE_VARIABILITY",
+      unit: "ms",
+      perDay: 12,
+      base: 45,
+      spread: 20,
+      idPrefix: "hk-",
+    },
+    {
+      type: "OXYGEN_SATURATION",
+      unit: "%",
+      perDay: 24,
+      base: 96,
+      spread: 2,
+      idPrefix: "hk-",
+    },
+    {
+      type: "RESPIRATORY_RATE",
+      unit: "breaths/min",
+      perDay: 48,
+      base: 14,
+      spread: 3,
+      idPrefix: "hk-",
+    },
+    {
+      type: "WALKING_SPEED",
+      unit: "km/h",
+      perDay: 30,
+      base: 4.8,
+      spread: 0.8,
+      idPrefix: "hk-",
+    },
+    {
+      type: "WALKING_STEP_LENGTH",
+      unit: "cm",
+      perDay: 30,
+      base: 72,
+      spread: 6,
+      idPrefix: "hk-",
+    },
+    {
+      type: "WALKING_ASYMMETRY",
+      unit: "%",
+      perDay: 30,
+      base: 3,
+      spread: 2,
+      idPrefix: "hk-",
+    },
+    {
+      type: "WALKING_DOUBLE_SUPPORT",
+      unit: "%",
+      perDay: 30,
+      base: 28,
+      spread: 3,
+      idPrefix: "hk-",
+    },
+    {
+      type: "AUDIO_EXPOSURE_ENV",
+      unit: "dB",
+      perDay: 60,
+      base: 62,
+      spread: 12,
+      idPrefix: "hk-",
+    },
+    // The iOS sync's per-sample cumulative rows: dense, recent, opaque ids.
+    {
+      type: "ACTIVITY_STEPS",
+      unit: "steps",
+      perDay: 100,
+      base: 90,
+      spread: 60,
+      days: 90,
+      idPrefix: "uuid-",
+    },
+    {
+      type: "ACTIVE_ENERGY_BURNED",
+      unit: "kcal",
+      perDay: 400,
+      base: 1.2,
+      spread: 0.9,
+      days: 90,
+      idPrefix: "uuid-",
+    },
+    {
+      type: "WALKING_RUNNING_DISTANCE",
+      unit: "m",
+      perDay: 100,
+      base: 70,
+      spread: 50,
+      days: 90,
+      idPrefix: "uuid-",
+    },
+    {
+      type: "WEIGHT",
+      unit: "kg",
+      perDay: 1,
+      base: 82,
+      spread: 1.5,
+      idPrefix: "hk-",
+    },
+  ];
+
+  let total = 0;
+  for (const s of spot) {
+    const n = await insertSpot(s, s.days ?? days);
+    total += n;
+    console.log(`${s.type.padEnd(28)} ${String(n).padStart(9)}`);
+  }
+
+  // One export-estimate `stats:` row per cumulative type and local day, at
+  // local noon — what `flushCumulativeBuckets` in the export import writes.
+  for (const [type, hk, unit, base] of [
+    ["ACTIVITY_STEPS", "HKQuantityTypeIdentifierStepCount", "steps", 8000],
+    [
+      "ACTIVE_ENERGY_BURNED",
+      "HKQuantityTypeIdentifierActiveEnergyBurned",
+      "kcal",
+      550,
+    ],
+    [
+      "WALKING_RUNNING_DISTANCE",
+      "HKQuantityTypeIdentifierDistanceWalkingRunning",
+      "m",
+      6000,
+    ],
+    ["FLIGHTS_CLIMBED", "HKQuantityTypeIdentifierFlightsClimbed", "count", 9],
+  ] as const) {
+    const n = await prisma.$executeRawUnsafe(
+      `
+      INSERT INTO measurements
+        (id, user_id, type, value, unit, source, measured_at, external_id,
+         aggregation_provenance, created_at, updated_at)
+      SELECT
+        'seed-' || $1 || '-' || $2 || '-stats-' || d,
+        $1, $2::measurement_type, $5 + (d * 7919) % ($5 / 2), $4, 'APPLE_HEALTH',
+        ((current_date - d)::timestamp + interval '12 hours')
+          AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC',
+        'stats:' || $3 || ':' || to_char(current_date - d, 'YYYY-MM-DD'),
+        'EXPORT_XML_SOURCE_MAX', now(), now()
+      FROM generate_series(91, $6::int) AS d
+      `,
+      USER_ID,
+      type,
+      hk,
+      unit,
+      base,
+      days,
+    );
+    total += n;
+    console.log(`${`${type} (stats:)`.padEnd(28)} ${String(n).padStart(9)}`);
+  }
+
+  // Sleep: 30 segments a night from 23:00 local, 16 minutes apart, rotating
+  // through the stages. Distinct measured_at per segment.
+  const sleep = await prisma.$executeRawUnsafe(
+    `
+    INSERT INTO measurements
+      (id, user_id, type, value, unit, source, measured_at, external_id,
+       sleep_stage, created_at, updated_at)
+    SELECT
+      'seed-' || $1 || '-sleep-' || d || '-' || s,
+      $1, 'SLEEP_DURATION', 16, 'minutes', 'APPLE_HEALTH',
+      ((current_date - d)::timestamp - interval '1 hour' + s * interval '16 minutes')
+        AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC',
+      'hk-' || md5('sleep' || d || '-' || s),
+      (ARRAY['CORE','DEEP','CORE','REM','AWAKE','CORE']::sleep_stage[])[1 + s % 6],
+      now(), now()
+    FROM generate_series(1, $2::int) AS d, generate_series(0, 29) AS s
+    `,
+    USER_ID,
+    days,
+  );
+  total += sleep;
+  console.log(`${"SLEEP_DURATION".padEnd(28)} ${String(sleep).padStart(9)}`);
+
+  await prisma.$executeRawUnsafe(`ANALYZE measurements`);
+  console.log(`${"total".padEnd(28)} ${String(total).padStart(9)}`);
+}
+
+main()
+  .catch((err) => {
+    console.error(err);
+    process.exitCode = 1;
+  })
+  .finally(() => prisma.$disconnect());

src/__tests__/backup-readers-stream-guard.test.ts

@@ -0,0 +1,45 @@
+/**
+ * Every reader of a stored or uploaded backup reads it as a stream (#1031).
+ *
+ * A backup's JSON grows with the record, and past 536 870 888 characters it
+ * cannot exist as one JavaScript string at all: the disaster-recovery JSON of
+ * an account with 1.25 million measurements is 662 MB. Each of the readers
+ * below used to turn the file into one string first, and so none of them
+ * could open that account's backup. This pins that none of them goes back to
+ * it: no whole-document decrypt-to-string, no whole-body text read.
+ */
+import { readFileSync } from "node:fs";
+import { describe, expect, it } from "vitest";
+
+const READERS = [
+  "src/app/api/admin/backups/[id]/restore/route.ts",
+  "src/app/api/admin/backups/[id]/summary/route.ts",
+  "src/app/api/admin/backups/[id]/download/route.ts",
+  "src/app/api/admin/backups/upload/route.ts",
+  "src/lib/jobs/restore-drill.ts",
+  "scripts/restore-backup.ts",
+] as const;
+
+/** Calls that hand back the whole document as one string or parse it whole. */
+const WHOLE_DOCUMENT = [
+  /\bunpackBackupBlob\s*\(/,
+  /\bdecryptBackup\s*\(/,
+  /\bJSON\.parse\s*\(\s*plaintext\b/,
+  /\bparseBackupPayload\s*\(\s*plaintext\b/,
+  /\bfile\.text\s*\(/,
+  /\bawait\s+request\.text\s*\(\s*\)[\s\S]{0,200}parseBackupPayload/,
+];
+
+describe("backup readers stream the file", () => {
+  it.each(READERS)("%s never holds the whole document", (path) => {
+    const source = readFileSync(path, "utf8");
+    for (const pattern of WHOLE_DOCUMENT) {
+      expect(source, `${path} matches ${pattern}`).not.toMatch(pattern);
+    }
+  });
+
+  it.each(READERS)("%s reads through the streaming reader", (path) => {
+    const source = readFileSync(path, "utf8");
+    expect(source).toMatch(/\b(readStreamedBackup|scanBackupJson)\b/);
+  });
+});

src/app/api/admin/backups/[id]/download/route.ts

@@ -14,6 +14,8 @@
  * at-rest contract: the ciphertext stays in the DB, the plaintext is only
  * materialised inside the request handler and streamed to the admin.
  */
+import { Readable } from "node:stream";
+
 import { NextRequest, NextResponse } from "next/server";
 import { prisma } from "@/lib/db";
 import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
@@ -22,8 +24,13 @@ import { auditLog } from "@/lib/auth/audit";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  unpackBackupBlob,
+  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  readStreamedBackup,
+  type BackupSource,
+  type StreamedBackup,
+} from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import { parseBackupPayload } from "@/lib/validations/backup";
 
@@ -54,9 +61,11 @@ export const GET = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    let plaintext: string;
+    // Opened and streamed, never unpacked into one string: a large record's
+    // JSON is longer than any string V8 can hold (#1031).
+    let source: BackupSource;
     try {
-      plaintext = unpackBackupBlob(backup.data);
+      source = openBackupBlob(backup.data);
     } catch (err) {
       // A rotated or dropped key, or a stored copy that is no longer the one
       // that was written. Both are bad stored input rather than a fault in
@@ -80,8 +89,10 @@ export const GET = apiHandler(
     // pathological case where the worker wrote something the upload
     // route would later reject — better to fail loudly here than to ship
     // a junk file that breaks an admin's restore plan.
+    let streamed: StreamedBackup;
     try {
-      parseBackupPayload(plaintext);
+      streamed = await readStreamedBackup(source);
+      parseBackupPayload(streamed.raw);
     } catch (err) {
       await auditLog("admin.backups.download.failed", {
         userId: admin.id,
@@ -107,22 +118,26 @@ export const GET = apiHandler(
         ownerId: backup.userId,
         ownerUsername: backup.user.username,
         type: backup.type,
-        sizeBytes: Buffer.byteLength(plaintext, "utf8"),
+        sizeBytes: streamed.bytes,
       },
     });
 
-    // Returning the plaintext as-is keeps the response a faithful copy of
+    // Returning the stored bytes as-is keeps the response a faithful copy of
     // what the worker wrote (formatting, key order). The file is the
     // canonical artefact admins will store / re-upload.
-    return new NextResponse(plaintext, {
-      status: 200,
-      headers: {
-        "content-type": "application/json; charset=utf-8",
-        "content-disposition": `attachment; filename="${filename}"`,
-        // Backups contain sensitive health data — keep them out of any
-        // shared cache (CDN, browser disk cache).
-        "cache-control": "no-store, max-age=0",
+    return new NextResponse(
+      Readable.toWeb(Readable.from(source())) as ReadableStream<Uint8Array>,
+      {
+        status: 200,
+        headers: {
+          "content-type": "application/json; charset=utf-8",
+          "content-disposition": `attachment; filename="${filename}"`,
+          // Backups contain sensitive health data — keep them out of any
+          // shared cache (CDN, browser disk cache).
+          "cache-control": "no-store, max-age=0",
+          "content-length": String(streamed.bytes),
+        },
       },
-    });
+    );
   },
 );

src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts

@@ -47,6 +47,9 @@ vi.mock("@/lib/validations/backup", () => ({
   parseBackupPayload: vi.fn(),
   isCompatibleSchemaVersion: vi.fn(() => true),
   summarizeBackup: vi.fn(() => ({})),
+  // The streamed reader checks measurements one at a time; this file has none.
+  BACKUP_SCHEMA_VERSION: "2",
+  backupMeasurementSchema: { safeParse: vi.fn() },
 }));
 vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
 vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));

src/app/api/admin/backups/[id]/restore/route.ts

@@ -26,8 +26,17 @@ import { auditLog } from "@/lib/auth/audit";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  unpackBackupBlob,
+  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  insertMeasurementRows,
+  type MeasurementInsertRow,
+} from "@/lib/export/measurement-bulk-insert";
+import {
+  readStreamedBackup,
+  type BackupSource,
+  type StreamedBackup,
+} from "@/lib/export/streamed-backup";
 import { encryptNote } from "@/lib/crypto/note-cipher";
 import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
 import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
@@ -38,6 +47,7 @@ import {
   parseBackupPayload,
   isCompatibleSchemaVersion,
   summarizeBackup,
+  type BackupMeasurement,
   type BackupSummary,
 } from "@/lib/validations/backup";
 import { recomputeUserMoodRollups } from "@/lib/rollups/mood-rollups";
@@ -137,6 +147,19 @@ interface RestoreResponse {
   };
 }
 
+/**
+ * How long the restore transaction may run. Two minutes was the whole budget
+ * when a record was a few hundred thousand rows; deleting and rewriting 1.25
+ * million measurements in one transaction takes longer than that (#1031). On
+ * the test host the whole restore of that account, reading the file twice
+ * included, took 94 s. The budget grows by a second per 5 000 measurements,
+ * which gives that account six minutes, about four times what it needed, and
+ * never drops below the old two minutes.
+ */
+function restoreTransactionTimeoutMs(measurementCount: number): number {
+  return 120_000 + Math.ceil(measurementCount / 5);
+}
+
 function decodeEncryptedBytes(encoded: string): Uint8Array<ArrayBuffer> {
   const decoded = Buffer.from(encoded, "base64");
   const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength));
@@ -198,9 +221,12 @@ const handler = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    let plaintext: string;
+    // Opened, not unpacked: the JSON of a large record is longer than any
+    // string V8 can hold (#1031), so it is read as a stream below. Opening
+    // authenticates the whole ciphertext first, as unpacking did.
+    let source: BackupSource;
     try {
-      plaintext = unpackBackupBlob(backup.data);
+      source = openBackupBlob(backup.data);
     } catch (err) {
       await auditLog("admin.backups.restore.failed", {
         userId: admin.id,
@@ -224,10 +250,16 @@ const handler = apiHandler(
     // Parsed once and kept, because the schema's per-section `.default([])`
     // erases the difference between a section that is absent and one that is
     // empty — and that difference is what the completeness check below reads.
+    //
+    // The measurements are the exception: `readStreamedBackup` checks each one
+    // against the element schema as it goes and keeps only their count, and
+    // the transaction reads them a second time, in batches, as it writes them.
     let raw: unknown;
     let payload;
+    let streamed: StreamedBackup;
     try {
-      raw = JSON.parse(plaintext);
+      streamed = await readStreamedBackup(source);
+      raw = streamed.raw;
       payload = parseBackupPayload(raw);
     } catch (err) {
       await auditLog("admin.backups.restore.failed", {
@@ -548,7 +580,7 @@ const handler = apiHandler(
           }
 
           const toRestoredMeasurementData = (
-            measurement: (typeof payload.measurements)[number],
+            measurement: BackupMeasurement,
           ) => ({
             type: measurement.type,
             value: measurement.value,
@@ -600,55 +632,80 @@ const handler = apiHandler(
               : {}),
           });
 
-          const stableRows = payload.measurements.flatMap((measurement) =>
-            measurement.id
-              ? [
-                  {
-                    id: measurement.id,
-                    userId: ownerId,
-                    ...toRestoredMeasurementData(measurement),
-                  },
-                ]
-              : [],
-          );
-          const measurementBatchSize = 1_000;
-          for (
-            let offset = 0;
-            offset < stableRows.length;
-            offset += measurementBatchSize
-          ) {
-            await tx.measurement.createMany({
-              data: stableRows.slice(offset, offset + measurementBatchSize),
-            });
-          }
-
-          // v1 payloads did not require stable ids. Preserve their historical
-          // natural-key reconciliation without routing canonical v2 rows
-          // through it.
-          for (const measurement of payload.measurements) {
-            if (measurement.id) continue;
-            const restoredData = toRestoredMeasurementData(measurement);
-            const existing = await tx.measurement.findFirst({
-              where: {
-                userId: ownerId,
-                type: measurement.type,
-                source: restoredData.source,
-                measuredAt: restoredData.measuredAt,
-                sleepStage: restoredData.sleepStage,
-              },
-              select: { id: true },
-            });
-            if (existing) {
-              await tx.measurement.update({
-                where: { id: existing.id, userId: ownerId },
-                data: restoredData,
-              });
-            } else {
-              await tx.measurement.create({
-                data: { userId: ownerId, ...restoredData },
-              });
+          // The measurements, a batch at a time from a second read of the
+          // file, so the restore never holds more than one batch of them.
+          //
+          // Two later sections point INTO the measurements (a record's
+          // `sourceMeasurementId`, an ECG strip's `measurementId`) and have
+          // to know which of those ids this transaction actually wrote. They
+          // used to be handed every written id; they only ever ask about the
+          // ones they reference, so only those are collected.
+          const referencedMeasurementIds = new Set<string>();
+          for (const record of payload.personalRecords) {
+            if (record.sourceMeasurementId) {
+              referencedMeasurementIds.add(record.sourceMeasurementId);
             }
           }
+          for (const recording of payload.ecgRecordings) {
+            if (recording.measurementId) {
+              referencedMeasurementIds.add(recording.measurementId);
+            }
+          }
+          const writtenReferencedIds = new Set<string>();
+          const measurementBatchSize = 1_000;
+          await streamed.forEachMeasurementBatch(
+            measurementBatchSize,
+            async (batch) => {
+              const stableRows = batch.flatMap(
+                (measurement): MeasurementInsertRow[] =>
+                  measurement.id
+                    ? [
+                        {
+                          id: measurement.id,
+                          userId: ownerId,
+                          ...toRestoredMeasurementData(measurement),
+                        },
+                      ]
+                    : [],
+              );
+              // One statement per batch, not `createMany`: see
+              // `insertMeasurementRows` for what the latter cost here.
+              await insertMeasurementRows(tx, stableRows);
+              for (const row of stableRows) {
+                if (referencedMeasurementIds.has(row.id)) {
+                  writtenReferencedIds.add(row.id);
+                }
+              }
+
+              // v1 payloads did not require stable ids. Preserve their
+              // historical natural-key reconciliation without routing
+              // canonical v2 rows through it.
+              for (const measurement of batch) {
+                if (measurement.id) continue;
+                const restoredData = toRestoredMeasurementData(measurement);
+                const existing = await tx.measurement.findFirst({
+                  where: {
+                    userId: ownerId,
+                    type: measurement.type,
+                    source: restoredData.source,
+                    measuredAt: restoredData.measuredAt,
+                    sleepStage: restoredData.sleepStage,
+                  },
+                  select: { id: true },
+                });
+                if (existing) {
+                  await tx.measurement.update({
+                    where: { id: existing.id, userId: ownerId },
+                    data: restoredData,
+                  });
+                } else {
+                  await tx.measurement.create({
+                    data: { userId: ownerId, ...restoredData },
+                  });
+                }
+              }
+            },
+          );
 
           const medByName = new Map<string, string>();
           const restoredMedicationIds = new Set<string>();
@@ -1932,7 +1989,7 @@ const handler = apiHandler(
             tx,
             ownerId,
             payload,
-            new Set(stableRows.map((row) => row.id)),
+            writtenReferencedIds,
             skips,
           );
 
@@ -1949,7 +2006,7 @@ const handler = apiHandler(
             tx,
             ownerId,
             payload,
-            new Set(stableRows.map((row) => row.id)),
+            writtenReferencedIds,
             skips,
           );
 
@@ -2024,7 +2081,7 @@ const handler = apiHandler(
         },
         {
           maxWait: 10_000,
-          timeout: 120_000,
+          timeout: restoreTransactionTimeoutMs(streamed.measurementCount),
         },
       );
     } catch (err) {
@@ -2071,7 +2128,7 @@ const handler = apiHandler(
     // outside the transaction (the 5-year fold would otherwise hold a
     // long write lock) and is best-effort so a populator hiccup never
     // undoes the restore. The boot-time backfill is the safety net.
-    if (payload.measurements.length > 0) {
+    if (streamed.measurementCount > 0) {
       try {
         await recomputeUserRollups(ownerId);
       } catch (err) {
@@ -2131,7 +2188,10 @@ const handler = apiHandler(
       }
     }
 
-    const summary = summarizeBackup(payload);
+    const summary = {
+      ...summarizeBackup(payload),
+      measurements: streamed.measurementCount,
+    };
 
     await auditLog("admin.backups.restore", {
       userId: admin.id,

src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts

@@ -44,6 +44,11 @@ vi.mock("@/lib/validations/backup", () => ({
   parseBackupPayload: (...a: unknown[]) => parseMock(...a),
   summarizeBackup: (...a: unknown[]) => summarizeMock(...a),
   isCompatibleSchemaVersion: vi.fn(() => true),
+  // The measurements are read as a stream and counted one by one.
+  BACKUP_SCHEMA_VERSION: "2",
+  backupMeasurementSchema: {
+    safeParse: (value: unknown) => ({ success: true, data: value }),
+  },
 }));
 
 import { GET } from "../route";
@@ -62,9 +67,14 @@ beforeEach(() => {
     createdAt: new Date("2026-08-01T00:00:00Z"),
     user: { id: "u1", username: "self-hoster" },
   });
-  decryptMock.mockReturnValue("{}");
+  // Twelve measurements in the file: the preview counts them from the stream.
+  decryptMock.mockReturnValue(
+    JSON.stringify({
+      measurements: Array.from({ length: 12 }, (_, i) => ({ id: `m${i}` })),
+    }),
+  );
   parseMock.mockReturnValue({ schemaVersion: 3 });
-  summarizeMock.mockReturnValue({ measurements: 12, moodEntries: 3 });
+  summarizeMock.mockReturnValue({ measurements: 0, moodEntries: 3 });
 });
 
 describe("GET /api/admin/backups/[id]/summary", () => {

src/app/api/admin/backups/[id]/summary/route.ts

@@ -19,8 +19,12 @@ import { apiError, apiSuccess } from "@/lib/api-response";
 import {
   BACKUP_UNDECRYPTABLE_CODE,
   BACKUP_UNDECRYPTABLE_ERROR,
-  unpackBackupBlob,
+  openBackupBlob,
 } from "@/lib/export/backup-blob";
+import {
+  readStreamedBackup,
+  type BackupSource,
+} from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import {
   isCompatibleSchemaVersion,
@@ -46,9 +50,11 @@ export const GET = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    let plaintext: string;
+    // Opened and read as a stream: a large record's JSON is longer than any
+    // string V8 can hold (#1031).
+    let source: BackupSource;
     try {
-      plaintext = unpackBackupBlob(backup.data);
+      source = openBackupBlob(backup.data);
     } catch {
       // Same refusal the restore and the download give, for the same reason:
       // a copy this instance cannot open is bad stored input, and the preview
@@ -59,8 +65,11 @@ export const GET = apiHandler(
     }
 
     let payload;
+    let measurementCount: number;
     try {
-      payload = parseBackupPayload(plaintext);
+      const streamed = await readStreamedBackup(source);
+      measurementCount = streamed.measurementCount;
+      payload = parseBackupPayload(streamed.raw);
     } catch {
       return apiError("Backup payload failed schema validation", 422);
     }
@@ -72,7 +81,10 @@ export const GET = apiHandler(
       );
     }
 
-    const summary = summarizeBackup(payload);
+    const summary = {
+      ...summarizeBackup(payload),
+      measurements: measurementCount,
+    };
 
     annotate({
       action: { name: "admin.backups.summary" },

src/app/api/admin/backups/upload/route.ts

@@ -1,17 +1,25 @@
 /**
  * POST /api/admin/backups/upload — admin-only ingest of a backup file.
  *
- * Accepts a `multipart/form-data` body with a single `file` field that
- * carries a JSON backup matching `backupPayloadSchema`. The route:
+ * Accepts the backup file either as the raw request body (JSON, or JSON
+ * compressed with gzip) or, for a small file, as a `multipart/form-data`
+ * body with a single `file` field. The route:
  *
  *   1. Authenticates the caller as an admin (cookie session only).
  *   2. Reads the file (size capped — see `MAX_UPLOAD_BYTES`).
  *   3. Parses and validates the JSON against the canonical schema.
  *   4. Rejects files written by a future schema version (incompatible).
- *   5. Encrypts the plaintext and inserts a new `DataBackup` row of
- *      type `MANUAL_UPLOAD_<unix-ms>` so multiple uploads coexist with
+ *   5. Encrypts the file as it arrives and inserts a new `DataBackup` row
+ *      of type `MANUAL_UPLOAD_<unix-ms>` so multiple uploads coexist with
  *      the rolling `WEEKLY_AUTO` snapshot AND with each other.
  *
+ * The file is read as a stream and never held whole (#1031): an export of an
+ * account with 1.25 million measurements is a few hundred megabytes of JSON,
+ * which neither fits a 1 GB container beside the running app nor, in the
+ * disaster-recovery shape, fits in one JavaScript string at all. It is
+ * checked as it goes and stored in the same pass; a file that fails any check
+ * rolls the store back, so nothing of it remains.
+ *
  * Crucially, this route does NOT execute a restore. It only stores the
  * file. The separate `POST /api/admin/backups/[id]/restore` endpoint
  * (criterion 3) actually replaces user data — keeping the two phases
@@ -20,46 +28,120 @@
  *
  * Phase B1 / criterion 2 of the v1.4.15 backup-completeness work.
  */
+import { Readable } from "node:stream";
+import { createGunzip } from "node:zlib";
+
 import { NextRequest } from "next/server";
 import { ZodError } from "zod/v4";
 import { prisma } from "@/lib/db";
 import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
 import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
 import { auditLog } from "@/lib/auth/audit";
-import { packBackupBlob } from "@/lib/export/backup-blob";
+import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { StreamedBackupInvalidError } from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import { checkRateLimit } from "@/lib/rate-limit";
 import {
+  BACKUP_SCHEMA_VERSION,
+  backupMeasurementSchema,
   backupPayloadSchema,
   isCompatibleSchemaVersion,
   summarizeBackup,
+  type BackupPayload,
   type BackupSummary,
 } from "@/lib/validations/backup";
 
 export const dynamic = "force-dynamic";
 
 /**
- * Cap on uploaded file size. The largest production backups today are
- * ~2 MB; 10 MB leaves headroom while keeping a malicious admin client
- * from blowing memory by streaming a 4 GB file. Enforced by reading the
- * `Content-Length` header AND the actual buffered size after decoding —
- * a missing/wrong content-length doesn't bypass the limit.
+ * Cap on a file sent as a multipart form. The form parser holds the whole
+ * file in memory, so this path stays for small files only; anything larger
+ * is sent as the raw request body, which is read as a stream.
  */
 const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
 
+/**
+ * Cap on a file sent as the raw request body, compressed or not. The same
+ * ceiling the app sets for any request body (`middlewareClientMaxBodySize` in
+ * `next.config.ts`), beyond which the body would arrive truncated. A
+ * disaster-recovery file of 1.25 million measurements is 662 MB as plain
+ * JSON and 64 MB compressed, so large files go up compressed.
+ */
+const MAX_RAW_UPLOAD_BYTES = 512 * 1024 * 1024;
+
 interface UploadResponse {
   id: string;
   valid: true;
   summary: BackupSummary;
 }
 
+/** A file the route refuses: the answer, and what the audit row records. */
+class UploadRefused extends Error {
+  constructor(
+    readonly status: number,
+    readonly answer: string,
+    readonly details: Record<string, unknown>,
+    readonly meta?: Record<string, unknown>,
+  ) {
+    super(answer);
+    this.name = "UploadRefused";
+  }
+}
+
+/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */
+async function* decodedBytes(
+  source: AsyncIterable<Uint8Array>,
+  limit: number,
+): AsyncGenerator<Uint8Array> {
+  let seen = 0;
+  async function* counted() {
+    for await (const chunk of source) {
+      seen += chunk.byteLength;
+      if (seen > limit) {
+        throw new UploadRefused(
+          413,
+          `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`,
+          { reason: "file_size_exceeded", size: seen },
+        );
+      }
+      yield chunk;
+    }
+  }
+  const iterator = counted()[Symbol.asyncIterator]();
+  const first = await iterator.next();
+  if (first.done) return;
+  const rest = (async function* () {
+    yield first.value;
+    for (;;) {
+      const next = await iterator.next();
+      if (next.done) return;
+      yield next.value;
+    }
+  })();
+  const head = first.value;
+  if (head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b) {
+    const gunzip = Readable.from(rest).pipe(createGunzip());
+    try {
+      for await (const chunk of gunzip) yield chunk as Buffer;
+    } catch (err) {
+      if (err instanceof UploadRefused) throw err;
+      throw new UploadRefused(422, "Uploaded file is not valid gzip", {
+        reason: "invalid_gzip",
+      });
+    }
+    return;
+  }
+  yield* rest;
+}
+
 export const POST = apiHandler(async (request: NextRequest) => {
   const { user: admin } = await requireAdmin();
   annotate({ action: { name: "admin.backups.upload" } });
+  const ipAddress = getClientIp(request);
 
-  // Rate-limit identical to the manual-run endpoint. An admin uploading
-  // a 10 MB blob 60× per minute would just be wasteful — three is plenty
-  // for a deliberate restore-prep workflow.
+  // Rate-limit identical to the manual-run endpoint. Three is plenty for a
+  // deliberate restore-prep workflow.
   const rl = await checkRateLimit(
     `admin-backups-upload:${admin.id}`,
     3,
@@ -69,120 +151,56 @@ export const POST = apiHandler(async (request: NextRequest) => {
     throw new HttpError(429, "Too many backup uploads");
   }
 
-  // Cheap pre-flight on the declared content length. The actual buffer
-  // size check below is the hard limit; this just rejects obvious abuse
-  // before allocating the FormData parser.
+  const denied = (reason: string, extra: Record<string, unknown> = {}) =>
+    auditLog("admin.backups.upload.denied", {
+      userId: admin.id,
+      ipAddress,
+      details: { reason, ...extra },
+    });
+
+  const contentType = request.headers.get("content-type") ?? "";
+  const multipart = contentType.startsWith("multipart/form-data");
+  const limit = multipart ? MAX_UPLOAD_BYTES : MAX_RAW_UPLOAD_BYTES;
+
+  // Cheap pre-flight on the declared content length. The byte count while
+  // reading is the hard limit; this just rejects obvious abuse early.
   const contentLength = Number(request.headers.get("content-length") ?? 0);
-  if (contentLength > MAX_UPLOAD_BYTES) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: { reason: "content_length_exceeded", contentLength },
-    });
-    return apiError("Upload exceeds 10 MB limit", 413);
-  }
-
-  let formData: FormData;
-  try {
-    formData = await request.formData();
-  } catch (err) {
+  if (contentLength > limit) {
+    await denied("content_length_exceeded", { contentLength });
     return apiError(
-      `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`,
-      400,
+      `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`,
+      413,
     );
   }
 
-  const file = formData.get("file");
-  if (!(file instanceof File)) {
-    return apiError("Field 'file' must be a file", 422);
-  }
-
-  if (file.size > MAX_UPLOAD_BYTES) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: { reason: "file_size_exceeded", size: file.size },
-    });
-    return apiError("Upload exceeds 10 MB limit", 413);
-  }
-
-  let plaintext: string;
-  try {
-    plaintext = await file.text();
-  } catch {
-    return apiError("Failed to read uploaded file", 400);
-  }
-
-  let parsedJson: unknown;
-  try {
-    parsedJson = JSON.parse(plaintext);
-  } catch (err) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: {
-        reason: "invalid_json",
-        message: err instanceof Error ? err.message : "parse_failed",
-      },
-    });
-    return apiError("Uploaded file is not valid JSON", 422);
-  }
-
-  let payload;
-  try {
-    payload = backupPayloadSchema.parse(parsedJson);
-  } catch (err) {
-    const issues =
-      err instanceof ZodError
-        ? err.issues.slice(0, 10).map((i) => ({
-            path: i.path.join("."),
-            message: i.message,
-          }))
-        : [];
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: { reason: "schema_invalid", issues },
-    });
-    return apiError("Backup payload failed schema validation", 422, {
-      issues,
-    });
-  }
-
-  if (!isCompatibleSchemaVersion(payload.schemaVersion)) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: {
-        reason: "incompatible_schema_version",
-        schemaVersion: payload.schemaVersion,
-      },
-    });
-    return apiError(
-      `Backup schema version '${payload.schemaVersion}' is not supported by this server`,
-      422,
+  let source: AsyncIterable<Uint8Array>;
+  if (multipart) {
+    let formData: FormData;
+    try {
+      formData = await request.formData();
+    } catch (err) {
+      return apiError(
+        `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`,
+        400,
+      );
+    }
+    const file = formData.get("file");
+    if (!(file instanceof File)) {
+      return apiError("Field 'file' must be a file", 422);
+    }
+    if (file.size > MAX_UPLOAD_BYTES) {
+      await denied("file_size_exceeded", { size: file.size });
+      return apiError("Upload exceeds 10 MB limit", 413);
+    }
+    source = Readable.fromWeb(
+      file.stream() as Parameters<typeof Readable.fromWeb>[0],
     );
-  }
-
-  // Make sure the userId on the file points at a user that exists in
-  // this DB. Restore would fail later anyway, but failing here gives
-  // the admin a precise error before any side-effect.
-  const owner = await prisma.user.findUnique({
-    where: { id: payload.userId },
-    select: { id: true, username: true },
-  });
-  if (!owner) {
-    await auditLog("admin.backups.upload.denied", {
-      userId: admin.id,
-      ipAddress: getClientIp(request),
-      details: {
-        reason: "owner_not_found",
-        ownerId: payload.userId,
-      },
-    });
-    return apiError(
-      `Backup is for user '${payload.userId}' which does not exist in this DB`,
-      422,
+  } else {
+    if (!request.body) {
+      return apiError("The backup file is missing from the request body", 422);
+    }
+    source = Readable.fromWeb(
+      request.body as Parameters<typeof Readable.fromWeb>[0],
     );
   }
 
@@ -190,25 +208,146 @@ export const POST = apiHandler(async (request: NextRequest) => {
   // intact so multiple uploads can coexist for the same user without
   // overwriting each other or the rolling WEEKLY_AUTO snapshot.
   const uploadType = `MANUAL_UPLOAD_${Date.now()}`;
-  const encrypted = packBackupBlob(JSON.stringify(payload));
+  let payload: BackupPayload | undefined;
+  let measurementCount = 0;
+  let owner: { id: string; username: string } | null = null;
 
-  const created = await prisma.dataBackup.create({
-    data: {
-      userId: owner.id,
-      type: uploadType,
-      data: encrypted,
-    },
-  });
+  let created: { id: string };
+  try {
+    created = await storeBackupBlob(
+      prisma,
+      () => ({ userId: owner!.id, type: uploadType }),
+      async (write) => {
+        // One pass: every chunk is stored as it is checked, and a check that
+        // fails throws, which rolls the store back.
+        async function* storedAsRead() {
+          for await (const chunk of decodedBytes(source, limit)) {
+            await write(Buffer.from(chunk));
+            yield chunk;
+          }
+        }
+        let firstWithoutId: number | null = null;
+        let scanned;
+        try {
+          scanned = await scanBackupJson(storedAsRead(), {
+            streamKeys: new Set(["measurements"]),
+            onElement: (_key, element, index) => {
+              const parsed = backupMeasurementSchema.safeParse(element);
+              if (!parsed.success) {
+                const issue = parsed.error.issues[0];
+                throw new StreamedBackupInvalidError(
+                  ["measurements", index, ...(issue?.path ?? [])].join("."),
+                  issue?.message ?? "invalid measurement",
+                );
+              }
+              if (!parsed.data.id && firstWithoutId === null) {
+                firstWithoutId = index;
+              }
+            },
+          });
+        } catch (err) {
+          if (err instanceof BackupJsonError) {
+            throw new UploadRefused(422, "Uploaded file is not valid JSON", {
+              reason: "invalid_json",
+              message: err.message,
+            });
+          }
+          if (err instanceof StreamedBackupInvalidError) {
+            const issues = [{ path: err.path, message: err.message }];
+            throw new UploadRefused(
+              422,
+              "Backup payload failed schema validation",
+              { reason: "schema_invalid", issues },
+              { issues },
+            );
+          }
+          throw err;
+        }
+        measurementCount = scanned.streamedCounts.measurements ?? 0;
 
-  const summary = summarizeBackup(payload);
+        try {
+          payload = backupPayloadSchema.parse(scanned.document);
+        } catch (err) {
+          const issues =
+            err instanceof ZodError
+              ? err.issues.slice(0, 10).map((i) => ({
+                  path: i.path.join("."),
+                  message: i.message,
+                }))
+              : [];
+          throw new UploadRefused(
+            422,
+            "Backup payload failed schema validation",
+            { reason: "schema_invalid", issues },
+            { issues },
+          );
+        }
+        if (
+          payload.schemaVersion === BACKUP_SCHEMA_VERSION &&
+          firstWithoutId !== null
+        ) {
+          const issues = [
+            {
+              path: `measurements.${firstWithoutId}.id`,
+              message: "Canonical v2 measurements require a stable id",
+            },
+          ];
+          throw new UploadRefused(
+            422,
+            "Backup payload failed schema validation",
+            { reason: "schema_invalid", issues },
+            { issues },
+          );
+        }
+
+        if (!isCompatibleSchemaVersion(payload.schemaVersion)) {
+          throw new UploadRefused(
+            422,
+            `Backup schema version '${payload.schemaVersion}' is not supported by this server`,
+            {
+              reason: "incompatible_schema_version",
+              schemaVersion: payload.schemaVersion,
+            },
+          );
+        }
+
+        // Make sure the userId on the file points at a user that exists in
+        // this DB. Restore would fail later anyway, but failing here gives
+        // the admin a precise error before any side-effect.
+        owner = await prisma.user.findUnique({
+          where: { id: payload.userId },
+          select: { id: true, username: true },
+        });
+        if (!owner) {
+          throw new UploadRefused(
+            422,
+            `Backup is for user '${payload.userId}' which does not exist in this DB`,
+            { reason: "owner_not_found", ownerId: payload.userId },
+          );
+        }
+      },
+    );
+  } catch (err) {
+    if (err instanceof UploadRefused) {
+      await denied(err.details.reason as string, err.details);
+      return apiError(err.answer, err.status, err.meta);
+    }
+    throw err;
+  }
+
+  const summary: BackupSummary = {
+    ...summarizeBackup(payload!),
+    measurements: measurementCount,
+  };
+  const storedOwner = owner as { id: string; username: string } | null;
 
   await auditLog("admin.backups.upload", {
     userId: admin.id,
-    ipAddress: getClientIp(request),
+    ipAddress,
     details: {
       backupId: created.id,
-      ownerId: owner.id,
-      ownerUsername: owner.username,
+      ownerId: storedOwner?.id,
+      ownerUsername: storedOwner?.username,
       type: uploadType,
       schemaVersion: summary.schemaVersion,
       counts: {

src/app/api/export/__tests__/soft-delete-filter.test.ts

@@ -64,6 +64,29 @@ vi.mock("@/lib/db", () => ({
     documentConditionLink: { findMany: vi.fn() },
     extractedFact: { findMany: vi.fn() },
     ecgRecording: { findMany: vi.fn() },
+    onboardingRecord: { findUnique: vi.fn().mockResolvedValue(null) },
+    environmentTravelLocation: { findMany: vi.fn().mockResolvedValue([]) },
+    environmentContext: { findMany: vi.fn().mockResolvedValue([]) },
+    personalRecord: { findMany: vi.fn().mockResolvedValue([]) },
+    coachReminder: { findMany: vi.fn().mockResolvedValue([]) },
+    coachPlan: { findMany: vi.fn().mockResolvedValue([]) },
+    coachFact: { findMany: vi.fn().mockResolvedValue([]) },
+    coachConversation: { findMany: vi.fn().mockResolvedValue([]) },
+    measurementReminderEvent: { findMany: vi.fn().mockResolvedValue([]) },
+    measurementReminder: { findMany: vi.fn().mockResolvedValue([]) },
+    vaccinationDocumentLink: { findMany: vi.fn().mockResolvedValue([]) },
+    vaccinationRecord: { findMany: vi.fn().mockResolvedValue([]) },
+    encounterConditionLink: { findMany: vi.fn().mockResolvedValue([]) },
+    encounterLabLink: { findMany: vi.fn().mockResolvedValue([]) },
+    encounterDocumentLink: { findMany: vi.fn().mockResolvedValue([]) },
+    encounter: { findMany: vi.fn().mockResolvedValue([]) },
+    practitioner: { findMany: vi.fn().mockResolvedValue([]) },
+    healthScoreRecord: { findMany: vi.fn().mockResolvedValue([]) },
+    intradayCumulativeProfile: { findMany: vi.fn().mockResolvedValue([]) },
+    correlationPattern: { findMany: vi.fn().mockResolvedValue([]) },
+    healthProfileFactRevision: { findMany: vi.fn().mockResolvedValue([]) },
+    customMetric: { findMany: vi.fn().mockResolvedValue([]) },
+    cycleSymptom: { findMany: vi.fn().mockResolvedValue([]) },
   },
 }));
 
@@ -177,6 +200,7 @@ beforeEach(() => {
   );
   vi.mocked(prisma.extractedFact.findMany).mockResolvedValue([] as never);
   vi.mocked(prisma.ecgRecording.findMany).mockResolvedValue([] as never);
+  vi.mocked(prisma.userAchievement.findMany).mockResolvedValue([] as never);
 });
 
 describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
@@ -192,7 +216,13 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
 
   it("/api/export/full-backup scopes the measurement read to deletedAt: null", async () => {
     const { GET } = await import("../full-backup/route");
-    await GET(mkReq("http://localhost/api/export/full-backup"));
+    // The file is written into the response as it is produced, so the reads
+    // happen while the body is consumed. The mocked client does not cover
+    // every section the builder reads, so the body errors part-way (as the
+    // whole response did before it streamed); the reads asserted here have
+    // been issued by then.
+    const res = await GET(mkReq("http://localhost/api/export/full-backup"));
+    await res.text();
     expect(prisma.measurement.findMany).toHaveBeenCalledWith(
       expect.objectContaining({
         where: expect.objectContaining({ deletedAt: null }),
@@ -202,7 +232,10 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
 
   it("/api/export/full-backup scopes every v1.28 records domain to userId + deletedAt: null", async () => {
     const { GET } = await import("../full-backup/route");
-    await GET(mkReq("http://localhost/api/export/full-backup"));
+    // The file is written into the response as it is produced, so the reads
+    // happen while the body is consumed.
+    const res = await GET(mkReq("http://localhost/api/export/full-backup"));
+    await res.text();
 
     expect(prisma.labResult.findMany).toHaveBeenCalledWith(
       expect.objectContaining({

src/app/api/export/encrypted/route.ts

@@ -42,13 +42,44 @@ import {
   safeJson,
 } from "@/lib/api-response";
 import { checkRateLimit } from "@/lib/rate-limit";
-import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
+import { randomBytes } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { readdir, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import {
-  encryptArchive,
+  encryptArchiveToFile,
   MIN_EXPORT_PASSPHRASE_LENGTH,
+  type SpooledArchive,
 } from "@/lib/export/passphrase-archive";
+import { streamToResponseBody } from "@/lib/export/response-stream";
 import { NextRequest, NextResponse } from "next/server";
 
+const SPOOL_PREFIX = "healthlog-export-";
+
+/**
+ * Remove spool files an earlier export left behind: a process that died
+ * mid-export never reached its cleanup. An hour is far longer than any export
+ * takes, so nothing still being written or sent is touched.
+ */
+async function removeStaleSpools(): Promise<void> {
+  try {
+    const dir = tmpdir();
+    const cutoff = Date.now() - 60 * 60 * 1000;
+    for (const name of await readdir(dir)) {
+      if (!name.startsWith(SPOOL_PREFIX)) continue;
+      const path = join(dir, name);
+      const info = await stat(path).catch(() => null);
+      if (info && info.mtimeMs < cutoff) await rm(path, { force: true });
+    }
+  } catch {
+    // Housekeeping only; an export must not fail over it.
+  }
+}
+
 const encryptedExportSchema = z
   .object({
     passphrase: z
@@ -89,10 +120,31 @@ export const POST = apiHandler(async (request: NextRequest) => {
   }
   const { passphrase } = parsed.data;
 
-  const { payload, counts } = await buildFullBackupPayload(prisma, user.id);
-
-  // Encrypt the JSON bytes. The passphrase goes no further than the KDF.
-  const archive = await encryptArchive(JSON.stringify(payload), passphrase);
+  // Sealed as it is produced and spooled to a temporary file, then sent: the
+  // format puts the tag in front of the ciphertext, so the archive cannot go
+  // out before its last byte is encrypted, and holding it in memory is what
+  // took the app down on an account of 1.25 million measurements (#1031). Only
+  // ciphertext touches the disk, and the file is removed once it is sent or
+  // the client leaves.
+  let counts: FullBackupCounts | undefined;
+  await removeStaleSpools();
+  const bodyPath = join(
+    tmpdir(),
+    `${SPOOL_PREFIX}${randomBytes(12).toString("hex")}.hlx.part`,
+  );
+  let archive: SpooledArchive;
+  try {
+    archive = await encryptArchiveToFile(
+      async (write) => {
+        counts = await streamFullBackupJson(prisma, user.id, write);
+      },
+      passphrase,
+      bodyPath,
+    );
+  } catch (err) {
+    await rm(bodyPath, { force: true });
+    throw err;
+  }
 
   await auditLog("user.export.encrypted", {
     userId: user.id,
@@ -102,38 +154,36 @@ export const POST = apiHandler(async (request: NextRequest) => {
 
   annotate({
     meta: {
-      export_measurements_count: counts.measurements,
-      export_medications_count: counts.medications,
-      export_intake_count: counts.intakeEvents,
-      export_medication_side_effect_count: counts.medicationSideEffects,
-      export_mood_count: counts.moodEntries,
-      export_cycle_count: counts.cycles,
-      export_cycle_day_log_count: counts.cycleDayLogs,
-      export_lab_result_count: counts.labResults,
-      export_biomarker_count: counts.biomarkers,
-      export_illness_episode_count: counts.illnessEpisodes,
-      export_illness_day_log_count: counts.illnessDayLogs,
-      export_allergy_count: counts.allergies,
-      export_family_history_count: counts.familyHistory,
-      export_workout_count: counts.workouts,
-      export_document_count: counts.documents,
-      export_nutrient_day_count: counts.nutrientDays,
-      export_health_profile_count: counts.healthProfile,
-      export_custom_metric_count: counts.customMetrics,
-      export_custom_metric_entry_count: counts.customMetricEntries,
-      export_intraday_profile_count: counts.intradayProfiles,
+      export_measurements_count: counts?.measurements,
+      export_medications_count: counts?.medications,
+      export_intake_count: counts?.intakeEvents,
+      export_mood_count: counts?.moodEntries,
       export_archive_bytes: archive.byteLength,
     },
   });
 
+  const archiveBody = streamToResponseBody(
+    async (write) => {
+      await write(archive.prefix);
+      for await (const chunk of createReadStream(archive.bodyPath)) {
+        await write(chunk as Buffer);
+      }
+    },
+    {
+      onComplete: () => rm(archive.bodyPath, { force: true }),
+      onError: () => rm(archive.bodyPath, { force: true }),
+    },
+  );
+
   const stamp = new Date().toISOString().slice(0, 10);
   // Return the raw binary archive (NOT the apiSuccess envelope). The file is a
   // self-contained `.hlx` archive openable with the user's passphrase via
   // scripts/decrypt-export.ts.
-  return new NextResponse(new Uint8Array(archive), {
+  return new NextResponse(archiveBody, {
     status: 200,
     headers: {
       "Content-Type": "application/octet-stream",
+      "Content-Length": String(archive.byteLength),
       "Content-Disposition": `attachment; filename="healthlog-backup-${user.id}-${stamp}.hlx"`,
       "Cache-Control": "no-store",
     },

src/app/api/export/full-backup/route.ts

@@ -35,7 +35,9 @@ import { annotate } from "@/lib/logging/context";
 import { auditLog } from "@/lib/auth/audit";
 import { apiError, getClientIp } from "@/lib/api-response";
 import { checkRateLimit } from "@/lib/rate-limit";
-import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
+import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
+import { streamToResponseBody } from "@/lib/export/response-stream";
 import { NextRequest, NextResponse } from "next/server";
 
 export const GET = apiHandler(async (request: NextRequest) => {
@@ -47,48 +49,44 @@ export const GET = apiHandler(async (request: NextRequest) => {
     return apiError("Maximum 10 exports per hour", 429);
   }
 
-  // v1.23 — the payload builder is shared with the passphrase-encrypted
-  // export route so both emit the byte-for-byte same shape (the measurement/
-  // medication/mood/cycle domains are restore-compatible; the newer records
-  // domains are export-only for now — see the module doc comment above).
-  const { payload, counts } = await buildFullBackupPayload(prisma, user.id);
-
-  await auditLog("user.export.full-backup", {
-    userId: user.id,
-    ipAddress: getClientIp(request),
-    details: { counts },
-  });
-
-  annotate({
-    meta: {
-      export_measurements_count: counts.measurements,
-      export_medications_count: counts.medications,
-      export_intake_count: counts.intakeEvents,
-      export_medication_side_effect_count: counts.medicationSideEffects,
-      export_mood_count: counts.moodEntries,
-      export_cycle_count: counts.cycles,
-      export_cycle_day_log_count: counts.cycleDayLogs,
-      export_lab_result_count: counts.labResults,
-      export_biomarker_count: counts.biomarkers,
-      export_illness_episode_count: counts.illnessEpisodes,
-      export_illness_day_log_count: counts.illnessDayLogs,
-      export_allergy_count: counts.allergies,
-      export_family_history_count: counts.familyHistory,
-      export_workout_count: counts.workouts,
-      export_document_count: counts.documents,
-      export_nutrient_day_count: counts.nutrientDays,
-      export_health_profile_count: counts.healthProfile,
-      export_custom_metric_count: counts.customMetrics,
-      export_custom_metric_entry_count: counts.customMetricEntries,
-      export_intraday_profile_count: counts.intradayProfiles,
+  // Written into the response as it is produced (`streamFullBackupJson`), not
+  // built and stringified first: for an account of 1.25 million measurements
+  // the built payload and its JSON string did not fit a 1 GB container, and
+  // the export took the whole app down (#1031). The file is byte for byte
+  // what `JSON.stringify` of the payload builder's object gives; the passphrase
+  // export beside this route writes the same bytes, encrypted.
+  //
+  // The counts are only known once the last row has gone out, so the audit
+  // row is written then. A file that stops early is audited as a failure.
+  const ipAddress = getClientIp(request);
+  let counts: FullBackupCounts | undefined;
+  const body = streamToResponseBody(
+    async (write) => {
+      counts = await streamFullBackupJson(prisma, user.id, write);
     },
-  });
+    {
+      onComplete: () =>
+        auditLog("user.export.full-backup", {
+          userId: user.id,
+          ipAddress,
+          details: { counts },
+        }),
+      onError: (err) =>
+        auditLog("user.export.full-backup.failed", {
+          userId: user.id,
+          ipAddress,
+          details: {
+            reason: err instanceof Error ? err.message : String(err),
+          },
+        }),
+    },
+  );
 
   const stamp = new Date().toISOString().slice(0, 10);
   // Stream the JSON directly (NOT wrapped in the apiSuccess envelope) so
   // the file is a self-contained backup — admin upload + restore expect
   // the raw payload, not `{ data: { ... } }`.
-  return new NextResponse(JSON.stringify(payload), {
+  return new NextResponse(body, {
     status: 200,
     headers: {
       "Content-Type": "application/json; charset=utf-8",

src/components/admin/__tests__/backups-section-empty.test.tsx

@@ -130,10 +130,10 @@ describe("BackupsSection — empty state", () => {
     const className = helpElement?.match(/\bclass="([^"]*)"/)?.[1] ?? "";
 
     expect(english).toContain(
-      "JSON file matching the current backup schema. Max 10 MB.",
+      "A HealthLog backup as .json or .json.gz. The file is compressed before upload; compressed, it may be up to 512 MB.",
     );
     expect(german).toContain(
-      "JSON-Datei passend zum aktuellen Backup-Schema. Max. 10 MB.",
+      "HealthLog-Backup als .json oder .json.gz. Die Datei wird vor dem Hochladen komprimiert; komprimiert darf sie bis zu 512 MB groß sein.",
     );
     expect(helpElement).toBeDefined();
     expect(className.split(/\s+/)).toContain("text-muted-foreground");

src/components/admin/backups-section.tsx

@@ -454,6 +454,20 @@ function missingSectionLabel(
  * the server may reword, and matching on it would turn a copy edit into a
  * silent loss of the panel.
  */
+
+/**
+ * The upload body for a backup file: the file itself when it is already
+ * gzip, otherwise the file compressed in the browser. The server accepts
+ * both; compressing keeps a large export well inside the request size limit.
+ */
+async function uploadBody(file: File): Promise<Blob> {
+  const head = new Uint8Array(await file.slice(0, 2).arrayBuffer());
+  if (head[0] === 0x1f && head[1] === 0x8b) return file;
+  return new Response(
+    file.stream().pipeThrough(new CompressionStream("gzip")),
+  ).blob();
+}
+
 export function missingSectionsOf(err: unknown): MissingBackupSection[] {
   if (!(err instanceof ApiError)) return [];
   if (err.meta?.errorCode !== "backup.section.missing") return [];
@@ -697,10 +711,11 @@ export function BackupsSection() {
   const fileInputRef = useRef<HTMLInputElement | null>(null);
   const upload = useMutation({
     mutationFn: async (file: File) => {
-      const fd = new FormData();
-      fd.append("file", file);
-      // apiFetch with a raw init: the multipart body must NOT be JSON
-      // re-encoded, so the verb helper (which JSON-stringifies) is out.
+      // Sent as the raw body, compressed on the way unless it already is: the
+      // server reads it as a stream, and a large export (hundreds of MB of
+      // JSON) shrinks about tenfold. No request timeout, because a file of
+      // that size takes longer to check and store than the default allows.
+      const body = await uploadBody(file);
       return apiFetch<{
         id: string;
         valid: true;
@@ -734,7 +749,12 @@ export function BackupsSection() {
           customMetricEntries?: number;
           intradayProfiles?: number;
         };
-      }>("/api/admin/backups/upload", { method: "POST", body: fd });
+      }>("/api/admin/backups/upload", {
+        method: "POST",
+        body,
+        headers: { "Content-Type": "application/gzip" },
+        signal: null,
+      });
     },
     onSuccess: (data) => {
       const total =
@@ -818,7 +838,8 @@ export function BackupsSection() {
       return apiPost<{ restored: true; skipped?: RestoreSkipSummary }>(
         `/api/admin/backups/${row.id}/restore`,
         { confirm: "RESTORE", restoreInstanceSettings },
-        { headers: { "Idempotency-Key": idempotencyKey } },
+        // No request timeout: rebuilding a large account takes minutes.
+        { headers: { "Idempotency-Key": idempotencyKey }, signal: null },
       );
     },
     onSuccess: (data) => {
@@ -959,7 +980,7 @@ export function BackupsSection() {
           <input
             ref={fileInputRef}
             type="file"
-            accept="application/json,.json"
+            accept="application/json,.json,application/gzip,.gz"
             className="sr-only"
             onChange={handleFileChange}
             aria-label={t("admin.section.backups.uploadButton")}

src/lib/crypto/__tests__/encrypted-column-writers.test.ts

@@ -66,6 +66,29 @@ const RELATION_ENVELOPE = new Set([
  */
 const NOT_ROTATED: ReadonlySet<string> = new Set<string>();
 
+/**
+ * Ciphertext written by raw SQL, which the Prisma-call walk below cannot see.
+ *
+ * `DataBackup.data` is the case: the stored backup is produced as ciphertext
+ * pieces (`packBackupBlobInto`) and assembled into the row by one SQL
+ * statement, because holding it as one string in the process is what a large
+ * record could not afford (#1031). Each entry names the file and the
+ * statement, and the scan fails if the statement is no longer there, so a
+ * moved or removed raw write cannot leave a stale entry that still passes.
+ */
+const RAW_CIPHERTEXT_WRITES: ReadonlyArray<{
+  key: string;
+  file: string;
+  statement: RegExp;
+}> = [
+  {
+    key: "DataBackup.data",
+    file: "src/lib/export/store-backup-blob.ts",
+    statement:
+      /UPDATE\s+data_backups\s+SET\s+data\s*=\s*\(\s*SELECT\s+string_agg/,
+  },
+];
+
 function sourceFiles(dir: string, out: string[] = []): string[] {
   for (const entry of readdirSync(dir)) {
     const p = join(dir, entry);
@@ -388,6 +411,12 @@ function scanCiphertextWrites(): Map<string, Set<string>> {
       `\\b(?:prisma|tx|db)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(?:${WRITE_METHODS})\\s*\\(`,
       "g",
     );
+    for (const raw of RAW_CIPHERTEXT_WRITES) {
+      if (rel !== raw.file) continue;
+      const hit = raw.statement.exec(src);
+      if (hit) record(...(raw.key.split(".") as [string, string]), hit.index);
+    }
+
     for (const m of src.matchAll(writes)) {
       const model = m[1][0].toUpperCase() + m[1].slice(1);
       if (!schema.models.has(model)) continue;
@@ -425,6 +454,17 @@ describe("ciphertext columns derived from the code", () => {
     }
   });
 
+  it("still finds every raw-SQL ciphertext write it was told about", () => {
+    for (const raw of RAW_CIPHERTEXT_WRITES) {
+      expect(
+        [...(written.get(raw.key) ?? [])].some((at) =>
+          at.startsWith(`${raw.file}:`),
+        ),
+        `${raw.key}: no matching statement left in ${raw.file}`,
+      ).toBe(true);
+    }
+  });
+
   it("finds ciphertext writes, including the non-suffixed columns", () => {
     // Anchored non-zero match count. An empty or collapsed result set is the
     // failure mode this whole file exists to make impossible, so name the

src/lib/crypto/__tests__/encryption-corpus.test.ts

@@ -214,4 +214,45 @@ describe("encryption-corpus scan + rotate", () => {
     expect(col.legacy).toBe(1);
     expect(col.byKeyId[LEGACY_BUCKET]).toBe(1);
   });
+
+  it("reads every column in bounded pages and counts only rows that hold ciphertext (#1031)", async () => {
+    // A notes column is read for every row of its table, 1.25 million on a
+    // large account, and was read in one findMany. 12 000 measurements, one
+    // in ten with a note written under the old key.
+    const measurements: Row[] = Array.from({ length: 12_000 }, (_, i) => ({
+      id: `m-${String(i).padStart(5, "0")}`,
+      notesEncrypted:
+        i % 10 === 0
+          ? new Uint8Array(Buffer.from(activeUnder("v1", `note ${i}`), "utf8"))
+          : null,
+    }));
+    const { client } = makeClient({ Measurement: measurements });
+    const takes: Array<number | undefined> = [];
+    const measurement = client.measurement!;
+    const findMany = measurement.findMany;
+    measurement.findMany = (args) => {
+      takes.push(args.take);
+      return findMany(args);
+    };
+
+    const scan = await scanCorpus(client);
+    const col = scan.columns.find(
+      (c) => c.model === "Measurement" && c.field === "notesEncrypted",
+    )!;
+    expect(col.total).toBe(1_200);
+    expect(col.byKeyId.v1).toBe(1_200);
+
+    const rotation = await rotateCorpus(client);
+    const result = rotation.results.find(
+      (r) => r.model === "Measurement" && r.field === "notesEncrypted",
+    )!;
+    expect(result.scanned).toBe(1_200);
+    expect(result.rotated).toBe(1_200);
+
+    expect(takes.length).toBeGreaterThan(2);
+    for (const take of takes) {
+      expect(take).toBeDefined();
+      expect(take!).toBeLessThanOrEqual(5_000);
+    }
+  });
 });

src/lib/crypto/encryption-corpus.ts

@@ -22,7 +22,7 @@
  *    `ENCRYPTION_KEYS` / `ENCRYPTION_ACTIVE_KEY_ID`. The operator's env key map
  *    is the only place keys live; a key drop stays a deliberate env + redeploy
  *    act, never a button.
- *  - IDEMPOTENT. `shouldRotate()` skips rows already on the active key, so a
+ *  - IDEMPOTENT. The walk skips rows already on the active key, so a
  *    second pass (or two racing workers) re-encrypts zero rows.
  *  - FAIL-CLOSED. A row written under a key id that is no longer configured
  *    throws on decrypt (counted as an error, the row is left untouched) rather
@@ -54,6 +54,15 @@ export const LEGACY_BUCKET = "legacy";
  */
 export const BLOB_ROTATION_BATCH_SIZE = 25;
 
+/**
+ * Batch size for every other column. These rows are short strings, but a
+ * column can have one per measurement: `Measurement.notesEncrypted` is read
+ * for every row of the table, 1.25 million on a large account (#1031). The
+ * walk used to read such a column in one `findMany`; it is id-cursor paged
+ * like the blob columns, only with larger pages.
+ */
+export const ROW_ROTATION_BATCH_SIZE = 5_000;
+
 /** Minimal Prisma delegate shape this module needs. */
 interface ColumnDelegate {
   findMany: (args: {
@@ -114,10 +123,6 @@ function fromCiphertext(
   return next;
 }
 
-function shouldRotate(ciphertext: string): boolean {
-  return extractKeyId(ciphertext) !== getActiveKeyId();
-}
-
 // ─── Codec-dispatched blob columns (document vault) ─────────────────────────
 
 /**
@@ -152,15 +157,16 @@ function reencryptBlob(value: Uint8Array, codec: string): Uint8Array {
 }
 
 /**
- * Walk a blob column in bounded id-cursor batches, invoking `onRow` per
- * non-empty row. At most `BLOB_ROTATION_BATCH_SIZE` rows are in memory per
- * step; an interrupted run resumes safely on re-invocation because processing
- * is idempotent (already-active rows are skipped by the callers).
+ * Walk a column in bounded id-cursor batches, invoking `onRow` per non-empty
+ * row. At most one batch is in memory per step (`BLOB_ROTATION_BATCH_SIZE`
+ * rows of a blob column, `ROW_ROTATION_BATCH_SIZE` of any other); an
+ * interrupted run resumes safely on re-invocation because processing is
+ * idempotent (already-active rows are skipped by the callers).
  *
  * `codec` is the row's own codec label for a codec-dispatched column and null
  * for a plain `batched` one — a backup blob has one layout, just a large one.
  */
-async function walkBlobColumn(
+async function walkColumn(
   delegate: ColumnDelegate,
   col: EncryptedColumn,
   onRow: (row: {
@@ -171,6 +177,9 @@ async function walkBlobColumn(
 ): Promise<void> {
   const codecField = col.codecField;
   const pk = pkField(col);
+  const batchSize = isBlobColumn(col)
+    ? BLOB_ROTATION_BATCH_SIZE
+    : ROW_ROTATION_BATCH_SIZE;
   let cursor: string | null = null;
   for (;;) {
     const rows = await delegate.findMany({
@@ -180,7 +189,7 @@ async function walkBlobColumn(
         ...(codecField ? { [codecField]: true } : {}),
       },
       orderBy: { [pk]: "asc" },
-      take: BLOB_ROTATION_BATCH_SIZE,
+      take: batchSize,
       ...(cursor ? { cursor: { [pk]: cursor }, skip: 1 } : {}),
     });
     if (rows.length === 0) break;
@@ -196,7 +205,7 @@ async function walkBlobColumn(
       });
     }
     cursor = rows[rows.length - 1]![pk] as string;
-    if (rows.length < BLOB_ROTATION_BATCH_SIZE) break;
+    if (rows.length < batchSize) break;
   }
 }
 
@@ -205,8 +214,8 @@ function pkField(col: EncryptedColumn): string {
   return col.pkField ?? "id";
 }
 
-/** True when the column's rows must be walked in bounded batches. */
-function isBatched(col: EncryptedColumn): boolean {
+/** True when the column's rows are blobs, walked in the small batches. */
+function isBlobColumn(col: EncryptedColumn): boolean {
   return Boolean(col.codecField ?? col.batched);
 }
 
@@ -242,25 +251,12 @@ export async function scanColumn(
   const byKeyId: Record<string, number> = {};
   let total = 0;
 
-  if (isBatched(col)) {
-    // Blob column: bounded batches, per-row codec where the column has one.
-    await walkBlobColumn(delegate, col, ({ value, codec }) => {
-      total += 1;
-      const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET;
-      byKeyId[id] = (byKeyId[id] ?? 0) + 1;
-    });
-  } else {
-    const rows = await delegate.findMany({
-      select: { [pkField(col)]: true, [col.field]: true },
-    });
-    for (const row of rows) {
-      const ciphertext = toCiphertext(row[col.field], col.kind);
-      if (ciphertext == null) continue;
-      total += 1;
-      const id = extractKeyId(ciphertext) ?? LEGACY_BUCKET;
-      byKeyId[id] = (byKeyId[id] ?? 0) + 1;
-    }
-  }
+  // Bounded batches for every column, per-row codec where the column has one.
+  await walkColumn(delegate, col, ({ value, codec }) => {
+    total += 1;
+    const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET;
+    byKeyId[id] = (byKeyId[id] ?? 0) + 1;
+  });
   return {
     model: col.model,
     field: col.field,
@@ -356,57 +352,33 @@ export async function rotateColumn(
     }
   };
 
-  if (isBatched(col)) {
-    // Blob column: bounded id-cursor batches (never an unbounded findMany),
-    // re-encrypted under each row's OWN codec where it has one. Idempotent —
-    // rows already on the active key are skipped, so an interrupted run
-    // resumes cleanly on the next invocation.
-    await walkBlobColumn(delegate, col, async ({ id, value, codec }) => {
-      result.scanned += 1;
-      if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return;
-      try {
-        // The plaintext is never inspected, only re-sealed, so every envelope
-        // a stored value can legitimately carry survives rotation untouched.
-        const next =
-          codec !== null
-            ? reencryptBlob(value as Uint8Array, codec)
-            : fromCiphertext(
-                encrypt(decrypt(toCiphertext(value, col.kind)!)),
-                col.kind,
-              );
-        await delegate.update({
-          where: { [pkField(col)]: id },
-          data: { [col.field]: next },
-        });
-        result.rotated += 1;
-      } catch {
-        await onUnreadable(id);
-      }
-    });
-    return result;
-  }
-
-  const pk = pkField(col);
-  const rows = await delegate.findMany({
-    select: { [pk]: true, [col.field]: true },
-  });
-  result.scanned = rows.length;
-  for (const row of rows) {
-    const ciphertext = toCiphertext(row[col.field], col.kind);
-    if (ciphertext == null || !shouldRotate(ciphertext)) continue;
-    const id = row[pk] as string;
+  // Bounded id-cursor batches for every column (never an unbounded
+  // findMany), re-encrypted under each row's OWN codec where it has one.
+  // Idempotent — rows already on the active key are skipped, so an
+  // interrupted run resumes cleanly on the next invocation. `scanned` counts
+  // the rows that hold ciphertext, whatever the column.
+  await walkColumn(delegate, col, async ({ id, value, codec }) => {
+    result.scanned += 1;
+    if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return;
     try {
-      // ACTIVE-KEY-ONLY: encrypt() always writes the active key id.
-      const reencrypted = encrypt(decrypt(ciphertext));
+      // The plaintext is never inspected, only re-sealed, so every envelope
+      // a stored value can legitimately carry survives rotation untouched.
+      const next =
+        codec !== null
+          ? reencryptBlob(value as Uint8Array, codec)
+          : fromCiphertext(
+              encrypt(decrypt(toCiphertext(value, col.kind)!)),
+              col.kind,
+            );
       await delegate.update({
-        where: { [pk]: id },
-        data: { [col.field]: fromCiphertext(reencrypted, col.kind) },
+        where: { [pkField(col)]: id },
+        data: { [col.field]: next },
       });
       result.rotated += 1;
     } catch {
       await onUnreadable(id);
     }
-  }
+  });
   return result;
 }
 

src/lib/export/__tests__/backup-blob.test.ts

@@ -17,6 +17,7 @@
 process.env.ENCRYPTION_KEY ??=
   "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
 
+import { randomBytes } from "node:crypto";
 import v8 from "node:v8";
 
 import { describe, expect, it } from "vitest";
@@ -26,6 +27,7 @@ import {
   BackupBlobTooLargeError,
   defaultBackupBlobLimit,
   packBackupBlob,
+  packBackupBlobInto,
   packBackupBlobStreaming,
   unpackBackupBlob,
 } from "../backup-blob";
@@ -213,3 +215,66 @@ describe("backup blob envelope", () => {
     expect(limit).toBeGreaterThan(16 * 1024 * 1024);
   });
 });
+
+/**
+ * Issue #1031: the weekly and manual backup kept the stored copy as one string
+ * before writing it, and at 1.25 million measurements that string and the
+ * driver's copies of it added about 640 MB to the process. `packBackupBlobInto`
+ * hands the stored copy on in bounded pieces instead, as it is produced.
+ */
+describe("piecewise envelope", () => {
+  /** JSON that gzip cannot shrink much, so the stored copy is several MB. */
+  function incompressibleJson(bytes: number): string {
+    return JSON.stringify({
+      noise: randomBytes(Math.ceil((bytes * 3) / 4)).toString("base64"),
+    });
+  }
+
+  it("delivers pieces that concatenate to a blob reading back the exact JSON", async () => {
+    const json = sampleJson(3_000);
+    const pieces: string[] = [];
+    await packBackupBlobInto(
+      (piece) => {
+        pieces.push(piece);
+      },
+      async (write) => {
+        for (let at = 0; at < json.length; at += 4_096) {
+          await write(json.slice(at, at + 4_096));
+        }
+      },
+    );
+    expect(unpackBackupBlob(pieces.join(""))).toBe(json);
+  });
+
+  it("hands pieces on while the producer is still writing, each one bounded", async () => {
+    const json = incompressibleJson(12 * 1024 * 1024);
+    const pieces: number[] = [];
+    let producing = false;
+    let piecesWhileProducing = 0;
+    const chunks: string[] = [];
+
+    await packBackupBlobInto(
+      (piece) => {
+        pieces.push(piece.length);
+        chunks.push(piece);
+        if (producing) piecesWhileProducing += 1;
+      },
+      async (write) => {
+        producing = true;
+        for (let at = 0; at < json.length; at += 64 * 1024) {
+          await write(json.slice(at, at + 64 * 1024));
+        }
+        producing = false;
+      },
+      { maxBytes: Number.MAX_SAFE_INTEGER },
+    );
+
+    expect(pieces.length).toBeGreaterThan(2);
+    expect(piecesWhileProducing).toBeGreaterThan(1);
+    // The flush threshold is 4 MB; one gzip chunk's worth of slack on top.
+    for (const length of pieces) {
+      expect(length).toBeLessThan(4 * 1024 * 1024 + 256 * 1024);
+    }
+    expect(unpackBackupBlob(chunks.join(""))).toBe(json);
+  });
+});

src/lib/export/__tests__/backup-json-scan.test.ts

@@ -0,0 +1,185 @@
+/**
+ * `scanBackupJson` must read a backup exactly as `JSON.parse` of the whole
+ * document would, with the streamed members handed out one element at a time
+ * (issue #1031: a 662 MB backup cannot exist as one string). These tests hold
+ * it to that over awkward values and arbitrary chunk boundaries, including
+ * boundaries inside strings, escapes and multi-byte characters.
+ */
+import { describe, expect, it } from "vitest";
+
+import { BackupJsonError, scanBackupJson } from "../backup-json-scan";
+
+async function* chunked(text: string, sizes: number[]) {
+  const bytes = Buffer.from(text, "utf8");
+  let at = 0;
+  let i = 0;
+  while (at < bytes.length) {
+    const size = sizes[i++ % sizes.length]!;
+    yield bytes.subarray(at, at + size);
+    at += size;
+  }
+}
+
+function rng(seed: number) {
+  let x = seed;
+  return () => {
+    x = (x * 1103515245 + 12345) % 2 ** 31;
+    return x / 2 ** 31;
+  };
+}
+
+const AWKWARD = [
+  'quote " inside',
+  "back\\slash and \\n",
+  "brackets ] } [ { , : in a string",
+  "umlaut ä ö ü and emoji 🫀",
+  "",
+  "\u0000\u001f control",
+];
+
+function randomValue(rand: () => number, depth: number): unknown {
+  const r = rand();
+  if (depth > 2 || r < 0.35) {
+    const p = rand();
+    if (p < 0.2) return null;
+    if (p < 0.3) return rand() < 0.5;
+    if (p < 0.55) return Math.round(rand() * 1e6) / 100 - 500;
+    return (
+      AWKWARD[Math.floor(rand() * AWKWARD.length)]! + Math.floor(rand() * 9)
+    );
+  }
+  if (r < 0.65) {
+    return Array.from({ length: Math.floor(rand() * 4) }, () =>
+      randomValue(rand, depth + 1),
+    );
+  }
+  const obj: Record<string, unknown> = {};
+  for (let i = 0; i < Math.floor(rand() * 4); i++) {
+    obj[`${AWKWARD[i % AWKWARD.length]}-${i}`] = randomValue(rand, depth + 1);
+  }
+  return obj;
+}
+
+function randomDocument(seed: number) {
+  const rand = rng(seed);
+  return {
+    schemaVersion: "2",
+    userId: "user-ä",
+    measurements: Array.from({ length: Math.floor(rand() * 30) }, (_, i) =>
+      i % 5 === 4
+        ? randomValue(rand, 1)
+        : { id: `m${i}`, notes: AWKWARD[i % 6] },
+    ),
+    medications: [randomValue(rand, 0), randomValue(rand, 0)],
+    moodEntries: [],
+    count: 42,
+    flag: false,
+    nothing: null,
+    text: AWKWARD[seed % AWKWARD.length],
+    nested: randomValue(rand, 0),
+  };
+}
+
+async function scan(text: string, sizes: number[], streamKeys: string[]) {
+  const elements: Record<string, unknown[]> = {};
+  const result = await scanBackupJson(chunked(text, sizes), {
+    streamKeys: new Set(streamKeys),
+    onElement: (key, element, index) => {
+      (elements[key] ??= [])[index] = element;
+    },
+  });
+  return { ...result, elements };
+}
+
+describe("scanBackupJson", () => {
+  it("equals JSON.parse over random documents, formats and chunk splits", async () => {
+    for (let seed = 1; seed <= 200; seed++) {
+      const doc = randomDocument(seed);
+      const text =
+        seed % 3 === 0
+          ? JSON.stringify(doc, null, seed % 2 ? 2 : "\t")
+          : JSON.stringify(doc);
+      const sizes = [1 + (seed % 7), 2, 13, 1, 64].slice(seed % 3);
+      const { document, streamedCounts, elements } = await scan(text, sizes, [
+        "measurements",
+        "moodEntries",
+      ]);
+      const expected = JSON.parse(text) as Record<string, unknown>;
+
+      expect(elements.measurements ?? [], `seed ${seed}`).toEqual(
+        expected.measurements,
+      );
+      expect(streamedCounts).toEqual({
+        measurements: (expected.measurements as unknown[]).length,
+        moodEntries: 0,
+      });
+      expect(document, `seed ${seed}`).toEqual({
+        ...expected,
+        measurements: [],
+        moodEntries: [],
+      });
+    }
+  });
+
+  it("hands out streamed elements before it has read the rest of the document", async () => {
+    const text = JSON.stringify({
+      a: 1,
+      measurements: [{ id: "x" }, { id: "y" }],
+      tail: "z".repeat(100),
+    });
+    const bytes = Buffer.from(text);
+    let delivered = 0;
+    const seenAt: number[] = [];
+    async function* source() {
+      for (let i = 0; i < bytes.length; i += 4) {
+        delivered = i + 4;
+        yield bytes.subarray(i, i + 4);
+      }
+    }
+    await scanBackupJson(source(), {
+      streamKeys: new Set(["measurements"]),
+      onElement: () => {
+        seenAt.push(delivered);
+      },
+    });
+    expect(seenAt).toHaveLength(2);
+    expect(seenAt[1]!).toBeLessThan(bytes.length - 100);
+  });
+
+  it("skips a member it was told to skip without keeping it", async () => {
+    const { document } = await scanBackupJson(
+      chunked('{"a":[1,{"b":"]"}],"big":{"x":[1,2,3]},"c":"d"}', [3]),
+      { skipKeys: new Set(["big"]) },
+    );
+    expect(document).toEqual({ a: [1, { b: "]" }], c: "d" });
+  });
+
+  it("refuses a truncated or malformed document", async () => {
+    for (const bad of [
+      '{"a":1',
+      '{"a":[1,2}',
+      "[1,2]",
+      '{"measurements":{"x":1}}',
+      '{"a":1}{',
+      '{"measurements":[{"id":1} {"id":2}]}',
+    ]) {
+      await expect(
+        scanBackupJson(chunked(bad, [2]), {
+          streamKeys: new Set(["measurements"]),
+        }),
+        bad,
+      ).rejects.toBeInstanceOf(BackupJsonError);
+    }
+  });
+
+  it("propagates a failure from the element consumer", async () => {
+    await expect(
+      scanBackupJson(chunked('{"measurements":[1,2,3]}', [5]), {
+        streamKeys: new Set(["measurements"]),
+        onElement: (_key, _element, index) => {
+          if (index === 1) throw new Error("consumer failed");
+        },
+      }),
+    ).rejects.toThrow("consumer failed");
+  });
+});

src/lib/export/__tests__/passphrase-archive.test.ts

@@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest";
 import { Buffer } from "node:buffer";
 import {
   encryptArchive,
+  encryptArchiveToFile,
   decryptArchive,
   parseArchiveHeader,
   EXPORT_ARGON2_PARAMS,
@@ -78,3 +79,45 @@ describe("passphrase-encrypted export archive (HLX1)", () => {
     expect(await decryptArchive(archive, PASSPHRASE)).toBe(PAYLOAD);
   });
 });
+
+/**
+ * #1031: the encrypted export is sealed as it is produced and spooled to a
+ * file, because holding it did not fit a 1 GB container for a large account.
+ * The format must not change: an archive written this way opens with the
+ * same `decryptArchive` every existing archive opens with.
+ */
+describe("encryptArchiveToFile", () => {
+  it("writes an archive decryptArchive opens, byte for byte the same document", async () => {
+    const { mkdtemp, readFile, rm } = await import("node:fs/promises");
+    const { tmpdir } = await import("node:os");
+    const { join } = await import("node:path");
+    const dir = await mkdtemp(join(tmpdir(), "hlx-spool-"));
+    try {
+      const pieces = Array.from({ length: 500 }, (_, i) =>
+        JSON.stringify({ i, note: "ümlaut 🫀 ".repeat(i % 7) }),
+      );
+      const bodyPath = join(dir, "body");
+      const spooled = await encryptArchiveToFile(
+        async (write) => {
+          await write("[");
+          for (let i = 0; i < pieces.length; i++) {
+            await write(i === 0 ? pieces[i]! : `,${pieces[i]}`);
+          }
+          await write(Buffer.from("]"));
+        },
+        PASSPHRASE,
+        bodyPath,
+      );
+      const archive = Buffer.concat([spooled.prefix, await readFile(bodyPath)]);
+      expect(archive.byteLength).toBe(spooled.byteLength);
+      expect(await decryptArchive(archive, PASSPHRASE)).toBe(
+        `[${pieces.join(",")}]`,
+      );
+      await expect(
+        decryptArchive(archive, "wrong passphrase!!"),
+      ).rejects.toThrow();
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

src/lib/export/backup-blob.ts

@@ -37,7 +37,8 @@
  */
 import { Buffer } from "node:buffer";
 import v8 from "node:v8";
-import { createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { Readable } from "node:stream";
+import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
 
 import {
   createStreamEncryptor,
@@ -125,7 +126,7 @@ export class BackupBlobTooLargeError extends Error {
  * counts) is the caller's business, not the envelope's.
  */
 export type BackupJsonProducer = (
-  write: (chunk: string) => Promise<void>,
+  write: (chunk: string | Buffer) => Promise<void>,
 ) => Promise<unknown>;
 
 export interface PackBackupBlobOptions {
@@ -141,26 +142,59 @@ export interface PackBackupBlobOptions {
  * Serialised backup JSON, produced in pieces → the stored string.
  *
  * The pipeline is JSON piece → gzip → AES-256-GCM → base64, with nothing
- * buffered end to end but the base64 answer. `producer` decides how big its
- * pieces are; the gzip stream applies backpressure through the promise this
- * hands back, so a fast producer cannot outrun the compressor and pile up
- * chunks in the stream's internal queue.
- *
- * The one copy that cannot be avoided is the answer itself. `data_backups.data`
- * is a single `text` column, so the row has to be one value, and one value has
- * to exist as one string before the driver can bind it. That copy is therefore
- * also the only thing here that grows without bound as a record grows, which
- * is why the size check lives on it: `maxBytes` counts the ciphertext this
- * call has actually produced and stops on the account whose backup does not
- * fit, rather than reading a heap gauge every account shares.
+ * buffered end to end but the base64 answer. Kept for callers that need the
+ * answer as one value; the weekly and manual backup no longer do, they hand
+ * the pieces to Postgres as they come (`packBackupBlobInto` below, via
+ * `storeBackupBlob`).
  */
 export async function packBackupBlobStreaming(
   producer: BackupJsonProducer,
   options: PackBackupBlobOptions = {},
 ): Promise<string> {
+  const pieces: string[] = [];
+  await packBackupBlobInto(
+    (piece) => {
+      pieces.push(piece);
+    },
+    producer,
+    options,
+  );
+  return pieces.join("");
+}
+
+/** Receives the stored string a piece at a time, in order. Awaited. */
+export type BackupBlobSink = (piece: string) => void | Promise<void>;
+
+/**
+ * How much base64 to gather before handing it to the sink. Large enough that
+ * a sink writing to the database makes a few dozen round trips for a large
+ * record rather than thousands, small enough to be irrelevant to the heap.
+ */
+const SINK_FLUSH_BYTES = 4 * 1024 * 1024;
+
+/**
+ * Serialised backup JSON, produced in pieces → the stored string, delivered
+ * in pieces.
+ *
+ * The concatenation of every piece `sink` receives is exactly the string
+ * `packBackupBlobStreaming` returns. `producer` decides how big its JSON
+ * pieces are; the gzip stream applies backpressure through the promise this
+ * hands back, so a fast producer cannot outrun the compressor, and a slow
+ * sink holds the producer back the same way, so neither side piles up.
+ *
+ * `maxBytes` still counts the whole answer. Nothing here holds it any more,
+ * but every reader of a stored copy (restore, download, summary) opens it as
+ * one value, so the limit is what keeps a copy restorable on this host.
+ */
+export async function packBackupBlobInto(
+  sink: BackupBlobSink,
+  producer: BackupJsonProducer,
+  options: PackBackupBlobOptions = {},
+): Promise<void> {
   const limitBytes = options.maxBytes ?? defaultBackupBlobLimit();
   const encryptor = createStreamEncryptor();
-  const pieces: string[] = [encryptor.header];
+  let pending: string[] = [encryptor.header];
+  let pendingBytes = encryptor.header.length;
   let heldBytes = encryptor.header.length;
   const gzip = createGzip();
 
@@ -169,12 +203,13 @@ export async function packBackupBlobStreaming(
     try {
       const piece = encryptor.update(chunk);
       if (piece === "") return;
-      // Base64 is ASCII, so a character counted here is a byte held here.
+      // Base64 is ASCII, so a character counted here is a byte written here.
       heldBytes += piece.length;
       if (heldBytes > limitBytes) {
         throw new BackupBlobTooLargeError(heldBytes, limitBytes);
       }
-      pieces.push(piece);
+      pending.push(piece);
+      pendingBytes += piece.length;
     } catch (err) {
       failure ??= err;
       gzip.destroy(err as Error);
@@ -192,8 +227,19 @@ export async function packBackupBlobStreaming(
   // rejection; `await finished` below still sees the rejection.
   finished.catch(() => {});
 
-  const write = async (chunk: string): Promise<void> => {
+  const flush = async (force: boolean): Promise<void> => {
+    if (pendingBytes === 0 || (!force && pendingBytes < SINK_FLUSH_BYTES)) {
+      return;
+    }
+    const piece = pending.join("");
+    pending = [];
+    pendingBytes = 0;
+    await sink(piece);
+  };
+
+  const write = async (chunk: string | Buffer): Promise<void> => {
     if (failure) throw failure;
+    await flush(false);
     if (gzip.write(chunk, "utf8")) return;
     await new Promise<void>((resolve, reject) => {
       const onDrain = () => {
@@ -219,8 +265,10 @@ export async function packBackupBlobStreaming(
   await finished;
   if (failure) throw failure;
 
-  pieces.push(encryptor.final());
-  return pieces.join("");
+  const tail = encryptor.final();
+  pending.push(tail);
+  pendingBytes += tail.length;
+  await flush(true);
 }
 
 /**
@@ -266,3 +314,34 @@ export function unpackBackupBlob(stored: string): string {
     Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"),
   ).toString("utf8");
 }
+
+/**
+ * A stored `DataBackup.data` string → a source of its JSON as byte chunks,
+ * which can be opened as many times as the caller needs to read it.
+ *
+ * `unpackBackupBlob` returns the JSON as one string, and a string is what a
+ * large record cannot be: the disaster-recovery JSON of an account with 1.25
+ * million measurements is 662 MB, past the 536 870 888 characters V8 allows
+ * in any string, so that call threw on every such backup (#1031). Here the
+ * authentication happens first and whole, on the compressed ciphertext, which
+ * stays small (48 MB for that record): no plaintext byte is released before
+ * the tag has verified. Only the decompression is streamed, on every open.
+ *
+ * Fail-closed exactly like `unpackBackupBlob`: a bad key or a tag that does
+ * not verify throws here, before a source exists; a truncated gzip member
+ * errors the stream.
+ */
+export function openBackupBlob(stored: string): () => AsyncIterable<Buffer> {
+  if (isStreamCiphertext(stored)) {
+    const gz = decryptStream(stored);
+    return () => Readable.from([gz]).pipe(createGunzip());
+  }
+  const plaintext = decrypt(stored);
+  if (!plaintext.startsWith(GZIP_MARKER)) {
+    // A pre-envelope copy was written from one string, so it is one.
+    const bytes = Buffer.from(plaintext, "utf8");
+    return () => Readable.from([bytes]);
+  }
+  const gz = Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64");
+  return () => Readable.from([gz]).pipe(createGunzip());
+}

src/lib/export/backup-json-scan.ts

@@ -0,0 +1,336 @@
+/**
+ * Read a backup's JSON as a stream, without ever holding the document.
+ *
+ * Why. A backup is one JSON object, and every reader used to turn it into
+ * one string and `JSON.parse` it. For an account of 1.25 million measurements
+ * (issue #1031) the disaster-recovery JSON is 662 MB. V8 cannot hold a string
+ * longer than 536 870 888 characters at all, so `unpackBackupBlob` threw
+ * before a byte was parsed: the weekly backup of that account could not be
+ * restored, downloaded or even summarised, and the restore route reported the
+ * copy as undecryptable. A portable export of the same account fits under
+ * the limit but not beside the rest of a 1 GB container.
+ *
+ * What this does. It scans the bytes once, tracking only string and nesting
+ * state. Every top-level member is parsed on its own with `JSON.parse` and
+ * kept, except the members named in `streamKeys`: those must be arrays, and
+ * each element is parsed and handed to `onElement` instead, so the caller
+ * sees them one at a time and the kept object holds an empty array in their
+ * place. A backup's bulk tables (measurements, intake events, mood entries)
+ * are the only members that grow with a record, so everything else is small.
+ *
+ * Structural characters are ASCII and UTF-8 never uses an ASCII byte inside a
+ * multi-byte sequence, so scanning bytes is exact. A value is decoded only
+ * once it is complete, which is where a character split across two chunks is
+ * put back together.
+ *
+ * The output is what `JSON.parse` of the whole document would give, with the
+ * streamed arrays emptied: the tests hold it to that over arbitrary splits.
+ */
+
+/** Characters the scanner cares about, as bytes. */
+const QUOTE = 0x22;
+const BACKSLASH = 0x5c;
+const OPEN_BRACE = 0x7b;
+const CLOSE_BRACE = 0x7d;
+const OPEN_BRACKET = 0x5b;
+const CLOSE_BRACKET = 0x5d;
+const COMMA = 0x2c;
+const COLON = 0x3a;
+
+function isWhitespace(byte: number): boolean {
+  return byte === 0x20 || byte === 0x0a || byte === 0x0d || byte === 0x09;
+}
+
+export interface ScanBackupJsonOptions {
+  /** Top-level members whose array elements are streamed, not kept. */
+  streamKeys?: ReadonlySet<string>;
+  /**
+   * Receives each element of a streamed member, parsed, with its index.
+   * Awaited, so a consumer that writes to a database applies backpressure.
+   */
+  onElement?: (key: string, element: unknown, index: number) => unknown;
+  /**
+   * Members to skip entirely: scanned past, never parsed or kept. For a
+   * reader that only needs a few sections of the document.
+   */
+  skipKeys?: ReadonlySet<string>;
+}
+
+export interface ScanBackupJsonResult {
+  /** The document, with streamed members as empty arrays and skipped ones absent. */
+  document: Record<string, unknown>;
+  /** How many elements each streamed member held. */
+  streamedCounts: Record<string, number>;
+}
+
+/** Thrown for input that is not a JSON object the scanner can walk. */
+export class BackupJsonError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "BackupJsonError";
+  }
+}
+
+/** Collects the bytes of one value across chunks. */
+class Collector {
+  private parts: Buffer[] = [];
+  private bytes = 0;
+  add(chunk: Buffer, from: number, to: number): void {
+    if (to > from) {
+      this.parts.push(chunk.subarray(from, to));
+      this.bytes += to - from;
+    }
+  }
+  take(): string {
+    const text =
+      this.parts.length === 1
+        ? this.parts[0]!.toString("utf8")
+        : Buffer.concat(this.parts, this.bytes).toString("utf8");
+    this.parts = [];
+    this.bytes = 0;
+    return text;
+  }
+}
+
+type Phase =
+  | "start" // before the opening brace
+  | "key" // expecting a key or the closing brace
+  | "colon"
+  | "value" // expecting the start of a member value
+  | "in-value" // inside a kept or skipped member value
+  | "array-start" // a streamed member: expecting its opening bracket
+  | "element" // inside a streamed array, expecting an element or `]`
+  | "in-element" // inside one element
+  | "after" // after a member, expecting `,` or `}`
+  | "done";
+
+/**
+ * Scan a backup document delivered as byte chunks. Resolves to the document
+ * (see the module comment) once the closing brace has been read; rejects with
+ * `BackupJsonError` on malformed input or with whatever `onElement` threw.
+ */
+export async function scanBackupJson(
+  chunks: AsyncIterable<Uint8Array>,
+  options: ScanBackupJsonOptions = {},
+): Promise<ScanBackupJsonResult> {
+  const streamKeys = options.streamKeys ?? new Set<string>();
+  const skipKeys = options.skipKeys ?? new Set<string>();
+  const document: Record<string, unknown> = {};
+  const streamedCounts: Record<string, number> = {};
+
+  let phase: Phase = "start";
+  let inString = false;
+  let escaped = false;
+  // Nesting depth inside the current value or element (0 = not inside one).
+  let depth = 0;
+  let key = "";
+  let skipping = false;
+  // Inside a key string.
+  let keyOpen = false;
+  // Inside a streamed array, after an element: the next byte must be `,` or `]`.
+  let elementNeedsComma = false;
+  const keyBytes = new Collector();
+  const value = new Collector();
+
+  const finishValue = () => {
+    const text = value.take();
+    if (skipping) return;
+    try {
+      document[key] = JSON.parse(text);
+    } catch {
+      throw new BackupJsonError(`Backup member '${key}' is not valid JSON`);
+    }
+  };
+  const finishElement = async () => {
+    const text = value.take();
+    let element: unknown;
+    try {
+      element = JSON.parse(text);
+    } catch {
+      throw new BackupJsonError(
+        `Element ${streamedCounts[key]} of backup member '${key}' is not valid JSON`,
+      );
+    }
+    const index = streamedCounts[key]!;
+    streamedCounts[key] = index + 1;
+    await options.onElement?.(key, element, index);
+  };
+
+  for await (const raw of chunks) {
+    const chunk = Buffer.isBuffer(raw)
+      ? raw
+      : Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength);
+    // Where the value or key being collected started in this chunk.
+    let mark = 0;
+    for (let i = 0; i < chunk.length; i++) {
+      const byte = chunk[i]!;
+
+      if (phase === "in-value" || phase === "in-element") {
+        if (inString) {
+          if (escaped) escaped = false;
+          else if (byte === BACKSLASH) escaped = true;
+          else if (byte === QUOTE) inString = false;
+          continue;
+        }
+        if (byte === QUOTE) {
+          inString = true;
+          continue;
+        }
+        if (byte === OPEN_BRACE || byte === OPEN_BRACKET) {
+          depth += 1;
+          continue;
+        }
+        if (byte === CLOSE_BRACE || byte === CLOSE_BRACKET) {
+          if (depth > 0) {
+            depth -= 1;
+            if (depth > 0) continue;
+            // The value's own closing bracket: it belongs to the value.
+            value.add(chunk, mark, i + 1);
+            if (phase === "in-value") {
+              finishValue();
+              phase = "after";
+            } else {
+              await finishElement();
+              phase = "element";
+              elementNeedsComma = true;
+            }
+            continue;
+          }
+          // A primitive value ended by the enclosing bracket.
+          value.add(chunk, mark, i);
+          if (phase === "in-value") {
+            if (byte !== CLOSE_BRACE) {
+              throw new BackupJsonError("Unbalanced bracket in backup");
+            }
+            finishValue();
+            phase = "done";
+          } else {
+            if (byte !== CLOSE_BRACKET) {
+              throw new BackupJsonError("Unbalanced bracket in backup");
+            }
+            await finishElement();
+            phase = "after";
+          }
+          continue;
+        }
+        if (depth === 0 && (byte === COMMA || isWhitespace(byte))) {
+          // A primitive value ended.
+          value.add(chunk, mark, i);
+          if (phase === "in-value") {
+            finishValue();
+            phase = byte === COMMA ? "key" : "after";
+          } else {
+            await finishElement();
+            phase = "element";
+            elementNeedsComma = byte !== COMMA;
+          }
+          continue;
+        }
+        continue;
+      }
+
+      if (phase === "key" && keyOpen) {
+        if (escaped) escaped = false;
+        else if (byte === BACKSLASH) escaped = true;
+        else if (byte === QUOTE) {
+          keyBytes.add(chunk, mark, i + 1);
+          key = JSON.parse(keyBytes.take()) as string;
+          keyOpen = false;
+          phase = "colon";
+        }
+        continue;
+      }
+
+      if (isWhitespace(byte)) continue;
+
+      switch (phase) {
+        case "start":
+          if (byte !== OPEN_BRACE) {
+            throw new BackupJsonError("Backup is not a JSON object");
+          }
+          phase = "key";
+          break;
+        case "key":
+          if (byte === CLOSE_BRACE) {
+            phase = "done";
+          } else if (byte === QUOTE) {
+            keyOpen = true;
+            escaped = false;
+            mark = i;
+          } else {
+            throw new BackupJsonError("Malformed backup: expected a key");
+          }
+          break;
+        case "colon":
+          if (byte !== COLON) {
+            throw new BackupJsonError("Malformed backup: expected ':'");
+          }
+          phase = streamKeys.has(key) ? "array-start" : "value";
+          skipping = skipKeys.has(key);
+          break;
+        case "value":
+          phase = "in-value";
+          mark = i;
+          depth = 0;
+          inString = false;
+          escaped = false;
+          // Re-read this byte as the first byte of the value.
+          i -= 1;
+          break;
+        case "array-start":
+          if (byte !== OPEN_BRACKET) {
+            throw new BackupJsonError(
+              `Backup member '${key}' must be an array`,
+            );
+          }
+          streamedCounts[key] = 0;
+          document[key] = [];
+          phase = "element";
+          elementNeedsComma = false;
+          break;
+        case "element":
+          if (byte === CLOSE_BRACKET) {
+            phase = "after";
+          } else if (byte === COMMA && elementNeedsComma) {
+            elementNeedsComma = false;
+          } else if (!elementNeedsComma) {
+            phase = "in-element";
+            mark = i;
+            depth = 0;
+            inString = false;
+            escaped = false;
+            i -= 1;
+          } else {
+            throw new BackupJsonError(
+              `Malformed backup: expected ',' in '${key}'`,
+            );
+          }
+          break;
+        case "after":
+          if (byte === COMMA) {
+            phase = "key";
+          } else if (byte === CLOSE_BRACE) {
+            phase = "done";
+          } else {
+            throw new BackupJsonError("Malformed backup: expected ',' or '}'");
+          }
+          break;
+        case "done":
+          throw new BackupJsonError("Unexpected content after the backup");
+      }
+    }
+    // Carry the unfinished key or value into the next chunk.
+    if (phase === "in-value" || phase === "in-element") {
+      value.add(chunk, mark, chunk.length);
+    } else if (phase === "key" && keyOpen) {
+      keyBytes.add(chunk, mark, chunk.length);
+    }
+  }
+
+  // Anything but the closing brace as the last structural byte is a
+  // truncated file, including one that ends inside a member value.
+  if ((phase as Phase) !== "done") {
+    throw new BackupJsonError("Backup ends before its closing brace");
+  }
+  return { document, streamedCounts };
+}

src/lib/export/full-backup-stream.ts

@@ -26,6 +26,7 @@
  * trusting that paragraph.
  */
 import type { PrismaClient } from "@/generated/prisma/client";
+import { reportJobProgress } from "@/lib/jobs/job-observer";
 import {
   buildFullBackupPayload,
   isDeferredRows,
@@ -45,6 +46,9 @@ export type BackupJsonSink = (chunk: string) => void | Promise<void>;
  */
 const FLUSH_BYTES = 256 * 1024;
 
+/** How often a bulk section reports how far it has got to a running job. */
+const PROGRESS_EVERY_ROWS = 50_000;
+
 /**
  * There is deliberately no heap reading in this writer.
  *
@@ -125,6 +129,9 @@ export async function streamFullBackupJson(
           rows === 0 ? JSON.stringify(row) : `,${JSON.stringify(row)}`,
         );
         rows++;
+        if (rows % PROGRESS_EVERY_ROWS === 0) {
+          reportJobProgress({ backup_section: key, backup_section_rows: rows });
+        }
       }
       await write("]");
       bulkCounts[key] = rows;

src/lib/export/measurement-bulk-insert.ts

@@ -0,0 +1,136 @@
+/**
+ * Insert many measurements in one statement, as column arrays.
+ *
+ * Why not `createMany`. Prisma compiles and caches a query plan per query
+ * shape, and a `createMany` of N rows is its own shape: measured, one
+ * 1 000-row `createMany` of measurements leaves about 145 MB of retained
+ * heap behind, a 2 000-row one about 290 MB, and the smaller last batch of a
+ * restore adds its own plan on top. In a 1 GB container that alone took a
+ * restore of 1.25 million measurements past the heap limit (#1031). One
+ * `INSERT … SELECT FROM unnest(…)` with an array per column is a single small
+ * statement whatever the batch size, and about four times faster.
+ *
+ * `COLUMNS` maps every field of `MeasurementInsertRow` to its column and
+ * type. It is typed as a complete record over the row's keys, so a field
+ * added to the row without a column here does not compile, and the restore
+ * builds its rows as `MeasurementInsertRow`, so a field it starts writing has
+ * to be added to the row first.
+ */
+import type { Prisma } from "@/generated/prisma/client";
+
+/** One measurement as the restore writes it. */
+export interface MeasurementInsertRow {
+  id: string;
+  userId: string;
+  type: string;
+  value: number;
+  valueMin: number | null;
+  valueMax: number | null;
+  unit: string;
+  source: string;
+  measuredAt: Date;
+  notes: string | null;
+  notesEncrypted: Uint8Array | null;
+  externalId: string | null;
+  externalSourceVersion: string | null;
+  aggregationProvenance: string | null;
+  glucoseContext: string | null;
+  sleepStage: string | null;
+  rhythmClassification: string | null;
+  deviceType: string | null;
+  syncVersion: number;
+  deletedAt: Date | null;
+  createdAt?: Date;
+  updatedAt?: Date;
+}
+
+type Kind =
+  "text" | "float8" | "int4" | "timestamp" | "bytea" | `enum:${string}`;
+
+/** Column and element type per field. Complete by construction. */
+const COLUMNS: { [K in keyof MeasurementInsertRow]-?: [string, Kind] } = {
+  id: ["id", "text"],
+  userId: ["user_id", "text"],
+  type: ["type", "enum:measurement_type"],
+  value: ["value", "float8"],
+  valueMin: ["value_min", "float8"],
+  valueMax: ["value_max", "float8"],
+  unit: ["unit", "text"],
+  source: ["source", "enum:measurement_source"],
+  measuredAt: ["measured_at", "timestamp"],
+  notes: ["notes", "text"],
+  notesEncrypted: ["notes_encrypted", "bytea"],
+  externalId: ["external_id", "text"],
+  externalSourceVersion: ["external_source_version", "text"],
+  aggregationProvenance: [
+    "aggregation_provenance",
+    "enum:measurement_aggregation_provenance",
+  ],
+  glucoseContext: ["glucose_context", "enum:glucose_context"],
+  sleepStage: ["sleep_stage", "enum:sleep_stage"],
+  rhythmClassification: ["rhythm_classification", "enum:rhythm_classification"],
+  deviceType: ["device_type", "text"],
+  syncVersion: ["sync_version", "int4"],
+  deletedAt: ["deleted_at", "timestamp"],
+  createdAt: ["created_at", "timestamp"],
+  updatedAt: ["updated_at", "timestamp"],
+};
+
+const FIELDS = Object.keys(COLUMNS) as Array<keyof MeasurementInsertRow>;
+
+/** How each array travels, and how its element is turned back into a value. */
+function arrayType(kind: Kind): string {
+  if (kind === "float8" || kind === "int4") return `${kind}[]`;
+  return "text[]"; // timestamps as ISO text, bytes as hex, enums as labels
+}
+
+function selectExpression(
+  field: keyof MeasurementInsertRow,
+  kind: Kind,
+): string {
+  const ref = `u."${field}"`;
+  if (kind === "timestamp") {
+    // Stored as UTC in `timestamp(3)`. ISO text with a `Z` casts to the same
+    // wall-clock instant; the zone suffix is ignored by the cast.
+    const cast = `${ref}::timestamp(3)`;
+    // Prisma fills both on create when the row does not carry them.
+    return field === "createdAt" || field === "updatedAt"
+      ? `COALESCE(${cast}, CURRENT_TIMESTAMP)`
+      : cast;
+  }
+  if (kind === "bytea") return `decode(${ref}, 'hex')`;
+  if (kind.startsWith("enum:")) return `${ref}::${kind.slice(5)}`;
+  return ref;
+}
+
+/**
+ * The statement, built once. Its text depends on nothing but `COLUMNS`, a
+ * compile-time table: no input reaches it, and every value travels as a bound
+ * array parameter.
+ */
+export const MEASUREMENT_BULK_INSERT_SQL = `
+  INSERT INTO measurements (${FIELDS.map((f) => COLUMNS[f][0]).join(", ")})
+  SELECT ${FIELDS.map((f) => selectExpression(f, COLUMNS[f][1])).join(", ")}
+  FROM unnest(${FIELDS.map((f, i) => `$${i + 1}::${arrayType(COLUMNS[f][1])}`).join(", ")})
+    AS u(${FIELDS.map((f) => `"${f}"`).join(", ")})
+`;
+
+function cell(value: unknown, kind: Kind): unknown {
+  if (value === null || value === undefined) return null;
+  if (kind === "timestamp") return (value as Date).toISOString();
+  if (kind === "bytea") return Buffer.from(value as Uint8Array).toString("hex");
+  return value;
+}
+
+/** Insert `rows` in one statement. Resolves to the number inserted. */
+export async function insertMeasurementRows(
+  tx: Prisma.TransactionClient,
+  rows: readonly MeasurementInsertRow[],
+): Promise<number> {
+  if (rows.length === 0) return 0;
+  const arrays = FIELDS.map((field) => {
+    const kind = COLUMNS[field][1];
+    return rows.map((row) => cell(row[field], kind));
+  });
+  return tx.$executeRawUnsafe(MEASUREMENT_BULK_INSERT_SQL, ...arrays);
+}

src/lib/export/passphrase-archive.ts

@@ -39,6 +39,8 @@
  * hashing params so tuning one never silently changes the other.
  */
 import { hashRaw, type Algorithm } from "@node-rs/argon2";
+import { once } from "node:events";
+import { createWriteStream } from "node:fs";
 import {
   createCipheriv,
   createDecipheriv,
@@ -128,6 +130,11 @@ export async function encryptArchive(
   const ciphertext = Buffer.concat([cipher.update(body), cipher.final()]);
   const tag = cipher.getAuthTag();
 
+  return Buffer.concat([archiveHeader(), salt, iv, tag, ciphertext]);
+}
+
+/** The fixed `HLX1` header for the current KDF parameters. */
+function archiveHeader(): Buffer {
   const header = Buffer.alloc(4 + 1 + 1 + 4 + 4 + 1 + 1);
   header.write(MAGIC, 0, "ascii");
   header.writeUInt8(VERSION, 4);
@@ -136,8 +143,88 @@ export async function encryptArchive(
   header.writeUInt32BE(EXPORT_ARGON2_PARAMS.timeCost, 10);
   header.writeUInt8(EXPORT_ARGON2_PARAMS.parallelism, 14);
   header.writeUInt8(SALT_LENGTH, 15);
+  return header;
+}
 
-  return Buffer.concat([header, salt, iv, tag, ciphertext]);
+/** Where `encryptArchiveToFile` left an archive, in two parts. */
+export interface SpooledArchive {
+  /** Header, salt, IV and tag: everything in front of the ciphertext. */
+  prefix: Buffer;
+  /** File holding the ciphertext, which follows the prefix. */
+  bodyPath: string;
+  /** Size of the whole archive in bytes. */
+  byteLength: number;
+}
+
+/**
+ * Seal a JSON document produced in pieces into an `HLX1` archive without
+ * holding it: the ciphertext goes to `bodyPath` as it is produced.
+ *
+ * Why a file. The format puts the GCM tag in front of the ciphertext, and the
+ * tag exists only once the last byte has been encrypted, so the archive
+ * cannot be sent as it is written. Holding it in memory instead is what the
+ * route did, and for an account of 1.25 million measurements that did not fit
+ * a 1 GB container (#1031). Spooling the ciphertext keeps the format, so every
+ * archive already on someone's disk still opens the same way. Only ciphertext
+ * ever touches the disk; the caller deletes the file once it is sent.
+ *
+ * Decrypts with `decryptArchive` exactly like an archive `encryptArchive`
+ * wrote; the tests hold the two to that.
+ */
+export async function encryptArchiveToFile(
+  producer: (
+    write: (chunk: string | Buffer) => Promise<void>,
+  ) => Promise<unknown>,
+  passphrase: string,
+  bodyPath: string,
+): Promise<SpooledArchive> {
+  if (passphrase.length < MIN_EXPORT_PASSPHRASE_LENGTH) {
+    throw new Error(
+      `Passphrase must be at least ${MIN_EXPORT_PASSPHRASE_LENGTH} characters`,
+    );
+  }
+  const salt = randomBytes(SALT_LENGTH);
+  const key = await deriveKey(passphrase, EXPORT_ARGON2_PARAMS, salt);
+  const iv = randomBytes(IV_LENGTH);
+  const cipher = createCipheriv(ALGORITHM, key, iv);
+  const file = createWriteStream(bodyPath, { mode: 0o600 });
+  let bodyBytes = 0;
+  const fileError = new Promise<never>((_, reject) => {
+    file.once("error", reject);
+  });
+  fileError.catch(() => {});
+
+  const put = async (bytes: Buffer): Promise<void> => {
+    if (bytes.length === 0) return;
+    bodyBytes += bytes.length;
+    if (!file.write(bytes)) {
+      await Promise.race([once(file, "drain"), fileError]);
+    }
+  };
+
+  try {
+    await producer(async (chunk) => {
+      await put(
+        cipher.update(
+          typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk,
+        ),
+      );
+    });
+    await put(cipher.final());
+    file.end();
+    await Promise.race([once(file, "finish"), fileError]);
+  } catch (err) {
+    file.destroy();
+    throw err;
+  }
+
+  const prefix = Buffer.concat([
+    archiveHeader(),
+    salt,
+    iv,
+    cipher.getAuthTag(),
+  ]);
+  return { prefix, bodyPath, byteLength: prefix.length + bodyBytes };
 }
 
 /** Parse + validate the fixed `HLX1` header. Throws on any structural fault. */

src/lib/export/response-stream.ts

@@ -0,0 +1,82 @@
+/**
+ * Turn a producer of text or bytes into a response body, with backpressure
+ * and cancellation.
+ *
+ * The export routes used to build their whole file as one string and hand it
+ * to the response. For an account of 1.25 million measurements that string
+ * and the object graph behind it did not fit a 1 GB container: one export
+ * took the app down for every user of the instance (#1031). The file is now
+ * written into the response as it is produced.
+ *
+ * Backpressure: the producer's `write` waits while the client is slower than
+ * the database, so nothing piles up in between. Cancellation: when the client
+ * goes away, the next `write` throws `ResponseStreamClosedError`, which stops
+ * the producer instead of leaving it to finish a file nobody reads.
+ */
+import { PassThrough, Readable } from "node:stream";
+
+/** The client closed the response before the producer finished. */
+export class ResponseStreamClosedError extends Error {
+  constructor() {
+    super("The response was closed before the file was complete");
+    this.name = "ResponseStreamClosedError";
+  }
+}
+
+export type ResponseProducer = (
+  write: (chunk: string | Buffer) => Promise<void>,
+) => Promise<unknown>;
+
+export interface ResponseStreamHooks {
+  /** Runs once the producer has written everything. */
+  onComplete?: () => unknown;
+  /**
+   * Runs when the producer failed or the client left. The response has
+   * already started, so the client receives a truncated file; a truncated
+   * JSON file does not parse, and a truncated archive fails its tag.
+   */
+  onError?: (err: unknown) => unknown;
+}
+
+/** Start `producer` and return the web stream its output flows into. */
+export function streamToResponseBody(
+  producer: ResponseProducer,
+  hooks: ResponseStreamHooks = {},
+): ReadableStream<Uint8Array> {
+  const out = new PassThrough();
+
+  const write = async (chunk: string | Buffer): Promise<void> => {
+    if (out.destroyed || out.writableEnded)
+      throw new ResponseStreamClosedError();
+    if (out.write(chunk)) return;
+    await new Promise<void>((resolve, reject) => {
+      const onDrain = () => {
+        out.off("close", onClose);
+        resolve();
+      };
+      const onClose = () => {
+        out.off("drain", onDrain);
+        reject(new ResponseStreamClosedError());
+      };
+      out.once("drain", onDrain);
+      out.once("close", onClose);
+    });
+  };
+
+  void (async () => {
+    try {
+      await producer(write);
+      out.end();
+      await hooks.onComplete?.();
+    } catch (err) {
+      out.destroy(err instanceof Error ? err : new Error(String(err)));
+      try {
+        await hooks.onError?.(err);
+      } catch {
+        // Reporting a failure must not raise another one.
+      }
+    }
+  })();
+
+  return Readable.toWeb(out) as ReadableStream<Uint8Array>;
+}

src/lib/export/store-backup-blob.ts

@@ -0,0 +1,119 @@
+/**
+ * Write one account's stored backup (`data_backups.data`) without ever
+ * holding the stored copy in this process.
+ *
+ * Why. `packBackupBlobStreaming` streams the JSON through gzip and the cipher,
+ * but its answer is one string, because the column takes one value. Measured
+ * on a seeded account of 1.25 million measurements: the answer is 64 MB of
+ * base64, and between the pieces, the joined string and the copies the
+ * database driver makes to bind it, the backup grew the process by about
+ * 640 MB of resident memory (309 MB → 946 MB). The default compose stack runs
+ * the web server and the worker in one container capped at 1 GB, so on a
+ * record of that size the weekly or manual backup is the thing that takes the
+ * container down; a killed worker leaves its job `active` until pg-boss
+ * expires it two hours later as `job timed out`.
+ *
+ * What instead. The pieces go to Postgres as they come, a few megabytes at a
+ * time, into a temporary table on the transaction's own connection, and one
+ * statement assembles them into the row with `string_agg`. The process never
+ * holds more than one piece; Postgres builds the value it was going to store
+ * anyway.
+ *
+ * All of it is one transaction, so the account's previous copy stays in place
+ * until the new one is complete: a failure halfway through rolls back to it,
+ * and the temporary table goes with the transaction either way.
+ */
+import type { PrismaClient } from "@/generated/prisma/client";
+
+import {
+  packBackupBlobInto,
+  type BackupJsonProducer,
+  type PackBackupBlobOptions,
+} from "@/lib/export/backup-blob";
+
+/**
+ * How long the storing transaction may stay open. The job around it expires
+ * after two hours (`DATA_BACKUP_SEND_OPTIONS`); the transaction has to give
+ * up before that so the failure is the transaction's own, with its own
+ * message, rather than an expired job.
+ */
+const STORE_TRANSACTION_TIMEOUT_MS = 90 * 60 * 1000;
+
+/**
+ * How long the transaction may sit idle between two pieces. The session
+ * default (`idle_in_transaction_session_timeout`, 60 s, see `src/lib/db.ts`)
+ * is right for a request and wrong here: the producer reads the whole record
+ * on other connections while this one waits for the next piece, and the
+ * sections before the first measurement can take longer than a minute on a
+ * large account. Scoped with SET LOCAL, so it ends with the transaction.
+ */
+const STORE_IDLE_TIMEOUT = "10min";
+
+export interface StoreBackupBlobInput {
+  userId: string;
+  /** `DataBackup.type`: `WEEKLY_AUTO` for the scheduled and manual pass. */
+  type: string;
+}
+
+/**
+ * Pack `producer`'s JSON into the stored envelope and upsert it as the
+ * `(userId, type)` backup. Resolves to the row's id and the stored size.
+ *
+ * `input` may be a function, read once the producer has finished: an
+ * uploaded file names its owner somewhere inside itself, and the upload
+ * route only knows who that is once it has read the file through.
+ */
+export async function storeBackupBlob(
+  prisma: PrismaClient,
+  input: StoreBackupBlobInput | (() => StoreBackupBlobInput),
+  producer: BackupJsonProducer,
+  options: PackBackupBlobOptions = {},
+): Promise<{ id: string; bytes: number }> {
+  return prisma.$transaction(
+    async (tx) => {
+      await tx.$executeRawUnsafe(
+        `SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`,
+      );
+      await tx.$executeRaw`
+        CREATE TEMP TABLE backup_blob_parts (
+          seq integer PRIMARY KEY,
+          piece text NOT NULL
+        ) ON COMMIT DROP
+      `;
+
+      let seq = 0;
+      let bytes = 0;
+      await packBackupBlobInto(
+        async (piece) => {
+          await tx.$executeRaw`
+            INSERT INTO backup_blob_parts (seq, piece) VALUES (${seq}, ${piece})
+          `;
+          seq += 1;
+          bytes += piece.length;
+        },
+        producer,
+        options,
+      );
+
+      const target = typeof input === "function" ? input() : input;
+      // The row first, through Prisma, so a new account's backup gets its id
+      // the same way every other row does; the data it briefly carries is
+      // never visible outside this transaction.
+      const row = await tx.dataBackup.upsert({
+        where: { userId_type: { userId: target.userId, type: target.type } },
+        update: { createdAt: new Date() },
+        create: { userId: target.userId, type: target.type, data: "" },
+        select: { id: true },
+      });
+      await tx.$executeRaw`
+        UPDATE data_backups
+        SET data = (
+          SELECT string_agg(piece, '' ORDER BY seq) FROM backup_blob_parts
+        )
+        WHERE id = ${row.id}
+      `;
+      return { id: row.id, bytes };
+    },
+    { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 },
+  );
+}

src/lib/export/streamed-backup.ts

@@ -0,0 +1,138 @@
+/**
+ * A backup file read as a stream: every section but the measurements in
+ * memory, the measurements handed over in batches.
+ *
+ * Why. The measurements are the one section that grows with a record without
+ * bound. For an account with 1.25 million of them the disaster-recovery JSON
+ * is 662 MB, which cannot exist as one JavaScript string, and the parsed rows
+ * alone would not fit a 1 GB container beside the running app (#1031). Every
+ * other section of the same file is a few megabytes.
+ *
+ * So the file is read twice. The first read keeps everything except the
+ * measurements, and checks each measurement against the same element schema
+ * the whole-file schema uses, counting them. Nothing is written on the
+ * strength of an unchecked row: a restore reads the file this way before it
+ * deletes anything. The second read, from `forEachMeasurementBatch`, hands
+ * the measurements over in order, in batches, for the writer.
+ *
+ * `raw` is the file exactly as `JSON.parse` would give it, with
+ * `measurements` replaced by an empty array when the file has that section
+ * (and still absent when it does not, which is what the section-presence
+ * check reads). Run it through `parseBackupPayload` for the rest of the
+ * validation; together that is the whole-file check.
+ */
+import { scanBackupJson } from "@/lib/export/backup-json-scan";
+import {
+  BACKUP_SCHEMA_VERSION,
+  backupMeasurementSchema,
+  type BackupMeasurement,
+} from "@/lib/validations/backup";
+
+/** The file's bytes, from the start, every time it is called. */
+export type BackupSource = () => AsyncIterable<Uint8Array>;
+
+const MEASUREMENTS = "measurements";
+const STREAMED = new Set([MEASUREMENTS]);
+
+/** A measurement the element schema refused, with where it is in the file. */
+export class StreamedBackupInvalidError extends Error {
+  readonly path: string;
+  constructor(path: string, message: string) {
+    super(`${path}: ${message}`);
+    this.name = "StreamedBackupInvalidError";
+    this.path = path;
+  }
+}
+
+export interface StreamedBackup {
+  /** The file without its measurements (see the module comment). */
+  raw: Record<string, unknown>;
+  /** How many measurements the file carries. */
+  measurementCount: number;
+  /** The file's size in bytes. */
+  bytes: number;
+  /**
+   * Read the file again and hand its measurements over in file order, in
+   * batches of at most `size`, each row parsed by the element schema.
+   * Awaits `fn` before reading further.
+   */
+  forEachMeasurementBatch(
+    size: number,
+    fn: (batch: BackupMeasurement[]) => Promise<void>,
+  ): Promise<void>;
+}
+
+function parseMeasurement(element: unknown, index: number): BackupMeasurement {
+  const parsed = backupMeasurementSchema.safeParse(element);
+  if (!parsed.success) {
+    const issue = parsed.error.issues[0];
+    throw new StreamedBackupInvalidError(
+      [MEASUREMENTS, index, ...(issue?.path ?? [])].join("."),
+      issue?.message ?? "invalid measurement",
+    );
+  }
+  return parsed.data;
+}
+
+/**
+ * First read of a backup file. Rejects with `BackupJsonError` for input that
+ * is not a JSON object and with `StreamedBackupInvalidError` for a
+ * measurement the schema refuses.
+ */
+export async function readStreamedBackup(
+  source: BackupSource,
+): Promise<StreamedBackup> {
+  let firstWithoutId: number | null = null;
+  let bytes = 0;
+  async function* counted() {
+    for await (const chunk of source()) {
+      bytes += chunk.byteLength;
+      yield chunk;
+    }
+  }
+  const { document, streamedCounts } = await scanBackupJson(counted(), {
+    streamKeys: STREAMED,
+    onElement: (_key, element, index) => {
+      const row = parseMeasurement(element, index);
+      if (!row.id && firstWithoutId === null) firstWithoutId = index;
+    },
+  });
+
+  // The whole-file schema's cross-row rule, applied the same way: a canonical
+  // v2 file must give every measurement a stable id.
+  if (
+    document.schemaVersion === BACKUP_SCHEMA_VERSION &&
+    firstWithoutId !== null
+  ) {
+    throw new StreamedBackupInvalidError(
+      `${MEASUREMENTS}.${firstWithoutId}.id`,
+      "Canonical v2 measurements require a stable id",
+    );
+  }
+
+  const skipKeys = new Set(
+    Object.keys(document).filter((key) => key !== MEASUREMENTS),
+  );
+
+  return {
+    raw: document,
+    measurementCount: streamedCounts[MEASUREMENTS] ?? 0,
+    bytes,
+    async forEachMeasurementBatch(size, fn) {
+      let batch: BackupMeasurement[] = [];
+      await scanBackupJson(source(), {
+        streamKeys: STREAMED,
+        skipKeys,
+        onElement: async (_key, element, index) => {
+          batch.push(parseMeasurement(element, index));
+          if (batch.length >= size) {
+            const full = batch;
+            batch = [];
+            await fn(full);
+          }
+        },
+      });
+      if (batch.length > 0) await fn(batch);
+    },
+  };
+}

src/lib/jobs/__tests__/drain-continuation.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  DRAIN_MAX_CONTINUATIONS,
+  nextDrainContinuation,
+} from "@/lib/jobs/drain-continuation";
+
+// A nightly drain tick that ran out of budget hands the rest to a follow-up
+// job, so a first-time backlog drains the same night (issue #1031). The chain
+// must not loop on a pass that cannot move, and must end.
+
+describe("nextDrainContinuation", () => {
+  it("hands on a tick that stopped early after folding something", () => {
+    expect(
+      nextDrainContinuation({
+        stoppedEarly: true,
+        daysFolded: 12,
+        continuation: 0,
+      }),
+    ).toBe(1);
+    expect(
+      nextDrainContinuation({
+        stoppedEarly: true,
+        daysFolded: 1,
+        continuation: 5,
+      }),
+    ).toBe(6);
+  });
+
+  it("does not hand on a tick that finished", () => {
+    expect(
+      nextDrainContinuation({
+        stoppedEarly: false,
+        daysFolded: 400,
+        continuation: 0,
+      }),
+    ).toBeNull();
+  });
+
+  it("does not hand on a tick that stopped without progress", () => {
+    expect(
+      nextDrainContinuation({
+        stoppedEarly: true,
+        daysFolded: 0,
+        continuation: 0,
+      }),
+    ).toBeNull();
+  });
+
+  it("ends the chain at the cap", () => {
+    expect(
+      nextDrainContinuation({
+        stoppedEarly: true,
+        daysFolded: 10,
+        continuation: DRAIN_MAX_CONTINUATIONS - 1,
+      }),
+    ).toBe(DRAIN_MAX_CONTINUATIONS);
+    expect(
+      nextDrainContinuation({
+        stoppedEarly: true,
+        daysFolded: 10,
+        continuation: DRAIN_MAX_CONTINUATIONS,
+      }),
+    ).toBeNull();
+  });
+});

src/lib/jobs/__tests__/drain-cumulative-queue.test.ts

@@ -52,7 +52,7 @@ describe("reminder-worker — drainPerSampleCumulative nightly schedule", () =>
 
   it("registers a createAndWork handler against the drain queue", () => {
     expect(source).toMatch(
-      /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,900}drainPerSampleCumulative/,
+      /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,1100}drainPerSampleCumulative/,
     );
   });
 

src/lib/jobs/__tests__/job-budget.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vitest";
+
+import { JOB_BUDGET_SHARE, jobBudget } from "@/lib/jobs/job-budget";
+
+// pg-boss declares a job dead at its expiry but does not stop the handler;
+// the long passes stop themselves on this budget instead (issue #1031).
+
+function job(expireInSeconds: number, signal = new AbortController().signal) {
+  return { expireInSeconds, signal };
+}
+
+describe("jobBudget", () => {
+  it("stops at the budget share of the job's expiry, not at the expiry", () => {
+    let now = 1_000_000;
+    const shouldStop = jobBudget([job(900)], () => now);
+
+    now += 900 * 1000 * JOB_BUDGET_SHARE - 1;
+    expect(shouldStop()).toBe(false);
+    now += 1;
+    expect(shouldStop()).toBe(true);
+    expect(JOB_BUDGET_SHARE).toBeLessThan(1);
+  });
+
+  it("stops as soon as pg-boss aborts the job, whatever the clock says", () => {
+    const controller = new AbortController();
+    const shouldStop = jobBudget([job(900, controller.signal)], () => 0);
+    expect(shouldStop()).toBe(false);
+    controller.abort();
+    expect(shouldStop()).toBe(true);
+  });
+
+  it("takes the tightest expiry of a batch", () => {
+    let now = 0;
+    const shouldStop = jobBudget([job(7200), job(60)], () => now);
+    now = 60 * 1000 * JOB_BUDGET_SHARE;
+    expect(shouldStop()).toBe(true);
+  });
+
+  it("never stops on the clock when there is no job to take an expiry from", () => {
+    let now = 0;
+    const shouldStop = jobBudget([], () => now);
+    now = Number.MAX_SAFE_INTEGER;
+    expect(shouldStop()).toBe(false);
+  });
+});

src/lib/jobs/__tests__/job-observer.test.ts

@@ -0,0 +1,200 @@
+/**
+ * Issue #1031: three queues failed with `job timed out` night after night and
+ * the operator found no line about any of them in the app log. These tests
+ * pin the lines `observeJob` now writes — progress while a job runs, a warning
+ * at its expiry, and a closing line for a long or expired run — and the boot
+ * line naming jobs a previous process was running when it stopped.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+  emitted: [] as Array<{
+    level: string;
+    background?: { task_name?: string };
+    meta?: Record<string, unknown>;
+  }>,
+  active: vi.fn(),
+}));
+
+vi.mock("@/lib/logging/transports", () => ({
+  emitIfSampled: (event: (typeof mocks.emitted)[number]) => {
+    mocks.emitted.push(event);
+  },
+}));
+vi.mock("@/lib/jobs/job-failures", () => ({
+  readActiveJobsStartedBefore: mocks.active,
+}));
+
+import {
+  JOB_LONG_RUN_MS,
+  JOB_PROGRESS_INTERVAL_MS,
+  observeJob,
+  reportJobProgress,
+  reportJobsCutOffAtBoot,
+  type JobObserverClock,
+} from "@/lib/jobs/job-observer";
+
+/** A manual clock: `advance` fires due timers in order. */
+function manualClock() {
+  let now = 0;
+  let seq = 0;
+  const timers = new Map<
+    number,
+    { at: number; every: number | null; fn: () => void }
+  >();
+  const clock: JobObserverClock = {
+    now: () => now,
+    setTimeout: (fn, ms) => {
+      seq += 1;
+      timers.set(seq, { at: now + ms, every: null, fn });
+      return seq;
+    },
+    setInterval: (fn, ms) => {
+      seq += 1;
+      timers.set(seq, { at: now + ms, every: ms, fn });
+      return seq;
+    },
+    clearTimeout: (handle) => timers.delete(handle as number),
+    clearInterval: (handle) => timers.delete(handle as number),
+  };
+  const advance = (ms: number) => {
+    const target = now + ms;
+    for (;;) {
+      const due = [...timers.entries()]
+        .filter(([, t]) => t.at <= target)
+        .sort((a, b) => a[1].at - b[1].at)[0];
+      if (!due) break;
+      const [id, timer] = due;
+      now = timer.at;
+      if (timer.every === null) timers.delete(id);
+      else timer.at += timer.every;
+      timer.fn();
+    }
+    now = target;
+  };
+  return { clock, advance };
+}
+
+function lines(task: string) {
+  return mocks.emitted.filter((e) => e.background?.task_name === task);
+}
+
+beforeEach(() => {
+  mocks.emitted.length = 0;
+  mocks.active.mockReset();
+});
+
+describe("observeJob", () => {
+  it("writes progress while a long job runs, a warning at its expiry, and a closing line", async () => {
+    const { clock, advance } = manualClock();
+    let release!: (value: { ok: true; did: { days: number } }) => void;
+
+    const running = observeJob(
+      "dense-intraday-retention",
+      [{ id: "job-1", expireInSeconds: 900 }],
+      () =>
+        new Promise<{ ok: true; did: { days: number } }>((resolve) => {
+          reportJobProgress({ consolidation_day: "2024-03-01" });
+          release = resolve;
+        }),
+      clock,
+    );
+
+    advance(JOB_PROGRESS_INTERVAL_MS);
+    expect(lines("job.progress")).toHaveLength(1);
+    expect(lines("job.progress")[0]!.meta).toMatchObject({
+      queue: "dense-intraday-retention",
+      job_ids: "job-1",
+      expire_s: 900,
+      consolidation_day: "2024-03-01",
+    });
+    expect(lines("job.progress")[0]!.meta?.heap_used_mb).toEqual(
+      expect.any(Number),
+    );
+
+    advance(900 * 1000 - JOB_PROGRESS_INTERVAL_MS);
+    const expired = lines("job.expired");
+    expect(expired).toHaveLength(1);
+    expect(expired[0]!.level).toBe("warn");
+    expect(expired[0]!.meta).toMatchObject({
+      queue: "dense-intraday-retention",
+      elapsed_s: 900,
+      consolidation_day: "2024-03-01",
+    });
+
+    release({ ok: true, did: { days: 12 } });
+    await running;
+    const finished = lines("job.finished");
+    expect(finished).toHaveLength(1);
+    expect(finished[0]!.level).toBe("warn");
+    expect(finished[0]!.meta).toMatchObject({
+      queue: "dense-intraday-retention",
+      finished_after_expiry: true,
+      outcome: "completed",
+      did: { days: 12 },
+    });
+  });
+
+  it("stays silent for a short job", async () => {
+    const { clock } = manualClock();
+    await observeJob(
+      "host-metric-sample",
+      [{ id: "j", expireInSeconds: 900 }],
+      async () => ({ ok: true }),
+      clock,
+    );
+    expect(mocks.emitted).toEqual([]);
+  });
+
+  it("closes a long job that threw with the error, and rethrows it", async () => {
+    const { clock, advance } = manualClock();
+    const running = observeJob(
+      "data-backup",
+      [{ id: "b", expireInSeconds: 7200 }],
+      async () => {
+        advance(JOB_LONG_RUN_MS);
+        throw new Error("write failed");
+      },
+      clock,
+    );
+    await expect(running).rejects.toThrow("write failed");
+    expect(lines("job.finished")[0]!.meta).toMatchObject({
+      queue: "data-backup",
+      outcome: "threw",
+      error: "write failed",
+    });
+    expect(lines("job.expired")).toEqual([]);
+  });
+});
+
+describe("reportJobsCutOffAtBoot", () => {
+  it("names each job a previous process was running", async () => {
+    mocks.active.mockResolvedValue([
+      {
+        queue: "data-backup",
+        id: "a",
+        startedAt: "2026-09-24T09:36:00.000Z",
+        expireSeconds: 7200,
+      },
+    ]);
+    await reportJobsCutOffAtBoot(new Date("2026-09-24T10:00:00.000Z"));
+    const line = lines("worker.boot.jobs_cut_off");
+    expect(line).toHaveLength(1);
+    expect(line[0]!.level).toBe("warn");
+    expect(line[0]!.meta).toMatchObject({
+      jobs_cut_off: 1,
+      queues: "data-backup",
+    });
+    expect(String(line[0]!.meta?.jobs)).toContain(
+      "data-backup@2026-09-24T09:36:00.000Z",
+    );
+  });
+
+  it("stays silent when nothing was cut off, or there is no queue schema", async () => {
+    mocks.active.mockResolvedValueOnce([]);
+    await reportJobsCutOffAtBoot();
+    mocks.active.mockResolvedValueOnce(null);
+    await reportJobsCutOffAtBoot();
+    expect(mocks.emitted).toEqual([]);
+  });
+});

src/lib/jobs/__tests__/run-job.test.ts

@@ -149,3 +149,25 @@ describe("runJob", () => {
     expect(handler).toHaveBeenCalledWith(jobs);
   });
 });
+
+describe("runJob observation (#1031)", () => {
+  it("runs every handler inside the job observer, so a long or expired job leaves a log line", async () => {
+    const observer = await import("../job-observer");
+    const observe = vi.spyOn(observer, "observeJob");
+    const jobs = [
+      { id: "j-1", expireInSeconds: 900 },
+    ] as unknown as Job<object>[];
+    const wrapped = runJob("dense-intraday-retention", async () =>
+      jobDone({ days_consolidated: 1 }),
+    );
+
+    await wrapped(jobs);
+
+    expect(observe).toHaveBeenCalledWith(
+      "dense-intraday-retention",
+      jobs,
+      expect.any(Function),
+    );
+    observe.mockRestore();
+  });
+});

src/lib/jobs/apple-health-import-worker.ts

@@ -41,7 +41,10 @@ import {
   type ImportJobProgress,
   type ImportJobResult,
 } from "@/lib/measurements/import-apple-health-export";
-import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
+import {
+  recomputeUserRollups,
+  ROLLUP_FOLD_WINDOW_MS,
+} from "@/lib/rollups/measurement-rollups";
 import { withBackgroundEvent } from "@/lib/logging/background";
 
 /**
@@ -502,19 +505,23 @@ export async function handleAppleHealthImport(
     // contradict the row the status poll surfaces to the operator.
     let rollupFailed = false;
     try {
-      const span = await prisma.measurement.aggregate({
-        where: { userId },
-        _min: { measuredAt: true },
-        _max: { measuredAt: true },
-      });
-      if (span._min.measuredAt && span._max.measuredAt) {
-        // Add a small tail buffer to `to` so the upper bound is
-        // exclusive-safe under the rollup aggregator's `< to` filter.
-        const to = new Date(span._max.measuredAt.getTime() + 1);
-        await recomputeUserRollups(userId, {
-          from: span._min.measuredAt,
-          to,
-        });
+      // Folded over what this import wrote, not over the account's whole
+      // history, and never further back than the fold window every other
+      // path keeps to: buckets older than `ROLLUP_FOLD_WINDOW_MS` are by
+      // design never written, so reads there fall back to live SQL. The old
+      // span ran from the account's first measurement, whatever the import
+      // carried, and folded any history older than the window as well.
+      const span = result.measuredSpan;
+      if (span) {
+        const windowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS);
+        const spanFrom = new Date(span.from);
+        const from = spanFrom > windowStart ? spanFrom : windowStart;
+        // `to` is exclusive in the aggregator, so one millisecond past the
+        // newest row.
+        const to = new Date(new Date(span.to).getTime() + 1);
+        if (to > from) {
+          await recomputeUserRollups(userId, { from, to });
+        }
       }
     } catch (rollupErr) {
       // Rollup failure is non-fatal — the next read falls through to

src/lib/jobs/dense-intraday-hourly-rebuild.ts

@@ -65,11 +65,14 @@ export interface DenseIntradayHourlyRebuildPayload {
  */
 export async function runDenseIntradayHourlyRebuildForUser(
   userId: string,
+  shouldStop?: () => boolean,
 ): Promise<{
   daysRebuilt: number;
   hourlyRowsUpserted: number;
   dailyRowsRetired: number;
   daysSkippedNoTombstones: number;
+  /** The job's budget ran out first; see the queue handler. */
+  stoppedEarly: boolean;
 }> {
   // Shares the retention drain's kill-switch: when an operator disables the
   // dense tier, no-op so any already-queued backlog drains cleanly.
@@ -79,10 +82,12 @@ export async function runDenseIntradayHourlyRebuildForUser(
       hourlyRowsUpserted: 0,
       dailyRowsRetired: 0,
       daysSkippedNoTombstones: 0,
+      stoppedEarly: false,
     };
   }
   const summary = await runDenseIntradayHourlyRebuild(prisma, {
     userId,
+    shouldStop,
     log: () => {
       // Silent inside the queue handler — the worker logs the totals.
     },
@@ -104,6 +109,7 @@ export async function runDenseIntradayHourlyRebuildForUser(
     hourlyRowsUpserted: summary.totals.hourlyRowsUpserted,
     dailyRowsRetired: summary.totals.dailyRowsRetired,
     daysSkippedNoTombstones: summary.totals.daysSkippedNoTombstones,
+    stoppedEarly: summary.stoppedEarly,
   };
 }
 

src/lib/jobs/dense-intraday-retention.ts

@@ -89,10 +89,13 @@ export interface DenseIntradayRetentionPayload {
  */
 export async function runDenseIntradayRetentionForUser(
   userId: string,
+  shouldStop?: () => boolean,
 ): Promise<{
   daysConsolidated: number;
   perSampleRowsSoftDeleted: number;
   derivedRestingRowsUpserted: number;
+  /** The job's budget ran out first; the next run continues from here. */
+  stoppedEarly: boolean;
 }> {
   // Kill-switch: when an operator opts out, no-op so any already-queued
   // backlog completes cleanly (drains the queue) instead of running the fold.
@@ -101,10 +104,12 @@ export async function runDenseIntradayRetentionForUser(
       daysConsolidated: 0,
       perSampleRowsSoftDeleted: 0,
       derivedRestingRowsUpserted: 0,
+      stoppedEarly: false,
     };
   }
   const summary = await runDenseIntradayRetention(prisma, {
     userId,
+    shouldStop,
     log: () => {
       // Silent inside the queue handler — the worker logs the totals.
     },
@@ -121,6 +126,7 @@ export async function runDenseIntradayRetentionForUser(
         // days for proxy users, preserving the resting signal post-fold.
         derived_resting_rows_upserted:
           summary.totals.derivedRestingRowsUpserted,
+        stopped_early: summary.stoppedEarly,
       },
     },
   });
@@ -128,6 +134,7 @@ export async function runDenseIntradayRetentionForUser(
     daysConsolidated: summary.totals.daysConsolidated,
     perSampleRowsSoftDeleted: summary.totals.perSampleRowsSoftDeleted,
     derivedRestingRowsUpserted: summary.totals.derivedRestingRowsUpserted,
+    stoppedEarly: summary.stoppedEarly,
   };
 }
 

src/lib/jobs/drain-continuation.ts

@@ -0,0 +1,33 @@
+/**
+ * When the nightly drain tick hands its remaining work to a follow-up job.
+ *
+ * The tick's three passes (cumulative drain, daily-mean consolidation, dense
+ * intra-day retention) stop at the job's time budget (`jobBudget`) instead of
+ * running into pg-boss's expiry. On a normal night that never happens. After
+ * a multi-year import it does, and without a follow-up the backlog would drain
+ * one budget per night. So a tick that stopped early sends itself again a
+ * minute later, and the chain is bounded twice: a follow-up is sent only when
+ * the run folded at least one day, so a pass that makes no progress cannot
+ * loop, and never more than `DRAIN_MAX_CONTINUATIONS` times in a row.
+ */
+
+/** Follow-ups one nightly tick may chain: about four and a half hours of budget. */
+export const DRAIN_MAX_CONTINUATIONS = 24;
+
+/** Delay before a follow-up tick starts, in seconds. */
+export const DRAIN_CONTINUATION_DELAY_SECONDS = 60;
+
+/**
+ * The follow-up's continuation number, or `null` when no follow-up is due.
+ * `continuation` is the current job's own number (0 for the cron's job).
+ */
+export function nextDrainContinuation(input: {
+  stoppedEarly: boolean;
+  daysFolded: number;
+  continuation: number;
+}): number | null {
+  if (!input.stoppedEarly) return null;
+  if (input.daysFolded <= 0) return null;
+  if (input.continuation >= DRAIN_MAX_CONTINUATIONS) return null;
+  return input.continuation + 1;
+}

src/lib/jobs/job-budget.ts

@@ -0,0 +1,48 @@
+/**
+ * A time budget for a long pass, derived from the pg-boss job it runs under.
+ *
+ * pg-boss gives every job an expiry (`expireInSeconds`, 900 s unless the send
+ * says otherwise). When it passes, pg-boss marks the job failed with
+ * `job timed out`, retries it, and fires the job's abort signal. It does NOT
+ * stop the handler: a handler that never looks at the signal keeps writing
+ * after its job has been declared dead, and the retry then runs beside it over
+ * the same rows. On a large account that is how one pass that could not
+ * finish turned into two or three copies of itself contending for the same
+ * days.
+ *
+ * The passes that walk a whole history therefore stop on their own, between
+ * units of work, once a share of the expiry has gone by, and report that they
+ * stopped early. Their units commit one at a time and a finished unit drops
+ * out of the next scan, so the following run starts where this one stopped.
+ * The share leaves room for the unit in flight and for the handler's own
+ * bookkeeping before pg-boss's deadline.
+ */
+import type { Job } from "pg-boss";
+
+/** Share of the job's expiry a pass may spend before it stops itself. */
+export const JOB_BUDGET_SHARE = 0.75;
+
+/** What the budget reads from a pg-boss job. */
+type BudgetedJob = Pick<Job<unknown>, "expireInSeconds" | "signal">;
+
+/**
+ * Build the `shouldStop` predicate for a batch of jobs handled together.
+ *
+ * True once the budget has elapsed (the tightest job's expiry decides) or once
+ * pg-boss has aborted any of the jobs. `now` is injectable for tests.
+ */
+export function jobBudget(
+  jobs: readonly BudgetedJob[],
+  now: () => number = Date.now,
+): () => boolean {
+  const expirySeconds = Math.min(
+    ...jobs.map((job) =>
+      Number.isFinite(job.expireInSeconds) && job.expireInSeconds > 0
+        ? job.expireInSeconds
+        : Number.POSITIVE_INFINITY,
+    ),
+  );
+  const deadline = now() + expirySeconds * 1000 * JOB_BUDGET_SHARE;
+  return () =>
+    now() >= deadline || jobs.some((job) => job.signal?.aborted === true);
+}

src/lib/jobs/job-failures.ts

@@ -66,6 +66,35 @@ interface FailingQueueRow {
   failures: number;
   last_failed_at: Date | null;
   last_error: string | null;
+  last_expire_seconds: number | null;
+}
+
+/**
+ * The two texts pg-boss writes when a job ran out of time: the supervisor's,
+ * for a job still `active` past its expiry, and the worker's own, for a
+ * handler that did not settle inside it.
+ */
+const TIMEOUT_MESSAGE = /^(job timed out|handler execution exceeded \d+s)$/;
+
+/**
+ * Say what a bare timeout means. "job timed out" alone sent an operator
+ * nowhere: pg-boss writes the same words for a pass that was too slow for its
+ * limit and for a job whose process died under it (a restart, or a container
+ * killed for memory), in which case the job sits `active` until the limit runs
+ * out. The limit and the two causes are what the operator can check.
+ */
+function explainTimeout(message: string, expireSeconds: number | null): string {
+  const limit =
+    expireSeconds !== null && expireSeconds > 0
+      ? expireSeconds >= 3600 && expireSeconds % 3600 === 0
+        ? `${expireSeconds / 3600} h`
+        : `${Math.round(expireSeconds / 60)} min`
+      : null;
+  return (
+    `${message}: did not finish within its ${limit ?? "time"} limit. ` +
+    `Either the run was too slow for the limit, or the app restarted while it ` +
+    `ran; a restart shows in the container's restart count and log.`
+  );
 }
 
 /**
@@ -73,10 +102,15 @@ interface FailingQueueRow {
  * serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at
  * all, so the raw text is the fallback rather than an invented placeholder.
  */
-function presentError(raw: string | null): string {
+function presentError(
+  raw: string | null,
+  expireSeconds: number | null = null,
+): string {
   if (raw === null) return "";
   const trimmed = raw.trim();
   if (trimmed.length === 0) return "";
+  if (TIMEOUT_MESSAGE.test(trimmed))
+    return explainTimeout(trimmed, expireSeconds);
   const redacted = redactSecrets(trimmed);
   return redacted.length > MAX_ERROR_CHARS
     ? `${redacted.slice(0, MAX_ERROR_CHARS)}…`
@@ -105,7 +139,10 @@ export async function readFailingQueues(
             )
             ORDER BY completed_on DESC NULLS LAST
           )
-        )[1] AS last_error
+        )[1] AS last_error,
+        (
+          array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST)
+        )[1] AS last_expire_seconds
       FROM pgboss.job
       WHERE state = 'failed'
         AND completed_on > now() - make_interval(hours => ${withinHours}::int)
@@ -119,7 +156,7 @@ export async function readFailingQueues(
         queue: row.name,
         failures: row.failures,
         lastFailedAt: row.last_failed_at!.toISOString(),
-        lastError: presentError(row.last_error),
+        lastError: presentError(row.last_error, row.last_expire_seconds),
       }));
   } catch {
     // No `pgboss` schema (web-only deployment) or no permission to read it.
@@ -160,11 +197,17 @@ export async function readLastQueueRun(
 ): Promise<LastQueueRun | null> {
   try {
     const rows = await prisma.$queryRaw<
-      Array<{ state: string; completed_on: Date | null; output: string | null }>
+      Array<{
+        state: string;
+        completed_on: Date | null;
+        output: string | null;
+        expire_seconds: number | null;
+      }>
     >`
       SELECT
         state,
         completed_on,
+        expire_seconds,
         -- pg-boss wraps a thrown value: an expired job's row reads
         -- {"value":{"message":"job timed out"}}, so a plain ->>'message'
         -- misses it and the operator gets raw JSON instead of the sentence.
@@ -189,7 +232,10 @@ export async function readLastQueueRun(
     return {
       at: row.completed_on.toISOString(),
       state,
-      error: state === "completed" ? null : presentError(row.output),
+      error:
+        state === "completed"
+          ? null
+          : presentError(row.output, row.expire_seconds),
     };
   } catch {
     // No `pgboss` schema (web-only deployment) or no permission to read it.
@@ -238,3 +284,58 @@ export async function readQueueFailureForUser(
     return null;
   }
 }
+
+/** A job pg-boss still has as running, from `readActiveJobsStartedBefore`. */
+export interface ActiveJob {
+  queue: string;
+  id: string;
+  /** ISO instant the job was picked up. */
+  startedAt: string;
+  /** The job's expiry, in seconds, as pg-boss holds it. */
+  expireSeconds: number;
+}
+
+/**
+ * Jobs still marked active that were picked up before `before` — at worker
+ * boot, before this process has taken any job, that is every job a previous
+ * process was running when it stopped.
+ *
+ * Why it matters (#1031): a worker killed for memory writes nothing, and
+ * pg-boss only marks its job `job timed out` once the expiry has passed,
+ * which for the backup is two hours later. Reading these rows at boot turns
+ * a silent restart into a line naming what was cut off. With more than one
+ * worker process on the same database, a job another live worker is running
+ * would show up here too, which the boot line says.
+ *
+ * `null` when the queue schema is absent, like the readers above.
+ */
+export async function readActiveJobsStartedBefore(
+  before: Date,
+): Promise<ActiveJob[] | null> {
+  try {
+    const rows = await prisma.$queryRaw<
+      Array<{
+        name: string;
+        id: string;
+        started_on: Date | null;
+        expire_seconds: number;
+      }>
+    >`
+      SELECT name, id::text AS id, started_on, expire_seconds
+      FROM pgboss.job
+      WHERE state = 'active' AND started_on < ${before}
+      ORDER BY started_on
+      LIMIT ${MAX_QUEUES}
+    `;
+    return rows
+      .filter((row) => row.started_on !== null)
+      .map((row) => ({
+        queue: row.name,
+        id: row.id,
+        startedAt: row.started_on!.toISOString(),
+        expireSeconds: row.expire_seconds,
+      }));
+  } catch {
+    return null;
+  }
+}

src/lib/jobs/job-observer.ts

@@ -0,0 +1,211 @@
+/**
+ * Log lines for long-running and cut-off jobs.
+ *
+ * Issue #1031: on a large account three queues failed night after night with
+ * `job timed out`, and the operator found not one line about any of them in
+ * the app log. Nothing wrote one. Most handlers annotate a wide event that
+ * nobody emits, pg-boss records an expiry only in its own table, and a worker
+ * killed for memory cannot say anything at all. The admin card showed the
+ * failure; the log showed nothing about what the job was doing, how far it
+ * got, or why it stopped.
+ *
+ * `observeJob` wraps every handler (through `runJob`) and writes three kinds
+ * of line, each a background wide event naming the queue:
+ *
+ *   - `job.progress`, every two minutes while a handler runs, with how long it
+ *     has been running, the process's memory, and whatever the handler last
+ *     reported through `reportJobProgress`. A job that runs for a minute never
+ *     writes one; a job that is killed leaves its last one behind.
+ *   - `job.expired`, at the job's expiry if the handler has not settled: the
+ *     moment pg-boss declares it timed out. pg-boss does not stop the handler,
+ *     so this line says it is still running and how far it has got.
+ *   - `job.finished`, when a handler that ran long or past its expiry
+ *     settles, with its outcome and the facts it returned.
+ *
+ * `reportOrphanedJobs` covers the case no running handler can: at worker boot,
+ * jobs still marked active were started by a process that is gone.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+
+import { WideEventBuilder } from "@/lib/logging/event-builder";
+import { emitIfSampled } from "@/lib/logging/transports";
+import { readActiveJobsStartedBefore } from "@/lib/jobs/job-failures";
+
+/** How often a running job writes `job.progress`. */
+export const JOB_PROGRESS_INTERVAL_MS = 2 * 60 * 1000;
+
+/** A handler that ran at least this long writes `job.finished`. */
+export const JOB_LONG_RUN_MS = 60 * 1000;
+
+type ProgressFacts = Record<string, string | number | boolean>;
+
+const progressStore = new AsyncLocalStorage<{ facts: ProgressFacts }>();
+
+/**
+ * Record how far the running job has got. Cheap: it only replaces the facts
+ * the next `job.progress` / `job.expired` line carries, so a pass can call it
+ * once per unit of work. A no-op outside an observed job.
+ */
+export function reportJobProgress(facts: ProgressFacts): void {
+  const cell = progressStore.getStore();
+  if (cell) cell.facts = { ...cell.facts, ...facts };
+}
+
+/** What `observeJob` reads from a pg-boss job. */
+interface ObservedJob {
+  id?: string;
+  expireInSeconds?: number;
+}
+
+/** Injectable clock and timers, for tests. */
+export interface JobObserverClock {
+  now: () => number;
+  setTimeout: (fn: () => void, ms: number) => unknown;
+  setInterval: (fn: () => void, ms: number) => unknown;
+  clearTimeout: (handle: unknown) => void;
+  clearInterval: (handle: unknown) => void;
+}
+
+const systemClock: JobObserverClock = {
+  now: () => Date.now(),
+  setTimeout: (fn, ms) => setTimeout(fn, ms).unref(),
+  setInterval: (fn, ms) => setInterval(fn, ms).unref(),
+  clearTimeout: (handle) => clearTimeout(handle as NodeJS.Timeout),
+  clearInterval: (handle) => clearInterval(handle as NodeJS.Timeout),
+};
+
+function megabytes(bytes: number): number {
+  return Math.round(bytes / 1024 / 1024);
+}
+
+function emitJobLine(
+  taskName: string,
+  level: "info" | "warn",
+  meta: Record<string, unknown>,
+): void {
+  try {
+    const event = new WideEventBuilder("background");
+    event.setBackground({ task_name: taskName });
+    for (const [key, value] of Object.entries(meta)) event.addMeta(key, value);
+    if (level === "warn") event.elevateLevel("warn");
+    event.finish();
+    emitIfSampled(event.toJSON());
+  } catch {
+    // A log line must never break the job it describes.
+  }
+}
+
+/**
+ * Run `run` for `jobs` of `queue`, writing the progress, expiry and finish
+ * lines described above. Resolves or rejects exactly as `run` does.
+ */
+export async function observeJob<R>(
+  queue: string,
+  jobs: readonly ObservedJob[],
+  run: () => Promise<R>,
+  clock: JobObserverClock = systemClock,
+): Promise<R> {
+  const startedAt = clock.now();
+  const cell: { facts: ProgressFacts } = { facts: {} };
+  const jobIds = jobs
+    .map((job) => job.id)
+    .filter(Boolean)
+    .join(",");
+  const expireSeconds = Math.min(
+    ...jobs.map((job) =>
+      typeof job.expireInSeconds === "number" && job.expireInSeconds > 0
+        ? job.expireInSeconds
+        : Number.POSITIVE_INFINITY,
+    ),
+  );
+  let expired = false;
+
+  const snapshot = () => {
+    const memory = process.memoryUsage();
+    return {
+      queue,
+      job_ids: jobIds,
+      elapsed_s: Math.round((clock.now() - startedAt) / 1000),
+      ...(Number.isFinite(expireSeconds) ? { expire_s: expireSeconds } : {}),
+      heap_used_mb: megabytes(memory.heapUsed),
+      rss_mb: megabytes(memory.rss),
+      ...cell.facts,
+    };
+  };
+
+  const heartbeat = clock.setInterval(() => {
+    emitJobLine("job.progress", "info", snapshot());
+  }, JOB_PROGRESS_INTERVAL_MS);
+  const expiry = Number.isFinite(expireSeconds)
+    ? clock.setTimeout(() => {
+        expired = true;
+        emitJobLine("job.expired", "warn", {
+          ...snapshot(),
+          reason:
+            "pg-boss has marked this job timed out; the handler is still running",
+        });
+      }, expireSeconds * 1000)
+    : null;
+
+  const finish = (outcome: Record<string, unknown>) => {
+    clock.clearInterval(heartbeat);
+    if (expiry !== null) clock.clearTimeout(expiry);
+    const elapsed = clock.now() - startedAt;
+    if (!expired && elapsed < JOB_LONG_RUN_MS) return;
+    emitJobLine("job.finished", expired ? "warn" : "info", {
+      ...snapshot(),
+      ...(expired ? { finished_after_expiry: true } : {}),
+      ...outcome,
+    });
+  };
+
+  try {
+    const result = await progressStore.run(cell, run);
+    const did =
+      result !== null && typeof result === "object" && "did" in result
+        ? (result as { did?: unknown }).did
+        : undefined;
+    const ok =
+      result !== null && typeof result === "object" && "ok" in result
+        ? (result as { ok?: unknown }).ok !== false
+        : true;
+    finish({
+      outcome: ok ? "completed" : "failed",
+      ...(did && typeof did === "object" ? { did } : {}),
+    });
+    return result;
+  } catch (err) {
+    finish({
+      outcome: "threw",
+      error: err instanceof Error ? err.message : String(err),
+    });
+    throw err;
+  }
+}
+
+/**
+ * At worker boot, write one line naming every job a previous process was
+ * running when it stopped. Must run before this process takes any job.
+ *
+ * A worker killed for memory cannot log its own death, and pg-boss marks the
+ * job `job timed out` only once the expiry has passed. This line is where a
+ * restart in the middle of a job becomes visible: the queue, when the job
+ * started, and its limit. Silent when nothing was cut off.
+ */
+export async function reportJobsCutOffAtBoot(
+  bootedAt: Date = new Date(),
+): Promise<void> {
+  const active = await readActiveJobsStartedBefore(bootedAt);
+  if (active === null || active.length === 0) return;
+  emitJobLine("worker.boot.jobs_cut_off", "warn", {
+    jobs_cut_off: active.length,
+    queues: [...new Set(active.map((job) => job.queue))].join(","),
+    jobs: active
+      .map(
+        (job) => `${job.queue}@${job.startedAt}(limit ${job.expireSeconds}s)`,
+      )
+      .join(" "),
+    reason:
+      "these jobs were still running when the previous worker process stopped (a restart, a deploy, or the container running out of memory); pg-boss retries or fails them once their limit passes. If another worker process shares this database, its running jobs are listed too.",
+  });
+}

src/lib/jobs/job-outcome.ts

@@ -55,6 +55,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "connections_deleted",
   "considered",
   "context_surfaced",
+  "continuation",
+  "continued",
   "created",
   "daily_rows_retired",
   "days_consolidated",
@@ -136,6 +138,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "notifications_dispatched",
   "notifications_failed",
   "notified",
+  "offhost_backup_already_uploaded",
   "offhost_backup_configured",
   "offhost_backup_failed",
   "offhost_backup_oversized",
@@ -200,6 +203,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "skipped_outside_window",
   "slots_collapsed",
   "states_deleted",
+  "stopped_early",
   "stored",
   "subscription_repair_failed",
   "summaries_enqueued",

src/lib/jobs/mean-consolidation.ts

@@ -44,9 +44,15 @@ export interface MeanConsolidationPayload {
  */
 export async function runMeanConsolidationForUser(
   userId: string,
-): Promise<{ daysConsolidated: number; perSampleRowsSoftDeleted: number }> {
+  shouldStop?: () => boolean,
+): Promise<{
+  daysConsolidated: number;
+  perSampleRowsSoftDeleted: number;
+  stoppedEarly: boolean;
+}> {
   const summary = await consolidateDailyMean(prisma, {
     userId,
+    shouldStop,
     cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS,
     log: () => {
       // Silent inside the queue handler — the worker logs the totals.
@@ -59,12 +65,14 @@ export async function runMeanConsolidationForUser(
         days: summary.totals.daysConsolidated,
         per_sample_rows_soft_deleted: summary.totals.perSampleRowsSoftDeleted,
         daily_rows_upserted: summary.totals.dailyRowsUpserted,
+        stopped_early: summary.stoppedEarly,
       },
     },
   });
   return {
     daysConsolidated: summary.totals.daysConsolidated,
     perSampleRowsSoftDeleted: summary.totals.perSampleRowsSoftDeleted,
+    stoppedEarly: summary.stoppedEarly,
   };
 }
 

src/lib/jobs/offhost-backup.ts

@@ -23,8 +23,8 @@
  */
 import { Buffer } from "node:buffer";
 import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
-import { Transform, type Readable } from "node:stream";
-import { createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { Readable, Transform } from "node:stream";
+import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
 import type { PrismaClient } from "@/generated/prisma/client";
 import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto";
 import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
@@ -193,6 +193,48 @@ export function decryptBackup(buf: Buffer, key: Buffer): string {
     : plaintext.toString("utf8");
 }
 
+/**
+ * An off-host object → a source of its JSON as byte chunks, openable as often
+ * as needed. `decryptBackup` answers one string, which the JSON of a large
+ * record cannot be: 662 MB for an account of 1.25 million measurements, past
+ * the longest string V8 allows (#1031). The tag is verified whole on the
+ * compressed ciphertext before a source exists, exactly as `decryptBackup`
+ * does; only the decompression is streamed.
+ */
+export function openBackupObject(
+  buf: Buffer,
+  key: Buffer,
+): () => AsyncIterable<Buffer> {
+  const magic = buf.subarray(0, 4).toString("binary");
+  const version = buf[PREAMBLE_LENGTH - 1];
+  if (
+    magic !== MAGIC ||
+    (version !== BACKUP_ENVELOPE_PLAIN &&
+      version !== BACKUP_ENVELOPE_GZIP &&
+      version !== BACKUP_ENVELOPE_STREAM)
+  ) {
+    throw new Error("Invalid backup envelope (bad magic or version)");
+  }
+  let plaintext: Buffer;
+  if (version === BACKUP_ENVELOPE_STREAM) {
+    plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key);
+  } else {
+    const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH);
+    const tag = buf.subarray(
+      PREAMBLE_LENGTH + IV_LENGTH,
+      PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH,
+    );
+    const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH);
+    const dec = createDecipheriv(ALGORITHM, key, iv);
+    dec.setAuthTag(tag);
+    plaintext = Buffer.concat([dec.update(ct), dec.final()]);
+  }
+  if (version === BACKUP_ENVELOPE_PLAIN) {
+    return () => Readable.from([plaintext]);
+  }
+  return () => Readable.from([plaintext]).pipe(createGunzip());
+}
+
 /**
  * How much of one object the upload holds at a time, and how many of those
  * windows are in flight. `@aws-sdk/lib-storage` buffers `partSize` bytes per
@@ -551,9 +593,33 @@ interface BackupRunReport {
   largestObjectBytes: number;
   /** Accounts refused for size rather than failed for a reason. */
   oversized: number;
+  /**
+   * Accounts an earlier attempt of the same run already uploaded, and this
+   * one therefore skipped (see `RunOffhostBackupOptions.runStartedAt`).
+   */
+  alreadyUploaded: number;
+  /** `shouldStop` ended the run before every account was reached. */
+  stoppedEarly: boolean;
 }
 
-export type RunOffhostBackupOptions = UploadBackupOptions;
+export interface RunOffhostBackupOptions extends UploadBackupOptions {
+  /**
+   * When this run first started: the pg-boss job's creation time, the same
+   * on every attempt of the job. An attempt skips each account whose ledger
+   * shows a successful upload since then, because an earlier attempt of the
+   * same run already put its object in the bucket, and the objects are keyed
+   * on this date so a retry after midnight still writes under the run's day.
+   * Without it a retry started again from the first account (#1031), and on
+   * a cohort with one large record the retry never got past the accounts the
+   * first attempt had already done.
+   */
+  runStartedAt?: Date;
+  /**
+   * Asked before each account. Returning `true` ends the run cleanly between
+   * accounts, before the job's expiry would cut one off halfway.
+   */
+  shouldStop?: () => boolean;
+}
 
 export async function runOffhostBackup(
   prisma: PrismaClient,
@@ -568,9 +634,23 @@ export async function runOffhostBackup(
     );
   }
   const s3 = s3Override ?? (await getS3Client(cfg));
-  const dateKey = now.toISOString().slice(0, 10);
+  const runStartedAt = options.runStartedAt;
+  const dateKey = (runStartedAt ?? now).toISOString().slice(0, 10);
 
-  const users = await prisma.user.findMany({ select: { id: true } });
+  const users = await prisma.user.findMany({
+    select: { id: true },
+    orderBy: { id: "asc" },
+  });
+  const doneThisRun = new Set<string>();
+  if (runStartedAt) {
+    const done = await prisma.offhostBackupState.findMany({
+      where: { lastSuccessAt: { gte: runStartedAt } },
+      select: { userId: true },
+    });
+    for (const row of done) doneThisRun.add(row.userId);
+  }
+  let alreadyUploaded = 0;
+  let stoppedEarly = false;
   let uploaded = 0;
   let failed = 0;
   let oversized = 0;
@@ -579,6 +659,14 @@ export async function runOffhostBackup(
   let ledgerWriteFailures = 0;
   const evt = getEvent();
   for (const user of users) {
+    if (doneThisRun.has(user.id)) {
+      alreadyUploaded++;
+      continue;
+    }
+    if (options.shouldStop?.()) {
+      stoppedEarly = true;
+      break;
+    }
     let objectBytes: number | null = null;
     try {
       objectBytes = await uploadEncryptedBackup(
@@ -679,6 +767,8 @@ export async function runOffhostBackup(
     totalUsers: users.length,
     largestObjectBytes,
     oversized,
+    alreadyUploaded,
+    stoppedEarly,
   };
 }
 

src/lib/jobs/reminder-worker.ts

@@ -16,6 +16,7 @@
  */
 import { markWorkerStarted, recordError } from "@/lib/jobs/worker-status";
 import { createWorkerBoss, setGlobalBoss } from "@/lib/jobs/boss-instance";
+import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer";
 import { assertSubsystemEnabled } from "@/lib/process-type";
 import { DATABASE_URL, workerLog } from "./reminder/shared";
 import {
@@ -55,7 +56,11 @@ export async function startReminderWorker() {
     recordError();
   });
 
+  // Read before any `boss.work` binding below can take a job: whatever is
+  // still active now was started by a process that is gone.
+  const bootedAt = new Date();
   await boss.start();
+  await reportJobsCutOffAtBoot(bootedAt);
   setGlobalBoss(boss);
   markWorkerStarted();
 

src/lib/jobs/reminder/__tests__/backup-handlers.test.ts

@@ -4,9 +4,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => ({
   buildFullBackupPayload: vi.fn(),
-  packBlob: vi.fn((value: string) => value),
   getWorkerPrisma: vi.fn(),
-  upsert: vi.fn(),
+  store: vi.fn(),
 }));
 
 // Only the payload builder is stubbed. The REAL streaming writer runs on top
@@ -19,27 +18,25 @@ vi.mock("@/lib/export/full-backup-payload", () => ({
   isDeferredRows: () => false,
 }));
 
-// The envelope is exercised end-to-end in
-// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so
-// the stored bytes can be read back as JSON. The size error is the real class
-// so the handler's `instanceof` arm is the one that runs.
-vi.mock("@/lib/export/backup-blob", async () => {
-  const actual = await vi.importActual<
-    typeof import("@/lib/export/backup-blob")
-  >("@/lib/export/backup-blob");
-  return {
-    BackupBlobTooLargeError: actual.BackupBlobTooLargeError,
-    packBackupBlobStreaming: async (
-      produce: (write: (chunk: string) => Promise<void>) => Promise<void>,
-    ) => {
-      let out = "";
-      await produce(async (chunk) => {
-        out += chunk;
-      });
-      return mocks.packBlob(out);
-    },
-  };
-});
+// The envelope and the piecewise store are exercised end-to-end in
+// `src/lib/export/__tests__/backup-blob.test.ts` and the integration suite;
+// here the store stays transparent so the stored bytes can be read back as
+// JSON. `mocks.store` receives what would have been stored and decides
+// whether the write succeeds.
+vi.mock("@/lib/export/store-backup-blob", () => ({
+  storeBackupBlob: async (
+    _prisma: unknown,
+    input: { userId: string; type: string },
+    produce: (write: (chunk: string) => Promise<void>) => Promise<void>,
+  ) => {
+    let out = "";
+    await produce(async (chunk) => {
+      out += chunk;
+    });
+    await mocks.store(input, out);
+    return { id: "backup-row", bytes: out.length };
+  },
+}));
 
 vi.mock("@/lib/logging/background", () => ({
   withBackgroundEvent: vi.fn(
@@ -70,7 +67,6 @@ function buildPrismaMock(
 ) {
   return {
     user: { findMany: vi.fn().mockResolvedValue(users) },
-    dataBackup: { upsert: mocks.upsert },
   };
 }
 
@@ -78,7 +74,7 @@ describe("handleDataBackup canonical DR payload", () => {
   beforeEach(() => {
     vi.clearAllMocks();
     mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock());
-    mocks.upsert.mockResolvedValue({});
+    mocks.store.mockResolvedValue(undefined);
     mocks.buildFullBackupPayload.mockResolvedValue({
       payload: {
         schemaVersion: "1",
@@ -117,8 +113,12 @@ describe("handleDataBackup canonical DR payload", () => {
         deferBulk: true,
       }),
     );
-    expect(mocks.upsert).toHaveBeenCalledOnce();
-    const encrypted = mocks.upsert.mock.calls[0]![0].create.data as string;
+    expect(mocks.store).toHaveBeenCalledOnce();
+    expect(mocks.store.mock.calls[0]![0]).toEqual({
+      userId: "user-dr",
+      type: "WEEKLY_AUTO",
+    });
+    const encrypted = mocks.store.mock.calls[0]![1] as string;
     const payload = JSON.parse(encrypted) as {
       moodEntries: Array<{
         id: string;
@@ -155,7 +155,7 @@ describe("handleDataBackup canonical DR payload", () => {
 describe("handleDataBackup outcome", () => {
   beforeEach(() => {
     vi.clearAllMocks();
-    mocks.upsert.mockResolvedValue({});
+    mocks.store.mockResolvedValue(undefined);
     mocks.buildFullBackupPayload.mockResolvedValue({
       payload: { schemaVersion: "1", userId: "user-dr" },
       counts: {},
@@ -164,7 +164,7 @@ describe("handleDataBackup outcome", () => {
 
   it("fails the run when not one account got a copy", async () => {
     mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock());
-    mocks.upsert.mockRejectedValue(new Error("write failed"));
+    mocks.store.mockRejectedValue(new Error("write failed"));
 
     const outcome = await handleDataBackup([]);
 
@@ -178,7 +178,7 @@ describe("handleDataBackup outcome", () => {
 
   it("counts an oversized record as the reason it could not", async () => {
     mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock());
-    mocks.upsert.mockRejectedValue(new BackupBlobTooLargeError(9_000, 4_096));
+    mocks.store.mockRejectedValue(new BackupBlobTooLargeError(9_000, 4_096));
 
     const outcome = await handleDataBackup([]);
 
@@ -198,9 +198,9 @@ describe("handleDataBackup outcome", () => {
         { id: "user-b", username: "b" },
       ]),
     );
-    mocks.upsert
+    mocks.store
       .mockRejectedValueOnce(new Error("write failed"))
-      .mockResolvedValueOnce({});
+      .mockResolvedValueOnce(undefined);
 
     const outcome = await handleDataBackup([]);
 

src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts

@@ -53,6 +53,8 @@ function report(over: Partial<Record<string, unknown>> = {}) {
     totalUsers: 2,
     largestObjectBytes: 9_000,
     oversized: 0,
+    alreadyUploaded: 0,
+    stoppedEarly: false,
     ...over,
   };
 }
@@ -67,6 +69,8 @@ describe("handleOffhostBackup", () => {
     const outcome = await handleOffhostBackup([]);
     expect(outcome.ok).toBe(true);
     expect(outcome.ok && outcome.did).toEqual({
+      offhost_backup_already_uploaded: 0,
+      stopped_early: false,
       offhost_backup_uploaded: 2,
       offhost_backup_failed: 0,
       offhost_backup_total_users: 2,
@@ -138,4 +142,23 @@ describe("handleOffhostBackup", () => {
       offhost_backup_configured: false,
     });
   });
+
+  it("fails an attempt that ran out of budget, so pg-boss retries and the retry resumes (#1031)", async () => {
+    mocks.runOffhostBackup.mockResolvedValue(
+      report({ uploaded: 1, totalUsers: 3, stoppedEarly: true }),
+    );
+    const outcome = await handleOffhostBackup([]);
+    expect(outcome).toMatchObject({
+      ok: false,
+      reason: expect.stringContaining("retry resumes"),
+    });
+  });
+
+  it("passes a retry whose accounts an earlier attempt had all uploaded", async () => {
+    mocks.runOffhostBackup.mockResolvedValue(
+      report({ uploaded: 0, alreadyUploaded: 2, totalUsers: 2 }),
+    );
+    const outcome = await handleOffhostBackup([]);
+    expect(outcome.ok).toBe(true);
+  });
 });

src/lib/jobs/reminder/backup-handlers.ts

@@ -4,12 +4,11 @@
  * Extracted from reminder-worker.ts, which owns the queue names, cron
  * schedules, and boss.work registrations.
  */
-import { type Job } from "pg-boss";
+import { type Job, type JobWithMetadata } from "pg-boss";
+import { jobBudget } from "@/lib/jobs/job-budget";
 import { recordError } from "@/lib/jobs/worker-status";
-import {
-  BackupBlobTooLargeError,
-  packBackupBlobStreaming,
-} from "@/lib/export/backup-blob";
+import { BackupBlobTooLargeError } from "@/lib/export/backup-blob";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
 import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
 import { withBackgroundEvent } from "@/lib/logging/background";
@@ -18,6 +17,7 @@ import {
   runOffhostBackup,
 } from "@/lib/jobs/offhost-backup";
 import { getWorkerPrisma } from "./shared";
+import { reportJobProgress } from "@/lib/jobs/job-observer";
 
 export interface DataBackupPayload {
   triggeredAt: string;
@@ -28,13 +28,18 @@ export interface OffhostBackupPayload {
 }
 
 export async function handleOffhostBackup(
-  jobs: Job<OffhostBackupPayload>[],
+  jobs: JobWithMetadata<OffhostBackupPayload>[],
 ): Promise<JobOutcome> {
-  void jobs;
   return withBackgroundEvent("job.offhost_backup", async (evt) => {
     const p = getWorkerPrisma();
     try {
-      const report = await runOffhostBackup(p);
+      // A retry of this job carries the same creation time, so it resumes
+      // after the accounts an earlier attempt uploaded, and the run stops
+      // between accounts before the job's expiry would cut one off.
+      const report = await runOffhostBackup(p, undefined, new Date(), {
+        runStartedAt: jobs[0]?.createdOn,
+        shouldStop: jobBudget(jobs),
+      });
       evt.addMeta("offhost_backup_uploaded", report.uploaded);
       evt.addMeta("offhost_backup_failed", report.failed);
       evt.addMeta("offhost_backup_total_users", report.totalUsers);
@@ -56,6 +61,8 @@ export async function handleOffhostBackup(
       }
 
       const did = {
+        offhost_backup_already_uploaded: report.alreadyUploaded,
+        stopped_early: report.stoppedEarly,
         offhost_backup_uploaded: report.uploaded,
         offhost_backup_failed: report.failed,
         offhost_backup_total_users: report.totalUsers,
@@ -70,7 +77,20 @@ export async function handleOffhostBackup(
       // than one. Per-account failures still ride out as counts when SOME
       // account got a copy: that is the fan-out rule, and retrying the whole
       // cohort over one object would re-upload everybody's.
-      if (report.totalUsers > 0 && report.uploaded === 0) {
+      // Out of budget before every account: fail, so pg-boss retries the job,
+      // and the retry resumes after the accounts this attempt uploaded.
+      if (report.stoppedEarly) {
+        return jobFailed(
+          "stopped before every account was uploaded; the retry resumes",
+          undefined,
+          did,
+        );
+      }
+
+      if (
+        report.totalUsers > 0 &&
+        report.uploaded + report.alreadyUploaded === 0
+      ) {
         // The SDK's own words, not a stack: `runJob` puts the cause message in
         // the reported meta, and "SignatureDoesNotMatch" is the sentence an
         // operator can act on.
@@ -118,38 +138,30 @@ export async function handleDataBackup(
       // six weeks in.
       let lastError: unknown;
       for (const user of users) {
+        reportJobProgress({
+          backup_user: user.id,
+          backup_users_done: backed + usersFailed,
+          backup_users_total: users.length,
+        });
         try {
           // Streamed, compressed, then encrypted (the record contains
-          // sensitive health information). Nothing between the database rows
-          // and this string exists as a whole: the payload goes into gzip a
-          // page at a time and the cipher consumes gzip's output as it comes,
-          // because a materialised copy of a large record is what used to take
-          // the process down. Both legs live in `packBackupBlobStreaming`,
-          // which is also what the restore path reads back through.
-          const encryptedBackup = await packBackupBlobStreaming((write) =>
-            streamFullBackupJson(prisma, user.id, write, {
-              purpose: "disaster-recovery",
-            }),
+          // sensitive health information), and stored a piece at a time.
+          // Nothing between the database rows and the stored row exists as a
+          // whole in this process: the payload goes into gzip a page at a
+          // time, the cipher consumes gzip's output as it comes, and the
+          // ciphertext goes back to Postgres in pieces that one statement
+          // assembles into the row (`storeBackupBlob`). The stored copy used
+          // to be built here as one string first, which on a large record
+          // added several hundred megabytes to a container capped at 1 GB.
+          const { bytes: storedBytes } = await storeBackupBlob(
+            prisma,
+            { userId: user.id, type: "WEEKLY_AUTO" },
+            (write) =>
+              streamFullBackupJson(prisma, user.id, write, {
+                purpose: "disaster-recovery",
+              }),
           );
-          largestBlobBytes = Math.max(
-            largestBlobBytes,
-            Buffer.byteLength(encryptedBackup, "utf8"),
-          );
-
-          await prisma.dataBackup.upsert({
-            where: {
-              userId_type: { userId: user.id, type: "WEEKLY_AUTO" },
-            },
-            update: {
-              data: encryptedBackup,
-              createdAt: new Date(),
-            },
-            create: {
-              userId: user.id,
-              type: "WEEKLY_AUTO",
-              data: encryptedBackup,
-            },
-          });
+          largestBlobBytes = Math.max(largestBlobBytes, storedBytes);
           backed++;
         } catch (err) {
           // One user's payload failing is that user's problem, not the pass's:

src/lib/jobs/reminder/register-maintenance.ts

@@ -724,10 +724,11 @@ export async function registerMaintenanceQueues(
     { localConcurrency: 1 },
     handleStepUpElevationCleanup,
   );
+  // With metadata: the handler resumes a retry from the job's creation time.
   await createAndWork<OffhostBackupPayload>(
     boss,
     OFFHOST_BACKUP_QUEUE,
-    { localConcurrency: 1 },
+    { localConcurrency: 1, includeMetadata: true },
     handleOffhostBackup,
   );
   await createAndWork(

src/lib/jobs/reminder/register-rollup.ts

@@ -110,6 +110,11 @@ import {
   type ScheduleEntry,
 } from "./registrar-shared";
 import { jobDone, jobFailed } from "@/lib/jobs/job-outcome";
+import { jobBudget } from "@/lib/jobs/job-budget";
+import {
+  DRAIN_CONTINUATION_DELAY_SECONDS,
+  nextDrainContinuation,
+} from "@/lib/jobs/drain-continuation";
 // v1.4.37 W7c — nightly drain of per-sample APPLE_HEALTH cumulative rows.
 // Collapses each user × cumulative-type × calendar-day bucket into one
 // `stats:…` row so the list view stops painting hundreds of step chunks per
@@ -123,7 +128,12 @@ const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative";
 const DRAIN_CUMULATIVE_CRON = "45 3 * * *";
 
 interface DrainCumulativePayload {
-  triggeredAt: string;
+  triggeredAt?: string;
+  /**
+   * How many times this tick has already handed itself on. Absent on the
+   * cron's own job. See `nextDrainContinuation`.
+   */
+  continuation?: number;
 }
 
 const allQueues = [
@@ -437,12 +447,15 @@ export async function registerRollupQueues(
     MEAN_CONSOLIDATION_QUEUE,
     { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY },
     async (jobs) => {
+      const shouldStop = jobBudget(jobs);
       let daysTotal = 0;
       let perSampleRowsTotal = 0;
+      let stoppedEarly = false;
       for (const job of jobs) {
         const { userId } = job.data;
-        const { daysConsolidated, perSampleRowsSoftDeleted } =
-          await runMeanConsolidationForUser(userId);
+        const summary = await runMeanConsolidationForUser(userId, shouldStop);
+        const { daysConsolidated, perSampleRowsSoftDeleted } = summary;
+        stoppedEarly ||= summary.stoppedEarly;
         daysTotal += daysConsolidated;
         perSampleRowsTotal += perSampleRowsSoftDeleted;
         workerLog(
@@ -450,10 +463,14 @@ export async function registerRollupQueues(
           `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`,
         );
       }
+      // Out of budget is a completed run, not a failed one: the folded days
+      // are committed, and the nightly tick carries on from the first day
+      // this run did not reach.
       return jobDone({
         users: jobs.length,
         days_consolidated: daysTotal,
         per_sample_rows_soft_deleted: perSampleRowsTotal,
+        stopped_early: stoppedEarly,
       });
     },
   );
@@ -470,17 +487,24 @@ export async function registerRollupQueues(
     DENSE_INTRADAY_RETENTION_QUEUE,
     { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY },
     async (jobs) => {
+      const shouldStop = jobBudget(jobs);
       let daysTotal = 0;
       let perSampleRowsTotal = 0;
       let derivedRestingRowsTotal = 0;
+      let stoppedEarly = false;
       for (const job of jobs) {
         const { userId } = job.data;
         try {
+          const summary = await runDenseIntradayRetentionForUser(
+            userId,
+            shouldStop,
+          );
           const {
             daysConsolidated,
             perSampleRowsSoftDeleted,
             derivedRestingRowsUpserted,
-          } = await runDenseIntradayRetentionForUser(userId);
+          } = summary;
+          stoppedEarly ||= summary.stoppedEarly;
           daysTotal += daysConsolidated;
           perSampleRowsTotal += perSampleRowsSoftDeleted;
           derivedRestingRowsTotal += derivedRestingRowsUpserted;
@@ -503,6 +527,10 @@ export async function registerRollupQueues(
         days_consolidated: daysTotal,
         per_sample_rows_soft_deleted: perSampleRowsTotal,
         derived_resting_rows_upserted: derivedRestingRowsTotal,
+        // Out of budget completes the job: the folded days are committed and
+        // the nightly tick, which hands itself on while work remains,
+        // continues from the first day this run did not reach.
+        stopped_early: stoppedEarly,
       });
     },
   );
@@ -519,6 +547,8 @@ export async function registerRollupQueues(
     DENSE_INTRADAY_HOURLY_REBUILD_QUEUE,
     { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY },
     async (jobs) => {
+      const shouldStop = jobBudget(jobs);
+      let stoppedEarly = false;
       let daysRebuiltTotal = 0;
       let hourlyRowsTotal = 0;
       let dailyRowsRetiredTotal = 0;
@@ -526,12 +556,17 @@ export async function registerRollupQueues(
       for (const job of jobs) {
         const { userId } = job.data;
         try {
+          const summary = await runDenseIntradayHourlyRebuildForUser(
+            userId,
+            shouldStop,
+          );
           const {
             daysRebuilt,
             hourlyRowsUpserted,
             dailyRowsRetired,
             daysSkippedNoTombstones,
-          } = await runDenseIntradayHourlyRebuildForUser(userId);
+          } = summary;
+          stoppedEarly ||= summary.stoppedEarly;
           daysRebuiltTotal += daysRebuilt;
           hourlyRowsTotal += hourlyRowsUpserted;
           dailyRowsRetiredTotal += dailyRowsRetired;
@@ -550,13 +585,25 @@ export async function registerRollupQueues(
           throw err;
         }
       }
-      return jobDone({
+      const did = {
         users: jobs.length,
+        stopped_early: stoppedEarly,
         days_rebuilt: daysRebuiltTotal,
         hourly_rows_upserted: hourlyRowsTotal,
         daily_rows_retired: dailyRowsRetiredTotal,
         days_skipped_no_tombstones: daysSkippedTotal,
-      });
+      };
+      // Out of budget: fail so pg-boss retries (the boot send allows three
+      // retries), and the retry starts at the first day not yet rebuilt. The
+      // queue is `exclusive`, so the job cannot hand itself on while active.
+      if (stoppedEarly) {
+        return jobFailed(
+          "stopped before every day was rebuilt; the retry resumes",
+          undefined,
+          did,
+        );
+      }
+      return jobDone(did);
     },
   );
 
@@ -679,6 +726,9 @@ export async function registerRollupQueues(
     DRAIN_CUMULATIVE_QUEUE,
     { localConcurrency: 1 },
     async (jobs) => {
+      // One budget shared by all three passes, run in turn.
+      const shouldStop = jobBudget(jobs);
+      let stoppedEarly = false;
       // The three drains below are independent passes sharing one nightly
       // tick, so one failing pass must not stop the other two from running.
       // The first failure is held back and reported once the tick is over —
@@ -699,8 +749,10 @@ export async function registerRollupQueues(
           const summary = await drainPerSampleCumulative(getWorkerPrisma(), {
             dryRun: false,
             cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS,
+            shouldStop,
             log: (line) => workerLog("info", line),
           });
+          stoppedEarly ||= summary.stoppedEarly;
           bucketsCollapsed += summary.totals.bucketsCollapsed;
           perSampleRowsDeleted += summary.totals.perSampleRowsDeleted;
           workerLog(
@@ -729,8 +781,10 @@ export async function registerRollupQueues(
           const meanSummary = await consolidateDailyMean(getWorkerPrisma(), {
             dryRun: false,
             cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS,
+            shouldStop,
             log: (line) => workerLog("info", line),
           });
+          stoppedEarly ||= meanSummary.stoppedEarly;
           meanDaysConsolidated += meanSummary.totals.daysConsolidated;
           meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted;
           workerLog(
@@ -769,9 +823,11 @@ export async function registerRollupQueues(
               getWorkerPrisma(),
               {
                 dryRun: false,
+                shouldStop,
                 log: (line) => workerLog("info", line),
               },
             );
+            stoppedEarly ||= denseSummary.stoppedEarly;
             denseDaysConsolidated += denseSummary.totals.daysConsolidated;
             denseRowsSoftDeleted +=
               denseSummary.totals.perSampleRowsSoftDeleted;
@@ -790,7 +846,41 @@ export async function registerRollupQueues(
         }
       }
 
+      // Hand the remaining work to a follow-up job when this one ran out of
+      // budget (bounded; see `nextDrainContinuation`).
+      const continuation = Math.max(
+        0,
+        ...jobs.map((job) => job.data.continuation ?? 0),
+      );
+      const next = nextDrainContinuation({
+        stoppedEarly,
+        daysFolded:
+          bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated,
+        continuation,
+      });
+      let continued = false;
+      if (next !== null) {
+        try {
+          const followUp: DrainCumulativePayload = {
+            triggeredAt: new Date().toISOString(),
+            continuation: next,
+          };
+          continued =
+            (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, {
+              ...cronIsTheRetry,
+              startAfter: DRAIN_CONTINUATION_DELAY_SECONDS,
+            })) !== null;
+        } catch (err) {
+          // The next nightly tick picks the work up regardless; a lost
+          // follow-up only delays it.
+          workerLog("error", "[drain-cumulative] follow-up send failed", err);
+        }
+      }
+
       const did = {
+        stopped_early: stoppedEarly,
+        continuation,
+        continued,
         passes_failed: passesFailed,
         buckets_collapsed: bucketsCollapsed,
         per_sample_rows_deleted: perSampleRowsDeleted,

src/lib/jobs/restore-drill.ts

@@ -22,8 +22,9 @@
  * no other cron uses.
  */
 import type { Job } from "pg-boss";
+import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan";
 import {
-  decryptBackup,
+  openBackupObject,
   getS3Client,
   loadOffhostConfig,
   OffhostBackupNotConfiguredError,
@@ -103,19 +104,36 @@ export async function runRestoreDrill(
   const dateKey = BACKUP_KEY_PATTERN.exec(objectKey)![1];
 
   const ciphertext = await s3.getObject(objectKey);
-  const plaintext = decryptBackup(ciphertext, cfg.encryptionKey);
-  const parsed: unknown = JSON.parse(plaintext);
-
-  if (typeof parsed !== "object" || parsed === null) {
-    throw new Error(
-      `Restore drill: backup object "${objectKey}" decrypted and parsed but is not a JSON object.`,
-    );
+  // Read as a stream, never as one string: the JSON of a large record is
+  // longer than any string V8 can hold (#1031). The bulk tables are counted,
+  // not kept.
+  const source = openBackupObject(ciphertext, cfg.encryptionKey);
+  let plaintextBytes = 0;
+  async function* counted() {
+    for await (const chunk of source()) {
+      plaintextBytes += chunk.byteLength;
+      yield chunk;
+    }
   }
-  const payload = parsed as Record<string, unknown>;
+  let scanned;
+  try {
+    scanned = await scanBackupJson(counted(), {
+      streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]),
+    });
+  } catch (err) {
+    if (err instanceof BackupJsonError) {
+      throw new Error(
+        `Restore drill: backup object "${objectKey}" decrypted but is not a JSON object: ${err.message}`,
+      );
+    }
+    throw err;
+  }
+  const payload = scanned.document;
+  const streamedCounts = scanned.streamedCounts;
   if (
     typeof payload.exportedAt !== "string" ||
     typeof payload.userId !== "string" ||
-    !Array.isArray(payload.measurements)
+    streamedCounts.measurements === undefined
   ) {
     throw new Error(
       `Restore drill: backup object "${objectKey}" parses but is missing core fields (exportedAt / userId / measurements).`,
@@ -133,12 +151,12 @@ export async function runRestoreDrill(
     ageDays,
     stale: ageDays > MAX_BACKUP_AGE_DAYS,
     ciphertextBytes: ciphertext.length,
-    plaintextBytes: Buffer.byteLength(plaintext, "utf8"),
+    plaintextBytes,
     recordCounts: {
-      measurements: payload.measurements.length,
+      measurements: streamedCounts.measurements,
       medications: countArray(payload.medications),
-      intakeEvents: countArray(payload.intakeEvents),
-      moodEntries: countArray(payload.moodEntries),
+      intakeEvents: streamedCounts.intakeEvents ?? 0,
+      moodEntries: streamedCounts.moodEntries ?? 0,
     },
   };
 }

src/lib/jobs/run-job.ts

@@ -28,6 +28,7 @@ import {
   type SerializedJobOutcome,
 } from "./job-outcome";
 import { reportWorkerError } from "./report-worker-error";
+import { observeJob } from "./job-observer";
 
 /** A converted handler: it says what it did instead of just finishing. */
 export type JobHandler<T> = (jobs: Job<T>[]) => Promise<JobOutcome>;
@@ -80,7 +81,13 @@ export function runJob<J>(
   handler: (jobs: J[]) => Promise<JobOutcome>,
 ): (jobs: J[]) => Promise<SerializedJobOutcome> {
   return async (jobs: J[]): Promise<SerializedJobOutcome> => {
-    const outcome = await handler(jobs);
+    // Observed, so a long run, an expiry and a cut-off leave a line in the
+    // log naming the queue and how far it got (see `job-observer.ts`).
+    const outcome = await observeJob(
+      queue,
+      jobs as ReadonlyArray<{ id?: string; expireInSeconds?: number }>,
+      () => handler(jobs),
+    );
     if (outcome.ok) return serializeJobOutcome(outcome);
 
     const failure = new JobFailure(queue, outcome.reason, outcome.cause);

src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts

@@ -0,0 +1,331 @@
+/**
+ * `adoptOrMintHourlyRows` resolves a whole folded day in one lookup and one
+ * insert. It replaced a per-slot loop (two lookups and a write per hour, about
+ * 75 statements a day), which was most of the 380 000 statements the dense
+ * retention spent on a 1.25-million-row account (issue #1031).
+ *
+ * Two things are pinned here. The statement shape: one read, at most one
+ * insert, no per-slot lookups. And the semantics: against an in-memory table
+ * that enforces both unique identities, the batch leaves exactly the rows the
+ * per-slot loop leaves and returns the same ids, over randomised days that
+ * include rows already sitting on an anchor, rows already carrying a target
+ * externalId, tombstones, and a row holding one hour's externalId while
+ * parked on another hour's anchor.
+ */
+import { describe, expect, it, vi } from "vitest";
+
+import type { MeasurementType, Prisma } from "@/generated/prisma/client";
+
+import {
+  adoptOrMintHourlyRows,
+  type HourlySlot,
+} from "../dense-intraday-retention";
+
+interface Row {
+  id: string;
+  externalId: string | null;
+  measuredAt: Date;
+  value: number;
+  unit: string;
+  deletedAt: Date | null;
+}
+
+/**
+ * A measurement table for one (user, type, APPLE_HEALTH), with the two unique
+ * identities the real one has: externalId and measuredAt (sleepStage is NULL
+ * for these types). A write that would duplicate either throws, as P2002 does.
+ */
+function fakeTable(seed: Row[]) {
+  const rows = seed.map((row) => ({ ...row }));
+  let minted = 0;
+  const calls = { findMany: 0, findFirst: 0, createMany: 0, create: 0 };
+
+  const assertUnique = () => {
+    const ext = rows
+      .filter((r) => r.externalId !== null)
+      .map((r) => r.externalId);
+    const at = rows.map((r) => r.measuredAt.getTime());
+    if (new Set(ext).size !== ext.length) throw new Error("P2002 externalId");
+    if (new Set(at).size !== at.length) throw new Error("P2002 measuredAt");
+  };
+  const mint = (data: {
+    externalId: string;
+    measuredAt: Date;
+    value: number;
+    unit: string;
+  }) => {
+    minted += 1;
+    const row: Row = {
+      id: `z-minted-${String(minted).padStart(3, "0")}`,
+      externalId: data.externalId,
+      measuredAt: data.measuredAt,
+      value: data.value,
+      unit: data.unit,
+      deletedAt: null,
+    };
+    rows.push(row);
+    assertUnique();
+    return row;
+  };
+  const byId = () => [...rows].sort((a, b) => (a.id < b.id ? -1 : 1));
+
+  const tx = {
+    measurement: {
+      findMany: vi.fn(
+        async (args: {
+          where: {
+            OR: [
+              { externalId: { in: string[] } },
+              { measuredAt: { in: Date[] } },
+            ];
+          };
+        }) => {
+          calls.findMany += 1;
+          const ext = new Set(args.where.OR[0].externalId.in);
+          const at = new Set(
+            args.where.OR[1].measuredAt.in.map((d) => d.getTime()),
+          );
+          return byId()
+            .filter(
+              (r) =>
+                (r.externalId !== null && ext.has(r.externalId)) ||
+                at.has(r.measuredAt.getTime()),
+            )
+            .map((r) => ({
+              id: r.id,
+              externalId: r.externalId,
+              measuredAt: r.measuredAt,
+            }));
+        },
+      ),
+      findFirst: vi.fn(
+        async (args: { where: { externalId?: string; measuredAt?: Date } }) => {
+          calls.findFirst += 1;
+          const hit = byId().find((r) =>
+            args.where.externalId !== undefined
+              ? r.externalId === args.where.externalId
+              : r.measuredAt.getTime() === args.where.measuredAt!.getTime(),
+          );
+          return hit ? { id: hit.id } : null;
+        },
+      ),
+      update: vi.fn(
+        async (args: {
+          where: { id: string };
+          data: Partial<Row> & { measuredAt?: Date };
+        }) => {
+          const row = rows.find((r) => r.id === args.where.id)!;
+          Object.assign(row, args.data);
+          assertUnique();
+          return row;
+        },
+      ),
+      create: vi.fn(
+        async (args: {
+          data: {
+            externalId: string;
+            measuredAt: Date;
+            value: number;
+            unit: string;
+          };
+        }) => {
+          calls.create += 1;
+          return { id: mint(args.data).id };
+        },
+      ),
+      createManyAndReturn: vi.fn(
+        async (args: {
+          data: Array<{
+            externalId: string;
+            measuredAt: Date;
+            value: number;
+            unit: string;
+          }>;
+        }) => {
+          calls.createMany += 1;
+          return args.data.map((d) => {
+            const row = mint(d);
+            return { id: row.id, externalId: row.externalId };
+          });
+        },
+      ),
+    },
+  };
+  return {
+    tx: tx as unknown as Prisma.TransactionClient,
+    rows,
+    calls,
+  };
+}
+
+/**
+ * The per-slot loop the batch replaced, kept verbatim in behaviour as the
+ * reference: externalId first, then the anchor occupant, lowest id on a tie;
+ * re-anchor only onto a free slot; otherwise mint.
+ */
+async function perSlotReference(
+  tx: Prisma.TransactionClient,
+  unit: string,
+  slots: readonly HourlySlot[],
+): Promise<string[]> {
+  const ids: string[] = [];
+  for (const slot of slots) {
+    const eidRow = await tx.measurement.findFirst({
+      where: { externalId: slot.externalId },
+    } as Prisma.MeasurementFindFirstArgs);
+    const slotRow = await tx.measurement.findFirst({
+      where: { measuredAt: slot.anchor },
+    } as Prisma.MeasurementFindFirstArgs);
+    const target = eidRow ?? slotRow;
+    if (target) {
+      const free = slotRow === null || slotRow.id === target.id;
+      await tx.measurement.update({
+        where: { id: target.id },
+        data: {
+          value: slot.value,
+          unit,
+          externalId: slot.externalId,
+          deletedAt: null,
+          ...(free ? { measuredAt: slot.anchor } : {}),
+        },
+      });
+      ids.push(target.id);
+    } else {
+      const created = await tx.measurement.create({
+        data: {
+          value: slot.value,
+          unit,
+          measuredAt: slot.anchor,
+          externalId: slot.externalId,
+        },
+      } as Prisma.MeasurementCreateArgs);
+      ids.push(created.id);
+    }
+  }
+  return ids;
+}
+
+const HOUR = 3_600_000;
+const DAY0 = Date.UTC(2026, 4, 1, 0, 30);
+const anchor = (h: number) => new Date(DAY0 + h * HOUR);
+const ext = (h: number) => `stats:HK:2026-05-01T${String(h).padStart(2, "0")}`;
+
+/** A deterministic pseudo-random generator, so a failure reproduces. */
+function rng(seed: number) {
+  let x = seed;
+  return () => {
+    x = (x * 1103515245 + 12345) % 2 ** 31;
+    return x / 2 ** 31;
+  };
+}
+
+function scenario(seed: number): { seed: Row[]; slots: HourlySlot[] } {
+  const rand = rng(seed);
+  const hours = [...Array(24).keys()].filter(() => rand() < 0.7);
+  const slots = hours.map((h) => ({
+    externalId: ext(h),
+    anchor: anchor(h),
+    value: Math.round(rand() * 100),
+  }));
+  const rows: Row[] = [];
+  const takenAt = new Set<number>();
+  const takenExt = new Set<string>();
+  let n = 0;
+  const add = (externalId: string | null, at: Date) => {
+    if (takenAt.has(at.getTime())) return;
+    if (externalId !== null && takenExt.has(externalId)) return;
+    takenAt.add(at.getTime());
+    if (externalId !== null) takenExt.add(externalId);
+    n += 1;
+    rows.push({
+      id: `r-${String(Math.floor(rand() * 900) + 100)}-${n}`,
+      externalId,
+      measuredAt: at,
+      value: 1,
+      unit: "bpm",
+      deletedAt: rand() < 0.3 ? new Date(DAY0) : null,
+    });
+  };
+  for (const h of hours) {
+    const roll = rand();
+    if (roll < 0.15)
+      add(ext(h), anchor(h)); // a prior fold's own row
+    else if (roll < 0.3)
+      add(`hk-sample-${h}`, anchor(h)); // a sample on the anchor
+    else if (roll < 0.4)
+      add(ext(h), new Date(DAY0 + h * HOUR + 60_000)); // stats row off-anchor
+    else if (roll < 0.5) {
+      // This hour's externalId parked on ANOTHER hour's anchor.
+      const other = hours[Math.floor(rand() * hours.length)]!;
+      add(ext(h), anchor(other));
+    }
+  }
+  return { seed: rows, slots };
+}
+
+function snapshot(rows: Row[]) {
+  return [...rows]
+    .map((r) => ({
+      id: r.id.startsWith("z-minted") ? "minted" : r.id,
+      externalId: r.externalId,
+      at: r.measuredAt.toISOString(),
+      value: r.value,
+      deleted: r.deletedAt !== null,
+    }))
+    .sort((a, b) =>
+      `${a.externalId}|${a.at}` < `${b.externalId}|${b.at}` ? -1 : 1,
+    );
+}
+
+describe("adoptOrMintHourlyRows", () => {
+  it("resolves a fresh day with one lookup and one insert, no per-slot reads", async () => {
+    const { tx, calls, rows } = fakeTable([]);
+    const slots = [...Array(24).keys()].map((h) => ({
+      externalId: ext(h),
+      anchor: anchor(h),
+      value: h,
+    }));
+
+    const ids = await adoptOrMintHourlyRows(tx, {
+      userId: "u",
+      type: "PULSE" as MeasurementType,
+      unit: "bpm",
+      slots,
+    });
+
+    expect(calls).toEqual({
+      findMany: 1,
+      findFirst: 0,
+      createMany: 1,
+      create: 0,
+    });
+    expect(rows).toHaveLength(24);
+    expect(ids).toEqual(
+      slots.map((s) => rows.find((r) => r.externalId === s.externalId)!.id),
+    );
+  });
+
+  it("leaves the same rows and returns the same ids as the per-slot loop", async () => {
+    for (let seed = 1; seed <= 300; seed++) {
+      const { seed: rows, slots } = scenario(seed);
+
+      const batch = fakeTable(rows);
+      const batchIds = await adoptOrMintHourlyRows(batch.tx, {
+        userId: "u",
+        type: "PULSE" as MeasurementType,
+        unit: "bpm",
+        slots,
+      });
+
+      const reference = fakeTable(rows);
+      const referenceIds = await perSlotReference(reference.tx, "bpm", slots);
+
+      const norm = (ids: string[]) =>
+        ids.map((id) => (id.startsWith("z-minted") ? "minted" : id));
+      expect(norm(batchIds), `seed ${seed}`).toEqual(norm(referenceIds));
+      expect(snapshot(batch.rows), `seed ${seed}`).toEqual(
+        snapshot(reference.rows),
+      );
+    }
+  });
+});

src/lib/measurements/__tests__/consolidation-scan-paged.test.ts

@@ -1,31 +1,50 @@
 import { describe, expect, it, vi } from "vitest";
 
-import { bucketRowsByDay, scanSourceRowsPaged } from "../consolidation-base";
+import {
+  bucketRowsByDay,
+  iterateDayBuckets,
+  iterateSourcePages,
+  runConsolidation,
+  type ConsolidationParams,
+} from "../consolidation-base";
 import type { PerSampleRow } from "../consolidation-tz";
-import type { Prisma, PrismaClient } from "@/generated/prisma/client";
+import type {
+  MeasurementType,
+  Prisma,
+  PrismaClient,
+} from "@/generated/prisma/client";
 
-// `scanSourceRowsPaged` keyset-paginates the per-(user, type) source scan so a
-// multi-year tenant no longer materialises its whole history in one Prisma
-// result set. These tests pin the two properties that matter: the accumulated
-// array equals a single unbounded scan (same rows, same order, no gaps /
-// duplicates), and the cursor advances on `(measuredAt, id)` after every full
-// page.
+// The consolidation drains walk a user's whole history of a type. Issue #1031:
+// on an account with 1.25 million rows the walk gathered every page into one
+// array before folding a single day (≈230 MB of heap for the heart-rate type
+// alone) and paged with a cursor Postgres could not use as an index condition,
+// so every page re-read the rows before it. These tests pin the three
+// properties the fix rests on: pages are handed on one at a time and a day is
+// folded before the scan reads past it, the cursor carries an index-usable
+// bound, and a pass stops cleanly when asked to.
 
-/** Build a deterministic per-sample row at a 1-minute-per-index offset. */
-function row(index: number): PerSampleRow {
+/** A deterministic per-sample row at `index` hours past 2026-01-01T00:00Z. */
+function row(index: number, minuteOffset = 0): PerSampleRow {
   return {
     id: `m-${String(index).padStart(4, "0")}`,
     type: "ACTIVITY_STEPS",
     value: index,
-    measuredAt: new Date(Date.UTC(2026, 0, 1, 0, index, 0)),
+    measuredAt: new Date(Date.UTC(2026, 0, 1, index, minuteOffset, 0)),
     externalId: `hk-uuid-${index}`,
   };
 }
 
+interface CursorWhere {
+  AND?: [
+    Prisma.MeasurementWhereInput,
+    { measuredAt?: { gte?: Date } },
+    { OR?: Array<{ measuredAt?: Date | { gt?: Date }; id?: { gt?: string } }> },
+  ];
+}
+
 /**
- * A findMany mock that serves `allRows` in keyset order, honouring `take`
- * (page size) and the `(measuredAt, id)` cursor clause the helper AND-combines
- * onto the base where. This mirrors how Postgres would resolve the keyset.
+ * A findMany mock that serves `allRows` in keyset order, honouring `take` and
+ * the `(measuredAt, id)` cursor the pager AND-combines onto the base where.
  */
 function buildPagingFindMany(allRows: PerSampleRow[]) {
   const ordered = [...allRows].sort((a, b) => {
@@ -34,43 +53,29 @@ function buildPagingFindMany(allRows: PerSampleRow[]) {
   });
 
   return vi.fn(
-    async (args: {
-      where: Prisma.MeasurementWhereInput;
-      take?: number;
-      orderBy?: unknown;
-    }) => {
+    async (args: { where: CursorWhere; take?: number; orderBy?: unknown }) => {
       const take = args.take ?? ordered.length;
-
-      // Extract the keyset cursor the helper threads through `where.AND[1].OR`.
-      let afterTime: number | null = null;
-      let afterId: string | null = null;
-      const and = (args.where as { AND?: unknown[] }).AND;
-      if (Array.isArray(and)) {
-        const or = (and[1] as { OR?: Array<Record<string, unknown>> }).OR;
-        const gt = or?.[0]?.measuredAt as { gt?: Date } | undefined;
-        const tie = or?.[1] as
-          { measuredAt?: Date; id?: { gt?: string } } | undefined;
-        afterTime = gt?.gt ? gt.gt.getTime() : null;
-        afterId = tie?.id?.gt ?? null;
-      }
-
-      const remaining =
-        afterTime === null
-          ? ordered
-          : ordered.filter((r) => {
-              const t = r.measuredAt.getTime();
-              return t > afterTime! || (t === afterTime! && r.id > afterId!);
-            });
-
-      return remaining.slice(0, take);
+      const and = args.where.AND;
+      if (!and) return ordered.slice(0, take);
+      const gte = and[1].measuredAt?.gte?.getTime();
+      const or = and[2].OR ?? [];
+      const gt = (
+        or[0]?.measuredAt as { gt?: Date } | undefined
+      )?.gt?.getTime();
+      const tieId = or[1]?.id?.gt;
+      return ordered
+        .filter((r) => {
+          const t = r.measuredAt.getTime();
+          if (gte !== undefined && t < gte) return false;
+          return t > gt! || (t === gt! && r.id > tieId!);
+        })
+        .slice(0, take);
     },
   );
 }
 
 function mockClient(findMany: ReturnType<typeof vi.fn>): PrismaClient {
-  return {
-    measurement: { findMany },
-  } as unknown as PrismaClient;
+  return { measurement: { findMany } } as unknown as PrismaClient;
 }
 
 const BASE_WHERE: Prisma.MeasurementWhereInput = {
@@ -85,21 +90,25 @@ const SELECT: Prisma.MeasurementSelect = {
   externalId: true,
 };
 
-describe("scanSourceRowsPaged", () => {
-  it("accumulates every row in keyset order across multiple pages with no gaps or duplicates", async () => {
+async function collectPages(
+  pages: AsyncIterable<PerSampleRow[]>,
+): Promise<PerSampleRow[][]> {
+  const out: PerSampleRow[][] = [];
+  for await (const page of pages) out.push(page);
+  return out;
+}
+
+describe("iterateSourcePages", () => {
+  it("yields every row once, in keyset order, one bounded page at a time", async () => {
     const all = Array.from({ length: 23 }, (_, i) => row(i));
     const findMany = buildPagingFindMany(all);
 
-    const result = await scanSourceRowsPaged(
-      mockClient(findMany),
-      BASE_WHERE,
-      SELECT,
-      10,
+    const pages = await collectPages(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10),
     );
 
-    // 23 rows at page size 10 → pages of 10, 10, 3 (the short page ends it).
-    expect(result.map((r) => r.id)).toEqual(all.map((r) => r.id));
-    expect(new Set(result.map((r) => r.id)).size).toBe(all.length);
+    expect(pages.map((p) => p.length)).toEqual([10, 10, 3]);
+    expect(pages.flat().map((r) => r.id)).toEqual(all.map((r) => r.id));
     expect(findMany).toHaveBeenCalledTimes(3);
   });
 
@@ -107,88 +116,191 @@ describe("scanSourceRowsPaged", () => {
     const all = Array.from({ length: 20 }, (_, i) => row(i));
     const findMany = buildPagingFindMany(all);
 
-    const result = await scanSourceRowsPaged(
-      mockClient(findMany),
-      BASE_WHERE,
-      SELECT,
-      10,
+    const pages = await collectPages(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10),
     );
 
-    // 20 rows at page size 10 → full pages 10, 10, then an empty probe page.
-    expect(result).toHaveLength(20);
+    expect(pages.flat()).toHaveLength(20);
     expect(findMany).toHaveBeenCalledTimes(3);
   });
 
-  it("advances the cursor on (measuredAt, id) after each full page", async () => {
+  it("yields nothing when the first page is empty", async () => {
+    const findMany = buildPagingFindMany([]);
+    const pages = await collectPages(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10),
+    );
+    expect(pages).toEqual([]);
+    expect(findMany).toHaveBeenCalledTimes(1);
+  });
+
+  it("bounds every later page with an index-usable measuredAt >= cursor conjunct", async () => {
+    // `measuredAt > c OR (measuredAt = c AND id > i)` alone is not an index
+    // condition: Postgres walked the index from the user's first row on
+    // every page. The redundant `>=` bound is what the planner can use.
     const all = Array.from({ length: 12 }, (_, i) => row(i));
     const findMany = buildPagingFindMany(all);
 
-    await scanSourceRowsPaged(mockClient(findMany), BASE_WHERE, SELECT, 5);
+    await collectPages(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 5),
+    );
 
-    // First page: no cursor (base where only).
-    const firstWhere = findMany.mock.calls[0]?.[0]?.where;
-    expect((firstWhere as { AND?: unknown }).AND).toBeUndefined();
-    expect(firstWhere).toEqual(BASE_WHERE);
-
-    // Second page: cursor resumes after the 5th row (id m-0004).
-    const secondWhere = findMany.mock.calls[1]?.[0]?.where as {
-      AND: unknown[];
-    };
-    expect(Array.isArray(secondWhere.AND)).toBe(true);
-    expect(secondWhere.AND[0]).toEqual(BASE_WHERE);
-    const cursorClause = secondWhere.AND[1] as {
-      OR: Array<{ measuredAt?: { gt?: Date }; id?: { gt?: string } }>;
-    };
-    expect(cursorClause.OR[0].measuredAt?.gt).toEqual(row(4).measuredAt);
-    expect(cursorClause.OR[1].id?.gt).toBe(row(4).id);
-
-    // Every page is requested in stable keyset order.
+    expect(findMany.mock.calls[0]?.[0]?.where).toEqual(BASE_WHERE);
+    const second = findMany.mock.calls[1]?.[0]?.where as CursorWhere;
+    expect(second.AND?.[0]).toEqual(BASE_WHERE);
+    expect(second.AND?.[1]).toEqual({
+      measuredAt: { gte: row(4).measuredAt },
+    });
+    expect(second.AND?.[2]).toEqual({
+      OR: [
+        { measuredAt: { gt: row(4).measuredAt } },
+        { measuredAt: row(4).measuredAt, id: { gt: row(4).id } },
+      ],
+    });
     for (const call of findMany.mock.calls) {
       expect(call[0]?.orderBy).toEqual([{ measuredAt: "asc" }, { id: "asc" }]);
     }
   });
 
-  it("returns the empty array when the first page is empty", async () => {
-    const findMany = buildPagingFindMany([]);
-    const result = await scanSourceRowsPaged(
-      mockClient(findMany),
-      BASE_WHERE,
-      SELECT,
-      10,
-    );
-    expect(result).toEqual([]);
-    expect(findMany).toHaveBeenCalledTimes(1);
-  });
-
-  it("produces output that buckets identically to a single unpaged scan", async () => {
-    // Rows spanning three calendar days; the paged accumulation must bucket
-    // byte-identically to the same rows fed in one array.
-    const all = [
-      { ...row(1), measuredAt: new Date("2026-05-16T08:00:00.000Z") },
-      { ...row(2), measuredAt: new Date("2026-05-16T14:00:00.000Z") },
-      { ...row(3), measuredAt: new Date("2026-05-17T06:00:00.000Z") },
-      { ...row(4), measuredAt: new Date("2026-05-17T20:00:00.000Z") },
-      { ...row(5), measuredAt: new Date("2026-05-18T09:00:00.000Z") },
-    ];
+  it("resumes past rows that share the boundary instant without gaps or duplicates", async () => {
+    // Five rows on the same instant straddle the page boundary.
+    const all = [0, 1, 2, 3, 4].map((i) => ({
+      ...row(i),
+      measuredAt: new Date("2026-05-16T08:00:00.000Z"),
+    }));
     const findMany = buildPagingFindMany(all);
-
-    const paged = await scanSourceRowsPaged(
-      mockClient(findMany),
-      BASE_WHERE,
-      SELECT,
-      2,
+    const pages = await collectPages(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2),
     );
+    expect(pages.flat().map((r) => r.id)).toEqual(all.map((r) => r.id));
+  });
+});
 
-    const baselineByDay = bucketRowsByDay(all, "Europe/Berlin", "stats:");
-    const pagedByDay = bucketRowsByDay(paged, "Europe/Berlin", "stats:");
+describe("iterateDayBuckets", () => {
+  const days = [
+    { ...row(1), measuredAt: new Date("2026-05-16T08:00:00.000Z") },
+    { ...row(2), measuredAt: new Date("2026-05-16T14:00:00.000Z") },
+    {
+      ...row(3),
+      externalId: "stats:HKQuantityTypeIdentifierStepCount:2026-05-16",
+      measuredAt: new Date("2026-05-16T15:00:00.000Z"),
+    },
+    { ...row(4), measuredAt: new Date("2026-05-17T06:00:00.000Z") },
+    { ...row(5), measuredAt: new Date("2026-05-17T20:00:00.000Z") },
+    { ...row(6), measuredAt: new Date("2026-05-18T09:00:00.000Z") },
+  ];
 
-    expect([...pagedByDay.keys()].sort()).toEqual(
-      [...baselineByDay.keys()].sort(),
-    );
-    for (const [dateKey, rows] of baselineByDay) {
-      expect(pagedByDay.get(dateKey)?.map((r) => r.id)).toEqual(
+  it("groups exactly as the whole-array bucketing does, stats rows skipped", async () => {
+    const findMany = buildPagingFindMany(days);
+    let scanned = 0;
+    const streamed = new Map<string, string[]>();
+    for await (const [key, rows] of iterateDayBuckets(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2),
+      "Europe/Berlin",
+      "stats:",
+      () => {
+        scanned += 1;
+      },
+    )) {
+      streamed.set(
+        key,
         rows.map((r) => r.id),
       );
     }
+
+    const baseline = bucketRowsByDay(days, "Europe/Berlin", "stats:");
+    expect([...streamed.keys()]).toEqual([...baseline.keys()]);
+    for (const [key, rows] of baseline) {
+      expect(streamed.get(key)).toEqual(rows.map((r) => r.id));
+    }
+    expect(scanned).toBe(days.length);
+  });
+
+  it("hands a day on before the scan reads the page after the one that closed it", async () => {
+    const findMany = buildPagingFindMany(days);
+    const pagesReadWhenYielded: number[] = [];
+    for await (const _day of iterateDayBuckets(
+      iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2),
+      "Europe/Berlin",
+      "stats:",
+    )) {
+      pagesReadWhenYielded.push(findMany.mock.calls.length);
+    }
+    // Pages: [r1 r2] [r3 r4] [r5 r6] [] — 05-16 closes on page 2 (r4),
+    // 05-17 on page 3 (r6), 05-18 at the end of the scan.
+    expect(pagesReadWhenYielded).toEqual([2, 3, 4]);
+  });
+});
+
+describe("runConsolidation — streamed walk", () => {
+  /** One row per hour for `dayCount` UTC days. */
+  function hourlyRows(dayCount: number): PerSampleRow[] {
+    return Array.from({ length: dayCount * 24 }, (_, i) => row(i, 30));
+  }
+
+  function params(
+    findMany: ReturnType<typeof vi.fn>,
+    writeDay: ConsolidationParams<MeasurementType>["writeDay"],
+    shouldStop?: () => boolean,
+  ): ConsolidationParams<MeasurementType> {
+    const client = {
+      user: {
+        findMany: vi
+          .fn()
+          .mockResolvedValue([{ id: "user-1", timezone: "UTC" }]),
+      },
+      measurement: { findMany },
+    } as unknown as PrismaClient;
+    return {
+      prismaClient: client,
+      options: { shouldStop },
+      types: ["ACTIVITY_STEPS"] as MeasurementType[],
+      hkIdentifierForType: () => "HKQuantityTypeIdentifierStepCount",
+      dailyStatsExternalId: (hk: string, day: string) => `stats:${hk}:${day}`,
+      statsPrefix: "stats:",
+      reduce: (rows: readonly PerSampleRow[]) => rows.length,
+      buildScanWhere: () => BASE_WHERE,
+      writeDay,
+      recordBucket: () => {},
+    };
+  }
+
+  it("folds the first day before the scan has read the whole history", async () => {
+    // 500 days × 24 rows = 12 000 rows → default pages of 5000, 5000, 2000.
+    const all = hourlyRows(500);
+    const findMany = buildPagingFindMany(all);
+    const pagesReadAtFirstWrite: number[] = [];
+    const writeDay = vi.fn(async () => {
+      if (pagesReadAtFirstWrite.length === 0) {
+        pagesReadAtFirstWrite.push(findMany.mock.calls.length);
+      }
+      return { kind: "written" as const, sourceRowsRemoved: 0 };
+    });
+
+    const result = await runConsolidation(params(findMany, writeDay));
+
+    expect(writeDay).toHaveBeenCalledTimes(500);
+    expect(findMany).toHaveBeenCalledTimes(3);
+    // The old walk read all three pages before the first write.
+    expect(pagesReadAtFirstWrite).toEqual([1]);
+    expect(result.stoppedEarly).toBe(false);
+  });
+
+  it("stops before the next day once shouldStop answers true, and reads no further", async () => {
+    const all = hourlyRows(500);
+    const findMany = buildPagingFindMany(all);
+    const writeDay = vi.fn(async () => ({
+      kind: "written" as const,
+      sourceRowsRemoved: 0,
+    }));
+    let budget = 3;
+    const shouldStop = () => budget-- <= 0;
+
+    const result = await runConsolidation(
+      params(findMany, writeDay, shouldStop),
+    );
+
+    expect(writeDay).toHaveBeenCalledTimes(3);
+    expect(result.stoppedEarly).toBe(true);
+    expect(findMany).toHaveBeenCalledTimes(1);
   });
 });

src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts

@@ -30,6 +30,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({
 
 import { runDenseIntradayHourlyRebuild } from "../dense-intraday-hourly-rebuild";
 import type { PrismaClient } from "@/generated/prisma/client";
+import { candidateLookup, createManyVia } from "./hourly-mint-mock";
 
 const HRV_DAILY_ID =
   "stats:HKQuantityTypeIdentifierHeartRateVariabilitySDNN:2026-05-01";
@@ -79,6 +80,8 @@ function buildPrismaMock(opts: {
       create: txCreate,
       update: txUpdate,
       findFirst: txFindFirst,
+      findMany: candidateLookup(),
+      createManyAndReturn: createManyVia(txCreate),
     },
   };
 
@@ -444,6 +447,10 @@ describe("runDenseIntradayHourlyRebuild — rebuild flow", () => {
             create: vi.fn().mockResolvedValue({ id: "minted" }),
             update: vi.fn().mockResolvedValue({}),
             findFirst: vi.fn().mockResolvedValue(null),
+            findMany: candidateLookup(),
+            createManyAndReturn: createManyVia(
+              vi.fn().mockResolvedValue({ id: "minted" }),
+            ),
           },
         });
       },
@@ -456,4 +463,44 @@ describe("runDenseIntradayHourlyRebuild — rebuild flow", () => {
     expect(summary.totals.daysFailed).toBe(1);
     expect(summary.totals.daysRebuilt).toBe(1);
   });
+
+  it("stops before the next day once its budget is spent (#1031)", async () => {
+    const { mock, $transaction } = buildPrismaMock({
+      dailyByType: {
+        HEART_RATE_VARIABILITY: [
+          { id: "daily-1", externalId: HRV_DAILY_ID },
+          {
+            id: "daily-2",
+            externalId:
+              "stats:HKQuantityTypeIdentifierHeartRateVariabilitySDNN:2026-05-02",
+          },
+        ],
+      },
+      tombstonesByType: {
+        HEART_RATE_VARIABILITY: [
+          tombstone(
+            "a",
+            40,
+            "2026-05-01T08:00:00.000Z",
+            "HEART_RATE_VARIABILITY",
+          ),
+          tombstone(
+            "b",
+            44,
+            "2026-05-02T08:00:00.000Z",
+            "HEART_RATE_VARIABILITY",
+          ),
+        ],
+      },
+    });
+    let asked = 0;
+    const summary = await runDenseIntradayHourlyRebuild(mock, {
+      log: () => {},
+      shouldStop: () => asked++ >= 1,
+    });
+
+    expect(summary.stoppedEarly).toBe(true);
+    expect(summary.totals.daysRebuilt).toBe(1);
+    expect($transaction).toHaveBeenCalledTimes(1);
+  });
 });

src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts

@@ -33,6 +33,7 @@ import {
 } from "../dense-intraday-retention";
 import type { PerSampleRow } from "../drain-per-sample-cumulative";
 import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
+import { candidateLookup, createManyVia } from "./hourly-mint-mock";
 
 function pulseRow(id: string, value: number, iso: string): PerSampleRow {
   return {
@@ -68,6 +69,8 @@ function buildPrismaMock(opts: {
       update: txUpdate,
       findFirst: txFindFirst,
       updateMany: txUpdateMany,
+      findMany: candidateLookup(),
+      createManyAndReturn: createManyVia(txCreate),
     },
   };
 
@@ -325,6 +328,10 @@ describe("PULSE fold — per-day failure boundary", () => {
             update: vi.fn(),
             findFirst: vi.fn().mockResolvedValue(null),
             updateMany: vi.fn().mockResolvedValue({ count: 5 }),
+            findMany: candidateLookup(),
+            createManyAndReturn: createManyVia(
+              vi.fn().mockResolvedValue({ id: "minted" }),
+            ),
           },
         });
       },

src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts

@@ -32,6 +32,7 @@ import {
   DENSE_INTRADAY_RETENTION_DAYS,
 } from "../dense-intraday-retention";
 import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
+import { candidateLookup, createManyVia } from "./hourly-mint-mock";
 
 /** A day well outside the retention window, so the fold definitely runs. */
 const FOLD_DAY = new Date(
@@ -76,7 +77,15 @@ function buildPrismaMock(existingResting: Array<{ source: string }>) {
   );
 
   const tx = {
-    measurement: { create, update, findFirst: txFindFirst, updateMany, upsert },
+    measurement: {
+      create,
+      update,
+      findFirst: txFindFirst,
+      updateMany,
+      upsert,
+      findMany: candidateLookup(),
+      createManyAndReturn: createManyVia(create),
+    },
   };
 
   return {

src/lib/measurements/__tests__/dense-intraday-retention.test.ts

@@ -25,6 +25,7 @@ import {
 } from "../dense-intraday-retention";
 import type { PerSampleRow } from "../drain-per-sample-cumulative";
 import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
+import { candidateLookup, createManyVia } from "./hourly-mint-mock";
 
 function row(
   id: string,
@@ -56,7 +57,16 @@ function buildPrismaMock(
     async (args: { where: { type: string } }) =>
       rowsByType[args.where.type] ?? [],
   );
-  const tx = { measurement: { create, update, findFirst, updateMany } };
+  const tx = {
+    measurement: {
+      create,
+      update,
+      findFirst,
+      updateMany,
+      findMany: candidateLookup(existingCanonicalId),
+      createManyAndReturn: createManyVia(create),
+    },
+  };
   return {
     mock: {
       user: {

src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts

@@ -29,6 +29,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({
 import { runDenseIntradayRetention } from "../dense-intraday-retention";
 import type { PerSampleRow } from "../drain-per-sample-cumulative";
 import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
+import { candidateLookup, createManyVia } from "./hourly-mint-mock";
 
 function spo2Row(id: string, value: number, iso: string): PerSampleRow {
   return {
@@ -60,6 +61,8 @@ function buildPrismaMock(opts: { spo2Rows: PerSampleRow[] }) {
       update: txUpdate,
       findFirst: txFindFirst,
       updateMany: txUpdateMany,
+      findMany: candidateLookup(),
+      createManyAndReturn: createManyVia(txCreate),
     },
   };
 
@@ -254,6 +257,10 @@ describe("SpO2 fold — per-day failure boundary", () => {
             update: vi.fn(),
             findFirst: vi.fn().mockResolvedValue(null),
             updateMany: vi.fn().mockResolvedValue({ count: 2 }),
+            findMany: candidateLookup(),
+            createManyAndReturn: createManyVia(
+              vi.fn().mockResolvedValue({ id: "minted" }),
+            ),
           },
         });
       },

src/lib/measurements/__tests__/hourly-mint-mock.ts

@@ -0,0 +1,49 @@
+/**
+ * Transaction-client stubs for `adoptOrMintHourlyRows`, shared by the mocked
+ * dense-tier fold tests.
+ *
+ * The hourly mint reads the day's candidate rows with one `findMany` and
+ * writes its mints with one `createManyAndReturn`. The tests were written
+ * against the per-slot loop and assert on individual `create` calls, so the
+ * batch insert here is routed through the test's own `create` spy, one call
+ * per minted row: the assertions keep reading what was minted, in slot order.
+ */
+import { vi } from "vitest";
+
+type CreateSpy = (args: {
+  data: { externalId: string };
+}) => Promise<{ id: string }>;
+
+/** `createManyAndReturn` that forwards every row to `create`. */
+export function createManyVia(create: CreateSpy) {
+  return vi.fn(async (args: { data: Array<{ externalId: string }> }) => {
+    const out: Array<{ id: string; externalId: string }> = [];
+    for (const data of args.data) {
+      const row = await create({ data });
+      out.push({ id: row.id, externalId: data.externalId });
+    }
+    return out;
+  });
+}
+
+/**
+ * The day's candidate lookup. With `existingId`, one row already occupies the
+ * first slot's anchor instant (the adopt-in-place case); otherwise nothing is
+ * in the way.
+ */
+export function candidateLookup(existingId: string | null = null) {
+  return vi.fn(
+    async (args: {
+      where: { OR: [unknown, { measuredAt: { in: Date[] } }] };
+    }) =>
+      existingId === null
+        ? []
+        : [
+            {
+              id: existingId,
+              externalId: null,
+              measuredAt: args.where.OR[1].measuredAt.in[0]!,
+            },
+          ],
+  );
+}

src/lib/measurements/consolidate-daily-mean.ts

@@ -93,6 +93,12 @@ export interface MeanConsolidationBucket {
 
 export interface MeanConsolidationSummary {
   dryRun: boolean;
+  /**
+   * `shouldStop` ended the walk before every day was reached. Nothing is
+   * lost: folded days committed one by one, and the next run starts at the
+   * first day this one did not reach.
+   */
+  stoppedEarly: boolean;
   buckets: MeanConsolidationBucket[];
   totals: {
     usersScanned: number;
@@ -123,6 +129,11 @@ export interface MeanConsolidationOptions {
    * `undefined` to drain everything.
    */
   cutoffHours?: number;
+  /**
+   * Ends the pass cleanly before the next day when it returns `true`; the
+   * summary then reports `stoppedEarly`. See `ConsolidationOptions`.
+   */
+  shouldStop?: () => boolean;
 }
 
 /** Arithmetic mean of a per-day bucket. Exposed for unit testing. */
@@ -205,6 +216,7 @@ export async function consolidateDailyMean(
 
   const summary: MeanConsolidationSummary = {
     dryRun: options.dryRun ?? false,
+    stoppedEarly: false,
     buckets: [],
     totals: {
       usersScanned: 0,
@@ -215,7 +227,7 @@ export async function consolidateDailyMean(
     },
   };
 
-  const { usersScanned, daysFailed } = await runConsolidation<MeasurementType>({
+  const walk = await runConsolidation<MeasurementType>({
     prismaClient,
     options,
     types: HIGH_FREQUENCY_MEAN_TYPES,
@@ -420,8 +432,9 @@ export async function consolidateDailyMean(
     },
   });
 
-  summary.totals.usersScanned = usersScanned;
-  summary.totals.daysFailed = daysFailed;
+  summary.totals.usersScanned = walk.usersScanned;
+  summary.stoppedEarly = walk.stoppedEarly;
+  summary.totals.daysFailed = walk.daysFailed;
 
   log(
     `[mean-consolidation] done — usersScanned=${summary.totals.usersScanned} daysConsolidated=${summary.totals.daysConsolidated} perSampleRowsSoftDeleted=${summary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted} daysFailed=${summary.totals.daysFailed}${options.dryRun ? " (dry-run)" : ""}`,

src/lib/measurements/consolidation-base.ts

@@ -32,6 +32,8 @@ import type {
   Prisma,
 } from "@/generated/prisma/client";
 
+import { reportJobProgress } from "@/lib/jobs/job-observer";
+
 import {
   CONSOLIDATION_GRACE_CUTOFF_HOURS,
   canonicalDailyTimestamp,
@@ -57,6 +59,18 @@ export interface ConsolidationOptions {
   dryRun?: boolean;
   /** Logger sink — defaults to `console.log`. */
   log?: (line: string) => void;
+  /**
+   * Asked before every day bucket. Returning `true` ends the pass cleanly
+   * after the day in flight has committed, and the result reports
+   * `stoppedEarly`. The queue handlers wire it to the job's time budget and
+   * to pg-boss's abort signal, so a pass that cannot finish inside its expiry
+   * stops on its own instead of being declared dead while it keeps writing.
+   *
+   * Stopping loses nothing: every day commits on its own, and a folded day's
+   * source rows leave the scan predicate, so the next run starts at the first
+   * day this one did not reach.
+   */
+  shouldStop?: () => boolean;
 }
 
 /**
@@ -229,7 +243,7 @@ export interface ConsolidationParams<TType extends MeasurementType> {
     dryRun: boolean;
   }) => void;
   /**
-   * Fired once per (user, type) immediately after the rows are bucketed,
+   * Fired once per (user, type) after the type's rows have been walked,
    * with the raw scanned-row count and per-day bucket count. Only invoked
    * for types that yielded at least one source row. Lets a single-type
    * drain (legacy steps) reproduce its scan-time log line. Optional.
@@ -275,35 +289,40 @@ const DEFAULT_SCAN_SELECT: Prisma.MeasurementSelect = {
 };
 
 /**
- * Page size for the keyset-paginated source-row scan. Heavy multi-year
- * tenants previously materialised the entire per-(user, type) history in a
- * single unbounded `findMany` result set — the dominant Prisma intermediate
- * cost and the root of the consolidation boot-storm on large accounts.
- * Scanning in bounded keyset pages caps the per-query result set while still
- * accumulating the same full row set.
+ * Page size for the keyset-paginated source-row scan.
  */
 const CONSOLIDATION_SCAN_PAGE_SIZE = 5000;
 
 /**
  * Keyset-paginate the per-(user, type) source-row scan on `(measuredAt, id)`
- * ascending, accumulating every page into one array. Output is identical to a
- * single unbounded `findMany` — same `where`, same `select`, same ascending
- * order — but the Prisma client never holds more than one page of rows at a
- * time, so a multi-year tenant no longer materialises its whole history in one
- * result set at worker boot.
+ * ascending, one page at a time.
  *
- * `(measuredAt, id)` is a stable keyset: `measuredAt` is the primary order and
- * `id` breaks ties, so each page resumes exactly after the last row with no
- * gaps or duplicates. The cursor clause is AND-combined with the caller's
- * `baseWhere` so the scope / soft-delete / grace filters always still apply.
+ * Two properties matter, and both used to be missing.
+ *
+ * The pages are yielded, not accumulated. The previous helper gathered every
+ * page into one array before a single day was folded, so the pass held a
+ * type's whole history at once: on an account with 766 000 heart-rate samples
+ * that is about 230 MB of live heap for the array alone, most of a 524 MB
+ * limit, and nothing was written until the whole scan had finished.
+ *
+ * The cursor is index-usable. `measuredAt > c OR (measuredAt = c AND id > i)`
+ * cannot be an index condition, so Postgres walked the index from the user's
+ * first row on every page and filtered its way to the cursor, which makes the
+ * whole scan quadratic (measured: 47 ms and 248 000 rows filtered for one
+ * page halfway through). The redundant `measuredAt >= c` conjunct is the
+ * index condition; the OR only breaks ties at the boundary instant.
+ *
+ * The caller's `baseWhere` is AND-combined, so the scope / soft-delete /
+ * grace filters always still apply.
  */
-export async function scanSourceRowsPaged(
+export async function* iterateSourcePages<
+  Row extends { id: string; measuredAt: Date } = PerSampleRow,
+>(
   prismaClient: PrismaClient,
   baseWhere: Prisma.MeasurementWhereInput,
   scanSelect: Prisma.MeasurementSelect,
   pageSize: number = CONSOLIDATION_SCAN_PAGE_SIZE,
-): Promise<PerSampleRow[]> {
-  const accumulated: PerSampleRow[] = [];
+): AsyncGenerator<Row[], void, void> {
   let cursor: { measuredAt: Date; id: string } | null = null;
 
   for (;;) {
@@ -311,6 +330,7 @@ export async function scanSourceRowsPaged(
       ? {
           AND: [
             baseWhere,
+            { measuredAt: { gte: cursor.measuredAt } },
             {
               OR: [
                 { measuredAt: { gt: cursor.measuredAt } },
@@ -326,20 +346,63 @@ export async function scanSourceRowsPaged(
       select: scanSelect,
       orderBy: [{ measuredAt: "asc" }, { id: "asc" }],
       take: pageSize,
-    })) as PerSampleRow[];
+    })) as unknown as Row[];
 
-    if (page.length === 0) break;
-
-    for (const row of page) accumulated.push(row);
-
-    // A short page is the last page — no further rows can satisfy the keyset.
-    if (page.length < pageSize) break;
+    if (page.length === 0) return;
 
     const last = page[page.length - 1]!;
+    const isLastPage = page.length < pageSize;
+    yield page;
+
+    // A short page is the last page — no further rows can satisfy the keyset.
+    if (isLastPage) return;
     cursor = { measuredAt: last.measuredAt, id: last.id };
   }
+}
 
-  return accumulated;
+/**
+ * Group a stream of ascending pages into complete per-day buckets, in the
+ * user's timezone, without holding more than one day at a time.
+ *
+ * The rows arrive in `measuredAt` order and a local calendar day is one
+ * contiguous span of instants, so a day is complete the moment a row from a
+ * later day shows up. Rows whose externalId starts with `statsPrefix` (the
+ * daily-stats shape) are skipped, the same as `bucketRowsByDay`; pass `null`
+ * to keep every row. `onRow` sees every scanned row, skipped ones included,
+ * for the scan-count log line.
+ */
+export async function* iterateDayBuckets<
+  Row extends { measuredAt: Date; externalId: string | null } = PerSampleRow,
+>(
+  pages: AsyncIterable<readonly Row[]>,
+  tz: string,
+  statsPrefix: string | null,
+  onRow?: () => void,
+): AsyncGenerator<[dateKey: string, rows: Row[]], void, void> {
+  let currentKey: string | null = null;
+  let current: Row[] = [];
+  for await (const page of pages) {
+    for (const row of page) {
+      onRow?.();
+      if (
+        statsPrefix !== null &&
+        row.externalId !== null &&
+        row.externalId.startsWith(statsPrefix)
+      ) {
+        continue;
+      }
+      const key = dayKeyForUserTz(row.measuredAt, tz);
+      if (key !== currentKey) {
+        if (currentKey !== null && current.length > 0) {
+          yield [currentKey, current];
+        }
+        currentKey = key;
+        current = [];
+      }
+      current.push(row);
+    }
+  }
+  if (currentKey !== null && current.length > 0) yield [currentKey, current];
 }
 
 /**
@@ -348,7 +411,8 @@ export async function scanSourceRowsPaged(
  * and delegates the per-day mint + delete to `writeDay`. Returns the
  * number of users scanned (so the caller can seed its summary totals)
  * plus the number of day buckets absorbed by `onBucketError` (0 when the
- * boundary is not supplied).
+ * boundary is not supplied), and whether `options.shouldStop` ended the
+ * walk before every day was reached.
  *
  * Idempotency, the grace-window cutoff, and the "skip already-collapsed
  * rows" predicate are all owned here; the divergent reducer / delete /
@@ -356,16 +420,24 @@ export async function scanSourceRowsPaged(
  */
 export async function runConsolidation<TType extends MeasurementType>(
   params: ConsolidationParams<TType>,
-): Promise<{ usersScanned: number; dryRun: boolean; daysFailed: number }> {
+): Promise<{
+  usersScanned: number;
+  dryRun: boolean;
+  daysFailed: number;
+  stoppedEarly: boolean;
+}> {
   const { prismaClient, options } = params;
   const dryRun = options.dryRun ?? false;
   const cutoffAt = resolveCutoffInstant(options.cutoffHours);
   const scanSelect = params.scanSelect ?? DEFAULT_SCAN_SELECT;
+  const shouldStop = options.shouldStop ?? (() => false);
   let daysFailed = 0;
+  let daysWalked = 0;
+  let stoppedEarly = false;
 
   const users = await loadConsolidationUsers(prismaClient, options.userId);
 
-  for (const user of users) {
+  walk: for (const user of users) {
     const tz = resolveUserTimezone(user.timezone);
     params.onUserStart?.({ userId: user.id, tz, dryRun });
 
@@ -373,36 +445,45 @@ export async function runConsolidation<TType extends MeasurementType>(
       const hkIdentifier = params.hkIdentifierForType(type);
       if (!hkIdentifier) continue;
 
-      // Keyset-paginate the scan instead of loading the whole per-(user, type)
-      // history in one result set; the accumulated array is byte-identical to a
-      // single unbounded `findMany`, but a multi-year tenant no longer spikes
-      // worker memory at boot. See `scanSourceRowsPaged`.
-      const sourceRows = await scanSourceRowsPaged(
-        prismaClient,
-        params.buildScanWhere({
-          userId: user.id,
-          type,
-          cutoffAt,
-          statsPrefix: params.statsPrefix,
-        }),
-        scanSelect,
+      // Streamed: each day is folded as soon as its last row has been read,
+      // so the pass holds one page and one day, never the type's history.
+      // See `iterateSourcePages` and `iterateDayBuckets`.
+      let rowCount = 0;
+      let dayCount = 0;
+      const days = iterateDayBuckets(
+        iterateSourcePages(
+          prismaClient,
+          params.buildScanWhere({
+            userId: user.id,
+            type,
+            cutoffAt,
+            statsPrefix: params.statsPrefix,
+          }),
+          scanSelect,
+        ),
+        tz,
+        params.statsPrefix,
+        () => {
+          rowCount += 1;
+        },
       );
 
-      if (sourceRows.length === 0) continue;
-
-      const byDay = bucketRowsByDay(sourceRows, tz, params.statsPrefix);
-
-      params.onScan?.({
-        userId: user.id,
-        type,
-        tz,
-        rowCount: sourceRows.length,
-        dayCount: byDay.size,
-        dryRun,
-      });
-
-      for (const [dateKey, dayRows] of byDay) {
-        if (dayRows.length === 0) continue;
+      for await (const [dateKey, dayRows] of days) {
+        if (shouldStop()) {
+          stoppedEarly = true;
+          // Leaving the loop closes both generators, so no further page is
+          // requested.
+          break walk;
+        }
+        dayCount += 1;
+        daysWalked += 1;
+        // How far the pass has got, for the job's progress and expiry lines.
+        reportJobProgress({
+          consolidation_user: user.id,
+          consolidation_type: type,
+          consolidation_day: dateKey,
+          consolidation_days_walked: daysWalked,
+        });
 
         try {
           const reducedValue = params.reduce(dayRows);
@@ -464,10 +545,21 @@ export async function runConsolidation<TType extends MeasurementType>(
           });
         }
       }
+
+      if (rowCount > 0) {
+        params.onScan?.({
+          userId: user.id,
+          type,
+          tz,
+          rowCount,
+          dayCount,
+          dryRun,
+        });
+      }
     }
 
     params.onUserComplete?.({ userId: user.id, tz, dryRun });
   }
 
-  return { usersScanned: users.length, dryRun, daysFailed };
+  return { usersScanned: users.length, dryRun, daysFailed, stoppedEarly };
 }

src/lib/measurements/dense-intraday-hourly-rebuild.ts

@@ -19,7 +19,7 @@
  *       day's local window (the fold's soft-deleted inputs).
  *
  * Per qualifying day: compute per-local-hour means from the tombstoned raw
- * rows (same `meanBucketValue` reducer + `adoptOrMintHourlyRow`
+ * rows (same `meanBucketValue` reducer + `adoptOrMintHourlyRows`
  * adopt-in-place slot logic as the live fold, so a re-run converges), then
  * tombstone the daily row IN THE SAME TRANSACTION as the hourly mint — at
  * no instant are the daily row and its hourly rows both live, so an
@@ -73,7 +73,7 @@ import { meanBucketValue } from "./consolidate-daily-mean";
 import {
   DENSE_INTRADAY_RETENTION_DAYS,
   DENSE_INTRADAY_RETENTION_TYPES,
-  adoptOrMintHourlyRow,
+  adoptOrMintHourlyRows,
   bucketRowsByLocalHour,
   hourlyStatsExternalId,
 } from "./dense-intraday-retention";
@@ -111,6 +111,8 @@ export interface DenseIntradayHourlyRebuildSummary {
     /** Days whose rebuild threw and was stepped over (retried next run). */
     daysFailed: number;
   };
+  /** `shouldStop` ended the pass before every candidate day was reached. */
+  stoppedEarly: boolean;
 }
 
 export interface DenseIntradayHourlyRebuildOptions {
@@ -126,6 +128,12 @@ export interface DenseIntradayHourlyRebuildOptions {
    * pass `0` to lift the bound.
    */
   retentionDays?: number;
+  /**
+   * Asked before each day. Returning `true` ends the pass cleanly; the
+   * summary reports `stoppedEarly`. A rebuilt day leaves the candidate set,
+   * so the next run starts at the first day this one did not reach.
+   */
+  shouldStop?: () => boolean;
 }
 
 /**
@@ -147,6 +155,7 @@ export async function runDenseIntradayHourlyRebuild(
 
   const summary: DenseIntradayHourlyRebuildSummary = {
     dryRun,
+    stoppedEarly: false,
     totals: {
       usersScanned: 0,
       daysRebuilt: 0,
@@ -160,7 +169,7 @@ export async function runDenseIntradayHourlyRebuild(
   const users = await loadConsolidationUsers(prismaClient, options.userId);
   summary.totals.usersScanned = users.length;
 
-  for (const user of users) {
+  walk: for (const user of users) {
     const tz = resolveUserTimezone(user.timezone);
 
     for (const type of DENSE_INTRADAY_RETENTION_TYPES) {
@@ -195,6 +204,10 @@ export async function runDenseIntradayHourlyRebuild(
       if (candidates.length === 0) continue;
 
       for (const candidate of candidates) {
+        if (options.shouldStop?.()) {
+          summary.stoppedEarly = true;
+          break walk;
+        }
         // Per-day failure boundary: one poisoned day is stepped over so the
         // walk keeps rebuilding every other user / type / day; the failed
         // day keeps its live daily row, so the discovery re-finds it.
@@ -318,11 +331,11 @@ async function rebuildDay(
   const rebuildOnce = async (): Promise<{ dailyRetired: boolean }> => {
     let dailyRetired = false;
     await prismaClient.$transaction(async (tx) => {
-      const canonicalRowIds: string[] = [];
-      for (const [hour, hourRows] of byHour) {
-        const rowId = await adoptOrMintHourlyRow(tx, {
-          userId: input.userId,
-          type: input.type,
+      const canonicalRowIds = await adoptOrMintHourlyRows(tx, {
+        userId: input.userId,
+        type: input.type,
+        unit,
+        slots: byHour.map(([hour, hourRows]) => ({
           externalId: hourlyStatsExternalId(
             input.hkIdentifier,
             input.dateKey,
@@ -330,10 +343,8 @@ async function rebuildDay(
           ),
           anchor: canonicalHourlyTimestamp(input.dateKey, hour, input.tz),
           value: meanBucketValue(hourRows),
-          unit,
-        });
-        canonicalRowIds.push(rowId);
-      }
+        })),
+      });
 
       // Retire the daily row in the SAME transaction — at no instant are
       // the daily row and the hourly rows both live (an AVG-over-live-rows
@@ -359,7 +370,7 @@ async function rebuildDay(
     if (!isUniqueConstraintViolation(err)) throw err;
     // A concurrent writer (the nightly fold racing this rebuild) won a slot
     // mid-transaction. Retry once: the deterministic lookup inside
-    // `adoptOrMintHourlyRow` now resolves the winning row and adopts it.
+    // `adoptOrMintHourlyRows` now resolves the winning row and adopts it.
     outcome = await rebuildOnce();
   }
 

src/lib/measurements/dense-intraday-retention.ts

@@ -191,10 +191,20 @@ export function bucketRowsByLocalHour(
   return byHour;
 }
 
+/** One hourly slot of a folded day, as `adoptOrMintHourlyRows` receives it. */
+export interface HourlySlot {
+  /** Target hourly `stats:` externalId (index-A identity). */
+  externalId: string;
+  /** Local HH:30 anchor instant (index-B identity). */
+  anchor: Date;
+  value: number;
+}
+
 /**
- * Adopt-or-mint one hourly `stats:` row inside the caller's transaction.
+ * Adopt-or-mint a day's hourly `stats:` rows inside the caller's transaction.
+ * Returns the canonical row id for every slot, in slot order.
  *
- * TWO unique indexes can collide on the mint:
+ * TWO unique indexes can collide on a mint:
  *   A) `(userId, type, source, externalId)` — a row already carrying the
  *      target hourly `stats:` externalId (a prior fold / rebuild pass).
  *   B) `(userId, type, measuredAt, source, sleepStage)` (NULLS NOT
@@ -202,98 +212,149 @@ export function bucketRowsByLocalHour(
  *      anchor instant, which may carry a DIFFERENT externalId (a
  *      per-sample row that happened at HH:30, or a sibling stats row).
  *
- * Determinism (VECTOR 1): resolve by index A FIRST — the row already
- * holding the target externalId IS the canonical hourly row by
- * construction. Only when no such row exists fall back to the index-B row
- * physically occupying the anchor. A stable `orderBy: id` pins the pick.
- * Adopting the wrong sibling and stamping the target externalId onto it
- * would otherwise collide with the real `stats:` row on index A.
+ * Determinism: per slot, resolve by index A FIRST — the row already holding
+ * the target externalId IS the canonical hourly row by construction. Only
+ * when no such row exists fall back to the index-B row physically occupying
+ * the anchor. The lowest id wins a tie. Adopting the wrong sibling and
+ * stamping the target externalId onto it would otherwise collide with the
+ * real `stats:` row on index A.
  *
- * Concurrency (VECTOR 2): the resolve→create is a check-then-act not
- * serialised by the unique index under READ COMMITTED. If a concurrent
- * writer wins the slot between the lookup and the INSERT, the `create`
- * throws P2002 and Postgres aborts the whole transaction. Callers
- * therefore catch P2002 OUTSIDE their transaction (a caught error inside
- * would leave the tx in the aborted `25P02` state) and retry the fold
- * once: on the retry the deterministic lookup finds the now-existing row
- * and ADOPTS it, so the create path never fires twice. Built
- * field-by-field (no spread) per the no-mass-assignment convention.
+ * Why a batch. This used to run per slot — two lookups and a write for each
+ * of up to 24 hours, about 75 statements for one day — and on a multi-year
+ * heart-rate history that loop was most of the pass's 380 000 statements.
+ * Now one lookup reads every row that could matter for the whole day, the
+ * slots are resolved against that snapshot in order, and the mints go out as
+ * one insert. The snapshot is kept up to date as each slot adopts or mints,
+ * so a later slot sees exactly what the one-by-one loop would have read back
+ * from the database: an adopted row that moves to its anchor frees the
+ * instant it left and occupies the one it moved to, and a minted row occupies
+ * its anchor and externalId. Updates run in slot order, as before, and no
+ * mint can collide with them because a slot mints only when nothing in the
+ * snapshot holds its externalId or its anchor.
+ *
+ * Concurrency: the resolve→write is a check-then-act not serialised by the
+ * unique index under READ COMMITTED. If a concurrent writer wins a slot
+ * between the lookup and the INSERT, the insert throws P2002 and Postgres
+ * aborts the whole transaction. Callers therefore catch P2002 OUTSIDE their
+ * transaction (a caught error inside would leave the tx in the aborted
+ * `25P02` state) and retry the fold once: on the retry the lookup finds the
+ * now-existing row and ADOPTS it. Built field-by-field (no spread) per the
+ * no-mass-assignment convention.
  */
-export async function adoptOrMintHourlyRow(
+export async function adoptOrMintHourlyRows(
   tx: Prisma.TransactionClient,
   input: {
     userId: string;
     type: MeasurementType;
-    /** Target hourly `stats:` externalId (index-A identity). */
-    externalId: string;
-    /** Local HH:30 anchor instant (index-B identity). */
-    anchor: Date;
-    value: number;
     unit: string;
+    slots: readonly HourlySlot[];
   },
-): Promise<string> {
-  const eidRow = await tx.measurement.findFirst({
+): Promise<string[]> {
+  if (input.slots.length === 0) return [];
+
+  // Every row that either index could put in the way, lowest id first — the
+  // order the per-slot `findFirst … orderBy id` used to resolve ties by.
+  // `sleepStage` is NULL for these continuous types, matched via the
+  // NULLS-NOT-DISTINCT index.
+  const existing = await tx.measurement.findMany({
     where: {
       userId: input.userId,
       type: input.type,
       source: "APPLE_HEALTH",
-      externalId: input.externalId,
+      OR: [
+        { externalId: { in: input.slots.map((slot) => slot.externalId) } },
+        {
+          measuredAt: { in: input.slots.map((slot) => slot.anchor) },
+          sleepStage: null,
+        },
+      ],
     },
-    select: { id: true },
-    orderBy: { id: "asc" },
-  });
-  // Index-B occupant (`sleepStage` is NULL for these continuous types,
-  // matched via the NULLS-NOT-DISTINCT index). At most one row can exist.
-  const slotRow = await tx.measurement.findFirst({
-    where: {
-      userId: input.userId,
-      type: input.type,
-      source: "APPLE_HEALTH",
-      measuredAt: input.anchor,
-      sleepStage: null,
-    },
-    select: { id: true },
+    select: { id: true, externalId: true, measuredAt: true },
     orderBy: { id: "asc" },
   });
 
-  const adoptTarget = eidRow ?? slotRow;
-  if (adoptTarget) {
-    // Pin the adopted row to the anchor, but only when the anchor slot is
-    // free or already this row — a DIFFERENT row occupying the slot would
-    // otherwise collide on index B. In that rare case the row keeps its
-    // existing instant; the `stats:` externalId is the identity,
-    // measuredAt is secondary.
-    const slotIsFreeForTarget =
-      slotRow === null || slotRow.id === adoptTarget.id;
-    // Adopt: refresh value, stamp the hourly externalId, optionally
-    // re-anchor, and un-tombstone. This coexists with any pre-existing row
-    // on the slot instead of colliding with it.
-    await tx.measurement.update({
-      where: { id: adoptTarget.id },
-      data: {
-        value: input.value,
-        unit: input.unit,
-        externalId: input.externalId,
-        deletedAt: null,
-        ...(slotIsFreeForTarget ? { measuredAt: input.anchor } : {}),
-      },
-    });
-    return adoptTarget.id;
+  // The snapshot the slots resolve against. `id: null` marks a row this
+  // call is about to mint.
+  const state: Array<{
+    id: string | null;
+    externalId: string | null;
+    at: number;
+  }> = existing.map((row) => ({
+    id: row.id,
+    externalId: row.externalId,
+    at: row.measuredAt.getTime(),
+  }));
+
+  const resolved: Array<string | null> = [];
+  const mints: HourlySlot[] = [];
+
+  for (const slot of input.slots) {
+    const anchorAt = slot.anchor.getTime();
+    const eidRow = state.find((row) => row.externalId === slot.externalId);
+    const slotRow = state.find((row) => row.at === anchorAt);
+    const adoptTarget = eidRow ?? slotRow;
+
+    if (adoptTarget && adoptTarget.id !== null) {
+      // Pin the adopted row to the anchor, but only when the anchor slot is
+      // free or already this row — a DIFFERENT row occupying the slot would
+      // otherwise collide on index B. In that rare case the row keeps its
+      // existing instant; the `stats:` externalId is the identity,
+      // measuredAt is secondary.
+      const slotIsFreeForTarget =
+        slotRow === undefined || slotRow === adoptTarget;
+      // Adopt: refresh value, stamp the hourly externalId, optionally
+      // re-anchor, and un-tombstone. This coexists with any pre-existing row
+      // on the slot instead of colliding with it.
+      await tx.measurement.update({
+        where: { id: adoptTarget.id },
+        data: {
+          value: slot.value,
+          unit: input.unit,
+          externalId: slot.externalId,
+          deletedAt: null,
+          ...(slotIsFreeForTarget ? { measuredAt: slot.anchor } : {}),
+        },
+      });
+      adoptTarget.externalId = slot.externalId;
+      if (slotIsFreeForTarget) adoptTarget.at = anchorAt;
+      resolved.push(adoptTarget.id);
+    } else {
+      // Nothing holds this slot's identity or instant — including rows this
+      // call already decided to mint, which carry a distinct anchor and
+      // externalId by construction (one slot per local hour).
+      state.push({ id: null, externalId: slot.externalId, at: anchorAt });
+      mints.push(slot);
+      resolved.push(null);
+    }
   }
 
-  const created = await tx.measurement.create({
-    data: {
-      userId: input.userId,
-      type: input.type,
-      value: input.value,
-      unit: input.unit,
-      source: "APPLE_HEALTH",
-      measuredAt: input.anchor,
-      externalId: input.externalId,
-    },
-    select: { id: true },
-  });
-  return created.id;
+  if (mints.length > 0) {
+    const created = await tx.measurement.createManyAndReturn({
+      data: mints.map((slot) => ({
+        userId: input.userId,
+        type: input.type,
+        value: slot.value,
+        unit: input.unit,
+        source: "APPLE_HEALTH" as const,
+        measuredAt: slot.anchor,
+        externalId: slot.externalId,
+      })),
+      select: { id: true, externalId: true },
+    });
+    const idByExternalId = new Map(
+      created.map((row) => [row.externalId, row.id] as const),
+    );
+    input.slots.forEach((slot, index) => {
+      if (resolved[index] !== null) return;
+      const id = idByExternalId.get(slot.externalId);
+      if (id === undefined) {
+        throw new Error(`hourly mint returned no row for ${slot.externalId}`);
+      }
+      resolved[index] = id;
+    });
+  }
+
+  return resolved as string[];
 }
 
 /**
@@ -347,6 +408,12 @@ export function deriveDailyRestingFromPulse(
 
 export interface DenseIntradayRetentionSummary {
   dryRun: boolean;
+  /**
+   * `shouldStop` ended the walk before every day was reached. Nothing is
+   * lost: folded days committed one by one, and the next run starts at the
+   * first day this one did not reach.
+   */
+  stoppedEarly: boolean;
   totals: {
     usersScanned: number;
     daysConsolidated: number;
@@ -385,6 +452,11 @@ export interface DenseIntradayRetentionOptions {
    * full-collapse, never by the scheduled pass.
    */
   retentionDays?: number;
+  /**
+   * Ends the pass cleanly before the next day when it returns `true`; the
+   * summary then reports `stoppedEarly`. See `ConsolidationOptions`.
+   */
+  shouldStop?: () => boolean;
 }
 
 /**
@@ -412,6 +484,7 @@ export async function runDenseIntradayRetention(
 
   const summary: DenseIntradayRetentionSummary = {
     dryRun: options.dryRun ?? false,
+    stoppedEarly: false,
     totals: {
       usersScanned: 0,
       daysConsolidated: 0,
@@ -462,7 +535,7 @@ export async function runDenseIntradayRetention(
   // for the dry-run hourly fan-out estimate (writeDay never runs on dry-run).
   let currentTz = "Europe/Berlin";
 
-  const { usersScanned } = await runConsolidation<MeasurementType>({
+  const walk = await runConsolidation<MeasurementType>({
     prismaClient,
     options: { ...options, cutoffHours },
     types: DENSE_INTRADAY_RETENTION_TYPES,
@@ -543,18 +616,16 @@ export async function runDenseIntradayRetention(
         let removed = 0;
         let retiredDaily = false;
         await pc.$transaction(async (tx) => {
-          const canonicalRowIds: string[] = [];
-          for (const [hour, hourRows] of byHour) {
-            const rowId = await adoptOrMintHourlyRow(tx, {
-              userId,
-              type,
+          const canonicalRowIds = await adoptOrMintHourlyRows(tx, {
+            userId,
+            type,
+            unit,
+            slots: byHour.map(([hour, hourRows]) => ({
               externalId: hourlyStatsExternalId(hkIdentifier, dateKey, hour),
               anchor: canonicalHourlyTimestamp(dateKey, hour, tz),
               value: meanBucketValue(hourRows),
-              unit,
-            });
-            canonicalRowIds.push(rowId);
-          }
+            })),
+          });
 
           // Retire a live pre-hourly DAILY `stats:` row for this day (the
           // late-sync path: raw samples arriving for a day folded to the
@@ -712,7 +783,8 @@ export async function runDenseIntradayRetention(
     },
   });
 
-  summary.totals.usersScanned = usersScanned;
+  summary.totals.usersScanned = walk.usersScanned;
+  summary.stoppedEarly = walk.stoppedEarly;
 
   log(
     `[dense-intraday-retention] done — usersScanned=${summary.totals.usersScanned} daysConsolidated=${summary.totals.daysConsolidated} perSampleRowsSoftDeleted=${summary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${summary.totals.hourlyRowsUpserted} dailyRowsRetired=${summary.totals.dailyRowsRetired} derivedRestingRowsUpserted=${summary.totals.derivedRestingRowsUpserted}${options.dryRun ? " (dry-run)" : ""}`,

src/lib/measurements/drain-per-sample-cumulative.ts

@@ -124,6 +124,12 @@ export interface DrainBucket {
 export interface DrainSummary {
   /** Did the run actually write to the DB (false for `dryRun`). */
   dryRun: boolean;
+  /**
+   * `shouldStop` ended the walk before every day was reached. Nothing is
+   * lost: folded days committed one by one, and the next run starts at the
+   * first day this one did not reach.
+   */
+  stoppedEarly: boolean;
   /** Per-user-day buckets the drain rewrote (or would rewrite). */
   buckets: DrainBucket[];
   /** Aggregate counts across the run. */
@@ -167,6 +173,11 @@ export interface DrainOptions {
    * the operator points at) for explicit one-shot use.
    */
   cutoffHours?: number;
+  /**
+   * Ends the pass cleanly before the next day when it returns `true`; the
+   * summary then reports `stoppedEarly`. See `ConsolidationOptions`.
+   */
+  shouldStop?: () => boolean;
 }
 
 export interface BucketedRows {
@@ -210,6 +221,7 @@ export async function drainPerSampleCumulative(
 
   const summary: DrainSummary = {
     dryRun: options.dryRun ?? false,
+    stoppedEarly: false,
     buckets: [],
     totals: {
       usersScanned: 0,
@@ -229,7 +241,7 @@ export async function drainPerSampleCumulative(
   let beforePerSampleDeleted = 0;
   let beforeDailyUpserted = 0;
 
-  const { usersScanned } = await runConsolidation<MeasurementType>({
+  const walk = await runConsolidation<MeasurementType>({
     prismaClient,
     options,
     types: CUMULATIVE_HK_TYPES,
@@ -521,7 +533,8 @@ export async function drainPerSampleCumulative(
     },
   });
 
-  summary.totals.usersScanned = usersScanned;
+  summary.totals.usersScanned = walk.usersScanned;
+  summary.stoppedEarly = walk.stoppedEarly;
 
   // v1.34.0 — bound the profile sidecar. One indexed range delete per run,
   // across every account, on the same nightly tick that writes the profiles.

src/lib/measurements/import-apple-health-export.ts

@@ -125,6 +125,11 @@ export interface ImportJobResult {
     /** Estimated `(measurement type, local day)` aggregate rows considered. */
     rows: number;
   };
+  /**
+   * Oldest and newest instant this import wrote a measurement at (ISO), or
+   * null when it wrote none. What the rollup fold after the import covers.
+   */
+  measuredSpan: { from: string; to: string } | null;
   /** Bounded auxiliary ECG outcomes. Never contains filenames or waveform data. */
   ecg: {
     discovered: number;
@@ -396,6 +401,14 @@ export async function streamParseExportXml(
   let rowsUpserted = 0;
   const cumulativeEstimatedDays = new Set<string>();
   let cumulativeEstimatedRows = 0;
+  // The instants this import wrote measurements at, oldest and newest. The
+  // worker folds the rollup table over exactly this span afterwards.
+  let spanFrom: Date | null = null;
+  let spanTo: Date | null = null;
+  const widenSpan = (at: Date) => {
+    if (spanFrom === null || at < spanFrom) spanFrom = at;
+    if (spanTo === null || at > spanTo) spanTo = at;
+  };
   // Per R-1 §8 the percent stays best-effort and may remain null
   // until the parser sees the closing `</HealthData>` tag. We don't
   // currently mutate this in v1.4.34 — the iOS app keeps polling on
@@ -438,6 +451,7 @@ export async function streamParseExportXml(
   const flushSpotBatch = async (): Promise<void> => {
     if (spotBatch.length === 0) return;
     const chunk = spotBatch.splice(0, spotBatch.length);
+    for (const row of chunk) widenSpan(row.measuredAt);
     const insertedArrivals: Array<{
       id: string;
       type: MeasurementType;
@@ -731,6 +745,7 @@ export async function streamParseExportXml(
         }
         const externalId = dailyStatsExternalId(mapping.hkIdentifier, dayKey);
         const measuredAt = canonicalDailyTimestamp(dayKey, userTimezone);
+        widenSpan(measuredAt);
         const rowStart = Date.now();
         const verdict = await prisma.$transaction((tx) =>
           reconcileExternalMeasurement(
@@ -1202,6 +1217,13 @@ export async function streamParseExportXml(
       days: cumulativeEstimatedDays.size,
       rows: cumulativeEstimatedRows,
     },
+    measuredSpan:
+      spanFrom !== null && spanTo !== null
+        ? {
+            from: (spanFrom as Date).toISOString(),
+            to: (spanTo as Date).toISOString(),
+          }
+        : null,
     ecg: {
       discovered: 0,
       imported: 0,

src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts

@@ -8,12 +8,20 @@ import { describe, expect, it, vi, beforeEach } from "vitest";
 
 const deleteMany = vi.fn();
 const findMany = vi.fn();
+// The delete and the re-derivation share one transaction; the stub hands the
+// callback a client that routes to the same spies.
+const txClient = {
+  personalRecord: {
+    deleteMany: (a: unknown) => deleteMany(a),
+  },
+};
 vi.mock("@/lib/db", () => ({
   prisma: {
     personalRecord: {
       deleteMany: (a: unknown) => deleteMany(a),
       findMany: (a: unknown) => findMany(a),
     },
+    $transaction: (fn: (tx: unknown) => Promise<unknown>) => fn(txClient),
   },
 }));
 
@@ -68,8 +76,10 @@ describe("runCumulativePrRederivationForUser", () => {
     });
     // Re-derivation always runs silent — this is data hygiene, not a new
     // achievement, so no push should fire for it.
+    // It reads through the transaction, so it sees the deletion.
     expect(detectPersonalRecordsForUser).toHaveBeenCalledWith("u1", {
       silent: true,
+      prisma: txClient,
     });
     expect(summary).toEqual({ rowsDeleted: 1, rowsReinserted: 1 });
   });

src/lib/personal-records/cumulative-pr-rederivation.ts

@@ -41,7 +41,10 @@ import { annotate } from "@/lib/logging/context";
 import { getGlobalBoss } from "@/lib/jobs/boss-instance";
 import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping";
 import { detectPersonalRecordsForUser } from "@/lib/personal-records/pr-detection-worker";
-import type { MeasurementType } from "@/generated/prisma/client";
+import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
+
+/** Budget for one account's delete-and-re-derive transaction. */
+const REDERIVE_TRANSACTION_TIMEOUT_MS = 5 * 60 * 1000;
 
 export const CUMULATIVE_PR_REDERIVE_QUEUE = "cumulative-pr-rederive";
 
@@ -90,22 +93,40 @@ export interface CumulativePrRederiveSummary {
 export async function runCumulativePrRederivationForUser(
   userId: string,
 ): Promise<CumulativePrRederiveSummary> {
-  const { count: rowsDeleted } = await prisma.personalRecord.deleteMany({
-    where: {
-      userId,
-      metricSlot: null,
-      metricType: { in: CUMULATIVE_TYPES_ARRAY },
-      createdAt: { lt: CUMULATIVE_PR_FIX_CUTOFF },
+  // One transaction: the suspect rows go and the re-derived ones arrive
+  // together, or neither happens. Deleting first and re-deriving after the
+  // commit lost the records whenever the detector failed (a timeout on a
+  // large account, a restart): discovery looks for the suspect rows, so once
+  // they were gone the account was never repaired again (#1031). The detector
+  // reads through the transaction, so it sees the suspect rows as deleted and
+  // compares against what is left. It only reads and inserts, and sends no
+  // notification when `silent`.
+  const { rowsDeleted, rowsReinserted } = await prisma.$transaction(
+    async (tx) => {
+      const { count } = await tx.personalRecord.deleteMany({
+        where: {
+          userId,
+          metricSlot: null,
+          metricType: { in: CUMULATIVE_TYPES_ARRAY },
+          createdAt: { lt: CUMULATIVE_PR_FIX_CUTOFF },
+        },
+      });
+      if (count === 0) return { rowsDeleted: 0, rowsReinserted: 0 };
+      const result = await detectPersonalRecordsForUser(userId, {
+        silent: true,
+        prisma: tx as unknown as PrismaClient,
+      });
+      return {
+        rowsDeleted: count,
+        rowsReinserted: result.inserted + result.ties,
+      };
     },
-  });
-
-  let rowsReinserted = 0;
-  if (rowsDeleted > 0) {
-    const result = await detectPersonalRecordsForUser(userId, {
-      silent: true,
-    });
-    rowsReinserted = result.inserted + result.ties;
-  }
+    // The detector walks every trackable type of the account; on a large
+    // record that is tens of seconds (19.8 s measured for 1.25 million
+    // measurements plus 784 000 energy rows), so the default five seconds
+    // would roll back every repair that matters.
+    { timeout: REDERIVE_TRANSACTION_TIMEOUT_MS, maxWait: 10_000 },
+  );
 
   annotate({
     action: {

src/lib/personal-records/pr-detection-worker.ts

@@ -40,6 +40,10 @@
 import { prisma as defaultPrisma } from "@/lib/db";
 import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping";
 import { dayKeyForUserTz } from "@/lib/measurements/consolidation-tz";
+import {
+  iterateDayBuckets,
+  iterateSourcePages,
+} from "@/lib/measurements/consolidation-base";
 import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum";
 import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority";
 import { measurementTypeEnum } from "@/lib/validations/measurement";
@@ -273,8 +277,12 @@ export async function detectPersonalRecordsForUser(
 
     // Warm-up gate. Cheap count first so we don't pull a 30k-row
     // scan on a brand-new account.
+    // Bounded: the gate only asks whether there are enough, so it stops
+    // counting at the threshold instead of counting a whole history (766 000
+    // heart-rate rows on a large account, every half hour).
     const sampleCount = await prisma.measurement.count({
       where: { userId, type: metricType, deletedAt: null },
+      take: PR_DETECTION_WARMUP_THRESHOLD,
     });
     if (sampleCount < PR_DETECTION_WARMUP_THRESHOLD) continue;
 
@@ -542,18 +550,28 @@ async function findBestMeasurement(
  * cumulative-day surface.
  *
  * Order of operations:
- *   1. Read every row for the type (already scoped by the caller's
- *      warm-up gate to a metric with at least
- *      `PR_DETECTION_WARMUP_THRESHOLD` samples).
- *   2. Collapse to the canonical source per local day.
- *   3. Bucket-and-sum the canonical rows per local day; pick the
- *      winning day by `direction`.
+ *   1. Read the type's rows one local day at a time, in time order
+ *      (already scoped by the caller's warm-up gate to a metric with at
+ *      least `PR_DETECTION_WARMUP_THRESHOLD` samples).
+ *   2. Collapse each day to its canonical source.
+ *   3. Sum the day's canonical rows; keep the winning day by `direction`.
  *   4. Re-read the winning day's latest slice so the PR row carries a
  *      real row's unit/source/externalId (the day-sum overrides
  *      `value` below).
  * The unique-index contract is unaffected — a re-run picks the same
  * day and the same latest-slice timestamp, so `achievedAt` is stable.
  */
+/** One row of the cumulative-day scan. */
+interface CumulativeRow {
+  id: string;
+  value: number;
+  unit: string;
+  measuredAt: Date;
+  source: MeasurementSource;
+  externalId: string | null;
+  deviceType: string | null;
+}
+
 async function findBestCumulativeDay(
   prisma: PrismaClient,
   userId: string,
@@ -562,46 +580,50 @@ async function findBestCumulativeDay(
   tz: string,
   priorityJson: unknown,
 ): Promise<MeasurementCandidate | null> {
-  const rows = await prisma.measurement.findMany({
-    where: { userId, type, deletedAt: null },
-    orderBy: { measuredAt: "asc" },
-    select: {
-      id: true,
-      value: true,
-      unit: true,
-      measuredAt: true,
-      source: true,
-      externalId: true,
-      deviceType: true,
-    },
-  });
-  if (rows.length === 0) return null;
-
+  // Read a day at a time, in time order, never the whole history: the
+  // canonical-source pick is per day, so running it on each day alone gives
+  // the same rows it gave on the whole list. Loading every row at once was
+  // unbounded; with 784 000 rows of one cumulative type the pass ran a
+  // 524 MB heap out of memory (#1031).
   const metricKey = metricKeyForType(type);
-  const canonicalRows = metricKey
-    ? pickCanonicalSourceRows(
-        rows.map((r) => ({ ...r, type })),
-        metricKey,
-        priorityJson,
-        (d) => dayKeyForUserTz(d, tz),
-      ).canonicalRows
-    : rows;
-
-  const byDay = new Map<string, { total: number; latest: Date }>();
-  for (const row of canonicalRows) {
-    const key = dayKeyForUserTz(row.measuredAt, tz);
-    const slot = byDay.get(key);
-    if (!slot) {
-      byDay.set(key, { total: row.value, latest: row.measuredAt });
-    } else {
-      slot.total += row.value;
-      if (row.measuredAt > slot.latest) slot.latest = row.measuredAt;
-    }
-  }
-
   let winnerTotal: number | null = null;
   let winnerLatest: Date | null = null;
-  for (const { total, latest } of byDay.values()) {
+  const days = iterateDayBuckets(
+    iterateSourcePages<CumulativeRow>(
+      prisma,
+      { userId, type, deletedAt: null },
+      {
+        id: true,
+        value: true,
+        unit: true,
+        measuredAt: true,
+        source: true,
+        externalId: true,
+        deviceType: true,
+      },
+    ),
+    tz,
+    null,
+  );
+  for await (const [, dayRows] of days) {
+    const canonicalRows = metricKey
+      ? pickCanonicalSourceRows(
+          dayRows.map((r) => ({ ...r, type })),
+          metricKey,
+          priorityJson,
+          (d) => dayKeyForUserTz(d, tz),
+        ).canonicalRows
+      : dayRows;
+    if (canonicalRows.length === 0) continue;
+
+    let total = 0;
+    let latest = canonicalRows[0]!.measuredAt;
+    for (const row of canonicalRows) {
+      total += row.value;
+      if (row.measuredAt > latest) latest = row.measuredAt;
+    }
+    // Days arrive in order, and only a strictly better day replaces the
+    // winner, so a tie keeps the earliest day, as before.
     const better =
       winnerTotal === null ||
       (direction === PersonalRecordDirection.MAX

src/lib/validations/backup.ts

@@ -1773,6 +1773,16 @@ export const backupPayloadSchema = z
 
 export type BackupPayload = z.infer<typeof backupPayloadSchema>;
 
+/**
+ * One element of `measurements`, validated on its own. The streamed readers
+ * (`src/lib/export/streamed-backup.ts`) check a large file's measurements
+ * one at a time with this, and the rest of the file with
+ * `backupPayloadSchema` over a document whose measurements are empty, which
+ * together is the same check `backupPayloadSchema` makes of the whole file.
+ */
+export const backupMeasurementSchema = measurementSchema;
+export type BackupMeasurement = z.infer<typeof measurementSchema>;
+
 /**
  * Numeric counts of each backed-up record kind. Returned in the
  * upload + restore API responses so the admin sees what they

tests/integration/apple-health-import-rollup-window.test.ts

@@ -0,0 +1,116 @@
+/**
+ * #1031 — the rollup fold after an import covers what the import wrote.
+ *
+ * The worker folded the rollup table from the account's first measurement to
+ * its last, whatever the import carried, and so also folded history older
+ * than the five-year window every other fold path keeps to. It now folds the
+ * span of the rows this import wrote, clamped to that window.
+ */
+import { createHash } from "node:crypto";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { strToU8, zipSync } from "fflate";
+import type { Job } from "pg-boss";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+import {
+  APPLE_HEALTH_IMPORT_PARSER_REVISION,
+  handleAppleHealthImport,
+  _setWorkerPrismaForTests,
+  type AppleHealthImportPayload,
+} from "@/lib/jobs/apple-health-import-worker";
+import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups";
+
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+vi.mock("@/lib/jobs/boss-instance", () => ({
+  getGlobalBoss: vi.fn(() => null),
+}));
+
+const USER = "user-import-rollup-window";
+
+function record(at: Date, value: number): string {
+  const stamp = at.toISOString().replace("T", " ").slice(0, 19) + " +0000";
+  return `<Record type="HKQuantityTypeIdentifierBodyMass" unit="kg" startDate="${stamp}" endDate="${stamp}" value="${value}" sourceName="Scale"/>`;
+}
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+  _setWorkerPrismaForTests(getPrismaClient());
+  await getPrismaClient().user.create({
+    data: {
+      id: USER,
+      username: USER,
+      email: `${USER}@example.test`,
+      timezone: "UTC",
+    },
+  });
+});
+
+describe("import rollup fold window", () => {
+  it("folds the imported span, never past the fold window, and leaves older history alone", async () => {
+    const prisma = getPrismaClient();
+    const now = Date.now();
+    const day = 86_400_000;
+    // Already in the account before the import: a reading eight years old.
+    await prisma.measurement.create({
+      data: {
+        userId: USER,
+        type: "WEIGHT",
+        value: 90,
+        unit: "kg",
+        source: "MANUAL",
+        measuredAt: new Date(now - 8 * 365 * day),
+      },
+    });
+    // The import: one reading seven years old, a few recent ones.
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<HealthData locale="en_US">
+  ${record(new Date(now - 7 * 365 * day), 88)}
+  ${record(new Date(now - 40 * day), 84)}
+  ${record(new Date(now - 30 * day), 83)}
+  ${record(new Date(now - 20 * day), 82)}
+</HealthData>`;
+    const dir = mkdtempSync(join(tmpdir(), "hl-import-window-"));
+    const path = join(dir, "export.zip");
+    const bytes = Buffer.from(
+      zipSync({ "apple_health_export/export.xml": strToU8(xml) }),
+    );
+    writeFileSync(path, bytes);
+    await prisma.importJob.create({
+      data: {
+        userId: USER,
+        pgBossJobId: "window-job",
+        status: "queued",
+        uploadBytes: bytes.length,
+        uploadSha256: createHash("sha256").update(bytes).digest("hex"),
+        parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
+      },
+    });
+
+    const outcome = await handleAppleHealthImport({
+      id: "window-job",
+      data: {
+        userId: USER,
+        uploadPath: path,
+        uploadBytes: bytes.length,
+        enqueuedAt: new Date().toISOString(),
+      },
+    } as unknown as Job<AppleHealthImportPayload>);
+    expect(outcome.ok).toBe(true);
+
+    const buckets = await prisma.measurementRollup.findMany({
+      where: { userId: USER, type: "WEIGHT", granularity: "DAY" },
+      orderBy: { bucketStart: "asc" },
+    });
+    const windowStart = now - ROLLUP_FOLD_WINDOW_MS;
+    expect(buckets.length).toBeGreaterThanOrEqual(3);
+    for (const bucket of buckets) {
+      expect(bucket.bucketStart.getTime()).toBeGreaterThan(windowStart - day);
+    }
+  });
+});

tests/integration/cumulative-pr-rederive-atomic.test.ts

@@ -0,0 +1,94 @@
+/**
+ * #1031 — the cumulative-record repair must not lose records when the
+ * re-derivation fails.
+ *
+ * The repair deleted every suspect cumulative record, committed, and only
+ * then re-ran the detector. A detector that failed (a timeout on a large
+ * account, a crash) left the records deleted, and because discovery looks
+ * for the suspect rows the deletion removed, the account was never
+ * repaired again: its step, energy and distance records were simply gone.
+ * The delete and the re-derivation now commit together or not at all.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+
+const detect = vi.hoisted(() => ({ fail: false }));
+vi.mock("@/lib/personal-records/pr-detection-worker", async (importActual) => {
+  const actual =
+    await importActual<
+      typeof import("@/lib/personal-records/pr-detection-worker")
+    >();
+  return {
+    ...actual,
+    detectPersonalRecordsForUser: async (
+      ...args: Parameters<typeof actual.detectPersonalRecordsForUser>
+    ) => {
+      if (detect.fail) throw new Error("detector ran out of time");
+      return actual.detectPersonalRecordsForUser(...args);
+    },
+  };
+});
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+const { runCumulativePrRederivationForUser, CUMULATIVE_PR_FIX_CUTOFF } =
+  await import("@/lib/personal-records/cumulative-pr-rederivation");
+
+const USER = "user-pr-rederive-atomic";
+
+beforeEach(async () => {
+  const prisma = getPrismaClient();
+  await truncateAllTables(prisma);
+  detect.fail = false;
+  await prisma.user.create({
+    data: { id: USER, username: USER, email: `${USER}@example.test` },
+  });
+  const before = new Date(CUMULATIVE_PR_FIX_CUTOFF.getTime() - 86_400_000);
+  await prisma.personalRecord.create({
+    data: {
+      userId: USER,
+      metricType: "ACTIVITY_STEPS",
+      direction: "MAX",
+      value: 99_999,
+      unit: "steps",
+      achievedAt: new Date("2025-03-01T12:00:00Z"),
+      createdAt: before,
+    },
+  });
+  // Enough steps for the detector's warm-up gate, one clear best day.
+  await prisma.measurement.createMany({
+    data: Array.from({ length: 10 }, (_, day) => ({
+      userId: USER,
+      type: "ACTIVITY_STEPS" as const,
+      value: day === 4 ? 12_000 : 5_000,
+      unit: "steps",
+      source: "APPLE_HEALTH" as const,
+      measuredAt: new Date(Date.UTC(2025, 3, 1 + day, 12)),
+      externalId: `s-${day}`,
+    })),
+  });
+});
+
+describe("runCumulativePrRederivationForUser", () => {
+  it("keeps the suspect record when the re-derivation fails", async () => {
+    detect.fail = true;
+    await expect(runCumulativePrRederivationForUser(USER)).rejects.toThrow(
+      "detector ran out of time",
+    );
+    const left = await getPrismaClient().personalRecord.findMany({
+      where: { userId: USER },
+    });
+    expect(left.map((r) => r.value)).toEqual([99_999]);
+  });
+
+  it("replaces the suspect record with the re-derived one when it succeeds", async () => {
+    const summary = await runCumulativePrRederivationForUser(USER);
+    expect(summary.rowsDeleted).toBe(1);
+    const left = await getPrismaClient().personalRecord.findMany({
+      where: { userId: USER },
+    });
+    expect(left.map((r) => r.value)).toEqual([12_000]);
+  });
+});

tests/integration/job-failures.test.ts

@@ -16,6 +16,7 @@ import { PgBoss } from "pg-boss";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 
 import {
+  readActiveJobsStartedBefore,
   readFailingQueues,
   readQueueFailureForUser,
 } from "@/lib/jobs/job-failures";
@@ -28,8 +29,16 @@ const OTHER_QUEUE = "reminder-check";
 let boss: PgBoss;
 
 /** Send a job, take it, and fail it terminally so it lands in `state='failed'`. */
-async function failOneJob(queue: string, data: object, error: string) {
-  const jobId = await boss.send(queue, data, { retryLimit: 0 });
+async function failOneJob(
+  queue: string,
+  data: object,
+  error: string,
+  expireInSeconds?: number,
+) {
+  const jobId = await boss.send(queue, data, {
+    retryLimit: 0,
+    ...(expireInSeconds === undefined ? {} : { expireInSeconds }),
+  });
   if (!jobId) throw new Error("queue rejected the job");
   const [job] = await boss.fetch(queue);
   if (!job) throw new Error("nothing to fetch");
@@ -81,6 +90,26 @@ describe("readFailingQueues", () => {
     expect(other?.lastError).toBe("reminder dispatch failed");
   });
 
+  it("says what a bare timeout means, with the job's own limit (#1031)", async () => {
+    await getPrismaClient().$executeRawUnsafe(
+      `DELETE FROM pgboss.job WHERE state = 'failed'`,
+    );
+    // pg-boss writes these words both for a pass too slow for its limit and
+    // for a job whose process died under it; alone they sent an operator
+    // nowhere.
+    await failOneJob(QUEUE, {}, "job timed out", 900);
+    await failOneJob(OTHER_QUEUE, {}, "job timed out", 7200);
+
+    const failing = await readFailingQueues();
+    const fifteen = failing?.find((entry) => entry.queue === QUEUE);
+    expect(fifteen?.lastError).toMatch(
+      /^job timed out: did not finish within its 15 min limit\./,
+    );
+    expect(fifteen?.lastError).toMatch(/restarted/);
+    const twoHours = failing?.find((entry) => entry.queue === OTHER_QUEUE);
+    expect(twoHours?.lastError).toMatch(/within its 2 h limit/);
+  });
+
   it("reports an empty list — not null — when nothing is failing", async () => {
     await getPrismaClient().$executeRawUnsafe(
       `DELETE FROM pgboss.job WHERE state = 'failed'`,
@@ -132,6 +161,34 @@ describe("readQueueFailureForUser", () => {
   });
 });
 
+describe("readActiveJobsStartedBefore (#1031)", () => {
+  it("lists a job still marked active from before the given instant, and nothing after it", async () => {
+    const jobId = await boss.send(
+      OTHER_QUEUE,
+      {},
+      { retryLimit: 0, expireInSeconds: 7200 },
+    );
+    const [job] = await boss.fetch(OTHER_QUEUE);
+    expect(job?.id).toBe(jobId);
+
+    const cutOff = await readActiveJobsStartedBefore(
+      new Date(Date.now() + 60_000),
+    );
+    expect(cutOff).toContainEqual(
+      expect.objectContaining({
+        queue: OTHER_QUEUE,
+        id: jobId,
+        expireSeconds: 7200,
+      }),
+    );
+    await expect(
+      readActiveJobsStartedBefore(new Date(Date.now() - 60 * 60_000)),
+    ).resolves.toEqual([]);
+
+    await boss.complete(OTHER_QUEUE, jobId!);
+  });
+});
+
 describe("no queue schema at all", () => {
   it("answers null rather than throwing, so the status page still renders", async () => {
     const prisma = getPrismaClient();

tests/integration/large-account-jobs.test.ts

@@ -0,0 +1,266 @@
+/**
+ * Issue #1031 — maintenance jobs on a large account.
+ *
+ * A self-hoster with 1.25 million measurement rows (a multi-year Apple Health
+ * export) saw the dense intra-day retention, the nightly drain tick and the
+ * data backup all fail with `job timed out`. Reproduced against a seeded
+ * account of the same size (`scripts/seed-large-account.ts`): the drains held
+ * a type's whole history in memory before folding a day, spent about 90
+ * statements per folded day, and paged with a cursor Postgres could not use as
+ * an index condition; the backup added about 640 MB to the process to store
+ * one 64 MB row. pg-boss declared the jobs dead at their expiry without
+ * stopping them.
+ *
+ * A million rows do not fit a test run, so this file pins the properties the
+ * fix rests on at a scale that does: a pass that runs out of budget stops
+ * cleanly and the next run finishes the job with the same result as one
+ * uninterrupted run, and the backup is stored piecewise with the previous
+ * copy intact if a run fails halfway.
+ */
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention";
+import { consolidateDailyMean } from "@/lib/measurements/consolidate-daily-mean";
+import { unpackBackupBlob } from "@/lib/export/backup-blob";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+
+const TZ = "Europe/Berlin";
+const RUN_A = "user-1031-stopped";
+const RUN_B = "user-1031-straight";
+const DAYS = 12;
+/** Oldest seeded instant: well outside the 90-day raw window. */
+const START = Date.UTC(2026, 0, 1, 0, 7);
+
+beforeEach(async () => {
+  const prisma = getPrismaClient();
+  await truncateAllTables(prisma);
+  for (const id of [RUN_A, RUN_B]) {
+    await prisma.user.create({
+      data: { id, username: id, email: `${id}@example.test`, timezone: TZ },
+    });
+  }
+});
+
+/** The same history for both users: heart rate every 20 minutes, HRV hourly. */
+async function seedHistory(userId: string): Promise<void> {
+  const rows = [];
+  for (let minute = 0; minute < DAYS * 24 * 60; minute += 20) {
+    const at = new Date(START + minute * 60_000);
+    rows.push({
+      userId,
+      type: "PULSE" as const,
+      value: 55 + ((minute / 20) % 40),
+      unit: "bpm",
+      source: "APPLE_HEALTH" as const,
+      measuredAt: at,
+      externalId: `hk-hr-${minute}`,
+    });
+    if (minute % 60 === 0) {
+      rows.push({
+        userId,
+        type: "HEART_RATE_VARIABILITY" as const,
+        value: 30 + ((minute / 60) % 25),
+        unit: "ms",
+        source: "APPLE_HEALTH" as const,
+        measuredAt: at,
+        externalId: `hk-hrv-${minute}`,
+      });
+    }
+    if (minute % 40 === 0) {
+      rows.push({
+        userId,
+        type: "WALKING_SPEED" as const,
+        value: 4 + ((minute / 40) % 10) / 10,
+        unit: "km/h",
+        source: "APPLE_HEALTH" as const,
+        measuredAt: at,
+        externalId: `hk-ws-${minute}`,
+      });
+    }
+  }
+  await getPrismaClient().measurement.createMany({ data: rows });
+}
+
+/** What a reader sees: every live row, identity-free. */
+async function liveRows(userId: string) {
+  const rows = await getPrismaClient().measurement.findMany({
+    where: { userId, deletedAt: null },
+    select: {
+      type: true,
+      value: true,
+      unit: true,
+      source: true,
+      measuredAt: true,
+      externalId: true,
+    },
+    orderBy: [{ type: "asc" }, { measuredAt: "asc" }],
+  });
+  return rows.map((row) => ({
+    ...row,
+    measuredAt: row.measuredAt.toISOString(),
+  }));
+}
+
+/** A `shouldStop` that lets `days` day buckets through, then says stop. */
+function afterDays(days: number): () => boolean {
+  let left = days;
+  return () => left-- <= 0;
+}
+
+describe("a pass that runs out of budget (#1031)", () => {
+  it("dense retention: stops cleanly, and the next run ends where one straight run ends", async () => {
+    await seedHistory(RUN_A);
+    await seedHistory(RUN_B);
+    const prisma = getPrismaClient();
+
+    const first = await runDenseIntradayRetention(prisma, {
+      userId: RUN_A,
+      shouldStop: afterDays(5),
+      log: () => {},
+    });
+    expect(first.stoppedEarly).toBe(true);
+    expect(first.totals.daysConsolidated).toBe(5);
+
+    // What the stopped run folded is committed: the next scan no longer sees
+    // it, so the second run folds only what is left.
+    const second = await runDenseIntradayRetention(prisma, {
+      userId: RUN_A,
+      log: () => {},
+    });
+    expect(second.stoppedEarly).toBe(false);
+
+    const straight = await runDenseIntradayRetention(prisma, {
+      userId: RUN_B,
+      log: () => {},
+    });
+    expect(first.totals.daysConsolidated + second.totals.daysConsolidated).toBe(
+      straight.totals.daysConsolidated,
+    );
+    expect(
+      first.totals.perSampleRowsSoftDeleted +
+        second.totals.perSampleRowsSoftDeleted,
+    ).toBe(straight.totals.perSampleRowsSoftDeleted);
+
+    // A third run finds nothing: the pass converged.
+    const third = await runDenseIntradayRetention(prisma, {
+      userId: RUN_A,
+      log: () => {},
+    });
+    expect(third.totals.daysConsolidated).toBe(0);
+
+    const stoppedResult = (await liveRows(RUN_A)).filter(
+      (row) => row.type !== "WALKING_SPEED",
+    );
+    const straightResult = (await liveRows(RUN_B)).filter(
+      (row) => row.type !== "WALKING_SPEED",
+    );
+    expect(stoppedResult.length).toBeGreaterThan(0);
+    expect(stoppedResult).toEqual(straightResult);
+  });
+
+  it("daily-mean consolidation: the same, for the nightly tick's middle pass", async () => {
+    await seedHistory(RUN_A);
+    await seedHistory(RUN_B);
+    const prisma = getPrismaClient();
+
+    const first = await consolidateDailyMean(prisma, {
+      userId: RUN_A,
+      shouldStop: afterDays(4),
+      log: () => {},
+    });
+    expect(first.stoppedEarly).toBe(true);
+    expect(first.totals.daysConsolidated).toBe(4);
+    await consolidateDailyMean(prisma, { userId: RUN_A, log: () => {} });
+    await consolidateDailyMean(prisma, { userId: RUN_B, log: () => {} });
+
+    const stopped = (await liveRows(RUN_A)).filter(
+      (row) => row.type === "WALKING_SPEED",
+    );
+    const straight = (await liveRows(RUN_B)).filter(
+      (row) => row.type === "WALKING_SPEED",
+    );
+    // One daily mean per local day: twelve UTC days touch thirteen Berlin days.
+    expect(stopped).toHaveLength(DAYS + 1);
+    expect(stopped).toEqual(straight);
+  });
+});
+
+describe("storeBackupBlob (#1031)", () => {
+  /** A producer writing `rows` JSON rows in small pieces. */
+  function producer(rows: number, failAfter?: number) {
+    return async (write: (chunk: string) => Promise<void>) => {
+      await write('{"measurements":[');
+      for (let i = 0; i < rows; i++) {
+        if (failAfter !== undefined && i === failAfter) {
+          throw new Error("record read failed halfway");
+        }
+        await write(
+          `${i === 0 ? "" : ","}${JSON.stringify({ id: `m-${i}`, value: i })}`,
+        );
+      }
+      await write("]}");
+    };
+  }
+
+  function expectedJson(rows: number): string {
+    return JSON.stringify({
+      measurements: Array.from({ length: rows }, (_, i) => ({
+        id: `m-${i}`,
+        value: i,
+      })),
+    });
+  }
+
+  it("stores a copy that reads back as exactly the JSON produced", async () => {
+    const prisma = getPrismaClient();
+    const { bytes } = await storeBackupBlob(
+      prisma,
+      { userId: RUN_A, type: "WEEKLY_AUTO" },
+      producer(20_000),
+    );
+
+    const row = await prisma.dataBackup.findUniqueOrThrow({
+      where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
+    });
+    expect(row.data.length).toBe(bytes);
+    expect(unpackBackupBlob(row.data)).toBe(expectedJson(20_000));
+  });
+
+  it("keeps the previous copy when a run fails halfway, and the next run still stores", async () => {
+    const prisma = getPrismaClient();
+    await storeBackupBlob(
+      prisma,
+      { userId: RUN_A, type: "WEEKLY_AUTO" },
+      producer(10),
+    );
+    const before = await prisma.dataBackup.findUniqueOrThrow({
+      where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
+    });
+
+    await expect(
+      storeBackupBlob(
+        prisma,
+        { userId: RUN_A, type: "WEEKLY_AUTO" },
+        producer(20_000, 15_000),
+      ),
+    ).rejects.toThrow("record read failed halfway");
+
+    const after = await prisma.dataBackup.findUniqueOrThrow({
+      where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
+    });
+    expect(after.data).toBe(before.data);
+    expect(after.createdAt.getTime()).toBe(before.createdAt.getTime());
+
+    // The temporary table went with the failed transaction.
+    await storeBackupBlob(
+      prisma,
+      { userId: RUN_A, type: "WEEKLY_AUTO" },
+      producer(30),
+    );
+    const replaced = await prisma.dataBackup.findUniqueOrThrow({
+      where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
+    });
+    expect(unpackBackupBlob(replaced.data)).toBe(expectedJson(30));
+  });
+});

tests/integration/large-backup-round-trip.test.ts

@@ -0,0 +1,203 @@
+/**
+ * #1031 — an account too large to export or restore.
+ *
+ * For an account of 1.25 million measurements the full export took the app
+ * down, and its weekly backup could not be restored, downloaded or summarised
+ * at all: the backup's JSON (662 MB) is longer than any string V8 can hold,
+ * and every reader turned the file into one string first. Measured on a
+ * seeded account of that size in a 1 GB container, the streamed paths now
+ * export in 25 s, upload in under 20 s and restore in 94 s, and the restored
+ * rows match the file exactly.
+ *
+ * A million rows do not fit a test run. This file runs the same paths at a
+ * size that spans many pages and batches, and checks the rows come back
+ * exactly: the export streamed into the response, uploaded as gzip through
+ * the raw-body path, and restored through the streamed two-pass reader; and a
+ * weekly backup restored the same way, every column intact, with a personal
+ * record still pointing at the measurement it was found in.
+ */
+import { gzipSync } from "node:zlib";
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+process.env.ENCRYPTION_KEY ??=
+  "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+import { cookieJar, headerJar } from "./mock-next-headers";
+import { getPrismaClient, truncateAllTables } from "./setup";
+import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
+import { storeBackupBlob } from "@/lib/export/store-backup-blob";
+import { encryptNote } from "@/lib/crypto/note-cipher";
+
+vi.mock("next/headers", async () => {
+  const { cookieJar, headerJar } = await import("./mock-next-headers");
+  return {
+    headers: vi.fn(async () => ({
+      get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
+    })),
+    cookies: vi.fn(async () => ({
+      get: (name: string) => {
+        const value = cookieJar.get(name);
+        return value ? { name, value } : undefined;
+      },
+      set: (name: string, value: string) => {
+        cookieJar.set(name, value);
+      },
+      delete: (name: string) => {
+        cookieJar.delete(name);
+      },
+    })),
+  };
+});
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+const COUNT = 7_300; // more than one read page and several write batches
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+  cookieJar.clear();
+  headerJar.clear();
+});
+
+async function seedAccount() {
+  const prisma = getPrismaClient();
+  const user = await prisma.user.create({
+    data: {
+      username: "large-account",
+      email: "large-account@example.test",
+      role: "ADMIN",
+    },
+  });
+  const session = await prisma.session.create({
+    data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) },
+  });
+  cookieJar.set("healthlog_session", session.id);
+
+  const start = Date.UTC(2024, 0, 1);
+  await prisma.measurement.createMany({
+    data: Array.from({ length: COUNT }, (_, i) => ({
+      id: `m-${String(i).padStart(5, "0")}`,
+      userId: user.id,
+      type: i % 3 === 0 ? ("PULSE" as const) : ("SLEEP_DURATION" as const),
+      value: 50 + (i % 40) + 0.25,
+      valueMax: i % 3 === 0 ? 120 : null,
+      unit: i % 3 === 0 ? "bpm" : "minutes",
+      source: "APPLE_HEALTH" as const,
+      measuredAt: new Date(start + i * 300_000),
+      externalId: `hk-${i}`,
+      sleepStage: i % 3 === 0 ? null : ("CORE" as const),
+      deviceType: "watch",
+      syncVersion: 1 + (i % 4),
+      notesEncrypted: i === 17 ? encryptNote("after the long run") : null,
+      deletedAt: i % 97 === 0 ? new Date(start) : null,
+    })),
+  });
+  await prisma.personalRecord.create({
+    data: {
+      userId: user.id,
+      metricType: "PULSE",
+      direction: "MAX",
+      value: 89.25,
+      unit: "bpm",
+      achievedAt: new Date(start + 3 * 300_000),
+      sourceMeasurementId: "m-00003",
+    },
+  });
+  return user;
+}
+
+async function measurementsOf(userId: string) {
+  return getPrismaClient().measurement.findMany({
+    where: { userId },
+    orderBy: { id: "asc" },
+    omit: { createdAt: true, updatedAt: true },
+  });
+}
+
+async function restore(id: string) {
+  const { POST } = await import("@/app/api/admin/backups/[id]/restore/route");
+  const res = await POST(
+    new Request(`http://localhost/api/admin/backups/${id}/restore`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ confirm: "RESTORE" }),
+    }) as never,
+    { params: Promise.resolve({ id }) },
+  );
+  return res;
+}
+
+describe("large backups round-trip through the streamed paths (#1031)", () => {
+  it("export → gzip upload → restore gives back every live measurement", async () => {
+    const user = await seedAccount();
+    const live = (await measurementsOf(user.id)).filter((m) => !m.deletedAt);
+
+    const { GET } = await import("@/app/api/export/full-backup/route");
+    const exported = await GET(
+      new Request("http://localhost/api/export/full-backup") as never,
+    );
+    expect(exported.status).toBe(200);
+    const file = Buffer.from(await exported.arrayBuffer());
+    expect(JSON.parse(file.toString("utf8")).measurements).toHaveLength(
+      live.length,
+    );
+
+    const { POST: upload } =
+      await import("@/app/api/admin/backups/upload/route");
+    const uploaded = await upload(
+      new Request("http://localhost/api/admin/backups/upload", {
+        method: "POST",
+        headers: { "content-type": "application/gzip" },
+        body: gzipSync(file),
+      }) as never,
+    );
+    expect(uploaded.status).toBe(201);
+    const { data } = (await uploaded.json()) as {
+      data: { id: string; summary: { measurements: number } };
+    };
+    expect(data.summary.measurements).toBe(live.length);
+
+    expect((await restore(data.id)).status).toBe(200);
+    const restored = await measurementsOf(user.id);
+    // A portable file carries the fields a person reads, not the sync
+    // bookkeeping, so those are what must come back.
+    const readable = (rows: typeof restored) =>
+      rows.map((m) => ({
+        id: m.id,
+        type: m.type,
+        value: m.value,
+        unit: m.unit,
+        measuredAt: m.measuredAt.toISOString(),
+        source: m.source,
+      }));
+    expect(readable(restored)).toEqual(readable(live));
+  });
+
+  it("a weekly backup restores every column, and a record's pointer survives", async () => {
+    const user = await seedAccount();
+    const before = await measurementsOf(user.id);
+    const prisma = getPrismaClient();
+
+    const { id } = await storeBackupBlob(
+      prisma,
+      { userId: user.id, type: "WEEKLY_AUTO" },
+      (write) =>
+        streamFullBackupJson(prisma, user.id, write, {
+          purpose: "disaster-recovery",
+        }),
+    );
+
+    // Change the account so the restore has something to undo.
+    await prisma.measurement.deleteMany({ where: { userId: user.id } });
+
+    const res = await restore(id);
+    expect(res.status).toBe(200);
+    expect(await measurementsOf(user.id)).toEqual(before);
+    const record = await prisma.personalRecord.findFirstOrThrow({
+      where: { userId: user.id },
+    });
+    expect(record.sourceMeasurementId).toBe("m-00003");
+  });
+});

tests/integration/measurement-bulk-insert.test.ts

@@ -0,0 +1,139 @@
+/**
+ * `insertMeasurementRows` replaced `createMany` in the restore because
+ * Prisma's cached plan for a 1 000-row `createMany` kept about 145 MB alive
+ * (#1031). It writes raw SQL, so this pins that a row written through it
+ * reads back exactly as the same row written through Prisma, every column
+ * set, and every column null where it may be.
+ */
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+import {
+  insertMeasurementRows,
+  type MeasurementInsertRow,
+} from "@/lib/export/measurement-bulk-insert";
+
+const USER = "user-bulk-insert";
+
+beforeEach(async () => {
+  const prisma = getPrismaClient();
+  await truncateAllTables(prisma);
+  await prisma.user.create({
+    data: { id: USER, username: USER, email: `${USER}@example.test` },
+  });
+});
+
+function full(id: string, at: string): MeasurementInsertRow {
+  return {
+    id,
+    userId: USER,
+    type: "SLEEP_DURATION",
+    value: 47.25,
+    valueMin: 1.5,
+    valueMax: 99.75,
+    unit: "minutes",
+    source: "APPLE_HEALTH",
+    measuredAt: new Date(at),
+    notes: "legacy note",
+    notesEncrypted: new Uint8Array([0, 1, 2, 254, 255]),
+    externalId: `ext-${id}`,
+    externalSourceVersion: "17.4",
+    aggregationProvenance: "HEALTHKIT_STATISTICS",
+    glucoseContext: null,
+    sleepStage: "REM",
+    rhythmClassification: null,
+    deviceType: "watch",
+    syncVersion: 7,
+    deletedAt: new Date("2026-02-03T04:05:06.789Z"),
+    createdAt: new Date("2025-01-02T03:04:05.678Z"),
+    updatedAt: new Date("2025-06-07T08:09:10.111Z"),
+  };
+}
+
+function sparse(id: string, at: string): MeasurementInsertRow {
+  return {
+    id,
+    userId: USER,
+    type: "BLOOD_GLUCOSE",
+    value: 5.4,
+    valueMin: null,
+    valueMax: null,
+    unit: "mmol/L",
+    source: "MANUAL",
+    measuredAt: new Date(at),
+    notes: null,
+    notesEncrypted: null,
+    externalId: null,
+    externalSourceVersion: null,
+    aggregationProvenance: null,
+    glucoseContext: "FASTING",
+    sleepStage: null,
+    rhythmClassification: null,
+    deviceType: null,
+    syncVersion: 1,
+    deletedAt: null,
+  };
+}
+
+describe("insertMeasurementRows", () => {
+  it("writes every column exactly as Prisma create does", async () => {
+    const prisma = getPrismaClient();
+    const rows = [
+      [
+        full("a-bulk", "2026-01-01T10:00:00.123Z"),
+        full("a-prisma", "2026-01-02T10:00:00.123Z"),
+      ],
+      [
+        sparse("b-bulk", "2026-01-03T10:00:00.000Z"),
+        sparse("b-prisma", "2026-01-04T10:00:00.000Z"),
+      ],
+    ] as const;
+
+    for (const [viaBulk, viaPrisma] of rows) {
+      await prisma.$transaction((tx) => insertMeasurementRows(tx, [viaBulk]));
+      await prisma.measurement.create({ data: viaPrisma as never });
+    }
+
+    for (const [viaBulk, viaPrisma] of rows) {
+      const a = await prisma.measurement.findUniqueOrThrow({
+        where: { id: viaBulk.id },
+      });
+      const b = await prisma.measurement.findUniqueOrThrow({
+        where: { id: viaPrisma.id },
+      });
+      const strip = (row: typeof a) => ({
+        ...row,
+        id: undefined,
+        externalId: row.externalId?.replace(/-(bulk|prisma)$/, ""),
+        measuredAt: undefined,
+        // Filled with the current time when the row carries none.
+        ...(viaBulk.createdAt
+          ? {}
+          : { createdAt: undefined, updatedAt: undefined }),
+      });
+      expect(strip(a)).toEqual(strip(b));
+      expect(a.measuredAt.toISOString()).toBe(viaBulk.measuredAt.toISOString());
+    }
+    const sparseRow = await prisma.measurement.findUniqueOrThrow({
+      where: { id: "b-bulk" },
+    });
+    expect(Date.now() - sparseRow.createdAt.getTime()).toBeLessThan(60_000);
+  });
+
+  it("inserts a whole batch in one statement and reports the count", async () => {
+    const prisma = getPrismaClient();
+    const batch = Array.from({ length: 2_500 }, (_, i) =>
+      sparse(
+        `m-${i}`,
+        new Date(Date.UTC(2026, 0, 1) + i * 60_000).toISOString(),
+      ),
+    );
+    const inserted = await prisma.$transaction((tx) =>
+      insertMeasurementRows(tx, batch),
+    );
+    expect(inserted).toBe(2_500);
+    expect(await prisma.measurement.count({ where: { userId: USER } })).toBe(
+      2_500,
+    );
+  });
+});

tests/integration/offhost-backup-ledger.test.ts

@@ -228,4 +228,50 @@ describe("off-host backup ledger (real Postgres)", () => {
     await prisma.user.delete({ where: { id: "account-one" } });
     expect(await prisma.offhostBackupState.count()).toBe(0);
   });
+
+  it("a retry of the same run resumes after the accounts it already uploaded (#1031)", async () => {
+    const prisma = getPrismaClient();
+    for (const id of ["acct-a", "acct-b", "acct-c"]) await seedUser(id);
+    const bucket = makeBucket();
+    const puts: string[] = [];
+    const putStream = bucket.putStream;
+    bucket.putStream = async (key, body) => {
+      puts.push(key);
+      return putStream(key, body);
+    };
+    // The job was created a moment ago; its retries share that instant.
+    const runStartedAt = new Date(Date.now() - 60_000);
+    const day = runStartedAt.toISOString().slice(0, 10);
+
+    // First attempt: out of budget after one account.
+    let asked = 0;
+    const first = await runOffhostBackup(prisma, bucket, new Date(), {
+      runStartedAt,
+      shouldStop: () => asked++ >= 1,
+    });
+    expect(first.stoppedEarly).toBe(true);
+    expect(first.uploaded).toBe(1);
+
+    // The retry: same run, so the account already done is skipped.
+    const retry = await runOffhostBackup(prisma, bucket, new Date(), {
+      runStartedAt,
+    });
+    expect(retry.stoppedEarly).toBe(false);
+    expect(retry.alreadyUploaded).toBe(1);
+    expect(retry.uploaded).toBe(2);
+
+    // Each account uploaded exactly once, all under the run's own day.
+    expect(puts.sort()).toEqual([
+      `${day}/user-acct-a.json.enc`,
+      `${day}/user-acct-b.json.enc`,
+      `${day}/user-acct-c.json.enc`,
+    ]);
+
+    // A new run (a later job) uploads everybody again.
+    const next = await runOffhostBackup(prisma, bucket, new Date(), {
+      runStartedAt: new Date(),
+    });
+    expect(next.alreadyUploaded).toBe(0);
+    expect(next.uploaded).toBe(3);
+  });
 });

tests/integration/pr-detection-cumulative-paged.test.ts

@@ -0,0 +1,111 @@
+/**
+ * #1031 — the best cumulative day, found a day at a time.
+ *
+ * `findBestCumulativeDay` read every row of a cumulative type into memory,
+ * every half hour, for every account. On an account holding 784 000 active
+ * energy rows it ran a 524 MB heap out of memory. It now reads a local day at
+ * a time in time order, which gives the same answer because the canonical-
+ * source pick it applies is per day.
+ *
+ * Pinned here at a size that spans several read pages: two sources report
+ * the same days, the source ladder must decide each day on its own, the
+ * best day sits in the middle of the history, and no read of the history is
+ * unbounded.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+const { detectPersonalRecordsForUser } =
+  await import("@/lib/personal-records/pr-detection-worker");
+
+const USER = "user-pr-cumulative-paged";
+const DAYS = 40;
+const PER_DAY = 300; // 12 000 Apple rows plus a Withings total per day
+
+beforeEach(async () => {
+  const prisma = getPrismaClient();
+  await truncateAllTables(prisma);
+  await prisma.user.create({
+    data: {
+      id: USER,
+      username: USER,
+      email: `${USER}@example.test`,
+      timezone: "UTC",
+    },
+  });
+});
+
+describe("best cumulative day across many pages", () => {
+  it("sums each day's canonical source and finds the best day without an unbounded read", async () => {
+    const prisma = getPrismaClient();
+    const start = Date.UTC(2025, 0, 1, 6);
+    const rows = [];
+    for (let day = 0; day < DAYS; day++) {
+      // Day 23 is the best Apple day; the Withings total on day 5 is larger
+      // still but loses to Apple on that day by the default ladder, so it
+      // must not win: if the days were pooled across sources it would.
+      const perSample = day === 23 ? 40 : 20;
+      for (let i = 0; i < PER_DAY; i++) {
+        rows.push({
+          userId: USER,
+          type: "ACTIVITY_STEPS" as const,
+          value: perSample,
+          unit: "steps",
+          source: "APPLE_HEALTH" as const,
+          measuredAt: new Date(start + day * 86_400_000 + i * 60_000),
+          externalId: `hk-${day}-${i}`,
+        });
+      }
+      rows.push({
+        userId: USER,
+        type: "ACTIVITY_STEPS" as const,
+        value: day === 5 ? 50_000 : 1_000,
+        unit: "steps",
+        source: "WITHINGS" as const,
+        measuredAt: new Date(start + day * 86_400_000 + 12 * 3_600_000),
+        externalId: `w-${day}`,
+      });
+    }
+    await prisma.measurement.createMany({ data: rows });
+
+    // Every read of the steps history, through a client that records it.
+    const reads: Array<number | undefined> = [];
+    const recording = prisma.$extends({
+      query: {
+        measurement: {
+          findMany({ args, query }) {
+            if (JSON.stringify(args.where ?? {}).includes("ACTIVITY_STEPS")) {
+              reads.push(args.take);
+            }
+            return query(args);
+          },
+        },
+      },
+    });
+
+    await detectPersonalRecordsForUser(USER, {
+      silent: true,
+      prisma: recording as unknown as typeof prisma,
+    });
+
+    const record = await prisma.personalRecord.findFirstOrThrow({
+      where: { userId: USER, metricType: "ACTIVITY_STEPS" },
+    });
+    expect(record.value).toBe(40 * PER_DAY);
+    expect(record.achievedAt.toISOString()).toBe(
+      new Date(start + 23 * 86_400_000 + (PER_DAY - 1) * 60_000).toISOString(),
+    );
+
+    // Several bounded pages, never one read of the whole history.
+    expect(reads.length).toBeGreaterThan(1);
+    for (const take of reads) {
+      expect(take).toBeDefined();
+      expect(take!).toBeLessThanOrEqual(5_000);
+    }
+  });
+});