diff --git a/docs/ops/encryption-key-rotation.md b/docs/ops/encryption-key-rotation.md index 007dd4fb9..5a62c55e0 100644 --- a/docs/ops/encryption-key-rotation.md +++ b/docs/ops/encryption-key-rotation.md @@ -60,8 +60,10 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`). Read three things off the output before going further: - - the per-column line, `scanned` / `rotated` / `errors` / `dropped`. Treat - `errors > 0` as a hard failure and re-run after fixing the cause. A + - the per-column line, `scanned` / `rotated` / `errors` / `dropped`. + `scanned` counts the rows that hold ciphertext in that column, not every + row of the table. Treat `errors > 0` as a hard failure and re-run after + fixing the cause. A `dropped` count is only ever a cache row the run could not read and deleted rather than leave unreadable. - `Columns walked: N/M registered`. The two numbers must match. diff --git a/messages/de.json b/messages/de.json index 7c7a8da86..2629c846f 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Backup hochladen", "uploadDescription": "Eine Backup-Datei validieren und als manuellen Snapshot speichern. Das Wiederherstellen ist ein separater Schritt.", "uploadButton": "Datei hochladen", - "uploadHelp": "JSON-Datei passend zum aktuellen Backup-Schema. Max. 10 MB.", + "uploadHelp": "HealthLog-Backup als .json oder .json.gz. Die Datei wird vor dem Hochladen komprimiert; komprimiert darf sie bis zu 512 MB groß sein.", "uploadSuccess": "Backup hochgeladen ({count} Datensätze)", "uploadFailed": "Upload fehlgeschlagen", "restore": "Wiederherstellen", diff --git a/messages/en.json b/messages/en.json index b84e3eb1d..b2d727fc7 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Upload backup", "uploadDescription": "Validate a backup file and store it as a manual snapshot. Restoring is a separate step.", "uploadButton": "Upload file", - "uploadHelp": "JSON file matching the current backup schema. Max 10 MB.", + "uploadHelp": "A HealthLog backup as .json or .json.gz. The file is compressed before upload; compressed, it may be up to 512 MB.", "uploadSuccess": "Backup uploaded ({count} records)", "uploadFailed": "Upload failed", "restore": "Restore", diff --git a/messages/es.json b/messages/es.json index 57961bd23..638d44389 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Subir copia de seguridad", "uploadDescription": "Valida un archivo de copia de seguridad y guárdalo como instantánea manual. La restauración es un paso aparte.", "uploadButton": "Subir archivo", - "uploadHelp": "Archivo JSON que coincida con el esquema de copia de seguridad actual. Máx. 10 MB.", + "uploadHelp": "Una copia de seguridad de HealthLog en .json o .json.gz. El archivo se comprime antes de subirlo; comprimido puede ocupar hasta 512 MB.", "uploadSuccess": "Copia de seguridad subida ({count} registros)", "uploadFailed": "La subida falló", "restore": "Restaurar", diff --git a/messages/fr.json b/messages/fr.json index a5cebbec3..a91aaba73 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Téléverser une sauvegarde", "uploadDescription": "Validez un fichier de sauvegarde et stockez-le comme instantané manuel. La restauration est une étape distincte.", "uploadButton": "Téléverser un fichier", - "uploadHelp": "Fichier JSON correspondant au schéma de sauvegarde actuel. Max 10 Mo.", + "uploadHelp": "Une sauvegarde HealthLog au format .json ou .json.gz. Le fichier est compressé avant l’envoi ; compressé, il peut atteindre 512 Mo.", "uploadSuccess": "Sauvegarde téléversée ({count} enregistrements)", "uploadFailed": "Échec du téléversement", "restore": "Restaurer", diff --git a/messages/it.json b/messages/it.json index b41bc95c7..ba55d1bd9 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Carica backup", "uploadDescription": "Convalida un file di backup e archivialo come istantanea manuale. Il ripristino è un passaggio separato.", "uploadButton": "Carica file", - "uploadHelp": "File JSON conforme allo schema di backup attuale. Max 10 MB.", + "uploadHelp": "Un backup di HealthLog in formato .json o .json.gz. Il file viene compresso prima del caricamento; compresso può arrivare a 512 MB.", "uploadSuccess": "Backup caricato ({count} record)", "uploadFailed": "Caricamento non riuscito", "restore": "Ripristina", diff --git a/messages/ko.json b/messages/ko.json index 3c418574d..78925959c 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7146,7 +7146,7 @@ "uploadTitle": "백업 업로드", "uploadDescription": "백업 파일을 검사한 뒤 수동 스냅샷으로 저장해요. 복원은 별도 단계예요.", "uploadButton": "파일 업로드", - "uploadHelp": "현재 백업 스키마에 맞는 JSON 파일이에요. 최대 10 MB.", + "uploadHelp": "HealthLog 백업 파일(.json 또는 .json.gz)이에요. 업로드 전에 압축되며, 압축된 크기는 최대 512 MB까지 가능해요.", "uploadSuccess": "백업을 업로드했어요 (기록 {count}개)", "uploadFailed": "업로드하지 못했어요", "restore": "복원", diff --git a/messages/pl.json b/messages/pl.json index 765f55ccc..89b15b449 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Prześlij kopię zapasową", "uploadDescription": "Sprawdź plik kopii zapasowej i zapisz go jako ręczną migawkę. Przywracanie to osobny krok.", "uploadButton": "Prześlij plik", - "uploadHelp": "Plik JSON zgodny z bieżącym schematem kopii zapasowej. Maks. 10 MB.", + "uploadHelp": "Kopia zapasowa HealthLog w formacie .json lub .json.gz. Plik jest kompresowany przed wysłaniem; po kompresji może mieć do 512 MB.", "uploadSuccess": "Przesłano kopię zapasową ({count} rekordów)", "uploadFailed": "Przesyłanie nie powiodło się", "restore": "Przywróć", diff --git a/scripts/restore-backup.ts b/scripts/restore-backup.ts index 354da0fff..c6f4a51fe 100644 --- a/scripts/restore-backup.ts +++ b/scripts/restore-backup.ts @@ -15,12 +15,23 @@ * BACKUP_S3_REGION=auto \ * BACKUP_ENCRYPTION_KEY= \ * pnpm dlx tsx scripts/restore-backup.ts 2026-05-08/user-clx123.json.enc /tmp/restored.json + * + * An output name ending in `.gz` writes the JSON compressed, which is the + * form to hand to the admin upload for a large account: the JSON of 1.25 + * million measurements is 662 MB, the compressed file about a tenth of it. + * The file is written as a stream in either case and never held whole. */ -import { writeFileSync } from "node:fs"; +import { createWriteStream } from "node:fs"; +import { rename } from "node:fs/promises"; +import { PassThrough, Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGzip } from "node:zlib"; + +import { scanBackupJson } from "@/lib/export/backup-json-scan"; import { loadOffhostConfig, getS3Client, - decryptBackup, + openBackupObject, OffhostBackupNotConfiguredError, } from "@/lib/jobs/offhost-backup"; @@ -98,28 +109,43 @@ async function main() { `Downloading s3://${cfg.bucket}/${key} from ${cfg.endpoint} (region=${cfg.region})`, ); const ciphertext = await s3.getObject(key); - const plaintext = decryptBackup(ciphertext, cfg.encryptionKey); + const source = openBackupObject(ciphertext, cfg.encryptionKey); - let parsed: unknown; - try { - parsed = JSON.parse(plaintext); - } catch (err) { - throw new Error( - `Decrypted payload is not valid JSON: ${(err as Error).message}`, - ); - } - const body = validateBackupShape(parsed); + // Checked first, in one streamed pass that keeps only the small sections, + // so nothing is written for an object that is not a backup. + const { document, streamedCounts } = await scanBackupJson(source(), { + streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), + }); + const body = validateBackupShape({ + ...document, + measurements: [], + intakeEvents: [], + moodEntries: [], + }); if (expectedUserId && body.userId !== expectedUserId) { throw new Error( `Refusing to write: backup.userId='${body.userId}' does not match --user-id='${expectedUserId}'`, ); } - // The restored file is decrypted PHI in plaintext. Write it 0o600 so it is - // not world-readable on the operator's disk while it sits there. - writeFileSync(out, plaintext, { encoding: "utf8", mode: 0o600 }); + // The restored file is decrypted PHI in plaintext. Written 0o600 so it is + // not world-readable on the operator's disk, to a temporary name first so + // an interrupted run never leaves a truncated file under the real one. + const partial = `${out}.part`; + let bytes = 0; + const counter = new PassThrough(); + counter.on("data", (chunk: Buffer) => { + bytes += chunk.length; + }); + await pipeline( + Readable.from(source()), + counter, + ...(out.endsWith(".gz") ? [createGzip()] : []), + createWriteStream(partial, { mode: 0o600 }), + ); + await rename(partial, out); console.log( - `Restored userId=${body.userId} (${plaintext.length} bytes) -> ${out}`, + `Restored userId=${body.userId} (${bytes} bytes of JSON, ${streamedCounts.measurements ?? 0} measurements) -> ${out}`, ); } diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index 1546d0e90..616af8582 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -108,6 +108,49 @@ function shouldRotate(value: string | null): boolean { return id !== getActiveKeyId(); } +/** The part of a Prisma delegate the per-column walks below need. */ +interface PagedDelegate { + findMany: (args: { + select: Record; + orderBy: { id: "asc" }; + take: number; + cursor?: { id: string }; + skip?: number; + }) => Promise>>; + update: (args: { + where: { id: string }; + data: Record; + }) => Promise; +} + +/** Rows per page when walking a column. */ +const PAGE_SIZE = 5_000; + +/** + * Every row of one column, `id` and value only, a page at a time in id order. + * The walks used to read a column in one `findMany`, which for + * `Measurement.notesEncrypted` is every measurement of the instance: 1.25 + * million rows on one large account (#1031). + */ +async function* pagedRows( + delegate: PagedDelegate, + field: string, +): AsyncGenerator> { + let cursor: string | null = null; + for (;;) { + const rows = await delegate.findMany({ + select: { id: true, [field]: true }, + orderBy: { id: "asc" }, + take: PAGE_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }); + if (rows.length === 0) return; + yield* rows; + if (rows.length < PAGE_SIZE) return; + cursor = rows[rows.length - 1]!.id as string; + } +} + /** * Rotate one `String` ciphertext column on a Prisma model. `delegate` is the * `prisma.` accessor; `field` is the column. Reads only `id` + the @@ -116,31 +159,28 @@ function shouldRotate(value: string | null): boolean { async function rotateStringColumn( table: string, field: string, - delegate: { - findMany: (args: { - select: Record; - }) => Promise>>; - update: (args: { - where: { id: string }; - data: Record; - }) => Promise; - }, + delegate: PagedDelegate, ): Promise { - const rows = await delegate.findMany({ select: { id: true, [field]: true } }); const result: RotationResult = { table, field, - scanned: rows.length, + scanned: 0, rotated: 0, errors: 0, dropped: 0, }; - for (const row of rows) { + for await (const row of pagedRows( + delegate as PagedDelegate, + field, + )) { const v = row[field] as string | null; + // `scanned` counts rows that hold ciphertext, as the in-app rotation does. + if (!v) continue; + result.scanned++; if (!shouldRotate(v)) continue; const id = row.id as string; try { - const re = encrypt(decrypt(v as string)); + const re = encrypt(decrypt(v)); await delegate.update({ where: { id }, data: { [field]: re } }); result.rotated++; } catch (err) { @@ -159,28 +199,23 @@ async function rotateStringColumn( async function rotateBytesColumn( table: string, field: string, - delegate: { - findMany: (args: { - select: Record; - }) => Promise>>; - update: (args: { - where: { id: string }; - data: Record; - }) => Promise; - }, + delegate: PagedDelegate, ): Promise { - const rows = await delegate.findMany({ select: { id: true, [field]: true } }); const result: RotationResult = { table, field, - scanned: rows.length, + scanned: 0, rotated: 0, errors: 0, dropped: 0, }; - for (const row of rows) { + for await (const row of pagedRows( + delegate as PagedDelegate, + field, + )) { const buf = row[field] as Uint8Array | null; if (!buf || buf.byteLength === 0) continue; + result.scanned++; const asString = Buffer.from(buf).toString("utf8"); if (!shouldRotate(asString)) continue; const id = row.id as string; diff --git a/scripts/seed-large-account.ts b/scripts/seed-large-account.ts new file mode 100644 index 000000000..fba2c078c --- /dev/null +++ b/scripts/seed-large-account.ts @@ -0,0 +1,296 @@ +/** + * Seed one account shaped like a multi-year Apple Health `export.xml` import, + * for reproducing the maintenance jobs at the size a real heavy account + * reaches (issue #1031: about 1.26 million measurement rows). + * + * DATABASE_URL=postgresql://… pnpm dlx tsx scripts/seed-large-account.ts \ + * [--user bench-large] [--years 4] [--pulse-seconds 165] + * + * Never point this at a database that holds real data: it inserts rows under + * the given user id and, with `--reset`, deletes that user first. + * + * The shape follows what the import and the iOS sync actually write: + * + * - dense spot rows (heart rate every `--pulse-seconds`, HRV, SpO2) keyed by + * a hashed sample id, the tier the dense-intraday retention folds; + * - high-frequency mean rows (respiratory rate, gait metrics, audio + * exposure), the tier the daily-mean consolidation folds; + * - one `stats:` daily row per cumulative type and day (what the export + * import writes for steps, energy, distance and flights); + * - per-sample cumulative rows with opaque sample ids for the most recent + * 90 days (what the iOS sync writes, and what the cumulative drain folds); + * - sleep-stage segments every night. + * + * Everything is generated inside Postgres with `generate_series`, so a million + * rows take seconds rather than a million round trips. `ANALYZE` runs at the + * end: a planner reading pre-seed statistics picks plans no production table + * of this size would get. + */ +import { prisma } from "@/lib/db"; + +function arg(name: string, fallback: string): string { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback; +} + +const USER_ID = arg("user", "bench-large"); +const YEARS = Number(arg("years", "4")); +const PULSE_SECONDS = Number(arg("pulse-seconds", "165")); +const RESET = process.argv.includes("--reset"); + +/** A spot series: `perDay` evenly spaced rows per day over the whole span. */ +interface SpotSeries { + type: string; + unit: string; + perDay: number; + base: number; + spread: number; + /** Days back from now the series covers. Defaults to the whole span. */ + days?: number; + /** externalId prefix; `stats:` rows are generated separately. */ + idPrefix: string; +} + +async function insertSpot(s: SpotSeries, days: number): Promise { + const step = Math.floor(86_400 / s.perDay); + const count = Math.floor((days * 86_400) / step); + // Jitter stays below the step so every measured_at is distinct per type + // (the (user, type, measured_at, source, sleep_stage) unique index). + const jitter = Math.max(1, Math.min(59, step - 1)); + return prisma.$executeRawUnsafe( + ` + INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, external_id, created_at, updated_at) + SELECT + 'seed-' || $1 || '-' || $2 || '-' || g, + $1, + $2::measurement_type, + round(($3 + $4 * sin(g / 37.0) + ($4 / 2) * ((g * 7919) % 100) / 100.0)::numeric, 2), + $5, + 'APPLE_HEALTH', + date_trunc('minute', now()) - make_interval(secs => $6 * $7::bigint) + + make_interval(secs => g * $6 + (g * 31) % $8), + $9 || md5($2 || g::text), + now(), now() + FROM generate_series(0, $7::bigint - 1) AS g + `, + USER_ID, + s.type, + s.base, + s.spread, + s.unit, + step, + count, + jitter, + s.idPrefix, + ); +} + +async function main(): Promise { + const days = Math.round(YEARS * 365); + + if (RESET) { + await prisma.$executeRawUnsafe(`DELETE FROM users WHERE id = $1`, USER_ID); + } + await prisma.$executeRawUnsafe( + `INSERT INTO users (id, username, timezone, updated_at) + VALUES ($1, $1, 'Europe/Berlin', now()) + ON CONFLICT (id) DO NOTHING`, + USER_ID, + ); + + const pulsePerDay = Math.floor(86_400 / PULSE_SECONDS); + const spot: SpotSeries[] = [ + { + type: "PULSE", + unit: "bpm", + perDay: pulsePerDay, + base: 62, + spread: 25, + idPrefix: "hk-", + }, + { + type: "HEART_RATE_VARIABILITY", + unit: "ms", + perDay: 12, + base: 45, + spread: 20, + idPrefix: "hk-", + }, + { + type: "OXYGEN_SATURATION", + unit: "%", + perDay: 24, + base: 96, + spread: 2, + idPrefix: "hk-", + }, + { + type: "RESPIRATORY_RATE", + unit: "breaths/min", + perDay: 48, + base: 14, + spread: 3, + idPrefix: "hk-", + }, + { + type: "WALKING_SPEED", + unit: "km/h", + perDay: 30, + base: 4.8, + spread: 0.8, + idPrefix: "hk-", + }, + { + type: "WALKING_STEP_LENGTH", + unit: "cm", + perDay: 30, + base: 72, + spread: 6, + idPrefix: "hk-", + }, + { + type: "WALKING_ASYMMETRY", + unit: "%", + perDay: 30, + base: 3, + spread: 2, + idPrefix: "hk-", + }, + { + type: "WALKING_DOUBLE_SUPPORT", + unit: "%", + perDay: 30, + base: 28, + spread: 3, + idPrefix: "hk-", + }, + { + type: "AUDIO_EXPOSURE_ENV", + unit: "dB", + perDay: 60, + base: 62, + spread: 12, + idPrefix: "hk-", + }, + // The iOS sync's per-sample cumulative rows: dense, recent, opaque ids. + { + type: "ACTIVITY_STEPS", + unit: "steps", + perDay: 100, + base: 90, + spread: 60, + days: 90, + idPrefix: "uuid-", + }, + { + type: "ACTIVE_ENERGY_BURNED", + unit: "kcal", + perDay: 400, + base: 1.2, + spread: 0.9, + days: 90, + idPrefix: "uuid-", + }, + { + type: "WALKING_RUNNING_DISTANCE", + unit: "m", + perDay: 100, + base: 70, + spread: 50, + days: 90, + idPrefix: "uuid-", + }, + { + type: "WEIGHT", + unit: "kg", + perDay: 1, + base: 82, + spread: 1.5, + idPrefix: "hk-", + }, + ]; + + let total = 0; + for (const s of spot) { + const n = await insertSpot(s, s.days ?? days); + total += n; + console.log(`${s.type.padEnd(28)} ${String(n).padStart(9)}`); + } + + // One export-estimate `stats:` row per cumulative type and local day, at + // local noon — what `flushCumulativeBuckets` in the export import writes. + for (const [type, hk, unit, base] of [ + ["ACTIVITY_STEPS", "HKQuantityTypeIdentifierStepCount", "steps", 8000], + [ + "ACTIVE_ENERGY_BURNED", + "HKQuantityTypeIdentifierActiveEnergyBurned", + "kcal", + 550, + ], + [ + "WALKING_RUNNING_DISTANCE", + "HKQuantityTypeIdentifierDistanceWalkingRunning", + "m", + 6000, + ], + ["FLIGHTS_CLIMBED", "HKQuantityTypeIdentifierFlightsClimbed", "count", 9], + ] as const) { + const n = await prisma.$executeRawUnsafe( + ` + INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, external_id, + aggregation_provenance, created_at, updated_at) + SELECT + 'seed-' || $1 || '-' || $2 || '-stats-' || d, + $1, $2::measurement_type, $5 + (d * 7919) % ($5 / 2), $4, 'APPLE_HEALTH', + ((current_date - d)::timestamp + interval '12 hours') + AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC', + 'stats:' || $3 || ':' || to_char(current_date - d, 'YYYY-MM-DD'), + 'EXPORT_XML_SOURCE_MAX', now(), now() + FROM generate_series(91, $6::int) AS d + `, + USER_ID, + type, + hk, + unit, + base, + days, + ); + total += n; + console.log(`${`${type} (stats:)`.padEnd(28)} ${String(n).padStart(9)}`); + } + + // Sleep: 30 segments a night from 23:00 local, 16 minutes apart, rotating + // through the stages. Distinct measured_at per segment. + const sleep = await prisma.$executeRawUnsafe( + ` + INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, external_id, + sleep_stage, created_at, updated_at) + SELECT + 'seed-' || $1 || '-sleep-' || d || '-' || s, + $1, 'SLEEP_DURATION', 16, 'minutes', 'APPLE_HEALTH', + ((current_date - d)::timestamp - interval '1 hour' + s * interval '16 minutes') + AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC', + 'hk-' || md5('sleep' || d || '-' || s), + (ARRAY['CORE','DEEP','CORE','REM','AWAKE','CORE']::sleep_stage[])[1 + s % 6], + now(), now() + FROM generate_series(1, $2::int) AS d, generate_series(0, 29) AS s + `, + USER_ID, + days, + ); + total += sleep; + console.log(`${"SLEEP_DURATION".padEnd(28)} ${String(sleep).padStart(9)}`); + + await prisma.$executeRawUnsafe(`ANALYZE measurements`); + console.log(`${"total".padEnd(28)} ${String(total).padStart(9)}`); +} + +main() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect()); diff --git a/src/__tests__/backup-readers-stream-guard.test.ts b/src/__tests__/backup-readers-stream-guard.test.ts new file mode 100644 index 000000000..8798c6586 --- /dev/null +++ b/src/__tests__/backup-readers-stream-guard.test.ts @@ -0,0 +1,45 @@ +/** + * Every reader of a stored or uploaded backup reads it as a stream (#1031). + * + * A backup's JSON grows with the record, and past 536 870 888 characters it + * cannot exist as one JavaScript string at all: the disaster-recovery JSON of + * an account with 1.25 million measurements is 662 MB. Each of the readers + * below used to turn the file into one string first, and so none of them + * could open that account's backup. This pins that none of them goes back to + * it: no whole-document decrypt-to-string, no whole-body text read. + */ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const READERS = [ + "src/app/api/admin/backups/[id]/restore/route.ts", + "src/app/api/admin/backups/[id]/summary/route.ts", + "src/app/api/admin/backups/[id]/download/route.ts", + "src/app/api/admin/backups/upload/route.ts", + "src/lib/jobs/restore-drill.ts", + "scripts/restore-backup.ts", +] as const; + +/** Calls that hand back the whole document as one string or parse it whole. */ +const WHOLE_DOCUMENT = [ + /\bunpackBackupBlob\s*\(/, + /\bdecryptBackup\s*\(/, + /\bJSON\.parse\s*\(\s*plaintext\b/, + /\bparseBackupPayload\s*\(\s*plaintext\b/, + /\bfile\.text\s*\(/, + /\bawait\s+request\.text\s*\(\s*\)[\s\S]{0,200}parseBackupPayload/, +]; + +describe("backup readers stream the file", () => { + it.each(READERS)("%s never holds the whole document", (path) => { + const source = readFileSync(path, "utf8"); + for (const pattern of WHOLE_DOCUMENT) { + expect(source, `${path} matches ${pattern}`).not.toMatch(pattern); + } + }); + + it.each(READERS)("%s reads through the streaming reader", (path) => { + const source = readFileSync(path, "utf8"); + expect(source).toMatch(/\b(readStreamedBackup|scanBackupJson)\b/); + }); +}); diff --git a/src/app/api/admin/backups/[id]/download/route.ts b/src/app/api/admin/backups/[id]/download/route.ts index 644313726..bb2a01736 100644 --- a/src/app/api/admin/backups/[id]/download/route.ts +++ b/src/app/api/admin/backups/[id]/download/route.ts @@ -14,6 +14,8 @@ * at-rest contract: the ciphertext stays in the DB, the plaintext is only * materialised inside the request handler and streamed to the admin. */ +import { Readable } from "node:stream"; + import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; @@ -22,8 +24,13 @@ import { auditLog } from "@/lib/auth/audit"; import { BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, - unpackBackupBlob, + openBackupBlob, } from "@/lib/export/backup-blob"; +import { + readStreamedBackup, + type BackupSource, + type StreamedBackup, +} from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { parseBackupPayload } from "@/lib/validations/backup"; @@ -54,9 +61,11 @@ export const GET = apiHandler( throw new HttpError(404, "Backup not found"); } - let plaintext: string; + // Opened and streamed, never unpacked into one string: a large record's + // JSON is longer than any string V8 can hold (#1031). + let source: BackupSource; try { - plaintext = unpackBackupBlob(backup.data); + source = openBackupBlob(backup.data); } catch (err) { // A rotated or dropped key, or a stored copy that is no longer the one // that was written. Both are bad stored input rather than a fault in @@ -80,8 +89,10 @@ export const GET = apiHandler( // pathological case where the worker wrote something the upload // route would later reject — better to fail loudly here than to ship // a junk file that breaks an admin's restore plan. + let streamed: StreamedBackup; try { - parseBackupPayload(plaintext); + streamed = await readStreamedBackup(source); + parseBackupPayload(streamed.raw); } catch (err) { await auditLog("admin.backups.download.failed", { userId: admin.id, @@ -107,22 +118,26 @@ export const GET = apiHandler( ownerId: backup.userId, ownerUsername: backup.user.username, type: backup.type, - sizeBytes: Buffer.byteLength(plaintext, "utf8"), + sizeBytes: streamed.bytes, }, }); - // Returning the plaintext as-is keeps the response a faithful copy of + // Returning the stored bytes as-is keeps the response a faithful copy of // what the worker wrote (formatting, key order). The file is the // canonical artefact admins will store / re-upload. - return new NextResponse(plaintext, { - status: 200, - headers: { - "content-type": "application/json; charset=utf-8", - "content-disposition": `attachment; filename="${filename}"`, - // Backups contain sensitive health data — keep them out of any - // shared cache (CDN, browser disk cache). - "cache-control": "no-store, max-age=0", + return new NextResponse( + Readable.toWeb(Readable.from(source())) as ReadableStream, + { + status: 200, + headers: { + "content-type": "application/json; charset=utf-8", + "content-disposition": `attachment; filename="${filename}"`, + // Backups contain sensitive health data — keep them out of any + // shared cache (CDN, browser disk cache). + "cache-control": "no-store, max-age=0", + "content-length": String(streamed.bytes), + }, }, - }); + ); }, ); diff --git a/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts b/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts index 5d52faa1a..70053fb02 100644 --- a/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts +++ b/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts @@ -47,6 +47,9 @@ vi.mock("@/lib/validations/backup", () => ({ parseBackupPayload: vi.fn(), isCompatibleSchemaVersion: vi.fn(() => true), summarizeBackup: vi.fn(() => ({})), + // The streamed reader checks measurements one at a time; this file has none. + BACKUP_SCHEMA_VERSION: "2", + backupMeasurementSchema: { safeParse: vi.fn() }, })); vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() })); vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index 7e45e49f9..7f8f2b4af 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -26,8 +26,17 @@ import { auditLog } from "@/lib/auth/audit"; import { BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, - unpackBackupBlob, + openBackupBlob, } from "@/lib/export/backup-blob"; +import { + insertMeasurementRows, + type MeasurementInsertRow, +} from "@/lib/export/measurement-bulk-insert"; +import { + readStreamedBackup, + type BackupSource, + type StreamedBackup, +} from "@/lib/export/streamed-backup"; import { encryptNote } from "@/lib/crypto/note-cipher"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; @@ -38,6 +47,7 @@ import { parseBackupPayload, isCompatibleSchemaVersion, summarizeBackup, + type BackupMeasurement, type BackupSummary, } from "@/lib/validations/backup"; import { recomputeUserMoodRollups } from "@/lib/rollups/mood-rollups"; @@ -137,6 +147,19 @@ interface RestoreResponse { }; } +/** + * How long the restore transaction may run. Two minutes was the whole budget + * when a record was a few hundred thousand rows; deleting and rewriting 1.25 + * million measurements in one transaction takes longer than that (#1031). On + * the test host the whole restore of that account, reading the file twice + * included, took 94 s. The budget grows by a second per 5 000 measurements, + * which gives that account six minutes, about four times what it needed, and + * never drops below the old two minutes. + */ +function restoreTransactionTimeoutMs(measurementCount: number): number { + return 120_000 + Math.ceil(measurementCount / 5); +} + function decodeEncryptedBytes(encoded: string): Uint8Array { const decoded = Buffer.from(encoded, "base64"); const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); @@ -198,9 +221,12 @@ const handler = apiHandler( throw new HttpError(404, "Backup not found"); } - let plaintext: string; + // Opened, not unpacked: the JSON of a large record is longer than any + // string V8 can hold (#1031), so it is read as a stream below. Opening + // authenticates the whole ciphertext first, as unpacking did. + let source: BackupSource; try { - plaintext = unpackBackupBlob(backup.data); + source = openBackupBlob(backup.data); } catch (err) { await auditLog("admin.backups.restore.failed", { userId: admin.id, @@ -224,10 +250,16 @@ const handler = apiHandler( // Parsed once and kept, because the schema's per-section `.default([])` // erases the difference between a section that is absent and one that is // empty — and that difference is what the completeness check below reads. + // + // The measurements are the exception: `readStreamedBackup` checks each one + // against the element schema as it goes and keeps only their count, and + // the transaction reads them a second time, in batches, as it writes them. let raw: unknown; let payload; + let streamed: StreamedBackup; try { - raw = JSON.parse(plaintext); + streamed = await readStreamedBackup(source); + raw = streamed.raw; payload = parseBackupPayload(raw); } catch (err) { await auditLog("admin.backups.restore.failed", { @@ -548,7 +580,7 @@ const handler = apiHandler( } const toRestoredMeasurementData = ( - measurement: (typeof payload.measurements)[number], + measurement: BackupMeasurement, ) => ({ type: measurement.type, value: measurement.value, @@ -600,55 +632,80 @@ const handler = apiHandler( : {}), }); - const stableRows = payload.measurements.flatMap((measurement) => - measurement.id - ? [ - { - id: measurement.id, - userId: ownerId, - ...toRestoredMeasurementData(measurement), - }, - ] - : [], - ); - const measurementBatchSize = 1_000; - for ( - let offset = 0; - offset < stableRows.length; - offset += measurementBatchSize - ) { - await tx.measurement.createMany({ - data: stableRows.slice(offset, offset + measurementBatchSize), - }); - } - - // v1 payloads did not require stable ids. Preserve their historical - // natural-key reconciliation without routing canonical v2 rows - // through it. - for (const measurement of payload.measurements) { - if (measurement.id) continue; - const restoredData = toRestoredMeasurementData(measurement); - const existing = await tx.measurement.findFirst({ - where: { - userId: ownerId, - type: measurement.type, - source: restoredData.source, - measuredAt: restoredData.measuredAt, - sleepStage: restoredData.sleepStage, - }, - select: { id: true }, - }); - if (existing) { - await tx.measurement.update({ - where: { id: existing.id, userId: ownerId }, - data: restoredData, - }); - } else { - await tx.measurement.create({ - data: { userId: ownerId, ...restoredData }, - }); + // The measurements, a batch at a time from a second read of the + // file, so the restore never holds more than one batch of them. + // + // Two later sections point INTO the measurements (a record's + // `sourceMeasurementId`, an ECG strip's `measurementId`) and have + // to know which of those ids this transaction actually wrote. They + // used to be handed every written id; they only ever ask about the + // ones they reference, so only those are collected. + const referencedMeasurementIds = new Set(); + for (const record of payload.personalRecords) { + if (record.sourceMeasurementId) { + referencedMeasurementIds.add(record.sourceMeasurementId); } } + for (const recording of payload.ecgRecordings) { + if (recording.measurementId) { + referencedMeasurementIds.add(recording.measurementId); + } + } + const writtenReferencedIds = new Set(); + const measurementBatchSize = 1_000; + await streamed.forEachMeasurementBatch( + measurementBatchSize, + async (batch) => { + const stableRows = batch.flatMap( + (measurement): MeasurementInsertRow[] => + measurement.id + ? [ + { + id: measurement.id, + userId: ownerId, + ...toRestoredMeasurementData(measurement), + }, + ] + : [], + ); + // One statement per batch, not `createMany`: see + // `insertMeasurementRows` for what the latter cost here. + await insertMeasurementRows(tx, stableRows); + for (const row of stableRows) { + if (referencedMeasurementIds.has(row.id)) { + writtenReferencedIds.add(row.id); + } + } + + // v1 payloads did not require stable ids. Preserve their + // historical natural-key reconciliation without routing + // canonical v2 rows through it. + for (const measurement of batch) { + if (measurement.id) continue; + const restoredData = toRestoredMeasurementData(measurement); + const existing = await tx.measurement.findFirst({ + where: { + userId: ownerId, + type: measurement.type, + source: restoredData.source, + measuredAt: restoredData.measuredAt, + sleepStage: restoredData.sleepStage, + }, + select: { id: true }, + }); + if (existing) { + await tx.measurement.update({ + where: { id: existing.id, userId: ownerId }, + data: restoredData, + }); + } else { + await tx.measurement.create({ + data: { userId: ownerId, ...restoredData }, + }); + } + } + }, + ); const medByName = new Map(); const restoredMedicationIds = new Set(); @@ -1932,7 +1989,7 @@ const handler = apiHandler( tx, ownerId, payload, - new Set(stableRows.map((row) => row.id)), + writtenReferencedIds, skips, ); @@ -1949,7 +2006,7 @@ const handler = apiHandler( tx, ownerId, payload, - new Set(stableRows.map((row) => row.id)), + writtenReferencedIds, skips, ); @@ -2024,7 +2081,7 @@ const handler = apiHandler( }, { maxWait: 10_000, - timeout: 120_000, + timeout: restoreTransactionTimeoutMs(streamed.measurementCount), }, ); } catch (err) { @@ -2071,7 +2128,7 @@ const handler = apiHandler( // outside the transaction (the 5-year fold would otherwise hold a // long write lock) and is best-effort so a populator hiccup never // undoes the restore. The boot-time backfill is the safety net. - if (payload.measurements.length > 0) { + if (streamed.measurementCount > 0) { try { await recomputeUserRollups(ownerId); } catch (err) { @@ -2131,7 +2188,10 @@ const handler = apiHandler( } } - const summary = summarizeBackup(payload); + const summary = { + ...summarizeBackup(payload), + measurements: streamed.measurementCount, + }; await auditLog("admin.backups.restore", { userId: admin.id, diff --git a/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts b/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts index 766e0d4de..16f704733 100644 --- a/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts +++ b/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts @@ -44,6 +44,11 @@ vi.mock("@/lib/validations/backup", () => ({ parseBackupPayload: (...a: unknown[]) => parseMock(...a), summarizeBackup: (...a: unknown[]) => summarizeMock(...a), isCompatibleSchemaVersion: vi.fn(() => true), + // The measurements are read as a stream and counted one by one. + BACKUP_SCHEMA_VERSION: "2", + backupMeasurementSchema: { + safeParse: (value: unknown) => ({ success: true, data: value }), + }, })); import { GET } from "../route"; @@ -62,9 +67,14 @@ beforeEach(() => { createdAt: new Date("2026-08-01T00:00:00Z"), user: { id: "u1", username: "self-hoster" }, }); - decryptMock.mockReturnValue("{}"); + // Twelve measurements in the file: the preview counts them from the stream. + decryptMock.mockReturnValue( + JSON.stringify({ + measurements: Array.from({ length: 12 }, (_, i) => ({ id: `m${i}` })), + }), + ); parseMock.mockReturnValue({ schemaVersion: 3 }); - summarizeMock.mockReturnValue({ measurements: 12, moodEntries: 3 }); + summarizeMock.mockReturnValue({ measurements: 0, moodEntries: 3 }); }); describe("GET /api/admin/backups/[id]/summary", () => { diff --git a/src/app/api/admin/backups/[id]/summary/route.ts b/src/app/api/admin/backups/[id]/summary/route.ts index 4b79fcc79..cb40b404c 100644 --- a/src/app/api/admin/backups/[id]/summary/route.ts +++ b/src/app/api/admin/backups/[id]/summary/route.ts @@ -19,8 +19,12 @@ import { apiError, apiSuccess } from "@/lib/api-response"; import { BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, - unpackBackupBlob, + openBackupBlob, } from "@/lib/export/backup-blob"; +import { + readStreamedBackup, + type BackupSource, +} from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { isCompatibleSchemaVersion, @@ -46,9 +50,11 @@ export const GET = apiHandler( throw new HttpError(404, "Backup not found"); } - let plaintext: string; + // Opened and read as a stream: a large record's JSON is longer than any + // string V8 can hold (#1031). + let source: BackupSource; try { - plaintext = unpackBackupBlob(backup.data); + source = openBackupBlob(backup.data); } catch { // Same refusal the restore and the download give, for the same reason: // a copy this instance cannot open is bad stored input, and the preview @@ -59,8 +65,11 @@ export const GET = apiHandler( } let payload; + let measurementCount: number; try { - payload = parseBackupPayload(plaintext); + const streamed = await readStreamedBackup(source); + measurementCount = streamed.measurementCount; + payload = parseBackupPayload(streamed.raw); } catch { return apiError("Backup payload failed schema validation", 422); } @@ -72,7 +81,10 @@ export const GET = apiHandler( ); } - const summary = summarizeBackup(payload); + const summary = { + ...summarizeBackup(payload), + measurements: measurementCount, + }; annotate({ action: { name: "admin.backups.summary" }, diff --git a/src/app/api/admin/backups/upload/route.ts b/src/app/api/admin/backups/upload/route.ts index b7b6443ea..8110b4bac 100644 --- a/src/app/api/admin/backups/upload/route.ts +++ b/src/app/api/admin/backups/upload/route.ts @@ -1,17 +1,25 @@ /** * POST /api/admin/backups/upload — admin-only ingest of a backup file. * - * Accepts a `multipart/form-data` body with a single `file` field that - * carries a JSON backup matching `backupPayloadSchema`. The route: + * Accepts the backup file either as the raw request body (JSON, or JSON + * compressed with gzip) or, for a small file, as a `multipart/form-data` + * body with a single `file` field. The route: * * 1. Authenticates the caller as an admin (cookie session only). * 2. Reads the file (size capped — see `MAX_UPLOAD_BYTES`). * 3. Parses and validates the JSON against the canonical schema. * 4. Rejects files written by a future schema version (incompatible). - * 5. Encrypts the plaintext and inserts a new `DataBackup` row of - * type `MANUAL_UPLOAD_` so multiple uploads coexist with + * 5. Encrypts the file as it arrives and inserts a new `DataBackup` row + * of type `MANUAL_UPLOAD_` so multiple uploads coexist with * the rolling `WEEKLY_AUTO` snapshot AND with each other. * + * The file is read as a stream and never held whole (#1031): an export of an + * account with 1.25 million measurements is a few hundred megabytes of JSON, + * which neither fits a 1 GB container beside the running app nor, in the + * disaster-recovery shape, fits in one JavaScript string at all. It is + * checked as it goes and stored in the same pass; a file that fails any check + * rolls the store back, so nothing of it remains. + * * Crucially, this route does NOT execute a restore. It only stores the * file. The separate `POST /api/admin/backups/[id]/restore` endpoint * (criterion 3) actually replaces user data — keeping the two phases @@ -20,46 +28,120 @@ * * Phase B1 / criterion 2 of the v1.4.15 backup-completeness work. */ +import { Readable } from "node:stream"; +import { createGunzip } from "node:zlib"; + import { NextRequest } from "next/server"; import { ZodError } from "zod/v4"; import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { packBackupBlob } from "@/lib/export/backup-blob"; +import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; +import { StreamedBackupInvalidError } from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit } from "@/lib/rate-limit"; import { + BACKUP_SCHEMA_VERSION, + backupMeasurementSchema, backupPayloadSchema, isCompatibleSchemaVersion, summarizeBackup, + type BackupPayload, type BackupSummary, } from "@/lib/validations/backup"; export const dynamic = "force-dynamic"; /** - * Cap on uploaded file size. The largest production backups today are - * ~2 MB; 10 MB leaves headroom while keeping a malicious admin client - * from blowing memory by streaming a 4 GB file. Enforced by reading the - * `Content-Length` header AND the actual buffered size after decoding — - * a missing/wrong content-length doesn't bypass the limit. + * Cap on a file sent as a multipart form. The form parser holds the whole + * file in memory, so this path stays for small files only; anything larger + * is sent as the raw request body, which is read as a stream. */ const MAX_UPLOAD_BYTES = 10 * 1024 * 1024; +/** + * Cap on a file sent as the raw request body, compressed or not. The same + * ceiling the app sets for any request body (`middlewareClientMaxBodySize` in + * `next.config.ts`), beyond which the body would arrive truncated. A + * disaster-recovery file of 1.25 million measurements is 662 MB as plain + * JSON and 64 MB compressed, so large files go up compressed. + */ +const MAX_RAW_UPLOAD_BYTES = 512 * 1024 * 1024; + interface UploadResponse { id: string; valid: true; summary: BackupSummary; } +/** A file the route refuses: the answer, and what the audit row records. */ +class UploadRefused extends Error { + constructor( + readonly status: number, + readonly answer: string, + readonly details: Record, + readonly meta?: Record, + ) { + super(answer); + this.name = "UploadRefused"; + } +} + +/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */ +async function* decodedBytes( + source: AsyncIterable, + limit: number, +): AsyncGenerator { + let seen = 0; + async function* counted() { + for await (const chunk of source) { + seen += chunk.byteLength; + if (seen > limit) { + throw new UploadRefused( + 413, + `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`, + { reason: "file_size_exceeded", size: seen }, + ); + } + yield chunk; + } + } + const iterator = counted()[Symbol.asyncIterator](); + const first = await iterator.next(); + if (first.done) return; + const rest = (async function* () { + yield first.value; + for (;;) { + const next = await iterator.next(); + if (next.done) return; + yield next.value; + } + })(); + const head = first.value; + if (head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b) { + const gunzip = Readable.from(rest).pipe(createGunzip()); + try { + for await (const chunk of gunzip) yield chunk as Buffer; + } catch (err) { + if (err instanceof UploadRefused) throw err; + throw new UploadRefused(422, "Uploaded file is not valid gzip", { + reason: "invalid_gzip", + }); + } + return; + } + yield* rest; +} + export const POST = apiHandler(async (request: NextRequest) => { const { user: admin } = await requireAdmin(); annotate({ action: { name: "admin.backups.upload" } }); + const ipAddress = getClientIp(request); - // Rate-limit identical to the manual-run endpoint. An admin uploading - // a 10 MB blob 60× per minute would just be wasteful — three is plenty - // for a deliberate restore-prep workflow. + // Rate-limit identical to the manual-run endpoint. Three is plenty for a + // deliberate restore-prep workflow. const rl = await checkRateLimit( `admin-backups-upload:${admin.id}`, 3, @@ -69,120 +151,56 @@ export const POST = apiHandler(async (request: NextRequest) => { throw new HttpError(429, "Too many backup uploads"); } - // Cheap pre-flight on the declared content length. The actual buffer - // size check below is the hard limit; this just rejects obvious abuse - // before allocating the FormData parser. + const denied = (reason: string, extra: Record = {}) => + auditLog("admin.backups.upload.denied", { + userId: admin.id, + ipAddress, + details: { reason, ...extra }, + }); + + const contentType = request.headers.get("content-type") ?? ""; + const multipart = contentType.startsWith("multipart/form-data"); + const limit = multipart ? MAX_UPLOAD_BYTES : MAX_RAW_UPLOAD_BYTES; + + // Cheap pre-flight on the declared content length. The byte count while + // reading is the hard limit; this just rejects obvious abuse early. const contentLength = Number(request.headers.get("content-length") ?? 0); - if (contentLength > MAX_UPLOAD_BYTES) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { reason: "content_length_exceeded", contentLength }, - }); - return apiError("Upload exceeds 10 MB limit", 413); - } - - let formData: FormData; - try { - formData = await request.formData(); - } catch (err) { + if (contentLength > limit) { + await denied("content_length_exceeded", { contentLength }); return apiError( - `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`, - 400, + `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`, + 413, ); } - const file = formData.get("file"); - if (!(file instanceof File)) { - return apiError("Field 'file' must be a file", 422); - } - - if (file.size > MAX_UPLOAD_BYTES) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { reason: "file_size_exceeded", size: file.size }, - }); - return apiError("Upload exceeds 10 MB limit", 413); - } - - let plaintext: string; - try { - plaintext = await file.text(); - } catch { - return apiError("Failed to read uploaded file", 400); - } - - let parsedJson: unknown; - try { - parsedJson = JSON.parse(plaintext); - } catch (err) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { - reason: "invalid_json", - message: err instanceof Error ? err.message : "parse_failed", - }, - }); - return apiError("Uploaded file is not valid JSON", 422); - } - - let payload; - try { - payload = backupPayloadSchema.parse(parsedJson); - } catch (err) { - const issues = - err instanceof ZodError - ? err.issues.slice(0, 10).map((i) => ({ - path: i.path.join("."), - message: i.message, - })) - : []; - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { reason: "schema_invalid", issues }, - }); - return apiError("Backup payload failed schema validation", 422, { - issues, - }); - } - - if (!isCompatibleSchemaVersion(payload.schemaVersion)) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { - reason: "incompatible_schema_version", - schemaVersion: payload.schemaVersion, - }, - }); - return apiError( - `Backup schema version '${payload.schemaVersion}' is not supported by this server`, - 422, + let source: AsyncIterable; + if (multipart) { + let formData: FormData; + try { + formData = await request.formData(); + } catch (err) { + return apiError( + `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`, + 400, + ); + } + const file = formData.get("file"); + if (!(file instanceof File)) { + return apiError("Field 'file' must be a file", 422); + } + if (file.size > MAX_UPLOAD_BYTES) { + await denied("file_size_exceeded", { size: file.size }); + return apiError("Upload exceeds 10 MB limit", 413); + } + source = Readable.fromWeb( + file.stream() as Parameters[0], ); - } - - // Make sure the userId on the file points at a user that exists in - // this DB. Restore would fail later anyway, but failing here gives - // the admin a precise error before any side-effect. - const owner = await prisma.user.findUnique({ - where: { id: payload.userId }, - select: { id: true, username: true }, - }); - if (!owner) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { - reason: "owner_not_found", - ownerId: payload.userId, - }, - }); - return apiError( - `Backup is for user '${payload.userId}' which does not exist in this DB`, - 422, + } else { + if (!request.body) { + return apiError("The backup file is missing from the request body", 422); + } + source = Readable.fromWeb( + request.body as Parameters[0], ); } @@ -190,25 +208,146 @@ export const POST = apiHandler(async (request: NextRequest) => { // intact so multiple uploads can coexist for the same user without // overwriting each other or the rolling WEEKLY_AUTO snapshot. const uploadType = `MANUAL_UPLOAD_${Date.now()}`; - const encrypted = packBackupBlob(JSON.stringify(payload)); + let payload: BackupPayload | undefined; + let measurementCount = 0; + let owner: { id: string; username: string } | null = null; - const created = await prisma.dataBackup.create({ - data: { - userId: owner.id, - type: uploadType, - data: encrypted, - }, - }); + let created: { id: string }; + try { + created = await storeBackupBlob( + prisma, + () => ({ userId: owner!.id, type: uploadType }), + async (write) => { + // One pass: every chunk is stored as it is checked, and a check that + // fails throws, which rolls the store back. + async function* storedAsRead() { + for await (const chunk of decodedBytes(source, limit)) { + await write(Buffer.from(chunk)); + yield chunk; + } + } + let firstWithoutId: number | null = null; + let scanned; + try { + scanned = await scanBackupJson(storedAsRead(), { + streamKeys: new Set(["measurements"]), + onElement: (_key, element, index) => { + const parsed = backupMeasurementSchema.safeParse(element); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + throw new StreamedBackupInvalidError( + ["measurements", index, ...(issue?.path ?? [])].join("."), + issue?.message ?? "invalid measurement", + ); + } + if (!parsed.data.id && firstWithoutId === null) { + firstWithoutId = index; + } + }, + }); + } catch (err) { + if (err instanceof BackupJsonError) { + throw new UploadRefused(422, "Uploaded file is not valid JSON", { + reason: "invalid_json", + message: err.message, + }); + } + if (err instanceof StreamedBackupInvalidError) { + const issues = [{ path: err.path, message: err.message }]; + throw new UploadRefused( + 422, + "Backup payload failed schema validation", + { reason: "schema_invalid", issues }, + { issues }, + ); + } + throw err; + } + measurementCount = scanned.streamedCounts.measurements ?? 0; - const summary = summarizeBackup(payload); + try { + payload = backupPayloadSchema.parse(scanned.document); + } catch (err) { + const issues = + err instanceof ZodError + ? err.issues.slice(0, 10).map((i) => ({ + path: i.path.join("."), + message: i.message, + })) + : []; + throw new UploadRefused( + 422, + "Backup payload failed schema validation", + { reason: "schema_invalid", issues }, + { issues }, + ); + } + if ( + payload.schemaVersion === BACKUP_SCHEMA_VERSION && + firstWithoutId !== null + ) { + const issues = [ + { + path: `measurements.${firstWithoutId}.id`, + message: "Canonical v2 measurements require a stable id", + }, + ]; + throw new UploadRefused( + 422, + "Backup payload failed schema validation", + { reason: "schema_invalid", issues }, + { issues }, + ); + } + + if (!isCompatibleSchemaVersion(payload.schemaVersion)) { + throw new UploadRefused( + 422, + `Backup schema version '${payload.schemaVersion}' is not supported by this server`, + { + reason: "incompatible_schema_version", + schemaVersion: payload.schemaVersion, + }, + ); + } + + // Make sure the userId on the file points at a user that exists in + // this DB. Restore would fail later anyway, but failing here gives + // the admin a precise error before any side-effect. + owner = await prisma.user.findUnique({ + where: { id: payload.userId }, + select: { id: true, username: true }, + }); + if (!owner) { + throw new UploadRefused( + 422, + `Backup is for user '${payload.userId}' which does not exist in this DB`, + { reason: "owner_not_found", ownerId: payload.userId }, + ); + } + }, + ); + } catch (err) { + if (err instanceof UploadRefused) { + await denied(err.details.reason as string, err.details); + return apiError(err.answer, err.status, err.meta); + } + throw err; + } + + const summary: BackupSummary = { + ...summarizeBackup(payload!), + measurements: measurementCount, + }; + const storedOwner = owner as { id: string; username: string } | null; await auditLog("admin.backups.upload", { userId: admin.id, - ipAddress: getClientIp(request), + ipAddress, details: { backupId: created.id, - ownerId: owner.id, - ownerUsername: owner.username, + ownerId: storedOwner?.id, + ownerUsername: storedOwner?.username, type: uploadType, schemaVersion: summary.schemaVersion, counts: { diff --git a/src/app/api/export/__tests__/soft-delete-filter.test.ts b/src/app/api/export/__tests__/soft-delete-filter.test.ts index bd4a5669c..ff386b3ec 100644 --- a/src/app/api/export/__tests__/soft-delete-filter.test.ts +++ b/src/app/api/export/__tests__/soft-delete-filter.test.ts @@ -64,6 +64,29 @@ vi.mock("@/lib/db", () => ({ documentConditionLink: { findMany: vi.fn() }, extractedFact: { findMany: vi.fn() }, ecgRecording: { findMany: vi.fn() }, + onboardingRecord: { findUnique: vi.fn().mockResolvedValue(null) }, + environmentTravelLocation: { findMany: vi.fn().mockResolvedValue([]) }, + environmentContext: { findMany: vi.fn().mockResolvedValue([]) }, + personalRecord: { findMany: vi.fn().mockResolvedValue([]) }, + coachReminder: { findMany: vi.fn().mockResolvedValue([]) }, + coachPlan: { findMany: vi.fn().mockResolvedValue([]) }, + coachFact: { findMany: vi.fn().mockResolvedValue([]) }, + coachConversation: { findMany: vi.fn().mockResolvedValue([]) }, + measurementReminderEvent: { findMany: vi.fn().mockResolvedValue([]) }, + measurementReminder: { findMany: vi.fn().mockResolvedValue([]) }, + vaccinationDocumentLink: { findMany: vi.fn().mockResolvedValue([]) }, + vaccinationRecord: { findMany: vi.fn().mockResolvedValue([]) }, + encounterConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, + encounterLabLink: { findMany: vi.fn().mockResolvedValue([]) }, + encounterDocumentLink: { findMany: vi.fn().mockResolvedValue([]) }, + encounter: { findMany: vi.fn().mockResolvedValue([]) }, + practitioner: { findMany: vi.fn().mockResolvedValue([]) }, + healthScoreRecord: { findMany: vi.fn().mockResolvedValue([]) }, + intradayCumulativeProfile: { findMany: vi.fn().mockResolvedValue([]) }, + correlationPattern: { findMany: vi.fn().mockResolvedValue([]) }, + healthProfileFactRevision: { findMany: vi.fn().mockResolvedValue([]) }, + customMetric: { findMany: vi.fn().mockResolvedValue([]) }, + cycleSymptom: { findMany: vi.fn().mockResolvedValue([]) }, }, })); @@ -177,6 +200,7 @@ beforeEach(() => { ); vi.mocked(prisma.extractedFact.findMany).mockResolvedValue([] as never); vi.mocked(prisma.ecgRecording.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.userAchievement.findMany).mockResolvedValue([] as never); }); describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { @@ -192,7 +216,13 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { it("/api/export/full-backup scopes the measurement read to deletedAt: null", async () => { const { GET } = await import("../full-backup/route"); - await GET(mkReq("http://localhost/api/export/full-backup")); + // The file is written into the response as it is produced, so the reads + // happen while the body is consumed. The mocked client does not cover + // every section the builder reads, so the body errors part-way (as the + // whole response did before it streamed); the reads asserted here have + // been issued by then. + const res = await GET(mkReq("http://localhost/api/export/full-backup")); + await res.text(); expect(prisma.measurement.findMany).toHaveBeenCalledWith( expect.objectContaining({ where: expect.objectContaining({ deletedAt: null }), @@ -202,7 +232,10 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { it("/api/export/full-backup scopes every v1.28 records domain to userId + deletedAt: null", async () => { const { GET } = await import("../full-backup/route"); - await GET(mkReq("http://localhost/api/export/full-backup")); + // The file is written into the response as it is produced, so the reads + // happen while the body is consumed. + const res = await GET(mkReq("http://localhost/api/export/full-backup")); + await res.text(); expect(prisma.labResult.findMany).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/app/api/export/encrypted/route.ts b/src/app/api/export/encrypted/route.ts index e68f4f987..bd19ae009 100644 --- a/src/app/api/export/encrypted/route.ts +++ b/src/app/api/export/encrypted/route.ts @@ -42,13 +42,44 @@ import { safeJson, } from "@/lib/api-response"; import { checkRateLimit } from "@/lib/rate-limit"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { randomBytes } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { FullBackupCounts } from "@/lib/export/full-backup-payload"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { - encryptArchive, + encryptArchiveToFile, MIN_EXPORT_PASSPHRASE_LENGTH, + type SpooledArchive, } from "@/lib/export/passphrase-archive"; +import { streamToResponseBody } from "@/lib/export/response-stream"; import { NextRequest, NextResponse } from "next/server"; +const SPOOL_PREFIX = "healthlog-export-"; + +/** + * Remove spool files an earlier export left behind: a process that died + * mid-export never reached its cleanup. An hour is far longer than any export + * takes, so nothing still being written or sent is touched. + */ +async function removeStaleSpools(): Promise { + try { + const dir = tmpdir(); + const cutoff = Date.now() - 60 * 60 * 1000; + for (const name of await readdir(dir)) { + if (!name.startsWith(SPOOL_PREFIX)) continue; + const path = join(dir, name); + const info = await stat(path).catch(() => null); + if (info && info.mtimeMs < cutoff) await rm(path, { force: true }); + } + } catch { + // Housekeeping only; an export must not fail over it. + } +} + const encryptedExportSchema = z .object({ passphrase: z @@ -89,10 +120,31 @@ export const POST = apiHandler(async (request: NextRequest) => { } const { passphrase } = parsed.data; - const { payload, counts } = await buildFullBackupPayload(prisma, user.id); - - // Encrypt the JSON bytes. The passphrase goes no further than the KDF. - const archive = await encryptArchive(JSON.stringify(payload), passphrase); + // Sealed as it is produced and spooled to a temporary file, then sent: the + // format puts the tag in front of the ciphertext, so the archive cannot go + // out before its last byte is encrypted, and holding it in memory is what + // took the app down on an account of 1.25 million measurements (#1031). Only + // ciphertext touches the disk, and the file is removed once it is sent or + // the client leaves. + let counts: FullBackupCounts | undefined; + await removeStaleSpools(); + const bodyPath = join( + tmpdir(), + `${SPOOL_PREFIX}${randomBytes(12).toString("hex")}.hlx.part`, + ); + let archive: SpooledArchive; + try { + archive = await encryptArchiveToFile( + async (write) => { + counts = await streamFullBackupJson(prisma, user.id, write); + }, + passphrase, + bodyPath, + ); + } catch (err) { + await rm(bodyPath, { force: true }); + throw err; + } await auditLog("user.export.encrypted", { userId: user.id, @@ -102,38 +154,36 @@ export const POST = apiHandler(async (request: NextRequest) => { annotate({ meta: { - export_measurements_count: counts.measurements, - export_medications_count: counts.medications, - export_intake_count: counts.intakeEvents, - export_medication_side_effect_count: counts.medicationSideEffects, - export_mood_count: counts.moodEntries, - export_cycle_count: counts.cycles, - export_cycle_day_log_count: counts.cycleDayLogs, - export_lab_result_count: counts.labResults, - export_biomarker_count: counts.biomarkers, - export_illness_episode_count: counts.illnessEpisodes, - export_illness_day_log_count: counts.illnessDayLogs, - export_allergy_count: counts.allergies, - export_family_history_count: counts.familyHistory, - export_workout_count: counts.workouts, - export_document_count: counts.documents, - export_nutrient_day_count: counts.nutrientDays, - export_health_profile_count: counts.healthProfile, - export_custom_metric_count: counts.customMetrics, - export_custom_metric_entry_count: counts.customMetricEntries, - export_intraday_profile_count: counts.intradayProfiles, + export_measurements_count: counts?.measurements, + export_medications_count: counts?.medications, + export_intake_count: counts?.intakeEvents, + export_mood_count: counts?.moodEntries, export_archive_bytes: archive.byteLength, }, }); + const archiveBody = streamToResponseBody( + async (write) => { + await write(archive.prefix); + for await (const chunk of createReadStream(archive.bodyPath)) { + await write(chunk as Buffer); + } + }, + { + onComplete: () => rm(archive.bodyPath, { force: true }), + onError: () => rm(archive.bodyPath, { force: true }), + }, + ); + const stamp = new Date().toISOString().slice(0, 10); // Return the raw binary archive (NOT the apiSuccess envelope). The file is a // self-contained `.hlx` archive openable with the user's passphrase via // scripts/decrypt-export.ts. - return new NextResponse(new Uint8Array(archive), { + return new NextResponse(archiveBody, { status: 200, headers: { "Content-Type": "application/octet-stream", + "Content-Length": String(archive.byteLength), "Content-Disposition": `attachment; filename="healthlog-backup-${user.id}-${stamp}.hlx"`, "Cache-Control": "no-store", }, diff --git a/src/app/api/export/full-backup/route.ts b/src/app/api/export/full-backup/route.ts index c7e83a262..b36201c92 100644 --- a/src/app/api/export/full-backup/route.ts +++ b/src/app/api/export/full-backup/route.ts @@ -35,7 +35,9 @@ import { annotate } from "@/lib/logging/context"; import { auditLog } from "@/lib/auth/audit"; import { apiError, getClientIp } from "@/lib/api-response"; import { checkRateLimit } from "@/lib/rate-limit"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import type { FullBackupCounts } from "@/lib/export/full-backup-payload"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; +import { streamToResponseBody } from "@/lib/export/response-stream"; import { NextRequest, NextResponse } from "next/server"; export const GET = apiHandler(async (request: NextRequest) => { @@ -47,48 +49,44 @@ export const GET = apiHandler(async (request: NextRequest) => { return apiError("Maximum 10 exports per hour", 429); } - // v1.23 — the payload builder is shared with the passphrase-encrypted - // export route so both emit the byte-for-byte same shape (the measurement/ - // medication/mood/cycle domains are restore-compatible; the newer records - // domains are export-only for now — see the module doc comment above). - const { payload, counts } = await buildFullBackupPayload(prisma, user.id); - - await auditLog("user.export.full-backup", { - userId: user.id, - ipAddress: getClientIp(request), - details: { counts }, - }); - - annotate({ - meta: { - export_measurements_count: counts.measurements, - export_medications_count: counts.medications, - export_intake_count: counts.intakeEvents, - export_medication_side_effect_count: counts.medicationSideEffects, - export_mood_count: counts.moodEntries, - export_cycle_count: counts.cycles, - export_cycle_day_log_count: counts.cycleDayLogs, - export_lab_result_count: counts.labResults, - export_biomarker_count: counts.biomarkers, - export_illness_episode_count: counts.illnessEpisodes, - export_illness_day_log_count: counts.illnessDayLogs, - export_allergy_count: counts.allergies, - export_family_history_count: counts.familyHistory, - export_workout_count: counts.workouts, - export_document_count: counts.documents, - export_nutrient_day_count: counts.nutrientDays, - export_health_profile_count: counts.healthProfile, - export_custom_metric_count: counts.customMetrics, - export_custom_metric_entry_count: counts.customMetricEntries, - export_intraday_profile_count: counts.intradayProfiles, + // Written into the response as it is produced (`streamFullBackupJson`), not + // built and stringified first: for an account of 1.25 million measurements + // the built payload and its JSON string did not fit a 1 GB container, and + // the export took the whole app down (#1031). The file is byte for byte + // what `JSON.stringify` of the payload builder's object gives; the passphrase + // export beside this route writes the same bytes, encrypted. + // + // The counts are only known once the last row has gone out, so the audit + // row is written then. A file that stops early is audited as a failure. + const ipAddress = getClientIp(request); + let counts: FullBackupCounts | undefined; + const body = streamToResponseBody( + async (write) => { + counts = await streamFullBackupJson(prisma, user.id, write); }, - }); + { + onComplete: () => + auditLog("user.export.full-backup", { + userId: user.id, + ipAddress, + details: { counts }, + }), + onError: (err) => + auditLog("user.export.full-backup.failed", { + userId: user.id, + ipAddress, + details: { + reason: err instanceof Error ? err.message : String(err), + }, + }), + }, + ); const stamp = new Date().toISOString().slice(0, 10); // Stream the JSON directly (NOT wrapped in the apiSuccess envelope) so // the file is a self-contained backup — admin upload + restore expect // the raw payload, not `{ data: { ... } }`. - return new NextResponse(JSON.stringify(payload), { + return new NextResponse(body, { status: 200, headers: { "Content-Type": "application/json; charset=utf-8", diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index a29f39741..85e81600d 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -130,10 +130,10 @@ describe("BackupsSection — empty state", () => { const className = helpElement?.match(/\bclass="([^"]*)"/)?.[1] ?? ""; expect(english).toContain( - "JSON file matching the current backup schema. Max 10 MB.", + "A HealthLog backup as .json or .json.gz. The file is compressed before upload; compressed, it may be up to 512 MB.", ); expect(german).toContain( - "JSON-Datei passend zum aktuellen Backup-Schema. Max. 10 MB.", + "HealthLog-Backup als .json oder .json.gz. Die Datei wird vor dem Hochladen komprimiert; komprimiert darf sie bis zu 512 MB groß sein.", ); expect(helpElement).toBeDefined(); expect(className.split(/\s+/)).toContain("text-muted-foreground"); diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index fb065f101..5fc81ab58 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -454,6 +454,20 @@ function missingSectionLabel( * the server may reword, and matching on it would turn a copy edit into a * silent loss of the panel. */ + +/** + * The upload body for a backup file: the file itself when it is already + * gzip, otherwise the file compressed in the browser. The server accepts + * both; compressing keeps a large export well inside the request size limit. + */ +async function uploadBody(file: File): Promise { + const head = new Uint8Array(await file.slice(0, 2).arrayBuffer()); + if (head[0] === 0x1f && head[1] === 0x8b) return file; + return new Response( + file.stream().pipeThrough(new CompressionStream("gzip")), + ).blob(); +} + export function missingSectionsOf(err: unknown): MissingBackupSection[] { if (!(err instanceof ApiError)) return []; if (err.meta?.errorCode !== "backup.section.missing") return []; @@ -697,10 +711,11 @@ export function BackupsSection() { const fileInputRef = useRef(null); const upload = useMutation({ mutationFn: async (file: File) => { - const fd = new FormData(); - fd.append("file", file); - // apiFetch with a raw init: the multipart body must NOT be JSON - // re-encoded, so the verb helper (which JSON-stringifies) is out. + // Sent as the raw body, compressed on the way unless it already is: the + // server reads it as a stream, and a large export (hundreds of MB of + // JSON) shrinks about tenfold. No request timeout, because a file of + // that size takes longer to check and store than the default allows. + const body = await uploadBody(file); return apiFetch<{ id: string; valid: true; @@ -734,7 +749,12 @@ export function BackupsSection() { customMetricEntries?: number; intradayProfiles?: number; }; - }>("/api/admin/backups/upload", { method: "POST", body: fd }); + }>("/api/admin/backups/upload", { + method: "POST", + body, + headers: { "Content-Type": "application/gzip" }, + signal: null, + }); }, onSuccess: (data) => { const total = @@ -818,7 +838,8 @@ export function BackupsSection() { return apiPost<{ restored: true; skipped?: RestoreSkipSummary }>( `/api/admin/backups/${row.id}/restore`, { confirm: "RESTORE", restoreInstanceSettings }, - { headers: { "Idempotency-Key": idempotencyKey } }, + // No request timeout: rebuilding a large account takes minutes. + { headers: { "Idempotency-Key": idempotencyKey }, signal: null }, ); }, onSuccess: (data) => { @@ -959,7 +980,7 @@ export function BackupsSection() { = new Set(); +/** + * Ciphertext written by raw SQL, which the Prisma-call walk below cannot see. + * + * `DataBackup.data` is the case: the stored backup is produced as ciphertext + * pieces (`packBackupBlobInto`) and assembled into the row by one SQL + * statement, because holding it as one string in the process is what a large + * record could not afford (#1031). Each entry names the file and the + * statement, and the scan fails if the statement is no longer there, so a + * moved or removed raw write cannot leave a stale entry that still passes. + */ +const RAW_CIPHERTEXT_WRITES: ReadonlyArray<{ + key: string; + file: string; + statement: RegExp; +}> = [ + { + key: "DataBackup.data", + file: "src/lib/export/store-backup-blob.ts", + statement: + /UPDATE\s+data_backups\s+SET\s+data\s*=\s*\(\s*SELECT\s+string_agg/, + }, +]; + function sourceFiles(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir)) { const p = join(dir, entry); @@ -388,6 +411,12 @@ function scanCiphertextWrites(): Map> { `\\b(?:prisma|tx|db)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(?:${WRITE_METHODS})\\s*\\(`, "g", ); + for (const raw of RAW_CIPHERTEXT_WRITES) { + if (rel !== raw.file) continue; + const hit = raw.statement.exec(src); + if (hit) record(...(raw.key.split(".") as [string, string]), hit.index); + } + for (const m of src.matchAll(writes)) { const model = m[1][0].toUpperCase() + m[1].slice(1); if (!schema.models.has(model)) continue; @@ -425,6 +454,17 @@ describe("ciphertext columns derived from the code", () => { } }); + it("still finds every raw-SQL ciphertext write it was told about", () => { + for (const raw of RAW_CIPHERTEXT_WRITES) { + expect( + [...(written.get(raw.key) ?? [])].some((at) => + at.startsWith(`${raw.file}:`), + ), + `${raw.key}: no matching statement left in ${raw.file}`, + ).toBe(true); + } + }); + it("finds ciphertext writes, including the non-suffixed columns", () => { // Anchored non-zero match count. An empty or collapsed result set is the // failure mode this whole file exists to make impossible, so name the diff --git a/src/lib/crypto/__tests__/encryption-corpus.test.ts b/src/lib/crypto/__tests__/encryption-corpus.test.ts index 39d9e6551..db764a92e 100644 --- a/src/lib/crypto/__tests__/encryption-corpus.test.ts +++ b/src/lib/crypto/__tests__/encryption-corpus.test.ts @@ -214,4 +214,45 @@ describe("encryption-corpus scan + rotate", () => { expect(col.legacy).toBe(1); expect(col.byKeyId[LEGACY_BUCKET]).toBe(1); }); + + it("reads every column in bounded pages and counts only rows that hold ciphertext (#1031)", async () => { + // A notes column is read for every row of its table, 1.25 million on a + // large account, and was read in one findMany. 12 000 measurements, one + // in ten with a note written under the old key. + const measurements: Row[] = Array.from({ length: 12_000 }, (_, i) => ({ + id: `m-${String(i).padStart(5, "0")}`, + notesEncrypted: + i % 10 === 0 + ? new Uint8Array(Buffer.from(activeUnder("v1", `note ${i}`), "utf8")) + : null, + })); + const { client } = makeClient({ Measurement: measurements }); + const takes: Array = []; + const measurement = client.measurement!; + const findMany = measurement.findMany; + measurement.findMany = (args) => { + takes.push(args.take); + return findMany(args); + }; + + const scan = await scanCorpus(client); + const col = scan.columns.find( + (c) => c.model === "Measurement" && c.field === "notesEncrypted", + )!; + expect(col.total).toBe(1_200); + expect(col.byKeyId.v1).toBe(1_200); + + const rotation = await rotateCorpus(client); + const result = rotation.results.find( + (r) => r.model === "Measurement" && r.field === "notesEncrypted", + )!; + expect(result.scanned).toBe(1_200); + expect(result.rotated).toBe(1_200); + + expect(takes.length).toBeGreaterThan(2); + for (const take of takes) { + expect(take).toBeDefined(); + expect(take!).toBeLessThanOrEqual(5_000); + } + }); }); diff --git a/src/lib/crypto/encryption-corpus.ts b/src/lib/crypto/encryption-corpus.ts index 31816b691..4efc0d46d 100644 --- a/src/lib/crypto/encryption-corpus.ts +++ b/src/lib/crypto/encryption-corpus.ts @@ -22,7 +22,7 @@ * `ENCRYPTION_KEYS` / `ENCRYPTION_ACTIVE_KEY_ID`. The operator's env key map * is the only place keys live; a key drop stays a deliberate env + redeploy * act, never a button. - * - IDEMPOTENT. `shouldRotate()` skips rows already on the active key, so a + * - IDEMPOTENT. The walk skips rows already on the active key, so a * second pass (or two racing workers) re-encrypts zero rows. * - FAIL-CLOSED. A row written under a key id that is no longer configured * throws on decrypt (counted as an error, the row is left untouched) rather @@ -54,6 +54,15 @@ export const LEGACY_BUCKET = "legacy"; */ export const BLOB_ROTATION_BATCH_SIZE = 25; +/** + * Batch size for every other column. These rows are short strings, but a + * column can have one per measurement: `Measurement.notesEncrypted` is read + * for every row of the table, 1.25 million on a large account (#1031). The + * walk used to read such a column in one `findMany`; it is id-cursor paged + * like the blob columns, only with larger pages. + */ +export const ROW_ROTATION_BATCH_SIZE = 5_000; + /** Minimal Prisma delegate shape this module needs. */ interface ColumnDelegate { findMany: (args: { @@ -114,10 +123,6 @@ function fromCiphertext( return next; } -function shouldRotate(ciphertext: string): boolean { - return extractKeyId(ciphertext) !== getActiveKeyId(); -} - // ─── Codec-dispatched blob columns (document vault) ───────────────────────── /** @@ -152,15 +157,16 @@ function reencryptBlob(value: Uint8Array, codec: string): Uint8Array { } /** - * Walk a blob column in bounded id-cursor batches, invoking `onRow` per - * non-empty row. At most `BLOB_ROTATION_BATCH_SIZE` rows are in memory per - * step; an interrupted run resumes safely on re-invocation because processing - * is idempotent (already-active rows are skipped by the callers). + * Walk a column in bounded id-cursor batches, invoking `onRow` per non-empty + * row. At most one batch is in memory per step (`BLOB_ROTATION_BATCH_SIZE` + * rows of a blob column, `ROW_ROTATION_BATCH_SIZE` of any other); an + * interrupted run resumes safely on re-invocation because processing is + * idempotent (already-active rows are skipped by the callers). * * `codec` is the row's own codec label for a codec-dispatched column and null * for a plain `batched` one — a backup blob has one layout, just a large one. */ -async function walkBlobColumn( +async function walkColumn( delegate: ColumnDelegate, col: EncryptedColumn, onRow: (row: { @@ -171,6 +177,9 @@ async function walkBlobColumn( ): Promise { const codecField = col.codecField; const pk = pkField(col); + const batchSize = isBlobColumn(col) + ? BLOB_ROTATION_BATCH_SIZE + : ROW_ROTATION_BATCH_SIZE; let cursor: string | null = null; for (;;) { const rows = await delegate.findMany({ @@ -180,7 +189,7 @@ async function walkBlobColumn( ...(codecField ? { [codecField]: true } : {}), }, orderBy: { [pk]: "asc" }, - take: BLOB_ROTATION_BATCH_SIZE, + take: batchSize, ...(cursor ? { cursor: { [pk]: cursor }, skip: 1 } : {}), }); if (rows.length === 0) break; @@ -196,7 +205,7 @@ async function walkBlobColumn( }); } cursor = rows[rows.length - 1]![pk] as string; - if (rows.length < BLOB_ROTATION_BATCH_SIZE) break; + if (rows.length < batchSize) break; } } @@ -205,8 +214,8 @@ function pkField(col: EncryptedColumn): string { return col.pkField ?? "id"; } -/** True when the column's rows must be walked in bounded batches. */ -function isBatched(col: EncryptedColumn): boolean { +/** True when the column's rows are blobs, walked in the small batches. */ +function isBlobColumn(col: EncryptedColumn): boolean { return Boolean(col.codecField ?? col.batched); } @@ -242,25 +251,12 @@ export async function scanColumn( const byKeyId: Record = {}; let total = 0; - if (isBatched(col)) { - // Blob column: bounded batches, per-row codec where the column has one. - await walkBlobColumn(delegate, col, ({ value, codec }) => { - total += 1; - const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET; - byKeyId[id] = (byKeyId[id] ?? 0) + 1; - }); - } else { - const rows = await delegate.findMany({ - select: { [pkField(col)]: true, [col.field]: true }, - }); - for (const row of rows) { - const ciphertext = toCiphertext(row[col.field], col.kind); - if (ciphertext == null) continue; - total += 1; - const id = extractKeyId(ciphertext) ?? LEGACY_BUCKET; - byKeyId[id] = (byKeyId[id] ?? 0) + 1; - } - } + // Bounded batches for every column, per-row codec where the column has one. + await walkColumn(delegate, col, ({ value, codec }) => { + total += 1; + const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET; + byKeyId[id] = (byKeyId[id] ?? 0) + 1; + }); return { model: col.model, field: col.field, @@ -356,57 +352,33 @@ export async function rotateColumn( } }; - if (isBatched(col)) { - // Blob column: bounded id-cursor batches (never an unbounded findMany), - // re-encrypted under each row's OWN codec where it has one. Idempotent — - // rows already on the active key are skipped, so an interrupted run - // resumes cleanly on the next invocation. - await walkBlobColumn(delegate, col, async ({ id, value, codec }) => { - result.scanned += 1; - if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return; - try { - // The plaintext is never inspected, only re-sealed, so every envelope - // a stored value can legitimately carry survives rotation untouched. - const next = - codec !== null - ? reencryptBlob(value as Uint8Array, codec) - : fromCiphertext( - encrypt(decrypt(toCiphertext(value, col.kind)!)), - col.kind, - ); - await delegate.update({ - where: { [pkField(col)]: id }, - data: { [col.field]: next }, - }); - result.rotated += 1; - } catch { - await onUnreadable(id); - } - }); - return result; - } - - const pk = pkField(col); - const rows = await delegate.findMany({ - select: { [pk]: true, [col.field]: true }, - }); - result.scanned = rows.length; - for (const row of rows) { - const ciphertext = toCiphertext(row[col.field], col.kind); - if (ciphertext == null || !shouldRotate(ciphertext)) continue; - const id = row[pk] as string; + // Bounded id-cursor batches for every column (never an unbounded + // findMany), re-encrypted under each row's OWN codec where it has one. + // Idempotent — rows already on the active key are skipped, so an + // interrupted run resumes cleanly on the next invocation. `scanned` counts + // the rows that hold ciphertext, whatever the column. + await walkColumn(delegate, col, async ({ id, value, codec }) => { + result.scanned += 1; + if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return; try { - // ACTIVE-KEY-ONLY: encrypt() always writes the active key id. - const reencrypted = encrypt(decrypt(ciphertext)); + // The plaintext is never inspected, only re-sealed, so every envelope + // a stored value can legitimately carry survives rotation untouched. + const next = + codec !== null + ? reencryptBlob(value as Uint8Array, codec) + : fromCiphertext( + encrypt(decrypt(toCiphertext(value, col.kind)!)), + col.kind, + ); await delegate.update({ - where: { [pk]: id }, - data: { [col.field]: fromCiphertext(reencrypted, col.kind) }, + where: { [pkField(col)]: id }, + data: { [col.field]: next }, }); result.rotated += 1; } catch { await onUnreadable(id); } - } + }); return result; } diff --git a/src/lib/export/__tests__/backup-blob.test.ts b/src/lib/export/__tests__/backup-blob.test.ts index cf3225a34..4f632bebb 100644 --- a/src/lib/export/__tests__/backup-blob.test.ts +++ b/src/lib/export/__tests__/backup-blob.test.ts @@ -17,6 +17,7 @@ process.env.ENCRYPTION_KEY ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +import { randomBytes } from "node:crypto"; import v8 from "node:v8"; import { describe, expect, it } from "vitest"; @@ -26,6 +27,7 @@ import { BackupBlobTooLargeError, defaultBackupBlobLimit, packBackupBlob, + packBackupBlobInto, packBackupBlobStreaming, unpackBackupBlob, } from "../backup-blob"; @@ -213,3 +215,66 @@ describe("backup blob envelope", () => { expect(limit).toBeGreaterThan(16 * 1024 * 1024); }); }); + +/** + * Issue #1031: the weekly and manual backup kept the stored copy as one string + * before writing it, and at 1.25 million measurements that string and the + * driver's copies of it added about 640 MB to the process. `packBackupBlobInto` + * hands the stored copy on in bounded pieces instead, as it is produced. + */ +describe("piecewise envelope", () => { + /** JSON that gzip cannot shrink much, so the stored copy is several MB. */ + function incompressibleJson(bytes: number): string { + return JSON.stringify({ + noise: randomBytes(Math.ceil((bytes * 3) / 4)).toString("base64"), + }); + } + + it("delivers pieces that concatenate to a blob reading back the exact JSON", async () => { + const json = sampleJson(3_000); + const pieces: string[] = []; + await packBackupBlobInto( + (piece) => { + pieces.push(piece); + }, + async (write) => { + for (let at = 0; at < json.length; at += 4_096) { + await write(json.slice(at, at + 4_096)); + } + }, + ); + expect(unpackBackupBlob(pieces.join(""))).toBe(json); + }); + + it("hands pieces on while the producer is still writing, each one bounded", async () => { + const json = incompressibleJson(12 * 1024 * 1024); + const pieces: number[] = []; + let producing = false; + let piecesWhileProducing = 0; + const chunks: string[] = []; + + await packBackupBlobInto( + (piece) => { + pieces.push(piece.length); + chunks.push(piece); + if (producing) piecesWhileProducing += 1; + }, + async (write) => { + producing = true; + for (let at = 0; at < json.length; at += 64 * 1024) { + await write(json.slice(at, at + 64 * 1024)); + } + producing = false; + }, + { maxBytes: Number.MAX_SAFE_INTEGER }, + ); + + expect(pieces.length).toBeGreaterThan(2); + expect(piecesWhileProducing).toBeGreaterThan(1); + // The flush threshold is 4 MB; one gzip chunk's worth of slack on top. + for (const length of pieces) { + expect(length).toBeLessThan(4 * 1024 * 1024 + 256 * 1024); + } + expect(unpackBackupBlob(chunks.join(""))).toBe(json); + }); +}); diff --git a/src/lib/export/__tests__/backup-json-scan.test.ts b/src/lib/export/__tests__/backup-json-scan.test.ts new file mode 100644 index 000000000..773e8080e --- /dev/null +++ b/src/lib/export/__tests__/backup-json-scan.test.ts @@ -0,0 +1,185 @@ +/** + * `scanBackupJson` must read a backup exactly as `JSON.parse` of the whole + * document would, with the streamed members handed out one element at a time + * (issue #1031: a 662 MB backup cannot exist as one string). These tests hold + * it to that over awkward values and arbitrary chunk boundaries, including + * boundaries inside strings, escapes and multi-byte characters. + */ +import { describe, expect, it } from "vitest"; + +import { BackupJsonError, scanBackupJson } from "../backup-json-scan"; + +async function* chunked(text: string, sizes: number[]) { + const bytes = Buffer.from(text, "utf8"); + let at = 0; + let i = 0; + while (at < bytes.length) { + const size = sizes[i++ % sizes.length]!; + yield bytes.subarray(at, at + size); + at += size; + } +} + +function rng(seed: number) { + let x = seed; + return () => { + x = (x * 1103515245 + 12345) % 2 ** 31; + return x / 2 ** 31; + }; +} + +const AWKWARD = [ + 'quote " inside', + "back\\slash and \\n", + "brackets ] } [ { , : in a string", + "umlaut ä ö ü and emoji 🫀", + "", + "\u0000\u001f control", +]; + +function randomValue(rand: () => number, depth: number): unknown { + const r = rand(); + if (depth > 2 || r < 0.35) { + const p = rand(); + if (p < 0.2) return null; + if (p < 0.3) return rand() < 0.5; + if (p < 0.55) return Math.round(rand() * 1e6) / 100 - 500; + return ( + AWKWARD[Math.floor(rand() * AWKWARD.length)]! + Math.floor(rand() * 9) + ); + } + if (r < 0.65) { + return Array.from({ length: Math.floor(rand() * 4) }, () => + randomValue(rand, depth + 1), + ); + } + const obj: Record = {}; + for (let i = 0; i < Math.floor(rand() * 4); i++) { + obj[`${AWKWARD[i % AWKWARD.length]}-${i}`] = randomValue(rand, depth + 1); + } + return obj; +} + +function randomDocument(seed: number) { + const rand = rng(seed); + return { + schemaVersion: "2", + userId: "user-ä", + measurements: Array.from({ length: Math.floor(rand() * 30) }, (_, i) => + i % 5 === 4 + ? randomValue(rand, 1) + : { id: `m${i}`, notes: AWKWARD[i % 6] }, + ), + medications: [randomValue(rand, 0), randomValue(rand, 0)], + moodEntries: [], + count: 42, + flag: false, + nothing: null, + text: AWKWARD[seed % AWKWARD.length], + nested: randomValue(rand, 0), + }; +} + +async function scan(text: string, sizes: number[], streamKeys: string[]) { + const elements: Record = {}; + const result = await scanBackupJson(chunked(text, sizes), { + streamKeys: new Set(streamKeys), + onElement: (key, element, index) => { + (elements[key] ??= [])[index] = element; + }, + }); + return { ...result, elements }; +} + +describe("scanBackupJson", () => { + it("equals JSON.parse over random documents, formats and chunk splits", async () => { + for (let seed = 1; seed <= 200; seed++) { + const doc = randomDocument(seed); + const text = + seed % 3 === 0 + ? JSON.stringify(doc, null, seed % 2 ? 2 : "\t") + : JSON.stringify(doc); + const sizes = [1 + (seed % 7), 2, 13, 1, 64].slice(seed % 3); + const { document, streamedCounts, elements } = await scan(text, sizes, [ + "measurements", + "moodEntries", + ]); + const expected = JSON.parse(text) as Record; + + expect(elements.measurements ?? [], `seed ${seed}`).toEqual( + expected.measurements, + ); + expect(streamedCounts).toEqual({ + measurements: (expected.measurements as unknown[]).length, + moodEntries: 0, + }); + expect(document, `seed ${seed}`).toEqual({ + ...expected, + measurements: [], + moodEntries: [], + }); + } + }); + + it("hands out streamed elements before it has read the rest of the document", async () => { + const text = JSON.stringify({ + a: 1, + measurements: [{ id: "x" }, { id: "y" }], + tail: "z".repeat(100), + }); + const bytes = Buffer.from(text); + let delivered = 0; + const seenAt: number[] = []; + async function* source() { + for (let i = 0; i < bytes.length; i += 4) { + delivered = i + 4; + yield bytes.subarray(i, i + 4); + } + } + await scanBackupJson(source(), { + streamKeys: new Set(["measurements"]), + onElement: () => { + seenAt.push(delivered); + }, + }); + expect(seenAt).toHaveLength(2); + expect(seenAt[1]!).toBeLessThan(bytes.length - 100); + }); + + it("skips a member it was told to skip without keeping it", async () => { + const { document } = await scanBackupJson( + chunked('{"a":[1,{"b":"]"}],"big":{"x":[1,2,3]},"c":"d"}', [3]), + { skipKeys: new Set(["big"]) }, + ); + expect(document).toEqual({ a: [1, { b: "]" }], c: "d" }); + }); + + it("refuses a truncated or malformed document", async () => { + for (const bad of [ + '{"a":1', + '{"a":[1,2}', + "[1,2]", + '{"measurements":{"x":1}}', + '{"a":1}{', + '{"measurements":[{"id":1} {"id":2}]}', + ]) { + await expect( + scanBackupJson(chunked(bad, [2]), { + streamKeys: new Set(["measurements"]), + }), + bad, + ).rejects.toBeInstanceOf(BackupJsonError); + } + }); + + it("propagates a failure from the element consumer", async () => { + await expect( + scanBackupJson(chunked('{"measurements":[1,2,3]}', [5]), { + streamKeys: new Set(["measurements"]), + onElement: (_key, _element, index) => { + if (index === 1) throw new Error("consumer failed"); + }, + }), + ).rejects.toThrow("consumer failed"); + }); +}); diff --git a/src/lib/export/__tests__/passphrase-archive.test.ts b/src/lib/export/__tests__/passphrase-archive.test.ts index 2cd158c97..dcdb84c5b 100644 --- a/src/lib/export/__tests__/passphrase-archive.test.ts +++ b/src/lib/export/__tests__/passphrase-archive.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest"; import { Buffer } from "node:buffer"; import { encryptArchive, + encryptArchiveToFile, decryptArchive, parseArchiveHeader, EXPORT_ARGON2_PARAMS, @@ -78,3 +79,45 @@ describe("passphrase-encrypted export archive (HLX1)", () => { expect(await decryptArchive(archive, PASSPHRASE)).toBe(PAYLOAD); }); }); + +/** + * #1031: the encrypted export is sealed as it is produced and spooled to a + * file, because holding it did not fit a 1 GB container for a large account. + * The format must not change: an archive written this way opens with the + * same `decryptArchive` every existing archive opens with. + */ +describe("encryptArchiveToFile", () => { + it("writes an archive decryptArchive opens, byte for byte the same document", async () => { + const { mkdtemp, readFile, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const dir = await mkdtemp(join(tmpdir(), "hlx-spool-")); + try { + const pieces = Array.from({ length: 500 }, (_, i) => + JSON.stringify({ i, note: "ümlaut 🫀 ".repeat(i % 7) }), + ); + const bodyPath = join(dir, "body"); + const spooled = await encryptArchiveToFile( + async (write) => { + await write("["); + for (let i = 0; i < pieces.length; i++) { + await write(i === 0 ? pieces[i]! : `,${pieces[i]}`); + } + await write(Buffer.from("]")); + }, + PASSPHRASE, + bodyPath, + ); + const archive = Buffer.concat([spooled.prefix, await readFile(bodyPath)]); + expect(archive.byteLength).toBe(spooled.byteLength); + expect(await decryptArchive(archive, PASSPHRASE)).toBe( + `[${pieces.join(",")}]`, + ); + await expect( + decryptArchive(archive, "wrong passphrase!!"), + ).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/export/backup-blob.ts b/src/lib/export/backup-blob.ts index a05760650..0c152ba45 100644 --- a/src/lib/export/backup-blob.ts +++ b/src/lib/export/backup-blob.ts @@ -37,7 +37,8 @@ */ import { Buffer } from "node:buffer"; import v8 from "node:v8"; -import { createGzip, gunzipSync, gzipSync } from "node:zlib"; +import { Readable } from "node:stream"; +import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import { createStreamEncryptor, @@ -125,7 +126,7 @@ export class BackupBlobTooLargeError extends Error { * counts) is the caller's business, not the envelope's. */ export type BackupJsonProducer = ( - write: (chunk: string) => Promise, + write: (chunk: string | Buffer) => Promise, ) => Promise; export interface PackBackupBlobOptions { @@ -141,26 +142,59 @@ export interface PackBackupBlobOptions { * Serialised backup JSON, produced in pieces → the stored string. * * The pipeline is JSON piece → gzip → AES-256-GCM → base64, with nothing - * buffered end to end but the base64 answer. `producer` decides how big its - * pieces are; the gzip stream applies backpressure through the promise this - * hands back, so a fast producer cannot outrun the compressor and pile up - * chunks in the stream's internal queue. - * - * The one copy that cannot be avoided is the answer itself. `data_backups.data` - * is a single `text` column, so the row has to be one value, and one value has - * to exist as one string before the driver can bind it. That copy is therefore - * also the only thing here that grows without bound as a record grows, which - * is why the size check lives on it: `maxBytes` counts the ciphertext this - * call has actually produced and stops on the account whose backup does not - * fit, rather than reading a heap gauge every account shares. + * buffered end to end but the base64 answer. Kept for callers that need the + * answer as one value; the weekly and manual backup no longer do, they hand + * the pieces to Postgres as they come (`packBackupBlobInto` below, via + * `storeBackupBlob`). */ export async function packBackupBlobStreaming( producer: BackupJsonProducer, options: PackBackupBlobOptions = {}, ): Promise { + const pieces: string[] = []; + await packBackupBlobInto( + (piece) => { + pieces.push(piece); + }, + producer, + options, + ); + return pieces.join(""); +} + +/** Receives the stored string a piece at a time, in order. Awaited. */ +export type BackupBlobSink = (piece: string) => void | Promise; + +/** + * How much base64 to gather before handing it to the sink. Large enough that + * a sink writing to the database makes a few dozen round trips for a large + * record rather than thousands, small enough to be irrelevant to the heap. + */ +const SINK_FLUSH_BYTES = 4 * 1024 * 1024; + +/** + * Serialised backup JSON, produced in pieces → the stored string, delivered + * in pieces. + * + * The concatenation of every piece `sink` receives is exactly the string + * `packBackupBlobStreaming` returns. `producer` decides how big its JSON + * pieces are; the gzip stream applies backpressure through the promise this + * hands back, so a fast producer cannot outrun the compressor, and a slow + * sink holds the producer back the same way, so neither side piles up. + * + * `maxBytes` still counts the whole answer. Nothing here holds it any more, + * but every reader of a stored copy (restore, download, summary) opens it as + * one value, so the limit is what keeps a copy restorable on this host. + */ +export async function packBackupBlobInto( + sink: BackupBlobSink, + producer: BackupJsonProducer, + options: PackBackupBlobOptions = {}, +): Promise { const limitBytes = options.maxBytes ?? defaultBackupBlobLimit(); const encryptor = createStreamEncryptor(); - const pieces: string[] = [encryptor.header]; + let pending: string[] = [encryptor.header]; + let pendingBytes = encryptor.header.length; let heldBytes = encryptor.header.length; const gzip = createGzip(); @@ -169,12 +203,13 @@ export async function packBackupBlobStreaming( try { const piece = encryptor.update(chunk); if (piece === "") return; - // Base64 is ASCII, so a character counted here is a byte held here. + // Base64 is ASCII, so a character counted here is a byte written here. heldBytes += piece.length; if (heldBytes > limitBytes) { throw new BackupBlobTooLargeError(heldBytes, limitBytes); } - pieces.push(piece); + pending.push(piece); + pendingBytes += piece.length; } catch (err) { failure ??= err; gzip.destroy(err as Error); @@ -192,8 +227,19 @@ export async function packBackupBlobStreaming( // rejection; `await finished` below still sees the rejection. finished.catch(() => {}); - const write = async (chunk: string): Promise => { + const flush = async (force: boolean): Promise => { + if (pendingBytes === 0 || (!force && pendingBytes < SINK_FLUSH_BYTES)) { + return; + } + const piece = pending.join(""); + pending = []; + pendingBytes = 0; + await sink(piece); + }; + + const write = async (chunk: string | Buffer): Promise => { if (failure) throw failure; + await flush(false); if (gzip.write(chunk, "utf8")) return; await new Promise((resolve, reject) => { const onDrain = () => { @@ -219,8 +265,10 @@ export async function packBackupBlobStreaming( await finished; if (failure) throw failure; - pieces.push(encryptor.final()); - return pieces.join(""); + const tail = encryptor.final(); + pending.push(tail); + pendingBytes += tail.length; + await flush(true); } /** @@ -266,3 +314,34 @@ export function unpackBackupBlob(stored: string): string { Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"), ).toString("utf8"); } + +/** + * A stored `DataBackup.data` string → a source of its JSON as byte chunks, + * which can be opened as many times as the caller needs to read it. + * + * `unpackBackupBlob` returns the JSON as one string, and a string is what a + * large record cannot be: the disaster-recovery JSON of an account with 1.25 + * million measurements is 662 MB, past the 536 870 888 characters V8 allows + * in any string, so that call threw on every such backup (#1031). Here the + * authentication happens first and whole, on the compressed ciphertext, which + * stays small (48 MB for that record): no plaintext byte is released before + * the tag has verified. Only the decompression is streamed, on every open. + * + * Fail-closed exactly like `unpackBackupBlob`: a bad key or a tag that does + * not verify throws here, before a source exists; a truncated gzip member + * errors the stream. + */ +export function openBackupBlob(stored: string): () => AsyncIterable { + if (isStreamCiphertext(stored)) { + const gz = decryptStream(stored); + return () => Readable.from([gz]).pipe(createGunzip()); + } + const plaintext = decrypt(stored); + if (!plaintext.startsWith(GZIP_MARKER)) { + // A pre-envelope copy was written from one string, so it is one. + const bytes = Buffer.from(plaintext, "utf8"); + return () => Readable.from([bytes]); + } + const gz = Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"); + return () => Readable.from([gz]).pipe(createGunzip()); +} diff --git a/src/lib/export/backup-json-scan.ts b/src/lib/export/backup-json-scan.ts new file mode 100644 index 000000000..60c3c967a --- /dev/null +++ b/src/lib/export/backup-json-scan.ts @@ -0,0 +1,336 @@ +/** + * Read a backup's JSON as a stream, without ever holding the document. + * + * Why. A backup is one JSON object, and every reader used to turn it into + * one string and `JSON.parse` it. For an account of 1.25 million measurements + * (issue #1031) the disaster-recovery JSON is 662 MB. V8 cannot hold a string + * longer than 536 870 888 characters at all, so `unpackBackupBlob` threw + * before a byte was parsed: the weekly backup of that account could not be + * restored, downloaded or even summarised, and the restore route reported the + * copy as undecryptable. A portable export of the same account fits under + * the limit but not beside the rest of a 1 GB container. + * + * What this does. It scans the bytes once, tracking only string and nesting + * state. Every top-level member is parsed on its own with `JSON.parse` and + * kept, except the members named in `streamKeys`: those must be arrays, and + * each element is parsed and handed to `onElement` instead, so the caller + * sees them one at a time and the kept object holds an empty array in their + * place. A backup's bulk tables (measurements, intake events, mood entries) + * are the only members that grow with a record, so everything else is small. + * + * Structural characters are ASCII and UTF-8 never uses an ASCII byte inside a + * multi-byte sequence, so scanning bytes is exact. A value is decoded only + * once it is complete, which is where a character split across two chunks is + * put back together. + * + * The output is what `JSON.parse` of the whole document would give, with the + * streamed arrays emptied: the tests hold it to that over arbitrary splits. + */ + +/** Characters the scanner cares about, as bytes. */ +const QUOTE = 0x22; +const BACKSLASH = 0x5c; +const OPEN_BRACE = 0x7b; +const CLOSE_BRACE = 0x7d; +const OPEN_BRACKET = 0x5b; +const CLOSE_BRACKET = 0x5d; +const COMMA = 0x2c; +const COLON = 0x3a; + +function isWhitespace(byte: number): boolean { + return byte === 0x20 || byte === 0x0a || byte === 0x0d || byte === 0x09; +} + +export interface ScanBackupJsonOptions { + /** Top-level members whose array elements are streamed, not kept. */ + streamKeys?: ReadonlySet; + /** + * Receives each element of a streamed member, parsed, with its index. + * Awaited, so a consumer that writes to a database applies backpressure. + */ + onElement?: (key: string, element: unknown, index: number) => unknown; + /** + * Members to skip entirely: scanned past, never parsed or kept. For a + * reader that only needs a few sections of the document. + */ + skipKeys?: ReadonlySet; +} + +export interface ScanBackupJsonResult { + /** The document, with streamed members as empty arrays and skipped ones absent. */ + document: Record; + /** How many elements each streamed member held. */ + streamedCounts: Record; +} + +/** Thrown for input that is not a JSON object the scanner can walk. */ +export class BackupJsonError extends Error { + constructor(message: string) { + super(message); + this.name = "BackupJsonError"; + } +} + +/** Collects the bytes of one value across chunks. */ +class Collector { + private parts: Buffer[] = []; + private bytes = 0; + add(chunk: Buffer, from: number, to: number): void { + if (to > from) { + this.parts.push(chunk.subarray(from, to)); + this.bytes += to - from; + } + } + take(): string { + const text = + this.parts.length === 1 + ? this.parts[0]!.toString("utf8") + : Buffer.concat(this.parts, this.bytes).toString("utf8"); + this.parts = []; + this.bytes = 0; + return text; + } +} + +type Phase = + | "start" // before the opening brace + | "key" // expecting a key or the closing brace + | "colon" + | "value" // expecting the start of a member value + | "in-value" // inside a kept or skipped member value + | "array-start" // a streamed member: expecting its opening bracket + | "element" // inside a streamed array, expecting an element or `]` + | "in-element" // inside one element + | "after" // after a member, expecting `,` or `}` + | "done"; + +/** + * Scan a backup document delivered as byte chunks. Resolves to the document + * (see the module comment) once the closing brace has been read; rejects with + * `BackupJsonError` on malformed input or with whatever `onElement` threw. + */ +export async function scanBackupJson( + chunks: AsyncIterable, + options: ScanBackupJsonOptions = {}, +): Promise { + const streamKeys = options.streamKeys ?? new Set(); + const skipKeys = options.skipKeys ?? new Set(); + const document: Record = {}; + const streamedCounts: Record = {}; + + let phase: Phase = "start"; + let inString = false; + let escaped = false; + // Nesting depth inside the current value or element (0 = not inside one). + let depth = 0; + let key = ""; + let skipping = false; + // Inside a key string. + let keyOpen = false; + // Inside a streamed array, after an element: the next byte must be `,` or `]`. + let elementNeedsComma = false; + const keyBytes = new Collector(); + const value = new Collector(); + + const finishValue = () => { + const text = value.take(); + if (skipping) return; + try { + document[key] = JSON.parse(text); + } catch { + throw new BackupJsonError(`Backup member '${key}' is not valid JSON`); + } + }; + const finishElement = async () => { + const text = value.take(); + let element: unknown; + try { + element = JSON.parse(text); + } catch { + throw new BackupJsonError( + `Element ${streamedCounts[key]} of backup member '${key}' is not valid JSON`, + ); + } + const index = streamedCounts[key]!; + streamedCounts[key] = index + 1; + await options.onElement?.(key, element, index); + }; + + for await (const raw of chunks) { + const chunk = Buffer.isBuffer(raw) + ? raw + : Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength); + // Where the value or key being collected started in this chunk. + let mark = 0; + for (let i = 0; i < chunk.length; i++) { + const byte = chunk[i]!; + + if (phase === "in-value" || phase === "in-element") { + if (inString) { + if (escaped) escaped = false; + else if (byte === BACKSLASH) escaped = true; + else if (byte === QUOTE) inString = false; + continue; + } + if (byte === QUOTE) { + inString = true; + continue; + } + if (byte === OPEN_BRACE || byte === OPEN_BRACKET) { + depth += 1; + continue; + } + if (byte === CLOSE_BRACE || byte === CLOSE_BRACKET) { + if (depth > 0) { + depth -= 1; + if (depth > 0) continue; + // The value's own closing bracket: it belongs to the value. + value.add(chunk, mark, i + 1); + if (phase === "in-value") { + finishValue(); + phase = "after"; + } else { + await finishElement(); + phase = "element"; + elementNeedsComma = true; + } + continue; + } + // A primitive value ended by the enclosing bracket. + value.add(chunk, mark, i); + if (phase === "in-value") { + if (byte !== CLOSE_BRACE) { + throw new BackupJsonError("Unbalanced bracket in backup"); + } + finishValue(); + phase = "done"; + } else { + if (byte !== CLOSE_BRACKET) { + throw new BackupJsonError("Unbalanced bracket in backup"); + } + await finishElement(); + phase = "after"; + } + continue; + } + if (depth === 0 && (byte === COMMA || isWhitespace(byte))) { + // A primitive value ended. + value.add(chunk, mark, i); + if (phase === "in-value") { + finishValue(); + phase = byte === COMMA ? "key" : "after"; + } else { + await finishElement(); + phase = "element"; + elementNeedsComma = byte !== COMMA; + } + continue; + } + continue; + } + + if (phase === "key" && keyOpen) { + if (escaped) escaped = false; + else if (byte === BACKSLASH) escaped = true; + else if (byte === QUOTE) { + keyBytes.add(chunk, mark, i + 1); + key = JSON.parse(keyBytes.take()) as string; + keyOpen = false; + phase = "colon"; + } + continue; + } + + if (isWhitespace(byte)) continue; + + switch (phase) { + case "start": + if (byte !== OPEN_BRACE) { + throw new BackupJsonError("Backup is not a JSON object"); + } + phase = "key"; + break; + case "key": + if (byte === CLOSE_BRACE) { + phase = "done"; + } else if (byte === QUOTE) { + keyOpen = true; + escaped = false; + mark = i; + } else { + throw new BackupJsonError("Malformed backup: expected a key"); + } + break; + case "colon": + if (byte !== COLON) { + throw new BackupJsonError("Malformed backup: expected ':'"); + } + phase = streamKeys.has(key) ? "array-start" : "value"; + skipping = skipKeys.has(key); + break; + case "value": + phase = "in-value"; + mark = i; + depth = 0; + inString = false; + escaped = false; + // Re-read this byte as the first byte of the value. + i -= 1; + break; + case "array-start": + if (byte !== OPEN_BRACKET) { + throw new BackupJsonError( + `Backup member '${key}' must be an array`, + ); + } + streamedCounts[key] = 0; + document[key] = []; + phase = "element"; + elementNeedsComma = false; + break; + case "element": + if (byte === CLOSE_BRACKET) { + phase = "after"; + } else if (byte === COMMA && elementNeedsComma) { + elementNeedsComma = false; + } else if (!elementNeedsComma) { + phase = "in-element"; + mark = i; + depth = 0; + inString = false; + escaped = false; + i -= 1; + } else { + throw new BackupJsonError( + `Malformed backup: expected ',' in '${key}'`, + ); + } + break; + case "after": + if (byte === COMMA) { + phase = "key"; + } else if (byte === CLOSE_BRACE) { + phase = "done"; + } else { + throw new BackupJsonError("Malformed backup: expected ',' or '}'"); + } + break; + case "done": + throw new BackupJsonError("Unexpected content after the backup"); + } + } + // Carry the unfinished key or value into the next chunk. + if (phase === "in-value" || phase === "in-element") { + value.add(chunk, mark, chunk.length); + } else if (phase === "key" && keyOpen) { + keyBytes.add(chunk, mark, chunk.length); + } + } + + // Anything but the closing brace as the last structural byte is a + // truncated file, including one that ends inside a member value. + if ((phase as Phase) !== "done") { + throw new BackupJsonError("Backup ends before its closing brace"); + } + return { document, streamedCounts }; +} diff --git a/src/lib/export/full-backup-stream.ts b/src/lib/export/full-backup-stream.ts index c5b364aa0..1f5b1cdcf 100644 --- a/src/lib/export/full-backup-stream.ts +++ b/src/lib/export/full-backup-stream.ts @@ -26,6 +26,7 @@ * trusting that paragraph. */ import type { PrismaClient } from "@/generated/prisma/client"; +import { reportJobProgress } from "@/lib/jobs/job-observer"; import { buildFullBackupPayload, isDeferredRows, @@ -45,6 +46,9 @@ export type BackupJsonSink = (chunk: string) => void | Promise; */ const FLUSH_BYTES = 256 * 1024; +/** How often a bulk section reports how far it has got to a running job. */ +const PROGRESS_EVERY_ROWS = 50_000; + /** * There is deliberately no heap reading in this writer. * @@ -125,6 +129,9 @@ export async function streamFullBackupJson( rows === 0 ? JSON.stringify(row) : `,${JSON.stringify(row)}`, ); rows++; + if (rows % PROGRESS_EVERY_ROWS === 0) { + reportJobProgress({ backup_section: key, backup_section_rows: rows }); + } } await write("]"); bulkCounts[key] = rows; diff --git a/src/lib/export/measurement-bulk-insert.ts b/src/lib/export/measurement-bulk-insert.ts new file mode 100644 index 000000000..f1db771d9 --- /dev/null +++ b/src/lib/export/measurement-bulk-insert.ts @@ -0,0 +1,136 @@ +/** + * Insert many measurements in one statement, as column arrays. + * + * Why not `createMany`. Prisma compiles and caches a query plan per query + * shape, and a `createMany` of N rows is its own shape: measured, one + * 1 000-row `createMany` of measurements leaves about 145 MB of retained + * heap behind, a 2 000-row one about 290 MB, and the smaller last batch of a + * restore adds its own plan on top. In a 1 GB container that alone took a + * restore of 1.25 million measurements past the heap limit (#1031). One + * `INSERT … SELECT FROM unnest(…)` with an array per column is a single small + * statement whatever the batch size, and about four times faster. + * + * `COLUMNS` maps every field of `MeasurementInsertRow` to its column and + * type. It is typed as a complete record over the row's keys, so a field + * added to the row without a column here does not compile, and the restore + * builds its rows as `MeasurementInsertRow`, so a field it starts writing has + * to be added to the row first. + */ +import type { Prisma } from "@/generated/prisma/client"; + +/** One measurement as the restore writes it. */ +export interface MeasurementInsertRow { + id: string; + userId: string; + type: string; + value: number; + valueMin: number | null; + valueMax: number | null; + unit: string; + source: string; + measuredAt: Date; + notes: string | null; + notesEncrypted: Uint8Array | null; + externalId: string | null; + externalSourceVersion: string | null; + aggregationProvenance: string | null; + glucoseContext: string | null; + sleepStage: string | null; + rhythmClassification: string | null; + deviceType: string | null; + syncVersion: number; + deletedAt: Date | null; + createdAt?: Date; + updatedAt?: Date; +} + +type Kind = + "text" | "float8" | "int4" | "timestamp" | "bytea" | `enum:${string}`; + +/** Column and element type per field. Complete by construction. */ +const COLUMNS: { [K in keyof MeasurementInsertRow]-?: [string, Kind] } = { + id: ["id", "text"], + userId: ["user_id", "text"], + type: ["type", "enum:measurement_type"], + value: ["value", "float8"], + valueMin: ["value_min", "float8"], + valueMax: ["value_max", "float8"], + unit: ["unit", "text"], + source: ["source", "enum:measurement_source"], + measuredAt: ["measured_at", "timestamp"], + notes: ["notes", "text"], + notesEncrypted: ["notes_encrypted", "bytea"], + externalId: ["external_id", "text"], + externalSourceVersion: ["external_source_version", "text"], + aggregationProvenance: [ + "aggregation_provenance", + "enum:measurement_aggregation_provenance", + ], + glucoseContext: ["glucose_context", "enum:glucose_context"], + sleepStage: ["sleep_stage", "enum:sleep_stage"], + rhythmClassification: ["rhythm_classification", "enum:rhythm_classification"], + deviceType: ["device_type", "text"], + syncVersion: ["sync_version", "int4"], + deletedAt: ["deleted_at", "timestamp"], + createdAt: ["created_at", "timestamp"], + updatedAt: ["updated_at", "timestamp"], +}; + +const FIELDS = Object.keys(COLUMNS) as Array; + +/** How each array travels, and how its element is turned back into a value. */ +function arrayType(kind: Kind): string { + if (kind === "float8" || kind === "int4") return `${kind}[]`; + return "text[]"; // timestamps as ISO text, bytes as hex, enums as labels +} + +function selectExpression( + field: keyof MeasurementInsertRow, + kind: Kind, +): string { + const ref = `u."${field}"`; + if (kind === "timestamp") { + // Stored as UTC in `timestamp(3)`. ISO text with a `Z` casts to the same + // wall-clock instant; the zone suffix is ignored by the cast. + const cast = `${ref}::timestamp(3)`; + // Prisma fills both on create when the row does not carry them. + return field === "createdAt" || field === "updatedAt" + ? `COALESCE(${cast}, CURRENT_TIMESTAMP)` + : cast; + } + if (kind === "bytea") return `decode(${ref}, 'hex')`; + if (kind.startsWith("enum:")) return `${ref}::${kind.slice(5)}`; + return ref; +} + +/** + * The statement, built once. Its text depends on nothing but `COLUMNS`, a + * compile-time table: no input reaches it, and every value travels as a bound + * array parameter. + */ +export const MEASUREMENT_BULK_INSERT_SQL = ` + INSERT INTO measurements (${FIELDS.map((f) => COLUMNS[f][0]).join(", ")}) + SELECT ${FIELDS.map((f) => selectExpression(f, COLUMNS[f][1])).join(", ")} + FROM unnest(${FIELDS.map((f, i) => `$${i + 1}::${arrayType(COLUMNS[f][1])}`).join(", ")}) + AS u(${FIELDS.map((f) => `"${f}"`).join(", ")}) +`; + +function cell(value: unknown, kind: Kind): unknown { + if (value === null || value === undefined) return null; + if (kind === "timestamp") return (value as Date).toISOString(); + if (kind === "bytea") return Buffer.from(value as Uint8Array).toString("hex"); + return value; +} + +/** Insert `rows` in one statement. Resolves to the number inserted. */ +export async function insertMeasurementRows( + tx: Prisma.TransactionClient, + rows: readonly MeasurementInsertRow[], +): Promise { + if (rows.length === 0) return 0; + const arrays = FIELDS.map((field) => { + const kind = COLUMNS[field][1]; + return rows.map((row) => cell(row[field], kind)); + }); + return tx.$executeRawUnsafe(MEASUREMENT_BULK_INSERT_SQL, ...arrays); +} diff --git a/src/lib/export/passphrase-archive.ts b/src/lib/export/passphrase-archive.ts index e30ee171b..f1793bdaf 100644 --- a/src/lib/export/passphrase-archive.ts +++ b/src/lib/export/passphrase-archive.ts @@ -39,6 +39,8 @@ * hashing params so tuning one never silently changes the other. */ import { hashRaw, type Algorithm } from "@node-rs/argon2"; +import { once } from "node:events"; +import { createWriteStream } from "node:fs"; import { createCipheriv, createDecipheriv, @@ -128,6 +130,11 @@ export async function encryptArchive( const ciphertext = Buffer.concat([cipher.update(body), cipher.final()]); const tag = cipher.getAuthTag(); + return Buffer.concat([archiveHeader(), salt, iv, tag, ciphertext]); +} + +/** The fixed `HLX1` header for the current KDF parameters. */ +function archiveHeader(): Buffer { const header = Buffer.alloc(4 + 1 + 1 + 4 + 4 + 1 + 1); header.write(MAGIC, 0, "ascii"); header.writeUInt8(VERSION, 4); @@ -136,8 +143,88 @@ export async function encryptArchive( header.writeUInt32BE(EXPORT_ARGON2_PARAMS.timeCost, 10); header.writeUInt8(EXPORT_ARGON2_PARAMS.parallelism, 14); header.writeUInt8(SALT_LENGTH, 15); + return header; +} - return Buffer.concat([header, salt, iv, tag, ciphertext]); +/** Where `encryptArchiveToFile` left an archive, in two parts. */ +export interface SpooledArchive { + /** Header, salt, IV and tag: everything in front of the ciphertext. */ + prefix: Buffer; + /** File holding the ciphertext, which follows the prefix. */ + bodyPath: string; + /** Size of the whole archive in bytes. */ + byteLength: number; +} + +/** + * Seal a JSON document produced in pieces into an `HLX1` archive without + * holding it: the ciphertext goes to `bodyPath` as it is produced. + * + * Why a file. The format puts the GCM tag in front of the ciphertext, and the + * tag exists only once the last byte has been encrypted, so the archive + * cannot be sent as it is written. Holding it in memory instead is what the + * route did, and for an account of 1.25 million measurements that did not fit + * a 1 GB container (#1031). Spooling the ciphertext keeps the format, so every + * archive already on someone's disk still opens the same way. Only ciphertext + * ever touches the disk; the caller deletes the file once it is sent. + * + * Decrypts with `decryptArchive` exactly like an archive `encryptArchive` + * wrote; the tests hold the two to that. + */ +export async function encryptArchiveToFile( + producer: ( + write: (chunk: string | Buffer) => Promise, + ) => Promise, + passphrase: string, + bodyPath: string, +): Promise { + if (passphrase.length < MIN_EXPORT_PASSPHRASE_LENGTH) { + throw new Error( + `Passphrase must be at least ${MIN_EXPORT_PASSPHRASE_LENGTH} characters`, + ); + } + const salt = randomBytes(SALT_LENGTH); + const key = await deriveKey(passphrase, EXPORT_ARGON2_PARAMS, salt); + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv(ALGORITHM, key, iv); + const file = createWriteStream(bodyPath, { mode: 0o600 }); + let bodyBytes = 0; + const fileError = new Promise((_, reject) => { + file.once("error", reject); + }); + fileError.catch(() => {}); + + const put = async (bytes: Buffer): Promise => { + if (bytes.length === 0) return; + bodyBytes += bytes.length; + if (!file.write(bytes)) { + await Promise.race([once(file, "drain"), fileError]); + } + }; + + try { + await producer(async (chunk) => { + await put( + cipher.update( + typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk, + ), + ); + }); + await put(cipher.final()); + file.end(); + await Promise.race([once(file, "finish"), fileError]); + } catch (err) { + file.destroy(); + throw err; + } + + const prefix = Buffer.concat([ + archiveHeader(), + salt, + iv, + cipher.getAuthTag(), + ]); + return { prefix, bodyPath, byteLength: prefix.length + bodyBytes }; } /** Parse + validate the fixed `HLX1` header. Throws on any structural fault. */ diff --git a/src/lib/export/response-stream.ts b/src/lib/export/response-stream.ts new file mode 100644 index 000000000..964bb2c41 --- /dev/null +++ b/src/lib/export/response-stream.ts @@ -0,0 +1,82 @@ +/** + * Turn a producer of text or bytes into a response body, with backpressure + * and cancellation. + * + * The export routes used to build their whole file as one string and hand it + * to the response. For an account of 1.25 million measurements that string + * and the object graph behind it did not fit a 1 GB container: one export + * took the app down for every user of the instance (#1031). The file is now + * written into the response as it is produced. + * + * Backpressure: the producer's `write` waits while the client is slower than + * the database, so nothing piles up in between. Cancellation: when the client + * goes away, the next `write` throws `ResponseStreamClosedError`, which stops + * the producer instead of leaving it to finish a file nobody reads. + */ +import { PassThrough, Readable } from "node:stream"; + +/** The client closed the response before the producer finished. */ +export class ResponseStreamClosedError extends Error { + constructor() { + super("The response was closed before the file was complete"); + this.name = "ResponseStreamClosedError"; + } +} + +export type ResponseProducer = ( + write: (chunk: string | Buffer) => Promise, +) => Promise; + +export interface ResponseStreamHooks { + /** Runs once the producer has written everything. */ + onComplete?: () => unknown; + /** + * Runs when the producer failed or the client left. The response has + * already started, so the client receives a truncated file; a truncated + * JSON file does not parse, and a truncated archive fails its tag. + */ + onError?: (err: unknown) => unknown; +} + +/** Start `producer` and return the web stream its output flows into. */ +export function streamToResponseBody( + producer: ResponseProducer, + hooks: ResponseStreamHooks = {}, +): ReadableStream { + const out = new PassThrough(); + + const write = async (chunk: string | Buffer): Promise => { + if (out.destroyed || out.writableEnded) + throw new ResponseStreamClosedError(); + if (out.write(chunk)) return; + await new Promise((resolve, reject) => { + const onDrain = () => { + out.off("close", onClose); + resolve(); + }; + const onClose = () => { + out.off("drain", onDrain); + reject(new ResponseStreamClosedError()); + }; + out.once("drain", onDrain); + out.once("close", onClose); + }); + }; + + void (async () => { + try { + await producer(write); + out.end(); + await hooks.onComplete?.(); + } catch (err) { + out.destroy(err instanceof Error ? err : new Error(String(err))); + try { + await hooks.onError?.(err); + } catch { + // Reporting a failure must not raise another one. + } + } + })(); + + return Readable.toWeb(out) as ReadableStream; +} diff --git a/src/lib/export/store-backup-blob.ts b/src/lib/export/store-backup-blob.ts new file mode 100644 index 000000000..82c8f0fae --- /dev/null +++ b/src/lib/export/store-backup-blob.ts @@ -0,0 +1,119 @@ +/** + * Write one account's stored backup (`data_backups.data`) without ever + * holding the stored copy in this process. + * + * Why. `packBackupBlobStreaming` streams the JSON through gzip and the cipher, + * but its answer is one string, because the column takes one value. Measured + * on a seeded account of 1.25 million measurements: the answer is 64 MB of + * base64, and between the pieces, the joined string and the copies the + * database driver makes to bind it, the backup grew the process by about + * 640 MB of resident memory (309 MB → 946 MB). The default compose stack runs + * the web server and the worker in one container capped at 1 GB, so on a + * record of that size the weekly or manual backup is the thing that takes the + * container down; a killed worker leaves its job `active` until pg-boss + * expires it two hours later as `job timed out`. + * + * What instead. The pieces go to Postgres as they come, a few megabytes at a + * time, into a temporary table on the transaction's own connection, and one + * statement assembles them into the row with `string_agg`. The process never + * holds more than one piece; Postgres builds the value it was going to store + * anyway. + * + * All of it is one transaction, so the account's previous copy stays in place + * until the new one is complete: a failure halfway through rolls back to it, + * and the temporary table goes with the transaction either way. + */ +import type { PrismaClient } from "@/generated/prisma/client"; + +import { + packBackupBlobInto, + type BackupJsonProducer, + type PackBackupBlobOptions, +} from "@/lib/export/backup-blob"; + +/** + * How long the storing transaction may stay open. The job around it expires + * after two hours (`DATA_BACKUP_SEND_OPTIONS`); the transaction has to give + * up before that so the failure is the transaction's own, with its own + * message, rather than an expired job. + */ +const STORE_TRANSACTION_TIMEOUT_MS = 90 * 60 * 1000; + +/** + * How long the transaction may sit idle between two pieces. The session + * default (`idle_in_transaction_session_timeout`, 60 s, see `src/lib/db.ts`) + * is right for a request and wrong here: the producer reads the whole record + * on other connections while this one waits for the next piece, and the + * sections before the first measurement can take longer than a minute on a + * large account. Scoped with SET LOCAL, so it ends with the transaction. + */ +const STORE_IDLE_TIMEOUT = "10min"; + +export interface StoreBackupBlobInput { + userId: string; + /** `DataBackup.type`: `WEEKLY_AUTO` for the scheduled and manual pass. */ + type: string; +} + +/** + * Pack `producer`'s JSON into the stored envelope and upsert it as the + * `(userId, type)` backup. Resolves to the row's id and the stored size. + * + * `input` may be a function, read once the producer has finished: an + * uploaded file names its owner somewhere inside itself, and the upload + * route only knows who that is once it has read the file through. + */ +export async function storeBackupBlob( + prisma: PrismaClient, + input: StoreBackupBlobInput | (() => StoreBackupBlobInput), + producer: BackupJsonProducer, + options: PackBackupBlobOptions = {}, +): Promise<{ id: string; bytes: number }> { + return prisma.$transaction( + async (tx) => { + await tx.$executeRawUnsafe( + `SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`, + ); + await tx.$executeRaw` + CREATE TEMP TABLE backup_blob_parts ( + seq integer PRIMARY KEY, + piece text NOT NULL + ) ON COMMIT DROP + `; + + let seq = 0; + let bytes = 0; + await packBackupBlobInto( + async (piece) => { + await tx.$executeRaw` + INSERT INTO backup_blob_parts (seq, piece) VALUES (${seq}, ${piece}) + `; + seq += 1; + bytes += piece.length; + }, + producer, + options, + ); + + const target = typeof input === "function" ? input() : input; + // The row first, through Prisma, so a new account's backup gets its id + // the same way every other row does; the data it briefly carries is + // never visible outside this transaction. + const row = await tx.dataBackup.upsert({ + where: { userId_type: { userId: target.userId, type: target.type } }, + update: { createdAt: new Date() }, + create: { userId: target.userId, type: target.type, data: "" }, + select: { id: true }, + }); + await tx.$executeRaw` + UPDATE data_backups + SET data = ( + SELECT string_agg(piece, '' ORDER BY seq) FROM backup_blob_parts + ) + WHERE id = ${row.id} + `; + return { id: row.id, bytes }; + }, + { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 }, + ); +} diff --git a/src/lib/export/streamed-backup.ts b/src/lib/export/streamed-backup.ts new file mode 100644 index 000000000..1e65d6150 --- /dev/null +++ b/src/lib/export/streamed-backup.ts @@ -0,0 +1,138 @@ +/** + * A backup file read as a stream: every section but the measurements in + * memory, the measurements handed over in batches. + * + * Why. The measurements are the one section that grows with a record without + * bound. For an account with 1.25 million of them the disaster-recovery JSON + * is 662 MB, which cannot exist as one JavaScript string, and the parsed rows + * alone would not fit a 1 GB container beside the running app (#1031). Every + * other section of the same file is a few megabytes. + * + * So the file is read twice. The first read keeps everything except the + * measurements, and checks each measurement against the same element schema + * the whole-file schema uses, counting them. Nothing is written on the + * strength of an unchecked row: a restore reads the file this way before it + * deletes anything. The second read, from `forEachMeasurementBatch`, hands + * the measurements over in order, in batches, for the writer. + * + * `raw` is the file exactly as `JSON.parse` would give it, with + * `measurements` replaced by an empty array when the file has that section + * (and still absent when it does not, which is what the section-presence + * check reads). Run it through `parseBackupPayload` for the rest of the + * validation; together that is the whole-file check. + */ +import { scanBackupJson } from "@/lib/export/backup-json-scan"; +import { + BACKUP_SCHEMA_VERSION, + backupMeasurementSchema, + type BackupMeasurement, +} from "@/lib/validations/backup"; + +/** The file's bytes, from the start, every time it is called. */ +export type BackupSource = () => AsyncIterable; + +const MEASUREMENTS = "measurements"; +const STREAMED = new Set([MEASUREMENTS]); + +/** A measurement the element schema refused, with where it is in the file. */ +export class StreamedBackupInvalidError extends Error { + readonly path: string; + constructor(path: string, message: string) { + super(`${path}: ${message}`); + this.name = "StreamedBackupInvalidError"; + this.path = path; + } +} + +export interface StreamedBackup { + /** The file without its measurements (see the module comment). */ + raw: Record; + /** How many measurements the file carries. */ + measurementCount: number; + /** The file's size in bytes. */ + bytes: number; + /** + * Read the file again and hand its measurements over in file order, in + * batches of at most `size`, each row parsed by the element schema. + * Awaits `fn` before reading further. + */ + forEachMeasurementBatch( + size: number, + fn: (batch: BackupMeasurement[]) => Promise, + ): Promise; +} + +function parseMeasurement(element: unknown, index: number): BackupMeasurement { + const parsed = backupMeasurementSchema.safeParse(element); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + throw new StreamedBackupInvalidError( + [MEASUREMENTS, index, ...(issue?.path ?? [])].join("."), + issue?.message ?? "invalid measurement", + ); + } + return parsed.data; +} + +/** + * First read of a backup file. Rejects with `BackupJsonError` for input that + * is not a JSON object and with `StreamedBackupInvalidError` for a + * measurement the schema refuses. + */ +export async function readStreamedBackup( + source: BackupSource, +): Promise { + let firstWithoutId: number | null = null; + let bytes = 0; + async function* counted() { + for await (const chunk of source()) { + bytes += chunk.byteLength; + yield chunk; + } + } + const { document, streamedCounts } = await scanBackupJson(counted(), { + streamKeys: STREAMED, + onElement: (_key, element, index) => { + const row = parseMeasurement(element, index); + if (!row.id && firstWithoutId === null) firstWithoutId = index; + }, + }); + + // The whole-file schema's cross-row rule, applied the same way: a canonical + // v2 file must give every measurement a stable id. + if ( + document.schemaVersion === BACKUP_SCHEMA_VERSION && + firstWithoutId !== null + ) { + throw new StreamedBackupInvalidError( + `${MEASUREMENTS}.${firstWithoutId}.id`, + "Canonical v2 measurements require a stable id", + ); + } + + const skipKeys = new Set( + Object.keys(document).filter((key) => key !== MEASUREMENTS), + ); + + return { + raw: document, + measurementCount: streamedCounts[MEASUREMENTS] ?? 0, + bytes, + async forEachMeasurementBatch(size, fn) { + let batch: BackupMeasurement[] = []; + await scanBackupJson(source(), { + streamKeys: STREAMED, + skipKeys, + onElement: async (_key, element, index) => { + batch.push(parseMeasurement(element, index)); + if (batch.length >= size) { + const full = batch; + batch = []; + await fn(full); + } + }, + }); + if (batch.length > 0) await fn(batch); + }, + }; +} diff --git a/src/lib/jobs/__tests__/drain-continuation.test.ts b/src/lib/jobs/__tests__/drain-continuation.test.ts new file mode 100644 index 000000000..c73e7ddeb --- /dev/null +++ b/src/lib/jobs/__tests__/drain-continuation.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { + DRAIN_MAX_CONTINUATIONS, + nextDrainContinuation, +} from "@/lib/jobs/drain-continuation"; + +// A nightly drain tick that ran out of budget hands the rest to a follow-up +// job, so a first-time backlog drains the same night (issue #1031). The chain +// must not loop on a pass that cannot move, and must end. + +describe("nextDrainContinuation", () => { + it("hands on a tick that stopped early after folding something", () => { + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 12, + continuation: 0, + }), + ).toBe(1); + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 1, + continuation: 5, + }), + ).toBe(6); + }); + + it("does not hand on a tick that finished", () => { + expect( + nextDrainContinuation({ + stoppedEarly: false, + daysFolded: 400, + continuation: 0, + }), + ).toBeNull(); + }); + + it("does not hand on a tick that stopped without progress", () => { + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 0, + continuation: 0, + }), + ).toBeNull(); + }); + + it("ends the chain at the cap", () => { + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 10, + continuation: DRAIN_MAX_CONTINUATIONS - 1, + }), + ).toBe(DRAIN_MAX_CONTINUATIONS); + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 10, + continuation: DRAIN_MAX_CONTINUATIONS, + }), + ).toBeNull(); + }); +}); diff --git a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts index b904a30dd..4421c5ebb 100644 --- a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts +++ b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts @@ -52,7 +52,7 @@ describe("reminder-worker — drainPerSampleCumulative nightly schedule", () => it("registers a createAndWork handler against the drain queue", () => { expect(source).toMatch( - /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,900}drainPerSampleCumulative/, + /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,1100}drainPerSampleCumulative/, ); }); diff --git a/src/lib/jobs/__tests__/job-budget.test.ts b/src/lib/jobs/__tests__/job-budget.test.ts new file mode 100644 index 000000000..27eec6bf6 --- /dev/null +++ b/src/lib/jobs/__tests__/job-budget.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; + +import { JOB_BUDGET_SHARE, jobBudget } from "@/lib/jobs/job-budget"; + +// pg-boss declares a job dead at its expiry but does not stop the handler; +// the long passes stop themselves on this budget instead (issue #1031). + +function job(expireInSeconds: number, signal = new AbortController().signal) { + return { expireInSeconds, signal }; +} + +describe("jobBudget", () => { + it("stops at the budget share of the job's expiry, not at the expiry", () => { + let now = 1_000_000; + const shouldStop = jobBudget([job(900)], () => now); + + now += 900 * 1000 * JOB_BUDGET_SHARE - 1; + expect(shouldStop()).toBe(false); + now += 1; + expect(shouldStop()).toBe(true); + expect(JOB_BUDGET_SHARE).toBeLessThan(1); + }); + + it("stops as soon as pg-boss aborts the job, whatever the clock says", () => { + const controller = new AbortController(); + const shouldStop = jobBudget([job(900, controller.signal)], () => 0); + expect(shouldStop()).toBe(false); + controller.abort(); + expect(shouldStop()).toBe(true); + }); + + it("takes the tightest expiry of a batch", () => { + let now = 0; + const shouldStop = jobBudget([job(7200), job(60)], () => now); + now = 60 * 1000 * JOB_BUDGET_SHARE; + expect(shouldStop()).toBe(true); + }); + + it("never stops on the clock when there is no job to take an expiry from", () => { + let now = 0; + const shouldStop = jobBudget([], () => now); + now = Number.MAX_SAFE_INTEGER; + expect(shouldStop()).toBe(false); + }); +}); diff --git a/src/lib/jobs/__tests__/job-observer.test.ts b/src/lib/jobs/__tests__/job-observer.test.ts new file mode 100644 index 000000000..9e7c20920 --- /dev/null +++ b/src/lib/jobs/__tests__/job-observer.test.ts @@ -0,0 +1,200 @@ +/** + * Issue #1031: three queues failed with `job timed out` night after night and + * the operator found no line about any of them in the app log. These tests + * pin the lines `observeJob` now writes — progress while a job runs, a warning + * at its expiry, and a closing line for a long or expired run — and the boot + * line naming jobs a previous process was running when it stopped. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + emitted: [] as Array<{ + level: string; + background?: { task_name?: string }; + meta?: Record; + }>, + active: vi.fn(), +})); + +vi.mock("@/lib/logging/transports", () => ({ + emitIfSampled: (event: (typeof mocks.emitted)[number]) => { + mocks.emitted.push(event); + }, +})); +vi.mock("@/lib/jobs/job-failures", () => ({ + readActiveJobsStartedBefore: mocks.active, +})); + +import { + JOB_LONG_RUN_MS, + JOB_PROGRESS_INTERVAL_MS, + observeJob, + reportJobProgress, + reportJobsCutOffAtBoot, + type JobObserverClock, +} from "@/lib/jobs/job-observer"; + +/** A manual clock: `advance` fires due timers in order. */ +function manualClock() { + let now = 0; + let seq = 0; + const timers = new Map< + number, + { at: number; every: number | null; fn: () => void } + >(); + const clock: JobObserverClock = { + now: () => now, + setTimeout: (fn, ms) => { + seq += 1; + timers.set(seq, { at: now + ms, every: null, fn }); + return seq; + }, + setInterval: (fn, ms) => { + seq += 1; + timers.set(seq, { at: now + ms, every: ms, fn }); + return seq; + }, + clearTimeout: (handle) => timers.delete(handle as number), + clearInterval: (handle) => timers.delete(handle as number), + }; + const advance = (ms: number) => { + const target = now + ms; + for (;;) { + const due = [...timers.entries()] + .filter(([, t]) => t.at <= target) + .sort((a, b) => a[1].at - b[1].at)[0]; + if (!due) break; + const [id, timer] = due; + now = timer.at; + if (timer.every === null) timers.delete(id); + else timer.at += timer.every; + timer.fn(); + } + now = target; + }; + return { clock, advance }; +} + +function lines(task: string) { + return mocks.emitted.filter((e) => e.background?.task_name === task); +} + +beforeEach(() => { + mocks.emitted.length = 0; + mocks.active.mockReset(); +}); + +describe("observeJob", () => { + it("writes progress while a long job runs, a warning at its expiry, and a closing line", async () => { + const { clock, advance } = manualClock(); + let release!: (value: { ok: true; did: { days: number } }) => void; + + const running = observeJob( + "dense-intraday-retention", + [{ id: "job-1", expireInSeconds: 900 }], + () => + new Promise<{ ok: true; did: { days: number } }>((resolve) => { + reportJobProgress({ consolidation_day: "2024-03-01" }); + release = resolve; + }), + clock, + ); + + advance(JOB_PROGRESS_INTERVAL_MS); + expect(lines("job.progress")).toHaveLength(1); + expect(lines("job.progress")[0]!.meta).toMatchObject({ + queue: "dense-intraday-retention", + job_ids: "job-1", + expire_s: 900, + consolidation_day: "2024-03-01", + }); + expect(lines("job.progress")[0]!.meta?.heap_used_mb).toEqual( + expect.any(Number), + ); + + advance(900 * 1000 - JOB_PROGRESS_INTERVAL_MS); + const expired = lines("job.expired"); + expect(expired).toHaveLength(1); + expect(expired[0]!.level).toBe("warn"); + expect(expired[0]!.meta).toMatchObject({ + queue: "dense-intraday-retention", + elapsed_s: 900, + consolidation_day: "2024-03-01", + }); + + release({ ok: true, did: { days: 12 } }); + await running; + const finished = lines("job.finished"); + expect(finished).toHaveLength(1); + expect(finished[0]!.level).toBe("warn"); + expect(finished[0]!.meta).toMatchObject({ + queue: "dense-intraday-retention", + finished_after_expiry: true, + outcome: "completed", + did: { days: 12 }, + }); + }); + + it("stays silent for a short job", async () => { + const { clock } = manualClock(); + await observeJob( + "host-metric-sample", + [{ id: "j", expireInSeconds: 900 }], + async () => ({ ok: true }), + clock, + ); + expect(mocks.emitted).toEqual([]); + }); + + it("closes a long job that threw with the error, and rethrows it", async () => { + const { clock, advance } = manualClock(); + const running = observeJob( + "data-backup", + [{ id: "b", expireInSeconds: 7200 }], + async () => { + advance(JOB_LONG_RUN_MS); + throw new Error("write failed"); + }, + clock, + ); + await expect(running).rejects.toThrow("write failed"); + expect(lines("job.finished")[0]!.meta).toMatchObject({ + queue: "data-backup", + outcome: "threw", + error: "write failed", + }); + expect(lines("job.expired")).toEqual([]); + }); +}); + +describe("reportJobsCutOffAtBoot", () => { + it("names each job a previous process was running", async () => { + mocks.active.mockResolvedValue([ + { + queue: "data-backup", + id: "a", + startedAt: "2026-09-24T09:36:00.000Z", + expireSeconds: 7200, + }, + ]); + await reportJobsCutOffAtBoot(new Date("2026-09-24T10:00:00.000Z")); + const line = lines("worker.boot.jobs_cut_off"); + expect(line).toHaveLength(1); + expect(line[0]!.level).toBe("warn"); + expect(line[0]!.meta).toMatchObject({ + jobs_cut_off: 1, + queues: "data-backup", + }); + expect(String(line[0]!.meta?.jobs)).toContain( + "data-backup@2026-09-24T09:36:00.000Z", + ); + }); + + it("stays silent when nothing was cut off, or there is no queue schema", async () => { + mocks.active.mockResolvedValueOnce([]); + await reportJobsCutOffAtBoot(); + mocks.active.mockResolvedValueOnce(null); + await reportJobsCutOffAtBoot(); + expect(mocks.emitted).toEqual([]); + }); +}); diff --git a/src/lib/jobs/__tests__/run-job.test.ts b/src/lib/jobs/__tests__/run-job.test.ts index 4558ccea9..9b32e591f 100644 --- a/src/lib/jobs/__tests__/run-job.test.ts +++ b/src/lib/jobs/__tests__/run-job.test.ts @@ -149,3 +149,25 @@ describe("runJob", () => { expect(handler).toHaveBeenCalledWith(jobs); }); }); + +describe("runJob observation (#1031)", () => { + it("runs every handler inside the job observer, so a long or expired job leaves a log line", async () => { + const observer = await import("../job-observer"); + const observe = vi.spyOn(observer, "observeJob"); + const jobs = [ + { id: "j-1", expireInSeconds: 900 }, + ] as unknown as Job[]; + const wrapped = runJob("dense-intraday-retention", async () => + jobDone({ days_consolidated: 1 }), + ); + + await wrapped(jobs); + + expect(observe).toHaveBeenCalledWith( + "dense-intraday-retention", + jobs, + expect.any(Function), + ); + observe.mockRestore(); + }); +}); diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index c610a4825..257649e72 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -41,7 +41,10 @@ import { type ImportJobProgress, type ImportJobResult, } from "@/lib/measurements/import-apple-health-export"; -import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { + recomputeUserRollups, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; import { withBackgroundEvent } from "@/lib/logging/background"; /** @@ -502,19 +505,23 @@ export async function handleAppleHealthImport( // contradict the row the status poll surfaces to the operator. let rollupFailed = false; try { - const span = await prisma.measurement.aggregate({ - where: { userId }, - _min: { measuredAt: true }, - _max: { measuredAt: true }, - }); - if (span._min.measuredAt && span._max.measuredAt) { - // Add a small tail buffer to `to` so the upper bound is - // exclusive-safe under the rollup aggregator's `< to` filter. - const to = new Date(span._max.measuredAt.getTime() + 1); - await recomputeUserRollups(userId, { - from: span._min.measuredAt, - to, - }); + // Folded over what this import wrote, not over the account's whole + // history, and never further back than the fold window every other + // path keeps to: buckets older than `ROLLUP_FOLD_WINDOW_MS` are by + // design never written, so reads there fall back to live SQL. The old + // span ran from the account's first measurement, whatever the import + // carried, and folded any history older than the window as well. + const span = result.measuredSpan; + if (span) { + const windowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const spanFrom = new Date(span.from); + const from = spanFrom > windowStart ? spanFrom : windowStart; + // `to` is exclusive in the aggregator, so one millisecond past the + // newest row. + const to = new Date(new Date(span.to).getTime() + 1); + if (to > from) { + await recomputeUserRollups(userId, { from, to }); + } } } catch (rollupErr) { // Rollup failure is non-fatal — the next read falls through to diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild.ts b/src/lib/jobs/dense-intraday-hourly-rebuild.ts index 9f451d577..a90de1ea7 100644 --- a/src/lib/jobs/dense-intraday-hourly-rebuild.ts +++ b/src/lib/jobs/dense-intraday-hourly-rebuild.ts @@ -65,11 +65,14 @@ export interface DenseIntradayHourlyRebuildPayload { */ export async function runDenseIntradayHourlyRebuildForUser( userId: string, + shouldStop?: () => boolean, ): Promise<{ daysRebuilt: number; hourlyRowsUpserted: number; dailyRowsRetired: number; daysSkippedNoTombstones: number; + /** The job's budget ran out first; see the queue handler. */ + stoppedEarly: boolean; }> { // Shares the retention drain's kill-switch: when an operator disables the // dense tier, no-op so any already-queued backlog drains cleanly. @@ -79,10 +82,12 @@ export async function runDenseIntradayHourlyRebuildForUser( hourlyRowsUpserted: 0, dailyRowsRetired: 0, daysSkippedNoTombstones: 0, + stoppedEarly: false, }; } const summary = await runDenseIntradayHourlyRebuild(prisma, { userId, + shouldStop, log: () => { // Silent inside the queue handler — the worker logs the totals. }, @@ -104,6 +109,7 @@ export async function runDenseIntradayHourlyRebuildForUser( hourlyRowsUpserted: summary.totals.hourlyRowsUpserted, dailyRowsRetired: summary.totals.dailyRowsRetired, daysSkippedNoTombstones: summary.totals.daysSkippedNoTombstones, + stoppedEarly: summary.stoppedEarly, }; } diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index bd86101b7..30eb96b14 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -89,10 +89,13 @@ export interface DenseIntradayRetentionPayload { */ export async function runDenseIntradayRetentionForUser( userId: string, + shouldStop?: () => boolean, ): Promise<{ daysConsolidated: number; perSampleRowsSoftDeleted: number; derivedRestingRowsUpserted: number; + /** The job's budget ran out first; the next run continues from here. */ + stoppedEarly: boolean; }> { // Kill-switch: when an operator opts out, no-op so any already-queued // backlog completes cleanly (drains the queue) instead of running the fold. @@ -101,10 +104,12 @@ export async function runDenseIntradayRetentionForUser( daysConsolidated: 0, perSampleRowsSoftDeleted: 0, derivedRestingRowsUpserted: 0, + stoppedEarly: false, }; } const summary = await runDenseIntradayRetention(prisma, { userId, + shouldStop, log: () => { // Silent inside the queue handler — the worker logs the totals. }, @@ -121,6 +126,7 @@ export async function runDenseIntradayRetentionForUser( // days for proxy users, preserving the resting signal post-fold. derived_resting_rows_upserted: summary.totals.derivedRestingRowsUpserted, + stopped_early: summary.stoppedEarly, }, }, }); @@ -128,6 +134,7 @@ export async function runDenseIntradayRetentionForUser( daysConsolidated: summary.totals.daysConsolidated, perSampleRowsSoftDeleted: summary.totals.perSampleRowsSoftDeleted, derivedRestingRowsUpserted: summary.totals.derivedRestingRowsUpserted, + stoppedEarly: summary.stoppedEarly, }; } diff --git a/src/lib/jobs/drain-continuation.ts b/src/lib/jobs/drain-continuation.ts new file mode 100644 index 000000000..35947dfcc --- /dev/null +++ b/src/lib/jobs/drain-continuation.ts @@ -0,0 +1,33 @@ +/** + * When the nightly drain tick hands its remaining work to a follow-up job. + * + * The tick's three passes (cumulative drain, daily-mean consolidation, dense + * intra-day retention) stop at the job's time budget (`jobBudget`) instead of + * running into pg-boss's expiry. On a normal night that never happens. After + * a multi-year import it does, and without a follow-up the backlog would drain + * one budget per night. So a tick that stopped early sends itself again a + * minute later, and the chain is bounded twice: a follow-up is sent only when + * the run folded at least one day, so a pass that makes no progress cannot + * loop, and never more than `DRAIN_MAX_CONTINUATIONS` times in a row. + */ + +/** Follow-ups one nightly tick may chain: about four and a half hours of budget. */ +export const DRAIN_MAX_CONTINUATIONS = 24; + +/** Delay before a follow-up tick starts, in seconds. */ +export const DRAIN_CONTINUATION_DELAY_SECONDS = 60; + +/** + * The follow-up's continuation number, or `null` when no follow-up is due. + * `continuation` is the current job's own number (0 for the cron's job). + */ +export function nextDrainContinuation(input: { + stoppedEarly: boolean; + daysFolded: number; + continuation: number; +}): number | null { + if (!input.stoppedEarly) return null; + if (input.daysFolded <= 0) return null; + if (input.continuation >= DRAIN_MAX_CONTINUATIONS) return null; + return input.continuation + 1; +} diff --git a/src/lib/jobs/job-budget.ts b/src/lib/jobs/job-budget.ts new file mode 100644 index 000000000..8607004b1 --- /dev/null +++ b/src/lib/jobs/job-budget.ts @@ -0,0 +1,48 @@ +/** + * A time budget for a long pass, derived from the pg-boss job it runs under. + * + * pg-boss gives every job an expiry (`expireInSeconds`, 900 s unless the send + * says otherwise). When it passes, pg-boss marks the job failed with + * `job timed out`, retries it, and fires the job's abort signal. It does NOT + * stop the handler: a handler that never looks at the signal keeps writing + * after its job has been declared dead, and the retry then runs beside it over + * the same rows. On a large account that is how one pass that could not + * finish turned into two or three copies of itself contending for the same + * days. + * + * The passes that walk a whole history therefore stop on their own, between + * units of work, once a share of the expiry has gone by, and report that they + * stopped early. Their units commit one at a time and a finished unit drops + * out of the next scan, so the following run starts where this one stopped. + * The share leaves room for the unit in flight and for the handler's own + * bookkeeping before pg-boss's deadline. + */ +import type { Job } from "pg-boss"; + +/** Share of the job's expiry a pass may spend before it stops itself. */ +export const JOB_BUDGET_SHARE = 0.75; + +/** What the budget reads from a pg-boss job. */ +type BudgetedJob = Pick, "expireInSeconds" | "signal">; + +/** + * Build the `shouldStop` predicate for a batch of jobs handled together. + * + * True once the budget has elapsed (the tightest job's expiry decides) or once + * pg-boss has aborted any of the jobs. `now` is injectable for tests. + */ +export function jobBudget( + jobs: readonly BudgetedJob[], + now: () => number = Date.now, +): () => boolean { + const expirySeconds = Math.min( + ...jobs.map((job) => + Number.isFinite(job.expireInSeconds) && job.expireInSeconds > 0 + ? job.expireInSeconds + : Number.POSITIVE_INFINITY, + ), + ); + const deadline = now() + expirySeconds * 1000 * JOB_BUDGET_SHARE; + return () => + now() >= deadline || jobs.some((job) => job.signal?.aborted === true); +} diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index d846c88be..6406e320c 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -66,6 +66,35 @@ interface FailingQueueRow { failures: number; last_failed_at: Date | null; last_error: string | null; + last_expire_seconds: number | null; +} + +/** + * The two texts pg-boss writes when a job ran out of time: the supervisor's, + * for a job still `active` past its expiry, and the worker's own, for a + * handler that did not settle inside it. + */ +const TIMEOUT_MESSAGE = /^(job timed out|handler execution exceeded \d+s)$/; + +/** + * Say what a bare timeout means. "job timed out" alone sent an operator + * nowhere: pg-boss writes the same words for a pass that was too slow for its + * limit and for a job whose process died under it (a restart, or a container + * killed for memory), in which case the job sits `active` until the limit runs + * out. The limit and the two causes are what the operator can check. + */ +function explainTimeout(message: string, expireSeconds: number | null): string { + const limit = + expireSeconds !== null && expireSeconds > 0 + ? expireSeconds >= 3600 && expireSeconds % 3600 === 0 + ? `${expireSeconds / 3600} h` + : `${Math.round(expireSeconds / 60)} min` + : null; + return ( + `${message}: did not finish within its ${limit ?? "time"} limit. ` + + `Either the run was too slow for the limit, or the app restarted while it ` + + `ran; a restart shows in the container's restart count and log.` + ); } /** @@ -73,10 +102,15 @@ interface FailingQueueRow { * serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at * all, so the raw text is the fallback rather than an invented placeholder. */ -function presentError(raw: string | null): string { +function presentError( + raw: string | null, + expireSeconds: number | null = null, +): string { if (raw === null) return ""; const trimmed = raw.trim(); if (trimmed.length === 0) return ""; + if (TIMEOUT_MESSAGE.test(trimmed)) + return explainTimeout(trimmed, expireSeconds); const redacted = redactSecrets(trimmed); return redacted.length > MAX_ERROR_CHARS ? `${redacted.slice(0, MAX_ERROR_CHARS)}…` @@ -105,7 +139,10 @@ export async function readFailingQueues( ) ORDER BY completed_on DESC NULLS LAST ) - )[1] AS last_error + )[1] AS last_error, + ( + array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST) + )[1] AS last_expire_seconds FROM pgboss.job WHERE state = 'failed' AND completed_on > now() - make_interval(hours => ${withinHours}::int) @@ -119,7 +156,7 @@ export async function readFailingQueues( queue: row.name, failures: row.failures, lastFailedAt: row.last_failed_at!.toISOString(), - lastError: presentError(row.last_error), + lastError: presentError(row.last_error, row.last_expire_seconds), })); } catch { // No `pgboss` schema (web-only deployment) or no permission to read it. @@ -160,11 +197,17 @@ export async function readLastQueueRun( ): Promise { try { const rows = await prisma.$queryRaw< - Array<{ state: string; completed_on: Date | null; output: string | null }> + Array<{ + state: string; + completed_on: Date | null; + output: string | null; + expire_seconds: number | null; + }> >` SELECT state, completed_on, + expire_seconds, -- pg-boss wraps a thrown value: an expired job's row reads -- {"value":{"message":"job timed out"}}, so a plain ->>'message' -- misses it and the operator gets raw JSON instead of the sentence. @@ -189,7 +232,10 @@ export async function readLastQueueRun( return { at: row.completed_on.toISOString(), state, - error: state === "completed" ? null : presentError(row.output), + error: + state === "completed" + ? null + : presentError(row.output, row.expire_seconds), }; } catch { // No `pgboss` schema (web-only deployment) or no permission to read it. @@ -238,3 +284,58 @@ export async function readQueueFailureForUser( return null; } } + +/** A job pg-boss still has as running, from `readActiveJobsStartedBefore`. */ +export interface ActiveJob { + queue: string; + id: string; + /** ISO instant the job was picked up. */ + startedAt: string; + /** The job's expiry, in seconds, as pg-boss holds it. */ + expireSeconds: number; +} + +/** + * Jobs still marked active that were picked up before `before` — at worker + * boot, before this process has taken any job, that is every job a previous + * process was running when it stopped. + * + * Why it matters (#1031): a worker killed for memory writes nothing, and + * pg-boss only marks its job `job timed out` once the expiry has passed, + * which for the backup is two hours later. Reading these rows at boot turns + * a silent restart into a line naming what was cut off. With more than one + * worker process on the same database, a job another live worker is running + * would show up here too, which the boot line says. + * + * `null` when the queue schema is absent, like the readers above. + */ +export async function readActiveJobsStartedBefore( + before: Date, +): Promise { + try { + const rows = await prisma.$queryRaw< + Array<{ + name: string; + id: string; + started_on: Date | null; + expire_seconds: number; + }> + >` + SELECT name, id::text AS id, started_on, expire_seconds + FROM pgboss.job + WHERE state = 'active' AND started_on < ${before} + ORDER BY started_on + LIMIT ${MAX_QUEUES} + `; + return rows + .filter((row) => row.started_on !== null) + .map((row) => ({ + queue: row.name, + id: row.id, + startedAt: row.started_on!.toISOString(), + expireSeconds: row.expire_seconds, + })); + } catch { + return null; + } +} diff --git a/src/lib/jobs/job-observer.ts b/src/lib/jobs/job-observer.ts new file mode 100644 index 000000000..18c4166d6 --- /dev/null +++ b/src/lib/jobs/job-observer.ts @@ -0,0 +1,211 @@ +/** + * Log lines for long-running and cut-off jobs. + * + * Issue #1031: on a large account three queues failed night after night with + * `job timed out`, and the operator found not one line about any of them in + * the app log. Nothing wrote one. Most handlers annotate a wide event that + * nobody emits, pg-boss records an expiry only in its own table, and a worker + * killed for memory cannot say anything at all. The admin card showed the + * failure; the log showed nothing about what the job was doing, how far it + * got, or why it stopped. + * + * `observeJob` wraps every handler (through `runJob`) and writes three kinds + * of line, each a background wide event naming the queue: + * + * - `job.progress`, every two minutes while a handler runs, with how long it + * has been running, the process's memory, and whatever the handler last + * reported through `reportJobProgress`. A job that runs for a minute never + * writes one; a job that is killed leaves its last one behind. + * - `job.expired`, at the job's expiry if the handler has not settled: the + * moment pg-boss declares it timed out. pg-boss does not stop the handler, + * so this line says it is still running and how far it has got. + * - `job.finished`, when a handler that ran long or past its expiry + * settles, with its outcome and the facts it returned. + * + * `reportOrphanedJobs` covers the case no running handler can: at worker boot, + * jobs still marked active were started by a process that is gone. + */ +import { AsyncLocalStorage } from "node:async_hooks"; + +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { emitIfSampled } from "@/lib/logging/transports"; +import { readActiveJobsStartedBefore } from "@/lib/jobs/job-failures"; + +/** How often a running job writes `job.progress`. */ +export const JOB_PROGRESS_INTERVAL_MS = 2 * 60 * 1000; + +/** A handler that ran at least this long writes `job.finished`. */ +export const JOB_LONG_RUN_MS = 60 * 1000; + +type ProgressFacts = Record; + +const progressStore = new AsyncLocalStorage<{ facts: ProgressFacts }>(); + +/** + * Record how far the running job has got. Cheap: it only replaces the facts + * the next `job.progress` / `job.expired` line carries, so a pass can call it + * once per unit of work. A no-op outside an observed job. + */ +export function reportJobProgress(facts: ProgressFacts): void { + const cell = progressStore.getStore(); + if (cell) cell.facts = { ...cell.facts, ...facts }; +} + +/** What `observeJob` reads from a pg-boss job. */ +interface ObservedJob { + id?: string; + expireInSeconds?: number; +} + +/** Injectable clock and timers, for tests. */ +export interface JobObserverClock { + now: () => number; + setTimeout: (fn: () => void, ms: number) => unknown; + setInterval: (fn: () => void, ms: number) => unknown; + clearTimeout: (handle: unknown) => void; + clearInterval: (handle: unknown) => void; +} + +const systemClock: JobObserverClock = { + now: () => Date.now(), + setTimeout: (fn, ms) => setTimeout(fn, ms).unref(), + setInterval: (fn, ms) => setInterval(fn, ms).unref(), + clearTimeout: (handle) => clearTimeout(handle as NodeJS.Timeout), + clearInterval: (handle) => clearInterval(handle as NodeJS.Timeout), +}; + +function megabytes(bytes: number): number { + return Math.round(bytes / 1024 / 1024); +} + +function emitJobLine( + taskName: string, + level: "info" | "warn", + meta: Record, +): void { + try { + const event = new WideEventBuilder("background"); + event.setBackground({ task_name: taskName }); + for (const [key, value] of Object.entries(meta)) event.addMeta(key, value); + if (level === "warn") event.elevateLevel("warn"); + event.finish(); + emitIfSampled(event.toJSON()); + } catch { + // A log line must never break the job it describes. + } +} + +/** + * Run `run` for `jobs` of `queue`, writing the progress, expiry and finish + * lines described above. Resolves or rejects exactly as `run` does. + */ +export async function observeJob( + queue: string, + jobs: readonly ObservedJob[], + run: () => Promise, + clock: JobObserverClock = systemClock, +): Promise { + const startedAt = clock.now(); + const cell: { facts: ProgressFacts } = { facts: {} }; + const jobIds = jobs + .map((job) => job.id) + .filter(Boolean) + .join(","); + const expireSeconds = Math.min( + ...jobs.map((job) => + typeof job.expireInSeconds === "number" && job.expireInSeconds > 0 + ? job.expireInSeconds + : Number.POSITIVE_INFINITY, + ), + ); + let expired = false; + + const snapshot = () => { + const memory = process.memoryUsage(); + return { + queue, + job_ids: jobIds, + elapsed_s: Math.round((clock.now() - startedAt) / 1000), + ...(Number.isFinite(expireSeconds) ? { expire_s: expireSeconds } : {}), + heap_used_mb: megabytes(memory.heapUsed), + rss_mb: megabytes(memory.rss), + ...cell.facts, + }; + }; + + const heartbeat = clock.setInterval(() => { + emitJobLine("job.progress", "info", snapshot()); + }, JOB_PROGRESS_INTERVAL_MS); + const expiry = Number.isFinite(expireSeconds) + ? clock.setTimeout(() => { + expired = true; + emitJobLine("job.expired", "warn", { + ...snapshot(), + reason: + "pg-boss has marked this job timed out; the handler is still running", + }); + }, expireSeconds * 1000) + : null; + + const finish = (outcome: Record) => { + clock.clearInterval(heartbeat); + if (expiry !== null) clock.clearTimeout(expiry); + const elapsed = clock.now() - startedAt; + if (!expired && elapsed < JOB_LONG_RUN_MS) return; + emitJobLine("job.finished", expired ? "warn" : "info", { + ...snapshot(), + ...(expired ? { finished_after_expiry: true } : {}), + ...outcome, + }); + }; + + try { + const result = await progressStore.run(cell, run); + const did = + result !== null && typeof result === "object" && "did" in result + ? (result as { did?: unknown }).did + : undefined; + const ok = + result !== null && typeof result === "object" && "ok" in result + ? (result as { ok?: unknown }).ok !== false + : true; + finish({ + outcome: ok ? "completed" : "failed", + ...(did && typeof did === "object" ? { did } : {}), + }); + return result; + } catch (err) { + finish({ + outcome: "threw", + error: err instanceof Error ? err.message : String(err), + }); + throw err; + } +} + +/** + * At worker boot, write one line naming every job a previous process was + * running when it stopped. Must run before this process takes any job. + * + * A worker killed for memory cannot log its own death, and pg-boss marks the + * job `job timed out` only once the expiry has passed. This line is where a + * restart in the middle of a job becomes visible: the queue, when the job + * started, and its limit. Silent when nothing was cut off. + */ +export async function reportJobsCutOffAtBoot( + bootedAt: Date = new Date(), +): Promise { + const active = await readActiveJobsStartedBefore(bootedAt); + if (active === null || active.length === 0) return; + emitJobLine("worker.boot.jobs_cut_off", "warn", { + jobs_cut_off: active.length, + queues: [...new Set(active.map((job) => job.queue))].join(","), + jobs: active + .map( + (job) => `${job.queue}@${job.startedAt}(limit ${job.expireSeconds}s)`, + ) + .join(" "), + reason: + "these jobs were still running when the previous worker process stopped (a restart, a deploy, or the container running out of memory); pg-boss retries or fails them once their limit passes. If another worker process shares this database, its running jobs are listed too.", + }); +} diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 7d372364a..2f25991f4 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -55,6 +55,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "connections_deleted", "considered", "context_surfaced", + "continuation", + "continued", "created", "daily_rows_retired", "days_consolidated", @@ -136,6 +138,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "notifications_dispatched", "notifications_failed", "notified", + "offhost_backup_already_uploaded", "offhost_backup_configured", "offhost_backup_failed", "offhost_backup_oversized", @@ -200,6 +203,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "skipped_outside_window", "slots_collapsed", "states_deleted", + "stopped_early", "stored", "subscription_repair_failed", "summaries_enqueued", diff --git a/src/lib/jobs/mean-consolidation.ts b/src/lib/jobs/mean-consolidation.ts index a001dc2b9..bbe5ab219 100644 --- a/src/lib/jobs/mean-consolidation.ts +++ b/src/lib/jobs/mean-consolidation.ts @@ -44,9 +44,15 @@ export interface MeanConsolidationPayload { */ export async function runMeanConsolidationForUser( userId: string, -): Promise<{ daysConsolidated: number; perSampleRowsSoftDeleted: number }> { + shouldStop?: () => boolean, +): Promise<{ + daysConsolidated: number; + perSampleRowsSoftDeleted: number; + stoppedEarly: boolean; +}> { const summary = await consolidateDailyMean(prisma, { userId, + shouldStop, cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, log: () => { // Silent inside the queue handler — the worker logs the totals. @@ -59,12 +65,14 @@ export async function runMeanConsolidationForUser( days: summary.totals.daysConsolidated, per_sample_rows_soft_deleted: summary.totals.perSampleRowsSoftDeleted, daily_rows_upserted: summary.totals.dailyRowsUpserted, + stopped_early: summary.stoppedEarly, }, }, }); return { daysConsolidated: summary.totals.daysConsolidated, perSampleRowsSoftDeleted: summary.totals.perSampleRowsSoftDeleted, + stoppedEarly: summary.stoppedEarly, }; } diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 3627b1d2b..1805448bf 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -23,8 +23,8 @@ */ import { Buffer } from "node:buffer"; import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; -import { Transform, type Readable } from "node:stream"; -import { createGzip, gunzipSync, gzipSync } from "node:zlib"; +import { Readable, Transform } from "node:stream"; +import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; @@ -193,6 +193,48 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { : plaintext.toString("utf8"); } +/** + * An off-host object → a source of its JSON as byte chunks, openable as often + * as needed. `decryptBackup` answers one string, which the JSON of a large + * record cannot be: 662 MB for an account of 1.25 million measurements, past + * the longest string V8 allows (#1031). The tag is verified whole on the + * compressed ciphertext before a source exists, exactly as `decryptBackup` + * does; only the decompression is streamed. + */ +export function openBackupObject( + buf: Buffer, + key: Buffer, +): () => AsyncIterable { + const magic = buf.subarray(0, 4).toString("binary"); + const version = buf[PREAMBLE_LENGTH - 1]; + if ( + magic !== MAGIC || + (version !== BACKUP_ENVELOPE_PLAIN && + version !== BACKUP_ENVELOPE_GZIP && + version !== BACKUP_ENVELOPE_STREAM) + ) { + throw new Error("Invalid backup envelope (bad magic or version)"); + } + let plaintext: Buffer; + if (version === BACKUP_ENVELOPE_STREAM) { + plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key); + } else { + const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH); + const tag = buf.subarray( + PREAMBLE_LENGTH + IV_LENGTH, + PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH, + ); + const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH); + const dec = createDecipheriv(ALGORITHM, key, iv); + dec.setAuthTag(tag); + plaintext = Buffer.concat([dec.update(ct), dec.final()]); + } + if (version === BACKUP_ENVELOPE_PLAIN) { + return () => Readable.from([plaintext]); + } + return () => Readable.from([plaintext]).pipe(createGunzip()); +} + /** * How much of one object the upload holds at a time, and how many of those * windows are in flight. `@aws-sdk/lib-storage` buffers `partSize` bytes per @@ -551,9 +593,33 @@ interface BackupRunReport { largestObjectBytes: number; /** Accounts refused for size rather than failed for a reason. */ oversized: number; + /** + * Accounts an earlier attempt of the same run already uploaded, and this + * one therefore skipped (see `RunOffhostBackupOptions.runStartedAt`). + */ + alreadyUploaded: number; + /** `shouldStop` ended the run before every account was reached. */ + stoppedEarly: boolean; } -export type RunOffhostBackupOptions = UploadBackupOptions; +export interface RunOffhostBackupOptions extends UploadBackupOptions { + /** + * When this run first started: the pg-boss job's creation time, the same + * on every attempt of the job. An attempt skips each account whose ledger + * shows a successful upload since then, because an earlier attempt of the + * same run already put its object in the bucket, and the objects are keyed + * on this date so a retry after midnight still writes under the run's day. + * Without it a retry started again from the first account (#1031), and on + * a cohort with one large record the retry never got past the accounts the + * first attempt had already done. + */ + runStartedAt?: Date; + /** + * Asked before each account. Returning `true` ends the run cleanly between + * accounts, before the job's expiry would cut one off halfway. + */ + shouldStop?: () => boolean; +} export async function runOffhostBackup( prisma: PrismaClient, @@ -568,9 +634,23 @@ export async function runOffhostBackup( ); } const s3 = s3Override ?? (await getS3Client(cfg)); - const dateKey = now.toISOString().slice(0, 10); + const runStartedAt = options.runStartedAt; + const dateKey = (runStartedAt ?? now).toISOString().slice(0, 10); - const users = await prisma.user.findMany({ select: { id: true } }); + const users = await prisma.user.findMany({ + select: { id: true }, + orderBy: { id: "asc" }, + }); + const doneThisRun = new Set(); + if (runStartedAt) { + const done = await prisma.offhostBackupState.findMany({ + where: { lastSuccessAt: { gte: runStartedAt } }, + select: { userId: true }, + }); + for (const row of done) doneThisRun.add(row.userId); + } + let alreadyUploaded = 0; + let stoppedEarly = false; let uploaded = 0; let failed = 0; let oversized = 0; @@ -579,6 +659,14 @@ export async function runOffhostBackup( let ledgerWriteFailures = 0; const evt = getEvent(); for (const user of users) { + if (doneThisRun.has(user.id)) { + alreadyUploaded++; + continue; + } + if (options.shouldStop?.()) { + stoppedEarly = true; + break; + } let objectBytes: number | null = null; try { objectBytes = await uploadEncryptedBackup( @@ -679,6 +767,8 @@ export async function runOffhostBackup( totalUsers: users.length, largestObjectBytes, oversized, + alreadyUploaded, + stoppedEarly, }; } diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index f8a20a384..e3e732d2c 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -16,6 +16,7 @@ */ import { markWorkerStarted, recordError } from "@/lib/jobs/worker-status"; import { createWorkerBoss, setGlobalBoss } from "@/lib/jobs/boss-instance"; +import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer"; import { assertSubsystemEnabled } from "@/lib/process-type"; import { DATABASE_URL, workerLog } from "./reminder/shared"; import { @@ -55,7 +56,11 @@ export async function startReminderWorker() { recordError(); }); + // Read before any `boss.work` binding below can take a job: whatever is + // still active now was started by a process that is gone. + const bootedAt = new Date(); await boss.start(); + await reportJobsCutOffAtBoot(bootedAt); setGlobalBoss(boss); markWorkerStarted(); diff --git a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts index 7692ca02d..743eae2a6 100644 --- a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts +++ b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts @@ -4,9 +4,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ buildFullBackupPayload: vi.fn(), - packBlob: vi.fn((value: string) => value), getWorkerPrisma: vi.fn(), - upsert: vi.fn(), + store: vi.fn(), })); // Only the payload builder is stubbed. The REAL streaming writer runs on top @@ -19,27 +18,25 @@ vi.mock("@/lib/export/full-backup-payload", () => ({ isDeferredRows: () => false, })); -// The envelope is exercised end-to-end in -// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so -// the stored bytes can be read back as JSON. The size error is the real class -// so the handler's `instanceof` arm is the one that runs. -vi.mock("@/lib/export/backup-blob", async () => { - const actual = await vi.importActual< - typeof import("@/lib/export/backup-blob") - >("@/lib/export/backup-blob"); - return { - BackupBlobTooLargeError: actual.BackupBlobTooLargeError, - packBackupBlobStreaming: async ( - produce: (write: (chunk: string) => Promise) => Promise, - ) => { - let out = ""; - await produce(async (chunk) => { - out += chunk; - }); - return mocks.packBlob(out); - }, - }; -}); +// The envelope and the piecewise store are exercised end-to-end in +// `src/lib/export/__tests__/backup-blob.test.ts` and the integration suite; +// here the store stays transparent so the stored bytes can be read back as +// JSON. `mocks.store` receives what would have been stored and decides +// whether the write succeeds. +vi.mock("@/lib/export/store-backup-blob", () => ({ + storeBackupBlob: async ( + _prisma: unknown, + input: { userId: string; type: string }, + produce: (write: (chunk: string) => Promise) => Promise, + ) => { + let out = ""; + await produce(async (chunk) => { + out += chunk; + }); + await mocks.store(input, out); + return { id: "backup-row", bytes: out.length }; + }, +})); vi.mock("@/lib/logging/background", () => ({ withBackgroundEvent: vi.fn( @@ -70,7 +67,6 @@ function buildPrismaMock( ) { return { user: { findMany: vi.fn().mockResolvedValue(users) }, - dataBackup: { upsert: mocks.upsert }, }; } @@ -78,7 +74,7 @@ describe("handleDataBackup canonical DR payload", () => { beforeEach(() => { vi.clearAllMocks(); mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock()); - mocks.upsert.mockResolvedValue({}); + mocks.store.mockResolvedValue(undefined); mocks.buildFullBackupPayload.mockResolvedValue({ payload: { schemaVersion: "1", @@ -117,8 +113,12 @@ describe("handleDataBackup canonical DR payload", () => { deferBulk: true, }), ); - expect(mocks.upsert).toHaveBeenCalledOnce(); - const encrypted = mocks.upsert.mock.calls[0]![0].create.data as string; + expect(mocks.store).toHaveBeenCalledOnce(); + expect(mocks.store.mock.calls[0]![0]).toEqual({ + userId: "user-dr", + type: "WEEKLY_AUTO", + }); + const encrypted = mocks.store.mock.calls[0]![1] as string; const payload = JSON.parse(encrypted) as { moodEntries: Array<{ id: string; @@ -155,7 +155,7 @@ describe("handleDataBackup canonical DR payload", () => { describe("handleDataBackup outcome", () => { beforeEach(() => { vi.clearAllMocks(); - mocks.upsert.mockResolvedValue({}); + mocks.store.mockResolvedValue(undefined); mocks.buildFullBackupPayload.mockResolvedValue({ payload: { schemaVersion: "1", userId: "user-dr" }, counts: {}, @@ -164,7 +164,7 @@ describe("handleDataBackup outcome", () => { it("fails the run when not one account got a copy", async () => { mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock()); - mocks.upsert.mockRejectedValue(new Error("write failed")); + mocks.store.mockRejectedValue(new Error("write failed")); const outcome = await handleDataBackup([]); @@ -178,7 +178,7 @@ describe("handleDataBackup outcome", () => { it("counts an oversized record as the reason it could not", async () => { mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock()); - mocks.upsert.mockRejectedValue(new BackupBlobTooLargeError(9_000, 4_096)); + mocks.store.mockRejectedValue(new BackupBlobTooLargeError(9_000, 4_096)); const outcome = await handleDataBackup([]); @@ -198,9 +198,9 @@ describe("handleDataBackup outcome", () => { { id: "user-b", username: "b" }, ]), ); - mocks.upsert + mocks.store .mockRejectedValueOnce(new Error("write failed")) - .mockResolvedValueOnce({}); + .mockResolvedValueOnce(undefined); const outcome = await handleDataBackup([]); diff --git a/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts b/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts index 41bfb935b..b1d880103 100644 --- a/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts +++ b/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts @@ -53,6 +53,8 @@ function report(over: Partial> = {}) { totalUsers: 2, largestObjectBytes: 9_000, oversized: 0, + alreadyUploaded: 0, + stoppedEarly: false, ...over, }; } @@ -67,6 +69,8 @@ describe("handleOffhostBackup", () => { const outcome = await handleOffhostBackup([]); expect(outcome.ok).toBe(true); expect(outcome.ok && outcome.did).toEqual({ + offhost_backup_already_uploaded: 0, + stopped_early: false, offhost_backup_uploaded: 2, offhost_backup_failed: 0, offhost_backup_total_users: 2, @@ -138,4 +142,23 @@ describe("handleOffhostBackup", () => { offhost_backup_configured: false, }); }); + + it("fails an attempt that ran out of budget, so pg-boss retries and the retry resumes (#1031)", async () => { + mocks.runOffhostBackup.mockResolvedValue( + report({ uploaded: 1, totalUsers: 3, stoppedEarly: true }), + ); + const outcome = await handleOffhostBackup([]); + expect(outcome).toMatchObject({ + ok: false, + reason: expect.stringContaining("retry resumes"), + }); + }); + + it("passes a retry whose accounts an earlier attempt had all uploaded", async () => { + mocks.runOffhostBackup.mockResolvedValue( + report({ uploaded: 0, alreadyUploaded: 2, totalUsers: 2 }), + ); + const outcome = await handleOffhostBackup([]); + expect(outcome.ok).toBe(true); + }); }); diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 48c6ae397..0a3b69ac5 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -4,12 +4,11 @@ * Extracted from reminder-worker.ts, which owns the queue names, cron * schedules, and boss.work registrations. */ -import { type Job } from "pg-boss"; +import { type Job, type JobWithMetadata } from "pg-boss"; +import { jobBudget } from "@/lib/jobs/job-budget"; import { recordError } from "@/lib/jobs/worker-status"; -import { - BackupBlobTooLargeError, - packBackupBlobStreaming, -} from "@/lib/export/backup-blob"; +import { BackupBlobTooLargeError } from "@/lib/export/backup-blob"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -18,6 +17,7 @@ import { runOffhostBackup, } from "@/lib/jobs/offhost-backup"; import { getWorkerPrisma } from "./shared"; +import { reportJobProgress } from "@/lib/jobs/job-observer"; export interface DataBackupPayload { triggeredAt: string; @@ -28,13 +28,18 @@ export interface OffhostBackupPayload { } export async function handleOffhostBackup( - jobs: Job[], + jobs: JobWithMetadata[], ): Promise { - void jobs; return withBackgroundEvent("job.offhost_backup", async (evt) => { const p = getWorkerPrisma(); try { - const report = await runOffhostBackup(p); + // A retry of this job carries the same creation time, so it resumes + // after the accounts an earlier attempt uploaded, and the run stops + // between accounts before the job's expiry would cut one off. + const report = await runOffhostBackup(p, undefined, new Date(), { + runStartedAt: jobs[0]?.createdOn, + shouldStop: jobBudget(jobs), + }); evt.addMeta("offhost_backup_uploaded", report.uploaded); evt.addMeta("offhost_backup_failed", report.failed); evt.addMeta("offhost_backup_total_users", report.totalUsers); @@ -56,6 +61,8 @@ export async function handleOffhostBackup( } const did = { + offhost_backup_already_uploaded: report.alreadyUploaded, + stopped_early: report.stoppedEarly, offhost_backup_uploaded: report.uploaded, offhost_backup_failed: report.failed, offhost_backup_total_users: report.totalUsers, @@ -70,7 +77,20 @@ export async function handleOffhostBackup( // than one. Per-account failures still ride out as counts when SOME // account got a copy: that is the fan-out rule, and retrying the whole // cohort over one object would re-upload everybody's. - if (report.totalUsers > 0 && report.uploaded === 0) { + // Out of budget before every account: fail, so pg-boss retries the job, + // and the retry resumes after the accounts this attempt uploaded. + if (report.stoppedEarly) { + return jobFailed( + "stopped before every account was uploaded; the retry resumes", + undefined, + did, + ); + } + + if ( + report.totalUsers > 0 && + report.uploaded + report.alreadyUploaded === 0 + ) { // The SDK's own words, not a stack: `runJob` puts the cause message in // the reported meta, and "SignatureDoesNotMatch" is the sentence an // operator can act on. @@ -118,38 +138,30 @@ export async function handleDataBackup( // six weeks in. let lastError: unknown; for (const user of users) { + reportJobProgress({ + backup_user: user.id, + backup_users_done: backed + usersFailed, + backup_users_total: users.length, + }); try { // Streamed, compressed, then encrypted (the record contains - // sensitive health information). Nothing between the database rows - // and this string exists as a whole: the payload goes into gzip a - // page at a time and the cipher consumes gzip's output as it comes, - // because a materialised copy of a large record is what used to take - // the process down. Both legs live in `packBackupBlobStreaming`, - // which is also what the restore path reads back through. - const encryptedBackup = await packBackupBlobStreaming((write) => - streamFullBackupJson(prisma, user.id, write, { - purpose: "disaster-recovery", - }), + // sensitive health information), and stored a piece at a time. + // Nothing between the database rows and the stored row exists as a + // whole in this process: the payload goes into gzip a page at a + // time, the cipher consumes gzip's output as it comes, and the + // ciphertext goes back to Postgres in pieces that one statement + // assembles into the row (`storeBackupBlob`). The stored copy used + // to be built here as one string first, which on a large record + // added several hundred megabytes to a container capped at 1 GB. + const { bytes: storedBytes } = await storeBackupBlob( + prisma, + { userId: user.id, type: "WEEKLY_AUTO" }, + (write) => + streamFullBackupJson(prisma, user.id, write, { + purpose: "disaster-recovery", + }), ); - largestBlobBytes = Math.max( - largestBlobBytes, - Buffer.byteLength(encryptedBackup, "utf8"), - ); - - await prisma.dataBackup.upsert({ - where: { - userId_type: { userId: user.id, type: "WEEKLY_AUTO" }, - }, - update: { - data: encryptedBackup, - createdAt: new Date(), - }, - create: { - userId: user.id, - type: "WEEKLY_AUTO", - data: encryptedBackup, - }, - }); + largestBlobBytes = Math.max(largestBlobBytes, storedBytes); backed++; } catch (err) { // One user's payload failing is that user's problem, not the pass's: diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 4d60ab352..30507c800 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -724,10 +724,11 @@ export async function registerMaintenanceQueues( { localConcurrency: 1 }, handleStepUpElevationCleanup, ); + // With metadata: the handler resumes a retry from the job's creation time. await createAndWork( boss, OFFHOST_BACKUP_QUEUE, - { localConcurrency: 1 }, + { localConcurrency: 1, includeMetadata: true }, handleOffhostBackup, ); await createAndWork( diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts index eca1bf4ff..ef38e1ddd 100644 --- a/src/lib/jobs/reminder/register-rollup.ts +++ b/src/lib/jobs/reminder/register-rollup.ts @@ -110,6 +110,11 @@ import { type ScheduleEntry, } from "./registrar-shared"; import { jobDone, jobFailed } from "@/lib/jobs/job-outcome"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { + DRAIN_CONTINUATION_DELAY_SECONDS, + nextDrainContinuation, +} from "@/lib/jobs/drain-continuation"; // v1.4.37 W7c — nightly drain of per-sample APPLE_HEALTH cumulative rows. // Collapses each user × cumulative-type × calendar-day bucket into one // `stats:…` row so the list view stops painting hundreds of step chunks per @@ -123,7 +128,12 @@ const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative"; const DRAIN_CUMULATIVE_CRON = "45 3 * * *"; interface DrainCumulativePayload { - triggeredAt: string; + triggeredAt?: string; + /** + * How many times this tick has already handed itself on. Absent on the + * cron's own job. See `nextDrainContinuation`. + */ + continuation?: number; } const allQueues = [ @@ -437,12 +447,15 @@ export async function registerRollupQueues( MEAN_CONSOLIDATION_QUEUE, { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, async (jobs) => { + const shouldStop = jobBudget(jobs); let daysTotal = 0; let perSampleRowsTotal = 0; + let stoppedEarly = false; for (const job of jobs) { const { userId } = job.data; - const { daysConsolidated, perSampleRowsSoftDeleted } = - await runMeanConsolidationForUser(userId); + const summary = await runMeanConsolidationForUser(userId, shouldStop); + const { daysConsolidated, perSampleRowsSoftDeleted } = summary; + stoppedEarly ||= summary.stoppedEarly; daysTotal += daysConsolidated; perSampleRowsTotal += perSampleRowsSoftDeleted; workerLog( @@ -450,10 +463,14 @@ export async function registerRollupQueues( `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, ); } + // Out of budget is a completed run, not a failed one: the folded days + // are committed, and the nightly tick carries on from the first day + // this run did not reach. return jobDone({ users: jobs.length, days_consolidated: daysTotal, per_sample_rows_soft_deleted: perSampleRowsTotal, + stopped_early: stoppedEarly, }); }, ); @@ -470,17 +487,24 @@ export async function registerRollupQueues( DENSE_INTRADAY_RETENTION_QUEUE, { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, async (jobs) => { + const shouldStop = jobBudget(jobs); let daysTotal = 0; let perSampleRowsTotal = 0; let derivedRestingRowsTotal = 0; + let stoppedEarly = false; for (const job of jobs) { const { userId } = job.data; try { + const summary = await runDenseIntradayRetentionForUser( + userId, + shouldStop, + ); const { daysConsolidated, perSampleRowsSoftDeleted, derivedRestingRowsUpserted, - } = await runDenseIntradayRetentionForUser(userId); + } = summary; + stoppedEarly ||= summary.stoppedEarly; daysTotal += daysConsolidated; perSampleRowsTotal += perSampleRowsSoftDeleted; derivedRestingRowsTotal += derivedRestingRowsUpserted; @@ -503,6 +527,10 @@ export async function registerRollupQueues( days_consolidated: daysTotal, per_sample_rows_soft_deleted: perSampleRowsTotal, derived_resting_rows_upserted: derivedRestingRowsTotal, + // Out of budget completes the job: the folded days are committed and + // the nightly tick, which hands itself on while work remains, + // continues from the first day this run did not reach. + stopped_early: stoppedEarly, }); }, ); @@ -519,6 +547,8 @@ export async function registerRollupQueues( DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, async (jobs) => { + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; let daysRebuiltTotal = 0; let hourlyRowsTotal = 0; let dailyRowsRetiredTotal = 0; @@ -526,12 +556,17 @@ export async function registerRollupQueues( for (const job of jobs) { const { userId } = job.data; try { + const summary = await runDenseIntradayHourlyRebuildForUser( + userId, + shouldStop, + ); const { daysRebuilt, hourlyRowsUpserted, dailyRowsRetired, daysSkippedNoTombstones, - } = await runDenseIntradayHourlyRebuildForUser(userId); + } = summary; + stoppedEarly ||= summary.stoppedEarly; daysRebuiltTotal += daysRebuilt; hourlyRowsTotal += hourlyRowsUpserted; dailyRowsRetiredTotal += dailyRowsRetired; @@ -550,13 +585,25 @@ export async function registerRollupQueues( throw err; } } - return jobDone({ + const did = { users: jobs.length, + stopped_early: stoppedEarly, days_rebuilt: daysRebuiltTotal, hourly_rows_upserted: hourlyRowsTotal, daily_rows_retired: dailyRowsRetiredTotal, days_skipped_no_tombstones: daysSkippedTotal, - }); + }; + // Out of budget: fail so pg-boss retries (the boot send allows three + // retries), and the retry starts at the first day not yet rebuilt. The + // queue is `exclusive`, so the job cannot hand itself on while active. + if (stoppedEarly) { + return jobFailed( + "stopped before every day was rebuilt; the retry resumes", + undefined, + did, + ); + } + return jobDone(did); }, ); @@ -679,6 +726,9 @@ export async function registerRollupQueues( DRAIN_CUMULATIVE_QUEUE, { localConcurrency: 1 }, async (jobs) => { + // One budget shared by all three passes, run in turn. + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; // The three drains below are independent passes sharing one nightly // tick, so one failing pass must not stop the other two from running. // The first failure is held back and reported once the tick is over — @@ -699,8 +749,10 @@ export async function registerRollupQueues( const summary = await drainPerSampleCumulative(getWorkerPrisma(), { dryRun: false, cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, + shouldStop, log: (line) => workerLog("info", line), }); + stoppedEarly ||= summary.stoppedEarly; bucketsCollapsed += summary.totals.bucketsCollapsed; perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; workerLog( @@ -729,8 +781,10 @@ export async function registerRollupQueues( const meanSummary = await consolidateDailyMean(getWorkerPrisma(), { dryRun: false, cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, + shouldStop, log: (line) => workerLog("info", line), }); + stoppedEarly ||= meanSummary.stoppedEarly; meanDaysConsolidated += meanSummary.totals.daysConsolidated; meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; workerLog( @@ -769,9 +823,11 @@ export async function registerRollupQueues( getWorkerPrisma(), { dryRun: false, + shouldStop, log: (line) => workerLog("info", line), }, ); + stoppedEarly ||= denseSummary.stoppedEarly; denseDaysConsolidated += denseSummary.totals.daysConsolidated; denseRowsSoftDeleted += denseSummary.totals.perSampleRowsSoftDeleted; @@ -790,7 +846,41 @@ export async function registerRollupQueues( } } + // Hand the remaining work to a follow-up job when this one ran out of + // budget (bounded; see `nextDrainContinuation`). + const continuation = Math.max( + 0, + ...jobs.map((job) => job.data.continuation ?? 0), + ); + const next = nextDrainContinuation({ + stoppedEarly, + daysFolded: + bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated, + continuation, + }); + let continued = false; + if (next !== null) { + try { + const followUp: DrainCumulativePayload = { + triggeredAt: new Date().toISOString(), + continuation: next, + }; + continued = + (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, { + ...cronIsTheRetry, + startAfter: DRAIN_CONTINUATION_DELAY_SECONDS, + })) !== null; + } catch (err) { + // The next nightly tick picks the work up regardless; a lost + // follow-up only delays it. + workerLog("error", "[drain-cumulative] follow-up send failed", err); + } + } + const did = { + stopped_early: stoppedEarly, + continuation, + continued, passes_failed: passesFailed, buckets_collapsed: bucketsCollapsed, per_sample_rows_deleted: perSampleRowsDeleted, diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 36056c56f..50450c630 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -22,8 +22,9 @@ * no other cron uses. */ import type { Job } from "pg-boss"; +import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; import { - decryptBackup, + openBackupObject, getS3Client, loadOffhostConfig, OffhostBackupNotConfiguredError, @@ -103,19 +104,36 @@ export async function runRestoreDrill( const dateKey = BACKUP_KEY_PATTERN.exec(objectKey)![1]; const ciphertext = await s3.getObject(objectKey); - const plaintext = decryptBackup(ciphertext, cfg.encryptionKey); - const parsed: unknown = JSON.parse(plaintext); - - if (typeof parsed !== "object" || parsed === null) { - throw new Error( - `Restore drill: backup object "${objectKey}" decrypted and parsed but is not a JSON object.`, - ); + // Read as a stream, never as one string: the JSON of a large record is + // longer than any string V8 can hold (#1031). The bulk tables are counted, + // not kept. + const source = openBackupObject(ciphertext, cfg.encryptionKey); + let plaintextBytes = 0; + async function* counted() { + for await (const chunk of source()) { + plaintextBytes += chunk.byteLength; + yield chunk; + } } - const payload = parsed as Record; + let scanned; + try { + scanned = await scanBackupJson(counted(), { + streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), + }); + } catch (err) { + if (err instanceof BackupJsonError) { + throw new Error( + `Restore drill: backup object "${objectKey}" decrypted but is not a JSON object: ${err.message}`, + ); + } + throw err; + } + const payload = scanned.document; + const streamedCounts = scanned.streamedCounts; if ( typeof payload.exportedAt !== "string" || typeof payload.userId !== "string" || - !Array.isArray(payload.measurements) + streamedCounts.measurements === undefined ) { throw new Error( `Restore drill: backup object "${objectKey}" parses but is missing core fields (exportedAt / userId / measurements).`, @@ -133,12 +151,12 @@ export async function runRestoreDrill( ageDays, stale: ageDays > MAX_BACKUP_AGE_DAYS, ciphertextBytes: ciphertext.length, - plaintextBytes: Buffer.byteLength(plaintext, "utf8"), + plaintextBytes, recordCounts: { - measurements: payload.measurements.length, + measurements: streamedCounts.measurements, medications: countArray(payload.medications), - intakeEvents: countArray(payload.intakeEvents), - moodEntries: countArray(payload.moodEntries), + intakeEvents: streamedCounts.intakeEvents ?? 0, + moodEntries: streamedCounts.moodEntries ?? 0, }, }; } diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts index 980bc5a1d..9a673bc8c 100644 --- a/src/lib/jobs/run-job.ts +++ b/src/lib/jobs/run-job.ts @@ -28,6 +28,7 @@ import { type SerializedJobOutcome, } from "./job-outcome"; import { reportWorkerError } from "./report-worker-error"; +import { observeJob } from "./job-observer"; /** A converted handler: it says what it did instead of just finishing. */ export type JobHandler = (jobs: Job[]) => Promise; @@ -80,7 +81,13 @@ export function runJob( handler: (jobs: J[]) => Promise, ): (jobs: J[]) => Promise { return async (jobs: J[]): Promise => { - const outcome = await handler(jobs); + // Observed, so a long run, an expiry and a cut-off leave a line in the + // log naming the queue and how far it got (see `job-observer.ts`). + const outcome = await observeJob( + queue, + jobs as ReadonlyArray<{ id?: string; expireInSeconds?: number }>, + () => handler(jobs), + ); if (outcome.ok) return serializeJobOutcome(outcome); const failure = new JobFailure(queue, outcome.reason, outcome.cause); diff --git a/src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts b/src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts new file mode 100644 index 000000000..a0396d9f1 --- /dev/null +++ b/src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts @@ -0,0 +1,331 @@ +/** + * `adoptOrMintHourlyRows` resolves a whole folded day in one lookup and one + * insert. It replaced a per-slot loop (two lookups and a write per hour, about + * 75 statements a day), which was most of the 380 000 statements the dense + * retention spent on a 1.25-million-row account (issue #1031). + * + * Two things are pinned here. The statement shape: one read, at most one + * insert, no per-slot lookups. And the semantics: against an in-memory table + * that enforces both unique identities, the batch leaves exactly the rows the + * per-slot loop leaves and returns the same ids, over randomised days that + * include rows already sitting on an anchor, rows already carrying a target + * externalId, tombstones, and a row holding one hour's externalId while + * parked on another hour's anchor. + */ +import { describe, expect, it, vi } from "vitest"; + +import type { MeasurementType, Prisma } from "@/generated/prisma/client"; + +import { + adoptOrMintHourlyRows, + type HourlySlot, +} from "../dense-intraday-retention"; + +interface Row { + id: string; + externalId: string | null; + measuredAt: Date; + value: number; + unit: string; + deletedAt: Date | null; +} + +/** + * A measurement table for one (user, type, APPLE_HEALTH), with the two unique + * identities the real one has: externalId and measuredAt (sleepStage is NULL + * for these types). A write that would duplicate either throws, as P2002 does. + */ +function fakeTable(seed: Row[]) { + const rows = seed.map((row) => ({ ...row })); + let minted = 0; + const calls = { findMany: 0, findFirst: 0, createMany: 0, create: 0 }; + + const assertUnique = () => { + const ext = rows + .filter((r) => r.externalId !== null) + .map((r) => r.externalId); + const at = rows.map((r) => r.measuredAt.getTime()); + if (new Set(ext).size !== ext.length) throw new Error("P2002 externalId"); + if (new Set(at).size !== at.length) throw new Error("P2002 measuredAt"); + }; + const mint = (data: { + externalId: string; + measuredAt: Date; + value: number; + unit: string; + }) => { + minted += 1; + const row: Row = { + id: `z-minted-${String(minted).padStart(3, "0")}`, + externalId: data.externalId, + measuredAt: data.measuredAt, + value: data.value, + unit: data.unit, + deletedAt: null, + }; + rows.push(row); + assertUnique(); + return row; + }; + const byId = () => [...rows].sort((a, b) => (a.id < b.id ? -1 : 1)); + + const tx = { + measurement: { + findMany: vi.fn( + async (args: { + where: { + OR: [ + { externalId: { in: string[] } }, + { measuredAt: { in: Date[] } }, + ]; + }; + }) => { + calls.findMany += 1; + const ext = new Set(args.where.OR[0].externalId.in); + const at = new Set( + args.where.OR[1].measuredAt.in.map((d) => d.getTime()), + ); + return byId() + .filter( + (r) => + (r.externalId !== null && ext.has(r.externalId)) || + at.has(r.measuredAt.getTime()), + ) + .map((r) => ({ + id: r.id, + externalId: r.externalId, + measuredAt: r.measuredAt, + })); + }, + ), + findFirst: vi.fn( + async (args: { where: { externalId?: string; measuredAt?: Date } }) => { + calls.findFirst += 1; + const hit = byId().find((r) => + args.where.externalId !== undefined + ? r.externalId === args.where.externalId + : r.measuredAt.getTime() === args.where.measuredAt!.getTime(), + ); + return hit ? { id: hit.id } : null; + }, + ), + update: vi.fn( + async (args: { + where: { id: string }; + data: Partial & { measuredAt?: Date }; + }) => { + const row = rows.find((r) => r.id === args.where.id)!; + Object.assign(row, args.data); + assertUnique(); + return row; + }, + ), + create: vi.fn( + async (args: { + data: { + externalId: string; + measuredAt: Date; + value: number; + unit: string; + }; + }) => { + calls.create += 1; + return { id: mint(args.data).id }; + }, + ), + createManyAndReturn: vi.fn( + async (args: { + data: Array<{ + externalId: string; + measuredAt: Date; + value: number; + unit: string; + }>; + }) => { + calls.createMany += 1; + return args.data.map((d) => { + const row = mint(d); + return { id: row.id, externalId: row.externalId }; + }); + }, + ), + }, + }; + return { + tx: tx as unknown as Prisma.TransactionClient, + rows, + calls, + }; +} + +/** + * The per-slot loop the batch replaced, kept verbatim in behaviour as the + * reference: externalId first, then the anchor occupant, lowest id on a tie; + * re-anchor only onto a free slot; otherwise mint. + */ +async function perSlotReference( + tx: Prisma.TransactionClient, + unit: string, + slots: readonly HourlySlot[], +): Promise { + const ids: string[] = []; + for (const slot of slots) { + const eidRow = await tx.measurement.findFirst({ + where: { externalId: slot.externalId }, + } as Prisma.MeasurementFindFirstArgs); + const slotRow = await tx.measurement.findFirst({ + where: { measuredAt: slot.anchor }, + } as Prisma.MeasurementFindFirstArgs); + const target = eidRow ?? slotRow; + if (target) { + const free = slotRow === null || slotRow.id === target.id; + await tx.measurement.update({ + where: { id: target.id }, + data: { + value: slot.value, + unit, + externalId: slot.externalId, + deletedAt: null, + ...(free ? { measuredAt: slot.anchor } : {}), + }, + }); + ids.push(target.id); + } else { + const created = await tx.measurement.create({ + data: { + value: slot.value, + unit, + measuredAt: slot.anchor, + externalId: slot.externalId, + }, + } as Prisma.MeasurementCreateArgs); + ids.push(created.id); + } + } + return ids; +} + +const HOUR = 3_600_000; +const DAY0 = Date.UTC(2026, 4, 1, 0, 30); +const anchor = (h: number) => new Date(DAY0 + h * HOUR); +const ext = (h: number) => `stats:HK:2026-05-01T${String(h).padStart(2, "0")}`; + +/** A deterministic pseudo-random generator, so a failure reproduces. */ +function rng(seed: number) { + let x = seed; + return () => { + x = (x * 1103515245 + 12345) % 2 ** 31; + return x / 2 ** 31; + }; +} + +function scenario(seed: number): { seed: Row[]; slots: HourlySlot[] } { + const rand = rng(seed); + const hours = [...Array(24).keys()].filter(() => rand() < 0.7); + const slots = hours.map((h) => ({ + externalId: ext(h), + anchor: anchor(h), + value: Math.round(rand() * 100), + })); + const rows: Row[] = []; + const takenAt = new Set(); + const takenExt = new Set(); + let n = 0; + const add = (externalId: string | null, at: Date) => { + if (takenAt.has(at.getTime())) return; + if (externalId !== null && takenExt.has(externalId)) return; + takenAt.add(at.getTime()); + if (externalId !== null) takenExt.add(externalId); + n += 1; + rows.push({ + id: `r-${String(Math.floor(rand() * 900) + 100)}-${n}`, + externalId, + measuredAt: at, + value: 1, + unit: "bpm", + deletedAt: rand() < 0.3 ? new Date(DAY0) : null, + }); + }; + for (const h of hours) { + const roll = rand(); + if (roll < 0.15) + add(ext(h), anchor(h)); // a prior fold's own row + else if (roll < 0.3) + add(`hk-sample-${h}`, anchor(h)); // a sample on the anchor + else if (roll < 0.4) + add(ext(h), new Date(DAY0 + h * HOUR + 60_000)); // stats row off-anchor + else if (roll < 0.5) { + // This hour's externalId parked on ANOTHER hour's anchor. + const other = hours[Math.floor(rand() * hours.length)]!; + add(ext(h), anchor(other)); + } + } + return { seed: rows, slots }; +} + +function snapshot(rows: Row[]) { + return [...rows] + .map((r) => ({ + id: r.id.startsWith("z-minted") ? "minted" : r.id, + externalId: r.externalId, + at: r.measuredAt.toISOString(), + value: r.value, + deleted: r.deletedAt !== null, + })) + .sort((a, b) => + `${a.externalId}|${a.at}` < `${b.externalId}|${b.at}` ? -1 : 1, + ); +} + +describe("adoptOrMintHourlyRows", () => { + it("resolves a fresh day with one lookup and one insert, no per-slot reads", async () => { + const { tx, calls, rows } = fakeTable([]); + const slots = [...Array(24).keys()].map((h) => ({ + externalId: ext(h), + anchor: anchor(h), + value: h, + })); + + const ids = await adoptOrMintHourlyRows(tx, { + userId: "u", + type: "PULSE" as MeasurementType, + unit: "bpm", + slots, + }); + + expect(calls).toEqual({ + findMany: 1, + findFirst: 0, + createMany: 1, + create: 0, + }); + expect(rows).toHaveLength(24); + expect(ids).toEqual( + slots.map((s) => rows.find((r) => r.externalId === s.externalId)!.id), + ); + }); + + it("leaves the same rows and returns the same ids as the per-slot loop", async () => { + for (let seed = 1; seed <= 300; seed++) { + const { seed: rows, slots } = scenario(seed); + + const batch = fakeTable(rows); + const batchIds = await adoptOrMintHourlyRows(batch.tx, { + userId: "u", + type: "PULSE" as MeasurementType, + unit: "bpm", + slots, + }); + + const reference = fakeTable(rows); + const referenceIds = await perSlotReference(reference.tx, "bpm", slots); + + const norm = (ids: string[]) => + ids.map((id) => (id.startsWith("z-minted") ? "minted" : id)); + expect(norm(batchIds), `seed ${seed}`).toEqual(norm(referenceIds)); + expect(snapshot(batch.rows), `seed ${seed}`).toEqual( + snapshot(reference.rows), + ); + } + }); +}); diff --git a/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts b/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts index ea8bf0563..1add7ea76 100644 --- a/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts +++ b/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts @@ -1,31 +1,50 @@ import { describe, expect, it, vi } from "vitest"; -import { bucketRowsByDay, scanSourceRowsPaged } from "../consolidation-base"; +import { + bucketRowsByDay, + iterateDayBuckets, + iterateSourcePages, + runConsolidation, + type ConsolidationParams, +} from "../consolidation-base"; import type { PerSampleRow } from "../consolidation-tz"; -import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import type { + MeasurementType, + Prisma, + PrismaClient, +} from "@/generated/prisma/client"; -// `scanSourceRowsPaged` keyset-paginates the per-(user, type) source scan so a -// multi-year tenant no longer materialises its whole history in one Prisma -// result set. These tests pin the two properties that matter: the accumulated -// array equals a single unbounded scan (same rows, same order, no gaps / -// duplicates), and the cursor advances on `(measuredAt, id)` after every full -// page. +// The consolidation drains walk a user's whole history of a type. Issue #1031: +// on an account with 1.25 million rows the walk gathered every page into one +// array before folding a single day (≈230 MB of heap for the heart-rate type +// alone) and paged with a cursor Postgres could not use as an index condition, +// so every page re-read the rows before it. These tests pin the three +// properties the fix rests on: pages are handed on one at a time and a day is +// folded before the scan reads past it, the cursor carries an index-usable +// bound, and a pass stops cleanly when asked to. -/** Build a deterministic per-sample row at a 1-minute-per-index offset. */ -function row(index: number): PerSampleRow { +/** A deterministic per-sample row at `index` hours past 2026-01-01T00:00Z. */ +function row(index: number, minuteOffset = 0): PerSampleRow { return { id: `m-${String(index).padStart(4, "0")}`, type: "ACTIVITY_STEPS", value: index, - measuredAt: new Date(Date.UTC(2026, 0, 1, 0, index, 0)), + measuredAt: new Date(Date.UTC(2026, 0, 1, index, minuteOffset, 0)), externalId: `hk-uuid-${index}`, }; } +interface CursorWhere { + AND?: [ + Prisma.MeasurementWhereInput, + { measuredAt?: { gte?: Date } }, + { OR?: Array<{ measuredAt?: Date | { gt?: Date }; id?: { gt?: string } }> }, + ]; +} + /** - * A findMany mock that serves `allRows` in keyset order, honouring `take` - * (page size) and the `(measuredAt, id)` cursor clause the helper AND-combines - * onto the base where. This mirrors how Postgres would resolve the keyset. + * A findMany mock that serves `allRows` in keyset order, honouring `take` and + * the `(measuredAt, id)` cursor the pager AND-combines onto the base where. */ function buildPagingFindMany(allRows: PerSampleRow[]) { const ordered = [...allRows].sort((a, b) => { @@ -34,43 +53,29 @@ function buildPagingFindMany(allRows: PerSampleRow[]) { }); return vi.fn( - async (args: { - where: Prisma.MeasurementWhereInput; - take?: number; - orderBy?: unknown; - }) => { + async (args: { where: CursorWhere; take?: number; orderBy?: unknown }) => { const take = args.take ?? ordered.length; - - // Extract the keyset cursor the helper threads through `where.AND[1].OR`. - let afterTime: number | null = null; - let afterId: string | null = null; - const and = (args.where as { AND?: unknown[] }).AND; - if (Array.isArray(and)) { - const or = (and[1] as { OR?: Array> }).OR; - const gt = or?.[0]?.measuredAt as { gt?: Date } | undefined; - const tie = or?.[1] as - { measuredAt?: Date; id?: { gt?: string } } | undefined; - afterTime = gt?.gt ? gt.gt.getTime() : null; - afterId = tie?.id?.gt ?? null; - } - - const remaining = - afterTime === null - ? ordered - : ordered.filter((r) => { - const t = r.measuredAt.getTime(); - return t > afterTime! || (t === afterTime! && r.id > afterId!); - }); - - return remaining.slice(0, take); + const and = args.where.AND; + if (!and) return ordered.slice(0, take); + const gte = and[1].measuredAt?.gte?.getTime(); + const or = and[2].OR ?? []; + const gt = ( + or[0]?.measuredAt as { gt?: Date } | undefined + )?.gt?.getTime(); + const tieId = or[1]?.id?.gt; + return ordered + .filter((r) => { + const t = r.measuredAt.getTime(); + if (gte !== undefined && t < gte) return false; + return t > gt! || (t === gt! && r.id > tieId!); + }) + .slice(0, take); }, ); } function mockClient(findMany: ReturnType): PrismaClient { - return { - measurement: { findMany }, - } as unknown as PrismaClient; + return { measurement: { findMany } } as unknown as PrismaClient; } const BASE_WHERE: Prisma.MeasurementWhereInput = { @@ -85,21 +90,25 @@ const SELECT: Prisma.MeasurementSelect = { externalId: true, }; -describe("scanSourceRowsPaged", () => { - it("accumulates every row in keyset order across multiple pages with no gaps or duplicates", async () => { +async function collectPages( + pages: AsyncIterable, +): Promise { + const out: PerSampleRow[][] = []; + for await (const page of pages) out.push(page); + return out; +} + +describe("iterateSourcePages", () => { + it("yields every row once, in keyset order, one bounded page at a time", async () => { const all = Array.from({ length: 23 }, (_, i) => row(i)); const findMany = buildPagingFindMany(all); - const result = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 10, + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10), ); - // 23 rows at page size 10 → pages of 10, 10, 3 (the short page ends it). - expect(result.map((r) => r.id)).toEqual(all.map((r) => r.id)); - expect(new Set(result.map((r) => r.id)).size).toBe(all.length); + expect(pages.map((p) => p.length)).toEqual([10, 10, 3]); + expect(pages.flat().map((r) => r.id)).toEqual(all.map((r) => r.id)); expect(findMany).toHaveBeenCalledTimes(3); }); @@ -107,88 +116,191 @@ describe("scanSourceRowsPaged", () => { const all = Array.from({ length: 20 }, (_, i) => row(i)); const findMany = buildPagingFindMany(all); - const result = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 10, + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10), ); - // 20 rows at page size 10 → full pages 10, 10, then an empty probe page. - expect(result).toHaveLength(20); + expect(pages.flat()).toHaveLength(20); expect(findMany).toHaveBeenCalledTimes(3); }); - it("advances the cursor on (measuredAt, id) after each full page", async () => { + it("yields nothing when the first page is empty", async () => { + const findMany = buildPagingFindMany([]); + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10), + ); + expect(pages).toEqual([]); + expect(findMany).toHaveBeenCalledTimes(1); + }); + + it("bounds every later page with an index-usable measuredAt >= cursor conjunct", async () => { + // `measuredAt > c OR (measuredAt = c AND id > i)` alone is not an index + // condition: Postgres walked the index from the user's first row on + // every page. The redundant `>=` bound is what the planner can use. const all = Array.from({ length: 12 }, (_, i) => row(i)); const findMany = buildPagingFindMany(all); - await scanSourceRowsPaged(mockClient(findMany), BASE_WHERE, SELECT, 5); + await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 5), + ); - // First page: no cursor (base where only). - const firstWhere = findMany.mock.calls[0]?.[0]?.where; - expect((firstWhere as { AND?: unknown }).AND).toBeUndefined(); - expect(firstWhere).toEqual(BASE_WHERE); - - // Second page: cursor resumes after the 5th row (id m-0004). - const secondWhere = findMany.mock.calls[1]?.[0]?.where as { - AND: unknown[]; - }; - expect(Array.isArray(secondWhere.AND)).toBe(true); - expect(secondWhere.AND[0]).toEqual(BASE_WHERE); - const cursorClause = secondWhere.AND[1] as { - OR: Array<{ measuredAt?: { gt?: Date }; id?: { gt?: string } }>; - }; - expect(cursorClause.OR[0].measuredAt?.gt).toEqual(row(4).measuredAt); - expect(cursorClause.OR[1].id?.gt).toBe(row(4).id); - - // Every page is requested in stable keyset order. + expect(findMany.mock.calls[0]?.[0]?.where).toEqual(BASE_WHERE); + const second = findMany.mock.calls[1]?.[0]?.where as CursorWhere; + expect(second.AND?.[0]).toEqual(BASE_WHERE); + expect(second.AND?.[1]).toEqual({ + measuredAt: { gte: row(4).measuredAt }, + }); + expect(second.AND?.[2]).toEqual({ + OR: [ + { measuredAt: { gt: row(4).measuredAt } }, + { measuredAt: row(4).measuredAt, id: { gt: row(4).id } }, + ], + }); for (const call of findMany.mock.calls) { expect(call[0]?.orderBy).toEqual([{ measuredAt: "asc" }, { id: "asc" }]); } }); - it("returns the empty array when the first page is empty", async () => { - const findMany = buildPagingFindMany([]); - const result = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 10, - ); - expect(result).toEqual([]); - expect(findMany).toHaveBeenCalledTimes(1); - }); - - it("produces output that buckets identically to a single unpaged scan", async () => { - // Rows spanning three calendar days; the paged accumulation must bucket - // byte-identically to the same rows fed in one array. - const all = [ - { ...row(1), measuredAt: new Date("2026-05-16T08:00:00.000Z") }, - { ...row(2), measuredAt: new Date("2026-05-16T14:00:00.000Z") }, - { ...row(3), measuredAt: new Date("2026-05-17T06:00:00.000Z") }, - { ...row(4), measuredAt: new Date("2026-05-17T20:00:00.000Z") }, - { ...row(5), measuredAt: new Date("2026-05-18T09:00:00.000Z") }, - ]; + it("resumes past rows that share the boundary instant without gaps or duplicates", async () => { + // Five rows on the same instant straddle the page boundary. + const all = [0, 1, 2, 3, 4].map((i) => ({ + ...row(i), + measuredAt: new Date("2026-05-16T08:00:00.000Z"), + })); const findMany = buildPagingFindMany(all); - - const paged = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 2, + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2), ); + expect(pages.flat().map((r) => r.id)).toEqual(all.map((r) => r.id)); + }); +}); - const baselineByDay = bucketRowsByDay(all, "Europe/Berlin", "stats:"); - const pagedByDay = bucketRowsByDay(paged, "Europe/Berlin", "stats:"); +describe("iterateDayBuckets", () => { + const days = [ + { ...row(1), measuredAt: new Date("2026-05-16T08:00:00.000Z") }, + { ...row(2), measuredAt: new Date("2026-05-16T14:00:00.000Z") }, + { + ...row(3), + externalId: "stats:HKQuantityTypeIdentifierStepCount:2026-05-16", + measuredAt: new Date("2026-05-16T15:00:00.000Z"), + }, + { ...row(4), measuredAt: new Date("2026-05-17T06:00:00.000Z") }, + { ...row(5), measuredAt: new Date("2026-05-17T20:00:00.000Z") }, + { ...row(6), measuredAt: new Date("2026-05-18T09:00:00.000Z") }, + ]; - expect([...pagedByDay.keys()].sort()).toEqual( - [...baselineByDay.keys()].sort(), - ); - for (const [dateKey, rows] of baselineByDay) { - expect(pagedByDay.get(dateKey)?.map((r) => r.id)).toEqual( + it("groups exactly as the whole-array bucketing does, stats rows skipped", async () => { + const findMany = buildPagingFindMany(days); + let scanned = 0; + const streamed = new Map(); + for await (const [key, rows] of iterateDayBuckets( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2), + "Europe/Berlin", + "stats:", + () => { + scanned += 1; + }, + )) { + streamed.set( + key, rows.map((r) => r.id), ); } + + const baseline = bucketRowsByDay(days, "Europe/Berlin", "stats:"); + expect([...streamed.keys()]).toEqual([...baseline.keys()]); + for (const [key, rows] of baseline) { + expect(streamed.get(key)).toEqual(rows.map((r) => r.id)); + } + expect(scanned).toBe(days.length); + }); + + it("hands a day on before the scan reads the page after the one that closed it", async () => { + const findMany = buildPagingFindMany(days); + const pagesReadWhenYielded: number[] = []; + for await (const _day of iterateDayBuckets( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2), + "Europe/Berlin", + "stats:", + )) { + pagesReadWhenYielded.push(findMany.mock.calls.length); + } + // Pages: [r1 r2] [r3 r4] [r5 r6] [] — 05-16 closes on page 2 (r4), + // 05-17 on page 3 (r6), 05-18 at the end of the scan. + expect(pagesReadWhenYielded).toEqual([2, 3, 4]); + }); +}); + +describe("runConsolidation — streamed walk", () => { + /** One row per hour for `dayCount` UTC days. */ + function hourlyRows(dayCount: number): PerSampleRow[] { + return Array.from({ length: dayCount * 24 }, (_, i) => row(i, 30)); + } + + function params( + findMany: ReturnType, + writeDay: ConsolidationParams["writeDay"], + shouldStop?: () => boolean, + ): ConsolidationParams { + const client = { + user: { + findMany: vi + .fn() + .mockResolvedValue([{ id: "user-1", timezone: "UTC" }]), + }, + measurement: { findMany }, + } as unknown as PrismaClient; + return { + prismaClient: client, + options: { shouldStop }, + types: ["ACTIVITY_STEPS"] as MeasurementType[], + hkIdentifierForType: () => "HKQuantityTypeIdentifierStepCount", + dailyStatsExternalId: (hk: string, day: string) => `stats:${hk}:${day}`, + statsPrefix: "stats:", + reduce: (rows: readonly PerSampleRow[]) => rows.length, + buildScanWhere: () => BASE_WHERE, + writeDay, + recordBucket: () => {}, + }; + } + + it("folds the first day before the scan has read the whole history", async () => { + // 500 days × 24 rows = 12 000 rows → default pages of 5000, 5000, 2000. + const all = hourlyRows(500); + const findMany = buildPagingFindMany(all); + const pagesReadAtFirstWrite: number[] = []; + const writeDay = vi.fn(async () => { + if (pagesReadAtFirstWrite.length === 0) { + pagesReadAtFirstWrite.push(findMany.mock.calls.length); + } + return { kind: "written" as const, sourceRowsRemoved: 0 }; + }); + + const result = await runConsolidation(params(findMany, writeDay)); + + expect(writeDay).toHaveBeenCalledTimes(500); + expect(findMany).toHaveBeenCalledTimes(3); + // The old walk read all three pages before the first write. + expect(pagesReadAtFirstWrite).toEqual([1]); + expect(result.stoppedEarly).toBe(false); + }); + + it("stops before the next day once shouldStop answers true, and reads no further", async () => { + const all = hourlyRows(500); + const findMany = buildPagingFindMany(all); + const writeDay = vi.fn(async () => ({ + kind: "written" as const, + sourceRowsRemoved: 0, + })); + let budget = 3; + const shouldStop = () => budget-- <= 0; + + const result = await runConsolidation( + params(findMany, writeDay, shouldStop), + ); + + expect(writeDay).toHaveBeenCalledTimes(3); + expect(result.stoppedEarly).toBe(true); + expect(findMany).toHaveBeenCalledTimes(1); }); }); diff --git a/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts b/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts index de394991c..9a149a2f4 100644 --- a/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts @@ -30,6 +30,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({ import { runDenseIntradayHourlyRebuild } from "../dense-intraday-hourly-rebuild"; import type { PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; const HRV_DAILY_ID = "stats:HKQuantityTypeIdentifierHeartRateVariabilitySDNN:2026-05-01"; @@ -79,6 +80,8 @@ function buildPrismaMock(opts: { create: txCreate, update: txUpdate, findFirst: txFindFirst, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(txCreate), }, }; @@ -444,6 +447,10 @@ describe("runDenseIntradayHourlyRebuild — rebuild flow", () => { create: vi.fn().mockResolvedValue({ id: "minted" }), update: vi.fn().mockResolvedValue({}), findFirst: vi.fn().mockResolvedValue(null), + findMany: candidateLookup(), + createManyAndReturn: createManyVia( + vi.fn().mockResolvedValue({ id: "minted" }), + ), }, }); }, @@ -456,4 +463,44 @@ describe("runDenseIntradayHourlyRebuild — rebuild flow", () => { expect(summary.totals.daysFailed).toBe(1); expect(summary.totals.daysRebuilt).toBe(1); }); + + it("stops before the next day once its budget is spent (#1031)", async () => { + const { mock, $transaction } = buildPrismaMock({ + dailyByType: { + HEART_RATE_VARIABILITY: [ + { id: "daily-1", externalId: HRV_DAILY_ID }, + { + id: "daily-2", + externalId: + "stats:HKQuantityTypeIdentifierHeartRateVariabilitySDNN:2026-05-02", + }, + ], + }, + tombstonesByType: { + HEART_RATE_VARIABILITY: [ + tombstone( + "a", + 40, + "2026-05-01T08:00:00.000Z", + "HEART_RATE_VARIABILITY", + ), + tombstone( + "b", + 44, + "2026-05-02T08:00:00.000Z", + "HEART_RATE_VARIABILITY", + ), + ], + }, + }); + let asked = 0; + const summary = await runDenseIntradayHourlyRebuild(mock, { + log: () => {}, + shouldStop: () => asked++ >= 1, + }); + + expect(summary.stoppedEarly).toBe(true); + expect(summary.totals.daysRebuilt).toBe(1); + expect($transaction).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts b/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts index 6d7973204..76f03037d 100644 --- a/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts @@ -33,6 +33,7 @@ import { } from "../dense-intraday-retention"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; function pulseRow(id: string, value: number, iso: string): PerSampleRow { return { @@ -68,6 +69,8 @@ function buildPrismaMock(opts: { update: txUpdate, findFirst: txFindFirst, updateMany: txUpdateMany, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(txCreate), }, }; @@ -325,6 +328,10 @@ describe("PULSE fold — per-day failure boundary", () => { update: vi.fn(), findFirst: vi.fn().mockResolvedValue(null), updateMany: vi.fn().mockResolvedValue({ count: 5 }), + findMany: candidateLookup(), + createManyAndReturn: createManyVia( + vi.fn().mockResolvedValue({ id: "minted" }), + ), }, }); }, diff --git a/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts b/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts index 92d88f07b..fb0579290 100644 --- a/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts @@ -32,6 +32,7 @@ import { DENSE_INTRADAY_RETENTION_DAYS, } from "../dense-intraday-retention"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; /** A day well outside the retention window, so the fold definitely runs. */ const FOLD_DAY = new Date( @@ -76,7 +77,15 @@ function buildPrismaMock(existingResting: Array<{ source: string }>) { ); const tx = { - measurement: { create, update, findFirst: txFindFirst, updateMany, upsert }, + measurement: { + create, + update, + findFirst: txFindFirst, + updateMany, + upsert, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(create), + }, }; return { diff --git a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts index 9f10409db..4ea0b1250 100644 --- a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts @@ -25,6 +25,7 @@ import { } from "../dense-intraday-retention"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; function row( id: string, @@ -56,7 +57,16 @@ function buildPrismaMock( async (args: { where: { type: string } }) => rowsByType[args.where.type] ?? [], ); - const tx = { measurement: { create, update, findFirst, updateMany } }; + const tx = { + measurement: { + create, + update, + findFirst, + updateMany, + findMany: candidateLookup(existingCanonicalId), + createManyAndReturn: createManyVia(create), + }, + }; return { mock: { user: { diff --git a/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts b/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts index cf19ba322..a61d7a1d8 100644 --- a/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts @@ -29,6 +29,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({ import { runDenseIntradayRetention } from "../dense-intraday-retention"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; function spo2Row(id: string, value: number, iso: string): PerSampleRow { return { @@ -60,6 +61,8 @@ function buildPrismaMock(opts: { spo2Rows: PerSampleRow[] }) { update: txUpdate, findFirst: txFindFirst, updateMany: txUpdateMany, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(txCreate), }, }; @@ -254,6 +257,10 @@ describe("SpO2 fold — per-day failure boundary", () => { update: vi.fn(), findFirst: vi.fn().mockResolvedValue(null), updateMany: vi.fn().mockResolvedValue({ count: 2 }), + findMany: candidateLookup(), + createManyAndReturn: createManyVia( + vi.fn().mockResolvedValue({ id: "minted" }), + ), }, }); }, diff --git a/src/lib/measurements/__tests__/hourly-mint-mock.ts b/src/lib/measurements/__tests__/hourly-mint-mock.ts new file mode 100644 index 000000000..a5131cee9 --- /dev/null +++ b/src/lib/measurements/__tests__/hourly-mint-mock.ts @@ -0,0 +1,49 @@ +/** + * Transaction-client stubs for `adoptOrMintHourlyRows`, shared by the mocked + * dense-tier fold tests. + * + * The hourly mint reads the day's candidate rows with one `findMany` and + * writes its mints with one `createManyAndReturn`. The tests were written + * against the per-slot loop and assert on individual `create` calls, so the + * batch insert here is routed through the test's own `create` spy, one call + * per minted row: the assertions keep reading what was minted, in slot order. + */ +import { vi } from "vitest"; + +type CreateSpy = (args: { + data: { externalId: string }; +}) => Promise<{ id: string }>; + +/** `createManyAndReturn` that forwards every row to `create`. */ +export function createManyVia(create: CreateSpy) { + return vi.fn(async (args: { data: Array<{ externalId: string }> }) => { + const out: Array<{ id: string; externalId: string }> = []; + for (const data of args.data) { + const row = await create({ data }); + out.push({ id: row.id, externalId: data.externalId }); + } + return out; + }); +} + +/** + * The day's candidate lookup. With `existingId`, one row already occupies the + * first slot's anchor instant (the adopt-in-place case); otherwise nothing is + * in the way. + */ +export function candidateLookup(existingId: string | null = null) { + return vi.fn( + async (args: { + where: { OR: [unknown, { measuredAt: { in: Date[] } }] }; + }) => + existingId === null + ? [] + : [ + { + id: existingId, + externalId: null, + measuredAt: args.where.OR[1].measuredAt.in[0]!, + }, + ], + ); +} diff --git a/src/lib/measurements/consolidate-daily-mean.ts b/src/lib/measurements/consolidate-daily-mean.ts index 097fed61a..dd8c376e0 100644 --- a/src/lib/measurements/consolidate-daily-mean.ts +++ b/src/lib/measurements/consolidate-daily-mean.ts @@ -93,6 +93,12 @@ export interface MeanConsolidationBucket { export interface MeanConsolidationSummary { dryRun: boolean; + /** + * `shouldStop` ended the walk before every day was reached. Nothing is + * lost: folded days committed one by one, and the next run starts at the + * first day this one did not reach. + */ + stoppedEarly: boolean; buckets: MeanConsolidationBucket[]; totals: { usersScanned: number; @@ -123,6 +129,11 @@ export interface MeanConsolidationOptions { * `undefined` to drain everything. */ cutoffHours?: number; + /** + * Ends the pass cleanly before the next day when it returns `true`; the + * summary then reports `stoppedEarly`. See `ConsolidationOptions`. + */ + shouldStop?: () => boolean; } /** Arithmetic mean of a per-day bucket. Exposed for unit testing. */ @@ -205,6 +216,7 @@ export async function consolidateDailyMean( const summary: MeanConsolidationSummary = { dryRun: options.dryRun ?? false, + stoppedEarly: false, buckets: [], totals: { usersScanned: 0, @@ -215,7 +227,7 @@ export async function consolidateDailyMean( }, }; - const { usersScanned, daysFailed } = await runConsolidation({ + const walk = await runConsolidation({ prismaClient, options, types: HIGH_FREQUENCY_MEAN_TYPES, @@ -420,8 +432,9 @@ export async function consolidateDailyMean( }, }); - summary.totals.usersScanned = usersScanned; - summary.totals.daysFailed = daysFailed; + summary.totals.usersScanned = walk.usersScanned; + summary.stoppedEarly = walk.stoppedEarly; + summary.totals.daysFailed = walk.daysFailed; log( `[mean-consolidation] done — usersScanned=${summary.totals.usersScanned} daysConsolidated=${summary.totals.daysConsolidated} perSampleRowsSoftDeleted=${summary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted} daysFailed=${summary.totals.daysFailed}${options.dryRun ? " (dry-run)" : ""}`, diff --git a/src/lib/measurements/consolidation-base.ts b/src/lib/measurements/consolidation-base.ts index a94dce1b2..382f2335e 100644 --- a/src/lib/measurements/consolidation-base.ts +++ b/src/lib/measurements/consolidation-base.ts @@ -32,6 +32,8 @@ import type { Prisma, } from "@/generated/prisma/client"; +import { reportJobProgress } from "@/lib/jobs/job-observer"; + import { CONSOLIDATION_GRACE_CUTOFF_HOURS, canonicalDailyTimestamp, @@ -57,6 +59,18 @@ export interface ConsolidationOptions { dryRun?: boolean; /** Logger sink — defaults to `console.log`. */ log?: (line: string) => void; + /** + * Asked before every day bucket. Returning `true` ends the pass cleanly + * after the day in flight has committed, and the result reports + * `stoppedEarly`. The queue handlers wire it to the job's time budget and + * to pg-boss's abort signal, so a pass that cannot finish inside its expiry + * stops on its own instead of being declared dead while it keeps writing. + * + * Stopping loses nothing: every day commits on its own, and a folded day's + * source rows leave the scan predicate, so the next run starts at the first + * day this one did not reach. + */ + shouldStop?: () => boolean; } /** @@ -229,7 +243,7 @@ export interface ConsolidationParams { dryRun: boolean; }) => void; /** - * Fired once per (user, type) immediately after the rows are bucketed, + * Fired once per (user, type) after the type's rows have been walked, * with the raw scanned-row count and per-day bucket count. Only invoked * for types that yielded at least one source row. Lets a single-type * drain (legacy steps) reproduce its scan-time log line. Optional. @@ -275,35 +289,40 @@ const DEFAULT_SCAN_SELECT: Prisma.MeasurementSelect = { }; /** - * Page size for the keyset-paginated source-row scan. Heavy multi-year - * tenants previously materialised the entire per-(user, type) history in a - * single unbounded `findMany` result set — the dominant Prisma intermediate - * cost and the root of the consolidation boot-storm on large accounts. - * Scanning in bounded keyset pages caps the per-query result set while still - * accumulating the same full row set. + * Page size for the keyset-paginated source-row scan. */ const CONSOLIDATION_SCAN_PAGE_SIZE = 5000; /** * Keyset-paginate the per-(user, type) source-row scan on `(measuredAt, id)` - * ascending, accumulating every page into one array. Output is identical to a - * single unbounded `findMany` — same `where`, same `select`, same ascending - * order — but the Prisma client never holds more than one page of rows at a - * time, so a multi-year tenant no longer materialises its whole history in one - * result set at worker boot. + * ascending, one page at a time. * - * `(measuredAt, id)` is a stable keyset: `measuredAt` is the primary order and - * `id` breaks ties, so each page resumes exactly after the last row with no - * gaps or duplicates. The cursor clause is AND-combined with the caller's - * `baseWhere` so the scope / soft-delete / grace filters always still apply. + * Two properties matter, and both used to be missing. + * + * The pages are yielded, not accumulated. The previous helper gathered every + * page into one array before a single day was folded, so the pass held a + * type's whole history at once: on an account with 766 000 heart-rate samples + * that is about 230 MB of live heap for the array alone, most of a 524 MB + * limit, and nothing was written until the whole scan had finished. + * + * The cursor is index-usable. `measuredAt > c OR (measuredAt = c AND id > i)` + * cannot be an index condition, so Postgres walked the index from the user's + * first row on every page and filtered its way to the cursor, which makes the + * whole scan quadratic (measured: 47 ms and 248 000 rows filtered for one + * page halfway through). The redundant `measuredAt >= c` conjunct is the + * index condition; the OR only breaks ties at the boundary instant. + * + * The caller's `baseWhere` is AND-combined, so the scope / soft-delete / + * grace filters always still apply. */ -export async function scanSourceRowsPaged( +export async function* iterateSourcePages< + Row extends { id: string; measuredAt: Date } = PerSampleRow, +>( prismaClient: PrismaClient, baseWhere: Prisma.MeasurementWhereInput, scanSelect: Prisma.MeasurementSelect, pageSize: number = CONSOLIDATION_SCAN_PAGE_SIZE, -): Promise { - const accumulated: PerSampleRow[] = []; +): AsyncGenerator { let cursor: { measuredAt: Date; id: string } | null = null; for (;;) { @@ -311,6 +330,7 @@ export async function scanSourceRowsPaged( ? { AND: [ baseWhere, + { measuredAt: { gte: cursor.measuredAt } }, { OR: [ { measuredAt: { gt: cursor.measuredAt } }, @@ -326,20 +346,63 @@ export async function scanSourceRowsPaged( select: scanSelect, orderBy: [{ measuredAt: "asc" }, { id: "asc" }], take: pageSize, - })) as PerSampleRow[]; + })) as unknown as Row[]; - if (page.length === 0) break; - - for (const row of page) accumulated.push(row); - - // A short page is the last page — no further rows can satisfy the keyset. - if (page.length < pageSize) break; + if (page.length === 0) return; const last = page[page.length - 1]!; + const isLastPage = page.length < pageSize; + yield page; + + // A short page is the last page — no further rows can satisfy the keyset. + if (isLastPage) return; cursor = { measuredAt: last.measuredAt, id: last.id }; } +} - return accumulated; +/** + * Group a stream of ascending pages into complete per-day buckets, in the + * user's timezone, without holding more than one day at a time. + * + * The rows arrive in `measuredAt` order and a local calendar day is one + * contiguous span of instants, so a day is complete the moment a row from a + * later day shows up. Rows whose externalId starts with `statsPrefix` (the + * daily-stats shape) are skipped, the same as `bucketRowsByDay`; pass `null` + * to keep every row. `onRow` sees every scanned row, skipped ones included, + * for the scan-count log line. + */ +export async function* iterateDayBuckets< + Row extends { measuredAt: Date; externalId: string | null } = PerSampleRow, +>( + pages: AsyncIterable, + tz: string, + statsPrefix: string | null, + onRow?: () => void, +): AsyncGenerator<[dateKey: string, rows: Row[]], void, void> { + let currentKey: string | null = null; + let current: Row[] = []; + for await (const page of pages) { + for (const row of page) { + onRow?.(); + if ( + statsPrefix !== null && + row.externalId !== null && + row.externalId.startsWith(statsPrefix) + ) { + continue; + } + const key = dayKeyForUserTz(row.measuredAt, tz); + if (key !== currentKey) { + if (currentKey !== null && current.length > 0) { + yield [currentKey, current]; + } + currentKey = key; + current = []; + } + current.push(row); + } + } + if (currentKey !== null && current.length > 0) yield [currentKey, current]; } /** @@ -348,7 +411,8 @@ export async function scanSourceRowsPaged( * and delegates the per-day mint + delete to `writeDay`. Returns the * number of users scanned (so the caller can seed its summary totals) * plus the number of day buckets absorbed by `onBucketError` (0 when the - * boundary is not supplied). + * boundary is not supplied), and whether `options.shouldStop` ended the + * walk before every day was reached. * * Idempotency, the grace-window cutoff, and the "skip already-collapsed * rows" predicate are all owned here; the divergent reducer / delete / @@ -356,16 +420,24 @@ export async function scanSourceRowsPaged( */ export async function runConsolidation( params: ConsolidationParams, -): Promise<{ usersScanned: number; dryRun: boolean; daysFailed: number }> { +): Promise<{ + usersScanned: number; + dryRun: boolean; + daysFailed: number; + stoppedEarly: boolean; +}> { const { prismaClient, options } = params; const dryRun = options.dryRun ?? false; const cutoffAt = resolveCutoffInstant(options.cutoffHours); const scanSelect = params.scanSelect ?? DEFAULT_SCAN_SELECT; + const shouldStop = options.shouldStop ?? (() => false); let daysFailed = 0; + let daysWalked = 0; + let stoppedEarly = false; const users = await loadConsolidationUsers(prismaClient, options.userId); - for (const user of users) { + walk: for (const user of users) { const tz = resolveUserTimezone(user.timezone); params.onUserStart?.({ userId: user.id, tz, dryRun }); @@ -373,36 +445,45 @@ export async function runConsolidation( const hkIdentifier = params.hkIdentifierForType(type); if (!hkIdentifier) continue; - // Keyset-paginate the scan instead of loading the whole per-(user, type) - // history in one result set; the accumulated array is byte-identical to a - // single unbounded `findMany`, but a multi-year tenant no longer spikes - // worker memory at boot. See `scanSourceRowsPaged`. - const sourceRows = await scanSourceRowsPaged( - prismaClient, - params.buildScanWhere({ - userId: user.id, - type, - cutoffAt, - statsPrefix: params.statsPrefix, - }), - scanSelect, + // Streamed: each day is folded as soon as its last row has been read, + // so the pass holds one page and one day, never the type's history. + // See `iterateSourcePages` and `iterateDayBuckets`. + let rowCount = 0; + let dayCount = 0; + const days = iterateDayBuckets( + iterateSourcePages( + prismaClient, + params.buildScanWhere({ + userId: user.id, + type, + cutoffAt, + statsPrefix: params.statsPrefix, + }), + scanSelect, + ), + tz, + params.statsPrefix, + () => { + rowCount += 1; + }, ); - if (sourceRows.length === 0) continue; - - const byDay = bucketRowsByDay(sourceRows, tz, params.statsPrefix); - - params.onScan?.({ - userId: user.id, - type, - tz, - rowCount: sourceRows.length, - dayCount: byDay.size, - dryRun, - }); - - for (const [dateKey, dayRows] of byDay) { - if (dayRows.length === 0) continue; + for await (const [dateKey, dayRows] of days) { + if (shouldStop()) { + stoppedEarly = true; + // Leaving the loop closes both generators, so no further page is + // requested. + break walk; + } + dayCount += 1; + daysWalked += 1; + // How far the pass has got, for the job's progress and expiry lines. + reportJobProgress({ + consolidation_user: user.id, + consolidation_type: type, + consolidation_day: dateKey, + consolidation_days_walked: daysWalked, + }); try { const reducedValue = params.reduce(dayRows); @@ -464,10 +545,21 @@ export async function runConsolidation( }); } } + + if (rowCount > 0) { + params.onScan?.({ + userId: user.id, + type, + tz, + rowCount, + dayCount, + dryRun, + }); + } } params.onUserComplete?.({ userId: user.id, tz, dryRun }); } - return { usersScanned: users.length, dryRun, daysFailed }; + return { usersScanned: users.length, dryRun, daysFailed, stoppedEarly }; } diff --git a/src/lib/measurements/dense-intraday-hourly-rebuild.ts b/src/lib/measurements/dense-intraday-hourly-rebuild.ts index 455e4b613..a5cbb7d78 100644 --- a/src/lib/measurements/dense-intraday-hourly-rebuild.ts +++ b/src/lib/measurements/dense-intraday-hourly-rebuild.ts @@ -19,7 +19,7 @@ * day's local window (the fold's soft-deleted inputs). * * Per qualifying day: compute per-local-hour means from the tombstoned raw - * rows (same `meanBucketValue` reducer + `adoptOrMintHourlyRow` + * rows (same `meanBucketValue` reducer + `adoptOrMintHourlyRows` * adopt-in-place slot logic as the live fold, so a re-run converges), then * tombstone the daily row IN THE SAME TRANSACTION as the hourly mint — at * no instant are the daily row and its hourly rows both live, so an @@ -73,7 +73,7 @@ import { meanBucketValue } from "./consolidate-daily-mean"; import { DENSE_INTRADAY_RETENTION_DAYS, DENSE_INTRADAY_RETENTION_TYPES, - adoptOrMintHourlyRow, + adoptOrMintHourlyRows, bucketRowsByLocalHour, hourlyStatsExternalId, } from "./dense-intraday-retention"; @@ -111,6 +111,8 @@ export interface DenseIntradayHourlyRebuildSummary { /** Days whose rebuild threw and was stepped over (retried next run). */ daysFailed: number; }; + /** `shouldStop` ended the pass before every candidate day was reached. */ + stoppedEarly: boolean; } export interface DenseIntradayHourlyRebuildOptions { @@ -126,6 +128,12 @@ export interface DenseIntradayHourlyRebuildOptions { * pass `0` to lift the bound. */ retentionDays?: number; + /** + * Asked before each day. Returning `true` ends the pass cleanly; the + * summary reports `stoppedEarly`. A rebuilt day leaves the candidate set, + * so the next run starts at the first day this one did not reach. + */ + shouldStop?: () => boolean; } /** @@ -147,6 +155,7 @@ export async function runDenseIntradayHourlyRebuild( const summary: DenseIntradayHourlyRebuildSummary = { dryRun, + stoppedEarly: false, totals: { usersScanned: 0, daysRebuilt: 0, @@ -160,7 +169,7 @@ export async function runDenseIntradayHourlyRebuild( const users = await loadConsolidationUsers(prismaClient, options.userId); summary.totals.usersScanned = users.length; - for (const user of users) { + walk: for (const user of users) { const tz = resolveUserTimezone(user.timezone); for (const type of DENSE_INTRADAY_RETENTION_TYPES) { @@ -195,6 +204,10 @@ export async function runDenseIntradayHourlyRebuild( if (candidates.length === 0) continue; for (const candidate of candidates) { + if (options.shouldStop?.()) { + summary.stoppedEarly = true; + break walk; + } // Per-day failure boundary: one poisoned day is stepped over so the // walk keeps rebuilding every other user / type / day; the failed // day keeps its live daily row, so the discovery re-finds it. @@ -318,11 +331,11 @@ async function rebuildDay( const rebuildOnce = async (): Promise<{ dailyRetired: boolean }> => { let dailyRetired = false; await prismaClient.$transaction(async (tx) => { - const canonicalRowIds: string[] = []; - for (const [hour, hourRows] of byHour) { - const rowId = await adoptOrMintHourlyRow(tx, { - userId: input.userId, - type: input.type, + const canonicalRowIds = await adoptOrMintHourlyRows(tx, { + userId: input.userId, + type: input.type, + unit, + slots: byHour.map(([hour, hourRows]) => ({ externalId: hourlyStatsExternalId( input.hkIdentifier, input.dateKey, @@ -330,10 +343,8 @@ async function rebuildDay( ), anchor: canonicalHourlyTimestamp(input.dateKey, hour, input.tz), value: meanBucketValue(hourRows), - unit, - }); - canonicalRowIds.push(rowId); - } + })), + }); // Retire the daily row in the SAME transaction — at no instant are // the daily row and the hourly rows both live (an AVG-over-live-rows @@ -359,7 +370,7 @@ async function rebuildDay( if (!isUniqueConstraintViolation(err)) throw err; // A concurrent writer (the nightly fold racing this rebuild) won a slot // mid-transaction. Retry once: the deterministic lookup inside - // `adoptOrMintHourlyRow` now resolves the winning row and adopts it. + // `adoptOrMintHourlyRows` now resolves the winning row and adopts it. outcome = await rebuildOnce(); } diff --git a/src/lib/measurements/dense-intraday-retention.ts b/src/lib/measurements/dense-intraday-retention.ts index 515f8ea19..5c6772c3c 100644 --- a/src/lib/measurements/dense-intraday-retention.ts +++ b/src/lib/measurements/dense-intraday-retention.ts @@ -191,10 +191,20 @@ export function bucketRowsByLocalHour( return byHour; } +/** One hourly slot of a folded day, as `adoptOrMintHourlyRows` receives it. */ +export interface HourlySlot { + /** Target hourly `stats:` externalId (index-A identity). */ + externalId: string; + /** Local HH:30 anchor instant (index-B identity). */ + anchor: Date; + value: number; +} + /** - * Adopt-or-mint one hourly `stats:` row inside the caller's transaction. + * Adopt-or-mint a day's hourly `stats:` rows inside the caller's transaction. + * Returns the canonical row id for every slot, in slot order. * - * TWO unique indexes can collide on the mint: + * TWO unique indexes can collide on a mint: * A) `(userId, type, source, externalId)` — a row already carrying the * target hourly `stats:` externalId (a prior fold / rebuild pass). * B) `(userId, type, measuredAt, source, sleepStage)` (NULLS NOT @@ -202,98 +212,149 @@ export function bucketRowsByLocalHour( * anchor instant, which may carry a DIFFERENT externalId (a * per-sample row that happened at HH:30, or a sibling stats row). * - * Determinism (VECTOR 1): resolve by index A FIRST — the row already - * holding the target externalId IS the canonical hourly row by - * construction. Only when no such row exists fall back to the index-B row - * physically occupying the anchor. A stable `orderBy: id` pins the pick. - * Adopting the wrong sibling and stamping the target externalId onto it - * would otherwise collide with the real `stats:` row on index A. + * Determinism: per slot, resolve by index A FIRST — the row already holding + * the target externalId IS the canonical hourly row by construction. Only + * when no such row exists fall back to the index-B row physically occupying + * the anchor. The lowest id wins a tie. Adopting the wrong sibling and + * stamping the target externalId onto it would otherwise collide with the + * real `stats:` row on index A. * - * Concurrency (VECTOR 2): the resolve→create is a check-then-act not - * serialised by the unique index under READ COMMITTED. If a concurrent - * writer wins the slot between the lookup and the INSERT, the `create` - * throws P2002 and Postgres aborts the whole transaction. Callers - * therefore catch P2002 OUTSIDE their transaction (a caught error inside - * would leave the tx in the aborted `25P02` state) and retry the fold - * once: on the retry the deterministic lookup finds the now-existing row - * and ADOPTS it, so the create path never fires twice. Built - * field-by-field (no spread) per the no-mass-assignment convention. + * Why a batch. This used to run per slot — two lookups and a write for each + * of up to 24 hours, about 75 statements for one day — and on a multi-year + * heart-rate history that loop was most of the pass's 380 000 statements. + * Now one lookup reads every row that could matter for the whole day, the + * slots are resolved against that snapshot in order, and the mints go out as + * one insert. The snapshot is kept up to date as each slot adopts or mints, + * so a later slot sees exactly what the one-by-one loop would have read back + * from the database: an adopted row that moves to its anchor frees the + * instant it left and occupies the one it moved to, and a minted row occupies + * its anchor and externalId. Updates run in slot order, as before, and no + * mint can collide with them because a slot mints only when nothing in the + * snapshot holds its externalId or its anchor. + * + * Concurrency: the resolve→write is a check-then-act not serialised by the + * unique index under READ COMMITTED. If a concurrent writer wins a slot + * between the lookup and the INSERT, the insert throws P2002 and Postgres + * aborts the whole transaction. Callers therefore catch P2002 OUTSIDE their + * transaction (a caught error inside would leave the tx in the aborted + * `25P02` state) and retry the fold once: on the retry the lookup finds the + * now-existing row and ADOPTS it. Built field-by-field (no spread) per the + * no-mass-assignment convention. */ -export async function adoptOrMintHourlyRow( +export async function adoptOrMintHourlyRows( tx: Prisma.TransactionClient, input: { userId: string; type: MeasurementType; - /** Target hourly `stats:` externalId (index-A identity). */ - externalId: string; - /** Local HH:30 anchor instant (index-B identity). */ - anchor: Date; - value: number; unit: string; + slots: readonly HourlySlot[]; }, -): Promise { - const eidRow = await tx.measurement.findFirst({ +): Promise { + if (input.slots.length === 0) return []; + + // Every row that either index could put in the way, lowest id first — the + // order the per-slot `findFirst … orderBy id` used to resolve ties by. + // `sleepStage` is NULL for these continuous types, matched via the + // NULLS-NOT-DISTINCT index. + const existing = await tx.measurement.findMany({ where: { userId: input.userId, type: input.type, source: "APPLE_HEALTH", - externalId: input.externalId, + OR: [ + { externalId: { in: input.slots.map((slot) => slot.externalId) } }, + { + measuredAt: { in: input.slots.map((slot) => slot.anchor) }, + sleepStage: null, + }, + ], }, - select: { id: true }, - orderBy: { id: "asc" }, - }); - // Index-B occupant (`sleepStage` is NULL for these continuous types, - // matched via the NULLS-NOT-DISTINCT index). At most one row can exist. - const slotRow = await tx.measurement.findFirst({ - where: { - userId: input.userId, - type: input.type, - source: "APPLE_HEALTH", - measuredAt: input.anchor, - sleepStage: null, - }, - select: { id: true }, + select: { id: true, externalId: true, measuredAt: true }, orderBy: { id: "asc" }, }); - const adoptTarget = eidRow ?? slotRow; - if (adoptTarget) { - // Pin the adopted row to the anchor, but only when the anchor slot is - // free or already this row — a DIFFERENT row occupying the slot would - // otherwise collide on index B. In that rare case the row keeps its - // existing instant; the `stats:` externalId is the identity, - // measuredAt is secondary. - const slotIsFreeForTarget = - slotRow === null || slotRow.id === adoptTarget.id; - // Adopt: refresh value, stamp the hourly externalId, optionally - // re-anchor, and un-tombstone. This coexists with any pre-existing row - // on the slot instead of colliding with it. - await tx.measurement.update({ - where: { id: adoptTarget.id }, - data: { - value: input.value, - unit: input.unit, - externalId: input.externalId, - deletedAt: null, - ...(slotIsFreeForTarget ? { measuredAt: input.anchor } : {}), - }, - }); - return adoptTarget.id; + // The snapshot the slots resolve against. `id: null` marks a row this + // call is about to mint. + const state: Array<{ + id: string | null; + externalId: string | null; + at: number; + }> = existing.map((row) => ({ + id: row.id, + externalId: row.externalId, + at: row.measuredAt.getTime(), + })); + + const resolved: Array = []; + const mints: HourlySlot[] = []; + + for (const slot of input.slots) { + const anchorAt = slot.anchor.getTime(); + const eidRow = state.find((row) => row.externalId === slot.externalId); + const slotRow = state.find((row) => row.at === anchorAt); + const adoptTarget = eidRow ?? slotRow; + + if (adoptTarget && adoptTarget.id !== null) { + // Pin the adopted row to the anchor, but only when the anchor slot is + // free or already this row — a DIFFERENT row occupying the slot would + // otherwise collide on index B. In that rare case the row keeps its + // existing instant; the `stats:` externalId is the identity, + // measuredAt is secondary. + const slotIsFreeForTarget = + slotRow === undefined || slotRow === adoptTarget; + // Adopt: refresh value, stamp the hourly externalId, optionally + // re-anchor, and un-tombstone. This coexists with any pre-existing row + // on the slot instead of colliding with it. + await tx.measurement.update({ + where: { id: adoptTarget.id }, + data: { + value: slot.value, + unit: input.unit, + externalId: slot.externalId, + deletedAt: null, + ...(slotIsFreeForTarget ? { measuredAt: slot.anchor } : {}), + }, + }); + adoptTarget.externalId = slot.externalId; + if (slotIsFreeForTarget) adoptTarget.at = anchorAt; + resolved.push(adoptTarget.id); + } else { + // Nothing holds this slot's identity or instant — including rows this + // call already decided to mint, which carry a distinct anchor and + // externalId by construction (one slot per local hour). + state.push({ id: null, externalId: slot.externalId, at: anchorAt }); + mints.push(slot); + resolved.push(null); + } } - const created = await tx.measurement.create({ - data: { - userId: input.userId, - type: input.type, - value: input.value, - unit: input.unit, - source: "APPLE_HEALTH", - measuredAt: input.anchor, - externalId: input.externalId, - }, - select: { id: true }, - }); - return created.id; + if (mints.length > 0) { + const created = await tx.measurement.createManyAndReturn({ + data: mints.map((slot) => ({ + userId: input.userId, + type: input.type, + value: slot.value, + unit: input.unit, + source: "APPLE_HEALTH" as const, + measuredAt: slot.anchor, + externalId: slot.externalId, + })), + select: { id: true, externalId: true }, + }); + const idByExternalId = new Map( + created.map((row) => [row.externalId, row.id] as const), + ); + input.slots.forEach((slot, index) => { + if (resolved[index] !== null) return; + const id = idByExternalId.get(slot.externalId); + if (id === undefined) { + throw new Error(`hourly mint returned no row for ${slot.externalId}`); + } + resolved[index] = id; + }); + } + + return resolved as string[]; } /** @@ -347,6 +408,12 @@ export function deriveDailyRestingFromPulse( export interface DenseIntradayRetentionSummary { dryRun: boolean; + /** + * `shouldStop` ended the walk before every day was reached. Nothing is + * lost: folded days committed one by one, and the next run starts at the + * first day this one did not reach. + */ + stoppedEarly: boolean; totals: { usersScanned: number; daysConsolidated: number; @@ -385,6 +452,11 @@ export interface DenseIntradayRetentionOptions { * full-collapse, never by the scheduled pass. */ retentionDays?: number; + /** + * Ends the pass cleanly before the next day when it returns `true`; the + * summary then reports `stoppedEarly`. See `ConsolidationOptions`. + */ + shouldStop?: () => boolean; } /** @@ -412,6 +484,7 @@ export async function runDenseIntradayRetention( const summary: DenseIntradayRetentionSummary = { dryRun: options.dryRun ?? false, + stoppedEarly: false, totals: { usersScanned: 0, daysConsolidated: 0, @@ -462,7 +535,7 @@ export async function runDenseIntradayRetention( // for the dry-run hourly fan-out estimate (writeDay never runs on dry-run). let currentTz = "Europe/Berlin"; - const { usersScanned } = await runConsolidation({ + const walk = await runConsolidation({ prismaClient, options: { ...options, cutoffHours }, types: DENSE_INTRADAY_RETENTION_TYPES, @@ -543,18 +616,16 @@ export async function runDenseIntradayRetention( let removed = 0; let retiredDaily = false; await pc.$transaction(async (tx) => { - const canonicalRowIds: string[] = []; - for (const [hour, hourRows] of byHour) { - const rowId = await adoptOrMintHourlyRow(tx, { - userId, - type, + const canonicalRowIds = await adoptOrMintHourlyRows(tx, { + userId, + type, + unit, + slots: byHour.map(([hour, hourRows]) => ({ externalId: hourlyStatsExternalId(hkIdentifier, dateKey, hour), anchor: canonicalHourlyTimestamp(dateKey, hour, tz), value: meanBucketValue(hourRows), - unit, - }); - canonicalRowIds.push(rowId); - } + })), + }); // Retire a live pre-hourly DAILY `stats:` row for this day (the // late-sync path: raw samples arriving for a day folded to the @@ -712,7 +783,8 @@ export async function runDenseIntradayRetention( }, }); - summary.totals.usersScanned = usersScanned; + summary.totals.usersScanned = walk.usersScanned; + summary.stoppedEarly = walk.stoppedEarly; log( `[dense-intraday-retention] done — usersScanned=${summary.totals.usersScanned} daysConsolidated=${summary.totals.daysConsolidated} perSampleRowsSoftDeleted=${summary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${summary.totals.hourlyRowsUpserted} dailyRowsRetired=${summary.totals.dailyRowsRetired} derivedRestingRowsUpserted=${summary.totals.derivedRestingRowsUpserted}${options.dryRun ? " (dry-run)" : ""}`, diff --git a/src/lib/measurements/drain-per-sample-cumulative.ts b/src/lib/measurements/drain-per-sample-cumulative.ts index aa0f9c91b..e0cf58135 100644 --- a/src/lib/measurements/drain-per-sample-cumulative.ts +++ b/src/lib/measurements/drain-per-sample-cumulative.ts @@ -124,6 +124,12 @@ export interface DrainBucket { export interface DrainSummary { /** Did the run actually write to the DB (false for `dryRun`). */ dryRun: boolean; + /** + * `shouldStop` ended the walk before every day was reached. Nothing is + * lost: folded days committed one by one, and the next run starts at the + * first day this one did not reach. + */ + stoppedEarly: boolean; /** Per-user-day buckets the drain rewrote (or would rewrite). */ buckets: DrainBucket[]; /** Aggregate counts across the run. */ @@ -167,6 +173,11 @@ export interface DrainOptions { * the operator points at) for explicit one-shot use. */ cutoffHours?: number; + /** + * Ends the pass cleanly before the next day when it returns `true`; the + * summary then reports `stoppedEarly`. See `ConsolidationOptions`. + */ + shouldStop?: () => boolean; } export interface BucketedRows { @@ -210,6 +221,7 @@ export async function drainPerSampleCumulative( const summary: DrainSummary = { dryRun: options.dryRun ?? false, + stoppedEarly: false, buckets: [], totals: { usersScanned: 0, @@ -229,7 +241,7 @@ export async function drainPerSampleCumulative( let beforePerSampleDeleted = 0; let beforeDailyUpserted = 0; - const { usersScanned } = await runConsolidation({ + const walk = await runConsolidation({ prismaClient, options, types: CUMULATIVE_HK_TYPES, @@ -521,7 +533,8 @@ export async function drainPerSampleCumulative( }, }); - summary.totals.usersScanned = usersScanned; + summary.totals.usersScanned = walk.usersScanned; + summary.stoppedEarly = walk.stoppedEarly; // v1.34.0 — bound the profile sidecar. One indexed range delete per run, // across every account, on the same nightly tick that writes the profiles. diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index 9bc4fe8cf..d175fc5c0 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -125,6 +125,11 @@ export interface ImportJobResult { /** Estimated `(measurement type, local day)` aggregate rows considered. */ rows: number; }; + /** + * Oldest and newest instant this import wrote a measurement at (ISO), or + * null when it wrote none. What the rollup fold after the import covers. + */ + measuredSpan: { from: string; to: string } | null; /** Bounded auxiliary ECG outcomes. Never contains filenames or waveform data. */ ecg: { discovered: number; @@ -396,6 +401,14 @@ export async function streamParseExportXml( let rowsUpserted = 0; const cumulativeEstimatedDays = new Set(); let cumulativeEstimatedRows = 0; + // The instants this import wrote measurements at, oldest and newest. The + // worker folds the rollup table over exactly this span afterwards. + let spanFrom: Date | null = null; + let spanTo: Date | null = null; + const widenSpan = (at: Date) => { + if (spanFrom === null || at < spanFrom) spanFrom = at; + if (spanTo === null || at > spanTo) spanTo = at; + }; // Per R-1 §8 the percent stays best-effort and may remain null // until the parser sees the closing `` tag. We don't // currently mutate this in v1.4.34 — the iOS app keeps polling on @@ -438,6 +451,7 @@ export async function streamParseExportXml( const flushSpotBatch = async (): Promise => { if (spotBatch.length === 0) return; const chunk = spotBatch.splice(0, spotBatch.length); + for (const row of chunk) widenSpan(row.measuredAt); const insertedArrivals: Array<{ id: string; type: MeasurementType; @@ -731,6 +745,7 @@ export async function streamParseExportXml( } const externalId = dailyStatsExternalId(mapping.hkIdentifier, dayKey); const measuredAt = canonicalDailyTimestamp(dayKey, userTimezone); + widenSpan(measuredAt); const rowStart = Date.now(); const verdict = await prisma.$transaction((tx) => reconcileExternalMeasurement( @@ -1202,6 +1217,13 @@ export async function streamParseExportXml( days: cumulativeEstimatedDays.size, rows: cumulativeEstimatedRows, }, + measuredSpan: + spanFrom !== null && spanTo !== null + ? { + from: (spanFrom as Date).toISOString(), + to: (spanTo as Date).toISOString(), + } + : null, ecg: { discovered: 0, imported: 0, diff --git a/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts b/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts index e4172f434..bcda29b20 100644 --- a/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts +++ b/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts @@ -8,12 +8,20 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; const deleteMany = vi.fn(); const findMany = vi.fn(); +// The delete and the re-derivation share one transaction; the stub hands the +// callback a client that routes to the same spies. +const txClient = { + personalRecord: { + deleteMany: (a: unknown) => deleteMany(a), + }, +}; vi.mock("@/lib/db", () => ({ prisma: { personalRecord: { deleteMany: (a: unknown) => deleteMany(a), findMany: (a: unknown) => findMany(a), }, + $transaction: (fn: (tx: unknown) => Promise) => fn(txClient), }, })); @@ -68,8 +76,10 @@ describe("runCumulativePrRederivationForUser", () => { }); // Re-derivation always runs silent — this is data hygiene, not a new // achievement, so no push should fire for it. + // It reads through the transaction, so it sees the deletion. expect(detectPersonalRecordsForUser).toHaveBeenCalledWith("u1", { silent: true, + prisma: txClient, }); expect(summary).toEqual({ rowsDeleted: 1, rowsReinserted: 1 }); }); diff --git a/src/lib/personal-records/cumulative-pr-rederivation.ts b/src/lib/personal-records/cumulative-pr-rederivation.ts index a6202ac24..b93f15755 100644 --- a/src/lib/personal-records/cumulative-pr-rederivation.ts +++ b/src/lib/personal-records/cumulative-pr-rederivation.ts @@ -41,7 +41,10 @@ import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; import { detectPersonalRecordsForUser } from "@/lib/personal-records/pr-detection-worker"; -import type { MeasurementType } from "@/generated/prisma/client"; +import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; + +/** Budget for one account's delete-and-re-derive transaction. */ +const REDERIVE_TRANSACTION_TIMEOUT_MS = 5 * 60 * 1000; export const CUMULATIVE_PR_REDERIVE_QUEUE = "cumulative-pr-rederive"; @@ -90,22 +93,40 @@ export interface CumulativePrRederiveSummary { export async function runCumulativePrRederivationForUser( userId: string, ): Promise { - const { count: rowsDeleted } = await prisma.personalRecord.deleteMany({ - where: { - userId, - metricSlot: null, - metricType: { in: CUMULATIVE_TYPES_ARRAY }, - createdAt: { lt: CUMULATIVE_PR_FIX_CUTOFF }, + // One transaction: the suspect rows go and the re-derived ones arrive + // together, or neither happens. Deleting first and re-deriving after the + // commit lost the records whenever the detector failed (a timeout on a + // large account, a restart): discovery looks for the suspect rows, so once + // they were gone the account was never repaired again (#1031). The detector + // reads through the transaction, so it sees the suspect rows as deleted and + // compares against what is left. It only reads and inserts, and sends no + // notification when `silent`. + const { rowsDeleted, rowsReinserted } = await prisma.$transaction( + async (tx) => { + const { count } = await tx.personalRecord.deleteMany({ + where: { + userId, + metricSlot: null, + metricType: { in: CUMULATIVE_TYPES_ARRAY }, + createdAt: { lt: CUMULATIVE_PR_FIX_CUTOFF }, + }, + }); + if (count === 0) return { rowsDeleted: 0, rowsReinserted: 0 }; + const result = await detectPersonalRecordsForUser(userId, { + silent: true, + prisma: tx as unknown as PrismaClient, + }); + return { + rowsDeleted: count, + rowsReinserted: result.inserted + result.ties, + }; }, - }); - - let rowsReinserted = 0; - if (rowsDeleted > 0) { - const result = await detectPersonalRecordsForUser(userId, { - silent: true, - }); - rowsReinserted = result.inserted + result.ties; - } + // The detector walks every trackable type of the account; on a large + // record that is tens of seconds (19.8 s measured for 1.25 million + // measurements plus 784 000 energy rows), so the default five seconds + // would roll back every repair that matters. + { timeout: REDERIVE_TRANSACTION_TIMEOUT_MS, maxWait: 10_000 }, + ); annotate({ action: { diff --git a/src/lib/personal-records/pr-detection-worker.ts b/src/lib/personal-records/pr-detection-worker.ts index 107fcc704..77739c6a5 100644 --- a/src/lib/personal-records/pr-detection-worker.ts +++ b/src/lib/personal-records/pr-detection-worker.ts @@ -40,6 +40,10 @@ import { prisma as defaultPrisma } from "@/lib/db"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; import { dayKeyForUserTz } from "@/lib/measurements/consolidation-tz"; +import { + iterateDayBuckets, + iterateSourcePages, +} from "@/lib/measurements/consolidation-base"; import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum"; import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority"; import { measurementTypeEnum } from "@/lib/validations/measurement"; @@ -273,8 +277,12 @@ export async function detectPersonalRecordsForUser( // Warm-up gate. Cheap count first so we don't pull a 30k-row // scan on a brand-new account. + // Bounded: the gate only asks whether there are enough, so it stops + // counting at the threshold instead of counting a whole history (766 000 + // heart-rate rows on a large account, every half hour). const sampleCount = await prisma.measurement.count({ where: { userId, type: metricType, deletedAt: null }, + take: PR_DETECTION_WARMUP_THRESHOLD, }); if (sampleCount < PR_DETECTION_WARMUP_THRESHOLD) continue; @@ -542,18 +550,28 @@ async function findBestMeasurement( * cumulative-day surface. * * Order of operations: - * 1. Read every row for the type (already scoped by the caller's - * warm-up gate to a metric with at least - * `PR_DETECTION_WARMUP_THRESHOLD` samples). - * 2. Collapse to the canonical source per local day. - * 3. Bucket-and-sum the canonical rows per local day; pick the - * winning day by `direction`. + * 1. Read the type's rows one local day at a time, in time order + * (already scoped by the caller's warm-up gate to a metric with at + * least `PR_DETECTION_WARMUP_THRESHOLD` samples). + * 2. Collapse each day to its canonical source. + * 3. Sum the day's canonical rows; keep the winning day by `direction`. * 4. Re-read the winning day's latest slice so the PR row carries a * real row's unit/source/externalId (the day-sum overrides * `value` below). * The unique-index contract is unaffected — a re-run picks the same * day and the same latest-slice timestamp, so `achievedAt` is stable. */ +/** One row of the cumulative-day scan. */ +interface CumulativeRow { + id: string; + value: number; + unit: string; + measuredAt: Date; + source: MeasurementSource; + externalId: string | null; + deviceType: string | null; +} + async function findBestCumulativeDay( prisma: PrismaClient, userId: string, @@ -562,46 +580,50 @@ async function findBestCumulativeDay( tz: string, priorityJson: unknown, ): Promise { - const rows = await prisma.measurement.findMany({ - where: { userId, type, deletedAt: null }, - orderBy: { measuredAt: "asc" }, - select: { - id: true, - value: true, - unit: true, - measuredAt: true, - source: true, - externalId: true, - deviceType: true, - }, - }); - if (rows.length === 0) return null; - + // Read a day at a time, in time order, never the whole history: the + // canonical-source pick is per day, so running it on each day alone gives + // the same rows it gave on the whole list. Loading every row at once was + // unbounded; with 784 000 rows of one cumulative type the pass ran a + // 524 MB heap out of memory (#1031). const metricKey = metricKeyForType(type); - const canonicalRows = metricKey - ? pickCanonicalSourceRows( - rows.map((r) => ({ ...r, type })), - metricKey, - priorityJson, - (d) => dayKeyForUserTz(d, tz), - ).canonicalRows - : rows; - - const byDay = new Map(); - for (const row of canonicalRows) { - const key = dayKeyForUserTz(row.measuredAt, tz); - const slot = byDay.get(key); - if (!slot) { - byDay.set(key, { total: row.value, latest: row.measuredAt }); - } else { - slot.total += row.value; - if (row.measuredAt > slot.latest) slot.latest = row.measuredAt; - } - } - let winnerTotal: number | null = null; let winnerLatest: Date | null = null; - for (const { total, latest } of byDay.values()) { + const days = iterateDayBuckets( + iterateSourcePages( + prisma, + { userId, type, deletedAt: null }, + { + id: true, + value: true, + unit: true, + measuredAt: true, + source: true, + externalId: true, + deviceType: true, + }, + ), + tz, + null, + ); + for await (const [, dayRows] of days) { + const canonicalRows = metricKey + ? pickCanonicalSourceRows( + dayRows.map((r) => ({ ...r, type })), + metricKey, + priorityJson, + (d) => dayKeyForUserTz(d, tz), + ).canonicalRows + : dayRows; + if (canonicalRows.length === 0) continue; + + let total = 0; + let latest = canonicalRows[0]!.measuredAt; + for (const row of canonicalRows) { + total += row.value; + if (row.measuredAt > latest) latest = row.measuredAt; + } + // Days arrive in order, and only a strictly better day replaces the + // winner, so a tie keeps the earliest day, as before. const better = winnerTotal === null || (direction === PersonalRecordDirection.MAX diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index 9873bbfbc..c2c9fab6e 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -1773,6 +1773,16 @@ export const backupPayloadSchema = z export type BackupPayload = z.infer; +/** + * One element of `measurements`, validated on its own. The streamed readers + * (`src/lib/export/streamed-backup.ts`) check a large file's measurements + * one at a time with this, and the rest of the file with + * `backupPayloadSchema` over a document whose measurements are empty, which + * together is the same check `backupPayloadSchema` makes of the whole file. + */ +export const backupMeasurementSchema = measurementSchema; +export type BackupMeasurement = z.infer; + /** * Numeric counts of each backed-up record kind. Returned in the * upload + restore API responses so the admin sees what they diff --git a/tests/integration/apple-health-import-rollup-window.test.ts b/tests/integration/apple-health-import-rollup-window.test.ts new file mode 100644 index 000000000..be558605e --- /dev/null +++ b/tests/integration/apple-health-import-rollup-window.test.ts @@ -0,0 +1,116 @@ +/** + * #1031 — the rollup fold after an import covers what the import wrote. + * + * The worker folded the rollup table from the account's first measurement to + * its last, whatever the import carried, and so also folded history older + * than the five-year window every other fold path keeps to. It now folds the + * span of the rows this import wrote, clamped to that window. + */ +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { strToU8, zipSync } from "fflate"; +import type { Job } from "pg-boss"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { + APPLE_HEALTH_IMPORT_PARSER_REVISION, + handleAppleHealthImport, + _setWorkerPrismaForTests, + type AppleHealthImportPayload, +} from "@/lib/jobs/apple-health-import-worker"; +import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); + +const USER = "user-import-rollup-window"; + +function record(at: Date, value: number): string { + const stamp = at.toISOString().replace("T", " ").slice(0, 19) + " +0000"; + return ``; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + _setWorkerPrismaForTests(getPrismaClient()); + await getPrismaClient().user.create({ + data: { + id: USER, + username: USER, + email: `${USER}@example.test`, + timezone: "UTC", + }, + }); +}); + +describe("import rollup fold window", () => { + it("folds the imported span, never past the fold window, and leaves older history alone", async () => { + const prisma = getPrismaClient(); + const now = Date.now(); + const day = 86_400_000; + // Already in the account before the import: a reading eight years old. + await prisma.measurement.create({ + data: { + userId: USER, + type: "WEIGHT", + value: 90, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(now - 8 * 365 * day), + }, + }); + // The import: one reading seven years old, a few recent ones. + const xml = ` + + ${record(new Date(now - 7 * 365 * day), 88)} + ${record(new Date(now - 40 * day), 84)} + ${record(new Date(now - 30 * day), 83)} + ${record(new Date(now - 20 * day), 82)} +`; + const dir = mkdtempSync(join(tmpdir(), "hl-import-window-")); + const path = join(dir, "export.zip"); + const bytes = Buffer.from( + zipSync({ "apple_health_export/export.xml": strToU8(xml) }), + ); + writeFileSync(path, bytes); + await prisma.importJob.create({ + data: { + userId: USER, + pgBossJobId: "window-job", + status: "queued", + uploadBytes: bytes.length, + uploadSha256: createHash("sha256").update(bytes).digest("hex"), + parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, + }, + }); + + const outcome = await handleAppleHealthImport({ + id: "window-job", + data: { + userId: USER, + uploadPath: path, + uploadBytes: bytes.length, + enqueuedAt: new Date().toISOString(), + }, + } as unknown as Job); + expect(outcome.ok).toBe(true); + + const buckets = await prisma.measurementRollup.findMany({ + where: { userId: USER, type: "WEIGHT", granularity: "DAY" }, + orderBy: { bucketStart: "asc" }, + }); + const windowStart = now - ROLLUP_FOLD_WINDOW_MS; + expect(buckets.length).toBeGreaterThanOrEqual(3); + for (const bucket of buckets) { + expect(bucket.bucketStart.getTime()).toBeGreaterThan(windowStart - day); + } + }); +}); diff --git a/tests/integration/cumulative-pr-rederive-atomic.test.ts b/tests/integration/cumulative-pr-rederive-atomic.test.ts new file mode 100644 index 000000000..77c8509a2 --- /dev/null +++ b/tests/integration/cumulative-pr-rederive-atomic.test.ts @@ -0,0 +1,94 @@ +/** + * #1031 — the cumulative-record repair must not lose records when the + * re-derivation fails. + * + * The repair deleted every suspect cumulative record, committed, and only + * then re-ran the detector. A detector that failed (a timeout on a large + * account, a crash) left the records deleted, and because discovery looks + * for the suspect rows the deletion removed, the account was never + * repaired again: its step, energy and distance records were simply gone. + * The delete and the re-derivation now commit together or not at all. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const detect = vi.hoisted(() => ({ fail: false })); +vi.mock("@/lib/personal-records/pr-detection-worker", async (importActual) => { + const actual = + await importActual< + typeof import("@/lib/personal-records/pr-detection-worker") + >(); + return { + ...actual, + detectPersonalRecordsForUser: async ( + ...args: Parameters + ) => { + if (detect.fail) throw new Error("detector ran out of time"); + return actual.detectPersonalRecordsForUser(...args); + }, + }; +}); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const { runCumulativePrRederivationForUser, CUMULATIVE_PR_FIX_CUTOFF } = + await import("@/lib/personal-records/cumulative-pr-rederivation"); + +const USER = "user-pr-rederive-atomic"; + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + detect.fail = false; + await prisma.user.create({ + data: { id: USER, username: USER, email: `${USER}@example.test` }, + }); + const before = new Date(CUMULATIVE_PR_FIX_CUTOFF.getTime() - 86_400_000); + await prisma.personalRecord.create({ + data: { + userId: USER, + metricType: "ACTIVITY_STEPS", + direction: "MAX", + value: 99_999, + unit: "steps", + achievedAt: new Date("2025-03-01T12:00:00Z"), + createdAt: before, + }, + }); + // Enough steps for the detector's warm-up gate, one clear best day. + await prisma.measurement.createMany({ + data: Array.from({ length: 10 }, (_, day) => ({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: day === 4 ? 12_000 : 5_000, + unit: "steps", + source: "APPLE_HEALTH" as const, + measuredAt: new Date(Date.UTC(2025, 3, 1 + day, 12)), + externalId: `s-${day}`, + })), + }); +}); + +describe("runCumulativePrRederivationForUser", () => { + it("keeps the suspect record when the re-derivation fails", async () => { + detect.fail = true; + await expect(runCumulativePrRederivationForUser(USER)).rejects.toThrow( + "detector ran out of time", + ); + const left = await getPrismaClient().personalRecord.findMany({ + where: { userId: USER }, + }); + expect(left.map((r) => r.value)).toEqual([99_999]); + }); + + it("replaces the suspect record with the re-derived one when it succeeds", async () => { + const summary = await runCumulativePrRederivationForUser(USER); + expect(summary.rowsDeleted).toBe(1); + const left = await getPrismaClient().personalRecord.findMany({ + where: { userId: USER }, + }); + expect(left.map((r) => r.value)).toEqual([12_000]); + }); +}); diff --git a/tests/integration/job-failures.test.ts b/tests/integration/job-failures.test.ts index 18a6f6812..a4c108986 100644 --- a/tests/integration/job-failures.test.ts +++ b/tests/integration/job-failures.test.ts @@ -16,6 +16,7 @@ import { PgBoss } from "pg-boss"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { + readActiveJobsStartedBefore, readFailingQueues, readQueueFailureForUser, } from "@/lib/jobs/job-failures"; @@ -28,8 +29,16 @@ const OTHER_QUEUE = "reminder-check"; let boss: PgBoss; /** Send a job, take it, and fail it terminally so it lands in `state='failed'`. */ -async function failOneJob(queue: string, data: object, error: string) { - const jobId = await boss.send(queue, data, { retryLimit: 0 }); +async function failOneJob( + queue: string, + data: object, + error: string, + expireInSeconds?: number, +) { + const jobId = await boss.send(queue, data, { + retryLimit: 0, + ...(expireInSeconds === undefined ? {} : { expireInSeconds }), + }); if (!jobId) throw new Error("queue rejected the job"); const [job] = await boss.fetch(queue); if (!job) throw new Error("nothing to fetch"); @@ -81,6 +90,26 @@ describe("readFailingQueues", () => { expect(other?.lastError).toBe("reminder dispatch failed"); }); + it("says what a bare timeout means, with the job's own limit (#1031)", async () => { + await getPrismaClient().$executeRawUnsafe( + `DELETE FROM pgboss.job WHERE state = 'failed'`, + ); + // pg-boss writes these words both for a pass too slow for its limit and + // for a job whose process died under it; alone they sent an operator + // nowhere. + await failOneJob(QUEUE, {}, "job timed out", 900); + await failOneJob(OTHER_QUEUE, {}, "job timed out", 7200); + + const failing = await readFailingQueues(); + const fifteen = failing?.find((entry) => entry.queue === QUEUE); + expect(fifteen?.lastError).toMatch( + /^job timed out: did not finish within its 15 min limit\./, + ); + expect(fifteen?.lastError).toMatch(/restarted/); + const twoHours = failing?.find((entry) => entry.queue === OTHER_QUEUE); + expect(twoHours?.lastError).toMatch(/within its 2 h limit/); + }); + it("reports an empty list — not null — when nothing is failing", async () => { await getPrismaClient().$executeRawUnsafe( `DELETE FROM pgboss.job WHERE state = 'failed'`, @@ -132,6 +161,34 @@ describe("readQueueFailureForUser", () => { }); }); +describe("readActiveJobsStartedBefore (#1031)", () => { + it("lists a job still marked active from before the given instant, and nothing after it", async () => { + const jobId = await boss.send( + OTHER_QUEUE, + {}, + { retryLimit: 0, expireInSeconds: 7200 }, + ); + const [job] = await boss.fetch(OTHER_QUEUE); + expect(job?.id).toBe(jobId); + + const cutOff = await readActiveJobsStartedBefore( + new Date(Date.now() + 60_000), + ); + expect(cutOff).toContainEqual( + expect.objectContaining({ + queue: OTHER_QUEUE, + id: jobId, + expireSeconds: 7200, + }), + ); + await expect( + readActiveJobsStartedBefore(new Date(Date.now() - 60 * 60_000)), + ).resolves.toEqual([]); + + await boss.complete(OTHER_QUEUE, jobId!); + }); +}); + describe("no queue schema at all", () => { it("answers null rather than throwing, so the status page still renders", async () => { const prisma = getPrismaClient(); diff --git a/tests/integration/large-account-jobs.test.ts b/tests/integration/large-account-jobs.test.ts new file mode 100644 index 000000000..c897ec973 --- /dev/null +++ b/tests/integration/large-account-jobs.test.ts @@ -0,0 +1,266 @@ +/** + * Issue #1031 — maintenance jobs on a large account. + * + * A self-hoster with 1.25 million measurement rows (a multi-year Apple Health + * export) saw the dense intra-day retention, the nightly drain tick and the + * data backup all fail with `job timed out`. Reproduced against a seeded + * account of the same size (`scripts/seed-large-account.ts`): the drains held + * a type's whole history in memory before folding a day, spent about 90 + * statements per folded day, and paged with a cursor Postgres could not use as + * an index condition; the backup added about 640 MB to the process to store + * one 64 MB row. pg-boss declared the jobs dead at their expiry without + * stopping them. + * + * A million rows do not fit a test run, so this file pins the properties the + * fix rests on at a scale that does: a pass that runs out of budget stops + * cleanly and the next run finishes the job with the same result as one + * uninterrupted run, and the backup is stored piecewise with the previous + * copy intact if a run fails halfway. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; +import { consolidateDailyMean } from "@/lib/measurements/consolidate-daily-mean"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; + +const TZ = "Europe/Berlin"; +const RUN_A = "user-1031-stopped"; +const RUN_B = "user-1031-straight"; +const DAYS = 12; +/** Oldest seeded instant: well outside the 90-day raw window. */ +const START = Date.UTC(2026, 0, 1, 0, 7); + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + for (const id of [RUN_A, RUN_B]) { + await prisma.user.create({ + data: { id, username: id, email: `${id}@example.test`, timezone: TZ }, + }); + } +}); + +/** The same history for both users: heart rate every 20 minutes, HRV hourly. */ +async function seedHistory(userId: string): Promise { + const rows = []; + for (let minute = 0; minute < DAYS * 24 * 60; minute += 20) { + const at = new Date(START + minute * 60_000); + rows.push({ + userId, + type: "PULSE" as const, + value: 55 + ((minute / 20) % 40), + unit: "bpm", + source: "APPLE_HEALTH" as const, + measuredAt: at, + externalId: `hk-hr-${minute}`, + }); + if (minute % 60 === 0) { + rows.push({ + userId, + type: "HEART_RATE_VARIABILITY" as const, + value: 30 + ((minute / 60) % 25), + unit: "ms", + source: "APPLE_HEALTH" as const, + measuredAt: at, + externalId: `hk-hrv-${minute}`, + }); + } + if (minute % 40 === 0) { + rows.push({ + userId, + type: "WALKING_SPEED" as const, + value: 4 + ((minute / 40) % 10) / 10, + unit: "km/h", + source: "APPLE_HEALTH" as const, + measuredAt: at, + externalId: `hk-ws-${minute}`, + }); + } + } + await getPrismaClient().measurement.createMany({ data: rows }); +} + +/** What a reader sees: every live row, identity-free. */ +async function liveRows(userId: string) { + const rows = await getPrismaClient().measurement.findMany({ + where: { userId, deletedAt: null }, + select: { + type: true, + value: true, + unit: true, + source: true, + measuredAt: true, + externalId: true, + }, + orderBy: [{ type: "asc" }, { measuredAt: "asc" }], + }); + return rows.map((row) => ({ + ...row, + measuredAt: row.measuredAt.toISOString(), + })); +} + +/** A `shouldStop` that lets `days` day buckets through, then says stop. */ +function afterDays(days: number): () => boolean { + let left = days; + return () => left-- <= 0; +} + +describe("a pass that runs out of budget (#1031)", () => { + it("dense retention: stops cleanly, and the next run ends where one straight run ends", async () => { + await seedHistory(RUN_A); + await seedHistory(RUN_B); + const prisma = getPrismaClient(); + + const first = await runDenseIntradayRetention(prisma, { + userId: RUN_A, + shouldStop: afterDays(5), + log: () => {}, + }); + expect(first.stoppedEarly).toBe(true); + expect(first.totals.daysConsolidated).toBe(5); + + // What the stopped run folded is committed: the next scan no longer sees + // it, so the second run folds only what is left. + const second = await runDenseIntradayRetention(prisma, { + userId: RUN_A, + log: () => {}, + }); + expect(second.stoppedEarly).toBe(false); + + const straight = await runDenseIntradayRetention(prisma, { + userId: RUN_B, + log: () => {}, + }); + expect(first.totals.daysConsolidated + second.totals.daysConsolidated).toBe( + straight.totals.daysConsolidated, + ); + expect( + first.totals.perSampleRowsSoftDeleted + + second.totals.perSampleRowsSoftDeleted, + ).toBe(straight.totals.perSampleRowsSoftDeleted); + + // A third run finds nothing: the pass converged. + const third = await runDenseIntradayRetention(prisma, { + userId: RUN_A, + log: () => {}, + }); + expect(third.totals.daysConsolidated).toBe(0); + + const stoppedResult = (await liveRows(RUN_A)).filter( + (row) => row.type !== "WALKING_SPEED", + ); + const straightResult = (await liveRows(RUN_B)).filter( + (row) => row.type !== "WALKING_SPEED", + ); + expect(stoppedResult.length).toBeGreaterThan(0); + expect(stoppedResult).toEqual(straightResult); + }); + + it("daily-mean consolidation: the same, for the nightly tick's middle pass", async () => { + await seedHistory(RUN_A); + await seedHistory(RUN_B); + const prisma = getPrismaClient(); + + const first = await consolidateDailyMean(prisma, { + userId: RUN_A, + shouldStop: afterDays(4), + log: () => {}, + }); + expect(first.stoppedEarly).toBe(true); + expect(first.totals.daysConsolidated).toBe(4); + await consolidateDailyMean(prisma, { userId: RUN_A, log: () => {} }); + await consolidateDailyMean(prisma, { userId: RUN_B, log: () => {} }); + + const stopped = (await liveRows(RUN_A)).filter( + (row) => row.type === "WALKING_SPEED", + ); + const straight = (await liveRows(RUN_B)).filter( + (row) => row.type === "WALKING_SPEED", + ); + // One daily mean per local day: twelve UTC days touch thirteen Berlin days. + expect(stopped).toHaveLength(DAYS + 1); + expect(stopped).toEqual(straight); + }); +}); + +describe("storeBackupBlob (#1031)", () => { + /** A producer writing `rows` JSON rows in small pieces. */ + function producer(rows: number, failAfter?: number) { + return async (write: (chunk: string) => Promise) => { + await write('{"measurements":['); + for (let i = 0; i < rows; i++) { + if (failAfter !== undefined && i === failAfter) { + throw new Error("record read failed halfway"); + } + await write( + `${i === 0 ? "" : ","}${JSON.stringify({ id: `m-${i}`, value: i })}`, + ); + } + await write("]}"); + }; + } + + function expectedJson(rows: number): string { + return JSON.stringify({ + measurements: Array.from({ length: rows }, (_, i) => ({ + id: `m-${i}`, + value: i, + })), + }); + } + + it("stores a copy that reads back as exactly the JSON produced", async () => { + const prisma = getPrismaClient(); + const { bytes } = await storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(20_000), + ); + + const row = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + expect(row.data.length).toBe(bytes); + expect(unpackBackupBlob(row.data)).toBe(expectedJson(20_000)); + }); + + it("keeps the previous copy when a run fails halfway, and the next run still stores", async () => { + const prisma = getPrismaClient(); + await storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(10), + ); + const before = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + + await expect( + storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(20_000, 15_000), + ), + ).rejects.toThrow("record read failed halfway"); + + const after = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + expect(after.data).toBe(before.data); + expect(after.createdAt.getTime()).toBe(before.createdAt.getTime()); + + // The temporary table went with the failed transaction. + await storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(30), + ); + const replaced = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + expect(unpackBackupBlob(replaced.data)).toBe(expectedJson(30)); + }); +}); diff --git a/tests/integration/large-backup-round-trip.test.ts b/tests/integration/large-backup-round-trip.test.ts new file mode 100644 index 000000000..61301ac42 --- /dev/null +++ b/tests/integration/large-backup-round-trip.test.ts @@ -0,0 +1,203 @@ +/** + * #1031 — an account too large to export or restore. + * + * For an account of 1.25 million measurements the full export took the app + * down, and its weekly backup could not be restored, downloaded or summarised + * at all: the backup's JSON (662 MB) is longer than any string V8 can hold, + * and every reader turned the file into one string first. Measured on a + * seeded account of that size in a 1 GB container, the streamed paths now + * export in 25 s, upload in under 20 s and restore in 94 s, and the restored + * rows match the file exactly. + * + * A million rows do not fit a test run. This file runs the same paths at a + * size that spans many pages and batches, and checks the rows come back + * exactly: the export streamed into the response, uploaded as gzip through + * the raw-body path, and restored through the streamed two-pass reader; and a + * weekly backup restored the same way, every column intact, with a personal + * record still pointing at the measurement it was found in. + */ +import { gzipSync } from "node:zlib"; + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; +import { encryptNote } from "@/lib/crypto/note-cipher"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const COUNT = 7_300; // more than one read page and several write batches + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +async function seedAccount() { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username: "large-account", + email: "large-account@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const start = Date.UTC(2024, 0, 1); + await prisma.measurement.createMany({ + data: Array.from({ length: COUNT }, (_, i) => ({ + id: `m-${String(i).padStart(5, "0")}`, + userId: user.id, + type: i % 3 === 0 ? ("PULSE" as const) : ("SLEEP_DURATION" as const), + value: 50 + (i % 40) + 0.25, + valueMax: i % 3 === 0 ? 120 : null, + unit: i % 3 === 0 ? "bpm" : "minutes", + source: "APPLE_HEALTH" as const, + measuredAt: new Date(start + i * 300_000), + externalId: `hk-${i}`, + sleepStage: i % 3 === 0 ? null : ("CORE" as const), + deviceType: "watch", + syncVersion: 1 + (i % 4), + notesEncrypted: i === 17 ? encryptNote("after the long run") : null, + deletedAt: i % 97 === 0 ? new Date(start) : null, + })), + }); + await prisma.personalRecord.create({ + data: { + userId: user.id, + metricType: "PULSE", + direction: "MAX", + value: 89.25, + unit: "bpm", + achievedAt: new Date(start + 3 * 300_000), + sourceMeasurementId: "m-00003", + }, + }); + return user; +} + +async function measurementsOf(userId: string) { + return getPrismaClient().measurement.findMany({ + where: { userId }, + orderBy: { id: "asc" }, + omit: { createdAt: true, updatedAt: true }, + }); +} + +async function restore(id: string) { + const { POST } = await import("@/app/api/admin/backups/[id]/restore/route"); + const res = await POST( + new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }) as never, + { params: Promise.resolve({ id }) }, + ); + return res; +} + +describe("large backups round-trip through the streamed paths (#1031)", () => { + it("export → gzip upload → restore gives back every live measurement", async () => { + const user = await seedAccount(); + const live = (await measurementsOf(user.id)).filter((m) => !m.deletedAt); + + const { GET } = await import("@/app/api/export/full-backup/route"); + const exported = await GET( + new Request("http://localhost/api/export/full-backup") as never, + ); + expect(exported.status).toBe(200); + const file = Buffer.from(await exported.arrayBuffer()); + expect(JSON.parse(file.toString("utf8")).measurements).toHaveLength( + live.length, + ); + + const { POST: upload } = + await import("@/app/api/admin/backups/upload/route"); + const uploaded = await upload( + new Request("http://localhost/api/admin/backups/upload", { + method: "POST", + headers: { "content-type": "application/gzip" }, + body: gzipSync(file), + }) as never, + ); + expect(uploaded.status).toBe(201); + const { data } = (await uploaded.json()) as { + data: { id: string; summary: { measurements: number } }; + }; + expect(data.summary.measurements).toBe(live.length); + + expect((await restore(data.id)).status).toBe(200); + const restored = await measurementsOf(user.id); + // A portable file carries the fields a person reads, not the sync + // bookkeeping, so those are what must come back. + const readable = (rows: typeof restored) => + rows.map((m) => ({ + id: m.id, + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: m.measuredAt.toISOString(), + source: m.source, + })); + expect(readable(restored)).toEqual(readable(live)); + }); + + it("a weekly backup restores every column, and a record's pointer survives", async () => { + const user = await seedAccount(); + const before = await measurementsOf(user.id); + const prisma = getPrismaClient(); + + const { id } = await storeBackupBlob( + prisma, + { userId: user.id, type: "WEEKLY_AUTO" }, + (write) => + streamFullBackupJson(prisma, user.id, write, { + purpose: "disaster-recovery", + }), + ); + + // Change the account so the restore has something to undo. + await prisma.measurement.deleteMany({ where: { userId: user.id } }); + + const res = await restore(id); + expect(res.status).toBe(200); + expect(await measurementsOf(user.id)).toEqual(before); + const record = await prisma.personalRecord.findFirstOrThrow({ + where: { userId: user.id }, + }); + expect(record.sourceMeasurementId).toBe("m-00003"); + }); +}); diff --git a/tests/integration/measurement-bulk-insert.test.ts b/tests/integration/measurement-bulk-insert.test.ts new file mode 100644 index 000000000..c0caae7b1 --- /dev/null +++ b/tests/integration/measurement-bulk-insert.test.ts @@ -0,0 +1,139 @@ +/** + * `insertMeasurementRows` replaced `createMany` in the restore because + * Prisma's cached plan for a 1 000-row `createMany` kept about 145 MB alive + * (#1031). It writes raw SQL, so this pins that a row written through it + * reads back exactly as the same row written through Prisma, every column + * set, and every column null where it may be. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { + insertMeasurementRows, + type MeasurementInsertRow, +} from "@/lib/export/measurement-bulk-insert"; + +const USER = "user-bulk-insert"; + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, email: `${USER}@example.test` }, + }); +}); + +function full(id: string, at: string): MeasurementInsertRow { + return { + id, + userId: USER, + type: "SLEEP_DURATION", + value: 47.25, + valueMin: 1.5, + valueMax: 99.75, + unit: "minutes", + source: "APPLE_HEALTH", + measuredAt: new Date(at), + notes: "legacy note", + notesEncrypted: new Uint8Array([0, 1, 2, 254, 255]), + externalId: `ext-${id}`, + externalSourceVersion: "17.4", + aggregationProvenance: "HEALTHKIT_STATISTICS", + glucoseContext: null, + sleepStage: "REM", + rhythmClassification: null, + deviceType: "watch", + syncVersion: 7, + deletedAt: new Date("2026-02-03T04:05:06.789Z"), + createdAt: new Date("2025-01-02T03:04:05.678Z"), + updatedAt: new Date("2025-06-07T08:09:10.111Z"), + }; +} + +function sparse(id: string, at: string): MeasurementInsertRow { + return { + id, + userId: USER, + type: "BLOOD_GLUCOSE", + value: 5.4, + valueMin: null, + valueMax: null, + unit: "mmol/L", + source: "MANUAL", + measuredAt: new Date(at), + notes: null, + notesEncrypted: null, + externalId: null, + externalSourceVersion: null, + aggregationProvenance: null, + glucoseContext: "FASTING", + sleepStage: null, + rhythmClassification: null, + deviceType: null, + syncVersion: 1, + deletedAt: null, + }; +} + +describe("insertMeasurementRows", () => { + it("writes every column exactly as Prisma create does", async () => { + const prisma = getPrismaClient(); + const rows = [ + [ + full("a-bulk", "2026-01-01T10:00:00.123Z"), + full("a-prisma", "2026-01-02T10:00:00.123Z"), + ], + [ + sparse("b-bulk", "2026-01-03T10:00:00.000Z"), + sparse("b-prisma", "2026-01-04T10:00:00.000Z"), + ], + ] as const; + + for (const [viaBulk, viaPrisma] of rows) { + await prisma.$transaction((tx) => insertMeasurementRows(tx, [viaBulk])); + await prisma.measurement.create({ data: viaPrisma as never }); + } + + for (const [viaBulk, viaPrisma] of rows) { + const a = await prisma.measurement.findUniqueOrThrow({ + where: { id: viaBulk.id }, + }); + const b = await prisma.measurement.findUniqueOrThrow({ + where: { id: viaPrisma.id }, + }); + const strip = (row: typeof a) => ({ + ...row, + id: undefined, + externalId: row.externalId?.replace(/-(bulk|prisma)$/, ""), + measuredAt: undefined, + // Filled with the current time when the row carries none. + ...(viaBulk.createdAt + ? {} + : { createdAt: undefined, updatedAt: undefined }), + }); + expect(strip(a)).toEqual(strip(b)); + expect(a.measuredAt.toISOString()).toBe(viaBulk.measuredAt.toISOString()); + } + const sparseRow = await prisma.measurement.findUniqueOrThrow({ + where: { id: "b-bulk" }, + }); + expect(Date.now() - sparseRow.createdAt.getTime()).toBeLessThan(60_000); + }); + + it("inserts a whole batch in one statement and reports the count", async () => { + const prisma = getPrismaClient(); + const batch = Array.from({ length: 2_500 }, (_, i) => + sparse( + `m-${i}`, + new Date(Date.UTC(2026, 0, 1) + i * 60_000).toISOString(), + ), + ); + const inserted = await prisma.$transaction((tx) => + insertMeasurementRows(tx, batch), + ); + expect(inserted).toBe(2_500); + expect(await prisma.measurement.count({ where: { userId: USER } })).toBe( + 2_500, + ); + }); +}); diff --git a/tests/integration/offhost-backup-ledger.test.ts b/tests/integration/offhost-backup-ledger.test.ts index f2047fc9a..a4718865c 100644 --- a/tests/integration/offhost-backup-ledger.test.ts +++ b/tests/integration/offhost-backup-ledger.test.ts @@ -228,4 +228,50 @@ describe("off-host backup ledger (real Postgres)", () => { await prisma.user.delete({ where: { id: "account-one" } }); expect(await prisma.offhostBackupState.count()).toBe(0); }); + + it("a retry of the same run resumes after the accounts it already uploaded (#1031)", async () => { + const prisma = getPrismaClient(); + for (const id of ["acct-a", "acct-b", "acct-c"]) await seedUser(id); + const bucket = makeBucket(); + const puts: string[] = []; + const putStream = bucket.putStream; + bucket.putStream = async (key, body) => { + puts.push(key); + return putStream(key, body); + }; + // The job was created a moment ago; its retries share that instant. + const runStartedAt = new Date(Date.now() - 60_000); + const day = runStartedAt.toISOString().slice(0, 10); + + // First attempt: out of budget after one account. + let asked = 0; + const first = await runOffhostBackup(prisma, bucket, new Date(), { + runStartedAt, + shouldStop: () => asked++ >= 1, + }); + expect(first.stoppedEarly).toBe(true); + expect(first.uploaded).toBe(1); + + // The retry: same run, so the account already done is skipped. + const retry = await runOffhostBackup(prisma, bucket, new Date(), { + runStartedAt, + }); + expect(retry.stoppedEarly).toBe(false); + expect(retry.alreadyUploaded).toBe(1); + expect(retry.uploaded).toBe(2); + + // Each account uploaded exactly once, all under the run's own day. + expect(puts.sort()).toEqual([ + `${day}/user-acct-a.json.enc`, + `${day}/user-acct-b.json.enc`, + `${day}/user-acct-c.json.enc`, + ]); + + // A new run (a later job) uploads everybody again. + const next = await runOffhostBackup(prisma, bucket, new Date(), { + runStartedAt: new Date(), + }); + expect(next.alreadyUploaded).toBe(0); + expect(next.uploaded).toBe(3); + }); }); diff --git a/tests/integration/pr-detection-cumulative-paged.test.ts b/tests/integration/pr-detection-cumulative-paged.test.ts new file mode 100644 index 000000000..9bfc38159 --- /dev/null +++ b/tests/integration/pr-detection-cumulative-paged.test.ts @@ -0,0 +1,111 @@ +/** + * #1031 — the best cumulative day, found a day at a time. + * + * `findBestCumulativeDay` read every row of a cumulative type into memory, + * every half hour, for every account. On an account holding 784 000 active + * energy rows it ran a 524 MB heap out of memory. It now reads a local day at + * a time in time order, which gives the same answer because the canonical- + * source pick it applies is per day. + * + * Pinned here at a size that spans several read pages: two sources report + * the same days, the source ladder must decide each day on its own, the + * best day sits in the middle of the history, and no read of the history is + * unbounded. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const { detectPersonalRecordsForUser } = + await import("@/lib/personal-records/pr-detection-worker"); + +const USER = "user-pr-cumulative-paged"; +const DAYS = 40; +const PER_DAY = 300; // 12 000 Apple rows plus a Withings total per day + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: USER, + username: USER, + email: `${USER}@example.test`, + timezone: "UTC", + }, + }); +}); + +describe("best cumulative day across many pages", () => { + it("sums each day's canonical source and finds the best day without an unbounded read", async () => { + const prisma = getPrismaClient(); + const start = Date.UTC(2025, 0, 1, 6); + const rows = []; + for (let day = 0; day < DAYS; day++) { + // Day 23 is the best Apple day; the Withings total on day 5 is larger + // still but loses to Apple on that day by the default ladder, so it + // must not win: if the days were pooled across sources it would. + const perSample = day === 23 ? 40 : 20; + for (let i = 0; i < PER_DAY; i++) { + rows.push({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: perSample, + unit: "steps", + source: "APPLE_HEALTH" as const, + measuredAt: new Date(start + day * 86_400_000 + i * 60_000), + externalId: `hk-${day}-${i}`, + }); + } + rows.push({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: day === 5 ? 50_000 : 1_000, + unit: "steps", + source: "WITHINGS" as const, + measuredAt: new Date(start + day * 86_400_000 + 12 * 3_600_000), + externalId: `w-${day}`, + }); + } + await prisma.measurement.createMany({ data: rows }); + + // Every read of the steps history, through a client that records it. + const reads: Array = []; + const recording = prisma.$extends({ + query: { + measurement: { + findMany({ args, query }) { + if (JSON.stringify(args.where ?? {}).includes("ACTIVITY_STEPS")) { + reads.push(args.take); + } + return query(args); + }, + }, + }, + }); + + await detectPersonalRecordsForUser(USER, { + silent: true, + prisma: recording as unknown as typeof prisma, + }); + + const record = await prisma.personalRecord.findFirstOrThrow({ + where: { userId: USER, metricType: "ACTIVITY_STEPS" }, + }); + expect(record.value).toBe(40 * PER_DAY); + expect(record.achievedAt.toISOString()).toBe( + new Date(start + 23 * 86_400_000 + (PER_DAY - 1) * 60_000).toISOString(), + ); + + // Several bounded pages, never one read of the whole history. + expect(reads.length).toBeGreaterThan(1); + for (const take of reads) { + expect(take).toBeDefined(); + expect(take!).toBeLessThanOrEqual(5_000); + } + }); +});