From 32c3248b57513dd82437e63ce2225b77bc088679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 00:31:30 +0200 Subject: [PATCH 01/12] feat(jobs): log long-running, expired and cut-off jobs by queue A job that ran into its pg-boss expiry left no trace in the app log. Most handlers annotate a wide event nobody emits, pg-boss records an expiry only in its own table, and a worker that dies of heap exhaustion writes one FATAL line that names no queue. On a large account three queues failed with "job timed out" for days and the log said nothing about them (#1031). Every handler now runs inside `observeJob` (through `runJob`), which writes `job.progress` every two minutes while a handler runs (elapsed time, heap, RSS and the last progress the pass reported), `job.expired` when the job passes its expiry while the handler is still running, and `job.finished` when a long or expired run settles. At worker boot, `worker.boot.jobs_cut_off` names every job a previous process was still running when it stopped. The admin card's "job timed out" now says what the limit was and that the cause is either a run too slow for it or a restart while it ran. --- src/lib/jobs/__tests__/job-observer.test.ts | 200 +++++++++++++++++++ src/lib/jobs/__tests__/run-job.test.ts | 22 ++ src/lib/jobs/job-failures.ts | 111 +++++++++- src/lib/jobs/job-observer.ts | 211 ++++++++++++++++++++ src/lib/jobs/reminder-worker.ts | 5 + src/lib/jobs/run-job.ts | 9 +- tests/integration/job-failures.test.ts | 61 +++++- 7 files changed, 611 insertions(+), 8 deletions(-) create mode 100644 src/lib/jobs/__tests__/job-observer.test.ts create mode 100644 src/lib/jobs/job-observer.ts diff --git a/src/lib/jobs/__tests__/job-observer.test.ts b/src/lib/jobs/__tests__/job-observer.test.ts new file mode 100644 index 000000000..9e7c20920 --- /dev/null +++ b/src/lib/jobs/__tests__/job-observer.test.ts @@ -0,0 +1,200 @@ +/** + * Issue #1031: three queues failed with `job timed out` night after night and + * the operator found no line about any of them in the app log. These tests + * pin the lines `observeJob` now writes — progress while a job runs, a warning + * at its expiry, and a closing line for a long or expired run — and the boot + * line naming jobs a previous process was running when it stopped. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + emitted: [] as Array<{ + level: string; + background?: { task_name?: string }; + meta?: Record; + }>, + active: vi.fn(), +})); + +vi.mock("@/lib/logging/transports", () => ({ + emitIfSampled: (event: (typeof mocks.emitted)[number]) => { + mocks.emitted.push(event); + }, +})); +vi.mock("@/lib/jobs/job-failures", () => ({ + readActiveJobsStartedBefore: mocks.active, +})); + +import { + JOB_LONG_RUN_MS, + JOB_PROGRESS_INTERVAL_MS, + observeJob, + reportJobProgress, + reportJobsCutOffAtBoot, + type JobObserverClock, +} from "@/lib/jobs/job-observer"; + +/** A manual clock: `advance` fires due timers in order. */ +function manualClock() { + let now = 0; + let seq = 0; + const timers = new Map< + number, + { at: number; every: number | null; fn: () => void } + >(); + const clock: JobObserverClock = { + now: () => now, + setTimeout: (fn, ms) => { + seq += 1; + timers.set(seq, { at: now + ms, every: null, fn }); + return seq; + }, + setInterval: (fn, ms) => { + seq += 1; + timers.set(seq, { at: now + ms, every: ms, fn }); + return seq; + }, + clearTimeout: (handle) => timers.delete(handle as number), + clearInterval: (handle) => timers.delete(handle as number), + }; + const advance = (ms: number) => { + const target = now + ms; + for (;;) { + const due = [...timers.entries()] + .filter(([, t]) => t.at <= target) + .sort((a, b) => a[1].at - b[1].at)[0]; + if (!due) break; + const [id, timer] = due; + now = timer.at; + if (timer.every === null) timers.delete(id); + else timer.at += timer.every; + timer.fn(); + } + now = target; + }; + return { clock, advance }; +} + +function lines(task: string) { + return mocks.emitted.filter((e) => e.background?.task_name === task); +} + +beforeEach(() => { + mocks.emitted.length = 0; + mocks.active.mockReset(); +}); + +describe("observeJob", () => { + it("writes progress while a long job runs, a warning at its expiry, and a closing line", async () => { + const { clock, advance } = manualClock(); + let release!: (value: { ok: true; did: { days: number } }) => void; + + const running = observeJob( + "dense-intraday-retention", + [{ id: "job-1", expireInSeconds: 900 }], + () => + new Promise<{ ok: true; did: { days: number } }>((resolve) => { + reportJobProgress({ consolidation_day: "2024-03-01" }); + release = resolve; + }), + clock, + ); + + advance(JOB_PROGRESS_INTERVAL_MS); + expect(lines("job.progress")).toHaveLength(1); + expect(lines("job.progress")[0]!.meta).toMatchObject({ + queue: "dense-intraday-retention", + job_ids: "job-1", + expire_s: 900, + consolidation_day: "2024-03-01", + }); + expect(lines("job.progress")[0]!.meta?.heap_used_mb).toEqual( + expect.any(Number), + ); + + advance(900 * 1000 - JOB_PROGRESS_INTERVAL_MS); + const expired = lines("job.expired"); + expect(expired).toHaveLength(1); + expect(expired[0]!.level).toBe("warn"); + expect(expired[0]!.meta).toMatchObject({ + queue: "dense-intraday-retention", + elapsed_s: 900, + consolidation_day: "2024-03-01", + }); + + release({ ok: true, did: { days: 12 } }); + await running; + const finished = lines("job.finished"); + expect(finished).toHaveLength(1); + expect(finished[0]!.level).toBe("warn"); + expect(finished[0]!.meta).toMatchObject({ + queue: "dense-intraday-retention", + finished_after_expiry: true, + outcome: "completed", + did: { days: 12 }, + }); + }); + + it("stays silent for a short job", async () => { + const { clock } = manualClock(); + await observeJob( + "host-metric-sample", + [{ id: "j", expireInSeconds: 900 }], + async () => ({ ok: true }), + clock, + ); + expect(mocks.emitted).toEqual([]); + }); + + it("closes a long job that threw with the error, and rethrows it", async () => { + const { clock, advance } = manualClock(); + const running = observeJob( + "data-backup", + [{ id: "b", expireInSeconds: 7200 }], + async () => { + advance(JOB_LONG_RUN_MS); + throw new Error("write failed"); + }, + clock, + ); + await expect(running).rejects.toThrow("write failed"); + expect(lines("job.finished")[0]!.meta).toMatchObject({ + queue: "data-backup", + outcome: "threw", + error: "write failed", + }); + expect(lines("job.expired")).toEqual([]); + }); +}); + +describe("reportJobsCutOffAtBoot", () => { + it("names each job a previous process was running", async () => { + mocks.active.mockResolvedValue([ + { + queue: "data-backup", + id: "a", + startedAt: "2026-09-24T09:36:00.000Z", + expireSeconds: 7200, + }, + ]); + await reportJobsCutOffAtBoot(new Date("2026-09-24T10:00:00.000Z")); + const line = lines("worker.boot.jobs_cut_off"); + expect(line).toHaveLength(1); + expect(line[0]!.level).toBe("warn"); + expect(line[0]!.meta).toMatchObject({ + jobs_cut_off: 1, + queues: "data-backup", + }); + expect(String(line[0]!.meta?.jobs)).toContain( + "data-backup@2026-09-24T09:36:00.000Z", + ); + }); + + it("stays silent when nothing was cut off, or there is no queue schema", async () => { + mocks.active.mockResolvedValueOnce([]); + await reportJobsCutOffAtBoot(); + mocks.active.mockResolvedValueOnce(null); + await reportJobsCutOffAtBoot(); + expect(mocks.emitted).toEqual([]); + }); +}); diff --git a/src/lib/jobs/__tests__/run-job.test.ts b/src/lib/jobs/__tests__/run-job.test.ts index 4558ccea9..9b32e591f 100644 --- a/src/lib/jobs/__tests__/run-job.test.ts +++ b/src/lib/jobs/__tests__/run-job.test.ts @@ -149,3 +149,25 @@ describe("runJob", () => { expect(handler).toHaveBeenCalledWith(jobs); }); }); + +describe("runJob observation (#1031)", () => { + it("runs every handler inside the job observer, so a long or expired job leaves a log line", async () => { + const observer = await import("../job-observer"); + const observe = vi.spyOn(observer, "observeJob"); + const jobs = [ + { id: "j-1", expireInSeconds: 900 }, + ] as unknown as Job[]; + const wrapped = runJob("dense-intraday-retention", async () => + jobDone({ days_consolidated: 1 }), + ); + + await wrapped(jobs); + + expect(observe).toHaveBeenCalledWith( + "dense-intraday-retention", + jobs, + expect.any(Function), + ); + observe.mockRestore(); + }); +}); diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index d846c88be..6406e320c 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -66,6 +66,35 @@ interface FailingQueueRow { failures: number; last_failed_at: Date | null; last_error: string | null; + last_expire_seconds: number | null; +} + +/** + * The two texts pg-boss writes when a job ran out of time: the supervisor's, + * for a job still `active` past its expiry, and the worker's own, for a + * handler that did not settle inside it. + */ +const TIMEOUT_MESSAGE = /^(job timed out|handler execution exceeded \d+s)$/; + +/** + * Say what a bare timeout means. "job timed out" alone sent an operator + * nowhere: pg-boss writes the same words for a pass that was too slow for its + * limit and for a job whose process died under it (a restart, or a container + * killed for memory), in which case the job sits `active` until the limit runs + * out. The limit and the two causes are what the operator can check. + */ +function explainTimeout(message: string, expireSeconds: number | null): string { + const limit = + expireSeconds !== null && expireSeconds > 0 + ? expireSeconds >= 3600 && expireSeconds % 3600 === 0 + ? `${expireSeconds / 3600} h` + : `${Math.round(expireSeconds / 60)} min` + : null; + return ( + `${message}: did not finish within its ${limit ?? "time"} limit. ` + + `Either the run was too slow for the limit, or the app restarted while it ` + + `ran; a restart shows in the container's restart count and log.` + ); } /** @@ -73,10 +102,15 @@ interface FailingQueueRow { * serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at * all, so the raw text is the fallback rather than an invented placeholder. */ -function presentError(raw: string | null): string { +function presentError( + raw: string | null, + expireSeconds: number | null = null, +): string { if (raw === null) return ""; const trimmed = raw.trim(); if (trimmed.length === 0) return ""; + if (TIMEOUT_MESSAGE.test(trimmed)) + return explainTimeout(trimmed, expireSeconds); const redacted = redactSecrets(trimmed); return redacted.length > MAX_ERROR_CHARS ? `${redacted.slice(0, MAX_ERROR_CHARS)}…` @@ -105,7 +139,10 @@ export async function readFailingQueues( ) ORDER BY completed_on DESC NULLS LAST ) - )[1] AS last_error + )[1] AS last_error, + ( + array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST) + )[1] AS last_expire_seconds FROM pgboss.job WHERE state = 'failed' AND completed_on > now() - make_interval(hours => ${withinHours}::int) @@ -119,7 +156,7 @@ export async function readFailingQueues( queue: row.name, failures: row.failures, lastFailedAt: row.last_failed_at!.toISOString(), - lastError: presentError(row.last_error), + lastError: presentError(row.last_error, row.last_expire_seconds), })); } catch { // No `pgboss` schema (web-only deployment) or no permission to read it. @@ -160,11 +197,17 @@ export async function readLastQueueRun( ): Promise { try { const rows = await prisma.$queryRaw< - Array<{ state: string; completed_on: Date | null; output: string | null }> + Array<{ + state: string; + completed_on: Date | null; + output: string | null; + expire_seconds: number | null; + }> >` SELECT state, completed_on, + expire_seconds, -- pg-boss wraps a thrown value: an expired job's row reads -- {"value":{"message":"job timed out"}}, so a plain ->>'message' -- misses it and the operator gets raw JSON instead of the sentence. @@ -189,7 +232,10 @@ export async function readLastQueueRun( return { at: row.completed_on.toISOString(), state, - error: state === "completed" ? null : presentError(row.output), + error: + state === "completed" + ? null + : presentError(row.output, row.expire_seconds), }; } catch { // No `pgboss` schema (web-only deployment) or no permission to read it. @@ -238,3 +284,58 @@ export async function readQueueFailureForUser( return null; } } + +/** A job pg-boss still has as running, from `readActiveJobsStartedBefore`. */ +export interface ActiveJob { + queue: string; + id: string; + /** ISO instant the job was picked up. */ + startedAt: string; + /** The job's expiry, in seconds, as pg-boss holds it. */ + expireSeconds: number; +} + +/** + * Jobs still marked active that were picked up before `before` — at worker + * boot, before this process has taken any job, that is every job a previous + * process was running when it stopped. + * + * Why it matters (#1031): a worker killed for memory writes nothing, and + * pg-boss only marks its job `job timed out` once the expiry has passed, + * which for the backup is two hours later. Reading these rows at boot turns + * a silent restart into a line naming what was cut off. With more than one + * worker process on the same database, a job another live worker is running + * would show up here too, which the boot line says. + * + * `null` when the queue schema is absent, like the readers above. + */ +export async function readActiveJobsStartedBefore( + before: Date, +): Promise { + try { + const rows = await prisma.$queryRaw< + Array<{ + name: string; + id: string; + started_on: Date | null; + expire_seconds: number; + }> + >` + SELECT name, id::text AS id, started_on, expire_seconds + FROM pgboss.job + WHERE state = 'active' AND started_on < ${before} + ORDER BY started_on + LIMIT ${MAX_QUEUES} + `; + return rows + .filter((row) => row.started_on !== null) + .map((row) => ({ + queue: row.name, + id: row.id, + startedAt: row.started_on!.toISOString(), + expireSeconds: row.expire_seconds, + })); + } catch { + return null; + } +} diff --git a/src/lib/jobs/job-observer.ts b/src/lib/jobs/job-observer.ts new file mode 100644 index 000000000..18c4166d6 --- /dev/null +++ b/src/lib/jobs/job-observer.ts @@ -0,0 +1,211 @@ +/** + * Log lines for long-running and cut-off jobs. + * + * Issue #1031: on a large account three queues failed night after night with + * `job timed out`, and the operator found not one line about any of them in + * the app log. Nothing wrote one. Most handlers annotate a wide event that + * nobody emits, pg-boss records an expiry only in its own table, and a worker + * killed for memory cannot say anything at all. The admin card showed the + * failure; the log showed nothing about what the job was doing, how far it + * got, or why it stopped. + * + * `observeJob` wraps every handler (through `runJob`) and writes three kinds + * of line, each a background wide event naming the queue: + * + * - `job.progress`, every two minutes while a handler runs, with how long it + * has been running, the process's memory, and whatever the handler last + * reported through `reportJobProgress`. A job that runs for a minute never + * writes one; a job that is killed leaves its last one behind. + * - `job.expired`, at the job's expiry if the handler has not settled: the + * moment pg-boss declares it timed out. pg-boss does not stop the handler, + * so this line says it is still running and how far it has got. + * - `job.finished`, when a handler that ran long or past its expiry + * settles, with its outcome and the facts it returned. + * + * `reportOrphanedJobs` covers the case no running handler can: at worker boot, + * jobs still marked active were started by a process that is gone. + */ +import { AsyncLocalStorage } from "node:async_hooks"; + +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { emitIfSampled } from "@/lib/logging/transports"; +import { readActiveJobsStartedBefore } from "@/lib/jobs/job-failures"; + +/** How often a running job writes `job.progress`. */ +export const JOB_PROGRESS_INTERVAL_MS = 2 * 60 * 1000; + +/** A handler that ran at least this long writes `job.finished`. */ +export const JOB_LONG_RUN_MS = 60 * 1000; + +type ProgressFacts = Record; + +const progressStore = new AsyncLocalStorage<{ facts: ProgressFacts }>(); + +/** + * Record how far the running job has got. Cheap: it only replaces the facts + * the next `job.progress` / `job.expired` line carries, so a pass can call it + * once per unit of work. A no-op outside an observed job. + */ +export function reportJobProgress(facts: ProgressFacts): void { + const cell = progressStore.getStore(); + if (cell) cell.facts = { ...cell.facts, ...facts }; +} + +/** What `observeJob` reads from a pg-boss job. */ +interface ObservedJob { + id?: string; + expireInSeconds?: number; +} + +/** Injectable clock and timers, for tests. */ +export interface JobObserverClock { + now: () => number; + setTimeout: (fn: () => void, ms: number) => unknown; + setInterval: (fn: () => void, ms: number) => unknown; + clearTimeout: (handle: unknown) => void; + clearInterval: (handle: unknown) => void; +} + +const systemClock: JobObserverClock = { + now: () => Date.now(), + setTimeout: (fn, ms) => setTimeout(fn, ms).unref(), + setInterval: (fn, ms) => setInterval(fn, ms).unref(), + clearTimeout: (handle) => clearTimeout(handle as NodeJS.Timeout), + clearInterval: (handle) => clearInterval(handle as NodeJS.Timeout), +}; + +function megabytes(bytes: number): number { + return Math.round(bytes / 1024 / 1024); +} + +function emitJobLine( + taskName: string, + level: "info" | "warn", + meta: Record, +): void { + try { + const event = new WideEventBuilder("background"); + event.setBackground({ task_name: taskName }); + for (const [key, value] of Object.entries(meta)) event.addMeta(key, value); + if (level === "warn") event.elevateLevel("warn"); + event.finish(); + emitIfSampled(event.toJSON()); + } catch { + // A log line must never break the job it describes. + } +} + +/** + * Run `run` for `jobs` of `queue`, writing the progress, expiry and finish + * lines described above. Resolves or rejects exactly as `run` does. + */ +export async function observeJob( + queue: string, + jobs: readonly ObservedJob[], + run: () => Promise, + clock: JobObserverClock = systemClock, +): Promise { + const startedAt = clock.now(); + const cell: { facts: ProgressFacts } = { facts: {} }; + const jobIds = jobs + .map((job) => job.id) + .filter(Boolean) + .join(","); + const expireSeconds = Math.min( + ...jobs.map((job) => + typeof job.expireInSeconds === "number" && job.expireInSeconds > 0 + ? job.expireInSeconds + : Number.POSITIVE_INFINITY, + ), + ); + let expired = false; + + const snapshot = () => { + const memory = process.memoryUsage(); + return { + queue, + job_ids: jobIds, + elapsed_s: Math.round((clock.now() - startedAt) / 1000), + ...(Number.isFinite(expireSeconds) ? { expire_s: expireSeconds } : {}), + heap_used_mb: megabytes(memory.heapUsed), + rss_mb: megabytes(memory.rss), + ...cell.facts, + }; + }; + + const heartbeat = clock.setInterval(() => { + emitJobLine("job.progress", "info", snapshot()); + }, JOB_PROGRESS_INTERVAL_MS); + const expiry = Number.isFinite(expireSeconds) + ? clock.setTimeout(() => { + expired = true; + emitJobLine("job.expired", "warn", { + ...snapshot(), + reason: + "pg-boss has marked this job timed out; the handler is still running", + }); + }, expireSeconds * 1000) + : null; + + const finish = (outcome: Record) => { + clock.clearInterval(heartbeat); + if (expiry !== null) clock.clearTimeout(expiry); + const elapsed = clock.now() - startedAt; + if (!expired && elapsed < JOB_LONG_RUN_MS) return; + emitJobLine("job.finished", expired ? "warn" : "info", { + ...snapshot(), + ...(expired ? { finished_after_expiry: true } : {}), + ...outcome, + }); + }; + + try { + const result = await progressStore.run(cell, run); + const did = + result !== null && typeof result === "object" && "did" in result + ? (result as { did?: unknown }).did + : undefined; + const ok = + result !== null && typeof result === "object" && "ok" in result + ? (result as { ok?: unknown }).ok !== false + : true; + finish({ + outcome: ok ? "completed" : "failed", + ...(did && typeof did === "object" ? { did } : {}), + }); + return result; + } catch (err) { + finish({ + outcome: "threw", + error: err instanceof Error ? err.message : String(err), + }); + throw err; + } +} + +/** + * At worker boot, write one line naming every job a previous process was + * running when it stopped. Must run before this process takes any job. + * + * A worker killed for memory cannot log its own death, and pg-boss marks the + * job `job timed out` only once the expiry has passed. This line is where a + * restart in the middle of a job becomes visible: the queue, when the job + * started, and its limit. Silent when nothing was cut off. + */ +export async function reportJobsCutOffAtBoot( + bootedAt: Date = new Date(), +): Promise { + const active = await readActiveJobsStartedBefore(bootedAt); + if (active === null || active.length === 0) return; + emitJobLine("worker.boot.jobs_cut_off", "warn", { + jobs_cut_off: active.length, + queues: [...new Set(active.map((job) => job.queue))].join(","), + jobs: active + .map( + (job) => `${job.queue}@${job.startedAt}(limit ${job.expireSeconds}s)`, + ) + .join(" "), + reason: + "these jobs were still running when the previous worker process stopped (a restart, a deploy, or the container running out of memory); pg-boss retries or fails them once their limit passes. If another worker process shares this database, its running jobs are listed too.", + }); +} diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index f8a20a384..e3e732d2c 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -16,6 +16,7 @@ */ import { markWorkerStarted, recordError } from "@/lib/jobs/worker-status"; import { createWorkerBoss, setGlobalBoss } from "@/lib/jobs/boss-instance"; +import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer"; import { assertSubsystemEnabled } from "@/lib/process-type"; import { DATABASE_URL, workerLog } from "./reminder/shared"; import { @@ -55,7 +56,11 @@ export async function startReminderWorker() { recordError(); }); + // Read before any `boss.work` binding below can take a job: whatever is + // still active now was started by a process that is gone. + const bootedAt = new Date(); await boss.start(); + await reportJobsCutOffAtBoot(bootedAt); setGlobalBoss(boss); markWorkerStarted(); diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts index 980bc5a1d..9a673bc8c 100644 --- a/src/lib/jobs/run-job.ts +++ b/src/lib/jobs/run-job.ts @@ -28,6 +28,7 @@ import { type SerializedJobOutcome, } from "./job-outcome"; import { reportWorkerError } from "./report-worker-error"; +import { observeJob } from "./job-observer"; /** A converted handler: it says what it did instead of just finishing. */ export type JobHandler = (jobs: Job[]) => Promise; @@ -80,7 +81,13 @@ export function runJob( handler: (jobs: J[]) => Promise, ): (jobs: J[]) => Promise { return async (jobs: J[]): Promise => { - const outcome = await handler(jobs); + // Observed, so a long run, an expiry and a cut-off leave a line in the + // log naming the queue and how far it got (see `job-observer.ts`). + const outcome = await observeJob( + queue, + jobs as ReadonlyArray<{ id?: string; expireInSeconds?: number }>, + () => handler(jobs), + ); if (outcome.ok) return serializeJobOutcome(outcome); const failure = new JobFailure(queue, outcome.reason, outcome.cause); diff --git a/tests/integration/job-failures.test.ts b/tests/integration/job-failures.test.ts index 18a6f6812..a4c108986 100644 --- a/tests/integration/job-failures.test.ts +++ b/tests/integration/job-failures.test.ts @@ -16,6 +16,7 @@ import { PgBoss } from "pg-boss"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { + readActiveJobsStartedBefore, readFailingQueues, readQueueFailureForUser, } from "@/lib/jobs/job-failures"; @@ -28,8 +29,16 @@ const OTHER_QUEUE = "reminder-check"; let boss: PgBoss; /** Send a job, take it, and fail it terminally so it lands in `state='failed'`. */ -async function failOneJob(queue: string, data: object, error: string) { - const jobId = await boss.send(queue, data, { retryLimit: 0 }); +async function failOneJob( + queue: string, + data: object, + error: string, + expireInSeconds?: number, +) { + const jobId = await boss.send(queue, data, { + retryLimit: 0, + ...(expireInSeconds === undefined ? {} : { expireInSeconds }), + }); if (!jobId) throw new Error("queue rejected the job"); const [job] = await boss.fetch(queue); if (!job) throw new Error("nothing to fetch"); @@ -81,6 +90,26 @@ describe("readFailingQueues", () => { expect(other?.lastError).toBe("reminder dispatch failed"); }); + it("says what a bare timeout means, with the job's own limit (#1031)", async () => { + await getPrismaClient().$executeRawUnsafe( + `DELETE FROM pgboss.job WHERE state = 'failed'`, + ); + // pg-boss writes these words both for a pass too slow for its limit and + // for a job whose process died under it; alone they sent an operator + // nowhere. + await failOneJob(QUEUE, {}, "job timed out", 900); + await failOneJob(OTHER_QUEUE, {}, "job timed out", 7200); + + const failing = await readFailingQueues(); + const fifteen = failing?.find((entry) => entry.queue === QUEUE); + expect(fifteen?.lastError).toMatch( + /^job timed out: did not finish within its 15 min limit\./, + ); + expect(fifteen?.lastError).toMatch(/restarted/); + const twoHours = failing?.find((entry) => entry.queue === OTHER_QUEUE); + expect(twoHours?.lastError).toMatch(/within its 2 h limit/); + }); + it("reports an empty list — not null — when nothing is failing", async () => { await getPrismaClient().$executeRawUnsafe( `DELETE FROM pgboss.job WHERE state = 'failed'`, @@ -132,6 +161,34 @@ describe("readQueueFailureForUser", () => { }); }); +describe("readActiveJobsStartedBefore (#1031)", () => { + it("lists a job still marked active from before the given instant, and nothing after it", async () => { + const jobId = await boss.send( + OTHER_QUEUE, + {}, + { retryLimit: 0, expireInSeconds: 7200 }, + ); + const [job] = await boss.fetch(OTHER_QUEUE); + expect(job?.id).toBe(jobId); + + const cutOff = await readActiveJobsStartedBefore( + new Date(Date.now() + 60_000), + ); + expect(cutOff).toContainEqual( + expect.objectContaining({ + queue: OTHER_QUEUE, + id: jobId, + expireSeconds: 7200, + }), + ); + await expect( + readActiveJobsStartedBefore(new Date(Date.now() - 60 * 60_000)), + ).resolves.toEqual([]); + + await boss.complete(OTHER_QUEUE, jobId!); + }); +}); + describe("no queue schema at all", () => { it("answers null rather than throwing, so the status page still renders", async () => { const prisma = getPrismaClient(); From f4b51cf13466be8d45688d2c8aa2b0349494a6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 00:31:40 +0200 Subject: [PATCH 02/12] perf(consolidation): fold a day at a time and batch the hourly writes The consolidation drains (cumulative, daily mean, dense intra-day retention, legacy steps) gathered a type's whole history into one array before folding the first day. On an account of 1.25 million rows the heart-rate array alone held about 230 MB, and the dense retention peaked at 460 MB of heap against a 524 MB limit; in a 1 GB container running the web server and the worker together the process died of heap exhaustion within a minute and the job was later marked "job timed out" (#1031). - Pages are now handed on one at a time and each local day is folded as soon as its last row has been read, so a pass holds one page and one day. - The keyset cursor gains a redundant `measuredAt >= cursor` bound. Without it Postgres could not use the cursor as an index condition and re-read every earlier row on each page (47 ms and 248 000 filtered rows for one page halfway through the heart-rate history; 5 ms now). - The dense fold resolves a day's hourly rows with one lookup and one insert instead of two lookups and a write per hour. On the seeded account the pass went from about 290 000 statements to 54 000. - `shouldStop` lets a caller end a pass between days; the result says `stoppedEarly`. Days commit one by one and a folded day leaves the scan, so the next run continues at the first day not reached. --- .../adopt-or-mint-hourly-rows.test.ts | 331 +++++++++++++++++ .../consolidation-scan-paged.test.ts | 346 ++++++++++++------ .../dense-intraday-hourly-rebuild.test.ts | 7 + .../dense-intraday-pulse-summary.test.ts | 7 + .../dense-intraday-resting-mint.test.ts | 11 +- .../dense-intraday-retention.test.ts | 12 +- .../dense-intraday-spo2-summary.test.ts | 7 + .../__tests__/hourly-mint-mock.ts | 49 +++ .../measurements/consolidate-daily-mean.ts | 19 +- src/lib/measurements/consolidation-base.ts | 199 +++++++--- .../dense-intraday-hourly-rebuild.ts | 22 +- .../measurements/dense-intraday-retention.ts | 246 ++++++++----- .../drain-per-sample-cumulative.ts | 17 +- 13 files changed, 992 insertions(+), 281 deletions(-) create mode 100644 src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts create mode 100644 src/lib/measurements/__tests__/hourly-mint-mock.ts diff --git a/src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts b/src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts new file mode 100644 index 000000000..a0396d9f1 --- /dev/null +++ b/src/lib/measurements/__tests__/adopt-or-mint-hourly-rows.test.ts @@ -0,0 +1,331 @@ +/** + * `adoptOrMintHourlyRows` resolves a whole folded day in one lookup and one + * insert. It replaced a per-slot loop (two lookups and a write per hour, about + * 75 statements a day), which was most of the 380 000 statements the dense + * retention spent on a 1.25-million-row account (issue #1031). + * + * Two things are pinned here. The statement shape: one read, at most one + * insert, no per-slot lookups. And the semantics: against an in-memory table + * that enforces both unique identities, the batch leaves exactly the rows the + * per-slot loop leaves and returns the same ids, over randomised days that + * include rows already sitting on an anchor, rows already carrying a target + * externalId, tombstones, and a row holding one hour's externalId while + * parked on another hour's anchor. + */ +import { describe, expect, it, vi } from "vitest"; + +import type { MeasurementType, Prisma } from "@/generated/prisma/client"; + +import { + adoptOrMintHourlyRows, + type HourlySlot, +} from "../dense-intraday-retention"; + +interface Row { + id: string; + externalId: string | null; + measuredAt: Date; + value: number; + unit: string; + deletedAt: Date | null; +} + +/** + * A measurement table for one (user, type, APPLE_HEALTH), with the two unique + * identities the real one has: externalId and measuredAt (sleepStage is NULL + * for these types). A write that would duplicate either throws, as P2002 does. + */ +function fakeTable(seed: Row[]) { + const rows = seed.map((row) => ({ ...row })); + let minted = 0; + const calls = { findMany: 0, findFirst: 0, createMany: 0, create: 0 }; + + const assertUnique = () => { + const ext = rows + .filter((r) => r.externalId !== null) + .map((r) => r.externalId); + const at = rows.map((r) => r.measuredAt.getTime()); + if (new Set(ext).size !== ext.length) throw new Error("P2002 externalId"); + if (new Set(at).size !== at.length) throw new Error("P2002 measuredAt"); + }; + const mint = (data: { + externalId: string; + measuredAt: Date; + value: number; + unit: string; + }) => { + minted += 1; + const row: Row = { + id: `z-minted-${String(minted).padStart(3, "0")}`, + externalId: data.externalId, + measuredAt: data.measuredAt, + value: data.value, + unit: data.unit, + deletedAt: null, + }; + rows.push(row); + assertUnique(); + return row; + }; + const byId = () => [...rows].sort((a, b) => (a.id < b.id ? -1 : 1)); + + const tx = { + measurement: { + findMany: vi.fn( + async (args: { + where: { + OR: [ + { externalId: { in: string[] } }, + { measuredAt: { in: Date[] } }, + ]; + }; + }) => { + calls.findMany += 1; + const ext = new Set(args.where.OR[0].externalId.in); + const at = new Set( + args.where.OR[1].measuredAt.in.map((d) => d.getTime()), + ); + return byId() + .filter( + (r) => + (r.externalId !== null && ext.has(r.externalId)) || + at.has(r.measuredAt.getTime()), + ) + .map((r) => ({ + id: r.id, + externalId: r.externalId, + measuredAt: r.measuredAt, + })); + }, + ), + findFirst: vi.fn( + async (args: { where: { externalId?: string; measuredAt?: Date } }) => { + calls.findFirst += 1; + const hit = byId().find((r) => + args.where.externalId !== undefined + ? r.externalId === args.where.externalId + : r.measuredAt.getTime() === args.where.measuredAt!.getTime(), + ); + return hit ? { id: hit.id } : null; + }, + ), + update: vi.fn( + async (args: { + where: { id: string }; + data: Partial & { measuredAt?: Date }; + }) => { + const row = rows.find((r) => r.id === args.where.id)!; + Object.assign(row, args.data); + assertUnique(); + return row; + }, + ), + create: vi.fn( + async (args: { + data: { + externalId: string; + measuredAt: Date; + value: number; + unit: string; + }; + }) => { + calls.create += 1; + return { id: mint(args.data).id }; + }, + ), + createManyAndReturn: vi.fn( + async (args: { + data: Array<{ + externalId: string; + measuredAt: Date; + value: number; + unit: string; + }>; + }) => { + calls.createMany += 1; + return args.data.map((d) => { + const row = mint(d); + return { id: row.id, externalId: row.externalId }; + }); + }, + ), + }, + }; + return { + tx: tx as unknown as Prisma.TransactionClient, + rows, + calls, + }; +} + +/** + * The per-slot loop the batch replaced, kept verbatim in behaviour as the + * reference: externalId first, then the anchor occupant, lowest id on a tie; + * re-anchor only onto a free slot; otherwise mint. + */ +async function perSlotReference( + tx: Prisma.TransactionClient, + unit: string, + slots: readonly HourlySlot[], +): Promise { + const ids: string[] = []; + for (const slot of slots) { + const eidRow = await tx.measurement.findFirst({ + where: { externalId: slot.externalId }, + } as Prisma.MeasurementFindFirstArgs); + const slotRow = await tx.measurement.findFirst({ + where: { measuredAt: slot.anchor }, + } as Prisma.MeasurementFindFirstArgs); + const target = eidRow ?? slotRow; + if (target) { + const free = slotRow === null || slotRow.id === target.id; + await tx.measurement.update({ + where: { id: target.id }, + data: { + value: slot.value, + unit, + externalId: slot.externalId, + deletedAt: null, + ...(free ? { measuredAt: slot.anchor } : {}), + }, + }); + ids.push(target.id); + } else { + const created = await tx.measurement.create({ + data: { + value: slot.value, + unit, + measuredAt: slot.anchor, + externalId: slot.externalId, + }, + } as Prisma.MeasurementCreateArgs); + ids.push(created.id); + } + } + return ids; +} + +const HOUR = 3_600_000; +const DAY0 = Date.UTC(2026, 4, 1, 0, 30); +const anchor = (h: number) => new Date(DAY0 + h * HOUR); +const ext = (h: number) => `stats:HK:2026-05-01T${String(h).padStart(2, "0")}`; + +/** A deterministic pseudo-random generator, so a failure reproduces. */ +function rng(seed: number) { + let x = seed; + return () => { + x = (x * 1103515245 + 12345) % 2 ** 31; + return x / 2 ** 31; + }; +} + +function scenario(seed: number): { seed: Row[]; slots: HourlySlot[] } { + const rand = rng(seed); + const hours = [...Array(24).keys()].filter(() => rand() < 0.7); + const slots = hours.map((h) => ({ + externalId: ext(h), + anchor: anchor(h), + value: Math.round(rand() * 100), + })); + const rows: Row[] = []; + const takenAt = new Set(); + const takenExt = new Set(); + let n = 0; + const add = (externalId: string | null, at: Date) => { + if (takenAt.has(at.getTime())) return; + if (externalId !== null && takenExt.has(externalId)) return; + takenAt.add(at.getTime()); + if (externalId !== null) takenExt.add(externalId); + n += 1; + rows.push({ + id: `r-${String(Math.floor(rand() * 900) + 100)}-${n}`, + externalId, + measuredAt: at, + value: 1, + unit: "bpm", + deletedAt: rand() < 0.3 ? new Date(DAY0) : null, + }); + }; + for (const h of hours) { + const roll = rand(); + if (roll < 0.15) + add(ext(h), anchor(h)); // a prior fold's own row + else if (roll < 0.3) + add(`hk-sample-${h}`, anchor(h)); // a sample on the anchor + else if (roll < 0.4) + add(ext(h), new Date(DAY0 + h * HOUR + 60_000)); // stats row off-anchor + else if (roll < 0.5) { + // This hour's externalId parked on ANOTHER hour's anchor. + const other = hours[Math.floor(rand() * hours.length)]!; + add(ext(h), anchor(other)); + } + } + return { seed: rows, slots }; +} + +function snapshot(rows: Row[]) { + return [...rows] + .map((r) => ({ + id: r.id.startsWith("z-minted") ? "minted" : r.id, + externalId: r.externalId, + at: r.measuredAt.toISOString(), + value: r.value, + deleted: r.deletedAt !== null, + })) + .sort((a, b) => + `${a.externalId}|${a.at}` < `${b.externalId}|${b.at}` ? -1 : 1, + ); +} + +describe("adoptOrMintHourlyRows", () => { + it("resolves a fresh day with one lookup and one insert, no per-slot reads", async () => { + const { tx, calls, rows } = fakeTable([]); + const slots = [...Array(24).keys()].map((h) => ({ + externalId: ext(h), + anchor: anchor(h), + value: h, + })); + + const ids = await adoptOrMintHourlyRows(tx, { + userId: "u", + type: "PULSE" as MeasurementType, + unit: "bpm", + slots, + }); + + expect(calls).toEqual({ + findMany: 1, + findFirst: 0, + createMany: 1, + create: 0, + }); + expect(rows).toHaveLength(24); + expect(ids).toEqual( + slots.map((s) => rows.find((r) => r.externalId === s.externalId)!.id), + ); + }); + + it("leaves the same rows and returns the same ids as the per-slot loop", async () => { + for (let seed = 1; seed <= 300; seed++) { + const { seed: rows, slots } = scenario(seed); + + const batch = fakeTable(rows); + const batchIds = await adoptOrMintHourlyRows(batch.tx, { + userId: "u", + type: "PULSE" as MeasurementType, + unit: "bpm", + slots, + }); + + const reference = fakeTable(rows); + const referenceIds = await perSlotReference(reference.tx, "bpm", slots); + + const norm = (ids: string[]) => + ids.map((id) => (id.startsWith("z-minted") ? "minted" : id)); + expect(norm(batchIds), `seed ${seed}`).toEqual(norm(referenceIds)); + expect(snapshot(batch.rows), `seed ${seed}`).toEqual( + snapshot(reference.rows), + ); + } + }); +}); diff --git a/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts b/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts index ea8bf0563..1add7ea76 100644 --- a/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts +++ b/src/lib/measurements/__tests__/consolidation-scan-paged.test.ts @@ -1,31 +1,50 @@ import { describe, expect, it, vi } from "vitest"; -import { bucketRowsByDay, scanSourceRowsPaged } from "../consolidation-base"; +import { + bucketRowsByDay, + iterateDayBuckets, + iterateSourcePages, + runConsolidation, + type ConsolidationParams, +} from "../consolidation-base"; import type { PerSampleRow } from "../consolidation-tz"; -import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import type { + MeasurementType, + Prisma, + PrismaClient, +} from "@/generated/prisma/client"; -// `scanSourceRowsPaged` keyset-paginates the per-(user, type) source scan so a -// multi-year tenant no longer materialises its whole history in one Prisma -// result set. These tests pin the two properties that matter: the accumulated -// array equals a single unbounded scan (same rows, same order, no gaps / -// duplicates), and the cursor advances on `(measuredAt, id)` after every full -// page. +// The consolidation drains walk a user's whole history of a type. Issue #1031: +// on an account with 1.25 million rows the walk gathered every page into one +// array before folding a single day (≈230 MB of heap for the heart-rate type +// alone) and paged with a cursor Postgres could not use as an index condition, +// so every page re-read the rows before it. These tests pin the three +// properties the fix rests on: pages are handed on one at a time and a day is +// folded before the scan reads past it, the cursor carries an index-usable +// bound, and a pass stops cleanly when asked to. -/** Build a deterministic per-sample row at a 1-minute-per-index offset. */ -function row(index: number): PerSampleRow { +/** A deterministic per-sample row at `index` hours past 2026-01-01T00:00Z. */ +function row(index: number, minuteOffset = 0): PerSampleRow { return { id: `m-${String(index).padStart(4, "0")}`, type: "ACTIVITY_STEPS", value: index, - measuredAt: new Date(Date.UTC(2026, 0, 1, 0, index, 0)), + measuredAt: new Date(Date.UTC(2026, 0, 1, index, minuteOffset, 0)), externalId: `hk-uuid-${index}`, }; } +interface CursorWhere { + AND?: [ + Prisma.MeasurementWhereInput, + { measuredAt?: { gte?: Date } }, + { OR?: Array<{ measuredAt?: Date | { gt?: Date }; id?: { gt?: string } }> }, + ]; +} + /** - * A findMany mock that serves `allRows` in keyset order, honouring `take` - * (page size) and the `(measuredAt, id)` cursor clause the helper AND-combines - * onto the base where. This mirrors how Postgres would resolve the keyset. + * A findMany mock that serves `allRows` in keyset order, honouring `take` and + * the `(measuredAt, id)` cursor the pager AND-combines onto the base where. */ function buildPagingFindMany(allRows: PerSampleRow[]) { const ordered = [...allRows].sort((a, b) => { @@ -34,43 +53,29 @@ function buildPagingFindMany(allRows: PerSampleRow[]) { }); return vi.fn( - async (args: { - where: Prisma.MeasurementWhereInput; - take?: number; - orderBy?: unknown; - }) => { + async (args: { where: CursorWhere; take?: number; orderBy?: unknown }) => { const take = args.take ?? ordered.length; - - // Extract the keyset cursor the helper threads through `where.AND[1].OR`. - let afterTime: number | null = null; - let afterId: string | null = null; - const and = (args.where as { AND?: unknown[] }).AND; - if (Array.isArray(and)) { - const or = (and[1] as { OR?: Array> }).OR; - const gt = or?.[0]?.measuredAt as { gt?: Date } | undefined; - const tie = or?.[1] as - { measuredAt?: Date; id?: { gt?: string } } | undefined; - afterTime = gt?.gt ? gt.gt.getTime() : null; - afterId = tie?.id?.gt ?? null; - } - - const remaining = - afterTime === null - ? ordered - : ordered.filter((r) => { - const t = r.measuredAt.getTime(); - return t > afterTime! || (t === afterTime! && r.id > afterId!); - }); - - return remaining.slice(0, take); + const and = args.where.AND; + if (!and) return ordered.slice(0, take); + const gte = and[1].measuredAt?.gte?.getTime(); + const or = and[2].OR ?? []; + const gt = ( + or[0]?.measuredAt as { gt?: Date } | undefined + )?.gt?.getTime(); + const tieId = or[1]?.id?.gt; + return ordered + .filter((r) => { + const t = r.measuredAt.getTime(); + if (gte !== undefined && t < gte) return false; + return t > gt! || (t === gt! && r.id > tieId!); + }) + .slice(0, take); }, ); } function mockClient(findMany: ReturnType): PrismaClient { - return { - measurement: { findMany }, - } as unknown as PrismaClient; + return { measurement: { findMany } } as unknown as PrismaClient; } const BASE_WHERE: Prisma.MeasurementWhereInput = { @@ -85,21 +90,25 @@ const SELECT: Prisma.MeasurementSelect = { externalId: true, }; -describe("scanSourceRowsPaged", () => { - it("accumulates every row in keyset order across multiple pages with no gaps or duplicates", async () => { +async function collectPages( + pages: AsyncIterable, +): Promise { + const out: PerSampleRow[][] = []; + for await (const page of pages) out.push(page); + return out; +} + +describe("iterateSourcePages", () => { + it("yields every row once, in keyset order, one bounded page at a time", async () => { const all = Array.from({ length: 23 }, (_, i) => row(i)); const findMany = buildPagingFindMany(all); - const result = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 10, + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10), ); - // 23 rows at page size 10 → pages of 10, 10, 3 (the short page ends it). - expect(result.map((r) => r.id)).toEqual(all.map((r) => r.id)); - expect(new Set(result.map((r) => r.id)).size).toBe(all.length); + expect(pages.map((p) => p.length)).toEqual([10, 10, 3]); + expect(pages.flat().map((r) => r.id)).toEqual(all.map((r) => r.id)); expect(findMany).toHaveBeenCalledTimes(3); }); @@ -107,88 +116,191 @@ describe("scanSourceRowsPaged", () => { const all = Array.from({ length: 20 }, (_, i) => row(i)); const findMany = buildPagingFindMany(all); - const result = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 10, + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10), ); - // 20 rows at page size 10 → full pages 10, 10, then an empty probe page. - expect(result).toHaveLength(20); + expect(pages.flat()).toHaveLength(20); expect(findMany).toHaveBeenCalledTimes(3); }); - it("advances the cursor on (measuredAt, id) after each full page", async () => { + it("yields nothing when the first page is empty", async () => { + const findMany = buildPagingFindMany([]); + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 10), + ); + expect(pages).toEqual([]); + expect(findMany).toHaveBeenCalledTimes(1); + }); + + it("bounds every later page with an index-usable measuredAt >= cursor conjunct", async () => { + // `measuredAt > c OR (measuredAt = c AND id > i)` alone is not an index + // condition: Postgres walked the index from the user's first row on + // every page. The redundant `>=` bound is what the planner can use. const all = Array.from({ length: 12 }, (_, i) => row(i)); const findMany = buildPagingFindMany(all); - await scanSourceRowsPaged(mockClient(findMany), BASE_WHERE, SELECT, 5); + await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 5), + ); - // First page: no cursor (base where only). - const firstWhere = findMany.mock.calls[0]?.[0]?.where; - expect((firstWhere as { AND?: unknown }).AND).toBeUndefined(); - expect(firstWhere).toEqual(BASE_WHERE); - - // Second page: cursor resumes after the 5th row (id m-0004). - const secondWhere = findMany.mock.calls[1]?.[0]?.where as { - AND: unknown[]; - }; - expect(Array.isArray(secondWhere.AND)).toBe(true); - expect(secondWhere.AND[0]).toEqual(BASE_WHERE); - const cursorClause = secondWhere.AND[1] as { - OR: Array<{ measuredAt?: { gt?: Date }; id?: { gt?: string } }>; - }; - expect(cursorClause.OR[0].measuredAt?.gt).toEqual(row(4).measuredAt); - expect(cursorClause.OR[1].id?.gt).toBe(row(4).id); - - // Every page is requested in stable keyset order. + expect(findMany.mock.calls[0]?.[0]?.where).toEqual(BASE_WHERE); + const second = findMany.mock.calls[1]?.[0]?.where as CursorWhere; + expect(second.AND?.[0]).toEqual(BASE_WHERE); + expect(second.AND?.[1]).toEqual({ + measuredAt: { gte: row(4).measuredAt }, + }); + expect(second.AND?.[2]).toEqual({ + OR: [ + { measuredAt: { gt: row(4).measuredAt } }, + { measuredAt: row(4).measuredAt, id: { gt: row(4).id } }, + ], + }); for (const call of findMany.mock.calls) { expect(call[0]?.orderBy).toEqual([{ measuredAt: "asc" }, { id: "asc" }]); } }); - it("returns the empty array when the first page is empty", async () => { - const findMany = buildPagingFindMany([]); - const result = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 10, - ); - expect(result).toEqual([]); - expect(findMany).toHaveBeenCalledTimes(1); - }); - - it("produces output that buckets identically to a single unpaged scan", async () => { - // Rows spanning three calendar days; the paged accumulation must bucket - // byte-identically to the same rows fed in one array. - const all = [ - { ...row(1), measuredAt: new Date("2026-05-16T08:00:00.000Z") }, - { ...row(2), measuredAt: new Date("2026-05-16T14:00:00.000Z") }, - { ...row(3), measuredAt: new Date("2026-05-17T06:00:00.000Z") }, - { ...row(4), measuredAt: new Date("2026-05-17T20:00:00.000Z") }, - { ...row(5), measuredAt: new Date("2026-05-18T09:00:00.000Z") }, - ]; + it("resumes past rows that share the boundary instant without gaps or duplicates", async () => { + // Five rows on the same instant straddle the page boundary. + const all = [0, 1, 2, 3, 4].map((i) => ({ + ...row(i), + measuredAt: new Date("2026-05-16T08:00:00.000Z"), + })); const findMany = buildPagingFindMany(all); - - const paged = await scanSourceRowsPaged( - mockClient(findMany), - BASE_WHERE, - SELECT, - 2, + const pages = await collectPages( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2), ); + expect(pages.flat().map((r) => r.id)).toEqual(all.map((r) => r.id)); + }); +}); - const baselineByDay = bucketRowsByDay(all, "Europe/Berlin", "stats:"); - const pagedByDay = bucketRowsByDay(paged, "Europe/Berlin", "stats:"); +describe("iterateDayBuckets", () => { + const days = [ + { ...row(1), measuredAt: new Date("2026-05-16T08:00:00.000Z") }, + { ...row(2), measuredAt: new Date("2026-05-16T14:00:00.000Z") }, + { + ...row(3), + externalId: "stats:HKQuantityTypeIdentifierStepCount:2026-05-16", + measuredAt: new Date("2026-05-16T15:00:00.000Z"), + }, + { ...row(4), measuredAt: new Date("2026-05-17T06:00:00.000Z") }, + { ...row(5), measuredAt: new Date("2026-05-17T20:00:00.000Z") }, + { ...row(6), measuredAt: new Date("2026-05-18T09:00:00.000Z") }, + ]; - expect([...pagedByDay.keys()].sort()).toEqual( - [...baselineByDay.keys()].sort(), - ); - for (const [dateKey, rows] of baselineByDay) { - expect(pagedByDay.get(dateKey)?.map((r) => r.id)).toEqual( + it("groups exactly as the whole-array bucketing does, stats rows skipped", async () => { + const findMany = buildPagingFindMany(days); + let scanned = 0; + const streamed = new Map(); + for await (const [key, rows] of iterateDayBuckets( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2), + "Europe/Berlin", + "stats:", + () => { + scanned += 1; + }, + )) { + streamed.set( + key, rows.map((r) => r.id), ); } + + const baseline = bucketRowsByDay(days, "Europe/Berlin", "stats:"); + expect([...streamed.keys()]).toEqual([...baseline.keys()]); + for (const [key, rows] of baseline) { + expect(streamed.get(key)).toEqual(rows.map((r) => r.id)); + } + expect(scanned).toBe(days.length); + }); + + it("hands a day on before the scan reads the page after the one that closed it", async () => { + const findMany = buildPagingFindMany(days); + const pagesReadWhenYielded: number[] = []; + for await (const _day of iterateDayBuckets( + iterateSourcePages(mockClient(findMany), BASE_WHERE, SELECT, 2), + "Europe/Berlin", + "stats:", + )) { + pagesReadWhenYielded.push(findMany.mock.calls.length); + } + // Pages: [r1 r2] [r3 r4] [r5 r6] [] — 05-16 closes on page 2 (r4), + // 05-17 on page 3 (r6), 05-18 at the end of the scan. + expect(pagesReadWhenYielded).toEqual([2, 3, 4]); + }); +}); + +describe("runConsolidation — streamed walk", () => { + /** One row per hour for `dayCount` UTC days. */ + function hourlyRows(dayCount: number): PerSampleRow[] { + return Array.from({ length: dayCount * 24 }, (_, i) => row(i, 30)); + } + + function params( + findMany: ReturnType, + writeDay: ConsolidationParams["writeDay"], + shouldStop?: () => boolean, + ): ConsolidationParams { + const client = { + user: { + findMany: vi + .fn() + .mockResolvedValue([{ id: "user-1", timezone: "UTC" }]), + }, + measurement: { findMany }, + } as unknown as PrismaClient; + return { + prismaClient: client, + options: { shouldStop }, + types: ["ACTIVITY_STEPS"] as MeasurementType[], + hkIdentifierForType: () => "HKQuantityTypeIdentifierStepCount", + dailyStatsExternalId: (hk: string, day: string) => `stats:${hk}:${day}`, + statsPrefix: "stats:", + reduce: (rows: readonly PerSampleRow[]) => rows.length, + buildScanWhere: () => BASE_WHERE, + writeDay, + recordBucket: () => {}, + }; + } + + it("folds the first day before the scan has read the whole history", async () => { + // 500 days × 24 rows = 12 000 rows → default pages of 5000, 5000, 2000. + const all = hourlyRows(500); + const findMany = buildPagingFindMany(all); + const pagesReadAtFirstWrite: number[] = []; + const writeDay = vi.fn(async () => { + if (pagesReadAtFirstWrite.length === 0) { + pagesReadAtFirstWrite.push(findMany.mock.calls.length); + } + return { kind: "written" as const, sourceRowsRemoved: 0 }; + }); + + const result = await runConsolidation(params(findMany, writeDay)); + + expect(writeDay).toHaveBeenCalledTimes(500); + expect(findMany).toHaveBeenCalledTimes(3); + // The old walk read all three pages before the first write. + expect(pagesReadAtFirstWrite).toEqual([1]); + expect(result.stoppedEarly).toBe(false); + }); + + it("stops before the next day once shouldStop answers true, and reads no further", async () => { + const all = hourlyRows(500); + const findMany = buildPagingFindMany(all); + const writeDay = vi.fn(async () => ({ + kind: "written" as const, + sourceRowsRemoved: 0, + })); + let budget = 3; + const shouldStop = () => budget-- <= 0; + + const result = await runConsolidation( + params(findMany, writeDay, shouldStop), + ); + + expect(writeDay).toHaveBeenCalledTimes(3); + expect(result.stoppedEarly).toBe(true); + expect(findMany).toHaveBeenCalledTimes(1); }); }); diff --git a/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts b/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts index de394991c..c69715be9 100644 --- a/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts @@ -30,6 +30,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({ import { runDenseIntradayHourlyRebuild } from "../dense-intraday-hourly-rebuild"; import type { PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; const HRV_DAILY_ID = "stats:HKQuantityTypeIdentifierHeartRateVariabilitySDNN:2026-05-01"; @@ -79,6 +80,8 @@ function buildPrismaMock(opts: { create: txCreate, update: txUpdate, findFirst: txFindFirst, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(txCreate), }, }; @@ -444,6 +447,10 @@ describe("runDenseIntradayHourlyRebuild — rebuild flow", () => { create: vi.fn().mockResolvedValue({ id: "minted" }), update: vi.fn().mockResolvedValue({}), findFirst: vi.fn().mockResolvedValue(null), + findMany: candidateLookup(), + createManyAndReturn: createManyVia( + vi.fn().mockResolvedValue({ id: "minted" }), + ), }, }); }, diff --git a/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts b/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts index 6d7973204..76f03037d 100644 --- a/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts @@ -33,6 +33,7 @@ import { } from "../dense-intraday-retention"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; function pulseRow(id: string, value: number, iso: string): PerSampleRow { return { @@ -68,6 +69,8 @@ function buildPrismaMock(opts: { update: txUpdate, findFirst: txFindFirst, updateMany: txUpdateMany, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(txCreate), }, }; @@ -325,6 +328,10 @@ describe("PULSE fold — per-day failure boundary", () => { update: vi.fn(), findFirst: vi.fn().mockResolvedValue(null), updateMany: vi.fn().mockResolvedValue({ count: 5 }), + findMany: candidateLookup(), + createManyAndReturn: createManyVia( + vi.fn().mockResolvedValue({ id: "minted" }), + ), }, }); }, diff --git a/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts b/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts index 92d88f07b..fb0579290 100644 --- a/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts @@ -32,6 +32,7 @@ import { DENSE_INTRADAY_RETENTION_DAYS, } from "../dense-intraday-retention"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; /** A day well outside the retention window, so the fold definitely runs. */ const FOLD_DAY = new Date( @@ -76,7 +77,15 @@ function buildPrismaMock(existingResting: Array<{ source: string }>) { ); const tx = { - measurement: { create, update, findFirst: txFindFirst, updateMany, upsert }, + measurement: { + create, + update, + findFirst: txFindFirst, + updateMany, + upsert, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(create), + }, }; return { diff --git a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts index 9f10409db..4ea0b1250 100644 --- a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts @@ -25,6 +25,7 @@ import { } from "../dense-intraday-retention"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; function row( id: string, @@ -56,7 +57,16 @@ function buildPrismaMock( async (args: { where: { type: string } }) => rowsByType[args.where.type] ?? [], ); - const tx = { measurement: { create, update, findFirst, updateMany } }; + const tx = { + measurement: { + create, + update, + findFirst, + updateMany, + findMany: candidateLookup(existingCanonicalId), + createManyAndReturn: createManyVia(create), + }, + }; return { mock: { user: { diff --git a/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts b/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts index cf19ba322..a61d7a1d8 100644 --- a/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts @@ -29,6 +29,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({ import { runDenseIntradayRetention } from "../dense-intraday-retention"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { candidateLookup, createManyVia } from "./hourly-mint-mock"; function spo2Row(id: string, value: number, iso: string): PerSampleRow { return { @@ -60,6 +61,8 @@ function buildPrismaMock(opts: { spo2Rows: PerSampleRow[] }) { update: txUpdate, findFirst: txFindFirst, updateMany: txUpdateMany, + findMany: candidateLookup(), + createManyAndReturn: createManyVia(txCreate), }, }; @@ -254,6 +257,10 @@ describe("SpO2 fold — per-day failure boundary", () => { update: vi.fn(), findFirst: vi.fn().mockResolvedValue(null), updateMany: vi.fn().mockResolvedValue({ count: 2 }), + findMany: candidateLookup(), + createManyAndReturn: createManyVia( + vi.fn().mockResolvedValue({ id: "minted" }), + ), }, }); }, diff --git a/src/lib/measurements/__tests__/hourly-mint-mock.ts b/src/lib/measurements/__tests__/hourly-mint-mock.ts new file mode 100644 index 000000000..a5131cee9 --- /dev/null +++ b/src/lib/measurements/__tests__/hourly-mint-mock.ts @@ -0,0 +1,49 @@ +/** + * Transaction-client stubs for `adoptOrMintHourlyRows`, shared by the mocked + * dense-tier fold tests. + * + * The hourly mint reads the day's candidate rows with one `findMany` and + * writes its mints with one `createManyAndReturn`. The tests were written + * against the per-slot loop and assert on individual `create` calls, so the + * batch insert here is routed through the test's own `create` spy, one call + * per minted row: the assertions keep reading what was minted, in slot order. + */ +import { vi } from "vitest"; + +type CreateSpy = (args: { + data: { externalId: string }; +}) => Promise<{ id: string }>; + +/** `createManyAndReturn` that forwards every row to `create`. */ +export function createManyVia(create: CreateSpy) { + return vi.fn(async (args: { data: Array<{ externalId: string }> }) => { + const out: Array<{ id: string; externalId: string }> = []; + for (const data of args.data) { + const row = await create({ data }); + out.push({ id: row.id, externalId: data.externalId }); + } + return out; + }); +} + +/** + * The day's candidate lookup. With `existingId`, one row already occupies the + * first slot's anchor instant (the adopt-in-place case); otherwise nothing is + * in the way. + */ +export function candidateLookup(existingId: string | null = null) { + return vi.fn( + async (args: { + where: { OR: [unknown, { measuredAt: { in: Date[] } }] }; + }) => + existingId === null + ? [] + : [ + { + id: existingId, + externalId: null, + measuredAt: args.where.OR[1].measuredAt.in[0]!, + }, + ], + ); +} diff --git a/src/lib/measurements/consolidate-daily-mean.ts b/src/lib/measurements/consolidate-daily-mean.ts index 097fed61a..dd8c376e0 100644 --- a/src/lib/measurements/consolidate-daily-mean.ts +++ b/src/lib/measurements/consolidate-daily-mean.ts @@ -93,6 +93,12 @@ export interface MeanConsolidationBucket { export interface MeanConsolidationSummary { dryRun: boolean; + /** + * `shouldStop` ended the walk before every day was reached. Nothing is + * lost: folded days committed one by one, and the next run starts at the + * first day this one did not reach. + */ + stoppedEarly: boolean; buckets: MeanConsolidationBucket[]; totals: { usersScanned: number; @@ -123,6 +129,11 @@ export interface MeanConsolidationOptions { * `undefined` to drain everything. */ cutoffHours?: number; + /** + * Ends the pass cleanly before the next day when it returns `true`; the + * summary then reports `stoppedEarly`. See `ConsolidationOptions`. + */ + shouldStop?: () => boolean; } /** Arithmetic mean of a per-day bucket. Exposed for unit testing. */ @@ -205,6 +216,7 @@ export async function consolidateDailyMean( const summary: MeanConsolidationSummary = { dryRun: options.dryRun ?? false, + stoppedEarly: false, buckets: [], totals: { usersScanned: 0, @@ -215,7 +227,7 @@ export async function consolidateDailyMean( }, }; - const { usersScanned, daysFailed } = await runConsolidation({ + const walk = await runConsolidation({ prismaClient, options, types: HIGH_FREQUENCY_MEAN_TYPES, @@ -420,8 +432,9 @@ export async function consolidateDailyMean( }, }); - summary.totals.usersScanned = usersScanned; - summary.totals.daysFailed = daysFailed; + summary.totals.usersScanned = walk.usersScanned; + summary.stoppedEarly = walk.stoppedEarly; + summary.totals.daysFailed = walk.daysFailed; log( `[mean-consolidation] done — usersScanned=${summary.totals.usersScanned} daysConsolidated=${summary.totals.daysConsolidated} perSampleRowsSoftDeleted=${summary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted} daysFailed=${summary.totals.daysFailed}${options.dryRun ? " (dry-run)" : ""}`, diff --git a/src/lib/measurements/consolidation-base.ts b/src/lib/measurements/consolidation-base.ts index a94dce1b2..01abebbbe 100644 --- a/src/lib/measurements/consolidation-base.ts +++ b/src/lib/measurements/consolidation-base.ts @@ -32,6 +32,8 @@ import type { Prisma, } from "@/generated/prisma/client"; +import { reportJobProgress } from "@/lib/jobs/job-observer"; + import { CONSOLIDATION_GRACE_CUTOFF_HOURS, canonicalDailyTimestamp, @@ -57,6 +59,18 @@ export interface ConsolidationOptions { dryRun?: boolean; /** Logger sink — defaults to `console.log`. */ log?: (line: string) => void; + /** + * Asked before every day bucket. Returning `true` ends the pass cleanly + * after the day in flight has committed, and the result reports + * `stoppedEarly`. The queue handlers wire it to the job's time budget and + * to pg-boss's abort signal, so a pass that cannot finish inside its expiry + * stops on its own instead of being declared dead while it keeps writing. + * + * Stopping loses nothing: every day commits on its own, and a folded day's + * source rows leave the scan predicate, so the next run starts at the first + * day this one did not reach. + */ + shouldStop?: () => boolean; } /** @@ -229,7 +243,7 @@ export interface ConsolidationParams { dryRun: boolean; }) => void; /** - * Fired once per (user, type) immediately after the rows are bucketed, + * Fired once per (user, type) after the type's rows have been walked, * with the raw scanned-row count and per-day bucket count. Only invoked * for types that yielded at least one source row. Lets a single-type * drain (legacy steps) reproduce its scan-time log line. Optional. @@ -275,35 +289,38 @@ const DEFAULT_SCAN_SELECT: Prisma.MeasurementSelect = { }; /** - * Page size for the keyset-paginated source-row scan. Heavy multi-year - * tenants previously materialised the entire per-(user, type) history in a - * single unbounded `findMany` result set — the dominant Prisma intermediate - * cost and the root of the consolidation boot-storm on large accounts. - * Scanning in bounded keyset pages caps the per-query result set while still - * accumulating the same full row set. + * Page size for the keyset-paginated source-row scan. */ const CONSOLIDATION_SCAN_PAGE_SIZE = 5000; /** * Keyset-paginate the per-(user, type) source-row scan on `(measuredAt, id)` - * ascending, accumulating every page into one array. Output is identical to a - * single unbounded `findMany` — same `where`, same `select`, same ascending - * order — but the Prisma client never holds more than one page of rows at a - * time, so a multi-year tenant no longer materialises its whole history in one - * result set at worker boot. + * ascending, one page at a time. * - * `(measuredAt, id)` is a stable keyset: `measuredAt` is the primary order and - * `id` breaks ties, so each page resumes exactly after the last row with no - * gaps or duplicates. The cursor clause is AND-combined with the caller's - * `baseWhere` so the scope / soft-delete / grace filters always still apply. + * Two properties matter, and both used to be missing. + * + * The pages are yielded, not accumulated. The previous helper gathered every + * page into one array before a single day was folded, so the pass held a + * type's whole history at once: on an account with 766 000 heart-rate samples + * that is about 230 MB of live heap for the array alone, most of a 524 MB + * limit, and nothing was written until the whole scan had finished. + * + * The cursor is index-usable. `measuredAt > c OR (measuredAt = c AND id > i)` + * cannot be an index condition, so Postgres walked the index from the user's + * first row on every page and filtered its way to the cursor, which makes the + * whole scan quadratic (measured: 47 ms and 248 000 rows filtered for one + * page halfway through). The redundant `measuredAt >= c` conjunct is the + * index condition; the OR only breaks ties at the boundary instant. + * + * The caller's `baseWhere` is AND-combined, so the scope / soft-delete / + * grace filters always still apply. */ -export async function scanSourceRowsPaged( +export async function* iterateSourcePages( prismaClient: PrismaClient, baseWhere: Prisma.MeasurementWhereInput, scanSelect: Prisma.MeasurementSelect, pageSize: number = CONSOLIDATION_SCAN_PAGE_SIZE, -): Promise { - const accumulated: PerSampleRow[] = []; +): AsyncGenerator { let cursor: { measuredAt: Date; id: string } | null = null; for (;;) { @@ -311,6 +328,7 @@ export async function scanSourceRowsPaged( ? { AND: [ baseWhere, + { measuredAt: { gte: cursor.measuredAt } }, { OR: [ { measuredAt: { gt: cursor.measuredAt } }, @@ -328,18 +346,54 @@ export async function scanSourceRowsPaged( take: pageSize, })) as PerSampleRow[]; - if (page.length === 0) break; - - for (const row of page) accumulated.push(row); - - // A short page is the last page — no further rows can satisfy the keyset. - if (page.length < pageSize) break; + if (page.length === 0) return; const last = page[page.length - 1]!; + const isLastPage = page.length < pageSize; + yield page; + + // A short page is the last page — no further rows can satisfy the keyset. + if (isLastPage) return; cursor = { measuredAt: last.measuredAt, id: last.id }; } +} - return accumulated; +/** + * Group a stream of ascending pages into complete per-day buckets, in the + * user's timezone, without holding more than one day at a time. + * + * The rows arrive in `measuredAt` order and a local calendar day is one + * contiguous span of instants, so a day is complete the moment a row from a + * later day shows up. Rows already in the daily-stats shape are skipped, the + * same as `bucketRowsByDay`. `onRow` sees every scanned row, stats rows + * included, for the scan-count log line. + */ +export async function* iterateDayBuckets( + pages: AsyncIterable, + tz: string, + statsPrefix: string, + onRow?: () => void, +): AsyncGenerator<[dateKey: string, rows: PerSampleRow[]], void, void> { + let currentKey: string | null = null; + let current: PerSampleRow[] = []; + for await (const page of pages) { + for (const row of page) { + onRow?.(); + if (row.externalId !== null && row.externalId.startsWith(statsPrefix)) { + continue; + } + const key = dayKeyForUserTz(row.measuredAt, tz); + if (key !== currentKey) { + if (currentKey !== null && current.length > 0) { + yield [currentKey, current]; + } + currentKey = key; + current = []; + } + current.push(row); + } + } + if (currentKey !== null && current.length > 0) yield [currentKey, current]; } /** @@ -348,7 +402,8 @@ export async function scanSourceRowsPaged( * and delegates the per-day mint + delete to `writeDay`. Returns the * number of users scanned (so the caller can seed its summary totals) * plus the number of day buckets absorbed by `onBucketError` (0 when the - * boundary is not supplied). + * boundary is not supplied), and whether `options.shouldStop` ended the + * walk before every day was reached. * * Idempotency, the grace-window cutoff, and the "skip already-collapsed * rows" predicate are all owned here; the divergent reducer / delete / @@ -356,16 +411,24 @@ export async function scanSourceRowsPaged( */ export async function runConsolidation( params: ConsolidationParams, -): Promise<{ usersScanned: number; dryRun: boolean; daysFailed: number }> { +): Promise<{ + usersScanned: number; + dryRun: boolean; + daysFailed: number; + stoppedEarly: boolean; +}> { const { prismaClient, options } = params; const dryRun = options.dryRun ?? false; const cutoffAt = resolveCutoffInstant(options.cutoffHours); const scanSelect = params.scanSelect ?? DEFAULT_SCAN_SELECT; + const shouldStop = options.shouldStop ?? (() => false); let daysFailed = 0; + let daysWalked = 0; + let stoppedEarly = false; const users = await loadConsolidationUsers(prismaClient, options.userId); - for (const user of users) { + walk: for (const user of users) { const tz = resolveUserTimezone(user.timezone); params.onUserStart?.({ userId: user.id, tz, dryRun }); @@ -373,36 +436,45 @@ export async function runConsolidation( const hkIdentifier = params.hkIdentifierForType(type); if (!hkIdentifier) continue; - // Keyset-paginate the scan instead of loading the whole per-(user, type) - // history in one result set; the accumulated array is byte-identical to a - // single unbounded `findMany`, but a multi-year tenant no longer spikes - // worker memory at boot. See `scanSourceRowsPaged`. - const sourceRows = await scanSourceRowsPaged( - prismaClient, - params.buildScanWhere({ - userId: user.id, - type, - cutoffAt, - statsPrefix: params.statsPrefix, - }), - scanSelect, + // Streamed: each day is folded as soon as its last row has been read, + // so the pass holds one page and one day, never the type's history. + // See `iterateSourcePages` and `iterateDayBuckets`. + let rowCount = 0; + let dayCount = 0; + const days = iterateDayBuckets( + iterateSourcePages( + prismaClient, + params.buildScanWhere({ + userId: user.id, + type, + cutoffAt, + statsPrefix: params.statsPrefix, + }), + scanSelect, + ), + tz, + params.statsPrefix, + () => { + rowCount += 1; + }, ); - if (sourceRows.length === 0) continue; - - const byDay = bucketRowsByDay(sourceRows, tz, params.statsPrefix); - - params.onScan?.({ - userId: user.id, - type, - tz, - rowCount: sourceRows.length, - dayCount: byDay.size, - dryRun, - }); - - for (const [dateKey, dayRows] of byDay) { - if (dayRows.length === 0) continue; + for await (const [dateKey, dayRows] of days) { + if (shouldStop()) { + stoppedEarly = true; + // Leaving the loop closes both generators, so no further page is + // requested. + break walk; + } + dayCount += 1; + daysWalked += 1; + // How far the pass has got, for the job's progress and expiry lines. + reportJobProgress({ + consolidation_user: user.id, + consolidation_type: type, + consolidation_day: dateKey, + consolidation_days_walked: daysWalked, + }); try { const reducedValue = params.reduce(dayRows); @@ -464,10 +536,21 @@ export async function runConsolidation( }); } } + + if (rowCount > 0) { + params.onScan?.({ + userId: user.id, + type, + tz, + rowCount, + dayCount, + dryRun, + }); + } } params.onUserComplete?.({ userId: user.id, tz, dryRun }); } - return { usersScanned: users.length, dryRun, daysFailed }; + return { usersScanned: users.length, dryRun, daysFailed, stoppedEarly }; } diff --git a/src/lib/measurements/dense-intraday-hourly-rebuild.ts b/src/lib/measurements/dense-intraday-hourly-rebuild.ts index 455e4b613..15e116724 100644 --- a/src/lib/measurements/dense-intraday-hourly-rebuild.ts +++ b/src/lib/measurements/dense-intraday-hourly-rebuild.ts @@ -19,7 +19,7 @@ * day's local window (the fold's soft-deleted inputs). * * Per qualifying day: compute per-local-hour means from the tombstoned raw - * rows (same `meanBucketValue` reducer + `adoptOrMintHourlyRow` + * rows (same `meanBucketValue` reducer + `adoptOrMintHourlyRows` * adopt-in-place slot logic as the live fold, so a re-run converges), then * tombstone the daily row IN THE SAME TRANSACTION as the hourly mint — at * no instant are the daily row and its hourly rows both live, so an @@ -73,7 +73,7 @@ import { meanBucketValue } from "./consolidate-daily-mean"; import { DENSE_INTRADAY_RETENTION_DAYS, DENSE_INTRADAY_RETENTION_TYPES, - adoptOrMintHourlyRow, + adoptOrMintHourlyRows, bucketRowsByLocalHour, hourlyStatsExternalId, } from "./dense-intraday-retention"; @@ -318,11 +318,11 @@ async function rebuildDay( const rebuildOnce = async (): Promise<{ dailyRetired: boolean }> => { let dailyRetired = false; await prismaClient.$transaction(async (tx) => { - const canonicalRowIds: string[] = []; - for (const [hour, hourRows] of byHour) { - const rowId = await adoptOrMintHourlyRow(tx, { - userId: input.userId, - type: input.type, + const canonicalRowIds = await adoptOrMintHourlyRows(tx, { + userId: input.userId, + type: input.type, + unit, + slots: byHour.map(([hour, hourRows]) => ({ externalId: hourlyStatsExternalId( input.hkIdentifier, input.dateKey, @@ -330,10 +330,8 @@ async function rebuildDay( ), anchor: canonicalHourlyTimestamp(input.dateKey, hour, input.tz), value: meanBucketValue(hourRows), - unit, - }); - canonicalRowIds.push(rowId); - } + })), + }); // Retire the daily row in the SAME transaction — at no instant are // the daily row and the hourly rows both live (an AVG-over-live-rows @@ -359,7 +357,7 @@ async function rebuildDay( if (!isUniqueConstraintViolation(err)) throw err; // A concurrent writer (the nightly fold racing this rebuild) won a slot // mid-transaction. Retry once: the deterministic lookup inside - // `adoptOrMintHourlyRow` now resolves the winning row and adopts it. + // `adoptOrMintHourlyRows` now resolves the winning row and adopts it. outcome = await rebuildOnce(); } diff --git a/src/lib/measurements/dense-intraday-retention.ts b/src/lib/measurements/dense-intraday-retention.ts index 515f8ea19..5c6772c3c 100644 --- a/src/lib/measurements/dense-intraday-retention.ts +++ b/src/lib/measurements/dense-intraday-retention.ts @@ -191,10 +191,20 @@ export function bucketRowsByLocalHour( return byHour; } +/** One hourly slot of a folded day, as `adoptOrMintHourlyRows` receives it. */ +export interface HourlySlot { + /** Target hourly `stats:` externalId (index-A identity). */ + externalId: string; + /** Local HH:30 anchor instant (index-B identity). */ + anchor: Date; + value: number; +} + /** - * Adopt-or-mint one hourly `stats:` row inside the caller's transaction. + * Adopt-or-mint a day's hourly `stats:` rows inside the caller's transaction. + * Returns the canonical row id for every slot, in slot order. * - * TWO unique indexes can collide on the mint: + * TWO unique indexes can collide on a mint: * A) `(userId, type, source, externalId)` — a row already carrying the * target hourly `stats:` externalId (a prior fold / rebuild pass). * B) `(userId, type, measuredAt, source, sleepStage)` (NULLS NOT @@ -202,98 +212,149 @@ export function bucketRowsByLocalHour( * anchor instant, which may carry a DIFFERENT externalId (a * per-sample row that happened at HH:30, or a sibling stats row). * - * Determinism (VECTOR 1): resolve by index A FIRST — the row already - * holding the target externalId IS the canonical hourly row by - * construction. Only when no such row exists fall back to the index-B row - * physically occupying the anchor. A stable `orderBy: id` pins the pick. - * Adopting the wrong sibling and stamping the target externalId onto it - * would otherwise collide with the real `stats:` row on index A. + * Determinism: per slot, resolve by index A FIRST — the row already holding + * the target externalId IS the canonical hourly row by construction. Only + * when no such row exists fall back to the index-B row physically occupying + * the anchor. The lowest id wins a tie. Adopting the wrong sibling and + * stamping the target externalId onto it would otherwise collide with the + * real `stats:` row on index A. * - * Concurrency (VECTOR 2): the resolve→create is a check-then-act not - * serialised by the unique index under READ COMMITTED. If a concurrent - * writer wins the slot between the lookup and the INSERT, the `create` - * throws P2002 and Postgres aborts the whole transaction. Callers - * therefore catch P2002 OUTSIDE their transaction (a caught error inside - * would leave the tx in the aborted `25P02` state) and retry the fold - * once: on the retry the deterministic lookup finds the now-existing row - * and ADOPTS it, so the create path never fires twice. Built - * field-by-field (no spread) per the no-mass-assignment convention. + * Why a batch. This used to run per slot — two lookups and a write for each + * of up to 24 hours, about 75 statements for one day — and on a multi-year + * heart-rate history that loop was most of the pass's 380 000 statements. + * Now one lookup reads every row that could matter for the whole day, the + * slots are resolved against that snapshot in order, and the mints go out as + * one insert. The snapshot is kept up to date as each slot adopts or mints, + * so a later slot sees exactly what the one-by-one loop would have read back + * from the database: an adopted row that moves to its anchor frees the + * instant it left and occupies the one it moved to, and a minted row occupies + * its anchor and externalId. Updates run in slot order, as before, and no + * mint can collide with them because a slot mints only when nothing in the + * snapshot holds its externalId or its anchor. + * + * Concurrency: the resolve→write is a check-then-act not serialised by the + * unique index under READ COMMITTED. If a concurrent writer wins a slot + * between the lookup and the INSERT, the insert throws P2002 and Postgres + * aborts the whole transaction. Callers therefore catch P2002 OUTSIDE their + * transaction (a caught error inside would leave the tx in the aborted + * `25P02` state) and retry the fold once: on the retry the lookup finds the + * now-existing row and ADOPTS it. Built field-by-field (no spread) per the + * no-mass-assignment convention. */ -export async function adoptOrMintHourlyRow( +export async function adoptOrMintHourlyRows( tx: Prisma.TransactionClient, input: { userId: string; type: MeasurementType; - /** Target hourly `stats:` externalId (index-A identity). */ - externalId: string; - /** Local HH:30 anchor instant (index-B identity). */ - anchor: Date; - value: number; unit: string; + slots: readonly HourlySlot[]; }, -): Promise { - const eidRow = await tx.measurement.findFirst({ +): Promise { + if (input.slots.length === 0) return []; + + // Every row that either index could put in the way, lowest id first — the + // order the per-slot `findFirst … orderBy id` used to resolve ties by. + // `sleepStage` is NULL for these continuous types, matched via the + // NULLS-NOT-DISTINCT index. + const existing = await tx.measurement.findMany({ where: { userId: input.userId, type: input.type, source: "APPLE_HEALTH", - externalId: input.externalId, + OR: [ + { externalId: { in: input.slots.map((slot) => slot.externalId) } }, + { + measuredAt: { in: input.slots.map((slot) => slot.anchor) }, + sleepStage: null, + }, + ], }, - select: { id: true }, - orderBy: { id: "asc" }, - }); - // Index-B occupant (`sleepStage` is NULL for these continuous types, - // matched via the NULLS-NOT-DISTINCT index). At most one row can exist. - const slotRow = await tx.measurement.findFirst({ - where: { - userId: input.userId, - type: input.type, - source: "APPLE_HEALTH", - measuredAt: input.anchor, - sleepStage: null, - }, - select: { id: true }, + select: { id: true, externalId: true, measuredAt: true }, orderBy: { id: "asc" }, }); - const adoptTarget = eidRow ?? slotRow; - if (adoptTarget) { - // Pin the adopted row to the anchor, but only when the anchor slot is - // free or already this row — a DIFFERENT row occupying the slot would - // otherwise collide on index B. In that rare case the row keeps its - // existing instant; the `stats:` externalId is the identity, - // measuredAt is secondary. - const slotIsFreeForTarget = - slotRow === null || slotRow.id === adoptTarget.id; - // Adopt: refresh value, stamp the hourly externalId, optionally - // re-anchor, and un-tombstone. This coexists with any pre-existing row - // on the slot instead of colliding with it. - await tx.measurement.update({ - where: { id: adoptTarget.id }, - data: { - value: input.value, - unit: input.unit, - externalId: input.externalId, - deletedAt: null, - ...(slotIsFreeForTarget ? { measuredAt: input.anchor } : {}), - }, - }); - return adoptTarget.id; + // The snapshot the slots resolve against. `id: null` marks a row this + // call is about to mint. + const state: Array<{ + id: string | null; + externalId: string | null; + at: number; + }> = existing.map((row) => ({ + id: row.id, + externalId: row.externalId, + at: row.measuredAt.getTime(), + })); + + const resolved: Array = []; + const mints: HourlySlot[] = []; + + for (const slot of input.slots) { + const anchorAt = slot.anchor.getTime(); + const eidRow = state.find((row) => row.externalId === slot.externalId); + const slotRow = state.find((row) => row.at === anchorAt); + const adoptTarget = eidRow ?? slotRow; + + if (adoptTarget && adoptTarget.id !== null) { + // Pin the adopted row to the anchor, but only when the anchor slot is + // free or already this row — a DIFFERENT row occupying the slot would + // otherwise collide on index B. In that rare case the row keeps its + // existing instant; the `stats:` externalId is the identity, + // measuredAt is secondary. + const slotIsFreeForTarget = + slotRow === undefined || slotRow === adoptTarget; + // Adopt: refresh value, stamp the hourly externalId, optionally + // re-anchor, and un-tombstone. This coexists with any pre-existing row + // on the slot instead of colliding with it. + await tx.measurement.update({ + where: { id: adoptTarget.id }, + data: { + value: slot.value, + unit: input.unit, + externalId: slot.externalId, + deletedAt: null, + ...(slotIsFreeForTarget ? { measuredAt: slot.anchor } : {}), + }, + }); + adoptTarget.externalId = slot.externalId; + if (slotIsFreeForTarget) adoptTarget.at = anchorAt; + resolved.push(adoptTarget.id); + } else { + // Nothing holds this slot's identity or instant — including rows this + // call already decided to mint, which carry a distinct anchor and + // externalId by construction (one slot per local hour). + state.push({ id: null, externalId: slot.externalId, at: anchorAt }); + mints.push(slot); + resolved.push(null); + } } - const created = await tx.measurement.create({ - data: { - userId: input.userId, - type: input.type, - value: input.value, - unit: input.unit, - source: "APPLE_HEALTH", - measuredAt: input.anchor, - externalId: input.externalId, - }, - select: { id: true }, - }); - return created.id; + if (mints.length > 0) { + const created = await tx.measurement.createManyAndReturn({ + data: mints.map((slot) => ({ + userId: input.userId, + type: input.type, + value: slot.value, + unit: input.unit, + source: "APPLE_HEALTH" as const, + measuredAt: slot.anchor, + externalId: slot.externalId, + })), + select: { id: true, externalId: true }, + }); + const idByExternalId = new Map( + created.map((row) => [row.externalId, row.id] as const), + ); + input.slots.forEach((slot, index) => { + if (resolved[index] !== null) return; + const id = idByExternalId.get(slot.externalId); + if (id === undefined) { + throw new Error(`hourly mint returned no row for ${slot.externalId}`); + } + resolved[index] = id; + }); + } + + return resolved as string[]; } /** @@ -347,6 +408,12 @@ export function deriveDailyRestingFromPulse( export interface DenseIntradayRetentionSummary { dryRun: boolean; + /** + * `shouldStop` ended the walk before every day was reached. Nothing is + * lost: folded days committed one by one, and the next run starts at the + * first day this one did not reach. + */ + stoppedEarly: boolean; totals: { usersScanned: number; daysConsolidated: number; @@ -385,6 +452,11 @@ export interface DenseIntradayRetentionOptions { * full-collapse, never by the scheduled pass. */ retentionDays?: number; + /** + * Ends the pass cleanly before the next day when it returns `true`; the + * summary then reports `stoppedEarly`. See `ConsolidationOptions`. + */ + shouldStop?: () => boolean; } /** @@ -412,6 +484,7 @@ export async function runDenseIntradayRetention( const summary: DenseIntradayRetentionSummary = { dryRun: options.dryRun ?? false, + stoppedEarly: false, totals: { usersScanned: 0, daysConsolidated: 0, @@ -462,7 +535,7 @@ export async function runDenseIntradayRetention( // for the dry-run hourly fan-out estimate (writeDay never runs on dry-run). let currentTz = "Europe/Berlin"; - const { usersScanned } = await runConsolidation({ + const walk = await runConsolidation({ prismaClient, options: { ...options, cutoffHours }, types: DENSE_INTRADAY_RETENTION_TYPES, @@ -543,18 +616,16 @@ export async function runDenseIntradayRetention( let removed = 0; let retiredDaily = false; await pc.$transaction(async (tx) => { - const canonicalRowIds: string[] = []; - for (const [hour, hourRows] of byHour) { - const rowId = await adoptOrMintHourlyRow(tx, { - userId, - type, + const canonicalRowIds = await adoptOrMintHourlyRows(tx, { + userId, + type, + unit, + slots: byHour.map(([hour, hourRows]) => ({ externalId: hourlyStatsExternalId(hkIdentifier, dateKey, hour), anchor: canonicalHourlyTimestamp(dateKey, hour, tz), value: meanBucketValue(hourRows), - unit, - }); - canonicalRowIds.push(rowId); - } + })), + }); // Retire a live pre-hourly DAILY `stats:` row for this day (the // late-sync path: raw samples arriving for a day folded to the @@ -712,7 +783,8 @@ export async function runDenseIntradayRetention( }, }); - summary.totals.usersScanned = usersScanned; + summary.totals.usersScanned = walk.usersScanned; + summary.stoppedEarly = walk.stoppedEarly; log( `[dense-intraday-retention] done — usersScanned=${summary.totals.usersScanned} daysConsolidated=${summary.totals.daysConsolidated} perSampleRowsSoftDeleted=${summary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${summary.totals.hourlyRowsUpserted} dailyRowsRetired=${summary.totals.dailyRowsRetired} derivedRestingRowsUpserted=${summary.totals.derivedRestingRowsUpserted}${options.dryRun ? " (dry-run)" : ""}`, diff --git a/src/lib/measurements/drain-per-sample-cumulative.ts b/src/lib/measurements/drain-per-sample-cumulative.ts index aa0f9c91b..e0cf58135 100644 --- a/src/lib/measurements/drain-per-sample-cumulative.ts +++ b/src/lib/measurements/drain-per-sample-cumulative.ts @@ -124,6 +124,12 @@ export interface DrainBucket { export interface DrainSummary { /** Did the run actually write to the DB (false for `dryRun`). */ dryRun: boolean; + /** + * `shouldStop` ended the walk before every day was reached. Nothing is + * lost: folded days committed one by one, and the next run starts at the + * first day this one did not reach. + */ + stoppedEarly: boolean; /** Per-user-day buckets the drain rewrote (or would rewrite). */ buckets: DrainBucket[]; /** Aggregate counts across the run. */ @@ -167,6 +173,11 @@ export interface DrainOptions { * the operator points at) for explicit one-shot use. */ cutoffHours?: number; + /** + * Ends the pass cleanly before the next day when it returns `true`; the + * summary then reports `stoppedEarly`. See `ConsolidationOptions`. + */ + shouldStop?: () => boolean; } export interface BucketedRows { @@ -210,6 +221,7 @@ export async function drainPerSampleCumulative( const summary: DrainSummary = { dryRun: options.dryRun ?? false, + stoppedEarly: false, buckets: [], totals: { usersScanned: 0, @@ -229,7 +241,7 @@ export async function drainPerSampleCumulative( let beforePerSampleDeleted = 0; let beforeDailyUpserted = 0; - const { usersScanned } = await runConsolidation({ + const walk = await runConsolidation({ prismaClient, options, types: CUMULATIVE_HK_TYPES, @@ -521,7 +533,8 @@ export async function drainPerSampleCumulative( }, }); - summary.totals.usersScanned = usersScanned; + summary.totals.usersScanned = walk.usersScanned; + summary.stoppedEarly = walk.stoppedEarly; // v1.34.0 — bound the profile sidecar. One indexed range delete per run, // across every account, on the same nightly tick that writes the profiles. From 1b45ebb99d8e9d3ec1cf947ed71bf9a72f40d4e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 00:31:48 +0200 Subject: [PATCH 03/12] fix(jobs): stop the long drains on a budget instead of at the expiry pg-boss marks a job "job timed out" at its expiry and fires its abort signal, but does not stop the handler. A drain that could not finish kept writing after its job was declared dead, and the retry then ran beside it over the same days. The nightly drain tick and the per-user dense-retention and daily-mean jobs now stop between days once 75 % of the job's expiry has passed, or as soon as pg-boss aborts the job, and complete with `stopped_early` instead of failing. What they folded is committed. A nightly tick that stopped early after folding something sends itself again a minute later, up to 24 times, so a first-time backlog after a large import drains the same night; a run that folded nothing does not hand on, so a pass that cannot move does not loop. The expiries themselves stay at their defaults: with the passes bounded per day and budgeted, a longer limit would only delay the signal. --- .../jobs/__tests__/drain-continuation.test.ts | 66 ++++++++++++++++ .../__tests__/drain-cumulative-queue.test.ts | 2 +- src/lib/jobs/__tests__/job-budget.test.ts | 45 +++++++++++ src/lib/jobs/dense-intraday-retention.ts | 7 ++ src/lib/jobs/drain-continuation.ts | 33 ++++++++ src/lib/jobs/job-budget.ts | 48 +++++++++++ src/lib/jobs/job-outcome.ts | 3 + src/lib/jobs/mean-consolidation.ts | 10 ++- src/lib/jobs/reminder/register-rollup.ts | 79 ++++++++++++++++++- 9 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 src/lib/jobs/__tests__/drain-continuation.test.ts create mode 100644 src/lib/jobs/__tests__/job-budget.test.ts create mode 100644 src/lib/jobs/drain-continuation.ts create mode 100644 src/lib/jobs/job-budget.ts diff --git a/src/lib/jobs/__tests__/drain-continuation.test.ts b/src/lib/jobs/__tests__/drain-continuation.test.ts new file mode 100644 index 000000000..c73e7ddeb --- /dev/null +++ b/src/lib/jobs/__tests__/drain-continuation.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { + DRAIN_MAX_CONTINUATIONS, + nextDrainContinuation, +} from "@/lib/jobs/drain-continuation"; + +// A nightly drain tick that ran out of budget hands the rest to a follow-up +// job, so a first-time backlog drains the same night (issue #1031). The chain +// must not loop on a pass that cannot move, and must end. + +describe("nextDrainContinuation", () => { + it("hands on a tick that stopped early after folding something", () => { + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 12, + continuation: 0, + }), + ).toBe(1); + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 1, + continuation: 5, + }), + ).toBe(6); + }); + + it("does not hand on a tick that finished", () => { + expect( + nextDrainContinuation({ + stoppedEarly: false, + daysFolded: 400, + continuation: 0, + }), + ).toBeNull(); + }); + + it("does not hand on a tick that stopped without progress", () => { + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 0, + continuation: 0, + }), + ).toBeNull(); + }); + + it("ends the chain at the cap", () => { + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 10, + continuation: DRAIN_MAX_CONTINUATIONS - 1, + }), + ).toBe(DRAIN_MAX_CONTINUATIONS); + expect( + nextDrainContinuation({ + stoppedEarly: true, + daysFolded: 10, + continuation: DRAIN_MAX_CONTINUATIONS, + }), + ).toBeNull(); + }); +}); diff --git a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts index b904a30dd..4421c5ebb 100644 --- a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts +++ b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts @@ -52,7 +52,7 @@ describe("reminder-worker — drainPerSampleCumulative nightly schedule", () => it("registers a createAndWork handler against the drain queue", () => { expect(source).toMatch( - /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,900}drainPerSampleCumulative/, + /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,1100}drainPerSampleCumulative/, ); }); diff --git a/src/lib/jobs/__tests__/job-budget.test.ts b/src/lib/jobs/__tests__/job-budget.test.ts new file mode 100644 index 000000000..27eec6bf6 --- /dev/null +++ b/src/lib/jobs/__tests__/job-budget.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; + +import { JOB_BUDGET_SHARE, jobBudget } from "@/lib/jobs/job-budget"; + +// pg-boss declares a job dead at its expiry but does not stop the handler; +// the long passes stop themselves on this budget instead (issue #1031). + +function job(expireInSeconds: number, signal = new AbortController().signal) { + return { expireInSeconds, signal }; +} + +describe("jobBudget", () => { + it("stops at the budget share of the job's expiry, not at the expiry", () => { + let now = 1_000_000; + const shouldStop = jobBudget([job(900)], () => now); + + now += 900 * 1000 * JOB_BUDGET_SHARE - 1; + expect(shouldStop()).toBe(false); + now += 1; + expect(shouldStop()).toBe(true); + expect(JOB_BUDGET_SHARE).toBeLessThan(1); + }); + + it("stops as soon as pg-boss aborts the job, whatever the clock says", () => { + const controller = new AbortController(); + const shouldStop = jobBudget([job(900, controller.signal)], () => 0); + expect(shouldStop()).toBe(false); + controller.abort(); + expect(shouldStop()).toBe(true); + }); + + it("takes the tightest expiry of a batch", () => { + let now = 0; + const shouldStop = jobBudget([job(7200), job(60)], () => now); + now = 60 * 1000 * JOB_BUDGET_SHARE; + expect(shouldStop()).toBe(true); + }); + + it("never stops on the clock when there is no job to take an expiry from", () => { + let now = 0; + const shouldStop = jobBudget([], () => now); + now = Number.MAX_SAFE_INTEGER; + expect(shouldStop()).toBe(false); + }); +}); diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index bd86101b7..30eb96b14 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -89,10 +89,13 @@ export interface DenseIntradayRetentionPayload { */ export async function runDenseIntradayRetentionForUser( userId: string, + shouldStop?: () => boolean, ): Promise<{ daysConsolidated: number; perSampleRowsSoftDeleted: number; derivedRestingRowsUpserted: number; + /** The job's budget ran out first; the next run continues from here. */ + stoppedEarly: boolean; }> { // Kill-switch: when an operator opts out, no-op so any already-queued // backlog completes cleanly (drains the queue) instead of running the fold. @@ -101,10 +104,12 @@ export async function runDenseIntradayRetentionForUser( daysConsolidated: 0, perSampleRowsSoftDeleted: 0, derivedRestingRowsUpserted: 0, + stoppedEarly: false, }; } const summary = await runDenseIntradayRetention(prisma, { userId, + shouldStop, log: () => { // Silent inside the queue handler — the worker logs the totals. }, @@ -121,6 +126,7 @@ export async function runDenseIntradayRetentionForUser( // days for proxy users, preserving the resting signal post-fold. derived_resting_rows_upserted: summary.totals.derivedRestingRowsUpserted, + stopped_early: summary.stoppedEarly, }, }, }); @@ -128,6 +134,7 @@ export async function runDenseIntradayRetentionForUser( daysConsolidated: summary.totals.daysConsolidated, perSampleRowsSoftDeleted: summary.totals.perSampleRowsSoftDeleted, derivedRestingRowsUpserted: summary.totals.derivedRestingRowsUpserted, + stoppedEarly: summary.stoppedEarly, }; } diff --git a/src/lib/jobs/drain-continuation.ts b/src/lib/jobs/drain-continuation.ts new file mode 100644 index 000000000..35947dfcc --- /dev/null +++ b/src/lib/jobs/drain-continuation.ts @@ -0,0 +1,33 @@ +/** + * When the nightly drain tick hands its remaining work to a follow-up job. + * + * The tick's three passes (cumulative drain, daily-mean consolidation, dense + * intra-day retention) stop at the job's time budget (`jobBudget`) instead of + * running into pg-boss's expiry. On a normal night that never happens. After + * a multi-year import it does, and without a follow-up the backlog would drain + * one budget per night. So a tick that stopped early sends itself again a + * minute later, and the chain is bounded twice: a follow-up is sent only when + * the run folded at least one day, so a pass that makes no progress cannot + * loop, and never more than `DRAIN_MAX_CONTINUATIONS` times in a row. + */ + +/** Follow-ups one nightly tick may chain: about four and a half hours of budget. */ +export const DRAIN_MAX_CONTINUATIONS = 24; + +/** Delay before a follow-up tick starts, in seconds. */ +export const DRAIN_CONTINUATION_DELAY_SECONDS = 60; + +/** + * The follow-up's continuation number, or `null` when no follow-up is due. + * `continuation` is the current job's own number (0 for the cron's job). + */ +export function nextDrainContinuation(input: { + stoppedEarly: boolean; + daysFolded: number; + continuation: number; +}): number | null { + if (!input.stoppedEarly) return null; + if (input.daysFolded <= 0) return null; + if (input.continuation >= DRAIN_MAX_CONTINUATIONS) return null; + return input.continuation + 1; +} diff --git a/src/lib/jobs/job-budget.ts b/src/lib/jobs/job-budget.ts new file mode 100644 index 000000000..8607004b1 --- /dev/null +++ b/src/lib/jobs/job-budget.ts @@ -0,0 +1,48 @@ +/** + * A time budget for a long pass, derived from the pg-boss job it runs under. + * + * pg-boss gives every job an expiry (`expireInSeconds`, 900 s unless the send + * says otherwise). When it passes, pg-boss marks the job failed with + * `job timed out`, retries it, and fires the job's abort signal. It does NOT + * stop the handler: a handler that never looks at the signal keeps writing + * after its job has been declared dead, and the retry then runs beside it over + * the same rows. On a large account that is how one pass that could not + * finish turned into two or three copies of itself contending for the same + * days. + * + * The passes that walk a whole history therefore stop on their own, between + * units of work, once a share of the expiry has gone by, and report that they + * stopped early. Their units commit one at a time and a finished unit drops + * out of the next scan, so the following run starts where this one stopped. + * The share leaves room for the unit in flight and for the handler's own + * bookkeeping before pg-boss's deadline. + */ +import type { Job } from "pg-boss"; + +/** Share of the job's expiry a pass may spend before it stops itself. */ +export const JOB_BUDGET_SHARE = 0.75; + +/** What the budget reads from a pg-boss job. */ +type BudgetedJob = Pick, "expireInSeconds" | "signal">; + +/** + * Build the `shouldStop` predicate for a batch of jobs handled together. + * + * True once the budget has elapsed (the tightest job's expiry decides) or once + * pg-boss has aborted any of the jobs. `now` is injectable for tests. + */ +export function jobBudget( + jobs: readonly BudgetedJob[], + now: () => number = Date.now, +): () => boolean { + const expirySeconds = Math.min( + ...jobs.map((job) => + Number.isFinite(job.expireInSeconds) && job.expireInSeconds > 0 + ? job.expireInSeconds + : Number.POSITIVE_INFINITY, + ), + ); + const deadline = now() + expirySeconds * 1000 * JOB_BUDGET_SHARE; + return () => + now() >= deadline || jobs.some((job) => job.signal?.aborted === true); +} diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 7d372364a..f57210e09 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -55,6 +55,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "connections_deleted", "considered", "context_surfaced", + "continuation", + "continued", "created", "daily_rows_retired", "days_consolidated", @@ -200,6 +202,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "skipped_outside_window", "slots_collapsed", "states_deleted", + "stopped_early", "stored", "subscription_repair_failed", "summaries_enqueued", diff --git a/src/lib/jobs/mean-consolidation.ts b/src/lib/jobs/mean-consolidation.ts index a001dc2b9..bbe5ab219 100644 --- a/src/lib/jobs/mean-consolidation.ts +++ b/src/lib/jobs/mean-consolidation.ts @@ -44,9 +44,15 @@ export interface MeanConsolidationPayload { */ export async function runMeanConsolidationForUser( userId: string, -): Promise<{ daysConsolidated: number; perSampleRowsSoftDeleted: number }> { + shouldStop?: () => boolean, +): Promise<{ + daysConsolidated: number; + perSampleRowsSoftDeleted: number; + stoppedEarly: boolean; +}> { const summary = await consolidateDailyMean(prisma, { userId, + shouldStop, cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, log: () => { // Silent inside the queue handler — the worker logs the totals. @@ -59,12 +65,14 @@ export async function runMeanConsolidationForUser( days: summary.totals.daysConsolidated, per_sample_rows_soft_deleted: summary.totals.perSampleRowsSoftDeleted, daily_rows_upserted: summary.totals.dailyRowsUpserted, + stopped_early: summary.stoppedEarly, }, }, }); return { daysConsolidated: summary.totals.daysConsolidated, perSampleRowsSoftDeleted: summary.totals.perSampleRowsSoftDeleted, + stoppedEarly: summary.stoppedEarly, }; } diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts index eca1bf4ff..3ddd5ea92 100644 --- a/src/lib/jobs/reminder/register-rollup.ts +++ b/src/lib/jobs/reminder/register-rollup.ts @@ -110,6 +110,11 @@ import { type ScheduleEntry, } from "./registrar-shared"; import { jobDone, jobFailed } from "@/lib/jobs/job-outcome"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { + DRAIN_CONTINUATION_DELAY_SECONDS, + nextDrainContinuation, +} from "@/lib/jobs/drain-continuation"; // v1.4.37 W7c — nightly drain of per-sample APPLE_HEALTH cumulative rows. // Collapses each user × cumulative-type × calendar-day bucket into one // `stats:…` row so the list view stops painting hundreds of step chunks per @@ -123,7 +128,12 @@ const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative"; const DRAIN_CUMULATIVE_CRON = "45 3 * * *"; interface DrainCumulativePayload { - triggeredAt: string; + triggeredAt?: string; + /** + * How many times this tick has already handed itself on. Absent on the + * cron's own job. See `nextDrainContinuation`. + */ + continuation?: number; } const allQueues = [ @@ -437,12 +447,15 @@ export async function registerRollupQueues( MEAN_CONSOLIDATION_QUEUE, { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, async (jobs) => { + const shouldStop = jobBudget(jobs); let daysTotal = 0; let perSampleRowsTotal = 0; + let stoppedEarly = false; for (const job of jobs) { const { userId } = job.data; - const { daysConsolidated, perSampleRowsSoftDeleted } = - await runMeanConsolidationForUser(userId); + const summary = await runMeanConsolidationForUser(userId, shouldStop); + const { daysConsolidated, perSampleRowsSoftDeleted } = summary; + stoppedEarly ||= summary.stoppedEarly; daysTotal += daysConsolidated; perSampleRowsTotal += perSampleRowsSoftDeleted; workerLog( @@ -450,10 +463,14 @@ export async function registerRollupQueues( `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, ); } + // Out of budget is a completed run, not a failed one: the folded days + // are committed, and the nightly tick carries on from the first day + // this run did not reach. return jobDone({ users: jobs.length, days_consolidated: daysTotal, per_sample_rows_soft_deleted: perSampleRowsTotal, + stopped_early: stoppedEarly, }); }, ); @@ -470,17 +487,24 @@ export async function registerRollupQueues( DENSE_INTRADAY_RETENTION_QUEUE, { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, async (jobs) => { + const shouldStop = jobBudget(jobs); let daysTotal = 0; let perSampleRowsTotal = 0; let derivedRestingRowsTotal = 0; + let stoppedEarly = false; for (const job of jobs) { const { userId } = job.data; try { + const summary = await runDenseIntradayRetentionForUser( + userId, + shouldStop, + ); const { daysConsolidated, perSampleRowsSoftDeleted, derivedRestingRowsUpserted, - } = await runDenseIntradayRetentionForUser(userId); + } = summary; + stoppedEarly ||= summary.stoppedEarly; daysTotal += daysConsolidated; perSampleRowsTotal += perSampleRowsSoftDeleted; derivedRestingRowsTotal += derivedRestingRowsUpserted; @@ -503,6 +527,10 @@ export async function registerRollupQueues( days_consolidated: daysTotal, per_sample_rows_soft_deleted: perSampleRowsTotal, derived_resting_rows_upserted: derivedRestingRowsTotal, + // Out of budget completes the job: the folded days are committed and + // the nightly tick, which hands itself on while work remains, + // continues from the first day this run did not reach. + stopped_early: stoppedEarly, }); }, ); @@ -679,6 +707,9 @@ export async function registerRollupQueues( DRAIN_CUMULATIVE_QUEUE, { localConcurrency: 1 }, async (jobs) => { + // One budget shared by all three passes, run in turn. + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; // The three drains below are independent passes sharing one nightly // tick, so one failing pass must not stop the other two from running. // The first failure is held back and reported once the tick is over — @@ -699,8 +730,10 @@ export async function registerRollupQueues( const summary = await drainPerSampleCumulative(getWorkerPrisma(), { dryRun: false, cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, + shouldStop, log: (line) => workerLog("info", line), }); + stoppedEarly ||= summary.stoppedEarly; bucketsCollapsed += summary.totals.bucketsCollapsed; perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; workerLog( @@ -729,8 +762,10 @@ export async function registerRollupQueues( const meanSummary = await consolidateDailyMean(getWorkerPrisma(), { dryRun: false, cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, + shouldStop, log: (line) => workerLog("info", line), }); + stoppedEarly ||= meanSummary.stoppedEarly; meanDaysConsolidated += meanSummary.totals.daysConsolidated; meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; workerLog( @@ -769,9 +804,11 @@ export async function registerRollupQueues( getWorkerPrisma(), { dryRun: false, + shouldStop, log: (line) => workerLog("info", line), }, ); + stoppedEarly ||= denseSummary.stoppedEarly; denseDaysConsolidated += denseSummary.totals.daysConsolidated; denseRowsSoftDeleted += denseSummary.totals.perSampleRowsSoftDeleted; @@ -790,7 +827,41 @@ export async function registerRollupQueues( } } + // Hand the remaining work to a follow-up job when this one ran out of + // budget (bounded; see `nextDrainContinuation`). + const continuation = Math.max( + 0, + ...jobs.map((job) => job.data.continuation ?? 0), + ); + const next = nextDrainContinuation({ + stoppedEarly, + daysFolded: + bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated, + continuation, + }); + let continued = false; + if (next !== null) { + try { + const followUp: DrainCumulativePayload = { + triggeredAt: new Date().toISOString(), + continuation: next, + }; + continued = + (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, { + ...cronIsTheRetry, + startAfter: DRAIN_CONTINUATION_DELAY_SECONDS, + })) !== null; + } catch (err) { + // The next nightly tick picks the work up regardless; a lost + // follow-up only delays it. + workerLog("error", "[drain-cumulative] follow-up send failed", err); + } + } + const did = { + stopped_early: stoppedEarly, + continuation, + continued, passes_failed: passesFailed, buckets_collapsed: bucketsCollapsed, per_sample_rows_deleted: perSampleRowsDeleted, From 9411a485c8536d5813e52173509bba1dccc57c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 00:31:55 +0200 Subject: [PATCH 04/12] fix(backup): store the weekly and manual backup piece by piece The backup streamed its JSON through gzip and the cipher, but kept the stored copy as one string and handed that to Postgres in one parameter. On a seeded account of 1.25 million measurements the copy is 64 MB of base64, and the pieces, the joined string and the driver's copies of it grew the process from 309 MB to 946 MB resident. A bare process needed a heap limit above 400 MB to finish it (the new path finishes under 220 MB). With the dense retention running beside it in a 1 GB container the process died of heap exhaustion, and the backup job sat "active" until its two-hour limit marked it "job timed out" (#1031). `packBackupBlobInto` hands the stored copy on in pieces of up to 4 MB as it is produced, and `storeBackupBlob` writes them into a temporary table on its own transaction and assembles the row with one `string_agg`. The process never holds the whole copy. The account's previous copy stays in place until the new one is complete: a failure rolls back to it. Measured on the seeded account in a 1 GB container, web and worker in one process: the backup completes in about 30 s with the dense retention running beside it, where v1.39.0 crashed within 30 s. The streamed JSON also reports its progress to the job every 50 000 rows. --- src/lib/export/__tests__/backup-blob.test.ts | 65 +++++ src/lib/export/backup-blob.ts | 81 ++++-- src/lib/export/full-backup-stream.ts | 7 + src/lib/export/store-backup-blob.ts | 113 ++++++++ .../__tests__/backup-handlers.test.ts | 64 ++--- src/lib/jobs/reminder/backup-handlers.ts | 57 ++-- tests/integration/large-account-jobs.test.ts | 266 ++++++++++++++++++ 7 files changed, 571 insertions(+), 82 deletions(-) create mode 100644 src/lib/export/store-backup-blob.ts create mode 100644 tests/integration/large-account-jobs.test.ts diff --git a/src/lib/export/__tests__/backup-blob.test.ts b/src/lib/export/__tests__/backup-blob.test.ts index cf3225a34..4f632bebb 100644 --- a/src/lib/export/__tests__/backup-blob.test.ts +++ b/src/lib/export/__tests__/backup-blob.test.ts @@ -17,6 +17,7 @@ process.env.ENCRYPTION_KEY ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +import { randomBytes } from "node:crypto"; import v8 from "node:v8"; import { describe, expect, it } from "vitest"; @@ -26,6 +27,7 @@ import { BackupBlobTooLargeError, defaultBackupBlobLimit, packBackupBlob, + packBackupBlobInto, packBackupBlobStreaming, unpackBackupBlob, } from "../backup-blob"; @@ -213,3 +215,66 @@ describe("backup blob envelope", () => { expect(limit).toBeGreaterThan(16 * 1024 * 1024); }); }); + +/** + * Issue #1031: the weekly and manual backup kept the stored copy as one string + * before writing it, and at 1.25 million measurements that string and the + * driver's copies of it added about 640 MB to the process. `packBackupBlobInto` + * hands the stored copy on in bounded pieces instead, as it is produced. + */ +describe("piecewise envelope", () => { + /** JSON that gzip cannot shrink much, so the stored copy is several MB. */ + function incompressibleJson(bytes: number): string { + return JSON.stringify({ + noise: randomBytes(Math.ceil((bytes * 3) / 4)).toString("base64"), + }); + } + + it("delivers pieces that concatenate to a blob reading back the exact JSON", async () => { + const json = sampleJson(3_000); + const pieces: string[] = []; + await packBackupBlobInto( + (piece) => { + pieces.push(piece); + }, + async (write) => { + for (let at = 0; at < json.length; at += 4_096) { + await write(json.slice(at, at + 4_096)); + } + }, + ); + expect(unpackBackupBlob(pieces.join(""))).toBe(json); + }); + + it("hands pieces on while the producer is still writing, each one bounded", async () => { + const json = incompressibleJson(12 * 1024 * 1024); + const pieces: number[] = []; + let producing = false; + let piecesWhileProducing = 0; + const chunks: string[] = []; + + await packBackupBlobInto( + (piece) => { + pieces.push(piece.length); + chunks.push(piece); + if (producing) piecesWhileProducing += 1; + }, + async (write) => { + producing = true; + for (let at = 0; at < json.length; at += 64 * 1024) { + await write(json.slice(at, at + 64 * 1024)); + } + producing = false; + }, + { maxBytes: Number.MAX_SAFE_INTEGER }, + ); + + expect(pieces.length).toBeGreaterThan(2); + expect(piecesWhileProducing).toBeGreaterThan(1); + // The flush threshold is 4 MB; one gzip chunk's worth of slack on top. + for (const length of pieces) { + expect(length).toBeLessThan(4 * 1024 * 1024 + 256 * 1024); + } + expect(unpackBackupBlob(chunks.join(""))).toBe(json); + }); +}); diff --git a/src/lib/export/backup-blob.ts b/src/lib/export/backup-blob.ts index a05760650..f305a67b4 100644 --- a/src/lib/export/backup-blob.ts +++ b/src/lib/export/backup-blob.ts @@ -141,26 +141,59 @@ export interface PackBackupBlobOptions { * Serialised backup JSON, produced in pieces → the stored string. * * The pipeline is JSON piece → gzip → AES-256-GCM → base64, with nothing - * buffered end to end but the base64 answer. `producer` decides how big its - * pieces are; the gzip stream applies backpressure through the promise this - * hands back, so a fast producer cannot outrun the compressor and pile up - * chunks in the stream's internal queue. - * - * The one copy that cannot be avoided is the answer itself. `data_backups.data` - * is a single `text` column, so the row has to be one value, and one value has - * to exist as one string before the driver can bind it. That copy is therefore - * also the only thing here that grows without bound as a record grows, which - * is why the size check lives on it: `maxBytes` counts the ciphertext this - * call has actually produced and stops on the account whose backup does not - * fit, rather than reading a heap gauge every account shares. + * buffered end to end but the base64 answer. Kept for callers that need the + * answer as one value; the weekly and manual backup no longer do, they hand + * the pieces to Postgres as they come (`packBackupBlobInto` below, via + * `storeBackupBlob`). */ export async function packBackupBlobStreaming( producer: BackupJsonProducer, options: PackBackupBlobOptions = {}, ): Promise { + const pieces: string[] = []; + await packBackupBlobInto( + (piece) => { + pieces.push(piece); + }, + producer, + options, + ); + return pieces.join(""); +} + +/** Receives the stored string a piece at a time, in order. Awaited. */ +export type BackupBlobSink = (piece: string) => void | Promise; + +/** + * How much base64 to gather before handing it to the sink. Large enough that + * a sink writing to the database makes a few dozen round trips for a large + * record rather than thousands, small enough to be irrelevant to the heap. + */ +const SINK_FLUSH_BYTES = 4 * 1024 * 1024; + +/** + * Serialised backup JSON, produced in pieces → the stored string, delivered + * in pieces. + * + * The concatenation of every piece `sink` receives is exactly the string + * `packBackupBlobStreaming` returns. `producer` decides how big its JSON + * pieces are; the gzip stream applies backpressure through the promise this + * hands back, so a fast producer cannot outrun the compressor, and a slow + * sink holds the producer back the same way, so neither side piles up. + * + * `maxBytes` still counts the whole answer. Nothing here holds it any more, + * but every reader of a stored copy (restore, download, summary) opens it as + * one value, so the limit is what keeps a copy restorable on this host. + */ +export async function packBackupBlobInto( + sink: BackupBlobSink, + producer: BackupJsonProducer, + options: PackBackupBlobOptions = {}, +): Promise { const limitBytes = options.maxBytes ?? defaultBackupBlobLimit(); const encryptor = createStreamEncryptor(); - const pieces: string[] = [encryptor.header]; + let pending: string[] = [encryptor.header]; + let pendingBytes = encryptor.header.length; let heldBytes = encryptor.header.length; const gzip = createGzip(); @@ -169,12 +202,13 @@ export async function packBackupBlobStreaming( try { const piece = encryptor.update(chunk); if (piece === "") return; - // Base64 is ASCII, so a character counted here is a byte held here. + // Base64 is ASCII, so a character counted here is a byte written here. heldBytes += piece.length; if (heldBytes > limitBytes) { throw new BackupBlobTooLargeError(heldBytes, limitBytes); } - pieces.push(piece); + pending.push(piece); + pendingBytes += piece.length; } catch (err) { failure ??= err; gzip.destroy(err as Error); @@ -192,8 +226,19 @@ export async function packBackupBlobStreaming( // rejection; `await finished` below still sees the rejection. finished.catch(() => {}); + const flush = async (force: boolean): Promise => { + if (pendingBytes === 0 || (!force && pendingBytes < SINK_FLUSH_BYTES)) { + return; + } + const piece = pending.join(""); + pending = []; + pendingBytes = 0; + await sink(piece); + }; + const write = async (chunk: string): Promise => { if (failure) throw failure; + await flush(false); if (gzip.write(chunk, "utf8")) return; await new Promise((resolve, reject) => { const onDrain = () => { @@ -219,8 +264,10 @@ export async function packBackupBlobStreaming( await finished; if (failure) throw failure; - pieces.push(encryptor.final()); - return pieces.join(""); + const tail = encryptor.final(); + pending.push(tail); + pendingBytes += tail.length; + await flush(true); } /** diff --git a/src/lib/export/full-backup-stream.ts b/src/lib/export/full-backup-stream.ts index c5b364aa0..1f5b1cdcf 100644 --- a/src/lib/export/full-backup-stream.ts +++ b/src/lib/export/full-backup-stream.ts @@ -26,6 +26,7 @@ * trusting that paragraph. */ import type { PrismaClient } from "@/generated/prisma/client"; +import { reportJobProgress } from "@/lib/jobs/job-observer"; import { buildFullBackupPayload, isDeferredRows, @@ -45,6 +46,9 @@ export type BackupJsonSink = (chunk: string) => void | Promise; */ const FLUSH_BYTES = 256 * 1024; +/** How often a bulk section reports how far it has got to a running job. */ +const PROGRESS_EVERY_ROWS = 50_000; + /** * There is deliberately no heap reading in this writer. * @@ -125,6 +129,9 @@ export async function streamFullBackupJson( rows === 0 ? JSON.stringify(row) : `,${JSON.stringify(row)}`, ); rows++; + if (rows % PROGRESS_EVERY_ROWS === 0) { + reportJobProgress({ backup_section: key, backup_section_rows: rows }); + } } await write("]"); bulkCounts[key] = rows; diff --git a/src/lib/export/store-backup-blob.ts b/src/lib/export/store-backup-blob.ts new file mode 100644 index 000000000..cf7989269 --- /dev/null +++ b/src/lib/export/store-backup-blob.ts @@ -0,0 +1,113 @@ +/** + * Write one account's stored backup (`data_backups.data`) without ever + * holding the stored copy in this process. + * + * Why. `packBackupBlobStreaming` streams the JSON through gzip and the cipher, + * but its answer is one string, because the column takes one value. Measured + * on a seeded account of 1.25 million measurements: the answer is 64 MB of + * base64, and between the pieces, the joined string and the copies the + * database driver makes to bind it, the backup grew the process by about + * 640 MB of resident memory (309 MB → 946 MB). The default compose stack runs + * the web server and the worker in one container capped at 1 GB, so on a + * record of that size the weekly or manual backup is the thing that takes the + * container down; a killed worker leaves its job `active` until pg-boss + * expires it two hours later as `job timed out`. + * + * What instead. The pieces go to Postgres as they come, a few megabytes at a + * time, into a temporary table on the transaction's own connection, and one + * statement assembles them into the row with `string_agg`. The process never + * holds more than one piece; Postgres builds the value it was going to store + * anyway. + * + * All of it is one transaction, so the account's previous copy stays in place + * until the new one is complete: a failure halfway through rolls back to it, + * and the temporary table goes with the transaction either way. + */ +import type { PrismaClient } from "@/generated/prisma/client"; + +import { + packBackupBlobInto, + type BackupJsonProducer, + type PackBackupBlobOptions, +} from "@/lib/export/backup-blob"; + +/** + * How long the storing transaction may stay open. The job around it expires + * after two hours (`DATA_BACKUP_SEND_OPTIONS`); the transaction has to give + * up before that so the failure is the transaction's own, with its own + * message, rather than an expired job. + */ +const STORE_TRANSACTION_TIMEOUT_MS = 90 * 60 * 1000; + +/** + * How long the transaction may sit idle between two pieces. The session + * default (`idle_in_transaction_session_timeout`, 60 s, see `src/lib/db.ts`) + * is right for a request and wrong here: the producer reads the whole record + * on other connections while this one waits for the next piece, and the + * sections before the first measurement can take longer than a minute on a + * large account. Scoped with SET LOCAL, so it ends with the transaction. + */ +const STORE_IDLE_TIMEOUT = "10min"; + +export interface StoreBackupBlobInput { + userId: string; + /** `DataBackup.type`: `WEEKLY_AUTO` for the scheduled and manual pass. */ + type: string; +} + +/** + * Pack `producer`'s JSON into the stored envelope and upsert it as the + * account's `(userId, type)` backup. Resolves to the stored size in bytes. + */ +export async function storeBackupBlob( + prisma: PrismaClient, + input: StoreBackupBlobInput, + producer: BackupJsonProducer, + options: PackBackupBlobOptions = {}, +): Promise { + return prisma.$transaction( + async (tx) => { + await tx.$executeRawUnsafe( + `SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`, + ); + await tx.$executeRaw` + CREATE TEMP TABLE backup_blob_parts ( + seq integer PRIMARY KEY, + piece text NOT NULL + ) ON COMMIT DROP + `; + + let seq = 0; + let bytes = 0; + await packBackupBlobInto( + async (piece) => { + await tx.$executeRaw` + INSERT INTO backup_blob_parts (seq, piece) VALUES (${seq}, ${piece}) + `; + seq += 1; + bytes += piece.length; + }, + producer, + options, + ); + + // The row first, through Prisma, so a new account's backup gets its id + // the same way every other row does; the data it briefly carries is + // never visible outside this transaction. + await tx.dataBackup.upsert({ + where: { userId_type: { userId: input.userId, type: input.type } }, + update: { createdAt: new Date() }, + create: { userId: input.userId, type: input.type, data: "" }, + }); + await tx.$executeRaw` + UPDATE data_backups + SET data = ( + SELECT string_agg(piece, '' ORDER BY seq) FROM backup_blob_parts + ) + WHERE user_id = ${input.userId} AND type = ${input.type} + `; + return bytes; + }, + { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 }, + ); +} diff --git a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts index 7692ca02d..fe2a94b55 100644 --- a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts +++ b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts @@ -4,9 +4,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ buildFullBackupPayload: vi.fn(), - packBlob: vi.fn((value: string) => value), getWorkerPrisma: vi.fn(), - upsert: vi.fn(), + store: vi.fn(), })); // Only the payload builder is stubbed. The REAL streaming writer runs on top @@ -19,27 +18,25 @@ vi.mock("@/lib/export/full-backup-payload", () => ({ isDeferredRows: () => false, })); -// The envelope is exercised end-to-end in -// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so -// the stored bytes can be read back as JSON. The size error is the real class -// so the handler's `instanceof` arm is the one that runs. -vi.mock("@/lib/export/backup-blob", async () => { - const actual = await vi.importActual< - typeof import("@/lib/export/backup-blob") - >("@/lib/export/backup-blob"); - return { - BackupBlobTooLargeError: actual.BackupBlobTooLargeError, - packBackupBlobStreaming: async ( - produce: (write: (chunk: string) => Promise) => Promise, - ) => { - let out = ""; - await produce(async (chunk) => { - out += chunk; - }); - return mocks.packBlob(out); - }, - }; -}); +// The envelope and the piecewise store are exercised end-to-end in +// `src/lib/export/__tests__/backup-blob.test.ts` and the integration suite; +// here the store stays transparent so the stored bytes can be read back as +// JSON. `mocks.store` receives what would have been stored and decides +// whether the write succeeds. +vi.mock("@/lib/export/store-backup-blob", () => ({ + storeBackupBlob: async ( + _prisma: unknown, + input: { userId: string; type: string }, + produce: (write: (chunk: string) => Promise) => Promise, + ) => { + let out = ""; + await produce(async (chunk) => { + out += chunk; + }); + await mocks.store(input, out); + return out.length; + }, +})); vi.mock("@/lib/logging/background", () => ({ withBackgroundEvent: vi.fn( @@ -70,7 +67,6 @@ function buildPrismaMock( ) { return { user: { findMany: vi.fn().mockResolvedValue(users) }, - dataBackup: { upsert: mocks.upsert }, }; } @@ -78,7 +74,7 @@ describe("handleDataBackup canonical DR payload", () => { beforeEach(() => { vi.clearAllMocks(); mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock()); - mocks.upsert.mockResolvedValue({}); + mocks.store.mockResolvedValue(undefined); mocks.buildFullBackupPayload.mockResolvedValue({ payload: { schemaVersion: "1", @@ -117,8 +113,12 @@ describe("handleDataBackup canonical DR payload", () => { deferBulk: true, }), ); - expect(mocks.upsert).toHaveBeenCalledOnce(); - const encrypted = mocks.upsert.mock.calls[0]![0].create.data as string; + expect(mocks.store).toHaveBeenCalledOnce(); + expect(mocks.store.mock.calls[0]![0]).toEqual({ + userId: "user-dr", + type: "WEEKLY_AUTO", + }); + const encrypted = mocks.store.mock.calls[0]![1] as string; const payload = JSON.parse(encrypted) as { moodEntries: Array<{ id: string; @@ -155,7 +155,7 @@ describe("handleDataBackup canonical DR payload", () => { describe("handleDataBackup outcome", () => { beforeEach(() => { vi.clearAllMocks(); - mocks.upsert.mockResolvedValue({}); + mocks.store.mockResolvedValue(undefined); mocks.buildFullBackupPayload.mockResolvedValue({ payload: { schemaVersion: "1", userId: "user-dr" }, counts: {}, @@ -164,7 +164,7 @@ describe("handleDataBackup outcome", () => { it("fails the run when not one account got a copy", async () => { mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock()); - mocks.upsert.mockRejectedValue(new Error("write failed")); + mocks.store.mockRejectedValue(new Error("write failed")); const outcome = await handleDataBackup([]); @@ -178,7 +178,7 @@ describe("handleDataBackup outcome", () => { it("counts an oversized record as the reason it could not", async () => { mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock()); - mocks.upsert.mockRejectedValue(new BackupBlobTooLargeError(9_000, 4_096)); + mocks.store.mockRejectedValue(new BackupBlobTooLargeError(9_000, 4_096)); const outcome = await handleDataBackup([]); @@ -198,9 +198,9 @@ describe("handleDataBackup outcome", () => { { id: "user-b", username: "b" }, ]), ); - mocks.upsert + mocks.store .mockRejectedValueOnce(new Error("write failed")) - .mockResolvedValueOnce({}); + .mockResolvedValueOnce(undefined); const outcome = await handleDataBackup([]); diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 48c6ae397..13c258894 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -6,10 +6,8 @@ */ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; -import { - BackupBlobTooLargeError, - packBackupBlobStreaming, -} from "@/lib/export/backup-blob"; +import { BackupBlobTooLargeError } from "@/lib/export/backup-blob"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -18,6 +16,7 @@ import { runOffhostBackup, } from "@/lib/jobs/offhost-backup"; import { getWorkerPrisma } from "./shared"; +import { reportJobProgress } from "@/lib/jobs/job-observer"; export interface DataBackupPayload { triggeredAt: string; @@ -118,38 +117,30 @@ export async function handleDataBackup( // six weeks in. let lastError: unknown; for (const user of users) { + reportJobProgress({ + backup_user: user.id, + backup_users_done: backed + usersFailed, + backup_users_total: users.length, + }); try { // Streamed, compressed, then encrypted (the record contains - // sensitive health information). Nothing between the database rows - // and this string exists as a whole: the payload goes into gzip a - // page at a time and the cipher consumes gzip's output as it comes, - // because a materialised copy of a large record is what used to take - // the process down. Both legs live in `packBackupBlobStreaming`, - // which is also what the restore path reads back through. - const encryptedBackup = await packBackupBlobStreaming((write) => - streamFullBackupJson(prisma, user.id, write, { - purpose: "disaster-recovery", - }), + // sensitive health information), and stored a piece at a time. + // Nothing between the database rows and the stored row exists as a + // whole in this process: the payload goes into gzip a page at a + // time, the cipher consumes gzip's output as it comes, and the + // ciphertext goes back to Postgres in pieces that one statement + // assembles into the row (`storeBackupBlob`). The stored copy used + // to be built here as one string first, which on a large record + // added several hundred megabytes to a container capped at 1 GB. + const storedBytes = await storeBackupBlob( + prisma, + { userId: user.id, type: "WEEKLY_AUTO" }, + (write) => + streamFullBackupJson(prisma, user.id, write, { + purpose: "disaster-recovery", + }), ); - largestBlobBytes = Math.max( - largestBlobBytes, - Buffer.byteLength(encryptedBackup, "utf8"), - ); - - await prisma.dataBackup.upsert({ - where: { - userId_type: { userId: user.id, type: "WEEKLY_AUTO" }, - }, - update: { - data: encryptedBackup, - createdAt: new Date(), - }, - create: { - userId: user.id, - type: "WEEKLY_AUTO", - data: encryptedBackup, - }, - }); + largestBlobBytes = Math.max(largestBlobBytes, storedBytes); backed++; } catch (err) { // One user's payload failing is that user's problem, not the pass's: diff --git a/tests/integration/large-account-jobs.test.ts b/tests/integration/large-account-jobs.test.ts new file mode 100644 index 000000000..9568c0235 --- /dev/null +++ b/tests/integration/large-account-jobs.test.ts @@ -0,0 +1,266 @@ +/** + * Issue #1031 — maintenance jobs on a large account. + * + * A self-hoster with 1.25 million measurement rows (a multi-year Apple Health + * export) saw the dense intra-day retention, the nightly drain tick and the + * data backup all fail with `job timed out`. Reproduced against a seeded + * account of the same size (`scripts/seed-large-account.ts`): the drains held + * a type's whole history in memory before folding a day, spent about 90 + * statements per folded day, and paged with a cursor Postgres could not use as + * an index condition; the backup added about 640 MB to the process to store + * one 64 MB row. pg-boss declared the jobs dead at their expiry without + * stopping them. + * + * A million rows do not fit a test run, so this file pins the properties the + * fix rests on at a scale that does: a pass that runs out of budget stops + * cleanly and the next run finishes the job with the same result as one + * uninterrupted run, and the backup is stored piecewise with the previous + * copy intact if a run fails halfway. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; +import { consolidateDailyMean } from "@/lib/measurements/consolidate-daily-mean"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; + +const TZ = "Europe/Berlin"; +const RUN_A = "user-1031-stopped"; +const RUN_B = "user-1031-straight"; +const DAYS = 12; +/** Oldest seeded instant: well outside the 90-day raw window. */ +const START = Date.UTC(2026, 0, 1, 0, 7); + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + for (const id of [RUN_A, RUN_B]) { + await prisma.user.create({ + data: { id, username: id, email: `${id}@example.test`, timezone: TZ }, + }); + } +}); + +/** The same history for both users: heart rate every 20 minutes, HRV hourly. */ +async function seedHistory(userId: string): Promise { + const rows = []; + for (let minute = 0; minute < DAYS * 24 * 60; minute += 20) { + const at = new Date(START + minute * 60_000); + rows.push({ + userId, + type: "PULSE" as const, + value: 55 + ((minute / 20) % 40), + unit: "bpm", + source: "APPLE_HEALTH" as const, + measuredAt: at, + externalId: `hk-hr-${minute}`, + }); + if (minute % 60 === 0) { + rows.push({ + userId, + type: "HEART_RATE_VARIABILITY" as const, + value: 30 + ((minute / 60) % 25), + unit: "ms", + source: "APPLE_HEALTH" as const, + measuredAt: at, + externalId: `hk-hrv-${minute}`, + }); + } + if (minute % 40 === 0) { + rows.push({ + userId, + type: "WALKING_SPEED" as const, + value: 4 + ((minute / 40) % 10) / 10, + unit: "km/h", + source: "APPLE_HEALTH" as const, + measuredAt: at, + externalId: `hk-ws-${minute}`, + }); + } + } + await getPrismaClient().measurement.createMany({ data: rows }); +} + +/** What a reader sees: every live row, identity-free. */ +async function liveRows(userId: string) { + const rows = await getPrismaClient().measurement.findMany({ + where: { userId, deletedAt: null }, + select: { + type: true, + value: true, + unit: true, + source: true, + measuredAt: true, + externalId: true, + }, + orderBy: [{ type: "asc" }, { measuredAt: "asc" }], + }); + return rows.map((row) => ({ + ...row, + measuredAt: row.measuredAt.toISOString(), + })); +} + +/** A `shouldStop` that lets `days` day buckets through, then says stop. */ +function afterDays(days: number): () => boolean { + let left = days; + return () => left-- <= 0; +} + +describe("a pass that runs out of budget (#1031)", () => { + it("dense retention: stops cleanly, and the next run ends where one straight run ends", async () => { + await seedHistory(RUN_A); + await seedHistory(RUN_B); + const prisma = getPrismaClient(); + + const first = await runDenseIntradayRetention(prisma, { + userId: RUN_A, + shouldStop: afterDays(5), + log: () => {}, + }); + expect(first.stoppedEarly).toBe(true); + expect(first.totals.daysConsolidated).toBe(5); + + // What the stopped run folded is committed: the next scan no longer sees + // it, so the second run folds only what is left. + const second = await runDenseIntradayRetention(prisma, { + userId: RUN_A, + log: () => {}, + }); + expect(second.stoppedEarly).toBe(false); + + const straight = await runDenseIntradayRetention(prisma, { + userId: RUN_B, + log: () => {}, + }); + expect(first.totals.daysConsolidated + second.totals.daysConsolidated).toBe( + straight.totals.daysConsolidated, + ); + expect( + first.totals.perSampleRowsSoftDeleted + + second.totals.perSampleRowsSoftDeleted, + ).toBe(straight.totals.perSampleRowsSoftDeleted); + + // A third run finds nothing: the pass converged. + const third = await runDenseIntradayRetention(prisma, { + userId: RUN_A, + log: () => {}, + }); + expect(third.totals.daysConsolidated).toBe(0); + + const stoppedResult = (await liveRows(RUN_A)).filter( + (row) => row.type !== "WALKING_SPEED", + ); + const straightResult = (await liveRows(RUN_B)).filter( + (row) => row.type !== "WALKING_SPEED", + ); + expect(stoppedResult.length).toBeGreaterThan(0); + expect(stoppedResult).toEqual(straightResult); + }); + + it("daily-mean consolidation: the same, for the nightly tick's middle pass", async () => { + await seedHistory(RUN_A); + await seedHistory(RUN_B); + const prisma = getPrismaClient(); + + const first = await consolidateDailyMean(prisma, { + userId: RUN_A, + shouldStop: afterDays(4), + log: () => {}, + }); + expect(first.stoppedEarly).toBe(true); + expect(first.totals.daysConsolidated).toBe(4); + await consolidateDailyMean(prisma, { userId: RUN_A, log: () => {} }); + await consolidateDailyMean(prisma, { userId: RUN_B, log: () => {} }); + + const stopped = (await liveRows(RUN_A)).filter( + (row) => row.type === "WALKING_SPEED", + ); + const straight = (await liveRows(RUN_B)).filter( + (row) => row.type === "WALKING_SPEED", + ); + // One daily mean per local day: twelve UTC days touch thirteen Berlin days. + expect(stopped).toHaveLength(DAYS + 1); + expect(stopped).toEqual(straight); + }); +}); + +describe("storeBackupBlob (#1031)", () => { + /** A producer writing `rows` JSON rows in small pieces. */ + function producer(rows: number, failAfter?: number) { + return async (write: (chunk: string) => Promise) => { + await write('{"measurements":['); + for (let i = 0; i < rows; i++) { + if (failAfter !== undefined && i === failAfter) { + throw new Error("record read failed halfway"); + } + await write( + `${i === 0 ? "" : ","}${JSON.stringify({ id: `m-${i}`, value: i })}`, + ); + } + await write("]}"); + }; + } + + function expectedJson(rows: number): string { + return JSON.stringify({ + measurements: Array.from({ length: rows }, (_, i) => ({ + id: `m-${i}`, + value: i, + })), + }); + } + + it("stores a copy that reads back as exactly the JSON produced", async () => { + const prisma = getPrismaClient(); + const bytes = await storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(20_000), + ); + + const row = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + expect(row.data.length).toBe(bytes); + expect(unpackBackupBlob(row.data)).toBe(expectedJson(20_000)); + }); + + it("keeps the previous copy when a run fails halfway, and the next run still stores", async () => { + const prisma = getPrismaClient(); + await storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(10), + ); + const before = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + + await expect( + storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(20_000, 15_000), + ), + ).rejects.toThrow("record read failed halfway"); + + const after = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + expect(after.data).toBe(before.data); + expect(after.createdAt.getTime()).toBe(before.createdAt.getTime()); + + // The temporary table went with the failed transaction. + await storeBackupBlob( + prisma, + { userId: RUN_A, type: "WEEKLY_AUTO" }, + producer(30), + ); + const replaced = await prisma.dataBackup.findUniqueOrThrow({ + where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } }, + }); + expect(unpackBackupBlob(replaced.data)).toBe(expectedJson(30)); + }); +}); From 79ed62b7f9e45928dd7f4dc224122169f61c5b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 00:32:10 +0200 Subject: [PATCH 05/12] chore(scripts): add a seed for an account the size of a multi-year import `scripts/seed-large-account.ts` inserts about 1.25 million measurements for one user, shaped like an Apple Health export plus the iOS sync: dense heart rate, HRV and SpO2, the high-frequency mean types, export `stats:` daily rows, recent per-sample cumulative rows and nightly sleep stages. Everything is generated in Postgres, so a run takes seconds, and it ends with ANALYZE. It is how the maintenance jobs in #1031 were reproduced and measured; never point it at a database holding real data. --- scripts/seed-large-account.ts | 296 ++++++++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 scripts/seed-large-account.ts diff --git a/scripts/seed-large-account.ts b/scripts/seed-large-account.ts new file mode 100644 index 000000000..fba2c078c --- /dev/null +++ b/scripts/seed-large-account.ts @@ -0,0 +1,296 @@ +/** + * Seed one account shaped like a multi-year Apple Health `export.xml` import, + * for reproducing the maintenance jobs at the size a real heavy account + * reaches (issue #1031: about 1.26 million measurement rows). + * + * DATABASE_URL=postgresql://… pnpm dlx tsx scripts/seed-large-account.ts \ + * [--user bench-large] [--years 4] [--pulse-seconds 165] + * + * Never point this at a database that holds real data: it inserts rows under + * the given user id and, with `--reset`, deletes that user first. + * + * The shape follows what the import and the iOS sync actually write: + * + * - dense spot rows (heart rate every `--pulse-seconds`, HRV, SpO2) keyed by + * a hashed sample id, the tier the dense-intraday retention folds; + * - high-frequency mean rows (respiratory rate, gait metrics, audio + * exposure), the tier the daily-mean consolidation folds; + * - one `stats:` daily row per cumulative type and day (what the export + * import writes for steps, energy, distance and flights); + * - per-sample cumulative rows with opaque sample ids for the most recent + * 90 days (what the iOS sync writes, and what the cumulative drain folds); + * - sleep-stage segments every night. + * + * Everything is generated inside Postgres with `generate_series`, so a million + * rows take seconds rather than a million round trips. `ANALYZE` runs at the + * end: a planner reading pre-seed statistics picks plans no production table + * of this size would get. + */ +import { prisma } from "@/lib/db"; + +function arg(name: string, fallback: string): string { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback; +} + +const USER_ID = arg("user", "bench-large"); +const YEARS = Number(arg("years", "4")); +const PULSE_SECONDS = Number(arg("pulse-seconds", "165")); +const RESET = process.argv.includes("--reset"); + +/** A spot series: `perDay` evenly spaced rows per day over the whole span. */ +interface SpotSeries { + type: string; + unit: string; + perDay: number; + base: number; + spread: number; + /** Days back from now the series covers. Defaults to the whole span. */ + days?: number; + /** externalId prefix; `stats:` rows are generated separately. */ + idPrefix: string; +} + +async function insertSpot(s: SpotSeries, days: number): Promise { + const step = Math.floor(86_400 / s.perDay); + const count = Math.floor((days * 86_400) / step); + // Jitter stays below the step so every measured_at is distinct per type + // (the (user, type, measured_at, source, sleep_stage) unique index). + const jitter = Math.max(1, Math.min(59, step - 1)); + return prisma.$executeRawUnsafe( + ` + INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, external_id, created_at, updated_at) + SELECT + 'seed-' || $1 || '-' || $2 || '-' || g, + $1, + $2::measurement_type, + round(($3 + $4 * sin(g / 37.0) + ($4 / 2) * ((g * 7919) % 100) / 100.0)::numeric, 2), + $5, + 'APPLE_HEALTH', + date_trunc('minute', now()) - make_interval(secs => $6 * $7::bigint) + + make_interval(secs => g * $6 + (g * 31) % $8), + $9 || md5($2 || g::text), + now(), now() + FROM generate_series(0, $7::bigint - 1) AS g + `, + USER_ID, + s.type, + s.base, + s.spread, + s.unit, + step, + count, + jitter, + s.idPrefix, + ); +} + +async function main(): Promise { + const days = Math.round(YEARS * 365); + + if (RESET) { + await prisma.$executeRawUnsafe(`DELETE FROM users WHERE id = $1`, USER_ID); + } + await prisma.$executeRawUnsafe( + `INSERT INTO users (id, username, timezone, updated_at) + VALUES ($1, $1, 'Europe/Berlin', now()) + ON CONFLICT (id) DO NOTHING`, + USER_ID, + ); + + const pulsePerDay = Math.floor(86_400 / PULSE_SECONDS); + const spot: SpotSeries[] = [ + { + type: "PULSE", + unit: "bpm", + perDay: pulsePerDay, + base: 62, + spread: 25, + idPrefix: "hk-", + }, + { + type: "HEART_RATE_VARIABILITY", + unit: "ms", + perDay: 12, + base: 45, + spread: 20, + idPrefix: "hk-", + }, + { + type: "OXYGEN_SATURATION", + unit: "%", + perDay: 24, + base: 96, + spread: 2, + idPrefix: "hk-", + }, + { + type: "RESPIRATORY_RATE", + unit: "breaths/min", + perDay: 48, + base: 14, + spread: 3, + idPrefix: "hk-", + }, + { + type: "WALKING_SPEED", + unit: "km/h", + perDay: 30, + base: 4.8, + spread: 0.8, + idPrefix: "hk-", + }, + { + type: "WALKING_STEP_LENGTH", + unit: "cm", + perDay: 30, + base: 72, + spread: 6, + idPrefix: "hk-", + }, + { + type: "WALKING_ASYMMETRY", + unit: "%", + perDay: 30, + base: 3, + spread: 2, + idPrefix: "hk-", + }, + { + type: "WALKING_DOUBLE_SUPPORT", + unit: "%", + perDay: 30, + base: 28, + spread: 3, + idPrefix: "hk-", + }, + { + type: "AUDIO_EXPOSURE_ENV", + unit: "dB", + perDay: 60, + base: 62, + spread: 12, + idPrefix: "hk-", + }, + // The iOS sync's per-sample cumulative rows: dense, recent, opaque ids. + { + type: "ACTIVITY_STEPS", + unit: "steps", + perDay: 100, + base: 90, + spread: 60, + days: 90, + idPrefix: "uuid-", + }, + { + type: "ACTIVE_ENERGY_BURNED", + unit: "kcal", + perDay: 400, + base: 1.2, + spread: 0.9, + days: 90, + idPrefix: "uuid-", + }, + { + type: "WALKING_RUNNING_DISTANCE", + unit: "m", + perDay: 100, + base: 70, + spread: 50, + days: 90, + idPrefix: "uuid-", + }, + { + type: "WEIGHT", + unit: "kg", + perDay: 1, + base: 82, + spread: 1.5, + idPrefix: "hk-", + }, + ]; + + let total = 0; + for (const s of spot) { + const n = await insertSpot(s, s.days ?? days); + total += n; + console.log(`${s.type.padEnd(28)} ${String(n).padStart(9)}`); + } + + // One export-estimate `stats:` row per cumulative type and local day, at + // local noon — what `flushCumulativeBuckets` in the export import writes. + for (const [type, hk, unit, base] of [ + ["ACTIVITY_STEPS", "HKQuantityTypeIdentifierStepCount", "steps", 8000], + [ + "ACTIVE_ENERGY_BURNED", + "HKQuantityTypeIdentifierActiveEnergyBurned", + "kcal", + 550, + ], + [ + "WALKING_RUNNING_DISTANCE", + "HKQuantityTypeIdentifierDistanceWalkingRunning", + "m", + 6000, + ], + ["FLIGHTS_CLIMBED", "HKQuantityTypeIdentifierFlightsClimbed", "count", 9], + ] as const) { + const n = await prisma.$executeRawUnsafe( + ` + INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, external_id, + aggregation_provenance, created_at, updated_at) + SELECT + 'seed-' || $1 || '-' || $2 || '-stats-' || d, + $1, $2::measurement_type, $5 + (d * 7919) % ($5 / 2), $4, 'APPLE_HEALTH', + ((current_date - d)::timestamp + interval '12 hours') + AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC', + 'stats:' || $3 || ':' || to_char(current_date - d, 'YYYY-MM-DD'), + 'EXPORT_XML_SOURCE_MAX', now(), now() + FROM generate_series(91, $6::int) AS d + `, + USER_ID, + type, + hk, + unit, + base, + days, + ); + total += n; + console.log(`${`${type} (stats:)`.padEnd(28)} ${String(n).padStart(9)}`); + } + + // Sleep: 30 segments a night from 23:00 local, 16 minutes apart, rotating + // through the stages. Distinct measured_at per segment. + const sleep = await prisma.$executeRawUnsafe( + ` + INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, external_id, + sleep_stage, created_at, updated_at) + SELECT + 'seed-' || $1 || '-sleep-' || d || '-' || s, + $1, 'SLEEP_DURATION', 16, 'minutes', 'APPLE_HEALTH', + ((current_date - d)::timestamp - interval '1 hour' + s * interval '16 minutes') + AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC', + 'hk-' || md5('sleep' || d || '-' || s), + (ARRAY['CORE','DEEP','CORE','REM','AWAKE','CORE']::sleep_stage[])[1 + s % 6], + now(), now() + FROM generate_series(1, $2::int) AS d, generate_series(0, 29) AS s + `, + USER_ID, + days, + ); + total += sleep; + console.log(`${"SLEEP_DURATION".padEnd(28)} ${String(sleep).padStart(9)}`); + + await prisma.$executeRawUnsafe(`ANALYZE measurements`); + console.log(`${"total".padEnd(28)} ${String(total).padStart(9)}`); +} + +main() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect()); From 237acc00024890c0d21e2f033f352e41b4f6d92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 01:38:27 +0200 Subject: [PATCH 06/12] fix(backup): export, upload and restore a large account as streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An account of 1.25 million measurements could not export its data or be restored from any backup (#1031). Measured in a 1 GB container: - The full export and the passphrase export built the whole payload and its JSON string in memory; the app ran out of heap and restarted. - The weekly backup's JSON is 662 MB, longer than any string V8 can hold, so `unpackBackupBlob` threw on every read. Restore, download and the restore preview all answered "could not be decrypted". - Upload accepted 10 MB, a fraction of a large export. Every reader and writer of a backup file now streams it: - `scanBackupJson` reads a backup's bytes once, keeps every section but the named bulk arrays, and hands their elements out one at a time. `readStreamedBackup` uses it twice: a first pass that validates each measurement against the element schema and keeps the rest of the file, and a second pass that hands the measurements to the writer in batches. Stored copies and off-host objects are authenticated whole on their compressed ciphertext before any plaintext is read; only the decompression is streamed. - Restore writes the measurements with one `INSERT … FROM unnest` per batch. Prisma caches a query plan per `createMany` row count, and one 1 000-row plan kept about 145 MB alive, which alone put the restore past the heap limit. The transaction budget grows with the measurement count. The awards and ECG sections are handed the ids they reference that were written, not every written id. - The export routes write the file into the response as it is produced. The passphrase export spools its ciphertext to a temporary file because the HLX1 format puts the tag first; the format is unchanged, so existing archives open as before. - Upload accepts the raw body, JSON or gzip, up to the 512 MB request ceiling, checks it as it arrives and stores it in the same pass; the admin page compresses the file in the browser before sending it. Multipart stays for small files. - Download streams the stored JSON; the preview counts measurements from the stream. The monthly restore drill and `scripts/restore-backup.ts` read the off-host object the same way; the script can write `.json.gz`. On the seeded account in a 1 GB container: export 25 s at 389 MB peak, passphrase export 25 s, both uploads 19 s, restore of the weekly backup 94 s at 554 MB peak, and export, upload and restore round-trip to exactly the same rows. The encrypted-column guard now also recognises the raw SQL statement that writes `DataBackup.data`. --- messages/de.json | 2 +- messages/en.json | 2 +- messages/es.json | 2 +- messages/fr.json | 2 +- messages/it.json | 2 +- messages/ko.json | 2 +- messages/pl.json | 2 +- scripts/restore-backup.ts | 58 ++- .../backup-readers-stream-guard.test.ts | 45 ++ .../api/admin/backups/[id]/download/route.ts | 45 +- .../restore/__tests__/owner-mismatch.test.ts | 3 + .../api/admin/backups/[id]/restore/route.ts | 174 +++++--- .../[id]/summary/__tests__/route.test.ts | 14 +- .../api/admin/backups/[id]/summary/route.ts | 22 +- src/app/api/admin/backups/upload/route.ts | 405 ++++++++++++------ .../__tests__/soft-delete-filter.test.ts | 37 +- src/app/api/export/encrypted/route.ts | 104 +++-- src/app/api/export/full-backup/route.ts | 72 ++-- .../__tests__/backups-section-empty.test.tsx | 4 +- src/components/admin/backups-section.tsx | 35 +- .../encrypted-column-writers.test.ts | 40 ++ .../export/__tests__/backup-json-scan.test.ts | 185 ++++++++ .../__tests__/passphrase-archive.test.ts | 43 ++ src/lib/export/backup-blob.ts | 38 +- src/lib/export/backup-json-scan.ts | 336 +++++++++++++++ src/lib/export/measurement-bulk-insert.ts | 136 ++++++ src/lib/export/passphrase-archive.ts | 89 +++- src/lib/export/response-stream.ts | 82 ++++ src/lib/export/store-backup-blob.ts | 22 +- src/lib/export/streamed-backup.ts | 138 ++++++ src/lib/jobs/offhost-backup.ts | 46 +- .../__tests__/backup-handlers.test.ts | 2 +- src/lib/jobs/reminder/backup-handlers.ts | 2 +- src/lib/jobs/restore-drill.ts | 46 +- src/lib/validations/backup.ts | 10 + tests/integration/large-account-jobs.test.ts | 2 +- .../large-backup-round-trip.test.ts | 203 +++++++++ .../measurement-bulk-insert.test.ts | 139 ++++++ 38 files changed, 2250 insertions(+), 341 deletions(-) create mode 100644 src/__tests__/backup-readers-stream-guard.test.ts create mode 100644 src/lib/export/__tests__/backup-json-scan.test.ts create mode 100644 src/lib/export/backup-json-scan.ts create mode 100644 src/lib/export/measurement-bulk-insert.ts create mode 100644 src/lib/export/response-stream.ts create mode 100644 src/lib/export/streamed-backup.ts create mode 100644 tests/integration/large-backup-round-trip.test.ts create mode 100644 tests/integration/measurement-bulk-insert.test.ts diff --git a/messages/de.json b/messages/de.json index 3af8f13ca..a95ac2bc6 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Backup hochladen", "uploadDescription": "Eine Backup-Datei validieren und als manuellen Snapshot speichern. Das Wiederherstellen ist ein separater Schritt.", "uploadButton": "Datei hochladen", - "uploadHelp": "JSON-Datei passend zum aktuellen Backup-Schema. Max. 10 MB.", + "uploadHelp": "HealthLog-Backup als .json oder .json.gz. Die Datei wird vor dem Hochladen komprimiert; komprimiert darf sie bis zu 512 MB groß sein.", "uploadSuccess": "Backup hochgeladen ({count} Datensätze)", "uploadFailed": "Upload fehlgeschlagen", "restore": "Wiederherstellen", diff --git a/messages/en.json b/messages/en.json index 9cd82bd03..ecbe39876 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Upload backup", "uploadDescription": "Validate a backup file and store it as a manual snapshot. Restoring is a separate step.", "uploadButton": "Upload file", - "uploadHelp": "JSON file matching the current backup schema. Max 10 MB.", + "uploadHelp": "A HealthLog backup as .json or .json.gz. The file is compressed before upload; compressed, it may be up to 512 MB.", "uploadSuccess": "Backup uploaded ({count} records)", "uploadFailed": "Upload failed", "restore": "Restore", diff --git a/messages/es.json b/messages/es.json index b69c2d44f..5efbc6d9e 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Subir copia de seguridad", "uploadDescription": "Valida un archivo de copia de seguridad y guárdalo como instantánea manual. La restauración es un paso aparte.", "uploadButton": "Subir archivo", - "uploadHelp": "Archivo JSON que coincida con el esquema de copia de seguridad actual. Máx. 10 MB.", + "uploadHelp": "Una copia de seguridad de HealthLog en .json o .json.gz. El archivo se comprime antes de subirlo; comprimido puede ocupar hasta 512 MB.", "uploadSuccess": "Copia de seguridad subida ({count} registros)", "uploadFailed": "La subida falló", "restore": "Restaurar", diff --git a/messages/fr.json b/messages/fr.json index e832e094e..6d7ae95e6 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Téléverser une sauvegarde", "uploadDescription": "Validez un fichier de sauvegarde et stockez-le comme instantané manuel. La restauration est une étape distincte.", "uploadButton": "Téléverser un fichier", - "uploadHelp": "Fichier JSON correspondant au schéma de sauvegarde actuel. Max 10 Mo.", + "uploadHelp": "Une sauvegarde HealthLog au format .json ou .json.gz. Le fichier est compressé avant l’envoi ; compressé, il peut atteindre 512 Mo.", "uploadSuccess": "Sauvegarde téléversée ({count} enregistrements)", "uploadFailed": "Échec du téléversement", "restore": "Restaurer", diff --git a/messages/it.json b/messages/it.json index 237f44644..bb6e9b094 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Carica backup", "uploadDescription": "Convalida un file di backup e archivialo come istantanea manuale. Il ripristino è un passaggio separato.", "uploadButton": "Carica file", - "uploadHelp": "File JSON conforme allo schema di backup attuale. Max 10 MB.", + "uploadHelp": "Un backup di HealthLog in formato .json o .json.gz. Il file viene compresso prima del caricamento; compresso può arrivare a 512 MB.", "uploadSuccess": "Backup caricato ({count} record)", "uploadFailed": "Caricamento non riuscito", "restore": "Ripristina", diff --git a/messages/ko.json b/messages/ko.json index 91b12f477..fcb2faf95 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7146,7 +7146,7 @@ "uploadTitle": "백업 업로드", "uploadDescription": "백업 파일을 검사한 뒤 수동 스냅샷으로 저장해요. 복원은 별도 단계예요.", "uploadButton": "파일 업로드", - "uploadHelp": "현재 백업 스키마에 맞는 JSON 파일이에요. 최대 10 MB.", + "uploadHelp": "HealthLog 백업 파일(.json 또는 .json.gz)이에요. 업로드 전에 압축되며, 압축된 크기는 최대 512 MB까지 가능해요.", "uploadSuccess": "백업을 업로드했어요 (기록 {count}개)", "uploadFailed": "업로드하지 못했어요", "restore": "복원", diff --git a/messages/pl.json b/messages/pl.json index ad7f00f51..de7694e66 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7167,7 +7167,7 @@ "uploadTitle": "Prześlij kopię zapasową", "uploadDescription": "Sprawdź plik kopii zapasowej i zapisz go jako ręczną migawkę. Przywracanie to osobny krok.", "uploadButton": "Prześlij plik", - "uploadHelp": "Plik JSON zgodny z bieżącym schematem kopii zapasowej. Maks. 10 MB.", + "uploadHelp": "Kopia zapasowa HealthLog w formacie .json lub .json.gz. Plik jest kompresowany przed wysłaniem; po kompresji może mieć do 512 MB.", "uploadSuccess": "Przesłano kopię zapasową ({count} rekordów)", "uploadFailed": "Przesyłanie nie powiodło się", "restore": "Przywróć", diff --git a/scripts/restore-backup.ts b/scripts/restore-backup.ts index 354da0fff..c6f4a51fe 100644 --- a/scripts/restore-backup.ts +++ b/scripts/restore-backup.ts @@ -15,12 +15,23 @@ * BACKUP_S3_REGION=auto \ * BACKUP_ENCRYPTION_KEY= \ * pnpm dlx tsx scripts/restore-backup.ts 2026-05-08/user-clx123.json.enc /tmp/restored.json + * + * An output name ending in `.gz` writes the JSON compressed, which is the + * form to hand to the admin upload for a large account: the JSON of 1.25 + * million measurements is 662 MB, the compressed file about a tenth of it. + * The file is written as a stream in either case and never held whole. */ -import { writeFileSync } from "node:fs"; +import { createWriteStream } from "node:fs"; +import { rename } from "node:fs/promises"; +import { PassThrough, Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGzip } from "node:zlib"; + +import { scanBackupJson } from "@/lib/export/backup-json-scan"; import { loadOffhostConfig, getS3Client, - decryptBackup, + openBackupObject, OffhostBackupNotConfiguredError, } from "@/lib/jobs/offhost-backup"; @@ -98,28 +109,43 @@ async function main() { `Downloading s3://${cfg.bucket}/${key} from ${cfg.endpoint} (region=${cfg.region})`, ); const ciphertext = await s3.getObject(key); - const plaintext = decryptBackup(ciphertext, cfg.encryptionKey); + const source = openBackupObject(ciphertext, cfg.encryptionKey); - let parsed: unknown; - try { - parsed = JSON.parse(plaintext); - } catch (err) { - throw new Error( - `Decrypted payload is not valid JSON: ${(err as Error).message}`, - ); - } - const body = validateBackupShape(parsed); + // Checked first, in one streamed pass that keeps only the small sections, + // so nothing is written for an object that is not a backup. + const { document, streamedCounts } = await scanBackupJson(source(), { + streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), + }); + const body = validateBackupShape({ + ...document, + measurements: [], + intakeEvents: [], + moodEntries: [], + }); if (expectedUserId && body.userId !== expectedUserId) { throw new Error( `Refusing to write: backup.userId='${body.userId}' does not match --user-id='${expectedUserId}'`, ); } - // The restored file is decrypted PHI in plaintext. Write it 0o600 so it is - // not world-readable on the operator's disk while it sits there. - writeFileSync(out, plaintext, { encoding: "utf8", mode: 0o600 }); + // The restored file is decrypted PHI in plaintext. Written 0o600 so it is + // not world-readable on the operator's disk, to a temporary name first so + // an interrupted run never leaves a truncated file under the real one. + const partial = `${out}.part`; + let bytes = 0; + const counter = new PassThrough(); + counter.on("data", (chunk: Buffer) => { + bytes += chunk.length; + }); + await pipeline( + Readable.from(source()), + counter, + ...(out.endsWith(".gz") ? [createGzip()] : []), + createWriteStream(partial, { mode: 0o600 }), + ); + await rename(partial, out); console.log( - `Restored userId=${body.userId} (${plaintext.length} bytes) -> ${out}`, + `Restored userId=${body.userId} (${bytes} bytes of JSON, ${streamedCounts.measurements ?? 0} measurements) -> ${out}`, ); } diff --git a/src/__tests__/backup-readers-stream-guard.test.ts b/src/__tests__/backup-readers-stream-guard.test.ts new file mode 100644 index 000000000..8798c6586 --- /dev/null +++ b/src/__tests__/backup-readers-stream-guard.test.ts @@ -0,0 +1,45 @@ +/** + * Every reader of a stored or uploaded backup reads it as a stream (#1031). + * + * A backup's JSON grows with the record, and past 536 870 888 characters it + * cannot exist as one JavaScript string at all: the disaster-recovery JSON of + * an account with 1.25 million measurements is 662 MB. Each of the readers + * below used to turn the file into one string first, and so none of them + * could open that account's backup. This pins that none of them goes back to + * it: no whole-document decrypt-to-string, no whole-body text read. + */ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const READERS = [ + "src/app/api/admin/backups/[id]/restore/route.ts", + "src/app/api/admin/backups/[id]/summary/route.ts", + "src/app/api/admin/backups/[id]/download/route.ts", + "src/app/api/admin/backups/upload/route.ts", + "src/lib/jobs/restore-drill.ts", + "scripts/restore-backup.ts", +] as const; + +/** Calls that hand back the whole document as one string or parse it whole. */ +const WHOLE_DOCUMENT = [ + /\bunpackBackupBlob\s*\(/, + /\bdecryptBackup\s*\(/, + /\bJSON\.parse\s*\(\s*plaintext\b/, + /\bparseBackupPayload\s*\(\s*plaintext\b/, + /\bfile\.text\s*\(/, + /\bawait\s+request\.text\s*\(\s*\)[\s\S]{0,200}parseBackupPayload/, +]; + +describe("backup readers stream the file", () => { + it.each(READERS)("%s never holds the whole document", (path) => { + const source = readFileSync(path, "utf8"); + for (const pattern of WHOLE_DOCUMENT) { + expect(source, `${path} matches ${pattern}`).not.toMatch(pattern); + } + }); + + it.each(READERS)("%s reads through the streaming reader", (path) => { + const source = readFileSync(path, "utf8"); + expect(source).toMatch(/\b(readStreamedBackup|scanBackupJson)\b/); + }); +}); diff --git a/src/app/api/admin/backups/[id]/download/route.ts b/src/app/api/admin/backups/[id]/download/route.ts index 644313726..bb2a01736 100644 --- a/src/app/api/admin/backups/[id]/download/route.ts +++ b/src/app/api/admin/backups/[id]/download/route.ts @@ -14,6 +14,8 @@ * at-rest contract: the ciphertext stays in the DB, the plaintext is only * materialised inside the request handler and streamed to the admin. */ +import { Readable } from "node:stream"; + import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; @@ -22,8 +24,13 @@ import { auditLog } from "@/lib/auth/audit"; import { BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, - unpackBackupBlob, + openBackupBlob, } from "@/lib/export/backup-blob"; +import { + readStreamedBackup, + type BackupSource, + type StreamedBackup, +} from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { parseBackupPayload } from "@/lib/validations/backup"; @@ -54,9 +61,11 @@ export const GET = apiHandler( throw new HttpError(404, "Backup not found"); } - let plaintext: string; + // Opened and streamed, never unpacked into one string: a large record's + // JSON is longer than any string V8 can hold (#1031). + let source: BackupSource; try { - plaintext = unpackBackupBlob(backup.data); + source = openBackupBlob(backup.data); } catch (err) { // A rotated or dropped key, or a stored copy that is no longer the one // that was written. Both are bad stored input rather than a fault in @@ -80,8 +89,10 @@ export const GET = apiHandler( // pathological case where the worker wrote something the upload // route would later reject — better to fail loudly here than to ship // a junk file that breaks an admin's restore plan. + let streamed: StreamedBackup; try { - parseBackupPayload(plaintext); + streamed = await readStreamedBackup(source); + parseBackupPayload(streamed.raw); } catch (err) { await auditLog("admin.backups.download.failed", { userId: admin.id, @@ -107,22 +118,26 @@ export const GET = apiHandler( ownerId: backup.userId, ownerUsername: backup.user.username, type: backup.type, - sizeBytes: Buffer.byteLength(plaintext, "utf8"), + sizeBytes: streamed.bytes, }, }); - // Returning the plaintext as-is keeps the response a faithful copy of + // Returning the stored bytes as-is keeps the response a faithful copy of // what the worker wrote (formatting, key order). The file is the // canonical artefact admins will store / re-upload. - return new NextResponse(plaintext, { - status: 200, - headers: { - "content-type": "application/json; charset=utf-8", - "content-disposition": `attachment; filename="${filename}"`, - // Backups contain sensitive health data — keep them out of any - // shared cache (CDN, browser disk cache). - "cache-control": "no-store, max-age=0", + return new NextResponse( + Readable.toWeb(Readable.from(source())) as ReadableStream, + { + status: 200, + headers: { + "content-type": "application/json; charset=utf-8", + "content-disposition": `attachment; filename="${filename}"`, + // Backups contain sensitive health data — keep them out of any + // shared cache (CDN, browser disk cache). + "cache-control": "no-store, max-age=0", + "content-length": String(streamed.bytes), + }, }, - }); + ); }, ); diff --git a/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts b/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts index 5d52faa1a..70053fb02 100644 --- a/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts +++ b/src/app/api/admin/backups/[id]/restore/__tests__/owner-mismatch.test.ts @@ -47,6 +47,9 @@ vi.mock("@/lib/validations/backup", () => ({ parseBackupPayload: vi.fn(), isCompatibleSchemaVersion: vi.fn(() => true), summarizeBackup: vi.fn(() => ({})), + // The streamed reader checks measurements one at a time; this file has none. + BACKUP_SCHEMA_VERSION: "2", + backupMeasurementSchema: { safeParse: vi.fn() }, })); vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() })); vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index 7e45e49f9..7f8f2b4af 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -26,8 +26,17 @@ import { auditLog } from "@/lib/auth/audit"; import { BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, - unpackBackupBlob, + openBackupBlob, } from "@/lib/export/backup-blob"; +import { + insertMeasurementRows, + type MeasurementInsertRow, +} from "@/lib/export/measurement-bulk-insert"; +import { + readStreamedBackup, + type BackupSource, + type StreamedBackup, +} from "@/lib/export/streamed-backup"; import { encryptNote } from "@/lib/crypto/note-cipher"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; @@ -38,6 +47,7 @@ import { parseBackupPayload, isCompatibleSchemaVersion, summarizeBackup, + type BackupMeasurement, type BackupSummary, } from "@/lib/validations/backup"; import { recomputeUserMoodRollups } from "@/lib/rollups/mood-rollups"; @@ -137,6 +147,19 @@ interface RestoreResponse { }; } +/** + * How long the restore transaction may run. Two minutes was the whole budget + * when a record was a few hundred thousand rows; deleting and rewriting 1.25 + * million measurements in one transaction takes longer than that (#1031). On + * the test host the whole restore of that account, reading the file twice + * included, took 94 s. The budget grows by a second per 5 000 measurements, + * which gives that account six minutes, about four times what it needed, and + * never drops below the old two minutes. + */ +function restoreTransactionTimeoutMs(measurementCount: number): number { + return 120_000 + Math.ceil(measurementCount / 5); +} + function decodeEncryptedBytes(encoded: string): Uint8Array { const decoded = Buffer.from(encoded, "base64"); const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); @@ -198,9 +221,12 @@ const handler = apiHandler( throw new HttpError(404, "Backup not found"); } - let plaintext: string; + // Opened, not unpacked: the JSON of a large record is longer than any + // string V8 can hold (#1031), so it is read as a stream below. Opening + // authenticates the whole ciphertext first, as unpacking did. + let source: BackupSource; try { - plaintext = unpackBackupBlob(backup.data); + source = openBackupBlob(backup.data); } catch (err) { await auditLog("admin.backups.restore.failed", { userId: admin.id, @@ -224,10 +250,16 @@ const handler = apiHandler( // Parsed once and kept, because the schema's per-section `.default([])` // erases the difference between a section that is absent and one that is // empty — and that difference is what the completeness check below reads. + // + // The measurements are the exception: `readStreamedBackup` checks each one + // against the element schema as it goes and keeps only their count, and + // the transaction reads them a second time, in batches, as it writes them. let raw: unknown; let payload; + let streamed: StreamedBackup; try { - raw = JSON.parse(plaintext); + streamed = await readStreamedBackup(source); + raw = streamed.raw; payload = parseBackupPayload(raw); } catch (err) { await auditLog("admin.backups.restore.failed", { @@ -548,7 +580,7 @@ const handler = apiHandler( } const toRestoredMeasurementData = ( - measurement: (typeof payload.measurements)[number], + measurement: BackupMeasurement, ) => ({ type: measurement.type, value: measurement.value, @@ -600,55 +632,80 @@ const handler = apiHandler( : {}), }); - const stableRows = payload.measurements.flatMap((measurement) => - measurement.id - ? [ - { - id: measurement.id, - userId: ownerId, - ...toRestoredMeasurementData(measurement), - }, - ] - : [], - ); - const measurementBatchSize = 1_000; - for ( - let offset = 0; - offset < stableRows.length; - offset += measurementBatchSize - ) { - await tx.measurement.createMany({ - data: stableRows.slice(offset, offset + measurementBatchSize), - }); - } - - // v1 payloads did not require stable ids. Preserve their historical - // natural-key reconciliation without routing canonical v2 rows - // through it. - for (const measurement of payload.measurements) { - if (measurement.id) continue; - const restoredData = toRestoredMeasurementData(measurement); - const existing = await tx.measurement.findFirst({ - where: { - userId: ownerId, - type: measurement.type, - source: restoredData.source, - measuredAt: restoredData.measuredAt, - sleepStage: restoredData.sleepStage, - }, - select: { id: true }, - }); - if (existing) { - await tx.measurement.update({ - where: { id: existing.id, userId: ownerId }, - data: restoredData, - }); - } else { - await tx.measurement.create({ - data: { userId: ownerId, ...restoredData }, - }); + // The measurements, a batch at a time from a second read of the + // file, so the restore never holds more than one batch of them. + // + // Two later sections point INTO the measurements (a record's + // `sourceMeasurementId`, an ECG strip's `measurementId`) and have + // to know which of those ids this transaction actually wrote. They + // used to be handed every written id; they only ever ask about the + // ones they reference, so only those are collected. + const referencedMeasurementIds = new Set(); + for (const record of payload.personalRecords) { + if (record.sourceMeasurementId) { + referencedMeasurementIds.add(record.sourceMeasurementId); } } + for (const recording of payload.ecgRecordings) { + if (recording.measurementId) { + referencedMeasurementIds.add(recording.measurementId); + } + } + const writtenReferencedIds = new Set(); + const measurementBatchSize = 1_000; + await streamed.forEachMeasurementBatch( + measurementBatchSize, + async (batch) => { + const stableRows = batch.flatMap( + (measurement): MeasurementInsertRow[] => + measurement.id + ? [ + { + id: measurement.id, + userId: ownerId, + ...toRestoredMeasurementData(measurement), + }, + ] + : [], + ); + // One statement per batch, not `createMany`: see + // `insertMeasurementRows` for what the latter cost here. + await insertMeasurementRows(tx, stableRows); + for (const row of stableRows) { + if (referencedMeasurementIds.has(row.id)) { + writtenReferencedIds.add(row.id); + } + } + + // v1 payloads did not require stable ids. Preserve their + // historical natural-key reconciliation without routing + // canonical v2 rows through it. + for (const measurement of batch) { + if (measurement.id) continue; + const restoredData = toRestoredMeasurementData(measurement); + const existing = await tx.measurement.findFirst({ + where: { + userId: ownerId, + type: measurement.type, + source: restoredData.source, + measuredAt: restoredData.measuredAt, + sleepStage: restoredData.sleepStage, + }, + select: { id: true }, + }); + if (existing) { + await tx.measurement.update({ + where: { id: existing.id, userId: ownerId }, + data: restoredData, + }); + } else { + await tx.measurement.create({ + data: { userId: ownerId, ...restoredData }, + }); + } + } + }, + ); const medByName = new Map(); const restoredMedicationIds = new Set(); @@ -1932,7 +1989,7 @@ const handler = apiHandler( tx, ownerId, payload, - new Set(stableRows.map((row) => row.id)), + writtenReferencedIds, skips, ); @@ -1949,7 +2006,7 @@ const handler = apiHandler( tx, ownerId, payload, - new Set(stableRows.map((row) => row.id)), + writtenReferencedIds, skips, ); @@ -2024,7 +2081,7 @@ const handler = apiHandler( }, { maxWait: 10_000, - timeout: 120_000, + timeout: restoreTransactionTimeoutMs(streamed.measurementCount), }, ); } catch (err) { @@ -2071,7 +2128,7 @@ const handler = apiHandler( // outside the transaction (the 5-year fold would otherwise hold a // long write lock) and is best-effort so a populator hiccup never // undoes the restore. The boot-time backfill is the safety net. - if (payload.measurements.length > 0) { + if (streamed.measurementCount > 0) { try { await recomputeUserRollups(ownerId); } catch (err) { @@ -2131,7 +2188,10 @@ const handler = apiHandler( } } - const summary = summarizeBackup(payload); + const summary = { + ...summarizeBackup(payload), + measurements: streamed.measurementCount, + }; await auditLog("admin.backups.restore", { userId: admin.id, diff --git a/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts b/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts index 766e0d4de..16f704733 100644 --- a/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts +++ b/src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts @@ -44,6 +44,11 @@ vi.mock("@/lib/validations/backup", () => ({ parseBackupPayload: (...a: unknown[]) => parseMock(...a), summarizeBackup: (...a: unknown[]) => summarizeMock(...a), isCompatibleSchemaVersion: vi.fn(() => true), + // The measurements are read as a stream and counted one by one. + BACKUP_SCHEMA_VERSION: "2", + backupMeasurementSchema: { + safeParse: (value: unknown) => ({ success: true, data: value }), + }, })); import { GET } from "../route"; @@ -62,9 +67,14 @@ beforeEach(() => { createdAt: new Date("2026-08-01T00:00:00Z"), user: { id: "u1", username: "self-hoster" }, }); - decryptMock.mockReturnValue("{}"); + // Twelve measurements in the file: the preview counts them from the stream. + decryptMock.mockReturnValue( + JSON.stringify({ + measurements: Array.from({ length: 12 }, (_, i) => ({ id: `m${i}` })), + }), + ); parseMock.mockReturnValue({ schemaVersion: 3 }); - summarizeMock.mockReturnValue({ measurements: 12, moodEntries: 3 }); + summarizeMock.mockReturnValue({ measurements: 0, moodEntries: 3 }); }); describe("GET /api/admin/backups/[id]/summary", () => { diff --git a/src/app/api/admin/backups/[id]/summary/route.ts b/src/app/api/admin/backups/[id]/summary/route.ts index 4b79fcc79..cb40b404c 100644 --- a/src/app/api/admin/backups/[id]/summary/route.ts +++ b/src/app/api/admin/backups/[id]/summary/route.ts @@ -19,8 +19,12 @@ import { apiError, apiSuccess } from "@/lib/api-response"; import { BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, - unpackBackupBlob, + openBackupBlob, } from "@/lib/export/backup-blob"; +import { + readStreamedBackup, + type BackupSource, +} from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { isCompatibleSchemaVersion, @@ -46,9 +50,11 @@ export const GET = apiHandler( throw new HttpError(404, "Backup not found"); } - let plaintext: string; + // Opened and read as a stream: a large record's JSON is longer than any + // string V8 can hold (#1031). + let source: BackupSource; try { - plaintext = unpackBackupBlob(backup.data); + source = openBackupBlob(backup.data); } catch { // Same refusal the restore and the download give, for the same reason: // a copy this instance cannot open is bad stored input, and the preview @@ -59,8 +65,11 @@ export const GET = apiHandler( } let payload; + let measurementCount: number; try { - payload = parseBackupPayload(plaintext); + const streamed = await readStreamedBackup(source); + measurementCount = streamed.measurementCount; + payload = parseBackupPayload(streamed.raw); } catch { return apiError("Backup payload failed schema validation", 422); } @@ -72,7 +81,10 @@ export const GET = apiHandler( ); } - const summary = summarizeBackup(payload); + const summary = { + ...summarizeBackup(payload), + measurements: measurementCount, + }; annotate({ action: { name: "admin.backups.summary" }, diff --git a/src/app/api/admin/backups/upload/route.ts b/src/app/api/admin/backups/upload/route.ts index b7b6443ea..8110b4bac 100644 --- a/src/app/api/admin/backups/upload/route.ts +++ b/src/app/api/admin/backups/upload/route.ts @@ -1,17 +1,25 @@ /** * POST /api/admin/backups/upload — admin-only ingest of a backup file. * - * Accepts a `multipart/form-data` body with a single `file` field that - * carries a JSON backup matching `backupPayloadSchema`. The route: + * Accepts the backup file either as the raw request body (JSON, or JSON + * compressed with gzip) or, for a small file, as a `multipart/form-data` + * body with a single `file` field. The route: * * 1. Authenticates the caller as an admin (cookie session only). * 2. Reads the file (size capped — see `MAX_UPLOAD_BYTES`). * 3. Parses and validates the JSON against the canonical schema. * 4. Rejects files written by a future schema version (incompatible). - * 5. Encrypts the plaintext and inserts a new `DataBackup` row of - * type `MANUAL_UPLOAD_` so multiple uploads coexist with + * 5. Encrypts the file as it arrives and inserts a new `DataBackup` row + * of type `MANUAL_UPLOAD_` so multiple uploads coexist with * the rolling `WEEKLY_AUTO` snapshot AND with each other. * + * The file is read as a stream and never held whole (#1031): an export of an + * account with 1.25 million measurements is a few hundred megabytes of JSON, + * which neither fits a 1 GB container beside the running app nor, in the + * disaster-recovery shape, fits in one JavaScript string at all. It is + * checked as it goes and stored in the same pass; a file that fails any check + * rolls the store back, so nothing of it remains. + * * Crucially, this route does NOT execute a restore. It only stores the * file. The separate `POST /api/admin/backups/[id]/restore` endpoint * (criterion 3) actually replaces user data — keeping the two phases @@ -20,46 +28,120 @@ * * Phase B1 / criterion 2 of the v1.4.15 backup-completeness work. */ +import { Readable } from "node:stream"; +import { createGunzip } from "node:zlib"; + import { NextRequest } from "next/server"; import { ZodError } from "zod/v4"; import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { packBackupBlob } from "@/lib/export/backup-blob"; +import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; +import { StreamedBackupInvalidError } from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit } from "@/lib/rate-limit"; import { + BACKUP_SCHEMA_VERSION, + backupMeasurementSchema, backupPayloadSchema, isCompatibleSchemaVersion, summarizeBackup, + type BackupPayload, type BackupSummary, } from "@/lib/validations/backup"; export const dynamic = "force-dynamic"; /** - * Cap on uploaded file size. The largest production backups today are - * ~2 MB; 10 MB leaves headroom while keeping a malicious admin client - * from blowing memory by streaming a 4 GB file. Enforced by reading the - * `Content-Length` header AND the actual buffered size after decoding — - * a missing/wrong content-length doesn't bypass the limit. + * Cap on a file sent as a multipart form. The form parser holds the whole + * file in memory, so this path stays for small files only; anything larger + * is sent as the raw request body, which is read as a stream. */ const MAX_UPLOAD_BYTES = 10 * 1024 * 1024; +/** + * Cap on a file sent as the raw request body, compressed or not. The same + * ceiling the app sets for any request body (`middlewareClientMaxBodySize` in + * `next.config.ts`), beyond which the body would arrive truncated. A + * disaster-recovery file of 1.25 million measurements is 662 MB as plain + * JSON and 64 MB compressed, so large files go up compressed. + */ +const MAX_RAW_UPLOAD_BYTES = 512 * 1024 * 1024; + interface UploadResponse { id: string; valid: true; summary: BackupSummary; } +/** A file the route refuses: the answer, and what the audit row records. */ +class UploadRefused extends Error { + constructor( + readonly status: number, + readonly answer: string, + readonly details: Record, + readonly meta?: Record, + ) { + super(answer); + this.name = "UploadRefused"; + } +} + +/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */ +async function* decodedBytes( + source: AsyncIterable, + limit: number, +): AsyncGenerator { + let seen = 0; + async function* counted() { + for await (const chunk of source) { + seen += chunk.byteLength; + if (seen > limit) { + throw new UploadRefused( + 413, + `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`, + { reason: "file_size_exceeded", size: seen }, + ); + } + yield chunk; + } + } + const iterator = counted()[Symbol.asyncIterator](); + const first = await iterator.next(); + if (first.done) return; + const rest = (async function* () { + yield first.value; + for (;;) { + const next = await iterator.next(); + if (next.done) return; + yield next.value; + } + })(); + const head = first.value; + if (head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b) { + const gunzip = Readable.from(rest).pipe(createGunzip()); + try { + for await (const chunk of gunzip) yield chunk as Buffer; + } catch (err) { + if (err instanceof UploadRefused) throw err; + throw new UploadRefused(422, "Uploaded file is not valid gzip", { + reason: "invalid_gzip", + }); + } + return; + } + yield* rest; +} + export const POST = apiHandler(async (request: NextRequest) => { const { user: admin } = await requireAdmin(); annotate({ action: { name: "admin.backups.upload" } }); + const ipAddress = getClientIp(request); - // Rate-limit identical to the manual-run endpoint. An admin uploading - // a 10 MB blob 60× per minute would just be wasteful — three is plenty - // for a deliberate restore-prep workflow. + // Rate-limit identical to the manual-run endpoint. Three is plenty for a + // deliberate restore-prep workflow. const rl = await checkRateLimit( `admin-backups-upload:${admin.id}`, 3, @@ -69,120 +151,56 @@ export const POST = apiHandler(async (request: NextRequest) => { throw new HttpError(429, "Too many backup uploads"); } - // Cheap pre-flight on the declared content length. The actual buffer - // size check below is the hard limit; this just rejects obvious abuse - // before allocating the FormData parser. + const denied = (reason: string, extra: Record = {}) => + auditLog("admin.backups.upload.denied", { + userId: admin.id, + ipAddress, + details: { reason, ...extra }, + }); + + const contentType = request.headers.get("content-type") ?? ""; + const multipart = contentType.startsWith("multipart/form-data"); + const limit = multipart ? MAX_UPLOAD_BYTES : MAX_RAW_UPLOAD_BYTES; + + // Cheap pre-flight on the declared content length. The byte count while + // reading is the hard limit; this just rejects obvious abuse early. const contentLength = Number(request.headers.get("content-length") ?? 0); - if (contentLength > MAX_UPLOAD_BYTES) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { reason: "content_length_exceeded", contentLength }, - }); - return apiError("Upload exceeds 10 MB limit", 413); - } - - let formData: FormData; - try { - formData = await request.formData(); - } catch (err) { + if (contentLength > limit) { + await denied("content_length_exceeded", { contentLength }); return apiError( - `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`, - 400, + `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`, + 413, ); } - const file = formData.get("file"); - if (!(file instanceof File)) { - return apiError("Field 'file' must be a file", 422); - } - - if (file.size > MAX_UPLOAD_BYTES) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { reason: "file_size_exceeded", size: file.size }, - }); - return apiError("Upload exceeds 10 MB limit", 413); - } - - let plaintext: string; - try { - plaintext = await file.text(); - } catch { - return apiError("Failed to read uploaded file", 400); - } - - let parsedJson: unknown; - try { - parsedJson = JSON.parse(plaintext); - } catch (err) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { - reason: "invalid_json", - message: err instanceof Error ? err.message : "parse_failed", - }, - }); - return apiError("Uploaded file is not valid JSON", 422); - } - - let payload; - try { - payload = backupPayloadSchema.parse(parsedJson); - } catch (err) { - const issues = - err instanceof ZodError - ? err.issues.slice(0, 10).map((i) => ({ - path: i.path.join("."), - message: i.message, - })) - : []; - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { reason: "schema_invalid", issues }, - }); - return apiError("Backup payload failed schema validation", 422, { - issues, - }); - } - - if (!isCompatibleSchemaVersion(payload.schemaVersion)) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { - reason: "incompatible_schema_version", - schemaVersion: payload.schemaVersion, - }, - }); - return apiError( - `Backup schema version '${payload.schemaVersion}' is not supported by this server`, - 422, + let source: AsyncIterable; + if (multipart) { + let formData: FormData; + try { + formData = await request.formData(); + } catch (err) { + return apiError( + `Invalid multipart body: ${err instanceof Error ? err.message : "unknown"}`, + 400, + ); + } + const file = formData.get("file"); + if (!(file instanceof File)) { + return apiError("Field 'file' must be a file", 422); + } + if (file.size > MAX_UPLOAD_BYTES) { + await denied("file_size_exceeded", { size: file.size }); + return apiError("Upload exceeds 10 MB limit", 413); + } + source = Readable.fromWeb( + file.stream() as Parameters[0], ); - } - - // Make sure the userId on the file points at a user that exists in - // this DB. Restore would fail later anyway, but failing here gives - // the admin a precise error before any side-effect. - const owner = await prisma.user.findUnique({ - where: { id: payload.userId }, - select: { id: true, username: true }, - }); - if (!owner) { - await auditLog("admin.backups.upload.denied", { - userId: admin.id, - ipAddress: getClientIp(request), - details: { - reason: "owner_not_found", - ownerId: payload.userId, - }, - }); - return apiError( - `Backup is for user '${payload.userId}' which does not exist in this DB`, - 422, + } else { + if (!request.body) { + return apiError("The backup file is missing from the request body", 422); + } + source = Readable.fromWeb( + request.body as Parameters[0], ); } @@ -190,25 +208,146 @@ export const POST = apiHandler(async (request: NextRequest) => { // intact so multiple uploads can coexist for the same user without // overwriting each other or the rolling WEEKLY_AUTO snapshot. const uploadType = `MANUAL_UPLOAD_${Date.now()}`; - const encrypted = packBackupBlob(JSON.stringify(payload)); + let payload: BackupPayload | undefined; + let measurementCount = 0; + let owner: { id: string; username: string } | null = null; - const created = await prisma.dataBackup.create({ - data: { - userId: owner.id, - type: uploadType, - data: encrypted, - }, - }); + let created: { id: string }; + try { + created = await storeBackupBlob( + prisma, + () => ({ userId: owner!.id, type: uploadType }), + async (write) => { + // One pass: every chunk is stored as it is checked, and a check that + // fails throws, which rolls the store back. + async function* storedAsRead() { + for await (const chunk of decodedBytes(source, limit)) { + await write(Buffer.from(chunk)); + yield chunk; + } + } + let firstWithoutId: number | null = null; + let scanned; + try { + scanned = await scanBackupJson(storedAsRead(), { + streamKeys: new Set(["measurements"]), + onElement: (_key, element, index) => { + const parsed = backupMeasurementSchema.safeParse(element); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + throw new StreamedBackupInvalidError( + ["measurements", index, ...(issue?.path ?? [])].join("."), + issue?.message ?? "invalid measurement", + ); + } + if (!parsed.data.id && firstWithoutId === null) { + firstWithoutId = index; + } + }, + }); + } catch (err) { + if (err instanceof BackupJsonError) { + throw new UploadRefused(422, "Uploaded file is not valid JSON", { + reason: "invalid_json", + message: err.message, + }); + } + if (err instanceof StreamedBackupInvalidError) { + const issues = [{ path: err.path, message: err.message }]; + throw new UploadRefused( + 422, + "Backup payload failed schema validation", + { reason: "schema_invalid", issues }, + { issues }, + ); + } + throw err; + } + measurementCount = scanned.streamedCounts.measurements ?? 0; - const summary = summarizeBackup(payload); + try { + payload = backupPayloadSchema.parse(scanned.document); + } catch (err) { + const issues = + err instanceof ZodError + ? err.issues.slice(0, 10).map((i) => ({ + path: i.path.join("."), + message: i.message, + })) + : []; + throw new UploadRefused( + 422, + "Backup payload failed schema validation", + { reason: "schema_invalid", issues }, + { issues }, + ); + } + if ( + payload.schemaVersion === BACKUP_SCHEMA_VERSION && + firstWithoutId !== null + ) { + const issues = [ + { + path: `measurements.${firstWithoutId}.id`, + message: "Canonical v2 measurements require a stable id", + }, + ]; + throw new UploadRefused( + 422, + "Backup payload failed schema validation", + { reason: "schema_invalid", issues }, + { issues }, + ); + } + + if (!isCompatibleSchemaVersion(payload.schemaVersion)) { + throw new UploadRefused( + 422, + `Backup schema version '${payload.schemaVersion}' is not supported by this server`, + { + reason: "incompatible_schema_version", + schemaVersion: payload.schemaVersion, + }, + ); + } + + // Make sure the userId on the file points at a user that exists in + // this DB. Restore would fail later anyway, but failing here gives + // the admin a precise error before any side-effect. + owner = await prisma.user.findUnique({ + where: { id: payload.userId }, + select: { id: true, username: true }, + }); + if (!owner) { + throw new UploadRefused( + 422, + `Backup is for user '${payload.userId}' which does not exist in this DB`, + { reason: "owner_not_found", ownerId: payload.userId }, + ); + } + }, + ); + } catch (err) { + if (err instanceof UploadRefused) { + await denied(err.details.reason as string, err.details); + return apiError(err.answer, err.status, err.meta); + } + throw err; + } + + const summary: BackupSummary = { + ...summarizeBackup(payload!), + measurements: measurementCount, + }; + const storedOwner = owner as { id: string; username: string } | null; await auditLog("admin.backups.upload", { userId: admin.id, - ipAddress: getClientIp(request), + ipAddress, details: { backupId: created.id, - ownerId: owner.id, - ownerUsername: owner.username, + ownerId: storedOwner?.id, + ownerUsername: storedOwner?.username, type: uploadType, schemaVersion: summary.schemaVersion, counts: { diff --git a/src/app/api/export/__tests__/soft-delete-filter.test.ts b/src/app/api/export/__tests__/soft-delete-filter.test.ts index bd4a5669c..ff386b3ec 100644 --- a/src/app/api/export/__tests__/soft-delete-filter.test.ts +++ b/src/app/api/export/__tests__/soft-delete-filter.test.ts @@ -64,6 +64,29 @@ vi.mock("@/lib/db", () => ({ documentConditionLink: { findMany: vi.fn() }, extractedFact: { findMany: vi.fn() }, ecgRecording: { findMany: vi.fn() }, + onboardingRecord: { findUnique: vi.fn().mockResolvedValue(null) }, + environmentTravelLocation: { findMany: vi.fn().mockResolvedValue([]) }, + environmentContext: { findMany: vi.fn().mockResolvedValue([]) }, + personalRecord: { findMany: vi.fn().mockResolvedValue([]) }, + coachReminder: { findMany: vi.fn().mockResolvedValue([]) }, + coachPlan: { findMany: vi.fn().mockResolvedValue([]) }, + coachFact: { findMany: vi.fn().mockResolvedValue([]) }, + coachConversation: { findMany: vi.fn().mockResolvedValue([]) }, + measurementReminderEvent: { findMany: vi.fn().mockResolvedValue([]) }, + measurementReminder: { findMany: vi.fn().mockResolvedValue([]) }, + vaccinationDocumentLink: { findMany: vi.fn().mockResolvedValue([]) }, + vaccinationRecord: { findMany: vi.fn().mockResolvedValue([]) }, + encounterConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, + encounterLabLink: { findMany: vi.fn().mockResolvedValue([]) }, + encounterDocumentLink: { findMany: vi.fn().mockResolvedValue([]) }, + encounter: { findMany: vi.fn().mockResolvedValue([]) }, + practitioner: { findMany: vi.fn().mockResolvedValue([]) }, + healthScoreRecord: { findMany: vi.fn().mockResolvedValue([]) }, + intradayCumulativeProfile: { findMany: vi.fn().mockResolvedValue([]) }, + correlationPattern: { findMany: vi.fn().mockResolvedValue([]) }, + healthProfileFactRevision: { findMany: vi.fn().mockResolvedValue([]) }, + customMetric: { findMany: vi.fn().mockResolvedValue([]) }, + cycleSymptom: { findMany: vi.fn().mockResolvedValue([]) }, }, })); @@ -177,6 +200,7 @@ beforeEach(() => { ); vi.mocked(prisma.extractedFact.findMany).mockResolvedValue([] as never); vi.mocked(prisma.ecgRecording.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.userAchievement.findMany).mockResolvedValue([] as never); }); describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { @@ -192,7 +216,13 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { it("/api/export/full-backup scopes the measurement read to deletedAt: null", async () => { const { GET } = await import("../full-backup/route"); - await GET(mkReq("http://localhost/api/export/full-backup")); + // The file is written into the response as it is produced, so the reads + // happen while the body is consumed. The mocked client does not cover + // every section the builder reads, so the body errors part-way (as the + // whole response did before it streamed); the reads asserted here have + // been issued by then. + const res = await GET(mkReq("http://localhost/api/export/full-backup")); + await res.text(); expect(prisma.measurement.findMany).toHaveBeenCalledWith( expect.objectContaining({ where: expect.objectContaining({ deletedAt: null }), @@ -202,7 +232,10 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { it("/api/export/full-backup scopes every v1.28 records domain to userId + deletedAt: null", async () => { const { GET } = await import("../full-backup/route"); - await GET(mkReq("http://localhost/api/export/full-backup")); + // The file is written into the response as it is produced, so the reads + // happen while the body is consumed. + const res = await GET(mkReq("http://localhost/api/export/full-backup")); + await res.text(); expect(prisma.labResult.findMany).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/app/api/export/encrypted/route.ts b/src/app/api/export/encrypted/route.ts index e68f4f987..bd19ae009 100644 --- a/src/app/api/export/encrypted/route.ts +++ b/src/app/api/export/encrypted/route.ts @@ -42,13 +42,44 @@ import { safeJson, } from "@/lib/api-response"; import { checkRateLimit } from "@/lib/rate-limit"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { randomBytes } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { FullBackupCounts } from "@/lib/export/full-backup-payload"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { - encryptArchive, + encryptArchiveToFile, MIN_EXPORT_PASSPHRASE_LENGTH, + type SpooledArchive, } from "@/lib/export/passphrase-archive"; +import { streamToResponseBody } from "@/lib/export/response-stream"; import { NextRequest, NextResponse } from "next/server"; +const SPOOL_PREFIX = "healthlog-export-"; + +/** + * Remove spool files an earlier export left behind: a process that died + * mid-export never reached its cleanup. An hour is far longer than any export + * takes, so nothing still being written or sent is touched. + */ +async function removeStaleSpools(): Promise { + try { + const dir = tmpdir(); + const cutoff = Date.now() - 60 * 60 * 1000; + for (const name of await readdir(dir)) { + if (!name.startsWith(SPOOL_PREFIX)) continue; + const path = join(dir, name); + const info = await stat(path).catch(() => null); + if (info && info.mtimeMs < cutoff) await rm(path, { force: true }); + } + } catch { + // Housekeeping only; an export must not fail over it. + } +} + const encryptedExportSchema = z .object({ passphrase: z @@ -89,10 +120,31 @@ export const POST = apiHandler(async (request: NextRequest) => { } const { passphrase } = parsed.data; - const { payload, counts } = await buildFullBackupPayload(prisma, user.id); - - // Encrypt the JSON bytes. The passphrase goes no further than the KDF. - const archive = await encryptArchive(JSON.stringify(payload), passphrase); + // Sealed as it is produced and spooled to a temporary file, then sent: the + // format puts the tag in front of the ciphertext, so the archive cannot go + // out before its last byte is encrypted, and holding it in memory is what + // took the app down on an account of 1.25 million measurements (#1031). Only + // ciphertext touches the disk, and the file is removed once it is sent or + // the client leaves. + let counts: FullBackupCounts | undefined; + await removeStaleSpools(); + const bodyPath = join( + tmpdir(), + `${SPOOL_PREFIX}${randomBytes(12).toString("hex")}.hlx.part`, + ); + let archive: SpooledArchive; + try { + archive = await encryptArchiveToFile( + async (write) => { + counts = await streamFullBackupJson(prisma, user.id, write); + }, + passphrase, + bodyPath, + ); + } catch (err) { + await rm(bodyPath, { force: true }); + throw err; + } await auditLog("user.export.encrypted", { userId: user.id, @@ -102,38 +154,36 @@ export const POST = apiHandler(async (request: NextRequest) => { annotate({ meta: { - export_measurements_count: counts.measurements, - export_medications_count: counts.medications, - export_intake_count: counts.intakeEvents, - export_medication_side_effect_count: counts.medicationSideEffects, - export_mood_count: counts.moodEntries, - export_cycle_count: counts.cycles, - export_cycle_day_log_count: counts.cycleDayLogs, - export_lab_result_count: counts.labResults, - export_biomarker_count: counts.biomarkers, - export_illness_episode_count: counts.illnessEpisodes, - export_illness_day_log_count: counts.illnessDayLogs, - export_allergy_count: counts.allergies, - export_family_history_count: counts.familyHistory, - export_workout_count: counts.workouts, - export_document_count: counts.documents, - export_nutrient_day_count: counts.nutrientDays, - export_health_profile_count: counts.healthProfile, - export_custom_metric_count: counts.customMetrics, - export_custom_metric_entry_count: counts.customMetricEntries, - export_intraday_profile_count: counts.intradayProfiles, + export_measurements_count: counts?.measurements, + export_medications_count: counts?.medications, + export_intake_count: counts?.intakeEvents, + export_mood_count: counts?.moodEntries, export_archive_bytes: archive.byteLength, }, }); + const archiveBody = streamToResponseBody( + async (write) => { + await write(archive.prefix); + for await (const chunk of createReadStream(archive.bodyPath)) { + await write(chunk as Buffer); + } + }, + { + onComplete: () => rm(archive.bodyPath, { force: true }), + onError: () => rm(archive.bodyPath, { force: true }), + }, + ); + const stamp = new Date().toISOString().slice(0, 10); // Return the raw binary archive (NOT the apiSuccess envelope). The file is a // self-contained `.hlx` archive openable with the user's passphrase via // scripts/decrypt-export.ts. - return new NextResponse(new Uint8Array(archive), { + return new NextResponse(archiveBody, { status: 200, headers: { "Content-Type": "application/octet-stream", + "Content-Length": String(archive.byteLength), "Content-Disposition": `attachment; filename="healthlog-backup-${user.id}-${stamp}.hlx"`, "Cache-Control": "no-store", }, diff --git a/src/app/api/export/full-backup/route.ts b/src/app/api/export/full-backup/route.ts index c7e83a262..b36201c92 100644 --- a/src/app/api/export/full-backup/route.ts +++ b/src/app/api/export/full-backup/route.ts @@ -35,7 +35,9 @@ import { annotate } from "@/lib/logging/context"; import { auditLog } from "@/lib/auth/audit"; import { apiError, getClientIp } from "@/lib/api-response"; import { checkRateLimit } from "@/lib/rate-limit"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import type { FullBackupCounts } from "@/lib/export/full-backup-payload"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; +import { streamToResponseBody } from "@/lib/export/response-stream"; import { NextRequest, NextResponse } from "next/server"; export const GET = apiHandler(async (request: NextRequest) => { @@ -47,48 +49,44 @@ export const GET = apiHandler(async (request: NextRequest) => { return apiError("Maximum 10 exports per hour", 429); } - // v1.23 — the payload builder is shared with the passphrase-encrypted - // export route so both emit the byte-for-byte same shape (the measurement/ - // medication/mood/cycle domains are restore-compatible; the newer records - // domains are export-only for now — see the module doc comment above). - const { payload, counts } = await buildFullBackupPayload(prisma, user.id); - - await auditLog("user.export.full-backup", { - userId: user.id, - ipAddress: getClientIp(request), - details: { counts }, - }); - - annotate({ - meta: { - export_measurements_count: counts.measurements, - export_medications_count: counts.medications, - export_intake_count: counts.intakeEvents, - export_medication_side_effect_count: counts.medicationSideEffects, - export_mood_count: counts.moodEntries, - export_cycle_count: counts.cycles, - export_cycle_day_log_count: counts.cycleDayLogs, - export_lab_result_count: counts.labResults, - export_biomarker_count: counts.biomarkers, - export_illness_episode_count: counts.illnessEpisodes, - export_illness_day_log_count: counts.illnessDayLogs, - export_allergy_count: counts.allergies, - export_family_history_count: counts.familyHistory, - export_workout_count: counts.workouts, - export_document_count: counts.documents, - export_nutrient_day_count: counts.nutrientDays, - export_health_profile_count: counts.healthProfile, - export_custom_metric_count: counts.customMetrics, - export_custom_metric_entry_count: counts.customMetricEntries, - export_intraday_profile_count: counts.intradayProfiles, + // Written into the response as it is produced (`streamFullBackupJson`), not + // built and stringified first: for an account of 1.25 million measurements + // the built payload and its JSON string did not fit a 1 GB container, and + // the export took the whole app down (#1031). The file is byte for byte + // what `JSON.stringify` of the payload builder's object gives; the passphrase + // export beside this route writes the same bytes, encrypted. + // + // The counts are only known once the last row has gone out, so the audit + // row is written then. A file that stops early is audited as a failure. + const ipAddress = getClientIp(request); + let counts: FullBackupCounts | undefined; + const body = streamToResponseBody( + async (write) => { + counts = await streamFullBackupJson(prisma, user.id, write); }, - }); + { + onComplete: () => + auditLog("user.export.full-backup", { + userId: user.id, + ipAddress, + details: { counts }, + }), + onError: (err) => + auditLog("user.export.full-backup.failed", { + userId: user.id, + ipAddress, + details: { + reason: err instanceof Error ? err.message : String(err), + }, + }), + }, + ); const stamp = new Date().toISOString().slice(0, 10); // Stream the JSON directly (NOT wrapped in the apiSuccess envelope) so // the file is a self-contained backup — admin upload + restore expect // the raw payload, not `{ data: { ... } }`. - return new NextResponse(JSON.stringify(payload), { + return new NextResponse(body, { status: 200, headers: { "Content-Type": "application/json; charset=utf-8", diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index a29f39741..85e81600d 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -130,10 +130,10 @@ describe("BackupsSection — empty state", () => { const className = helpElement?.match(/\bclass="([^"]*)"/)?.[1] ?? ""; expect(english).toContain( - "JSON file matching the current backup schema. Max 10 MB.", + "A HealthLog backup as .json or .json.gz. The file is compressed before upload; compressed, it may be up to 512 MB.", ); expect(german).toContain( - "JSON-Datei passend zum aktuellen Backup-Schema. Max. 10 MB.", + "HealthLog-Backup als .json oder .json.gz. Die Datei wird vor dem Hochladen komprimiert; komprimiert darf sie bis zu 512 MB groß sein.", ); expect(helpElement).toBeDefined(); expect(className.split(/\s+/)).toContain("text-muted-foreground"); diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index fb065f101..5fc81ab58 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -454,6 +454,20 @@ function missingSectionLabel( * the server may reword, and matching on it would turn a copy edit into a * silent loss of the panel. */ + +/** + * The upload body for a backup file: the file itself when it is already + * gzip, otherwise the file compressed in the browser. The server accepts + * both; compressing keeps a large export well inside the request size limit. + */ +async function uploadBody(file: File): Promise { + const head = new Uint8Array(await file.slice(0, 2).arrayBuffer()); + if (head[0] === 0x1f && head[1] === 0x8b) return file; + return new Response( + file.stream().pipeThrough(new CompressionStream("gzip")), + ).blob(); +} + export function missingSectionsOf(err: unknown): MissingBackupSection[] { if (!(err instanceof ApiError)) return []; if (err.meta?.errorCode !== "backup.section.missing") return []; @@ -697,10 +711,11 @@ export function BackupsSection() { const fileInputRef = useRef(null); const upload = useMutation({ mutationFn: async (file: File) => { - const fd = new FormData(); - fd.append("file", file); - // apiFetch with a raw init: the multipart body must NOT be JSON - // re-encoded, so the verb helper (which JSON-stringifies) is out. + // Sent as the raw body, compressed on the way unless it already is: the + // server reads it as a stream, and a large export (hundreds of MB of + // JSON) shrinks about tenfold. No request timeout, because a file of + // that size takes longer to check and store than the default allows. + const body = await uploadBody(file); return apiFetch<{ id: string; valid: true; @@ -734,7 +749,12 @@ export function BackupsSection() { customMetricEntries?: number; intradayProfiles?: number; }; - }>("/api/admin/backups/upload", { method: "POST", body: fd }); + }>("/api/admin/backups/upload", { + method: "POST", + body, + headers: { "Content-Type": "application/gzip" }, + signal: null, + }); }, onSuccess: (data) => { const total = @@ -818,7 +838,8 @@ export function BackupsSection() { return apiPost<{ restored: true; skipped?: RestoreSkipSummary }>( `/api/admin/backups/${row.id}/restore`, { confirm: "RESTORE", restoreInstanceSettings }, - { headers: { "Idempotency-Key": idempotencyKey } }, + // No request timeout: rebuilding a large account takes minutes. + { headers: { "Idempotency-Key": idempotencyKey }, signal: null }, ); }, onSuccess: (data) => { @@ -959,7 +980,7 @@ export function BackupsSection() { = new Set(); +/** + * Ciphertext written by raw SQL, which the Prisma-call walk below cannot see. + * + * `DataBackup.data` is the case: the stored backup is produced as ciphertext + * pieces (`packBackupBlobInto`) and assembled into the row by one SQL + * statement, because holding it as one string in the process is what a large + * record could not afford (#1031). Each entry names the file and the + * statement, and the scan fails if the statement is no longer there, so a + * moved or removed raw write cannot leave a stale entry that still passes. + */ +const RAW_CIPHERTEXT_WRITES: ReadonlyArray<{ + key: string; + file: string; + statement: RegExp; +}> = [ + { + key: "DataBackup.data", + file: "src/lib/export/store-backup-blob.ts", + statement: + /UPDATE\s+data_backups\s+SET\s+data\s*=\s*\(\s*SELECT\s+string_agg/, + }, +]; + function sourceFiles(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir)) { const p = join(dir, entry); @@ -388,6 +411,12 @@ function scanCiphertextWrites(): Map> { `\\b(?:prisma|tx|db)\\s*\\.\\s*(\\w+)\\s*\\.\\s*(?:${WRITE_METHODS})\\s*\\(`, "g", ); + for (const raw of RAW_CIPHERTEXT_WRITES) { + if (rel !== raw.file) continue; + const hit = raw.statement.exec(src); + if (hit) record(...(raw.key.split(".") as [string, string]), hit.index); + } + for (const m of src.matchAll(writes)) { const model = m[1][0].toUpperCase() + m[1].slice(1); if (!schema.models.has(model)) continue; @@ -425,6 +454,17 @@ describe("ciphertext columns derived from the code", () => { } }); + it("still finds every raw-SQL ciphertext write it was told about", () => { + for (const raw of RAW_CIPHERTEXT_WRITES) { + expect( + [...(written.get(raw.key) ?? [])].some((at) => + at.startsWith(`${raw.file}:`), + ), + `${raw.key}: no matching statement left in ${raw.file}`, + ).toBe(true); + } + }); + it("finds ciphertext writes, including the non-suffixed columns", () => { // Anchored non-zero match count. An empty or collapsed result set is the // failure mode this whole file exists to make impossible, so name the diff --git a/src/lib/export/__tests__/backup-json-scan.test.ts b/src/lib/export/__tests__/backup-json-scan.test.ts new file mode 100644 index 000000000..773e8080e --- /dev/null +++ b/src/lib/export/__tests__/backup-json-scan.test.ts @@ -0,0 +1,185 @@ +/** + * `scanBackupJson` must read a backup exactly as `JSON.parse` of the whole + * document would, with the streamed members handed out one element at a time + * (issue #1031: a 662 MB backup cannot exist as one string). These tests hold + * it to that over awkward values and arbitrary chunk boundaries, including + * boundaries inside strings, escapes and multi-byte characters. + */ +import { describe, expect, it } from "vitest"; + +import { BackupJsonError, scanBackupJson } from "../backup-json-scan"; + +async function* chunked(text: string, sizes: number[]) { + const bytes = Buffer.from(text, "utf8"); + let at = 0; + let i = 0; + while (at < bytes.length) { + const size = sizes[i++ % sizes.length]!; + yield bytes.subarray(at, at + size); + at += size; + } +} + +function rng(seed: number) { + let x = seed; + return () => { + x = (x * 1103515245 + 12345) % 2 ** 31; + return x / 2 ** 31; + }; +} + +const AWKWARD = [ + 'quote " inside', + "back\\slash and \\n", + "brackets ] } [ { , : in a string", + "umlaut ä ö ü and emoji 🫀", + "", + "\u0000\u001f control", +]; + +function randomValue(rand: () => number, depth: number): unknown { + const r = rand(); + if (depth > 2 || r < 0.35) { + const p = rand(); + if (p < 0.2) return null; + if (p < 0.3) return rand() < 0.5; + if (p < 0.55) return Math.round(rand() * 1e6) / 100 - 500; + return ( + AWKWARD[Math.floor(rand() * AWKWARD.length)]! + Math.floor(rand() * 9) + ); + } + if (r < 0.65) { + return Array.from({ length: Math.floor(rand() * 4) }, () => + randomValue(rand, depth + 1), + ); + } + const obj: Record = {}; + for (let i = 0; i < Math.floor(rand() * 4); i++) { + obj[`${AWKWARD[i % AWKWARD.length]}-${i}`] = randomValue(rand, depth + 1); + } + return obj; +} + +function randomDocument(seed: number) { + const rand = rng(seed); + return { + schemaVersion: "2", + userId: "user-ä", + measurements: Array.from({ length: Math.floor(rand() * 30) }, (_, i) => + i % 5 === 4 + ? randomValue(rand, 1) + : { id: `m${i}`, notes: AWKWARD[i % 6] }, + ), + medications: [randomValue(rand, 0), randomValue(rand, 0)], + moodEntries: [], + count: 42, + flag: false, + nothing: null, + text: AWKWARD[seed % AWKWARD.length], + nested: randomValue(rand, 0), + }; +} + +async function scan(text: string, sizes: number[], streamKeys: string[]) { + const elements: Record = {}; + const result = await scanBackupJson(chunked(text, sizes), { + streamKeys: new Set(streamKeys), + onElement: (key, element, index) => { + (elements[key] ??= [])[index] = element; + }, + }); + return { ...result, elements }; +} + +describe("scanBackupJson", () => { + it("equals JSON.parse over random documents, formats and chunk splits", async () => { + for (let seed = 1; seed <= 200; seed++) { + const doc = randomDocument(seed); + const text = + seed % 3 === 0 + ? JSON.stringify(doc, null, seed % 2 ? 2 : "\t") + : JSON.stringify(doc); + const sizes = [1 + (seed % 7), 2, 13, 1, 64].slice(seed % 3); + const { document, streamedCounts, elements } = await scan(text, sizes, [ + "measurements", + "moodEntries", + ]); + const expected = JSON.parse(text) as Record; + + expect(elements.measurements ?? [], `seed ${seed}`).toEqual( + expected.measurements, + ); + expect(streamedCounts).toEqual({ + measurements: (expected.measurements as unknown[]).length, + moodEntries: 0, + }); + expect(document, `seed ${seed}`).toEqual({ + ...expected, + measurements: [], + moodEntries: [], + }); + } + }); + + it("hands out streamed elements before it has read the rest of the document", async () => { + const text = JSON.stringify({ + a: 1, + measurements: [{ id: "x" }, { id: "y" }], + tail: "z".repeat(100), + }); + const bytes = Buffer.from(text); + let delivered = 0; + const seenAt: number[] = []; + async function* source() { + for (let i = 0; i < bytes.length; i += 4) { + delivered = i + 4; + yield bytes.subarray(i, i + 4); + } + } + await scanBackupJson(source(), { + streamKeys: new Set(["measurements"]), + onElement: () => { + seenAt.push(delivered); + }, + }); + expect(seenAt).toHaveLength(2); + expect(seenAt[1]!).toBeLessThan(bytes.length - 100); + }); + + it("skips a member it was told to skip without keeping it", async () => { + const { document } = await scanBackupJson( + chunked('{"a":[1,{"b":"]"}],"big":{"x":[1,2,3]},"c":"d"}', [3]), + { skipKeys: new Set(["big"]) }, + ); + expect(document).toEqual({ a: [1, { b: "]" }], c: "d" }); + }); + + it("refuses a truncated or malformed document", async () => { + for (const bad of [ + '{"a":1', + '{"a":[1,2}', + "[1,2]", + '{"measurements":{"x":1}}', + '{"a":1}{', + '{"measurements":[{"id":1} {"id":2}]}', + ]) { + await expect( + scanBackupJson(chunked(bad, [2]), { + streamKeys: new Set(["measurements"]), + }), + bad, + ).rejects.toBeInstanceOf(BackupJsonError); + } + }); + + it("propagates a failure from the element consumer", async () => { + await expect( + scanBackupJson(chunked('{"measurements":[1,2,3]}', [5]), { + streamKeys: new Set(["measurements"]), + onElement: (_key, _element, index) => { + if (index === 1) throw new Error("consumer failed"); + }, + }), + ).rejects.toThrow("consumer failed"); + }); +}); diff --git a/src/lib/export/__tests__/passphrase-archive.test.ts b/src/lib/export/__tests__/passphrase-archive.test.ts index 2cd158c97..dcdb84c5b 100644 --- a/src/lib/export/__tests__/passphrase-archive.test.ts +++ b/src/lib/export/__tests__/passphrase-archive.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest"; import { Buffer } from "node:buffer"; import { encryptArchive, + encryptArchiveToFile, decryptArchive, parseArchiveHeader, EXPORT_ARGON2_PARAMS, @@ -78,3 +79,45 @@ describe("passphrase-encrypted export archive (HLX1)", () => { expect(await decryptArchive(archive, PASSPHRASE)).toBe(PAYLOAD); }); }); + +/** + * #1031: the encrypted export is sealed as it is produced and spooled to a + * file, because holding it did not fit a 1 GB container for a large account. + * The format must not change: an archive written this way opens with the + * same `decryptArchive` every existing archive opens with. + */ +describe("encryptArchiveToFile", () => { + it("writes an archive decryptArchive opens, byte for byte the same document", async () => { + const { mkdtemp, readFile, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const dir = await mkdtemp(join(tmpdir(), "hlx-spool-")); + try { + const pieces = Array.from({ length: 500 }, (_, i) => + JSON.stringify({ i, note: "ümlaut 🫀 ".repeat(i % 7) }), + ); + const bodyPath = join(dir, "body"); + const spooled = await encryptArchiveToFile( + async (write) => { + await write("["); + for (let i = 0; i < pieces.length; i++) { + await write(i === 0 ? pieces[i]! : `,${pieces[i]}`); + } + await write(Buffer.from("]")); + }, + PASSPHRASE, + bodyPath, + ); + const archive = Buffer.concat([spooled.prefix, await readFile(bodyPath)]); + expect(archive.byteLength).toBe(spooled.byteLength); + expect(await decryptArchive(archive, PASSPHRASE)).toBe( + `[${pieces.join(",")}]`, + ); + await expect( + decryptArchive(archive, "wrong passphrase!!"), + ).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/export/backup-blob.ts b/src/lib/export/backup-blob.ts index f305a67b4..0c152ba45 100644 --- a/src/lib/export/backup-blob.ts +++ b/src/lib/export/backup-blob.ts @@ -37,7 +37,8 @@ */ import { Buffer } from "node:buffer"; import v8 from "node:v8"; -import { createGzip, gunzipSync, gzipSync } from "node:zlib"; +import { Readable } from "node:stream"; +import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import { createStreamEncryptor, @@ -125,7 +126,7 @@ export class BackupBlobTooLargeError extends Error { * counts) is the caller's business, not the envelope's. */ export type BackupJsonProducer = ( - write: (chunk: string) => Promise, + write: (chunk: string | Buffer) => Promise, ) => Promise; export interface PackBackupBlobOptions { @@ -236,7 +237,7 @@ export async function packBackupBlobInto( await sink(piece); }; - const write = async (chunk: string): Promise => { + const write = async (chunk: string | Buffer): Promise => { if (failure) throw failure; await flush(false); if (gzip.write(chunk, "utf8")) return; @@ -313,3 +314,34 @@ export function unpackBackupBlob(stored: string): string { Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"), ).toString("utf8"); } + +/** + * A stored `DataBackup.data` string → a source of its JSON as byte chunks, + * which can be opened as many times as the caller needs to read it. + * + * `unpackBackupBlob` returns the JSON as one string, and a string is what a + * large record cannot be: the disaster-recovery JSON of an account with 1.25 + * million measurements is 662 MB, past the 536 870 888 characters V8 allows + * in any string, so that call threw on every such backup (#1031). Here the + * authentication happens first and whole, on the compressed ciphertext, which + * stays small (48 MB for that record): no plaintext byte is released before + * the tag has verified. Only the decompression is streamed, on every open. + * + * Fail-closed exactly like `unpackBackupBlob`: a bad key or a tag that does + * not verify throws here, before a source exists; a truncated gzip member + * errors the stream. + */ +export function openBackupBlob(stored: string): () => AsyncIterable { + if (isStreamCiphertext(stored)) { + const gz = decryptStream(stored); + return () => Readable.from([gz]).pipe(createGunzip()); + } + const plaintext = decrypt(stored); + if (!plaintext.startsWith(GZIP_MARKER)) { + // A pre-envelope copy was written from one string, so it is one. + const bytes = Buffer.from(plaintext, "utf8"); + return () => Readable.from([bytes]); + } + const gz = Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"); + return () => Readable.from([gz]).pipe(createGunzip()); +} diff --git a/src/lib/export/backup-json-scan.ts b/src/lib/export/backup-json-scan.ts new file mode 100644 index 000000000..60c3c967a --- /dev/null +++ b/src/lib/export/backup-json-scan.ts @@ -0,0 +1,336 @@ +/** + * Read a backup's JSON as a stream, without ever holding the document. + * + * Why. A backup is one JSON object, and every reader used to turn it into + * one string and `JSON.parse` it. For an account of 1.25 million measurements + * (issue #1031) the disaster-recovery JSON is 662 MB. V8 cannot hold a string + * longer than 536 870 888 characters at all, so `unpackBackupBlob` threw + * before a byte was parsed: the weekly backup of that account could not be + * restored, downloaded or even summarised, and the restore route reported the + * copy as undecryptable. A portable export of the same account fits under + * the limit but not beside the rest of a 1 GB container. + * + * What this does. It scans the bytes once, tracking only string and nesting + * state. Every top-level member is parsed on its own with `JSON.parse` and + * kept, except the members named in `streamKeys`: those must be arrays, and + * each element is parsed and handed to `onElement` instead, so the caller + * sees them one at a time and the kept object holds an empty array in their + * place. A backup's bulk tables (measurements, intake events, mood entries) + * are the only members that grow with a record, so everything else is small. + * + * Structural characters are ASCII and UTF-8 never uses an ASCII byte inside a + * multi-byte sequence, so scanning bytes is exact. A value is decoded only + * once it is complete, which is where a character split across two chunks is + * put back together. + * + * The output is what `JSON.parse` of the whole document would give, with the + * streamed arrays emptied: the tests hold it to that over arbitrary splits. + */ + +/** Characters the scanner cares about, as bytes. */ +const QUOTE = 0x22; +const BACKSLASH = 0x5c; +const OPEN_BRACE = 0x7b; +const CLOSE_BRACE = 0x7d; +const OPEN_BRACKET = 0x5b; +const CLOSE_BRACKET = 0x5d; +const COMMA = 0x2c; +const COLON = 0x3a; + +function isWhitespace(byte: number): boolean { + return byte === 0x20 || byte === 0x0a || byte === 0x0d || byte === 0x09; +} + +export interface ScanBackupJsonOptions { + /** Top-level members whose array elements are streamed, not kept. */ + streamKeys?: ReadonlySet; + /** + * Receives each element of a streamed member, parsed, with its index. + * Awaited, so a consumer that writes to a database applies backpressure. + */ + onElement?: (key: string, element: unknown, index: number) => unknown; + /** + * Members to skip entirely: scanned past, never parsed or kept. For a + * reader that only needs a few sections of the document. + */ + skipKeys?: ReadonlySet; +} + +export interface ScanBackupJsonResult { + /** The document, with streamed members as empty arrays and skipped ones absent. */ + document: Record; + /** How many elements each streamed member held. */ + streamedCounts: Record; +} + +/** Thrown for input that is not a JSON object the scanner can walk. */ +export class BackupJsonError extends Error { + constructor(message: string) { + super(message); + this.name = "BackupJsonError"; + } +} + +/** Collects the bytes of one value across chunks. */ +class Collector { + private parts: Buffer[] = []; + private bytes = 0; + add(chunk: Buffer, from: number, to: number): void { + if (to > from) { + this.parts.push(chunk.subarray(from, to)); + this.bytes += to - from; + } + } + take(): string { + const text = + this.parts.length === 1 + ? this.parts[0]!.toString("utf8") + : Buffer.concat(this.parts, this.bytes).toString("utf8"); + this.parts = []; + this.bytes = 0; + return text; + } +} + +type Phase = + | "start" // before the opening brace + | "key" // expecting a key or the closing brace + | "colon" + | "value" // expecting the start of a member value + | "in-value" // inside a kept or skipped member value + | "array-start" // a streamed member: expecting its opening bracket + | "element" // inside a streamed array, expecting an element or `]` + | "in-element" // inside one element + | "after" // after a member, expecting `,` or `}` + | "done"; + +/** + * Scan a backup document delivered as byte chunks. Resolves to the document + * (see the module comment) once the closing brace has been read; rejects with + * `BackupJsonError` on malformed input or with whatever `onElement` threw. + */ +export async function scanBackupJson( + chunks: AsyncIterable, + options: ScanBackupJsonOptions = {}, +): Promise { + const streamKeys = options.streamKeys ?? new Set(); + const skipKeys = options.skipKeys ?? new Set(); + const document: Record = {}; + const streamedCounts: Record = {}; + + let phase: Phase = "start"; + let inString = false; + let escaped = false; + // Nesting depth inside the current value or element (0 = not inside one). + let depth = 0; + let key = ""; + let skipping = false; + // Inside a key string. + let keyOpen = false; + // Inside a streamed array, after an element: the next byte must be `,` or `]`. + let elementNeedsComma = false; + const keyBytes = new Collector(); + const value = new Collector(); + + const finishValue = () => { + const text = value.take(); + if (skipping) return; + try { + document[key] = JSON.parse(text); + } catch { + throw new BackupJsonError(`Backup member '${key}' is not valid JSON`); + } + }; + const finishElement = async () => { + const text = value.take(); + let element: unknown; + try { + element = JSON.parse(text); + } catch { + throw new BackupJsonError( + `Element ${streamedCounts[key]} of backup member '${key}' is not valid JSON`, + ); + } + const index = streamedCounts[key]!; + streamedCounts[key] = index + 1; + await options.onElement?.(key, element, index); + }; + + for await (const raw of chunks) { + const chunk = Buffer.isBuffer(raw) + ? raw + : Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength); + // Where the value or key being collected started in this chunk. + let mark = 0; + for (let i = 0; i < chunk.length; i++) { + const byte = chunk[i]!; + + if (phase === "in-value" || phase === "in-element") { + if (inString) { + if (escaped) escaped = false; + else if (byte === BACKSLASH) escaped = true; + else if (byte === QUOTE) inString = false; + continue; + } + if (byte === QUOTE) { + inString = true; + continue; + } + if (byte === OPEN_BRACE || byte === OPEN_BRACKET) { + depth += 1; + continue; + } + if (byte === CLOSE_BRACE || byte === CLOSE_BRACKET) { + if (depth > 0) { + depth -= 1; + if (depth > 0) continue; + // The value's own closing bracket: it belongs to the value. + value.add(chunk, mark, i + 1); + if (phase === "in-value") { + finishValue(); + phase = "after"; + } else { + await finishElement(); + phase = "element"; + elementNeedsComma = true; + } + continue; + } + // A primitive value ended by the enclosing bracket. + value.add(chunk, mark, i); + if (phase === "in-value") { + if (byte !== CLOSE_BRACE) { + throw new BackupJsonError("Unbalanced bracket in backup"); + } + finishValue(); + phase = "done"; + } else { + if (byte !== CLOSE_BRACKET) { + throw new BackupJsonError("Unbalanced bracket in backup"); + } + await finishElement(); + phase = "after"; + } + continue; + } + if (depth === 0 && (byte === COMMA || isWhitespace(byte))) { + // A primitive value ended. + value.add(chunk, mark, i); + if (phase === "in-value") { + finishValue(); + phase = byte === COMMA ? "key" : "after"; + } else { + await finishElement(); + phase = "element"; + elementNeedsComma = byte !== COMMA; + } + continue; + } + continue; + } + + if (phase === "key" && keyOpen) { + if (escaped) escaped = false; + else if (byte === BACKSLASH) escaped = true; + else if (byte === QUOTE) { + keyBytes.add(chunk, mark, i + 1); + key = JSON.parse(keyBytes.take()) as string; + keyOpen = false; + phase = "colon"; + } + continue; + } + + if (isWhitespace(byte)) continue; + + switch (phase) { + case "start": + if (byte !== OPEN_BRACE) { + throw new BackupJsonError("Backup is not a JSON object"); + } + phase = "key"; + break; + case "key": + if (byte === CLOSE_BRACE) { + phase = "done"; + } else if (byte === QUOTE) { + keyOpen = true; + escaped = false; + mark = i; + } else { + throw new BackupJsonError("Malformed backup: expected a key"); + } + break; + case "colon": + if (byte !== COLON) { + throw new BackupJsonError("Malformed backup: expected ':'"); + } + phase = streamKeys.has(key) ? "array-start" : "value"; + skipping = skipKeys.has(key); + break; + case "value": + phase = "in-value"; + mark = i; + depth = 0; + inString = false; + escaped = false; + // Re-read this byte as the first byte of the value. + i -= 1; + break; + case "array-start": + if (byte !== OPEN_BRACKET) { + throw new BackupJsonError( + `Backup member '${key}' must be an array`, + ); + } + streamedCounts[key] = 0; + document[key] = []; + phase = "element"; + elementNeedsComma = false; + break; + case "element": + if (byte === CLOSE_BRACKET) { + phase = "after"; + } else if (byte === COMMA && elementNeedsComma) { + elementNeedsComma = false; + } else if (!elementNeedsComma) { + phase = "in-element"; + mark = i; + depth = 0; + inString = false; + escaped = false; + i -= 1; + } else { + throw new BackupJsonError( + `Malformed backup: expected ',' in '${key}'`, + ); + } + break; + case "after": + if (byte === COMMA) { + phase = "key"; + } else if (byte === CLOSE_BRACE) { + phase = "done"; + } else { + throw new BackupJsonError("Malformed backup: expected ',' or '}'"); + } + break; + case "done": + throw new BackupJsonError("Unexpected content after the backup"); + } + } + // Carry the unfinished key or value into the next chunk. + if (phase === "in-value" || phase === "in-element") { + value.add(chunk, mark, chunk.length); + } else if (phase === "key" && keyOpen) { + keyBytes.add(chunk, mark, chunk.length); + } + } + + // Anything but the closing brace as the last structural byte is a + // truncated file, including one that ends inside a member value. + if ((phase as Phase) !== "done") { + throw new BackupJsonError("Backup ends before its closing brace"); + } + return { document, streamedCounts }; +} diff --git a/src/lib/export/measurement-bulk-insert.ts b/src/lib/export/measurement-bulk-insert.ts new file mode 100644 index 000000000..f1db771d9 --- /dev/null +++ b/src/lib/export/measurement-bulk-insert.ts @@ -0,0 +1,136 @@ +/** + * Insert many measurements in one statement, as column arrays. + * + * Why not `createMany`. Prisma compiles and caches a query plan per query + * shape, and a `createMany` of N rows is its own shape: measured, one + * 1 000-row `createMany` of measurements leaves about 145 MB of retained + * heap behind, a 2 000-row one about 290 MB, and the smaller last batch of a + * restore adds its own plan on top. In a 1 GB container that alone took a + * restore of 1.25 million measurements past the heap limit (#1031). One + * `INSERT … SELECT FROM unnest(…)` with an array per column is a single small + * statement whatever the batch size, and about four times faster. + * + * `COLUMNS` maps every field of `MeasurementInsertRow` to its column and + * type. It is typed as a complete record over the row's keys, so a field + * added to the row without a column here does not compile, and the restore + * builds its rows as `MeasurementInsertRow`, so a field it starts writing has + * to be added to the row first. + */ +import type { Prisma } from "@/generated/prisma/client"; + +/** One measurement as the restore writes it. */ +export interface MeasurementInsertRow { + id: string; + userId: string; + type: string; + value: number; + valueMin: number | null; + valueMax: number | null; + unit: string; + source: string; + measuredAt: Date; + notes: string | null; + notesEncrypted: Uint8Array | null; + externalId: string | null; + externalSourceVersion: string | null; + aggregationProvenance: string | null; + glucoseContext: string | null; + sleepStage: string | null; + rhythmClassification: string | null; + deviceType: string | null; + syncVersion: number; + deletedAt: Date | null; + createdAt?: Date; + updatedAt?: Date; +} + +type Kind = + "text" | "float8" | "int4" | "timestamp" | "bytea" | `enum:${string}`; + +/** Column and element type per field. Complete by construction. */ +const COLUMNS: { [K in keyof MeasurementInsertRow]-?: [string, Kind] } = { + id: ["id", "text"], + userId: ["user_id", "text"], + type: ["type", "enum:measurement_type"], + value: ["value", "float8"], + valueMin: ["value_min", "float8"], + valueMax: ["value_max", "float8"], + unit: ["unit", "text"], + source: ["source", "enum:measurement_source"], + measuredAt: ["measured_at", "timestamp"], + notes: ["notes", "text"], + notesEncrypted: ["notes_encrypted", "bytea"], + externalId: ["external_id", "text"], + externalSourceVersion: ["external_source_version", "text"], + aggregationProvenance: [ + "aggregation_provenance", + "enum:measurement_aggregation_provenance", + ], + glucoseContext: ["glucose_context", "enum:glucose_context"], + sleepStage: ["sleep_stage", "enum:sleep_stage"], + rhythmClassification: ["rhythm_classification", "enum:rhythm_classification"], + deviceType: ["device_type", "text"], + syncVersion: ["sync_version", "int4"], + deletedAt: ["deleted_at", "timestamp"], + createdAt: ["created_at", "timestamp"], + updatedAt: ["updated_at", "timestamp"], +}; + +const FIELDS = Object.keys(COLUMNS) as Array; + +/** How each array travels, and how its element is turned back into a value. */ +function arrayType(kind: Kind): string { + if (kind === "float8" || kind === "int4") return `${kind}[]`; + return "text[]"; // timestamps as ISO text, bytes as hex, enums as labels +} + +function selectExpression( + field: keyof MeasurementInsertRow, + kind: Kind, +): string { + const ref = `u."${field}"`; + if (kind === "timestamp") { + // Stored as UTC in `timestamp(3)`. ISO text with a `Z` casts to the same + // wall-clock instant; the zone suffix is ignored by the cast. + const cast = `${ref}::timestamp(3)`; + // Prisma fills both on create when the row does not carry them. + return field === "createdAt" || field === "updatedAt" + ? `COALESCE(${cast}, CURRENT_TIMESTAMP)` + : cast; + } + if (kind === "bytea") return `decode(${ref}, 'hex')`; + if (kind.startsWith("enum:")) return `${ref}::${kind.slice(5)}`; + return ref; +} + +/** + * The statement, built once. Its text depends on nothing but `COLUMNS`, a + * compile-time table: no input reaches it, and every value travels as a bound + * array parameter. + */ +export const MEASUREMENT_BULK_INSERT_SQL = ` + INSERT INTO measurements (${FIELDS.map((f) => COLUMNS[f][0]).join(", ")}) + SELECT ${FIELDS.map((f) => selectExpression(f, COLUMNS[f][1])).join(", ")} + FROM unnest(${FIELDS.map((f, i) => `$${i + 1}::${arrayType(COLUMNS[f][1])}`).join(", ")}) + AS u(${FIELDS.map((f) => `"${f}"`).join(", ")}) +`; + +function cell(value: unknown, kind: Kind): unknown { + if (value === null || value === undefined) return null; + if (kind === "timestamp") return (value as Date).toISOString(); + if (kind === "bytea") return Buffer.from(value as Uint8Array).toString("hex"); + return value; +} + +/** Insert `rows` in one statement. Resolves to the number inserted. */ +export async function insertMeasurementRows( + tx: Prisma.TransactionClient, + rows: readonly MeasurementInsertRow[], +): Promise { + if (rows.length === 0) return 0; + const arrays = FIELDS.map((field) => { + const kind = COLUMNS[field][1]; + return rows.map((row) => cell(row[field], kind)); + }); + return tx.$executeRawUnsafe(MEASUREMENT_BULK_INSERT_SQL, ...arrays); +} diff --git a/src/lib/export/passphrase-archive.ts b/src/lib/export/passphrase-archive.ts index e30ee171b..f1793bdaf 100644 --- a/src/lib/export/passphrase-archive.ts +++ b/src/lib/export/passphrase-archive.ts @@ -39,6 +39,8 @@ * hashing params so tuning one never silently changes the other. */ import { hashRaw, type Algorithm } from "@node-rs/argon2"; +import { once } from "node:events"; +import { createWriteStream } from "node:fs"; import { createCipheriv, createDecipheriv, @@ -128,6 +130,11 @@ export async function encryptArchive( const ciphertext = Buffer.concat([cipher.update(body), cipher.final()]); const tag = cipher.getAuthTag(); + return Buffer.concat([archiveHeader(), salt, iv, tag, ciphertext]); +} + +/** The fixed `HLX1` header for the current KDF parameters. */ +function archiveHeader(): Buffer { const header = Buffer.alloc(4 + 1 + 1 + 4 + 4 + 1 + 1); header.write(MAGIC, 0, "ascii"); header.writeUInt8(VERSION, 4); @@ -136,8 +143,88 @@ export async function encryptArchive( header.writeUInt32BE(EXPORT_ARGON2_PARAMS.timeCost, 10); header.writeUInt8(EXPORT_ARGON2_PARAMS.parallelism, 14); header.writeUInt8(SALT_LENGTH, 15); + return header; +} - return Buffer.concat([header, salt, iv, tag, ciphertext]); +/** Where `encryptArchiveToFile` left an archive, in two parts. */ +export interface SpooledArchive { + /** Header, salt, IV and tag: everything in front of the ciphertext. */ + prefix: Buffer; + /** File holding the ciphertext, which follows the prefix. */ + bodyPath: string; + /** Size of the whole archive in bytes. */ + byteLength: number; +} + +/** + * Seal a JSON document produced in pieces into an `HLX1` archive without + * holding it: the ciphertext goes to `bodyPath` as it is produced. + * + * Why a file. The format puts the GCM tag in front of the ciphertext, and the + * tag exists only once the last byte has been encrypted, so the archive + * cannot be sent as it is written. Holding it in memory instead is what the + * route did, and for an account of 1.25 million measurements that did not fit + * a 1 GB container (#1031). Spooling the ciphertext keeps the format, so every + * archive already on someone's disk still opens the same way. Only ciphertext + * ever touches the disk; the caller deletes the file once it is sent. + * + * Decrypts with `decryptArchive` exactly like an archive `encryptArchive` + * wrote; the tests hold the two to that. + */ +export async function encryptArchiveToFile( + producer: ( + write: (chunk: string | Buffer) => Promise, + ) => Promise, + passphrase: string, + bodyPath: string, +): Promise { + if (passphrase.length < MIN_EXPORT_PASSPHRASE_LENGTH) { + throw new Error( + `Passphrase must be at least ${MIN_EXPORT_PASSPHRASE_LENGTH} characters`, + ); + } + const salt = randomBytes(SALT_LENGTH); + const key = await deriveKey(passphrase, EXPORT_ARGON2_PARAMS, salt); + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv(ALGORITHM, key, iv); + const file = createWriteStream(bodyPath, { mode: 0o600 }); + let bodyBytes = 0; + const fileError = new Promise((_, reject) => { + file.once("error", reject); + }); + fileError.catch(() => {}); + + const put = async (bytes: Buffer): Promise => { + if (bytes.length === 0) return; + bodyBytes += bytes.length; + if (!file.write(bytes)) { + await Promise.race([once(file, "drain"), fileError]); + } + }; + + try { + await producer(async (chunk) => { + await put( + cipher.update( + typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk, + ), + ); + }); + await put(cipher.final()); + file.end(); + await Promise.race([once(file, "finish"), fileError]); + } catch (err) { + file.destroy(); + throw err; + } + + const prefix = Buffer.concat([ + archiveHeader(), + salt, + iv, + cipher.getAuthTag(), + ]); + return { prefix, bodyPath, byteLength: prefix.length + bodyBytes }; } /** Parse + validate the fixed `HLX1` header. Throws on any structural fault. */ diff --git a/src/lib/export/response-stream.ts b/src/lib/export/response-stream.ts new file mode 100644 index 000000000..964bb2c41 --- /dev/null +++ b/src/lib/export/response-stream.ts @@ -0,0 +1,82 @@ +/** + * Turn a producer of text or bytes into a response body, with backpressure + * and cancellation. + * + * The export routes used to build their whole file as one string and hand it + * to the response. For an account of 1.25 million measurements that string + * and the object graph behind it did not fit a 1 GB container: one export + * took the app down for every user of the instance (#1031). The file is now + * written into the response as it is produced. + * + * Backpressure: the producer's `write` waits while the client is slower than + * the database, so nothing piles up in between. Cancellation: when the client + * goes away, the next `write` throws `ResponseStreamClosedError`, which stops + * the producer instead of leaving it to finish a file nobody reads. + */ +import { PassThrough, Readable } from "node:stream"; + +/** The client closed the response before the producer finished. */ +export class ResponseStreamClosedError extends Error { + constructor() { + super("The response was closed before the file was complete"); + this.name = "ResponseStreamClosedError"; + } +} + +export type ResponseProducer = ( + write: (chunk: string | Buffer) => Promise, +) => Promise; + +export interface ResponseStreamHooks { + /** Runs once the producer has written everything. */ + onComplete?: () => unknown; + /** + * Runs when the producer failed or the client left. The response has + * already started, so the client receives a truncated file; a truncated + * JSON file does not parse, and a truncated archive fails its tag. + */ + onError?: (err: unknown) => unknown; +} + +/** Start `producer` and return the web stream its output flows into. */ +export function streamToResponseBody( + producer: ResponseProducer, + hooks: ResponseStreamHooks = {}, +): ReadableStream { + const out = new PassThrough(); + + const write = async (chunk: string | Buffer): Promise => { + if (out.destroyed || out.writableEnded) + throw new ResponseStreamClosedError(); + if (out.write(chunk)) return; + await new Promise((resolve, reject) => { + const onDrain = () => { + out.off("close", onClose); + resolve(); + }; + const onClose = () => { + out.off("drain", onDrain); + reject(new ResponseStreamClosedError()); + }; + out.once("drain", onDrain); + out.once("close", onClose); + }); + }; + + void (async () => { + try { + await producer(write); + out.end(); + await hooks.onComplete?.(); + } catch (err) { + out.destroy(err instanceof Error ? err : new Error(String(err))); + try { + await hooks.onError?.(err); + } catch { + // Reporting a failure must not raise another one. + } + } + })(); + + return Readable.toWeb(out) as ReadableStream; +} diff --git a/src/lib/export/store-backup-blob.ts b/src/lib/export/store-backup-blob.ts index cf7989269..82c8f0fae 100644 --- a/src/lib/export/store-backup-blob.ts +++ b/src/lib/export/store-backup-blob.ts @@ -57,14 +57,18 @@ export interface StoreBackupBlobInput { /** * Pack `producer`'s JSON into the stored envelope and upsert it as the - * account's `(userId, type)` backup. Resolves to the stored size in bytes. + * `(userId, type)` backup. Resolves to the row's id and the stored size. + * + * `input` may be a function, read once the producer has finished: an + * uploaded file names its owner somewhere inside itself, and the upload + * route only knows who that is once it has read the file through. */ export async function storeBackupBlob( prisma: PrismaClient, - input: StoreBackupBlobInput, + input: StoreBackupBlobInput | (() => StoreBackupBlobInput), producer: BackupJsonProducer, options: PackBackupBlobOptions = {}, -): Promise { +): Promise<{ id: string; bytes: number }> { return prisma.$transaction( async (tx) => { await tx.$executeRawUnsafe( @@ -91,22 +95,24 @@ export async function storeBackupBlob( options, ); + const target = typeof input === "function" ? input() : input; // The row first, through Prisma, so a new account's backup gets its id // the same way every other row does; the data it briefly carries is // never visible outside this transaction. - await tx.dataBackup.upsert({ - where: { userId_type: { userId: input.userId, type: input.type } }, + const row = await tx.dataBackup.upsert({ + where: { userId_type: { userId: target.userId, type: target.type } }, update: { createdAt: new Date() }, - create: { userId: input.userId, type: input.type, data: "" }, + create: { userId: target.userId, type: target.type, data: "" }, + select: { id: true }, }); await tx.$executeRaw` UPDATE data_backups SET data = ( SELECT string_agg(piece, '' ORDER BY seq) FROM backup_blob_parts ) - WHERE user_id = ${input.userId} AND type = ${input.type} + WHERE id = ${row.id} `; - return bytes; + return { id: row.id, bytes }; }, { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 }, ); diff --git a/src/lib/export/streamed-backup.ts b/src/lib/export/streamed-backup.ts new file mode 100644 index 000000000..1e65d6150 --- /dev/null +++ b/src/lib/export/streamed-backup.ts @@ -0,0 +1,138 @@ +/** + * A backup file read as a stream: every section but the measurements in + * memory, the measurements handed over in batches. + * + * Why. The measurements are the one section that grows with a record without + * bound. For an account with 1.25 million of them the disaster-recovery JSON + * is 662 MB, which cannot exist as one JavaScript string, and the parsed rows + * alone would not fit a 1 GB container beside the running app (#1031). Every + * other section of the same file is a few megabytes. + * + * So the file is read twice. The first read keeps everything except the + * measurements, and checks each measurement against the same element schema + * the whole-file schema uses, counting them. Nothing is written on the + * strength of an unchecked row: a restore reads the file this way before it + * deletes anything. The second read, from `forEachMeasurementBatch`, hands + * the measurements over in order, in batches, for the writer. + * + * `raw` is the file exactly as `JSON.parse` would give it, with + * `measurements` replaced by an empty array when the file has that section + * (and still absent when it does not, which is what the section-presence + * check reads). Run it through `parseBackupPayload` for the rest of the + * validation; together that is the whole-file check. + */ +import { scanBackupJson } from "@/lib/export/backup-json-scan"; +import { + BACKUP_SCHEMA_VERSION, + backupMeasurementSchema, + type BackupMeasurement, +} from "@/lib/validations/backup"; + +/** The file's bytes, from the start, every time it is called. */ +export type BackupSource = () => AsyncIterable; + +const MEASUREMENTS = "measurements"; +const STREAMED = new Set([MEASUREMENTS]); + +/** A measurement the element schema refused, with where it is in the file. */ +export class StreamedBackupInvalidError extends Error { + readonly path: string; + constructor(path: string, message: string) { + super(`${path}: ${message}`); + this.name = "StreamedBackupInvalidError"; + this.path = path; + } +} + +export interface StreamedBackup { + /** The file without its measurements (see the module comment). */ + raw: Record; + /** How many measurements the file carries. */ + measurementCount: number; + /** The file's size in bytes. */ + bytes: number; + /** + * Read the file again and hand its measurements over in file order, in + * batches of at most `size`, each row parsed by the element schema. + * Awaits `fn` before reading further. + */ + forEachMeasurementBatch( + size: number, + fn: (batch: BackupMeasurement[]) => Promise, + ): Promise; +} + +function parseMeasurement(element: unknown, index: number): BackupMeasurement { + const parsed = backupMeasurementSchema.safeParse(element); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + throw new StreamedBackupInvalidError( + [MEASUREMENTS, index, ...(issue?.path ?? [])].join("."), + issue?.message ?? "invalid measurement", + ); + } + return parsed.data; +} + +/** + * First read of a backup file. Rejects with `BackupJsonError` for input that + * is not a JSON object and with `StreamedBackupInvalidError` for a + * measurement the schema refuses. + */ +export async function readStreamedBackup( + source: BackupSource, +): Promise { + let firstWithoutId: number | null = null; + let bytes = 0; + async function* counted() { + for await (const chunk of source()) { + bytes += chunk.byteLength; + yield chunk; + } + } + const { document, streamedCounts } = await scanBackupJson(counted(), { + streamKeys: STREAMED, + onElement: (_key, element, index) => { + const row = parseMeasurement(element, index); + if (!row.id && firstWithoutId === null) firstWithoutId = index; + }, + }); + + // The whole-file schema's cross-row rule, applied the same way: a canonical + // v2 file must give every measurement a stable id. + if ( + document.schemaVersion === BACKUP_SCHEMA_VERSION && + firstWithoutId !== null + ) { + throw new StreamedBackupInvalidError( + `${MEASUREMENTS}.${firstWithoutId}.id`, + "Canonical v2 measurements require a stable id", + ); + } + + const skipKeys = new Set( + Object.keys(document).filter((key) => key !== MEASUREMENTS), + ); + + return { + raw: document, + measurementCount: streamedCounts[MEASUREMENTS] ?? 0, + bytes, + async forEachMeasurementBatch(size, fn) { + let batch: BackupMeasurement[] = []; + await scanBackupJson(source(), { + streamKeys: STREAMED, + skipKeys, + onElement: async (_key, element, index) => { + batch.push(parseMeasurement(element, index)); + if (batch.length >= size) { + const full = batch; + batch = []; + await fn(full); + } + }, + }); + if (batch.length > 0) await fn(batch); + }, + }; +} diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 3627b1d2b..19bb54f84 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -23,8 +23,8 @@ */ import { Buffer } from "node:buffer"; import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; -import { Transform, type Readable } from "node:stream"; -import { createGzip, gunzipSync, gzipSync } from "node:zlib"; +import { Readable, Transform } from "node:stream"; +import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; @@ -193,6 +193,48 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { : plaintext.toString("utf8"); } +/** + * An off-host object → a source of its JSON as byte chunks, openable as often + * as needed. `decryptBackup` answers one string, which the JSON of a large + * record cannot be: 662 MB for an account of 1.25 million measurements, past + * the longest string V8 allows (#1031). The tag is verified whole on the + * compressed ciphertext before a source exists, exactly as `decryptBackup` + * does; only the decompression is streamed. + */ +export function openBackupObject( + buf: Buffer, + key: Buffer, +): () => AsyncIterable { + const magic = buf.subarray(0, 4).toString("binary"); + const version = buf[PREAMBLE_LENGTH - 1]; + if ( + magic !== MAGIC || + (version !== BACKUP_ENVELOPE_PLAIN && + version !== BACKUP_ENVELOPE_GZIP && + version !== BACKUP_ENVELOPE_STREAM) + ) { + throw new Error("Invalid backup envelope (bad magic or version)"); + } + let plaintext: Buffer; + if (version === BACKUP_ENVELOPE_STREAM) { + plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key); + } else { + const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH); + const tag = buf.subarray( + PREAMBLE_LENGTH + IV_LENGTH, + PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH, + ); + const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH); + const dec = createDecipheriv(ALGORITHM, key, iv); + dec.setAuthTag(tag); + plaintext = Buffer.concat([dec.update(ct), dec.final()]); + } + if (version === BACKUP_ENVELOPE_PLAIN) { + return () => Readable.from([plaintext]); + } + return () => Readable.from([plaintext]).pipe(createGunzip()); +} + /** * How much of one object the upload holds at a time, and how many of those * windows are in flight. `@aws-sdk/lib-storage` buffers `partSize` bytes per diff --git a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts index fe2a94b55..743eae2a6 100644 --- a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts +++ b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts @@ -34,7 +34,7 @@ vi.mock("@/lib/export/store-backup-blob", () => ({ out += chunk; }); await mocks.store(input, out); - return out.length; + return { id: "backup-row", bytes: out.length }; }, })); diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 13c258894..41a114b1d 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -132,7 +132,7 @@ export async function handleDataBackup( // assembles into the row (`storeBackupBlob`). The stored copy used // to be built here as one string first, which on a large record // added several hundred megabytes to a container capped at 1 GB. - const storedBytes = await storeBackupBlob( + const { bytes: storedBytes } = await storeBackupBlob( prisma, { userId: user.id, type: "WEEKLY_AUTO" }, (write) => diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 36056c56f..50450c630 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -22,8 +22,9 @@ * no other cron uses. */ import type { Job } from "pg-boss"; +import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; import { - decryptBackup, + openBackupObject, getS3Client, loadOffhostConfig, OffhostBackupNotConfiguredError, @@ -103,19 +104,36 @@ export async function runRestoreDrill( const dateKey = BACKUP_KEY_PATTERN.exec(objectKey)![1]; const ciphertext = await s3.getObject(objectKey); - const plaintext = decryptBackup(ciphertext, cfg.encryptionKey); - const parsed: unknown = JSON.parse(plaintext); - - if (typeof parsed !== "object" || parsed === null) { - throw new Error( - `Restore drill: backup object "${objectKey}" decrypted and parsed but is not a JSON object.`, - ); + // Read as a stream, never as one string: the JSON of a large record is + // longer than any string V8 can hold (#1031). The bulk tables are counted, + // not kept. + const source = openBackupObject(ciphertext, cfg.encryptionKey); + let plaintextBytes = 0; + async function* counted() { + for await (const chunk of source()) { + plaintextBytes += chunk.byteLength; + yield chunk; + } } - const payload = parsed as Record; + let scanned; + try { + scanned = await scanBackupJson(counted(), { + streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), + }); + } catch (err) { + if (err instanceof BackupJsonError) { + throw new Error( + `Restore drill: backup object "${objectKey}" decrypted but is not a JSON object: ${err.message}`, + ); + } + throw err; + } + const payload = scanned.document; + const streamedCounts = scanned.streamedCounts; if ( typeof payload.exportedAt !== "string" || typeof payload.userId !== "string" || - !Array.isArray(payload.measurements) + streamedCounts.measurements === undefined ) { throw new Error( `Restore drill: backup object "${objectKey}" parses but is missing core fields (exportedAt / userId / measurements).`, @@ -133,12 +151,12 @@ export async function runRestoreDrill( ageDays, stale: ageDays > MAX_BACKUP_AGE_DAYS, ciphertextBytes: ciphertext.length, - plaintextBytes: Buffer.byteLength(plaintext, "utf8"), + plaintextBytes, recordCounts: { - measurements: payload.measurements.length, + measurements: streamedCounts.measurements, medications: countArray(payload.medications), - intakeEvents: countArray(payload.intakeEvents), - moodEntries: countArray(payload.moodEntries), + intakeEvents: streamedCounts.intakeEvents ?? 0, + moodEntries: streamedCounts.moodEntries ?? 0, }, }; } diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index 9873bbfbc..c2c9fab6e 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -1773,6 +1773,16 @@ export const backupPayloadSchema = z export type BackupPayload = z.infer; +/** + * One element of `measurements`, validated on its own. The streamed readers + * (`src/lib/export/streamed-backup.ts`) check a large file's measurements + * one at a time with this, and the rest of the file with + * `backupPayloadSchema` over a document whose measurements are empty, which + * together is the same check `backupPayloadSchema` makes of the whole file. + */ +export const backupMeasurementSchema = measurementSchema; +export type BackupMeasurement = z.infer; + /** * Numeric counts of each backed-up record kind. Returned in the * upload + restore API responses so the admin sees what they diff --git a/tests/integration/large-account-jobs.test.ts b/tests/integration/large-account-jobs.test.ts index 9568c0235..c897ec973 100644 --- a/tests/integration/large-account-jobs.test.ts +++ b/tests/integration/large-account-jobs.test.ts @@ -214,7 +214,7 @@ describe("storeBackupBlob (#1031)", () => { it("stores a copy that reads back as exactly the JSON produced", async () => { const prisma = getPrismaClient(); - const bytes = await storeBackupBlob( + const { bytes } = await storeBackupBlob( prisma, { userId: RUN_A, type: "WEEKLY_AUTO" }, producer(20_000), diff --git a/tests/integration/large-backup-round-trip.test.ts b/tests/integration/large-backup-round-trip.test.ts new file mode 100644 index 000000000..61301ac42 --- /dev/null +++ b/tests/integration/large-backup-round-trip.test.ts @@ -0,0 +1,203 @@ +/** + * #1031 — an account too large to export or restore. + * + * For an account of 1.25 million measurements the full export took the app + * down, and its weekly backup could not be restored, downloaded or summarised + * at all: the backup's JSON (662 MB) is longer than any string V8 can hold, + * and every reader turned the file into one string first. Measured on a + * seeded account of that size in a 1 GB container, the streamed paths now + * export in 25 s, upload in under 20 s and restore in 94 s, and the restored + * rows match the file exactly. + * + * A million rows do not fit a test run. This file runs the same paths at a + * size that spans many pages and batches, and checks the rows come back + * exactly: the export streamed into the response, uploaded as gzip through + * the raw-body path, and restored through the streamed two-pass reader; and a + * weekly backup restored the same way, every column intact, with a personal + * record still pointing at the measurement it was found in. + */ +import { gzipSync } from "node:zlib"; + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; +import { storeBackupBlob } from "@/lib/export/store-backup-blob"; +import { encryptNote } from "@/lib/crypto/note-cipher"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const COUNT = 7_300; // more than one read page and several write batches + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +async function seedAccount() { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username: "large-account", + email: "large-account@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const start = Date.UTC(2024, 0, 1); + await prisma.measurement.createMany({ + data: Array.from({ length: COUNT }, (_, i) => ({ + id: `m-${String(i).padStart(5, "0")}`, + userId: user.id, + type: i % 3 === 0 ? ("PULSE" as const) : ("SLEEP_DURATION" as const), + value: 50 + (i % 40) + 0.25, + valueMax: i % 3 === 0 ? 120 : null, + unit: i % 3 === 0 ? "bpm" : "minutes", + source: "APPLE_HEALTH" as const, + measuredAt: new Date(start + i * 300_000), + externalId: `hk-${i}`, + sleepStage: i % 3 === 0 ? null : ("CORE" as const), + deviceType: "watch", + syncVersion: 1 + (i % 4), + notesEncrypted: i === 17 ? encryptNote("after the long run") : null, + deletedAt: i % 97 === 0 ? new Date(start) : null, + })), + }); + await prisma.personalRecord.create({ + data: { + userId: user.id, + metricType: "PULSE", + direction: "MAX", + value: 89.25, + unit: "bpm", + achievedAt: new Date(start + 3 * 300_000), + sourceMeasurementId: "m-00003", + }, + }); + return user; +} + +async function measurementsOf(userId: string) { + return getPrismaClient().measurement.findMany({ + where: { userId }, + orderBy: { id: "asc" }, + omit: { createdAt: true, updatedAt: true }, + }); +} + +async function restore(id: string) { + const { POST } = await import("@/app/api/admin/backups/[id]/restore/route"); + const res = await POST( + new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }) as never, + { params: Promise.resolve({ id }) }, + ); + return res; +} + +describe("large backups round-trip through the streamed paths (#1031)", () => { + it("export → gzip upload → restore gives back every live measurement", async () => { + const user = await seedAccount(); + const live = (await measurementsOf(user.id)).filter((m) => !m.deletedAt); + + const { GET } = await import("@/app/api/export/full-backup/route"); + const exported = await GET( + new Request("http://localhost/api/export/full-backup") as never, + ); + expect(exported.status).toBe(200); + const file = Buffer.from(await exported.arrayBuffer()); + expect(JSON.parse(file.toString("utf8")).measurements).toHaveLength( + live.length, + ); + + const { POST: upload } = + await import("@/app/api/admin/backups/upload/route"); + const uploaded = await upload( + new Request("http://localhost/api/admin/backups/upload", { + method: "POST", + headers: { "content-type": "application/gzip" }, + body: gzipSync(file), + }) as never, + ); + expect(uploaded.status).toBe(201); + const { data } = (await uploaded.json()) as { + data: { id: string; summary: { measurements: number } }; + }; + expect(data.summary.measurements).toBe(live.length); + + expect((await restore(data.id)).status).toBe(200); + const restored = await measurementsOf(user.id); + // A portable file carries the fields a person reads, not the sync + // bookkeeping, so those are what must come back. + const readable = (rows: typeof restored) => + rows.map((m) => ({ + id: m.id, + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: m.measuredAt.toISOString(), + source: m.source, + })); + expect(readable(restored)).toEqual(readable(live)); + }); + + it("a weekly backup restores every column, and a record's pointer survives", async () => { + const user = await seedAccount(); + const before = await measurementsOf(user.id); + const prisma = getPrismaClient(); + + const { id } = await storeBackupBlob( + prisma, + { userId: user.id, type: "WEEKLY_AUTO" }, + (write) => + streamFullBackupJson(prisma, user.id, write, { + purpose: "disaster-recovery", + }), + ); + + // Change the account so the restore has something to undo. + await prisma.measurement.deleteMany({ where: { userId: user.id } }); + + const res = await restore(id); + expect(res.status).toBe(200); + expect(await measurementsOf(user.id)).toEqual(before); + const record = await prisma.personalRecord.findFirstOrThrow({ + where: { userId: user.id }, + }); + expect(record.sourceMeasurementId).toBe("m-00003"); + }); +}); diff --git a/tests/integration/measurement-bulk-insert.test.ts b/tests/integration/measurement-bulk-insert.test.ts new file mode 100644 index 000000000..c0caae7b1 --- /dev/null +++ b/tests/integration/measurement-bulk-insert.test.ts @@ -0,0 +1,139 @@ +/** + * `insertMeasurementRows` replaced `createMany` in the restore because + * Prisma's cached plan for a 1 000-row `createMany` kept about 145 MB alive + * (#1031). It writes raw SQL, so this pins that a row written through it + * reads back exactly as the same row written through Prisma, every column + * set, and every column null where it may be. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { + insertMeasurementRows, + type MeasurementInsertRow, +} from "@/lib/export/measurement-bulk-insert"; + +const USER = "user-bulk-insert"; + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, email: `${USER}@example.test` }, + }); +}); + +function full(id: string, at: string): MeasurementInsertRow { + return { + id, + userId: USER, + type: "SLEEP_DURATION", + value: 47.25, + valueMin: 1.5, + valueMax: 99.75, + unit: "minutes", + source: "APPLE_HEALTH", + measuredAt: new Date(at), + notes: "legacy note", + notesEncrypted: new Uint8Array([0, 1, 2, 254, 255]), + externalId: `ext-${id}`, + externalSourceVersion: "17.4", + aggregationProvenance: "HEALTHKIT_STATISTICS", + glucoseContext: null, + sleepStage: "REM", + rhythmClassification: null, + deviceType: "watch", + syncVersion: 7, + deletedAt: new Date("2026-02-03T04:05:06.789Z"), + createdAt: new Date("2025-01-02T03:04:05.678Z"), + updatedAt: new Date("2025-06-07T08:09:10.111Z"), + }; +} + +function sparse(id: string, at: string): MeasurementInsertRow { + return { + id, + userId: USER, + type: "BLOOD_GLUCOSE", + value: 5.4, + valueMin: null, + valueMax: null, + unit: "mmol/L", + source: "MANUAL", + measuredAt: new Date(at), + notes: null, + notesEncrypted: null, + externalId: null, + externalSourceVersion: null, + aggregationProvenance: null, + glucoseContext: "FASTING", + sleepStage: null, + rhythmClassification: null, + deviceType: null, + syncVersion: 1, + deletedAt: null, + }; +} + +describe("insertMeasurementRows", () => { + it("writes every column exactly as Prisma create does", async () => { + const prisma = getPrismaClient(); + const rows = [ + [ + full("a-bulk", "2026-01-01T10:00:00.123Z"), + full("a-prisma", "2026-01-02T10:00:00.123Z"), + ], + [ + sparse("b-bulk", "2026-01-03T10:00:00.000Z"), + sparse("b-prisma", "2026-01-04T10:00:00.000Z"), + ], + ] as const; + + for (const [viaBulk, viaPrisma] of rows) { + await prisma.$transaction((tx) => insertMeasurementRows(tx, [viaBulk])); + await prisma.measurement.create({ data: viaPrisma as never }); + } + + for (const [viaBulk, viaPrisma] of rows) { + const a = await prisma.measurement.findUniqueOrThrow({ + where: { id: viaBulk.id }, + }); + const b = await prisma.measurement.findUniqueOrThrow({ + where: { id: viaPrisma.id }, + }); + const strip = (row: typeof a) => ({ + ...row, + id: undefined, + externalId: row.externalId?.replace(/-(bulk|prisma)$/, ""), + measuredAt: undefined, + // Filled with the current time when the row carries none. + ...(viaBulk.createdAt + ? {} + : { createdAt: undefined, updatedAt: undefined }), + }); + expect(strip(a)).toEqual(strip(b)); + expect(a.measuredAt.toISOString()).toBe(viaBulk.measuredAt.toISOString()); + } + const sparseRow = await prisma.measurement.findUniqueOrThrow({ + where: { id: "b-bulk" }, + }); + expect(Date.now() - sparseRow.createdAt.getTime()).toBeLessThan(60_000); + }); + + it("inserts a whole batch in one statement and reports the count", async () => { + const prisma = getPrismaClient(); + const batch = Array.from({ length: 2_500 }, (_, i) => + sparse( + `m-${i}`, + new Date(Date.UTC(2026, 0, 1) + i * 60_000).toISOString(), + ), + ); + const inserted = await prisma.$transaction((tx) => + insertMeasurementRows(tx, batch), + ); + expect(inserted).toBe(2_500); + expect(await prisma.measurement.count({ where: { userId: USER } })).toBe( + 2_500, + ); + }); +}); From 77bfa2f4e1ca2b9f425af7cae7b25826afbc3fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 01:44:39 +0200 Subject: [PATCH 07/12] perf(records): find the best cumulative day a day at a time The personal-record pass, every half hour for every account, read every live row of each cumulative type into memory to find the best day. With 784 000 active-energy rows on one account (the per-sample rows pile up when the nightly drain cannot finish, #1031) it ran a 524 MB heap out of memory; the old code needed 1.1 GB of heap for that account. It now reads the type in bounded keyset pages and handles one local day at a time. The canonical-source pick it applies is per day, so the day totals and the winner are the same; a tie still keeps the earliest day. Measured on the same account: 189 MB peak heap, the same record. The warm-up gate counts only up to its threshold instead of counting a whole history. `iterateSourcePages` and `iterateDayBuckets` in the consolidation base take the row type as a parameter, and the day grouping can keep every row. --- src/lib/measurements/consolidation-base.ts | 33 ++++-- .../personal-records/pr-detection-worker.ts | 108 ++++++++++------- .../pr-detection-cumulative-paged.test.ts | 111 ++++++++++++++++++ 3 files changed, 197 insertions(+), 55 deletions(-) create mode 100644 tests/integration/pr-detection-cumulative-paged.test.ts diff --git a/src/lib/measurements/consolidation-base.ts b/src/lib/measurements/consolidation-base.ts index 01abebbbe..382f2335e 100644 --- a/src/lib/measurements/consolidation-base.ts +++ b/src/lib/measurements/consolidation-base.ts @@ -315,12 +315,14 @@ const CONSOLIDATION_SCAN_PAGE_SIZE = 5000; * The caller's `baseWhere` is AND-combined, so the scope / soft-delete / * grace filters always still apply. */ -export async function* iterateSourcePages( +export async function* iterateSourcePages< + Row extends { id: string; measuredAt: Date } = PerSampleRow, +>( prismaClient: PrismaClient, baseWhere: Prisma.MeasurementWhereInput, scanSelect: Prisma.MeasurementSelect, pageSize: number = CONSOLIDATION_SCAN_PAGE_SIZE, -): AsyncGenerator { +): AsyncGenerator { let cursor: { measuredAt: Date; id: string } | null = null; for (;;) { @@ -344,7 +346,7 @@ export async function* iterateSourcePages( select: scanSelect, orderBy: [{ measuredAt: "asc" }, { id: "asc" }], take: pageSize, - })) as PerSampleRow[]; + })) as unknown as Row[]; if (page.length === 0) return; @@ -364,22 +366,29 @@ export async function* iterateSourcePages( * * The rows arrive in `measuredAt` order and a local calendar day is one * contiguous span of instants, so a day is complete the moment a row from a - * later day shows up. Rows already in the daily-stats shape are skipped, the - * same as `bucketRowsByDay`. `onRow` sees every scanned row, stats rows - * included, for the scan-count log line. + * later day shows up. Rows whose externalId starts with `statsPrefix` (the + * daily-stats shape) are skipped, the same as `bucketRowsByDay`; pass `null` + * to keep every row. `onRow` sees every scanned row, skipped ones included, + * for the scan-count log line. */ -export async function* iterateDayBuckets( - pages: AsyncIterable, +export async function* iterateDayBuckets< + Row extends { measuredAt: Date; externalId: string | null } = PerSampleRow, +>( + pages: AsyncIterable, tz: string, - statsPrefix: string, + statsPrefix: string | null, onRow?: () => void, -): AsyncGenerator<[dateKey: string, rows: PerSampleRow[]], void, void> { +): AsyncGenerator<[dateKey: string, rows: Row[]], void, void> { let currentKey: string | null = null; - let current: PerSampleRow[] = []; + let current: Row[] = []; for await (const page of pages) { for (const row of page) { onRow?.(); - if (row.externalId !== null && row.externalId.startsWith(statsPrefix)) { + if ( + statsPrefix !== null && + row.externalId !== null && + row.externalId.startsWith(statsPrefix) + ) { continue; } const key = dayKeyForUserTz(row.measuredAt, tz); diff --git a/src/lib/personal-records/pr-detection-worker.ts b/src/lib/personal-records/pr-detection-worker.ts index 107fcc704..77739c6a5 100644 --- a/src/lib/personal-records/pr-detection-worker.ts +++ b/src/lib/personal-records/pr-detection-worker.ts @@ -40,6 +40,10 @@ import { prisma as defaultPrisma } from "@/lib/db"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; import { dayKeyForUserTz } from "@/lib/measurements/consolidation-tz"; +import { + iterateDayBuckets, + iterateSourcePages, +} from "@/lib/measurements/consolidation-base"; import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum"; import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority"; import { measurementTypeEnum } from "@/lib/validations/measurement"; @@ -273,8 +277,12 @@ export async function detectPersonalRecordsForUser( // Warm-up gate. Cheap count first so we don't pull a 30k-row // scan on a brand-new account. + // Bounded: the gate only asks whether there are enough, so it stops + // counting at the threshold instead of counting a whole history (766 000 + // heart-rate rows on a large account, every half hour). const sampleCount = await prisma.measurement.count({ where: { userId, type: metricType, deletedAt: null }, + take: PR_DETECTION_WARMUP_THRESHOLD, }); if (sampleCount < PR_DETECTION_WARMUP_THRESHOLD) continue; @@ -542,18 +550,28 @@ async function findBestMeasurement( * cumulative-day surface. * * Order of operations: - * 1. Read every row for the type (already scoped by the caller's - * warm-up gate to a metric with at least - * `PR_DETECTION_WARMUP_THRESHOLD` samples). - * 2. Collapse to the canonical source per local day. - * 3. Bucket-and-sum the canonical rows per local day; pick the - * winning day by `direction`. + * 1. Read the type's rows one local day at a time, in time order + * (already scoped by the caller's warm-up gate to a metric with at + * least `PR_DETECTION_WARMUP_THRESHOLD` samples). + * 2. Collapse each day to its canonical source. + * 3. Sum the day's canonical rows; keep the winning day by `direction`. * 4. Re-read the winning day's latest slice so the PR row carries a * real row's unit/source/externalId (the day-sum overrides * `value` below). * The unique-index contract is unaffected — a re-run picks the same * day and the same latest-slice timestamp, so `achievedAt` is stable. */ +/** One row of the cumulative-day scan. */ +interface CumulativeRow { + id: string; + value: number; + unit: string; + measuredAt: Date; + source: MeasurementSource; + externalId: string | null; + deviceType: string | null; +} + async function findBestCumulativeDay( prisma: PrismaClient, userId: string, @@ -562,46 +580,50 @@ async function findBestCumulativeDay( tz: string, priorityJson: unknown, ): Promise { - const rows = await prisma.measurement.findMany({ - where: { userId, type, deletedAt: null }, - orderBy: { measuredAt: "asc" }, - select: { - id: true, - value: true, - unit: true, - measuredAt: true, - source: true, - externalId: true, - deviceType: true, - }, - }); - if (rows.length === 0) return null; - + // Read a day at a time, in time order, never the whole history: the + // canonical-source pick is per day, so running it on each day alone gives + // the same rows it gave on the whole list. Loading every row at once was + // unbounded; with 784 000 rows of one cumulative type the pass ran a + // 524 MB heap out of memory (#1031). const metricKey = metricKeyForType(type); - const canonicalRows = metricKey - ? pickCanonicalSourceRows( - rows.map((r) => ({ ...r, type })), - metricKey, - priorityJson, - (d) => dayKeyForUserTz(d, tz), - ).canonicalRows - : rows; - - const byDay = new Map(); - for (const row of canonicalRows) { - const key = dayKeyForUserTz(row.measuredAt, tz); - const slot = byDay.get(key); - if (!slot) { - byDay.set(key, { total: row.value, latest: row.measuredAt }); - } else { - slot.total += row.value; - if (row.measuredAt > slot.latest) slot.latest = row.measuredAt; - } - } - let winnerTotal: number | null = null; let winnerLatest: Date | null = null; - for (const { total, latest } of byDay.values()) { + const days = iterateDayBuckets( + iterateSourcePages( + prisma, + { userId, type, deletedAt: null }, + { + id: true, + value: true, + unit: true, + measuredAt: true, + source: true, + externalId: true, + deviceType: true, + }, + ), + tz, + null, + ); + for await (const [, dayRows] of days) { + const canonicalRows = metricKey + ? pickCanonicalSourceRows( + dayRows.map((r) => ({ ...r, type })), + metricKey, + priorityJson, + (d) => dayKeyForUserTz(d, tz), + ).canonicalRows + : dayRows; + if (canonicalRows.length === 0) continue; + + let total = 0; + let latest = canonicalRows[0]!.measuredAt; + for (const row of canonicalRows) { + total += row.value; + if (row.measuredAt > latest) latest = row.measuredAt; + } + // Days arrive in order, and only a strictly better day replaces the + // winner, so a tie keeps the earliest day, as before. const better = winnerTotal === null || (direction === PersonalRecordDirection.MAX diff --git a/tests/integration/pr-detection-cumulative-paged.test.ts b/tests/integration/pr-detection-cumulative-paged.test.ts new file mode 100644 index 000000000..9bfc38159 --- /dev/null +++ b/tests/integration/pr-detection-cumulative-paged.test.ts @@ -0,0 +1,111 @@ +/** + * #1031 — the best cumulative day, found a day at a time. + * + * `findBestCumulativeDay` read every row of a cumulative type into memory, + * every half hour, for every account. On an account holding 784 000 active + * energy rows it ran a 524 MB heap out of memory. It now reads a local day at + * a time in time order, which gives the same answer because the canonical- + * source pick it applies is per day. + * + * Pinned here at a size that spans several read pages: two sources report + * the same days, the source ladder must decide each day on its own, the + * best day sits in the middle of the history, and no read of the history is + * unbounded. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const { detectPersonalRecordsForUser } = + await import("@/lib/personal-records/pr-detection-worker"); + +const USER = "user-pr-cumulative-paged"; +const DAYS = 40; +const PER_DAY = 300; // 12 000 Apple rows plus a Withings total per day + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: USER, + username: USER, + email: `${USER}@example.test`, + timezone: "UTC", + }, + }); +}); + +describe("best cumulative day across many pages", () => { + it("sums each day's canonical source and finds the best day without an unbounded read", async () => { + const prisma = getPrismaClient(); + const start = Date.UTC(2025, 0, 1, 6); + const rows = []; + for (let day = 0; day < DAYS; day++) { + // Day 23 is the best Apple day; the Withings total on day 5 is larger + // still but loses to Apple on that day by the default ladder, so it + // must not win: if the days were pooled across sources it would. + const perSample = day === 23 ? 40 : 20; + for (let i = 0; i < PER_DAY; i++) { + rows.push({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: perSample, + unit: "steps", + source: "APPLE_HEALTH" as const, + measuredAt: new Date(start + day * 86_400_000 + i * 60_000), + externalId: `hk-${day}-${i}`, + }); + } + rows.push({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: day === 5 ? 50_000 : 1_000, + unit: "steps", + source: "WITHINGS" as const, + measuredAt: new Date(start + day * 86_400_000 + 12 * 3_600_000), + externalId: `w-${day}`, + }); + } + await prisma.measurement.createMany({ data: rows }); + + // Every read of the steps history, through a client that records it. + const reads: Array = []; + const recording = prisma.$extends({ + query: { + measurement: { + findMany({ args, query }) { + if (JSON.stringify(args.where ?? {}).includes("ACTIVITY_STEPS")) { + reads.push(args.take); + } + return query(args); + }, + }, + }, + }); + + await detectPersonalRecordsForUser(USER, { + silent: true, + prisma: recording as unknown as typeof prisma, + }); + + const record = await prisma.personalRecord.findFirstOrThrow({ + where: { userId: USER, metricType: "ACTIVITY_STEPS" }, + }); + expect(record.value).toBe(40 * PER_DAY); + expect(record.achievedAt.toISOString()).toBe( + new Date(start + 23 * 86_400_000 + (PER_DAY - 1) * 60_000).toISOString(), + ); + + // Several bounded pages, never one read of the whole history. + expect(reads.length).toBeGreaterThan(1); + for (const take of reads) { + expect(take).toBeDefined(); + expect(take!).toBeLessThanOrEqual(5_000); + } + }); +}); From 7071fa91af41577b27da2687dd375242f27399cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 01:46:09 +0200 Subject: [PATCH 08/12] fix(records): repair cumulative records in one transaction The repair for the inflated cumulative records deleted the suspect rows, committed, and then re-ran the detector. When the detector failed, which a timeout on a large account or a restart can cause, the records stayed deleted, and since discovery looks for the suspect rows the account was never repaired again (#1031). The delete and the re-derivation now run in one transaction, with the detector reading through it, so they commit together or not at all. The transaction gets five minutes; the detector took 19.8 s on the largest seeded account. --- .../cumulative-pr-rederivation.test.ts | 10 ++ .../cumulative-pr-rederivation.ts | 53 +++++++---- .../cumulative-pr-rederive-atomic.test.ts | 94 +++++++++++++++++++ 3 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 tests/integration/cumulative-pr-rederive-atomic.test.ts diff --git a/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts b/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts index e4172f434..bcda29b20 100644 --- a/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts +++ b/src/lib/personal-records/__tests__/cumulative-pr-rederivation.test.ts @@ -8,12 +8,20 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; const deleteMany = vi.fn(); const findMany = vi.fn(); +// The delete and the re-derivation share one transaction; the stub hands the +// callback a client that routes to the same spies. +const txClient = { + personalRecord: { + deleteMany: (a: unknown) => deleteMany(a), + }, +}; vi.mock("@/lib/db", () => ({ prisma: { personalRecord: { deleteMany: (a: unknown) => deleteMany(a), findMany: (a: unknown) => findMany(a), }, + $transaction: (fn: (tx: unknown) => Promise) => fn(txClient), }, })); @@ -68,8 +76,10 @@ describe("runCumulativePrRederivationForUser", () => { }); // Re-derivation always runs silent — this is data hygiene, not a new // achievement, so no push should fire for it. + // It reads through the transaction, so it sees the deletion. expect(detectPersonalRecordsForUser).toHaveBeenCalledWith("u1", { silent: true, + prisma: txClient, }); expect(summary).toEqual({ rowsDeleted: 1, rowsReinserted: 1 }); }); diff --git a/src/lib/personal-records/cumulative-pr-rederivation.ts b/src/lib/personal-records/cumulative-pr-rederivation.ts index a6202ac24..b93f15755 100644 --- a/src/lib/personal-records/cumulative-pr-rederivation.ts +++ b/src/lib/personal-records/cumulative-pr-rederivation.ts @@ -41,7 +41,10 @@ import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; import { detectPersonalRecordsForUser } from "@/lib/personal-records/pr-detection-worker"; -import type { MeasurementType } from "@/generated/prisma/client"; +import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; + +/** Budget for one account's delete-and-re-derive transaction. */ +const REDERIVE_TRANSACTION_TIMEOUT_MS = 5 * 60 * 1000; export const CUMULATIVE_PR_REDERIVE_QUEUE = "cumulative-pr-rederive"; @@ -90,22 +93,40 @@ export interface CumulativePrRederiveSummary { export async function runCumulativePrRederivationForUser( userId: string, ): Promise { - const { count: rowsDeleted } = await prisma.personalRecord.deleteMany({ - where: { - userId, - metricSlot: null, - metricType: { in: CUMULATIVE_TYPES_ARRAY }, - createdAt: { lt: CUMULATIVE_PR_FIX_CUTOFF }, + // One transaction: the suspect rows go and the re-derived ones arrive + // together, or neither happens. Deleting first and re-deriving after the + // commit lost the records whenever the detector failed (a timeout on a + // large account, a restart): discovery looks for the suspect rows, so once + // they were gone the account was never repaired again (#1031). The detector + // reads through the transaction, so it sees the suspect rows as deleted and + // compares against what is left. It only reads and inserts, and sends no + // notification when `silent`. + const { rowsDeleted, rowsReinserted } = await prisma.$transaction( + async (tx) => { + const { count } = await tx.personalRecord.deleteMany({ + where: { + userId, + metricSlot: null, + metricType: { in: CUMULATIVE_TYPES_ARRAY }, + createdAt: { lt: CUMULATIVE_PR_FIX_CUTOFF }, + }, + }); + if (count === 0) return { rowsDeleted: 0, rowsReinserted: 0 }; + const result = await detectPersonalRecordsForUser(userId, { + silent: true, + prisma: tx as unknown as PrismaClient, + }); + return { + rowsDeleted: count, + rowsReinserted: result.inserted + result.ties, + }; }, - }); - - let rowsReinserted = 0; - if (rowsDeleted > 0) { - const result = await detectPersonalRecordsForUser(userId, { - silent: true, - }); - rowsReinserted = result.inserted + result.ties; - } + // The detector walks every trackable type of the account; on a large + // record that is tens of seconds (19.8 s measured for 1.25 million + // measurements plus 784 000 energy rows), so the default five seconds + // would roll back every repair that matters. + { timeout: REDERIVE_TRANSACTION_TIMEOUT_MS, maxWait: 10_000 }, + ); annotate({ action: { diff --git a/tests/integration/cumulative-pr-rederive-atomic.test.ts b/tests/integration/cumulative-pr-rederive-atomic.test.ts new file mode 100644 index 000000000..77c8509a2 --- /dev/null +++ b/tests/integration/cumulative-pr-rederive-atomic.test.ts @@ -0,0 +1,94 @@ +/** + * #1031 — the cumulative-record repair must not lose records when the + * re-derivation fails. + * + * The repair deleted every suspect cumulative record, committed, and only + * then re-ran the detector. A detector that failed (a timeout on a large + * account, a crash) left the records deleted, and because discovery looks + * for the suspect rows the deletion removed, the account was never + * repaired again: its step, energy and distance records were simply gone. + * The delete and the re-derivation now commit together or not at all. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const detect = vi.hoisted(() => ({ fail: false })); +vi.mock("@/lib/personal-records/pr-detection-worker", async (importActual) => { + const actual = + await importActual< + typeof import("@/lib/personal-records/pr-detection-worker") + >(); + return { + ...actual, + detectPersonalRecordsForUser: async ( + ...args: Parameters + ) => { + if (detect.fail) throw new Error("detector ran out of time"); + return actual.detectPersonalRecordsForUser(...args); + }, + }; +}); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const { runCumulativePrRederivationForUser, CUMULATIVE_PR_FIX_CUTOFF } = + await import("@/lib/personal-records/cumulative-pr-rederivation"); + +const USER = "user-pr-rederive-atomic"; + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + detect.fail = false; + await prisma.user.create({ + data: { id: USER, username: USER, email: `${USER}@example.test` }, + }); + const before = new Date(CUMULATIVE_PR_FIX_CUTOFF.getTime() - 86_400_000); + await prisma.personalRecord.create({ + data: { + userId: USER, + metricType: "ACTIVITY_STEPS", + direction: "MAX", + value: 99_999, + unit: "steps", + achievedAt: new Date("2025-03-01T12:00:00Z"), + createdAt: before, + }, + }); + // Enough steps for the detector's warm-up gate, one clear best day. + await prisma.measurement.createMany({ + data: Array.from({ length: 10 }, (_, day) => ({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: day === 4 ? 12_000 : 5_000, + unit: "steps", + source: "APPLE_HEALTH" as const, + measuredAt: new Date(Date.UTC(2025, 3, 1 + day, 12)), + externalId: `s-${day}`, + })), + }); +}); + +describe("runCumulativePrRederivationForUser", () => { + it("keeps the suspect record when the re-derivation fails", async () => { + detect.fail = true; + await expect(runCumulativePrRederivationForUser(USER)).rejects.toThrow( + "detector ran out of time", + ); + const left = await getPrismaClient().personalRecord.findMany({ + where: { userId: USER }, + }); + expect(left.map((r) => r.value)).toEqual([99_999]); + }); + + it("replaces the suspect record with the re-derived one when it succeeds", async () => { + const summary = await runCumulativePrRederivationForUser(USER); + expect(summary.rowsDeleted).toBe(1); + const left = await getPrismaClient().personalRecord.findMany({ + where: { userId: USER }, + }); + expect(left.map((r) => r.value)).toEqual([12_000]); + }); +}); From 0a54a43799729bf57a221a636608b00ca3dd215f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 01:50:37 +0200 Subject: [PATCH 09/12] fix(crypto): walk every encrypted column in bounded pages Key rotation and the rotation status read every column that was not flagged as a blob with one `findMany` over the whole table. `Measurement.notesEncrypted` and `MoodEntry.noteEncrypted` are such columns, so rotation loaded every measurement of the instance: 1.25 million rows on one large account (#1031), and the rotation status page did the same on every view. Every column is now walked with the id-cursor pages the blob columns already used: 25 rows a page for blobs, 5 000 for everything else. The maintenance script's own per-column walks page the same way. On the seeded account with 1 460 notes to rotate, peak heap fell from 272 MB to 153 MB, and it no longer grows with the table. `scanned` now counts the rows that hold ciphertext for every column, as it already did for the blob columns, rather than every row of the table for some columns and only the ciphertext rows for others. The runbook says so. --- docs/ops/encryption-key-rotation.md | 6 +- scripts/rotate-encryption-key.ts | 85 ++++++++---- .../__tests__/encryption-corpus.test.ts | 41 ++++++ src/lib/crypto/encryption-corpus.ts | 126 +++++++----------- 4 files changed, 154 insertions(+), 104 deletions(-) diff --git a/docs/ops/encryption-key-rotation.md b/docs/ops/encryption-key-rotation.md index 007dd4fb9..5a62c55e0 100644 --- a/docs/ops/encryption-key-rotation.md +++ b/docs/ops/encryption-key-rotation.md @@ -60,8 +60,10 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`). Read three things off the output before going further: - - the per-column line, `scanned` / `rotated` / `errors` / `dropped`. Treat - `errors > 0` as a hard failure and re-run after fixing the cause. A + - the per-column line, `scanned` / `rotated` / `errors` / `dropped`. + `scanned` counts the rows that hold ciphertext in that column, not every + row of the table. Treat `errors > 0` as a hard failure and re-run after + fixing the cause. A `dropped` count is only ever a cache row the run could not read and deleted rather than leave unreadable. - `Columns walked: N/M registered`. The two numbers must match. diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index 1546d0e90..616af8582 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -108,6 +108,49 @@ function shouldRotate(value: string | null): boolean { return id !== getActiveKeyId(); } +/** The part of a Prisma delegate the per-column walks below need. */ +interface PagedDelegate { + findMany: (args: { + select: Record; + orderBy: { id: "asc" }; + take: number; + cursor?: { id: string }; + skip?: number; + }) => Promise>>; + update: (args: { + where: { id: string }; + data: Record; + }) => Promise; +} + +/** Rows per page when walking a column. */ +const PAGE_SIZE = 5_000; + +/** + * Every row of one column, `id` and value only, a page at a time in id order. + * The walks used to read a column in one `findMany`, which for + * `Measurement.notesEncrypted` is every measurement of the instance: 1.25 + * million rows on one large account (#1031). + */ +async function* pagedRows( + delegate: PagedDelegate, + field: string, +): AsyncGenerator> { + let cursor: string | null = null; + for (;;) { + const rows = await delegate.findMany({ + select: { id: true, [field]: true }, + orderBy: { id: "asc" }, + take: PAGE_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }); + if (rows.length === 0) return; + yield* rows; + if (rows.length < PAGE_SIZE) return; + cursor = rows[rows.length - 1]!.id as string; + } +} + /** * Rotate one `String` ciphertext column on a Prisma model. `delegate` is the * `prisma.` accessor; `field` is the column. Reads only `id` + the @@ -116,31 +159,28 @@ function shouldRotate(value: string | null): boolean { async function rotateStringColumn( table: string, field: string, - delegate: { - findMany: (args: { - select: Record; - }) => Promise>>; - update: (args: { - where: { id: string }; - data: Record; - }) => Promise; - }, + delegate: PagedDelegate, ): Promise { - const rows = await delegate.findMany({ select: { id: true, [field]: true } }); const result: RotationResult = { table, field, - scanned: rows.length, + scanned: 0, rotated: 0, errors: 0, dropped: 0, }; - for (const row of rows) { + for await (const row of pagedRows( + delegate as PagedDelegate, + field, + )) { const v = row[field] as string | null; + // `scanned` counts rows that hold ciphertext, as the in-app rotation does. + if (!v) continue; + result.scanned++; if (!shouldRotate(v)) continue; const id = row.id as string; try { - const re = encrypt(decrypt(v as string)); + const re = encrypt(decrypt(v)); await delegate.update({ where: { id }, data: { [field]: re } }); result.rotated++; } catch (err) { @@ -159,28 +199,23 @@ async function rotateStringColumn( async function rotateBytesColumn( table: string, field: string, - delegate: { - findMany: (args: { - select: Record; - }) => Promise>>; - update: (args: { - where: { id: string }; - data: Record; - }) => Promise; - }, + delegate: PagedDelegate, ): Promise { - const rows = await delegate.findMany({ select: { id: true, [field]: true } }); const result: RotationResult = { table, field, - scanned: rows.length, + scanned: 0, rotated: 0, errors: 0, dropped: 0, }; - for (const row of rows) { + for await (const row of pagedRows( + delegate as PagedDelegate, + field, + )) { const buf = row[field] as Uint8Array | null; if (!buf || buf.byteLength === 0) continue; + result.scanned++; const asString = Buffer.from(buf).toString("utf8"); if (!shouldRotate(asString)) continue; const id = row.id as string; diff --git a/src/lib/crypto/__tests__/encryption-corpus.test.ts b/src/lib/crypto/__tests__/encryption-corpus.test.ts index 39d9e6551..db764a92e 100644 --- a/src/lib/crypto/__tests__/encryption-corpus.test.ts +++ b/src/lib/crypto/__tests__/encryption-corpus.test.ts @@ -214,4 +214,45 @@ describe("encryption-corpus scan + rotate", () => { expect(col.legacy).toBe(1); expect(col.byKeyId[LEGACY_BUCKET]).toBe(1); }); + + it("reads every column in bounded pages and counts only rows that hold ciphertext (#1031)", async () => { + // A notes column is read for every row of its table, 1.25 million on a + // large account, and was read in one findMany. 12 000 measurements, one + // in ten with a note written under the old key. + const measurements: Row[] = Array.from({ length: 12_000 }, (_, i) => ({ + id: `m-${String(i).padStart(5, "0")}`, + notesEncrypted: + i % 10 === 0 + ? new Uint8Array(Buffer.from(activeUnder("v1", `note ${i}`), "utf8")) + : null, + })); + const { client } = makeClient({ Measurement: measurements }); + const takes: Array = []; + const measurement = client.measurement!; + const findMany = measurement.findMany; + measurement.findMany = (args) => { + takes.push(args.take); + return findMany(args); + }; + + const scan = await scanCorpus(client); + const col = scan.columns.find( + (c) => c.model === "Measurement" && c.field === "notesEncrypted", + )!; + expect(col.total).toBe(1_200); + expect(col.byKeyId.v1).toBe(1_200); + + const rotation = await rotateCorpus(client); + const result = rotation.results.find( + (r) => r.model === "Measurement" && r.field === "notesEncrypted", + )!; + expect(result.scanned).toBe(1_200); + expect(result.rotated).toBe(1_200); + + expect(takes.length).toBeGreaterThan(2); + for (const take of takes) { + expect(take).toBeDefined(); + expect(take!).toBeLessThanOrEqual(5_000); + } + }); }); diff --git a/src/lib/crypto/encryption-corpus.ts b/src/lib/crypto/encryption-corpus.ts index 31816b691..4efc0d46d 100644 --- a/src/lib/crypto/encryption-corpus.ts +++ b/src/lib/crypto/encryption-corpus.ts @@ -22,7 +22,7 @@ * `ENCRYPTION_KEYS` / `ENCRYPTION_ACTIVE_KEY_ID`. The operator's env key map * is the only place keys live; a key drop stays a deliberate env + redeploy * act, never a button. - * - IDEMPOTENT. `shouldRotate()` skips rows already on the active key, so a + * - IDEMPOTENT. The walk skips rows already on the active key, so a * second pass (or two racing workers) re-encrypts zero rows. * - FAIL-CLOSED. A row written under a key id that is no longer configured * throws on decrypt (counted as an error, the row is left untouched) rather @@ -54,6 +54,15 @@ export const LEGACY_BUCKET = "legacy"; */ export const BLOB_ROTATION_BATCH_SIZE = 25; +/** + * Batch size for every other column. These rows are short strings, but a + * column can have one per measurement: `Measurement.notesEncrypted` is read + * for every row of the table, 1.25 million on a large account (#1031). The + * walk used to read such a column in one `findMany`; it is id-cursor paged + * like the blob columns, only with larger pages. + */ +export const ROW_ROTATION_BATCH_SIZE = 5_000; + /** Minimal Prisma delegate shape this module needs. */ interface ColumnDelegate { findMany: (args: { @@ -114,10 +123,6 @@ function fromCiphertext( return next; } -function shouldRotate(ciphertext: string): boolean { - return extractKeyId(ciphertext) !== getActiveKeyId(); -} - // ─── Codec-dispatched blob columns (document vault) ───────────────────────── /** @@ -152,15 +157,16 @@ function reencryptBlob(value: Uint8Array, codec: string): Uint8Array { } /** - * Walk a blob column in bounded id-cursor batches, invoking `onRow` per - * non-empty row. At most `BLOB_ROTATION_BATCH_SIZE` rows are in memory per - * step; an interrupted run resumes safely on re-invocation because processing - * is idempotent (already-active rows are skipped by the callers). + * Walk a column in bounded id-cursor batches, invoking `onRow` per non-empty + * row. At most one batch is in memory per step (`BLOB_ROTATION_BATCH_SIZE` + * rows of a blob column, `ROW_ROTATION_BATCH_SIZE` of any other); an + * interrupted run resumes safely on re-invocation because processing is + * idempotent (already-active rows are skipped by the callers). * * `codec` is the row's own codec label for a codec-dispatched column and null * for a plain `batched` one — a backup blob has one layout, just a large one. */ -async function walkBlobColumn( +async function walkColumn( delegate: ColumnDelegate, col: EncryptedColumn, onRow: (row: { @@ -171,6 +177,9 @@ async function walkBlobColumn( ): Promise { const codecField = col.codecField; const pk = pkField(col); + const batchSize = isBlobColumn(col) + ? BLOB_ROTATION_BATCH_SIZE + : ROW_ROTATION_BATCH_SIZE; let cursor: string | null = null; for (;;) { const rows = await delegate.findMany({ @@ -180,7 +189,7 @@ async function walkBlobColumn( ...(codecField ? { [codecField]: true } : {}), }, orderBy: { [pk]: "asc" }, - take: BLOB_ROTATION_BATCH_SIZE, + take: batchSize, ...(cursor ? { cursor: { [pk]: cursor }, skip: 1 } : {}), }); if (rows.length === 0) break; @@ -196,7 +205,7 @@ async function walkBlobColumn( }); } cursor = rows[rows.length - 1]![pk] as string; - if (rows.length < BLOB_ROTATION_BATCH_SIZE) break; + if (rows.length < batchSize) break; } } @@ -205,8 +214,8 @@ function pkField(col: EncryptedColumn): string { return col.pkField ?? "id"; } -/** True when the column's rows must be walked in bounded batches. */ -function isBatched(col: EncryptedColumn): boolean { +/** True when the column's rows are blobs, walked in the small batches. */ +function isBlobColumn(col: EncryptedColumn): boolean { return Boolean(col.codecField ?? col.batched); } @@ -242,25 +251,12 @@ export async function scanColumn( const byKeyId: Record = {}; let total = 0; - if (isBatched(col)) { - // Blob column: bounded batches, per-row codec where the column has one. - await walkBlobColumn(delegate, col, ({ value, codec }) => { - total += 1; - const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET; - byKeyId[id] = (byKeyId[id] ?? 0) + 1; - }); - } else { - const rows = await delegate.findMany({ - select: { [pkField(col)]: true, [col.field]: true }, - }); - for (const row of rows) { - const ciphertext = toCiphertext(row[col.field], col.kind); - if (ciphertext == null) continue; - total += 1; - const id = extractKeyId(ciphertext) ?? LEGACY_BUCKET; - byKeyId[id] = (byKeyId[id] ?? 0) + 1; - } - } + // Bounded batches for every column, per-row codec where the column has one. + await walkColumn(delegate, col, ({ value, codec }) => { + total += 1; + const id = walkedKeyId(value, codec, col.kind) ?? LEGACY_BUCKET; + byKeyId[id] = (byKeyId[id] ?? 0) + 1; + }); return { model: col.model, field: col.field, @@ -356,57 +352,33 @@ export async function rotateColumn( } }; - if (isBatched(col)) { - // Blob column: bounded id-cursor batches (never an unbounded findMany), - // re-encrypted under each row's OWN codec where it has one. Idempotent — - // rows already on the active key are skipped, so an interrupted run - // resumes cleanly on the next invocation. - await walkBlobColumn(delegate, col, async ({ id, value, codec }) => { - result.scanned += 1; - if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return; - try { - // The plaintext is never inspected, only re-sealed, so every envelope - // a stored value can legitimately carry survives rotation untouched. - const next = - codec !== null - ? reencryptBlob(value as Uint8Array, codec) - : fromCiphertext( - encrypt(decrypt(toCiphertext(value, col.kind)!)), - col.kind, - ); - await delegate.update({ - where: { [pkField(col)]: id }, - data: { [col.field]: next }, - }); - result.rotated += 1; - } catch { - await onUnreadable(id); - } - }); - return result; - } - - const pk = pkField(col); - const rows = await delegate.findMany({ - select: { [pk]: true, [col.field]: true }, - }); - result.scanned = rows.length; - for (const row of rows) { - const ciphertext = toCiphertext(row[col.field], col.kind); - if (ciphertext == null || !shouldRotate(ciphertext)) continue; - const id = row[pk] as string; + // Bounded id-cursor batches for every column (never an unbounded + // findMany), re-encrypted under each row's OWN codec where it has one. + // Idempotent — rows already on the active key are skipped, so an + // interrupted run resumes cleanly on the next invocation. `scanned` counts + // the rows that hold ciphertext, whatever the column. + await walkColumn(delegate, col, async ({ id, value, codec }) => { + result.scanned += 1; + if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return; try { - // ACTIVE-KEY-ONLY: encrypt() always writes the active key id. - const reencrypted = encrypt(decrypt(ciphertext)); + // The plaintext is never inspected, only re-sealed, so every envelope + // a stored value can legitimately carry survives rotation untouched. + const next = + codec !== null + ? reencryptBlob(value as Uint8Array, codec) + : fromCiphertext( + encrypt(decrypt(toCiphertext(value, col.kind)!)), + col.kind, + ); await delegate.update({ - where: { [pk]: id }, - data: { [col.field]: fromCiphertext(reencrypted, col.kind) }, + where: { [pkField(col)]: id }, + data: { [col.field]: next }, }); result.rotated += 1; } catch { await onUnreadable(id); } - } + }); return result; } From 4242b454ce460bb0a8856605b71917a918343553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 01:54:27 +0200 Subject: [PATCH 10/12] fix(import): fold rollups over the span the import wrote After an Apple Health import the worker folded the rollup table from the account's first measurement to its last, whatever the import carried, and so also folded history older than the five-year window that the boot backfill, its discovery and every read path keep to. The import now records the oldest and newest instant it wrote (`measuredSpan` on the job result), and the fold covers that span, clamped to the fold window. On the seeded account of 1.25 million rows the full fold took 12 s, the slowest statement 2.7 s against the 60 s statement limit, so this is about the window rather than time. --- src/lib/jobs/apple-health-import-worker.ts | 35 +++--- .../import-apple-health-export.ts | 22 ++++ .../apple-health-import-rollup-window.test.ts | 116 ++++++++++++++++++ 3 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 tests/integration/apple-health-import-rollup-window.test.ts diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index c610a4825..257649e72 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -41,7 +41,10 @@ import { type ImportJobProgress, type ImportJobResult, } from "@/lib/measurements/import-apple-health-export"; -import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { + recomputeUserRollups, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; import { withBackgroundEvent } from "@/lib/logging/background"; /** @@ -502,19 +505,23 @@ export async function handleAppleHealthImport( // contradict the row the status poll surfaces to the operator. let rollupFailed = false; try { - const span = await prisma.measurement.aggregate({ - where: { userId }, - _min: { measuredAt: true }, - _max: { measuredAt: true }, - }); - if (span._min.measuredAt && span._max.measuredAt) { - // Add a small tail buffer to `to` so the upper bound is - // exclusive-safe under the rollup aggregator's `< to` filter. - const to = new Date(span._max.measuredAt.getTime() + 1); - await recomputeUserRollups(userId, { - from: span._min.measuredAt, - to, - }); + // Folded over what this import wrote, not over the account's whole + // history, and never further back than the fold window every other + // path keeps to: buckets older than `ROLLUP_FOLD_WINDOW_MS` are by + // design never written, so reads there fall back to live SQL. The old + // span ran from the account's first measurement, whatever the import + // carried, and folded any history older than the window as well. + const span = result.measuredSpan; + if (span) { + const windowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const spanFrom = new Date(span.from); + const from = spanFrom > windowStart ? spanFrom : windowStart; + // `to` is exclusive in the aggregator, so one millisecond past the + // newest row. + const to = new Date(new Date(span.to).getTime() + 1); + if (to > from) { + await recomputeUserRollups(userId, { from, to }); + } } } catch (rollupErr) { // Rollup failure is non-fatal — the next read falls through to diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index 9bc4fe8cf..d175fc5c0 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -125,6 +125,11 @@ export interface ImportJobResult { /** Estimated `(measurement type, local day)` aggregate rows considered. */ rows: number; }; + /** + * Oldest and newest instant this import wrote a measurement at (ISO), or + * null when it wrote none. What the rollup fold after the import covers. + */ + measuredSpan: { from: string; to: string } | null; /** Bounded auxiliary ECG outcomes. Never contains filenames or waveform data. */ ecg: { discovered: number; @@ -396,6 +401,14 @@ export async function streamParseExportXml( let rowsUpserted = 0; const cumulativeEstimatedDays = new Set(); let cumulativeEstimatedRows = 0; + // The instants this import wrote measurements at, oldest and newest. The + // worker folds the rollup table over exactly this span afterwards. + let spanFrom: Date | null = null; + let spanTo: Date | null = null; + const widenSpan = (at: Date) => { + if (spanFrom === null || at < spanFrom) spanFrom = at; + if (spanTo === null || at > spanTo) spanTo = at; + }; // Per R-1 §8 the percent stays best-effort and may remain null // until the parser sees the closing `` tag. We don't // currently mutate this in v1.4.34 — the iOS app keeps polling on @@ -438,6 +451,7 @@ export async function streamParseExportXml( const flushSpotBatch = async (): Promise => { if (spotBatch.length === 0) return; const chunk = spotBatch.splice(0, spotBatch.length); + for (const row of chunk) widenSpan(row.measuredAt); const insertedArrivals: Array<{ id: string; type: MeasurementType; @@ -731,6 +745,7 @@ export async function streamParseExportXml( } const externalId = dailyStatsExternalId(mapping.hkIdentifier, dayKey); const measuredAt = canonicalDailyTimestamp(dayKey, userTimezone); + widenSpan(measuredAt); const rowStart = Date.now(); const verdict = await prisma.$transaction((tx) => reconcileExternalMeasurement( @@ -1202,6 +1217,13 @@ export async function streamParseExportXml( days: cumulativeEstimatedDays.size, rows: cumulativeEstimatedRows, }, + measuredSpan: + spanFrom !== null && spanTo !== null + ? { + from: (spanFrom as Date).toISOString(), + to: (spanTo as Date).toISOString(), + } + : null, ecg: { discovered: 0, imported: 0, diff --git a/tests/integration/apple-health-import-rollup-window.test.ts b/tests/integration/apple-health-import-rollup-window.test.ts new file mode 100644 index 000000000..be558605e --- /dev/null +++ b/tests/integration/apple-health-import-rollup-window.test.ts @@ -0,0 +1,116 @@ +/** + * #1031 — the rollup fold after an import covers what the import wrote. + * + * The worker folded the rollup table from the account's first measurement to + * its last, whatever the import carried, and so also folded history older + * than the five-year window every other fold path keeps to. It now folds the + * span of the rows this import wrote, clamped to that window. + */ +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { strToU8, zipSync } from "fflate"; +import type { Job } from "pg-boss"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { + APPLE_HEALTH_IMPORT_PARSER_REVISION, + handleAppleHealthImport, + _setWorkerPrismaForTests, + type AppleHealthImportPayload, +} from "@/lib/jobs/apple-health-import-worker"; +import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); + +const USER = "user-import-rollup-window"; + +function record(at: Date, value: number): string { + const stamp = at.toISOString().replace("T", " ").slice(0, 19) + " +0000"; + return ``; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + _setWorkerPrismaForTests(getPrismaClient()); + await getPrismaClient().user.create({ + data: { + id: USER, + username: USER, + email: `${USER}@example.test`, + timezone: "UTC", + }, + }); +}); + +describe("import rollup fold window", () => { + it("folds the imported span, never past the fold window, and leaves older history alone", async () => { + const prisma = getPrismaClient(); + const now = Date.now(); + const day = 86_400_000; + // Already in the account before the import: a reading eight years old. + await prisma.measurement.create({ + data: { + userId: USER, + type: "WEIGHT", + value: 90, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(now - 8 * 365 * day), + }, + }); + // The import: one reading seven years old, a few recent ones. + const xml = ` + + ${record(new Date(now - 7 * 365 * day), 88)} + ${record(new Date(now - 40 * day), 84)} + ${record(new Date(now - 30 * day), 83)} + ${record(new Date(now - 20 * day), 82)} +`; + const dir = mkdtempSync(join(tmpdir(), "hl-import-window-")); + const path = join(dir, "export.zip"); + const bytes = Buffer.from( + zipSync({ "apple_health_export/export.xml": strToU8(xml) }), + ); + writeFileSync(path, bytes); + await prisma.importJob.create({ + data: { + userId: USER, + pgBossJobId: "window-job", + status: "queued", + uploadBytes: bytes.length, + uploadSha256: createHash("sha256").update(bytes).digest("hex"), + parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, + }, + }); + + const outcome = await handleAppleHealthImport({ + id: "window-job", + data: { + userId: USER, + uploadPath: path, + uploadBytes: bytes.length, + enqueuedAt: new Date().toISOString(), + }, + } as unknown as Job); + expect(outcome.ok).toBe(true); + + const buckets = await prisma.measurementRollup.findMany({ + where: { userId: USER, type: "WEIGHT", granularity: "DAY" }, + orderBy: { bucketStart: "asc" }, + }); + const windowStart = now - ROLLUP_FOLD_WINDOW_MS; + expect(buckets.length).toBeGreaterThanOrEqual(3); + for (const bucket of buckets) { + expect(bucket.bucketStart.getTime()).toBeGreaterThan(windowStart - day); + } + }); +}); From b7a26d3a4d105f5f5c1613c9b35a3e71602be565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 01:58:06 +0200 Subject: [PATCH 11/12] fix(backup): resume an off-host run where its last attempt stopped The off-host backup walked every account in one job with the default fifteen-minute expiry, and a retry started again from the first account. On an instance with one large record next to others, an attempt that ran out of time spent its retry uploading the accounts it had already done (#1031). The run now stops between accounts once the job's time budget is spent and fails, so pg-boss retries it. Every attempt of a job carries the same creation time; an attempt skips the accounts whose ledger shows a success since then, and keys its objects on that day, so a retry after midnight still writes under the run's date. A new job uploads every account again, and a retry whose accounts were all done passes. --- src/lib/jobs/job-outcome.ts | 1 + src/lib/jobs/offhost-backup.ts | 54 +++++++++++++++++-- .../__tests__/offhost-backup-handler.test.ts | 23 ++++++++ src/lib/jobs/reminder/backup-handlers.ts | 31 +++++++++-- src/lib/jobs/reminder/register-maintenance.ts | 3 +- .../integration/offhost-backup-ledger.test.ts | 46 ++++++++++++++++ 6 files changed, 149 insertions(+), 9 deletions(-) diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index f57210e09..2f25991f4 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -138,6 +138,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "notifications_dispatched", "notifications_failed", "notified", + "offhost_backup_already_uploaded", "offhost_backup_configured", "offhost_backup_failed", "offhost_backup_oversized", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 19bb54f84..1805448bf 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -593,9 +593,33 @@ interface BackupRunReport { largestObjectBytes: number; /** Accounts refused for size rather than failed for a reason. */ oversized: number; + /** + * Accounts an earlier attempt of the same run already uploaded, and this + * one therefore skipped (see `RunOffhostBackupOptions.runStartedAt`). + */ + alreadyUploaded: number; + /** `shouldStop` ended the run before every account was reached. */ + stoppedEarly: boolean; } -export type RunOffhostBackupOptions = UploadBackupOptions; +export interface RunOffhostBackupOptions extends UploadBackupOptions { + /** + * When this run first started: the pg-boss job's creation time, the same + * on every attempt of the job. An attempt skips each account whose ledger + * shows a successful upload since then, because an earlier attempt of the + * same run already put its object in the bucket, and the objects are keyed + * on this date so a retry after midnight still writes under the run's day. + * Without it a retry started again from the first account (#1031), and on + * a cohort with one large record the retry never got past the accounts the + * first attempt had already done. + */ + runStartedAt?: Date; + /** + * Asked before each account. Returning `true` ends the run cleanly between + * accounts, before the job's expiry would cut one off halfway. + */ + shouldStop?: () => boolean; +} export async function runOffhostBackup( prisma: PrismaClient, @@ -610,9 +634,23 @@ export async function runOffhostBackup( ); } const s3 = s3Override ?? (await getS3Client(cfg)); - const dateKey = now.toISOString().slice(0, 10); + const runStartedAt = options.runStartedAt; + const dateKey = (runStartedAt ?? now).toISOString().slice(0, 10); - const users = await prisma.user.findMany({ select: { id: true } }); + const users = await prisma.user.findMany({ + select: { id: true }, + orderBy: { id: "asc" }, + }); + const doneThisRun = new Set(); + if (runStartedAt) { + const done = await prisma.offhostBackupState.findMany({ + where: { lastSuccessAt: { gte: runStartedAt } }, + select: { userId: true }, + }); + for (const row of done) doneThisRun.add(row.userId); + } + let alreadyUploaded = 0; + let stoppedEarly = false; let uploaded = 0; let failed = 0; let oversized = 0; @@ -621,6 +659,14 @@ export async function runOffhostBackup( let ledgerWriteFailures = 0; const evt = getEvent(); for (const user of users) { + if (doneThisRun.has(user.id)) { + alreadyUploaded++; + continue; + } + if (options.shouldStop?.()) { + stoppedEarly = true; + break; + } let objectBytes: number | null = null; try { objectBytes = await uploadEncryptedBackup( @@ -721,6 +767,8 @@ export async function runOffhostBackup( totalUsers: users.length, largestObjectBytes, oversized, + alreadyUploaded, + stoppedEarly, }; } diff --git a/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts b/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts index 41bfb935b..b1d880103 100644 --- a/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts +++ b/src/lib/jobs/reminder/__tests__/offhost-backup-handler.test.ts @@ -53,6 +53,8 @@ function report(over: Partial> = {}) { totalUsers: 2, largestObjectBytes: 9_000, oversized: 0, + alreadyUploaded: 0, + stoppedEarly: false, ...over, }; } @@ -67,6 +69,8 @@ describe("handleOffhostBackup", () => { const outcome = await handleOffhostBackup([]); expect(outcome.ok).toBe(true); expect(outcome.ok && outcome.did).toEqual({ + offhost_backup_already_uploaded: 0, + stopped_early: false, offhost_backup_uploaded: 2, offhost_backup_failed: 0, offhost_backup_total_users: 2, @@ -138,4 +142,23 @@ describe("handleOffhostBackup", () => { offhost_backup_configured: false, }); }); + + it("fails an attempt that ran out of budget, so pg-boss retries and the retry resumes (#1031)", async () => { + mocks.runOffhostBackup.mockResolvedValue( + report({ uploaded: 1, totalUsers: 3, stoppedEarly: true }), + ); + const outcome = await handleOffhostBackup([]); + expect(outcome).toMatchObject({ + ok: false, + reason: expect.stringContaining("retry resumes"), + }); + }); + + it("passes a retry whose accounts an earlier attempt had all uploaded", async () => { + mocks.runOffhostBackup.mockResolvedValue( + report({ uploaded: 0, alreadyUploaded: 2, totalUsers: 2 }), + ); + const outcome = await handleOffhostBackup([]); + expect(outcome.ok).toBe(true); + }); }); diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 41a114b1d..0a3b69ac5 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -4,7 +4,8 @@ * Extracted from reminder-worker.ts, which owns the queue names, cron * schedules, and boss.work registrations. */ -import { type Job } from "pg-boss"; +import { type Job, type JobWithMetadata } from "pg-boss"; +import { jobBudget } from "@/lib/jobs/job-budget"; import { recordError } from "@/lib/jobs/worker-status"; import { BackupBlobTooLargeError } from "@/lib/export/backup-blob"; import { storeBackupBlob } from "@/lib/export/store-backup-blob"; @@ -27,13 +28,18 @@ export interface OffhostBackupPayload { } export async function handleOffhostBackup( - jobs: Job[], + jobs: JobWithMetadata[], ): Promise { - void jobs; return withBackgroundEvent("job.offhost_backup", async (evt) => { const p = getWorkerPrisma(); try { - const report = await runOffhostBackup(p); + // A retry of this job carries the same creation time, so it resumes + // after the accounts an earlier attempt uploaded, and the run stops + // between accounts before the job's expiry would cut one off. + const report = await runOffhostBackup(p, undefined, new Date(), { + runStartedAt: jobs[0]?.createdOn, + shouldStop: jobBudget(jobs), + }); evt.addMeta("offhost_backup_uploaded", report.uploaded); evt.addMeta("offhost_backup_failed", report.failed); evt.addMeta("offhost_backup_total_users", report.totalUsers); @@ -55,6 +61,8 @@ export async function handleOffhostBackup( } const did = { + offhost_backup_already_uploaded: report.alreadyUploaded, + stopped_early: report.stoppedEarly, offhost_backup_uploaded: report.uploaded, offhost_backup_failed: report.failed, offhost_backup_total_users: report.totalUsers, @@ -69,7 +77,20 @@ export async function handleOffhostBackup( // than one. Per-account failures still ride out as counts when SOME // account got a copy: that is the fan-out rule, and retrying the whole // cohort over one object would re-upload everybody's. - if (report.totalUsers > 0 && report.uploaded === 0) { + // Out of budget before every account: fail, so pg-boss retries the job, + // and the retry resumes after the accounts this attempt uploaded. + if (report.stoppedEarly) { + return jobFailed( + "stopped before every account was uploaded; the retry resumes", + undefined, + did, + ); + } + + if ( + report.totalUsers > 0 && + report.uploaded + report.alreadyUploaded === 0 + ) { // The SDK's own words, not a stack: `runJob` puts the cause message in // the reported meta, and "SignatureDoesNotMatch" is the sentence an // operator can act on. diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 4d60ab352..30507c800 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -724,10 +724,11 @@ export async function registerMaintenanceQueues( { localConcurrency: 1 }, handleStepUpElevationCleanup, ); + // With metadata: the handler resumes a retry from the job's creation time. await createAndWork( boss, OFFHOST_BACKUP_QUEUE, - { localConcurrency: 1 }, + { localConcurrency: 1, includeMetadata: true }, handleOffhostBackup, ); await createAndWork( diff --git a/tests/integration/offhost-backup-ledger.test.ts b/tests/integration/offhost-backup-ledger.test.ts index f2047fc9a..a4718865c 100644 --- a/tests/integration/offhost-backup-ledger.test.ts +++ b/tests/integration/offhost-backup-ledger.test.ts @@ -228,4 +228,50 @@ describe("off-host backup ledger (real Postgres)", () => { await prisma.user.delete({ where: { id: "account-one" } }); expect(await prisma.offhostBackupState.count()).toBe(0); }); + + it("a retry of the same run resumes after the accounts it already uploaded (#1031)", async () => { + const prisma = getPrismaClient(); + for (const id of ["acct-a", "acct-b", "acct-c"]) await seedUser(id); + const bucket = makeBucket(); + const puts: string[] = []; + const putStream = bucket.putStream; + bucket.putStream = async (key, body) => { + puts.push(key); + return putStream(key, body); + }; + // The job was created a moment ago; its retries share that instant. + const runStartedAt = new Date(Date.now() - 60_000); + const day = runStartedAt.toISOString().slice(0, 10); + + // First attempt: out of budget after one account. + let asked = 0; + const first = await runOffhostBackup(prisma, bucket, new Date(), { + runStartedAt, + shouldStop: () => asked++ >= 1, + }); + expect(first.stoppedEarly).toBe(true); + expect(first.uploaded).toBe(1); + + // The retry: same run, so the account already done is skipped. + const retry = await runOffhostBackup(prisma, bucket, new Date(), { + runStartedAt, + }); + expect(retry.stoppedEarly).toBe(false); + expect(retry.alreadyUploaded).toBe(1); + expect(retry.uploaded).toBe(2); + + // Each account uploaded exactly once, all under the run's own day. + expect(puts.sort()).toEqual([ + `${day}/user-acct-a.json.enc`, + `${day}/user-acct-b.json.enc`, + `${day}/user-acct-c.json.enc`, + ]); + + // A new run (a later job) uploads everybody again. + const next = await runOffhostBackup(prisma, bucket, new Date(), { + runStartedAt: new Date(), + }); + expect(next.alreadyUploaded).toBe(0); + expect(next.uploaded).toBe(3); + }); }); From c757981638fcf9d547edd0efd02af9d469bfa837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 02:00:27 +0200 Subject: [PATCH 12/12] fix(jobs): give the hourly rebuild the same time budget The one-shot hourly rebuild of already-folded heart-rate, HRV and SpO2 days walks one account's whole folded history in one job with the default fifteen-minute expiry, about fifty statements per day. Like the drains it now stops between days once 75 % of the expiry has passed or pg-boss aborts the job, and fails so pg-boss retries it (the boot send allows three retries); a rebuilt day leaves the candidate set, so the retry starts at the first day not yet rebuilt. --- src/lib/jobs/dense-intraday-hourly-rebuild.ts | 6 +++ src/lib/jobs/reminder/register-rollup.ts | 25 ++++++++++-- .../dense-intraday-hourly-rebuild.test.ts | 40 +++++++++++++++++++ .../dense-intraday-hourly-rebuild.ts | 15 ++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild.ts b/src/lib/jobs/dense-intraday-hourly-rebuild.ts index 9f451d577..a90de1ea7 100644 --- a/src/lib/jobs/dense-intraday-hourly-rebuild.ts +++ b/src/lib/jobs/dense-intraday-hourly-rebuild.ts @@ -65,11 +65,14 @@ export interface DenseIntradayHourlyRebuildPayload { */ export async function runDenseIntradayHourlyRebuildForUser( userId: string, + shouldStop?: () => boolean, ): Promise<{ daysRebuilt: number; hourlyRowsUpserted: number; dailyRowsRetired: number; daysSkippedNoTombstones: number; + /** The job's budget ran out first; see the queue handler. */ + stoppedEarly: boolean; }> { // Shares the retention drain's kill-switch: when an operator disables the // dense tier, no-op so any already-queued backlog drains cleanly. @@ -79,10 +82,12 @@ export async function runDenseIntradayHourlyRebuildForUser( hourlyRowsUpserted: 0, dailyRowsRetired: 0, daysSkippedNoTombstones: 0, + stoppedEarly: false, }; } const summary = await runDenseIntradayHourlyRebuild(prisma, { userId, + shouldStop, log: () => { // Silent inside the queue handler — the worker logs the totals. }, @@ -104,6 +109,7 @@ export async function runDenseIntradayHourlyRebuildForUser( hourlyRowsUpserted: summary.totals.hourlyRowsUpserted, dailyRowsRetired: summary.totals.dailyRowsRetired, daysSkippedNoTombstones: summary.totals.daysSkippedNoTombstones, + stoppedEarly: summary.stoppedEarly, }; } diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts index 3ddd5ea92..ef38e1ddd 100644 --- a/src/lib/jobs/reminder/register-rollup.ts +++ b/src/lib/jobs/reminder/register-rollup.ts @@ -547,6 +547,8 @@ export async function registerRollupQueues( DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, async (jobs) => { + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; let daysRebuiltTotal = 0; let hourlyRowsTotal = 0; let dailyRowsRetiredTotal = 0; @@ -554,12 +556,17 @@ export async function registerRollupQueues( for (const job of jobs) { const { userId } = job.data; try { + const summary = await runDenseIntradayHourlyRebuildForUser( + userId, + shouldStop, + ); const { daysRebuilt, hourlyRowsUpserted, dailyRowsRetired, daysSkippedNoTombstones, - } = await runDenseIntradayHourlyRebuildForUser(userId); + } = summary; + stoppedEarly ||= summary.stoppedEarly; daysRebuiltTotal += daysRebuilt; hourlyRowsTotal += hourlyRowsUpserted; dailyRowsRetiredTotal += dailyRowsRetired; @@ -578,13 +585,25 @@ export async function registerRollupQueues( throw err; } } - return jobDone({ + const did = { users: jobs.length, + stopped_early: stoppedEarly, days_rebuilt: daysRebuiltTotal, hourly_rows_upserted: hourlyRowsTotal, daily_rows_retired: dailyRowsRetiredTotal, days_skipped_no_tombstones: daysSkippedTotal, - }); + }; + // Out of budget: fail so pg-boss retries (the boot send allows three + // retries), and the retry starts at the first day not yet rebuilt. The + // queue is `exclusive`, so the job cannot hand itself on while active. + if (stoppedEarly) { + return jobFailed( + "stopped before every day was rebuilt; the retry resumes", + undefined, + did, + ); + } + return jobDone(did); }, ); diff --git a/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts b/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts index c69715be9..9a149a2f4 100644 --- a/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-hourly-rebuild.test.ts @@ -463,4 +463,44 @@ describe("runDenseIntradayHourlyRebuild — rebuild flow", () => { expect(summary.totals.daysFailed).toBe(1); expect(summary.totals.daysRebuilt).toBe(1); }); + + it("stops before the next day once its budget is spent (#1031)", async () => { + const { mock, $transaction } = buildPrismaMock({ + dailyByType: { + HEART_RATE_VARIABILITY: [ + { id: "daily-1", externalId: HRV_DAILY_ID }, + { + id: "daily-2", + externalId: + "stats:HKQuantityTypeIdentifierHeartRateVariabilitySDNN:2026-05-02", + }, + ], + }, + tombstonesByType: { + HEART_RATE_VARIABILITY: [ + tombstone( + "a", + 40, + "2026-05-01T08:00:00.000Z", + "HEART_RATE_VARIABILITY", + ), + tombstone( + "b", + 44, + "2026-05-02T08:00:00.000Z", + "HEART_RATE_VARIABILITY", + ), + ], + }, + }); + let asked = 0; + const summary = await runDenseIntradayHourlyRebuild(mock, { + log: () => {}, + shouldStop: () => asked++ >= 1, + }); + + expect(summary.stoppedEarly).toBe(true); + expect(summary.totals.daysRebuilt).toBe(1); + expect($transaction).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/lib/measurements/dense-intraday-hourly-rebuild.ts b/src/lib/measurements/dense-intraday-hourly-rebuild.ts index 15e116724..a5cbb7d78 100644 --- a/src/lib/measurements/dense-intraday-hourly-rebuild.ts +++ b/src/lib/measurements/dense-intraday-hourly-rebuild.ts @@ -111,6 +111,8 @@ export interface DenseIntradayHourlyRebuildSummary { /** Days whose rebuild threw and was stepped over (retried next run). */ daysFailed: number; }; + /** `shouldStop` ended the pass before every candidate day was reached. */ + stoppedEarly: boolean; } export interface DenseIntradayHourlyRebuildOptions { @@ -126,6 +128,12 @@ export interface DenseIntradayHourlyRebuildOptions { * pass `0` to lift the bound. */ retentionDays?: number; + /** + * Asked before each day. Returning `true` ends the pass cleanly; the + * summary reports `stoppedEarly`. A rebuilt day leaves the candidate set, + * so the next run starts at the first day this one did not reach. + */ + shouldStop?: () => boolean; } /** @@ -147,6 +155,7 @@ export async function runDenseIntradayHourlyRebuild( const summary: DenseIntradayHourlyRebuildSummary = { dryRun, + stoppedEarly: false, totals: { usersScanned: 0, daysRebuilt: 0, @@ -160,7 +169,7 @@ export async function runDenseIntradayHourlyRebuild( const users = await loadConsolidationUsers(prismaClient, options.userId); summary.totals.usersScanned = users.length; - for (const user of users) { + walk: for (const user of users) { const tz = resolveUserTimezone(user.timezone); for (const type of DENSE_INTRADAY_RETENTION_TYPES) { @@ -195,6 +204,10 @@ export async function runDenseIntradayHourlyRebuild( if (candidates.length === 0) continue; for (const candidate of candidates) { + if (options.shouldStop?.()) { + summary.stoppedEarly = true; + break walk; + } // Per-day failure boundary: one poisoned day is stepped over so the // walk keeps rebuilding every other user / type / day; the failed // day keeps its live daily row, so the discovery re-finds it.