From 50ff90a40900fcf9ed946d17b7bdb2e08933b3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 13 Aug 2026 01:52:51 +0200 Subject: [PATCH] feat(import): refuse an Apple Health import the heap cannot carry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before the multi-minute unpack starts, compare the archive's declared uncompressed export.xml size against the V8 heap limit. The streaming parse keeps resident state span-bounded, so the model is generous — a fixed 192 MiB base plus 1/64 of the XML size; the 8 GiB extraction cap needs ~320 MiB, which any default heap clears. The check only refuses the truly impossible (an operator-pinned tiny heap against a huge export) and does so with an actionable failureReason naming NODE_OPTIONS=--max-old-space-size, instead of an OOM-killed worker and a job stuck in a running state (#775). Document the memory model, the insufficient_memory refusal, and the interrupted-import reconcile in the self-hosting scaling guide. --- docs/self-hosting/scaling.md | 30 +++++++++ .../import/__tests__/unzip-export-xml.test.ts | 24 +++++++ ...ple-health-import-memory-preflight.test.ts | 62 ++++++++++++++++++ src/lib/jobs/apple-health-import-worker.ts | 64 ++++++++++++++++++- 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 src/lib/jobs/__tests__/apple-health-import-memory-preflight.test.ts diff --git a/docs/self-hosting/scaling.md b/docs/self-hosting/scaling.md index 9d0527047..7d9f00408 100644 --- a/docs/self-hosting/scaling.md +++ b/docs/self-hosting/scaling.md @@ -56,6 +56,36 @@ know: Without a shared staging volume, imports in split mode cannot complete. +## Apple Health import memory + +The import parses `export.xml` as a stream: the archive is never +buffered whole, records flush to the database in fixed-size batches, +and what stays resident scales with the data's calendar span, not its +record count. Multi-year exports with millions of records parse within +the default Node.js heap. + +Before parsing starts, the worker compares the archive's declared +uncompressed size against the runtime heap limit. If the limit is too +low to carry the export — realistically only when +`--max-old-space-size` was pinned far below the default — the import +fails immediately with a `failureReason` starting with +`insufficient_memory`, naming the required and available amounts. +Raise the limit on the container that runs the worker and re-upload: + +```yaml +# docker-compose.yml +services: + app: + environment: + NODE_OPTIONS: --max-old-space-size=1024 +``` + +An import interrupted by a container restart or an out-of-memory kill +does not stay stuck: the worker reconciles non-terminal import jobs at +boot and every 15 minutes, flipping orphaned rows to `failed` +(`interrupted_by_restart`) so the UI offers a retry instead of an +endless spinner. + ## Healthchecks - `app` (web) healthcheck: `wget /api/version` every 30s. diff --git a/src/lib/import/__tests__/unzip-export-xml.test.ts b/src/lib/import/__tests__/unzip-export-xml.test.ts index 3863f79b8..10046cbaf 100644 --- a/src/lib/import/__tests__/unzip-export-xml.test.ts +++ b/src/lib/import/__tests__/unzip-export-xml.test.ts @@ -142,6 +142,30 @@ describe("extractExportXml", () => { expect(extracted.toString("utf8")).toBe(payload.toString("utf8")); }); + // (issue #775) — the preflight hook sees the declared sizes before + // any extraction work starts; throwing there must abort the extract + // with the hook's error. + it("aborts before extraction when the preflight hook throws", async () => { + const payload = Buffer.from( + ``, + ); + const zip = buildMinimalZip("apple_health_export/export.xml", payload); + const tmp = mkdtempSync(join(tmpdir(), "healthlog-unzip-")); + const zipPath = join(tmp, "export.zip"); + writeFileSync(zipPath, zip); + + let seenDeclaredBytes = -1; + await expect( + extractExportXml(zipPath, { + preflight: ({ declaredXmlBytes }) => { + seenDeclaredBytes = declaredXmlBytes; + throw new Error("insufficient_memory: preflight refused"); + }, + }), + ).rejects.toThrow(/insufficient_memory/); + expect(seenDeclaredBytes).toBe(payload.length); + }); + it("throws when the export.xml member is missing", async () => { const payload = Buffer.from("noop"); const zip = buildMinimalZip("other-file.txt", payload); diff --git a/src/lib/jobs/__tests__/apple-health-import-memory-preflight.test.ts b/src/lib/jobs/__tests__/apple-health-import-memory-preflight.test.ts new file mode 100644 index 000000000..2e871db9d --- /dev/null +++ b/src/lib/jobs/__tests__/apple-health-import-memory-preflight.test.ts @@ -0,0 +1,62 @@ +/** + * (issue #775) Memory preflight for the Apple Health import. The + * streaming parse keeps memory bounded, so the preflight is generous — + * it exists to refuse only the truly impossible combination (a tiny + * pinned heap against a huge declared export) with an actionable + * reason, instead of letting the worker die mid-parse and leave the + * job stuck in a running state. + */ +import { describe, expect, it } from "vitest"; + +import { + IMPORT_MEMORY_PREFLIGHT_BASE_BYTES, + IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR, + INSUFFICIENT_MEMORY_REASON_PREFIX, + importMemoryPreflightReason, +} from "../apple-health-import-worker"; + +const MIB = 1024 * 1024; +const GIB = 1024 * MIB; + +describe("importMemoryPreflightReason", () => { + it("passes a large real-world export against any default heap", () => { + // A 3 GiB export.xml (238 MB export.zip scale) against a 1 GiB + // default heap — the reported self-hoster setup — must NOT be + // refused: the streaming parse carries it. + expect(importMemoryPreflightReason(3 * GIB, 1 * GIB)).toBeNull(); + }); + + it("passes the largest extractable export against the default heap", () => { + // 8 GiB is the archive-level extraction cap; the model needs + // ~320 MiB for it, far below any default heap. + expect(importMemoryPreflightReason(8 * GIB, 1 * GIB)).toBeNull(); + }); + + it("refuses when the pinned heap cannot carry the declared export", () => { + const reason = importMemoryPreflightReason(8 * GIB, 256 * MIB); + expect(reason).not.toBeNull(); + expect(reason).toMatch( + new RegExp(`^${INSUFFICIENT_MEMORY_REASON_PREFIX}: `), + ); + // The reason must be actionable for the operator: name the knob. + expect(reason).toContain("NODE_OPTIONS=--max-old-space-size="); + // And honest about the numbers involved. + expect(reason).toContain("8192 MiB uncompressed"); + expect(reason).toContain("256 MiB"); + }); + + it("scales the requirement with the declared XML size", () => { + const declared = 4 * GIB; + const required = + IMPORT_MEMORY_PREFLIGHT_BASE_BYTES + + Math.ceil(declared / IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR); + expect(importMemoryPreflightReason(declared, required)).toBeNull(); + expect(importMemoryPreflightReason(declared, required - 1)).not.toBeNull(); + }); + + it("stays comfortably under the 1000-char failureReason column slice", () => { + const reason = importMemoryPreflightReason(8 * GIB, 1); + expect(reason).not.toBeNull(); + expect((reason as string).length).toBeLessThan(500); + }); +}); diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index 542ecb941..60cc8b8e9 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -25,6 +25,7 @@ * Locks per `.planning/research/v1434-r-1-xml-import.md` §5.1. */ import { unlinkSync } from "node:fs"; +import { getHeapStatistics } from "node:v8"; import type { PrismaClient } from "@/generated/prisma/client"; import { prisma, toJson } from "@/lib/db"; import type { Job } from "pg-boss"; @@ -115,6 +116,59 @@ const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000; /** Concurrency cap per worker host. */ export const APPLE_HEALTH_IMPORT_CONCURRENCY = 1; +/** + * (issue #775) Memory preflight for the parse. The streaming pipeline + * keeps per-record work off the heap, so what remains resident scales + * with data SPAN (cumulative day-buckets, per-type counters, batch + * buffers), not record count — a small fraction of the XML size. The + * model below is deliberately generous: a fixed base for parser + + * batches + framework, plus 1/64 of the declared XML size for the + * span-bounded fold state. An 8 GiB export (the archive-level cap) + * needs ~320 MiB under this model — any default Node heap clears it. + * The preflight only refuses the truly impossible, e.g. an operator + * who pinned --max-old-space-size far below that, and it refuses + * BEFORE minutes of unpack work and with an actionable reason instead + * of an OOM-killed worker and a job stuck in "running". + */ +export const IMPORT_MEMORY_PREFLIGHT_BASE_BYTES = 192 * 1024 * 1024; +export const IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR = 64; + +/** + * Machine-readable prefix of the preflight refusal. The web UI keys + * its translated message off this prefix (mirroring the bare + * `interrupted_by_restart` code the reconcile writes); the rest of the + * string stays honest English for the status API and logs. + */ +export const INSUFFICIENT_MEMORY_REASON_PREFIX = "insufficient_memory"; + +const MIB = 1024 * 1024; + +/** + * Returns `null` when the runtime heap can carry the declared export, + * otherwise the `failureReason` string for the refused import. + * `heapLimitBytes` is injectable for tests; production reads the real + * V8 limit, which reflects `NODE_OPTIONS=--max-old-space-size`. + */ +export function importMemoryPreflightReason( + declaredXmlBytes: number, + heapLimitBytes: number = getHeapStatistics().heap_size_limit, +): string | null { + const requiredBytes = + IMPORT_MEMORY_PREFLIGHT_BASE_BYTES + + Math.ceil(declaredXmlBytes / IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR); + if (heapLimitBytes >= requiredBytes) return null; + const requiredMib = Math.ceil(requiredBytes / MIB); + const suggestedMib = Math.max(512, requiredMib); + return ( + `${INSUFFICIENT_MEMORY_REASON_PREFIX}: export.xml declares ` + + `${Math.ceil(declaredXmlBytes / MIB)} MiB uncompressed; parsing it needs ` + + `roughly ${requiredMib} MiB of heap but the Node.js heap limit is ` + + `${Math.floor(heapLimitBytes / MIB)} MiB. Raise it with ` + + `NODE_OPTIONS=--max-old-space-size=${suggestedMib} on the container ` + + `that runs the worker, then upload the export again.` + ); +} + /** * v1.28.33 (issue #486) — job-level pg-boss overrides for the import * sends. The queue defaults (retryLimit 2, expireInSeconds 900) are @@ -325,7 +379,15 @@ export async function handleAppleHealthImport( elapsedMs: 0, }); - const unzip = await extractExportXml(uploadPath); + const unzip = await extractExportXml(uploadPath, { + // Refuse an export the heap demonstrably cannot carry BEFORE the + // multi-minute unpack, with an actionable reason instead of an + // OOM-killed worker (issue #775). + preflight: ({ declaredXmlBytes }) => { + const reason = importMemoryPreflightReason(declaredXmlBytes); + if (reason) throw new Error(reason); + }, + }); extractedXmlPath = unzip.xmlPath; // Phase 2 + 3: parsing + upserting (the parser tracks both