feat(import): refuse an Apple Health import the heap cannot carry

Before the multi-minute unpack starts, compare the archive's declared
uncompressed export.xml size against the V8 heap limit. The streaming
parse keeps resident state span-bounded, so the model is generous —
a fixed 192 MiB base plus 1/64 of the XML size; the 8 GiB extraction
cap needs ~320 MiB, which any default heap clears. The check only
refuses the truly impossible (an operator-pinned tiny heap against a
huge export) and does so with an actionable failureReason naming
NODE_OPTIONS=--max-old-space-size, instead of an OOM-killed worker
and a job stuck in a running state (#775).

Document the memory model, the insufficient_memory refusal, and the
interrupted-import reconcile in the self-hosting scaling guide.
This commit is contained in:
Marc-André Bombeck committed 2026-08-13 06:16:44 +02:00
1 parent dd02f983cb
commit 50ff90a409
4 files changed
+179 -1

No files matched your search

@@ -142,6 +142,30 @@ describe("extractExportXml", () => {
expect(extracted.toString("utf8")).toBe(payload.toString("utf8"));
});
// (issue #775) — the preflight hook sees the declared sizes before
// any extraction work starts; throwing there must abort the extract
// with the hook's error.
it("aborts before extraction when the preflight hook throws", async () => {
const payload = Buffer.from(
`<?xml version="1.0"?><HealthData><ExportDate value="2026-05-15"/></HealthData>`,
);
const zip = buildMinimalZip("apple_health_export/export.xml", payload);
const tmp = mkdtempSync(join(tmpdir(), "healthlog-unzip-"));
const zipPath = join(tmp, "export.zip");
writeFileSync(zipPath, zip);
let seenDeclaredBytes = -1;
await expect(
extractExportXml(zipPath, {
preflight: ({ declaredXmlBytes }) => {
seenDeclaredBytes = declaredXmlBytes;
throw new Error("insufficient_memory: preflight refused");
},
}),
).rejects.toThrow(/insufficient_memory/);
expect(seenDeclaredBytes).toBe(payload.length);
});
it("throws when the export.xml member is missing", async () => {
const payload = Buffer.from("noop");
const zip = buildMinimalZip("other-file.txt", payload);
@@ -0,0 +1,62 @@
/**
* (issue #775) Memory preflight for the Apple Health import. The
* streaming parse keeps memory bounded, so the preflight is generous —
* it exists to refuse only the truly impossible combination (a tiny
* pinned heap against a huge declared export) with an actionable
* reason, instead of letting the worker die mid-parse and leave the
* job stuck in a running state.
*/
import { describe, expect, it } from "vitest";
import {
IMPORT_MEMORY_PREFLIGHT_BASE_BYTES,
IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR,
INSUFFICIENT_MEMORY_REASON_PREFIX,
importMemoryPreflightReason,
} from "../apple-health-import-worker";
const MIB = 1024 * 1024;
const GIB = 1024 * MIB;
describe("importMemoryPreflightReason", () => {
it("passes a large real-world export against any default heap", () => {
// A 3 GiB export.xml (238 MB export.zip scale) against a 1 GiB
// default heap — the reported self-hoster setup — must NOT be
// refused: the streaming parse carries it.
expect(importMemoryPreflightReason(3 * GIB, 1 * GIB)).toBeNull();
});
it("passes the largest extractable export against the default heap", () => {
// 8 GiB is the archive-level extraction cap; the model needs
// ~320 MiB for it, far below any default heap.
expect(importMemoryPreflightReason(8 * GIB, 1 * GIB)).toBeNull();
});
it("refuses when the pinned heap cannot carry the declared export", () => {
const reason = importMemoryPreflightReason(8 * GIB, 256 * MIB);
expect(reason).not.toBeNull();
expect(reason).toMatch(
new RegExp(`^${INSUFFICIENT_MEMORY_REASON_PREFIX}: `),
);
// The reason must be actionable for the operator: name the knob.
expect(reason).toContain("NODE_OPTIONS=--max-old-space-size=");
// And honest about the numbers involved.
expect(reason).toContain("8192 MiB uncompressed");
expect(reason).toContain("256 MiB");
});
it("scales the requirement with the declared XML size", () => {
const declared = 4 * GIB;
const required =
IMPORT_MEMORY_PREFLIGHT_BASE_BYTES +
Math.ceil(declared / IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR);
expect(importMemoryPreflightReason(declared, required)).toBeNull();
expect(importMemoryPreflightReason(declared, required - 1)).not.toBeNull();
});
it("stays comfortably under the 1000-char failureReason column slice", () => {
const reason = importMemoryPreflightReason(8 * GIB, 1);
expect(reason).not.toBeNull();
expect((reason as string).length).toBeLessThan(500);
});
});
+63 -1
View File
@@ -25,6 +25,7 @@
* Locks per `.planning/research/v1434-r-1-xml-import.md` §5.1.
*/
import { unlinkSync } from "node:fs";
import { getHeapStatistics } from "node:v8";
import type { PrismaClient } from "@/generated/prisma/client";
import { prisma, toJson } from "@/lib/db";
import type { Job } from "pg-boss";
@@ -115,6 +116,59 @@ const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
/** Concurrency cap per worker host. */
export const APPLE_HEALTH_IMPORT_CONCURRENCY = 1;
/**
* (issue #775) Memory preflight for the parse. The streaming pipeline
* keeps per-record work off the heap, so what remains resident scales
* with data SPAN (cumulative day-buckets, per-type counters, batch
* buffers), not record count — a small fraction of the XML size. The
* model below is deliberately generous: a fixed base for parser +
* batches + framework, plus 1/64 of the declared XML size for the
* span-bounded fold state. An 8 GiB export (the archive-level cap)
* needs ~320 MiB under this model — any default Node heap clears it.
* The preflight only refuses the truly impossible, e.g. an operator
* who pinned --max-old-space-size far below that, and it refuses
* BEFORE minutes of unpack work and with an actionable reason instead
* of an OOM-killed worker and a job stuck in "running".
*/
export const IMPORT_MEMORY_PREFLIGHT_BASE_BYTES = 192 * 1024 * 1024;
export const IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR = 64;
/**
* Machine-readable prefix of the preflight refusal. The web UI keys
* its translated message off this prefix (mirroring the bare
* `interrupted_by_restart` code the reconcile writes); the rest of the
* string stays honest English for the status API and logs.
*/
export const INSUFFICIENT_MEMORY_REASON_PREFIX = "insufficient_memory";
const MIB = 1024 * 1024;
/**
* Returns `null` when the runtime heap can carry the declared export,
* otherwise the `failureReason` string for the refused import.
* `heapLimitBytes` is injectable for tests; production reads the real
* V8 limit, which reflects `NODE_OPTIONS=--max-old-space-size`.
*/
export function importMemoryPreflightReason(
declaredXmlBytes: number,
heapLimitBytes: number = getHeapStatistics().heap_size_limit,
): string | null {
const requiredBytes =
IMPORT_MEMORY_PREFLIGHT_BASE_BYTES +
Math.ceil(declaredXmlBytes / IMPORT_MEMORY_PREFLIGHT_XML_DIVISOR);
if (heapLimitBytes >= requiredBytes) return null;
const requiredMib = Math.ceil(requiredBytes / MIB);
const suggestedMib = Math.max(512, requiredMib);
return (
`${INSUFFICIENT_MEMORY_REASON_PREFIX}: export.xml declares ` +
`${Math.ceil(declaredXmlBytes / MIB)} MiB uncompressed; parsing it needs ` +
`roughly ${requiredMib} MiB of heap but the Node.js heap limit is ` +
`${Math.floor(heapLimitBytes / MIB)} MiB. Raise it with ` +
`NODE_OPTIONS=--max-old-space-size=${suggestedMib} on the container ` +
`that runs the worker, then upload the export again.`
);
}
/**
* v1.28.33 (issue #486) — job-level pg-boss overrides for the import
* sends. The queue defaults (retryLimit 2, expireInSeconds 900) are
@@ -325,7 +379,15 @@ export async function handleAppleHealthImport(
elapsedMs: 0,
});
const unzip = await extractExportXml(uploadPath);
const unzip = await extractExportXml(uploadPath, {
// Refuse an export the heap demonstrably cannot carry BEFORE the
// multi-minute unpack, with an actionable reason instead of an
// OOM-killed worker (issue #775).
preflight: ({ declaredXmlBytes }) => {
const reason = importMemoryPreflightReason(declaredXmlBytes);
if (reason) throw new Error(reason);
},
});
extractedXmlPath = unzip.xmlPath;
// Phase 2 + 3: parsing + upserting (the parser tracks both