feat(apple-health): import lab results from Health Records in the export

The export parser counted every <ClinicalRecord> as skipped. Lab-result
records are now handed to a pass after the export.xml rows are written:
each referenced clinical-records/ JSON file is read from the archive
one at a time, mapped with the shared FHIR mapper, given its panel's
name and written through the shared writer. Other clinical types stay
skipped. The pass is fail-soft like the ECG and route passes, and the
job result carries bounded lab counts only.

The ECG pass moves out of the worker into ecg-import.ts, and the parser
revision goes to 4 so an archive imported before can be uploaded again
to pick up its lab results.
This commit is contained in:
Marc-André Bombeck committed 2026-11-17 18:42:21 +01:00
1 parent e40de2a654
commit 736e6e9c86
16 files changed
+922 -212

No files matched your search

@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 84,
"total": 93816,
"total": 93825,
"entries": {
"app/medications/page-client.tsx": 842,
"components/charts/mood-chart.tsx": 1166,
@@ -72,7 +72,7 @@
"lib/mcp/record-search.ts": 1397,
"lib/mcp/rich-reads.ts": 1471,
"lib/measurements/dense-intraday-retention.ts": 1016,
"lib/measurements/import-apple-health-export.ts": 1722,
"lib/measurements/import-apple-health-export.ts": 1731,
"lib/medications/scheduling/cadence.ts": 976,
"lib/medications/scheduling/recurrence.ts": 1100,
"lib/medications/scheduling/slot-upsert.ts": 912,
@@ -1,7 +1,7 @@
{
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 335,
"total": 109025,
"total": 109021,
"entries": {
"app/achievements/page.tsx :: AchievementsPage": 206,
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 275,
@@ -285,7 +285,7 @@
"lib/insights/status-provider.ts :: runStatusCompletion": 215,
"lib/integrations/connection-health-load.ts :: loadConnectionFacts": 229,
"lib/integrations/status.ts :: recordSyncFailure": 292,
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 300,
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 292,
"lib/jobs/coach-nudge.ts :: findTriggerForUser": 279,
"lib/jobs/coach-nudge.ts :: runCoachNudgeTick": 292,
"lib/jobs/coach-reminder-sweep.ts :: runCoachReminderSweep": 222,
@@ -314,8 +314,8 @@
"lib/measurements/dense-intraday-retention.ts :: runDenseIntradayRetention > walkSource > writeDay > foldOnce": 212,
"lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative": 344,
"lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative > writeDay": 205,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1245,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 323,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1247,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 325,
"lib/medications/efficacy/build-efficacy.ts :: buildMedicationEfficacy": 268,
"lib/medications/intake-slot-dedup.ts :: dedupeUserIntakeSlots": 319,
"lib/medications/list-read.ts :: buildMedicationsList": 310,
@@ -198,7 +198,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
expect(mocks.importJobCreate).toHaveBeenCalledTimes(1);
expect(mocks.importJobCreate).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ parserRevision: 3 }),
data: expect.objectContaining({ parserRevision: 4 }),
}),
);
expect(mocks.bossSend).toHaveBeenCalledWith(
@@ -217,7 +217,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
expect.objectContaining({
where: expect.objectContaining({
status: { not: "failed" },
parserRevision: 3,
parserRevision: 4,
}),
}),
);
@@ -241,7 +241,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({ parserRevision: 3 }),
where: expect.objectContaining({ parserRevision: 4 }),
}),
);
});
@@ -93,6 +93,54 @@ describe("GET /api/import/apple-health-export/[jobId]/status", () => {
expect(body.data.status).toBe("parsing");
});
it("bounds the lab counts and drops anything else in the block", async () => {
vi.mocked(getSession).mockResolvedValue(OWNER_SESSION as never);
mocks.importJobFindUnique.mockResolvedValue(
makeRow({
status: "done",
result: {
labs: {
discovered: 3,
imported: 2,
duplicate: -1,
skipped: "x",
failed: 1.5,
reasons: { leaked: "content" },
},
},
}),
);
const req = new NextRequest(
"http://localhost/api/import/apple-health-export/ij-1/status",
);
const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) });
const body = (await res.json()) as {
data: { result: { labs: Record<string, unknown> } };
};
expect(body.data.result.labs).toEqual({
discovered: 3,
imported: 2,
duplicate: 0,
skipped: 0,
failed: 0,
});
});
it("reports zeroed lab counts for a job from before labs were read", async () => {
vi.mocked(getSession).mockResolvedValue(OWNER_SESSION as never);
mocks.importJobFindUnique.mockResolvedValue(
makeRow({ status: "done", result: { totals: { recordsRead: 1 } } }),
);
const req = new NextRequest(
"http://localhost/api/import/apple-health-export/ij-1/status",
);
const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) });
const body = (await res.json()) as {
data: { result: { labs: Record<string, number> } };
};
expect(body.data.result.labs.imported).toBe(0);
});
it("returns 404 to a non-owner non-admin user", async () => {
vi.mocked(getSession).mockResolvedValue(OTHER_SESSION as never);
mocks.importJobFindUnique.mockResolvedValue(makeRow());
@@ -46,10 +46,12 @@ function privacySafeResult(
value: Record<string, unknown> | null,
): Record<string, unknown> | null {
if (!value) return null;
const ecg =
typeof value.ecg === "object" && value.ecg !== null
? (value.ecg as Record<string, unknown>)
const block = (key: string): Record<string, unknown> =>
typeof value[key] === "object" && value[key] !== null
? (value[key] as Record<string, unknown>)
: {};
const ecg = block("ecg");
const labs = block("labs");
return {
...value,
ecg: {
@@ -59,6 +61,13 @@ function privacySafeResult(
skipped: boundedCount(ecg.skipped),
failed: boundedCount(ecg.failed),
},
labs: {
discovered: boundedCount(labs.discovered),
imported: boundedCount(labs.imported),
duplicate: boundedCount(labs.duplicate),
skipped: boundedCount(labs.skipped),
failed: boundedCount(labs.failed),
},
};
}
@@ -200,7 +200,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
expect(mocks.importJobCreate).toHaveBeenCalledTimes(1);
expect(mocks.importJobCreate).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ parserRevision: 3 }),
data: expect.objectContaining({ parserRevision: 4 }),
}),
);
expect(mocks.bossSend).toHaveBeenCalledWith(
@@ -217,7 +217,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
expect.objectContaining({
where: expect.objectContaining({
status: { not: "failed" },
parserRevision: 3,
parserRevision: 4,
}),
}),
);
@@ -243,7 +243,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({ parserRevision: 3 }),
where: expect.objectContaining({ parserRevision: 4 }),
}),
);
});
@@ -0,0 +1,225 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { strToU8, zipSync } from "fflate";
const mocks = vi.hoisted(() => ({
write: vi.fn(),
after: vi.fn(),
annotate: vi.fn(),
}));
vi.mock("@/lib/labs/fhir-import/write", () => ({
writeFhirLabReadings: mocks.write,
}));
vi.mock("@/lib/labs/fhir-import/after-write", () => ({
afterLabReadingsWritten: mocks.after,
}));
vi.mock("@/lib/logging/context", () => ({ annotate: mocks.annotate }));
import { importAppleHealthClinicalLabs } from "../clinical-labs-import";
import type { ClinicalLabReference } from "../clinical-record-ref";
import { zip64Archive } from "./helpers/zip64";
const dirs: string[] = [];
afterEach(() => {
for (const dir of dirs.splice(0))
rmSync(dir, { recursive: true, force: true });
});
beforeEach(() => {
vi.clearAllMocks();
mocks.write.mockImplementation(
async ({ readings }: { readings: unknown[] }) => ({
statuses: readings.map(() => ({ status: "inserted" })),
inserted: readings.length,
duplicate: 0,
failed: 0,
newestTakenAt: new Date("2025-01-01T00:00:00Z"),
}),
);
});
const ROOT = "apple_health_export/clinical-records/";
function observation(id: string, extra: Record<string, unknown> = {}) {
return JSON.stringify({
resourceType: "Observation",
id,
status: "final",
category: [{ coding: [{ code: "laboratory" }] }],
code: { text: `Analyte ${id}` },
effectiveDateTime: "2025-01-01T08:00:00Z",
valueQuantity: { value: 1, unit: "mmol/L" },
...extra,
});
}
function archive(files: Record<string, string>, zip64 = false): string {
const dir = mkdtempSync(join(tmpdir(), "healthlog-clinical-"));
dirs.push(dir);
const path = join(dir, "export.zip");
writeFileSync(
path,
zip64
? zip64Archive(files)
: zipSync(
Object.fromEntries(
Object.entries(files).map(([name, body]) => [name, strToU8(body)]),
),
),
);
return path;
}
function ref(
path: string | null,
identifier: string | null = null,
): ClinicalLabReference {
return {
path,
identifier,
sourceURL: "https://fhir.example.test",
sourceName: "Example",
fhirVersion: "4.0.1",
};
}
function run(path: string, references: ClinicalLabReference[], limits = {}) {
return importAppleHealthClinicalLabs({
archivePath: path,
userId: "user-1",
tz: "Europe/Berlin",
references,
limits: {
maxMembers: 50,
maxClinicalMembers: 20,
maxMemberBytes: 64 * 1024,
maxTotalClinicalBytes: 1024 * 1024,
maxCompressionRatio: 200,
...limits,
},
});
}
describe("importAppleHealthClinicalLabs", () => {
it("reads only referenced members and writes their readings with panels", async () => {
const path = archive({
"apple_health_export/export.xml": "<HealthData/>",
[`${ROOT}Observation-a.json`]: observation("a"),
[`${ROOT}Observation-panel.json`]: observation("panel", {
code: { text: "Electrolytes" },
valueQuantity: undefined,
hasMember: [{ reference: "Observation/a" }],
}),
[`${ROOT}Condition-c.json`]: '{"resourceType":"Condition"}',
});
const counts = await run(path, [
ref("/clinical-records/Observation-a.json", "a"),
ref("/clinical-records/Observation-panel.json", "panel"),
]);
expect(counts).toEqual({
discovered: 2,
imported: 1,
duplicate: 0,
skipped: 1,
failed: 0,
});
const [{ readings, tz, userId }] = mocks.write.mock.calls[0];
expect(userId).toBe("user-1");
expect(tz).toBe("Europe/Berlin");
expect(readings).toHaveLength(1);
expect(readings[0]).toMatchObject({
analyte: "Analyte a",
panel: "Electrolytes",
});
expect(mocks.after).toHaveBeenCalledWith(
expect.objectContaining({ userId: "user-1", inserted: 1 }),
);
});
it("works on a ZIP64 archive", async () => {
const path = archive(
{ [`${ROOT}Observation-a.json`]: observation("a") },
true,
);
const counts = await run(path, [
ref("/clinical-records/Observation-a.json"),
]);
expect(counts.imported).toBe(1);
});
it("counts a missing member, malformed JSON and an unsafe path as failed", async () => {
const path = archive({ [`${ROOT}Observation-bad.json`]: "{not json" });
const counts = await run(path, [
ref("/clinical-records/Observation-bad.json"),
ref("/clinical-records/Observation-gone.json"),
ref("/clinical-records/../export.xml"),
]);
expect(counts).toMatchObject({ discovered: 3, imported: 0, failed: 3 });
expect(mocks.write).not.toHaveBeenCalled();
});
it("finds a reference without a path by the resource id", async () => {
const path = archive({
[`${ROOT}one.json`]: observation("a"),
[`${ROOT}two.json`]: observation("unreferenced"),
});
const counts = await run(path, [ref(null, "a")]);
expect(counts).toMatchObject({ discovered: 1, imported: 1, failed: 0 });
expect(mocks.write.mock.calls[0][0].readings).toHaveLength(1);
});
it("refuses references past the clinical member ceiling", async () => {
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
const counts = await run(
path,
[
ref("/clinical-records/Observation-a.json"),
ref("/clinical-records/Observation-b.json"),
],
{ maxClinicalMembers: 1 },
);
expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 });
});
it("fails soft when the archive breaks a limit", async () => {
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
const counts = await run(
path,
[ref("/clinical-records/Observation-a.json")],
{
maxMemberBytes: 10,
},
);
expect(counts).toMatchObject({ discovered: 1, imported: 0, failed: 1 });
});
it("annotates counts and closed reasons only", async () => {
const path = archive({
[`${ROOT}Observation-a.json`]: observation("a", {
code: { text: "Secret analyte name" },
status: "preliminary",
}),
});
await run(path, [ref("/clinical-records/Observation-a.json")]);
const serialised = JSON.stringify(mocks.annotate.mock.calls);
expect(serialised).not.toContain("Secret analyte name");
expect(serialised).not.toContain("fhir.example.test");
expect(mocks.annotate).toHaveBeenCalledWith(
expect.objectContaining({
action: { name: "apple_health.import.clinical_labs" },
meta: expect.objectContaining({
skipped: 1,
reasons: { not_final: 1 },
}),
}),
);
});
it("does nothing for no references", async () => {
const counts = await run("/nonexistent.zip", []);
expect(counts.discovered).toBe(0);
expect(mocks.write).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,220 @@
/**
* Lab results from Apple Health's Health Records, out of an `export.zip`
* (#1400).
*
* `export.xml` lists each Health Records resource as a `<ClinicalRecord>`
* whose `resourceFilePath` names a FHIR JSON file under `clinical-records/`.
* The XML pass hands the lab-result references here
* (`routeClinicalRecord`); this module then reads the referenced members one
* at a time, maps each with the shared FHIR mapper, gives members their
* panel's name, and writes through the shared writer, so a result lands the
* same way it would from the iOS app (`POST /api/labs/fhir-batch`).
*
* A reference with no path (an export from a Health app that did not write
* one) is matched by the resource's own id among the clinical files instead.
*
* Memory: one member is held at a time, capped by the archive reader; what is
* kept across members is the mapped readings (a few hundred bytes each) and
* the panel headers' member lists.
*
* A failure here never fails the import: the measurements are committed. A
* missing, oversized or malformed file is counted as failed. Nothing about a
* resource reaches the result, the logs or a wide event: only counts and
* closed reasons.
*/
import type { PrismaClient } from "@/generated/prisma/client";
import {
clinicalMemberNameFromReference,
isClinicalRecordJson,
streamAppleHealthClinicalMembers,
type AppleHealthClinicalArchiveLimits,
} from "@/lib/apple-health/archive-stream";
import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref";
import { afterLabReadingsWritten } from "@/lib/labs/fhir-import/after-write";
import {
assignPanels,
mapFhirLabRecord,
type FhirLabMapping,
} from "@/lib/labs/fhir-import/map-observation";
import {
emptyFhirLabCounts,
type FhirLabCounts,
} from "@/lib/labs/fhir-import/types";
import { writeFhirLabReadings } from "@/lib/labs/fhir-import/write";
import { annotate } from "@/lib/logging/context";
import type { ImportJobResult } from "@/lib/measurements/import-apple-health-export";
/** The terminal result of an Apple Health import job, labs included. */
export type AppleHealthImportJobResult = ImportJobResult & {
labs: FhirLabCounts;
};
/** Per-archive ceilings for the Health Records members. */
export const APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS: AppleHealthClinicalArchiveLimits =
{
maxMembers: 200_000,
maxClinicalMembers: 50_000,
// A single Observation is a few KB; 2 MiB leaves room for a long note.
maxMemberBytes: 2 * 1024 * 1024,
maxTotalClinicalBytes: 256 * 1024 * 1024,
maxCompressionRatio: 200,
};
async function readJson(stream: AsyncIterable<unknown>): Promise<unknown> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
}
const text = Buffer.concat(chunks).toString("utf8");
for (const chunk of chunks) chunk.fill(0);
return JSON.parse(text) as unknown;
}
function resourceId(resource: unknown): string | null {
if (typeof resource !== "object" || resource === null) return null;
const id = (resource as { id?: unknown }).id;
return typeof id === "string" && id.trim() !== "" ? id.trim() : null;
}
export async function importAppleHealthClinicalLabs(input: {
archivePath: string;
userId: string;
tz: string;
references: readonly ClinicalLabReference[];
prisma?: Pick<PrismaClient, "labResult" | "biomarker" | "biomarkerAlias">;
limits?: AppleHealthClinicalArchiveLimits;
}): Promise<FhirLabCounts> {
const counts = emptyFhirLabCounts();
const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS;
counts.discovered = input.references.length;
if (input.references.length === 0) return counts;
const tally: Record<string, number> = {};
const note = (reason: string) => {
tally[reason] = (tally[reason] ?? 0) + 1;
};
// More references than the archive may hold clinical files: the overflow
// is refused, the rest is read.
const usable = input.references.slice(0, limits.maxClinicalMembers);
const overflow = input.references.length - usable.length;
counts.failed += overflow;
if (overflow > 0) tally.over_limit = overflow;
const byMember = new Map<string, ClinicalLabReference>();
const byId = new Map<string, ClinicalLabReference>();
for (const ref of usable) {
const member = ref.path ? clinicalMemberNameFromReference(ref.path) : null;
if (member && !byMember.has(member)) byMember.set(member, ref);
else if (!ref.path && ref.identifier && !byId.has(ref.identifier)) {
byId.set(ref.identifier, ref);
} else {
// An unsafe path, or a second reference to one file.
counts.failed += 1;
note("bad_reference");
}
}
const expected = byMember.size + byId.size;
let handled = 0;
const mapped: FhirLabMapping[] = [];
try {
for await (const member of streamAppleHealthClinicalMembers({
archivePath: input.archivePath,
limits,
wanted: (name) =>
byMember.has(name) || (byId.size > 0 && isClinicalRecordJson(name)),
})) {
const byPath = byMember.get(member.name);
let resource: unknown;
try {
resource = await readJson(member.stream);
} catch {
if (byPath) {
handled += 1;
counts.failed += 1;
note("malformed_json");
}
continue;
}
let ref = byPath;
if (!ref) {
const id = resourceId(resource);
ref = id ? byId.get(id) : undefined;
if (!ref || !id) continue;
byId.delete(id);
}
handled += 1;
const result = mapFhirLabRecord({
resource,
fhirVersion: ref.fhirVersion,
identifier: ref.identifier,
sourceURL: ref.sourceURL,
sourceName: ref.sourceName,
});
resource = null;
if (result.kind === "reading") {
mapped.push(result);
} else if (result.kind === "skip") {
counts.skipped += 1;
note(result.reason);
if (result.reason === "panel_header") mapped.push(result);
} else {
counts.failed += 1;
note(result.reason);
}
}
} catch {
// The archive itself is unreadable for this pass (a limit hit in the
// directory, an inflate failure): the files not yet read are counted
// below as missing. What was read is still written.
note("archive_unreadable");
}
const missing = Math.max(0, expected - handled);
counts.failed += missing;
if (missing > 0) tally.missing_member = (tally.missing_member ?? 0) + missing;
assignPanels(mapped);
const readings = mapped.flatMap((m) =>
m.kind === "reading" ? [m.reading] : [],
);
mapped.length = 0;
if (readings.length > 0) {
try {
const written = await writeFhirLabReadings({
userId: input.userId,
tz: input.tz,
readings,
prisma: input.prisma,
});
counts.imported += written.inserted;
counts.duplicate += written.duplicate;
counts.failed += written.failed;
for (const status of written.statuses) {
if (status.status === "failed") note(status.reason);
}
afterLabReadingsWritten({
userId: input.userId,
inserted: written.inserted,
newestTakenAt: written.newestTakenAt,
source: "apple_health_export",
});
} catch {
counts.failed += readings.length;
note("write_failed");
}
}
annotate({
action: { name: "apple_health.import.clinical_labs" },
meta: {
discovered: counts.discovered,
imported: counts.imported,
duplicate: counts.duplicate,
skipped: counts.skipped,
failed: counts.failed,
reasons: tally,
},
});
return counts;
}
@@ -0,0 +1,61 @@
/**
* A `<ClinicalRecord>` in `export.xml`, as a reference to the lab result it
* describes (#1400).
*
* Apple writes one element per Health Records resource:
*
* <ClinicalRecord type="HKClinicalTypeIdentifierLabResultRecord"
* identifier="…" sourceName="…" sourceURL="…" fhirVersion="4.0.1"
* receivedDate="…" resourceFilePath="/clinical-records/….json"/>
*
* The resource itself is the JSON file the path names. Only lab results are
* handed on; every other clinical type (conditions, medications, vital signs,
* immunisations) is left to the caller's skipped count. The attributes are
* capped here so a hostile export cannot make the parser hold long strings.
*/
export const HK_LAB_RESULT_RECORD = "HKClinicalTypeIdentifierLabResultRecord";
export interface ClinicalLabReference {
/** `resourceFilePath`, from the export root. Null when Apple wrote none. */
path: string | null;
identifier: string | null;
sourceURL: string | null;
sourceName: string | null;
fhirVersion: string | null;
}
const CAPS = {
path: 512,
identifier: 256,
sourceURL: 2048,
sourceName: 200,
fhirVersion: 16,
} as const;
function capped(raw: string | undefined, max: number): string | null {
if (raw === undefined) return null;
const trimmed = raw.trim();
if (trimmed === "" || trimmed.length > max) return null;
return trimmed;
}
/**
* Hand a lab-result record to `onLab` and return true; return false for any
* other record, or when there is no callback, so the caller counts it as
* skipped.
*/
export function routeClinicalRecord(
attrs: Record<string, string | undefined>,
onLab?: (ref: ClinicalLabReference) => void,
): boolean {
if (!onLab || attrs.type !== HK_LAB_RESULT_RECORD) return false;
onLab({
path: capped(attrs.resourceFilePath, CAPS.path),
identifier: capped(attrs.identifier, CAPS.identifier),
sourceURL: capped(attrs.sourceURL, CAPS.sourceURL),
sourceName: capped(attrs.sourceName, CAPS.sourceName),
fhirVersion: capped(attrs.fhirVersion, CAPS.fhirVersion),
});
return true;
}
+74 -1
View File
@@ -17,7 +17,11 @@
*/
import { createHash } from "node:crypto";
import type { PrismaClient } from "@/generated/prisma/client";
import type { NormalizedAppleHealthEcg } from "@/lib/apple-health/ecg-csv";
import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream";
import {
parseAppleHealthEcgCsv,
type NormalizedAppleHealthEcg,
} from "@/lib/apple-health/ecg-csv";
import { persistEcgRecording } from "@/lib/ecg/persist-recording";
export type AppleHealthEcgImportOutcome = "imported" | "updated" | "skipped";
@@ -68,3 +72,72 @@ export async function importAppleHealthEcg(input: {
if (result.outcome === "duplicate") return "skipped";
return "updated";
}
const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = {
maxMembers: 20_000,
maxEcgMembers: 2_000,
maxMemberBytes: 16 * 1024 * 1024,
maxTotalEcgBytes: 512 * 1024 * 1024,
maxCompressionRatio: 200,
} as const;
const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
export interface AppleHealthEcgCounts {
discovered: number;
imported: number;
updated: number;
skipped: number;
failed: number;
}
/**
* Every ECG recording in an export archive. Intentionally fail-soft: the
* canonical export.xml pass has already completed, and malformed, oversized
* or unwritable ECG members affect only bounded scalar counters in the
* terminal result. `memberNames` are the archive's other members, which the
* discovered count is taken from.
*/
export async function importAppleHealthArchiveEcgs(input: {
archivePath: string;
userId: string;
memberNames: readonly string[];
prisma: Pick<PrismaClient, "$transaction">;
}): Promise<AppleHealthEcgCounts> {
const ecg: AppleHealthEcgCounts = {
discovered: 0,
imported: 0,
updated: 0,
skipped: 0,
failed: 0,
};
ecg.discovered = input.memberNames.filter((name) => {
const lower = name.toLowerCase();
return lower.includes("electrocardiograms/") && lower.endsWith(".csv");
}).length;
try {
for await (const member of streamAppleHealthEcgMembers({
archivePath: input.archivePath,
limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS,
})) {
try {
const parsed = await parseAppleHealthEcgCsv({
memberName: member.name,
stream: member.stream,
maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES,
});
const outcome = await importAppleHealthEcg({
userId: input.userId,
ecg: parsed,
prisma: input.prisma,
});
ecg[outcome] += 1;
} catch {
ecg.failed += 1;
}
}
} catch {
const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed;
ecg.failed += Math.max(0, ecg.discovered - completed);
}
return ecg;
}
@@ -8,6 +8,10 @@ const workerSource = readFileSync(
join(process.cwd(), "src/lib/jobs/apple-health-import-worker.ts"),
"utf8",
);
const ecgSource = readFileSync(
join(process.cwd(), "src/lib/apple-health/ecg-import.ts"),
"utf8",
);
const parserSource = readFileSync(
join(process.cwd(), "src/lib/measurements/import-apple-health-export.ts"),
"utf8",
@@ -19,18 +23,22 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
});
it("runs bounded ECG archive ingestion beside the established XML parser", () => {
expect(workerSource).toMatch(/streamAppleHealthEcgMembers/);
expect(workerSource).toMatch(/parseAppleHealthEcgCsv/);
expect(workerSource).toMatch(/importAppleHealthEcg/);
expect(workerSource).toMatch(/streamParseExportXml/);
expect(workerSource).toMatch(/importAppleHealthArchiveEcgs/);
expect(ecgSource).toMatch(/streamAppleHealthEcgMembers/);
expect(ecgSource).toMatch(/parseAppleHealthEcgCsv/);
expect(ecgSource).toMatch(/importAppleHealthEcg\(/);
});
it("catches per-ECG failures without routing the valid XML result to the fatal job catch", () => {
expect(workerSource).toMatch(
expect(ecgSource).toMatch(
/for await[\s\S]*streamAppleHealthEcgMembers[\s\S]*try[\s\S]*parseAppleHealthEcgCsv[\s\S]*catch[\s\S]*ecg\.failed/,
);
expect(ecgSource).toMatch(
/ecg:\s*AppleHealthEcgCounts\s*=\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/,
);
expect(workerSource).toMatch(
/result[\s\S]*ecg:\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/,
/result:\s*AppleHealthImportJobResult\s*=\s*\{[\s\S]*ecg,[\s\S]*labs:/,
);
});
@@ -45,7 +53,9 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
it("does not log ECG filenames, source labels, samples, or raw parser errors", () => {
const loggingCalls =
workerSource.match(/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g) ?? [];
(workerSource + ecgSource).match(
/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g,
) ?? [];
const ecgLogging = loggingCalls.filter((call) =>
/ecg|electrocardiogram/i.test(call),
);
@@ -56,7 +66,7 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
});
it("does not claim to diagnose or reclassify a waveform", () => {
expect(workerSource).not.toMatch(
expect(workerSource + ecgSource).not.toMatch(
/(?:diagnose|infer|classify)Ecg|analy[sz]eWaveform|detectA(?:fib|trial)/i,
);
});
@@ -80,8 +80,8 @@ describe("apple health import — pg-boss send policy", () => {
});
describe("apple health import — parser revision boundary", () => {
it("isolates revision-3 sends from legacy workers", () => {
expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(3);
it("isolates revision-4 sends from legacy workers", () => {
expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(4);
expect(APPLE_HEALTH_IMPORT_V2_QUEUE).toBe("apple-health-import-v2");
expect(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).toBe("apple-health-import");
});
+71 -82
View File
@@ -30,9 +30,12 @@ import type { PrismaClient } from "@/generated/prisma/client";
import { prisma, toJson } from "@/lib/db";
import type { Job } from "pg-boss";
import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream";
import { parseAppleHealthEcgCsv } from "@/lib/apple-health/ecg-csv";
import { importAppleHealthEcg } from "@/lib/apple-health/ecg-import";
import {
importAppleHealthClinicalLabs,
type AppleHealthImportJobResult,
} from "@/lib/apple-health/clinical-labs-import";
import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref";
import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import";
import { extractExportXml } from "@/lib/import/unzip-export-xml";
import { annotate } from "@/lib/logging/context";
import {
@@ -45,7 +48,6 @@ import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
streamParseExportXml,
type ImportJobProgress,
type ImportJobResult,
} from "@/lib/measurements/import-apple-health-export";
import { withBackgroundEvent } from "@/lib/logging/background";
import { DEFAULT_TIMEZONE } from "@/lib/tz/format";
@@ -125,17 +127,12 @@ export async function handleImportJobReconcileTick(
);
}
/** Parser semantics carried by every newly-created ImportJob. */
export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 3;
const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = {
maxMembers: 20_000,
maxEcgMembers: 2_000,
maxMemberBytes: 16 * 1024 * 1024,
maxTotalEcgBytes: 512 * 1024 * 1024,
maxCompressionRatio: 200,
} as const;
const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
/**
* Parser semantics carried by every newly-created ImportJob. 4: lab results
* from Health Records are read (#1400), so an archive imported before is
* worth uploading again.
*/
export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 4;
/**
* (issue #775) Memory preflight for the parse. The streaming pipeline
@@ -404,6 +401,7 @@ export async function handleAppleHealthImport(
// GPS tracks are read from the archive only when the person keeps them.
const keepRoutes = userRow?.storeWorkoutRoutes === true;
const routeReferences: WorkoutRouteReference[] = [];
const labReferences: ClinicalLabReference[] = [];
const userTimezone =
userRow?.timezone && userRow.timezone.length > 0
? userRow.timezone
@@ -439,80 +437,58 @@ export async function handleAppleHealthImport(
elapsedMs: 0,
});
const result: ImportJobResult = {
...(await streamParseExportXml({
xmlPath: unzip.xmlPath,
userId,
userTimezone,
prisma,
...(keepRoutes
? { onWorkoutRoute: (ref) => void routeReferences.push(ref) }
: {}),
onProgress: async (snapshot) => {
// The phase label here is what the polling endpoint surfaces;
// map "parsing" / "upserting" through verbatim.
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: snapshot.currentPhase,
progress: toJson(snapshot),
},
});
},
// The instant the Health app stamped on the archive. Persisted as
// soon as the parser reads it, so the status endpoint answers with
// it for the rest of the run and a job that later fails still
// records which export it was working from. Best-effort: the
// stamp is provenance, and losing it must not fail an import.
onExportDate: async (exportedAt) => {
await prisma.importJob
.update({ where: { id: importJobId }, data: { exportedAt } })
.catch(caughtAs("apple_health.import.export_date_failed"));
},
})),
ecg: {
const parsed = await streamParseExportXml({
xmlPath: unzip.xmlPath,
userId,
userTimezone,
prisma,
...(keepRoutes
? { onWorkoutRoute: (ref) => void routeReferences.push(ref) }
: {}),
onClinicalLabRecord: (ref) => void labReferences.push(ref),
onProgress: async (snapshot) => {
// The phase label here is what the polling endpoint surfaces;
// map "parsing" / "upserting" through verbatim.
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: snapshot.currentPhase,
progress: toJson(snapshot),
},
});
},
// The instant the Health app stamped on the archive. Persisted as
// soon as the parser reads it, so the status endpoint answers with
// it for the rest of the run and a job that later fails still
// records which export it was working from. Best-effort: the
// stamp is provenance, and losing it must not fail an import.
onExportDate: async (exportedAt) => {
await prisma.importJob
.update({ where: { id: importJobId }, data: { exportedAt } })
.catch(caughtAs("apple_health.import.export_date_failed"));
},
});
// The auxiliary passes below are fail-soft: the export.xml rows are
// committed, and a member that cannot be read is only counted.
const ecg = await importAppleHealthArchiveEcgs({
archivePath: uploadPath,
userId,
memberNames: unzip.otherMembers.map((member) => member.name),
prisma,
});
const result: AppleHealthImportJobResult = {
...parsed,
ecg,
labs: {
discovered: 0,
imported: 0,
updated: 0,
duplicate: 0,
skipped: 0,
failed: 0,
},
};
// Auxiliary ECGs are intentionally fail-soft. The canonical export.xml
// transaction has already completed; malformed, oversized, or unwritable
// ECG members affect only bounded scalar counters in the terminal result.
const ecg = result.ecg;
ecg.discovered = unzip.otherMembers.filter((member) => {
const lower = member.name.toLowerCase();
return lower.includes("electrocardiograms/") && lower.endsWith(".csv");
}).length;
try {
for await (const member of streamAppleHealthEcgMembers({
archivePath: uploadPath,
limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS,
})) {
try {
const parsed = await parseAppleHealthEcgCsv({
memberName: member.name,
stream: member.stream,
maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES,
});
const outcome = await importAppleHealthEcg({
userId,
ecg: parsed,
prisma,
});
ecg[outcome] += 1;
} catch {
ecg.failed += 1;
}
}
} catch {
const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed;
ecg.failed += Math.max(0, ecg.discovered - completed);
}
// Workout GPS tracks, on opt-in only. Fail-soft like the ECGs: the
// workouts are committed, a track that cannot be read is only counted.
// Read one member at a time from the archive; nothing about a track
@@ -538,6 +514,19 @@ export async function handleAppleHealthImport(
routeReferences.length = 0;
}
// Lab results from Health Records, one FHIR file at a time; only counts
// reach the result.
if (labReferences.length > 0) {
result.labs = await importAppleHealthClinicalLabs({
archivePath: uploadPath,
userId,
tz: userTimezone,
references: labReferences,
prisma,
});
labReferences.length = 0;
}
// Done. Persist the terminal envelope.
await prisma.importJob.update({
where: { id: importJobId },
+132 -104
View File
@@ -70,7 +70,6 @@ interface Pending {
at: number;
reading: MappedFhirLab;
markerId: string | null;
keys: string[];
}
export async function writeFhirLabReadings(args: {
@@ -168,51 +167,11 @@ export async function writeFhirLabReadings(args: {
at,
reading,
markerId: hit?.marker.id ?? null,
keys: [],
});
}
// 3. The reading itself, against the live record.
const existingKeys = new Set<string>();
if (planned.length > 0) {
const days = planned
.map((p) => labReadingDay(p.reading.takenAt, tz))
.sort();
const ids = [
...new Set(
planned.map((p) => p.markerId).filter((id): id is string => !!id),
),
];
const names = [...new Set(planned.map((p) => p.reading.analyte))];
const existing = await client.labResult.findMany({
where: {
userId,
deletedAt: null,
takenAt: {
gte: labReadingDaySearchRange(days[0]).gte,
lt: labReadingDaySearchRange(days[days.length - 1]).lt,
},
OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }],
},
select: {
biomarkerId: true,
analyte: true,
takenAt: true,
value: true,
valueText: true,
sourceValue: true,
sourceUnit: true,
},
});
for (const e of existing) {
for (const k of readingKeys(e, tz)) existingKeys.add(k);
if (e.biomarkerId) {
for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) {
existingKeys.add(k);
}
}
}
}
const existingKeys = await liveReadingKeys(client, userId, tz, planned);
// 4. Build the rows, field by field.
const rows: Prisma.LabResultCreateManyInput[] = [];
@@ -237,72 +196,16 @@ export async function writeFhirLabReadings(args: {
continue;
}
const marker = await resolveOrMintBiomarker(
userId,
{
analyte: reading.analyte,
unit: reading.unit ?? "",
referenceLow: reading.sourceReferenceLow,
referenceHigh: reading.sourceReferenceHigh,
panel: reading.panel,
},
{ index, catalogKey: reading.catalogKey },
);
let storedValue = reading.value;
let storedLow =
reading.valueText === null ? reading.sourceReferenceLow : null;
let storedHigh =
reading.valueText === null ? reading.sourceReferenceHigh : null;
let convertedFrom: { sourceValue: number; sourceUnit: string } | null =
null;
if (reading.value !== null) {
const converted = await readingInMarkerUnit(client, {
marker,
value: reading.value,
unit: reading.unit ?? undefined,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
});
if (!converted.ok) {
failed(plan.at, "unit_mismatch");
continue;
}
storedValue = converted.value;
storedLow = converted.sourceReferenceLow;
storedHigh = converted.sourceReferenceHigh;
if (converted.converted) {
convertedFrom = {
sourceValue: converted.sourceValue,
sourceUnit: converted.sourceUnit,
};
}
const built = await buildRow(client, userId, reading, index);
if (!built.ok) {
failed(plan.at, "unit_mismatch");
continue;
}
rows.push({
userId,
biomarkerId: marker.id,
panel: reading.panel ?? marker.panel,
analyte: marker.name,
value: storedValue,
valueText: reading.valueText,
unit: marker.unit,
sourceValue: convertedFrom?.sourceValue ?? null,
sourceUnit: convertedFrom?.sourceUnit ?? null,
referenceLow: marker.lowerBound,
referenceHigh: marker.upperBound,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
sourceReferenceText: reading.sourceReferenceText,
takenAt: reading.takenAt,
source: APPLE_HEALTH_LAB_SOURCE,
externalId: reading.externalId,
noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null,
});
rows.push(built.row);
rowAt.set(reading.externalId, plan);
for (const k of [
...keys,
...readingKeys({ ...probe, biomarkerId: marker.id }, tz),
...readingKeys({ ...probe, biomarkerId: built.markerId }, tz),
]) {
seenKeys.add(k);
}
@@ -335,3 +238,128 @@ export async function writeFhirLabReadings(args: {
}
return result;
}
/** Duplicate keys of the live readings a chunk could collide with. */
async function liveReadingKeys(
client: LabWriteClient,
userId: string,
tz: string,
planned: readonly Pending[],
): Promise<Set<string>> {
const keys = new Set<string>();
if (planned.length === 0) return keys;
const days = planned.map((p) => labReadingDay(p.reading.takenAt, tz)).sort();
const ids = [
...new Set(
planned.map((p) => p.markerId).filter((id): id is string => !!id),
),
];
const names = [...new Set(planned.map((p) => p.reading.analyte))];
const existing = await client.labResult.findMany({
where: {
userId,
deletedAt: null,
takenAt: {
gte: labReadingDaySearchRange(days[0]).gte,
lt: labReadingDaySearchRange(days[days.length - 1]).lt,
},
OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }],
},
select: {
biomarkerId: true,
analyte: true,
takenAt: true,
value: true,
valueText: true,
sourceValue: true,
sourceUnit: true,
},
});
for (const e of existing) {
for (const k of readingKeys(e, tz)) keys.add(k);
// A reading filed under a marker also answers to its printed name.
if (e.biomarkerId) {
for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) keys.add(k);
}
}
return keys;
}
/**
* The row for one reading: its marker resolved or minted, the value moved
* into the marker's unit when it states another. Built field by field.
*/
async function buildRow(
client: LabWriteClient,
userId: string,
reading: MappedFhirLab,
index: MarkerIndex,
): Promise<
| { ok: true; row: Prisma.LabResultCreateManyInput; markerId: string }
| { ok: false }
> {
const marker = await resolveOrMintBiomarker(
userId,
{
analyte: reading.analyte,
unit: reading.unit ?? "",
referenceLow: reading.sourceReferenceLow,
referenceHigh: reading.sourceReferenceHigh,
panel: reading.panel,
},
{ index, catalogKey: reading.catalogKey },
);
// Like the CSV import: on a text result the window only seeds a new
// marker; the reading itself records none.
let storedValue = reading.value;
let storedLow =
reading.valueText === null ? reading.sourceReferenceLow : null;
let storedHigh =
reading.valueText === null ? reading.sourceReferenceHigh : null;
let convertedFrom: { sourceValue: number; sourceUnit: string } | null = null;
if (reading.value !== null) {
const converted = await readingInMarkerUnit(client, {
marker,
value: reading.value,
unit: reading.unit ?? undefined,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
});
if (!converted.ok) return { ok: false };
storedValue = converted.value;
storedLow = converted.sourceReferenceLow;
storedHigh = converted.sourceReferenceHigh;
if (converted.converted) {
convertedFrom = {
sourceValue: converted.sourceValue,
sourceUnit: converted.sourceUnit,
};
}
}
return {
ok: true,
markerId: marker.id,
row: {
userId,
biomarkerId: marker.id,
panel: reading.panel ?? marker.panel,
analyte: marker.name,
value: storedValue,
valueText: reading.valueText,
unit: marker.unit,
sourceValue: convertedFrom?.sourceValue ?? null,
sourceUnit: convertedFrom?.sourceUnit ?? null,
referenceLow: marker.lowerBound,
referenceHigh: marker.upperBound,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
sourceReferenceText: reading.sourceReferenceText,
takenAt: reading.takenAt,
source: APPLE_HEALTH_LAB_SOURCE,
externalId: reading.externalId,
noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null,
},
};
}
@@ -952,6 +952,44 @@ describe("streamParseExportXml — end-to-end", () => {
expect(result.totals.recordsRead).toBeGreaterThanOrEqual(7);
});
it("hands lab-result clinical records to the callback and skips the rest", async () => {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
const xmlPath = join(tmp, "export.xml");
writeFileSync(
xmlPath,
`<?xml version="1.0" encoding="UTF-8"?>
<HealthData locale="en_US">
<ClinicalRecord type="HKClinicalTypeIdentifierLabResultRecord"
identifier="obs-1" sourceName="Example Clinic"
sourceURL="https://fhir.example-clinic.test/R4" fhirVersion="4.0.1"
receivedDate="2025-03-05 10:00:00 +0100"
resourceFilePath="/clinical-records/Observation-obs-1.json"/>
<ClinicalRecord type="HKClinicalTypeIdentifierConditionRecord"
identifier="cond-1" sourceName="Example Clinic" fhirVersion="4.0.1"
resourceFilePath="/clinical-records/Condition-cond-1.json"/>
</HealthData>`,
);
const refs: unknown[] = [];
const result = await streamParseExportXml({
xmlPath,
userId: "user-1",
userTimezone: "Europe/Berlin",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
prisma: makeFakePrisma() as any,
onClinicalLabRecord: (ref) => void refs.push(ref),
});
expect(refs).toEqual([
{
path: "/clinical-records/Observation-obs-1.json",
identifier: "obs-1",
sourceURL: "https://fhir.example-clinic.test/R4",
sourceName: "Example Clinic",
fhirVersion: "4.0.1",
},
]);
expect(result.clinical.skipped).toBe(1);
});
it("re-importing the same export reports 0 inserts and N updates", async () => {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
const xmlPath = join(tmp, "export.xml");
@@ -45,6 +45,10 @@ import {
canonicalDailyTimestamp,
} from "@/lib/measurements/drain-per-sample-cumulative";
import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin";
import {
routeClinicalRecord,
type ClinicalLabReference,
} from "@/lib/apple-health/clinical-record-ref";
import {
isSameReadingAcrossSource,
MEASURED_AT_TOLERANCE_MS,
@@ -460,6 +464,9 @@ export interface StreamParseInput {
* pass, from the archive (`src/lib/apple-health/route-import.ts`).
*/
onWorkoutRoute?: (ref: { externalId: string; path: string }) => void;
/** Called once per lab-result `<ClinicalRecord>`; without it every clinical
* record is counted as skipped (`src/lib/apple-health/clinical-labs-import.ts`). */
onClinicalLabRecord?: (ref: ClinicalLabReference) => void;
}
/**
@@ -1449,7 +1456,9 @@ export async function streamParseExportXml(
if (name === "ClinicalRecord") {
recordsRead += 1;
clinical.skipped += 1;
if (!routeClinicalRecord(attrs, input.onClinicalLabRecord)) {
clinical.skipped += 1;
}
return;
}