diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 6f5698932..65e588018 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 84, - "total": 93816, + "total": 93825, "entries": { "app/medications/page-client.tsx": 842, "components/charts/mood-chart.tsx": 1166, @@ -72,7 +72,7 @@ "lib/mcp/record-search.ts": 1397, "lib/mcp/rich-reads.ts": 1471, "lib/measurements/dense-intraday-retention.ts": 1016, - "lib/measurements/import-apple-health-export.ts": 1722, + "lib/measurements/import-apple-health-export.ts": 1731, "lib/medications/scheduling/cadence.ts": 976, "lib/medications/scheduling/recurrence.ts": 1100, "lib/medications/scheduling/slot-upsert.ts": 912, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index 89d2702b0..2b56cc68d 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 335, - "total": 109025, + "total": 109021, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 275, @@ -285,7 +285,7 @@ "lib/insights/status-provider.ts :: runStatusCompletion": 215, "lib/integrations/connection-health-load.ts :: loadConnectionFacts": 229, "lib/integrations/status.ts :: recordSyncFailure": 292, - "lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 300, + "lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 292, "lib/jobs/coach-nudge.ts :: findTriggerForUser": 279, "lib/jobs/coach-nudge.ts :: runCoachNudgeTick": 292, "lib/jobs/coach-reminder-sweep.ts :: runCoachReminderSweep": 222, @@ -314,8 +314,8 @@ "lib/measurements/dense-intraday-retention.ts :: runDenseIntradayRetention > walkSource > writeDay > foldOnce": 212, "lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative": 344, "lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative > writeDay": 205, - "lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1245, - "lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 323, + "lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1247, + "lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 325, "lib/medications/efficacy/build-efficacy.ts :: buildMedicationEfficacy": 268, "lib/medications/intake-slot-dedup.ts :: dedupeUserIntakeSlots": 319, "lib/medications/list-read.ts :: buildMedicationsList": 310, diff --git a/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts b/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts index dcbada60d..8376385a7 100644 --- a/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts +++ b/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts @@ -198,7 +198,7 @@ describe("POST /api/admin/import-apple-health-export", () => { expect(mocks.importJobCreate).toHaveBeenCalledTimes(1); expect(mocks.importJobCreate).toHaveBeenCalledWith( expect.objectContaining({ - data: expect.objectContaining({ parserRevision: 3 }), + data: expect.objectContaining({ parserRevision: 4 }), }), ); expect(mocks.bossSend).toHaveBeenCalledWith( @@ -217,7 +217,7 @@ describe("POST /api/admin/import-apple-health-export", () => { expect.objectContaining({ where: expect.objectContaining({ status: { not: "failed" }, - parserRevision: 3, + parserRevision: 4, }), }), ); @@ -241,7 +241,7 @@ describe("POST /api/admin/import-apple-health-export", () => { expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH); expect(mocks.importJobFindFirst).toHaveBeenCalledWith( expect.objectContaining({ - where: expect.objectContaining({ parserRevision: 3 }), + where: expect.objectContaining({ parserRevision: 4 }), }), ); }); diff --git a/src/app/api/import/apple-health-export/[jobId]/status/__tests__/route.test.ts b/src/app/api/import/apple-health-export/[jobId]/status/__tests__/route.test.ts index 68b53197d..892758aee 100644 --- a/src/app/api/import/apple-health-export/[jobId]/status/__tests__/route.test.ts +++ b/src/app/api/import/apple-health-export/[jobId]/status/__tests__/route.test.ts @@ -93,6 +93,54 @@ describe("GET /api/import/apple-health-export/[jobId]/status", () => { expect(body.data.status).toBe("parsing"); }); + it("bounds the lab counts and drops anything else in the block", async () => { + vi.mocked(getSession).mockResolvedValue(OWNER_SESSION as never); + mocks.importJobFindUnique.mockResolvedValue( + makeRow({ + status: "done", + result: { + labs: { + discovered: 3, + imported: 2, + duplicate: -1, + skipped: "x", + failed: 1.5, + reasons: { leaked: "content" }, + }, + }, + }), + ); + const req = new NextRequest( + "http://localhost/api/import/apple-health-export/ij-1/status", + ); + const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) }); + const body = (await res.json()) as { + data: { result: { labs: Record } }; + }; + expect(body.data.result.labs).toEqual({ + discovered: 3, + imported: 2, + duplicate: 0, + skipped: 0, + failed: 0, + }); + }); + + it("reports zeroed lab counts for a job from before labs were read", async () => { + vi.mocked(getSession).mockResolvedValue(OWNER_SESSION as never); + mocks.importJobFindUnique.mockResolvedValue( + makeRow({ status: "done", result: { totals: { recordsRead: 1 } } }), + ); + const req = new NextRequest( + "http://localhost/api/import/apple-health-export/ij-1/status", + ); + const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) }); + const body = (await res.json()) as { + data: { result: { labs: Record } }; + }; + expect(body.data.result.labs.imported).toBe(0); + }); + it("returns 404 to a non-owner non-admin user", async () => { vi.mocked(getSession).mockResolvedValue(OTHER_SESSION as never); mocks.importJobFindUnique.mockResolvedValue(makeRow()); diff --git a/src/app/api/import/apple-health-export/[jobId]/status/route.ts b/src/app/api/import/apple-health-export/[jobId]/status/route.ts index 4cb8c2276..3614a6041 100644 --- a/src/app/api/import/apple-health-export/[jobId]/status/route.ts +++ b/src/app/api/import/apple-health-export/[jobId]/status/route.ts @@ -46,10 +46,12 @@ function privacySafeResult( value: Record | null, ): Record | null { if (!value) return null; - const ecg = - typeof value.ecg === "object" && value.ecg !== null - ? (value.ecg as Record) + const block = (key: string): Record => + typeof value[key] === "object" && value[key] !== null + ? (value[key] as Record) : {}; + const ecg = block("ecg"); + const labs = block("labs"); return { ...value, ecg: { @@ -59,6 +61,13 @@ function privacySafeResult( skipped: boundedCount(ecg.skipped), failed: boundedCount(ecg.failed), }, + labs: { + discovered: boundedCount(labs.discovered), + imported: boundedCount(labs.imported), + duplicate: boundedCount(labs.duplicate), + skipped: boundedCount(labs.skipped), + failed: boundedCount(labs.failed), + }, }; } diff --git a/src/app/api/import/apple-health-export/__tests__/route.test.ts b/src/app/api/import/apple-health-export/__tests__/route.test.ts index cc26e6570..b848a33e5 100644 --- a/src/app/api/import/apple-health-export/__tests__/route.test.ts +++ b/src/app/api/import/apple-health-export/__tests__/route.test.ts @@ -200,7 +200,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue expect(mocks.importJobCreate).toHaveBeenCalledTimes(1); expect(mocks.importJobCreate).toHaveBeenCalledWith( expect.objectContaining({ - data: expect.objectContaining({ parserRevision: 3 }), + data: expect.objectContaining({ parserRevision: 4 }), }), ); expect(mocks.bossSend).toHaveBeenCalledWith( @@ -217,7 +217,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue expect.objectContaining({ where: expect.objectContaining({ status: { not: "failed" }, - parserRevision: 3, + parserRevision: 4, }), }), ); @@ -243,7 +243,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH); expect(mocks.importJobFindFirst).toHaveBeenCalledWith( expect.objectContaining({ - where: expect.objectContaining({ parserRevision: 3 }), + where: expect.objectContaining({ parserRevision: 4 }), }), ); }); diff --git a/src/lib/apple-health/__tests__/clinical-labs-import.test.ts b/src/lib/apple-health/__tests__/clinical-labs-import.test.ts new file mode 100644 index 000000000..6842fa600 --- /dev/null +++ b/src/lib/apple-health/__tests__/clinical-labs-import.test.ts @@ -0,0 +1,225 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { strToU8, zipSync } from "fflate"; + +const mocks = vi.hoisted(() => ({ + write: vi.fn(), + after: vi.fn(), + annotate: vi.fn(), +})); +vi.mock("@/lib/labs/fhir-import/write", () => ({ + writeFhirLabReadings: mocks.write, +})); +vi.mock("@/lib/labs/fhir-import/after-write", () => ({ + afterLabReadingsWritten: mocks.after, +})); +vi.mock("@/lib/logging/context", () => ({ annotate: mocks.annotate })); + +import { importAppleHealthClinicalLabs } from "../clinical-labs-import"; +import type { ClinicalLabReference } from "../clinical-record-ref"; +import { zip64Archive } from "./helpers/zip64"; + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); +}); +beforeEach(() => { + vi.clearAllMocks(); + mocks.write.mockImplementation( + async ({ readings }: { readings: unknown[] }) => ({ + statuses: readings.map(() => ({ status: "inserted" })), + inserted: readings.length, + duplicate: 0, + failed: 0, + newestTakenAt: new Date("2025-01-01T00:00:00Z"), + }), + ); +}); + +const ROOT = "apple_health_export/clinical-records/"; + +function observation(id: string, extra: Record = {}) { + return JSON.stringify({ + resourceType: "Observation", + id, + status: "final", + category: [{ coding: [{ code: "laboratory" }] }], + code: { text: `Analyte ${id}` }, + effectiveDateTime: "2025-01-01T08:00:00Z", + valueQuantity: { value: 1, unit: "mmol/L" }, + ...extra, + }); +} + +function archive(files: Record, zip64 = false): string { + const dir = mkdtempSync(join(tmpdir(), "healthlog-clinical-")); + dirs.push(dir); + const path = join(dir, "export.zip"); + writeFileSync( + path, + zip64 + ? zip64Archive(files) + : zipSync( + Object.fromEntries( + Object.entries(files).map(([name, body]) => [name, strToU8(body)]), + ), + ), + ); + return path; +} + +function ref( + path: string | null, + identifier: string | null = null, +): ClinicalLabReference { + return { + path, + identifier, + sourceURL: "https://fhir.example.test", + sourceName: "Example", + fhirVersion: "4.0.1", + }; +} + +function run(path: string, references: ClinicalLabReference[], limits = {}) { + return importAppleHealthClinicalLabs({ + archivePath: path, + userId: "user-1", + tz: "Europe/Berlin", + references, + limits: { + maxMembers: 50, + maxClinicalMembers: 20, + maxMemberBytes: 64 * 1024, + maxTotalClinicalBytes: 1024 * 1024, + maxCompressionRatio: 200, + ...limits, + }, + }); +} + +describe("importAppleHealthClinicalLabs", () => { + it("reads only referenced members and writes their readings with panels", async () => { + const path = archive({ + "apple_health_export/export.xml": "", + [`${ROOT}Observation-a.json`]: observation("a"), + [`${ROOT}Observation-panel.json`]: observation("panel", { + code: { text: "Electrolytes" }, + valueQuantity: undefined, + hasMember: [{ reference: "Observation/a" }], + }), + [`${ROOT}Condition-c.json`]: '{"resourceType":"Condition"}', + }); + const counts = await run(path, [ + ref("/clinical-records/Observation-a.json", "a"), + ref("/clinical-records/Observation-panel.json", "panel"), + ]); + expect(counts).toEqual({ + discovered: 2, + imported: 1, + duplicate: 0, + skipped: 1, + failed: 0, + }); + const [{ readings, tz, userId }] = mocks.write.mock.calls[0]; + expect(userId).toBe("user-1"); + expect(tz).toBe("Europe/Berlin"); + expect(readings).toHaveLength(1); + expect(readings[0]).toMatchObject({ + analyte: "Analyte a", + panel: "Electrolytes", + }); + expect(mocks.after).toHaveBeenCalledWith( + expect.objectContaining({ userId: "user-1", inserted: 1 }), + ); + }); + + it("works on a ZIP64 archive", async () => { + const path = archive( + { [`${ROOT}Observation-a.json`]: observation("a") }, + true, + ); + const counts = await run(path, [ + ref("/clinical-records/Observation-a.json"), + ]); + expect(counts.imported).toBe(1); + }); + + it("counts a missing member, malformed JSON and an unsafe path as failed", async () => { + const path = archive({ [`${ROOT}Observation-bad.json`]: "{not json" }); + const counts = await run(path, [ + ref("/clinical-records/Observation-bad.json"), + ref("/clinical-records/Observation-gone.json"), + ref("/clinical-records/../export.xml"), + ]); + expect(counts).toMatchObject({ discovered: 3, imported: 0, failed: 3 }); + expect(mocks.write).not.toHaveBeenCalled(); + }); + + it("finds a reference without a path by the resource id", async () => { + const path = archive({ + [`${ROOT}one.json`]: observation("a"), + [`${ROOT}two.json`]: observation("unreferenced"), + }); + const counts = await run(path, [ref(null, "a")]); + expect(counts).toMatchObject({ discovered: 1, imported: 1, failed: 0 }); + expect(mocks.write.mock.calls[0][0].readings).toHaveLength(1); + }); + + it("refuses references past the clinical member ceiling", async () => { + const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") }); + const counts = await run( + path, + [ + ref("/clinical-records/Observation-a.json"), + ref("/clinical-records/Observation-b.json"), + ], + { maxClinicalMembers: 1 }, + ); + expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 }); + }); + + it("fails soft when the archive breaks a limit", async () => { + const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") }); + const counts = await run( + path, + [ref("/clinical-records/Observation-a.json")], + { + maxMemberBytes: 10, + }, + ); + expect(counts).toMatchObject({ discovered: 1, imported: 0, failed: 1 }); + }); + + it("annotates counts and closed reasons only", async () => { + const path = archive({ + [`${ROOT}Observation-a.json`]: observation("a", { + code: { text: "Secret analyte name" }, + status: "preliminary", + }), + }); + await run(path, [ref("/clinical-records/Observation-a.json")]); + const serialised = JSON.stringify(mocks.annotate.mock.calls); + expect(serialised).not.toContain("Secret analyte name"); + expect(serialised).not.toContain("fhir.example.test"); + expect(mocks.annotate).toHaveBeenCalledWith( + expect.objectContaining({ + action: { name: "apple_health.import.clinical_labs" }, + meta: expect.objectContaining({ + skipped: 1, + reasons: { not_final: 1 }, + }), + }), + ); + }); + + it("does nothing for no references", async () => { + const counts = await run("/nonexistent.zip", []); + expect(counts.discovered).toBe(0); + expect(mocks.write).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/apple-health/clinical-labs-import.ts b/src/lib/apple-health/clinical-labs-import.ts new file mode 100644 index 000000000..1ae8aea72 --- /dev/null +++ b/src/lib/apple-health/clinical-labs-import.ts @@ -0,0 +1,220 @@ +/** + * Lab results from Apple Health's Health Records, out of an `export.zip` + * (#1400). + * + * `export.xml` lists each Health Records resource as a `` + * whose `resourceFilePath` names a FHIR JSON file under `clinical-records/`. + * The XML pass hands the lab-result references here + * (`routeClinicalRecord`); this module then reads the referenced members one + * at a time, maps each with the shared FHIR mapper, gives members their + * panel's name, and writes through the shared writer, so a result lands the + * same way it would from the iOS app (`POST /api/labs/fhir-batch`). + * + * A reference with no path (an export from a Health app that did not write + * one) is matched by the resource's own id among the clinical files instead. + * + * Memory: one member is held at a time, capped by the archive reader; what is + * kept across members is the mapped readings (a few hundred bytes each) and + * the panel headers' member lists. + * + * A failure here never fails the import: the measurements are committed. A + * missing, oversized or malformed file is counted as failed. Nothing about a + * resource reaches the result, the logs or a wide event: only counts and + * closed reasons. + */ +import type { PrismaClient } from "@/generated/prisma/client"; +import { + clinicalMemberNameFromReference, + isClinicalRecordJson, + streamAppleHealthClinicalMembers, + type AppleHealthClinicalArchiveLimits, +} from "@/lib/apple-health/archive-stream"; +import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref"; +import { afterLabReadingsWritten } from "@/lib/labs/fhir-import/after-write"; +import { + assignPanels, + mapFhirLabRecord, + type FhirLabMapping, +} from "@/lib/labs/fhir-import/map-observation"; +import { + emptyFhirLabCounts, + type FhirLabCounts, +} from "@/lib/labs/fhir-import/types"; +import { writeFhirLabReadings } from "@/lib/labs/fhir-import/write"; +import { annotate } from "@/lib/logging/context"; +import type { ImportJobResult } from "@/lib/measurements/import-apple-health-export"; + +/** The terminal result of an Apple Health import job, labs included. */ +export type AppleHealthImportJobResult = ImportJobResult & { + labs: FhirLabCounts; +}; + +/** Per-archive ceilings for the Health Records members. */ +export const APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS: AppleHealthClinicalArchiveLimits = + { + maxMembers: 200_000, + maxClinicalMembers: 50_000, + // A single Observation is a few KB; 2 MiB leaves room for a long note. + maxMemberBytes: 2 * 1024 * 1024, + maxTotalClinicalBytes: 256 * 1024 * 1024, + maxCompressionRatio: 200, + }; + +async function readJson(stream: AsyncIterable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk))); + } + const text = Buffer.concat(chunks).toString("utf8"); + for (const chunk of chunks) chunk.fill(0); + return JSON.parse(text) as unknown; +} + +function resourceId(resource: unknown): string | null { + if (typeof resource !== "object" || resource === null) return null; + const id = (resource as { id?: unknown }).id; + return typeof id === "string" && id.trim() !== "" ? id.trim() : null; +} + +export async function importAppleHealthClinicalLabs(input: { + archivePath: string; + userId: string; + tz: string; + references: readonly ClinicalLabReference[]; + prisma?: Pick; + limits?: AppleHealthClinicalArchiveLimits; +}): Promise { + const counts = emptyFhirLabCounts(); + const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS; + counts.discovered = input.references.length; + if (input.references.length === 0) return counts; + + const tally: Record = {}; + const note = (reason: string) => { + tally[reason] = (tally[reason] ?? 0) + 1; + }; + + // More references than the archive may hold clinical files: the overflow + // is refused, the rest is read. + const usable = input.references.slice(0, limits.maxClinicalMembers); + const overflow = input.references.length - usable.length; + counts.failed += overflow; + if (overflow > 0) tally.over_limit = overflow; + + const byMember = new Map(); + const byId = new Map(); + for (const ref of usable) { + const member = ref.path ? clinicalMemberNameFromReference(ref.path) : null; + if (member && !byMember.has(member)) byMember.set(member, ref); + else if (!ref.path && ref.identifier && !byId.has(ref.identifier)) { + byId.set(ref.identifier, ref); + } else { + // An unsafe path, or a second reference to one file. + counts.failed += 1; + note("bad_reference"); + } + } + const expected = byMember.size + byId.size; + let handled = 0; + + const mapped: FhirLabMapping[] = []; + try { + for await (const member of streamAppleHealthClinicalMembers({ + archivePath: input.archivePath, + limits, + wanted: (name) => + byMember.has(name) || (byId.size > 0 && isClinicalRecordJson(name)), + })) { + const byPath = byMember.get(member.name); + let resource: unknown; + try { + resource = await readJson(member.stream); + } catch { + if (byPath) { + handled += 1; + counts.failed += 1; + note("malformed_json"); + } + continue; + } + let ref = byPath; + if (!ref) { + const id = resourceId(resource); + ref = id ? byId.get(id) : undefined; + if (!ref || !id) continue; + byId.delete(id); + } + handled += 1; + const result = mapFhirLabRecord({ + resource, + fhirVersion: ref.fhirVersion, + identifier: ref.identifier, + sourceURL: ref.sourceURL, + sourceName: ref.sourceName, + }); + resource = null; + if (result.kind === "reading") { + mapped.push(result); + } else if (result.kind === "skip") { + counts.skipped += 1; + note(result.reason); + if (result.reason === "panel_header") mapped.push(result); + } else { + counts.failed += 1; + note(result.reason); + } + } + } catch { + // The archive itself is unreadable for this pass (a limit hit in the + // directory, an inflate failure): the files not yet read are counted + // below as missing. What was read is still written. + note("archive_unreadable"); + } + const missing = Math.max(0, expected - handled); + counts.failed += missing; + if (missing > 0) tally.missing_member = (tally.missing_member ?? 0) + missing; + + assignPanels(mapped); + const readings = mapped.flatMap((m) => + m.kind === "reading" ? [m.reading] : [], + ); + mapped.length = 0; + if (readings.length > 0) { + try { + const written = await writeFhirLabReadings({ + userId: input.userId, + tz: input.tz, + readings, + prisma: input.prisma, + }); + counts.imported += written.inserted; + counts.duplicate += written.duplicate; + counts.failed += written.failed; + for (const status of written.statuses) { + if (status.status === "failed") note(status.reason); + } + afterLabReadingsWritten({ + userId: input.userId, + inserted: written.inserted, + newestTakenAt: written.newestTakenAt, + source: "apple_health_export", + }); + } catch { + counts.failed += readings.length; + note("write_failed"); + } + } + + annotate({ + action: { name: "apple_health.import.clinical_labs" }, + meta: { + discovered: counts.discovered, + imported: counts.imported, + duplicate: counts.duplicate, + skipped: counts.skipped, + failed: counts.failed, + reasons: tally, + }, + }); + return counts; +} diff --git a/src/lib/apple-health/clinical-record-ref.ts b/src/lib/apple-health/clinical-record-ref.ts new file mode 100644 index 000000000..f8326d7c5 --- /dev/null +++ b/src/lib/apple-health/clinical-record-ref.ts @@ -0,0 +1,61 @@ +/** + * A `` in `export.xml`, as a reference to the lab result it + * describes (#1400). + * + * Apple writes one element per Health Records resource: + * + * + * + * The resource itself is the JSON file the path names. Only lab results are + * handed on; every other clinical type (conditions, medications, vital signs, + * immunisations) is left to the caller's skipped count. The attributes are + * capped here so a hostile export cannot make the parser hold long strings. + */ + +export const HK_LAB_RESULT_RECORD = "HKClinicalTypeIdentifierLabResultRecord"; + +export interface ClinicalLabReference { + /** `resourceFilePath`, from the export root. Null when Apple wrote none. */ + path: string | null; + identifier: string | null; + sourceURL: string | null; + sourceName: string | null; + fhirVersion: string | null; +} + +const CAPS = { + path: 512, + identifier: 256, + sourceURL: 2048, + sourceName: 200, + fhirVersion: 16, +} as const; + +function capped(raw: string | undefined, max: number): string | null { + if (raw === undefined) return null; + const trimmed = raw.trim(); + if (trimmed === "" || trimmed.length > max) return null; + return trimmed; +} + +/** + * Hand a lab-result record to `onLab` and return true; return false for any + * other record, or when there is no callback, so the caller counts it as + * skipped. + */ +export function routeClinicalRecord( + attrs: Record, + onLab?: (ref: ClinicalLabReference) => void, +): boolean { + if (!onLab || attrs.type !== HK_LAB_RESULT_RECORD) return false; + onLab({ + path: capped(attrs.resourceFilePath, CAPS.path), + identifier: capped(attrs.identifier, CAPS.identifier), + sourceURL: capped(attrs.sourceURL, CAPS.sourceURL), + sourceName: capped(attrs.sourceName, CAPS.sourceName), + fhirVersion: capped(attrs.fhirVersion, CAPS.fhirVersion), + }); + return true; +} diff --git a/src/lib/apple-health/ecg-import.ts b/src/lib/apple-health/ecg-import.ts index 4e512b404..cfad6062e 100644 --- a/src/lib/apple-health/ecg-import.ts +++ b/src/lib/apple-health/ecg-import.ts @@ -17,7 +17,11 @@ */ import { createHash } from "node:crypto"; import type { PrismaClient } from "@/generated/prisma/client"; -import type { NormalizedAppleHealthEcg } from "@/lib/apple-health/ecg-csv"; +import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream"; +import { + parseAppleHealthEcgCsv, + type NormalizedAppleHealthEcg, +} from "@/lib/apple-health/ecg-csv"; import { persistEcgRecording } from "@/lib/ecg/persist-recording"; export type AppleHealthEcgImportOutcome = "imported" | "updated" | "skipped"; @@ -68,3 +72,72 @@ export async function importAppleHealthEcg(input: { if (result.outcome === "duplicate") return "skipped"; return "updated"; } + +const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = { + maxMembers: 20_000, + maxEcgMembers: 2_000, + maxMemberBytes: 16 * 1024 * 1024, + maxTotalEcgBytes: 512 * 1024 * 1024, + maxCompressionRatio: 200, +} as const; +const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000; + +export interface AppleHealthEcgCounts { + discovered: number; + imported: number; + updated: number; + skipped: number; + failed: number; +} + +/** + * Every ECG recording in an export archive. Intentionally fail-soft: the + * canonical export.xml pass has already completed, and malformed, oversized + * or unwritable ECG members affect only bounded scalar counters in the + * terminal result. `memberNames` are the archive's other members, which the + * discovered count is taken from. + */ +export async function importAppleHealthArchiveEcgs(input: { + archivePath: string; + userId: string; + memberNames: readonly string[]; + prisma: Pick; +}): Promise { + const ecg: AppleHealthEcgCounts = { + discovered: 0, + imported: 0, + updated: 0, + skipped: 0, + failed: 0, + }; + ecg.discovered = input.memberNames.filter((name) => { + const lower = name.toLowerCase(); + return lower.includes("electrocardiograms/") && lower.endsWith(".csv"); + }).length; + try { + for await (const member of streamAppleHealthEcgMembers({ + archivePath: input.archivePath, + limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS, + })) { + try { + const parsed = await parseAppleHealthEcgCsv({ + memberName: member.name, + stream: member.stream, + maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES, + }); + const outcome = await importAppleHealthEcg({ + userId: input.userId, + ecg: parsed, + prisma: input.prisma, + }); + ecg[outcome] += 1; + } catch { + ecg.failed += 1; + } + } + } catch { + const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed; + ecg.failed += Math.max(0, ecg.discovered - completed); + } + return ecg; +} diff --git a/src/lib/jobs/__tests__/apple-health-import-ecg.test.ts b/src/lib/jobs/__tests__/apple-health-import-ecg.test.ts index 5c0cac646..b0db870db 100644 --- a/src/lib/jobs/__tests__/apple-health-import-ecg.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-ecg.test.ts @@ -8,6 +8,10 @@ const workerSource = readFileSync( join(process.cwd(), "src/lib/jobs/apple-health-import-worker.ts"), "utf8", ); +const ecgSource = readFileSync( + join(process.cwd(), "src/lib/apple-health/ecg-import.ts"), + "utf8", +); const parserSource = readFileSync( join(process.cwd(), "src/lib/measurements/import-apple-health-export.ts"), "utf8", @@ -19,18 +23,22 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => { }); it("runs bounded ECG archive ingestion beside the established XML parser", () => { - expect(workerSource).toMatch(/streamAppleHealthEcgMembers/); - expect(workerSource).toMatch(/parseAppleHealthEcgCsv/); - expect(workerSource).toMatch(/importAppleHealthEcg/); expect(workerSource).toMatch(/streamParseExportXml/); + expect(workerSource).toMatch(/importAppleHealthArchiveEcgs/); + expect(ecgSource).toMatch(/streamAppleHealthEcgMembers/); + expect(ecgSource).toMatch(/parseAppleHealthEcgCsv/); + expect(ecgSource).toMatch(/importAppleHealthEcg\(/); }); it("catches per-ECG failures without routing the valid XML result to the fatal job catch", () => { - expect(workerSource).toMatch( + expect(ecgSource).toMatch( /for await[\s\S]*streamAppleHealthEcgMembers[\s\S]*try[\s\S]*parseAppleHealthEcgCsv[\s\S]*catch[\s\S]*ecg\.failed/, ); + expect(ecgSource).toMatch( + /ecg:\s*AppleHealthEcgCounts\s*=\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/, + ); expect(workerSource).toMatch( - /result[\s\S]*ecg:\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/, + /result:\s*AppleHealthImportJobResult\s*=\s*\{[\s\S]*ecg,[\s\S]*labs:/, ); }); @@ -45,7 +53,9 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => { it("does not log ECG filenames, source labels, samples, or raw parser errors", () => { const loggingCalls = - workerSource.match(/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g) ?? []; + (workerSource + ecgSource).match( + /console\.(?:log|warn|error)\([\s\S]{0,250}\)/g, + ) ?? []; const ecgLogging = loggingCalls.filter((call) => /ecg|electrocardiogram/i.test(call), ); @@ -56,7 +66,7 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => { }); it("does not claim to diagnose or reclassify a waveform", () => { - expect(workerSource).not.toMatch( + expect(workerSource + ecgSource).not.toMatch( /(?:diagnose|infer|classify)Ecg|analy[sz]eWaveform|detectA(?:fib|trial)/i, ); }); diff --git a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts index 1d645df5c..dfec8c2cc 100644 --- a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts @@ -80,8 +80,8 @@ describe("apple health import — pg-boss send policy", () => { }); describe("apple health import — parser revision boundary", () => { - it("isolates revision-3 sends from legacy workers", () => { - expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(3); + it("isolates revision-4 sends from legacy workers", () => { + expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(4); expect(APPLE_HEALTH_IMPORT_V2_QUEUE).toBe("apple-health-import-v2"); expect(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).toBe("apple-health-import"); }); diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index 15435665c..c8a2986ec 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -30,9 +30,12 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { prisma, toJson } from "@/lib/db"; import type { Job } from "pg-boss"; -import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream"; -import { parseAppleHealthEcgCsv } from "@/lib/apple-health/ecg-csv"; -import { importAppleHealthEcg } from "@/lib/apple-health/ecg-import"; +import { + importAppleHealthClinicalLabs, + type AppleHealthImportJobResult, +} from "@/lib/apple-health/clinical-labs-import"; +import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref"; +import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import"; import { extractExportXml } from "@/lib/import/unzip-export-xml"; import { annotate } from "@/lib/logging/context"; import { @@ -45,7 +48,6 @@ import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { streamParseExportXml, type ImportJobProgress, - type ImportJobResult, } from "@/lib/measurements/import-apple-health-export"; import { withBackgroundEvent } from "@/lib/logging/background"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; @@ -125,17 +127,12 @@ export async function handleImportJobReconcileTick( ); } -/** Parser semantics carried by every newly-created ImportJob. */ -export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 3; - -const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = { - maxMembers: 20_000, - maxEcgMembers: 2_000, - maxMemberBytes: 16 * 1024 * 1024, - maxTotalEcgBytes: 512 * 1024 * 1024, - maxCompressionRatio: 200, -} as const; -const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000; +/** + * Parser semantics carried by every newly-created ImportJob. 4: lab results + * from Health Records are read (#1400), so an archive imported before is + * worth uploading again. + */ +export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 4; /** * (issue #775) Memory preflight for the parse. The streaming pipeline @@ -404,6 +401,7 @@ export async function handleAppleHealthImport( // GPS tracks are read from the archive only when the person keeps them. const keepRoutes = userRow?.storeWorkoutRoutes === true; const routeReferences: WorkoutRouteReference[] = []; + const labReferences: ClinicalLabReference[] = []; const userTimezone = userRow?.timezone && userRow.timezone.length > 0 ? userRow.timezone @@ -439,80 +437,58 @@ export async function handleAppleHealthImport( elapsedMs: 0, }); - const result: ImportJobResult = { - ...(await streamParseExportXml({ - xmlPath: unzip.xmlPath, - userId, - userTimezone, - prisma, - ...(keepRoutes - ? { onWorkoutRoute: (ref) => void routeReferences.push(ref) } - : {}), - onProgress: async (snapshot) => { - // The phase label here is what the polling endpoint surfaces; - // map "parsing" / "upserting" through verbatim. - await prisma.importJob.update({ - where: { id: importJobId }, - data: { - status: snapshot.currentPhase, - progress: toJson(snapshot), - }, - }); - }, - // The instant the Health app stamped on the archive. Persisted as - // soon as the parser reads it, so the status endpoint answers with - // it for the rest of the run and a job that later fails still - // records which export it was working from. Best-effort: the - // stamp is provenance, and losing it must not fail an import. - onExportDate: async (exportedAt) => { - await prisma.importJob - .update({ where: { id: importJobId }, data: { exportedAt } }) - .catch(caughtAs("apple_health.import.export_date_failed")); - }, - })), - ecg: { + const parsed = await streamParseExportXml({ + xmlPath: unzip.xmlPath, + userId, + userTimezone, + prisma, + ...(keepRoutes + ? { onWorkoutRoute: (ref) => void routeReferences.push(ref) } + : {}), + onClinicalLabRecord: (ref) => void labReferences.push(ref), + onProgress: async (snapshot) => { + // The phase label here is what the polling endpoint surfaces; + // map "parsing" / "upserting" through verbatim. + await prisma.importJob.update({ + where: { id: importJobId }, + data: { + status: snapshot.currentPhase, + progress: toJson(snapshot), + }, + }); + }, + // The instant the Health app stamped on the archive. Persisted as + // soon as the parser reads it, so the status endpoint answers with + // it for the rest of the run and a job that later fails still + // records which export it was working from. Best-effort: the + // stamp is provenance, and losing it must not fail an import. + onExportDate: async (exportedAt) => { + await prisma.importJob + .update({ where: { id: importJobId }, data: { exportedAt } }) + .catch(caughtAs("apple_health.import.export_date_failed")); + }, + }); + + // The auxiliary passes below are fail-soft: the export.xml rows are + // committed, and a member that cannot be read is only counted. + const ecg = await importAppleHealthArchiveEcgs({ + archivePath: uploadPath, + userId, + memberNames: unzip.otherMembers.map((member) => member.name), + prisma, + }); + const result: AppleHealthImportJobResult = { + ...parsed, + ecg, + labs: { discovered: 0, imported: 0, - updated: 0, + duplicate: 0, skipped: 0, failed: 0, }, }; - // Auxiliary ECGs are intentionally fail-soft. The canonical export.xml - // transaction has already completed; malformed, oversized, or unwritable - // ECG members affect only bounded scalar counters in the terminal result. - const ecg = result.ecg; - ecg.discovered = unzip.otherMembers.filter((member) => { - const lower = member.name.toLowerCase(); - return lower.includes("electrocardiograms/") && lower.endsWith(".csv"); - }).length; - try { - for await (const member of streamAppleHealthEcgMembers({ - archivePath: uploadPath, - limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS, - })) { - try { - const parsed = await parseAppleHealthEcgCsv({ - memberName: member.name, - stream: member.stream, - maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES, - }); - const outcome = await importAppleHealthEcg({ - userId, - ecg: parsed, - prisma, - }); - ecg[outcome] += 1; - } catch { - ecg.failed += 1; - } - } - } catch { - const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed; - ecg.failed += Math.max(0, ecg.discovered - completed); - } - // Workout GPS tracks, on opt-in only. Fail-soft like the ECGs: the // workouts are committed, a track that cannot be read is only counted. // Read one member at a time from the archive; nothing about a track @@ -538,6 +514,19 @@ export async function handleAppleHealthImport( routeReferences.length = 0; } + // Lab results from Health Records, one FHIR file at a time; only counts + // reach the result. + if (labReferences.length > 0) { + result.labs = await importAppleHealthClinicalLabs({ + archivePath: uploadPath, + userId, + tz: userTimezone, + references: labReferences, + prisma, + }); + labReferences.length = 0; + } + // Done. Persist the terminal envelope. await prisma.importJob.update({ where: { id: importJobId }, diff --git a/src/lib/labs/fhir-import/write.ts b/src/lib/labs/fhir-import/write.ts index 5f3151569..0a4e440c1 100644 --- a/src/lib/labs/fhir-import/write.ts +++ b/src/lib/labs/fhir-import/write.ts @@ -70,7 +70,6 @@ interface Pending { at: number; reading: MappedFhirLab; markerId: string | null; - keys: string[]; } export async function writeFhirLabReadings(args: { @@ -168,51 +167,11 @@ export async function writeFhirLabReadings(args: { at, reading, markerId: hit?.marker.id ?? null, - keys: [], }); } // 3. The reading itself, against the live record. - const existingKeys = new Set(); - if (planned.length > 0) { - const days = planned - .map((p) => labReadingDay(p.reading.takenAt, tz)) - .sort(); - const ids = [ - ...new Set( - planned.map((p) => p.markerId).filter((id): id is string => !!id), - ), - ]; - const names = [...new Set(planned.map((p) => p.reading.analyte))]; - const existing = await client.labResult.findMany({ - where: { - userId, - deletedAt: null, - takenAt: { - gte: labReadingDaySearchRange(days[0]).gte, - lt: labReadingDaySearchRange(days[days.length - 1]).lt, - }, - OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }], - }, - select: { - biomarkerId: true, - analyte: true, - takenAt: true, - value: true, - valueText: true, - sourceValue: true, - sourceUnit: true, - }, - }); - for (const e of existing) { - for (const k of readingKeys(e, tz)) existingKeys.add(k); - if (e.biomarkerId) { - for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) { - existingKeys.add(k); - } - } - } - } + const existingKeys = await liveReadingKeys(client, userId, tz, planned); // 4. Build the rows, field by field. const rows: Prisma.LabResultCreateManyInput[] = []; @@ -237,72 +196,16 @@ export async function writeFhirLabReadings(args: { continue; } - const marker = await resolveOrMintBiomarker( - userId, - { - analyte: reading.analyte, - unit: reading.unit ?? "", - referenceLow: reading.sourceReferenceLow, - referenceHigh: reading.sourceReferenceHigh, - panel: reading.panel, - }, - { index, catalogKey: reading.catalogKey }, - ); - - let storedValue = reading.value; - let storedLow = - reading.valueText === null ? reading.sourceReferenceLow : null; - let storedHigh = - reading.valueText === null ? reading.sourceReferenceHigh : null; - let convertedFrom: { sourceValue: number; sourceUnit: string } | null = - null; - if (reading.value !== null) { - const converted = await readingInMarkerUnit(client, { - marker, - value: reading.value, - unit: reading.unit ?? undefined, - sourceReferenceLow: storedLow, - sourceReferenceHigh: storedHigh, - }); - if (!converted.ok) { - failed(plan.at, "unit_mismatch"); - continue; - } - storedValue = converted.value; - storedLow = converted.sourceReferenceLow; - storedHigh = converted.sourceReferenceHigh; - if (converted.converted) { - convertedFrom = { - sourceValue: converted.sourceValue, - sourceUnit: converted.sourceUnit, - }; - } + const built = await buildRow(client, userId, reading, index); + if (!built.ok) { + failed(plan.at, "unit_mismatch"); + continue; } - - rows.push({ - userId, - biomarkerId: marker.id, - panel: reading.panel ?? marker.panel, - analyte: marker.name, - value: storedValue, - valueText: reading.valueText, - unit: marker.unit, - sourceValue: convertedFrom?.sourceValue ?? null, - sourceUnit: convertedFrom?.sourceUnit ?? null, - referenceLow: marker.lowerBound, - referenceHigh: marker.upperBound, - sourceReferenceLow: storedLow, - sourceReferenceHigh: storedHigh, - sourceReferenceText: reading.sourceReferenceText, - takenAt: reading.takenAt, - source: APPLE_HEALTH_LAB_SOURCE, - externalId: reading.externalId, - noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null, - }); + rows.push(built.row); rowAt.set(reading.externalId, plan); for (const k of [ ...keys, - ...readingKeys({ ...probe, biomarkerId: marker.id }, tz), + ...readingKeys({ ...probe, biomarkerId: built.markerId }, tz), ]) { seenKeys.add(k); } @@ -335,3 +238,128 @@ export async function writeFhirLabReadings(args: { } return result; } + +/** Duplicate keys of the live readings a chunk could collide with. */ +async function liveReadingKeys( + client: LabWriteClient, + userId: string, + tz: string, + planned: readonly Pending[], +): Promise> { + const keys = new Set(); + if (planned.length === 0) return keys; + const days = planned.map((p) => labReadingDay(p.reading.takenAt, tz)).sort(); + const ids = [ + ...new Set( + planned.map((p) => p.markerId).filter((id): id is string => !!id), + ), + ]; + const names = [...new Set(planned.map((p) => p.reading.analyte))]; + const existing = await client.labResult.findMany({ + where: { + userId, + deletedAt: null, + takenAt: { + gte: labReadingDaySearchRange(days[0]).gte, + lt: labReadingDaySearchRange(days[days.length - 1]).lt, + }, + OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }], + }, + select: { + biomarkerId: true, + analyte: true, + takenAt: true, + value: true, + valueText: true, + sourceValue: true, + sourceUnit: true, + }, + }); + for (const e of existing) { + for (const k of readingKeys(e, tz)) keys.add(k); + // A reading filed under a marker also answers to its printed name. + if (e.biomarkerId) { + for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) keys.add(k); + } + } + return keys; +} + +/** + * The row for one reading: its marker resolved or minted, the value moved + * into the marker's unit when it states another. Built field by field. + */ +async function buildRow( + client: LabWriteClient, + userId: string, + reading: MappedFhirLab, + index: MarkerIndex, +): Promise< + | { ok: true; row: Prisma.LabResultCreateManyInput; markerId: string } + | { ok: false } +> { + const marker = await resolveOrMintBiomarker( + userId, + { + analyte: reading.analyte, + unit: reading.unit ?? "", + referenceLow: reading.sourceReferenceLow, + referenceHigh: reading.sourceReferenceHigh, + panel: reading.panel, + }, + { index, catalogKey: reading.catalogKey }, + ); + + // Like the CSV import: on a text result the window only seeds a new + // marker; the reading itself records none. + let storedValue = reading.value; + let storedLow = + reading.valueText === null ? reading.sourceReferenceLow : null; + let storedHigh = + reading.valueText === null ? reading.sourceReferenceHigh : null; + let convertedFrom: { sourceValue: number; sourceUnit: string } | null = null; + if (reading.value !== null) { + const converted = await readingInMarkerUnit(client, { + marker, + value: reading.value, + unit: reading.unit ?? undefined, + sourceReferenceLow: storedLow, + sourceReferenceHigh: storedHigh, + }); + if (!converted.ok) return { ok: false }; + storedValue = converted.value; + storedLow = converted.sourceReferenceLow; + storedHigh = converted.sourceReferenceHigh; + if (converted.converted) { + convertedFrom = { + sourceValue: converted.sourceValue, + sourceUnit: converted.sourceUnit, + }; + } + } + + return { + ok: true, + markerId: marker.id, + row: { + userId, + biomarkerId: marker.id, + panel: reading.panel ?? marker.panel, + analyte: marker.name, + value: storedValue, + valueText: reading.valueText, + unit: marker.unit, + sourceValue: convertedFrom?.sourceValue ?? null, + sourceUnit: convertedFrom?.sourceUnit ?? null, + referenceLow: marker.lowerBound, + referenceHigh: marker.upperBound, + sourceReferenceLow: storedLow, + sourceReferenceHigh: storedHigh, + sourceReferenceText: reading.sourceReferenceText, + takenAt: reading.takenAt, + source: APPLE_HEALTH_LAB_SOURCE, + externalId: reading.externalId, + noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null, + }, + }; +} diff --git a/src/lib/measurements/__tests__/import-apple-health-export.test.ts b/src/lib/measurements/__tests__/import-apple-health-export.test.ts index 2ceb05fc1..486d435e2 100644 --- a/src/lib/measurements/__tests__/import-apple-health-export.test.ts +++ b/src/lib/measurements/__tests__/import-apple-health-export.test.ts @@ -952,6 +952,44 @@ describe("streamParseExportXml — end-to-end", () => { expect(result.totals.recordsRead).toBeGreaterThanOrEqual(7); }); + it("hands lab-result clinical records to the callback and skips the rest", async () => { + const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); + const xmlPath = join(tmp, "export.xml"); + writeFileSync( + xmlPath, + ` + + + +`, + ); + const refs: unknown[] = []; + const result = await streamParseExportXml({ + xmlPath, + userId: "user-1", + userTimezone: "Europe/Berlin", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma: makeFakePrisma() as any, + onClinicalLabRecord: (ref) => void refs.push(ref), + }); + expect(refs).toEqual([ + { + path: "/clinical-records/Observation-obs-1.json", + identifier: "obs-1", + sourceURL: "https://fhir.example-clinic.test/R4", + sourceName: "Example Clinic", + fhirVersion: "4.0.1", + }, + ]); + expect(result.clinical.skipped).toBe(1); + }); + it("re-importing the same export reports 0 inserts and N updates", async () => { const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); const xmlPath = join(tmp, "export.xml"); diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index 931e62f75..bbb1035d8 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -45,6 +45,10 @@ import { canonicalDailyTimestamp, } from "@/lib/measurements/drain-per-sample-cumulative"; import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin"; +import { + routeClinicalRecord, + type ClinicalLabReference, +} from "@/lib/apple-health/clinical-record-ref"; import { isSameReadingAcrossSource, MEASURED_AT_TOLERANCE_MS, @@ -460,6 +464,9 @@ export interface StreamParseInput { * pass, from the archive (`src/lib/apple-health/route-import.ts`). */ onWorkoutRoute?: (ref: { externalId: string; path: string }) => void; + /** Called once per lab-result ``; without it every clinical + * record is counted as skipped (`src/lib/apple-health/clinical-labs-import.ts`). */ + onClinicalLabRecord?: (ref: ClinicalLabReference) => void; } /** @@ -1449,7 +1456,9 @@ export async function streamParseExportXml( if (name === "ClinicalRecord") { recordsRead += 1; - clinical.skipped += 1; + if (!routeClinicalRecord(attrs, input.onClinicalLabRecord)) { + clinical.skipped += 1; + } return; }