mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 08:12:52 +00:00
feat(apple-health): import lab results from Health Records in the export
The export parser counted every <ClinicalRecord> as skipped. Lab-result records are now handed to a pass after the export.xml rows are written: each referenced clinical-records/ JSON file is read from the archive one at a time, mapped with the shared FHIR mapper, given its panel's name and written through the shared writer. Other clinical types stay skipped. The pass is fail-soft like the ECG and route passes, and the job result carries bounded lab counts only. The ECG pass moves out of the worker into ecg-import.ts, and the parser revision goes to 4 so an archive imported before can be uploaded again to pick up its lab results.
This commit is contained in:
1 parent
e40de2a654
commit
736e6e9c86
16 files changed
+922
-212
No files matched your search
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 84,
|
||||
"total": 93816,
|
||||
"total": 93825,
|
||||
"entries": {
|
||||
"app/medications/page-client.tsx": 842,
|
||||
"components/charts/mood-chart.tsx": 1166,
|
||||
@@ -72,7 +72,7 @@
|
||||
"lib/mcp/record-search.ts": 1397,
|
||||
"lib/mcp/rich-reads.ts": 1471,
|
||||
"lib/measurements/dense-intraday-retention.ts": 1016,
|
||||
"lib/measurements/import-apple-health-export.ts": 1722,
|
||||
"lib/measurements/import-apple-health-export.ts": 1731,
|
||||
"lib/medications/scheduling/cadence.ts": 976,
|
||||
"lib/medications/scheduling/recurrence.ts": 1100,
|
||||
"lib/medications/scheduling/slot-upsert.ts": 912,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 335,
|
||||
"total": 109025,
|
||||
"total": 109021,
|
||||
"entries": {
|
||||
"app/achievements/page.tsx :: AchievementsPage": 206,
|
||||
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 275,
|
||||
@@ -285,7 +285,7 @@
|
||||
"lib/insights/status-provider.ts :: runStatusCompletion": 215,
|
||||
"lib/integrations/connection-health-load.ts :: loadConnectionFacts": 229,
|
||||
"lib/integrations/status.ts :: recordSyncFailure": 292,
|
||||
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 300,
|
||||
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 292,
|
||||
"lib/jobs/coach-nudge.ts :: findTriggerForUser": 279,
|
||||
"lib/jobs/coach-nudge.ts :: runCoachNudgeTick": 292,
|
||||
"lib/jobs/coach-reminder-sweep.ts :: runCoachReminderSweep": 222,
|
||||
@@ -314,8 +314,8 @@
|
||||
"lib/measurements/dense-intraday-retention.ts :: runDenseIntradayRetention > walkSource > writeDay > foldOnce": 212,
|
||||
"lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative": 344,
|
||||
"lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative > writeDay": 205,
|
||||
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1245,
|
||||
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 323,
|
||||
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1247,
|
||||
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 325,
|
||||
"lib/medications/efficacy/build-efficacy.ts :: buildMedicationEfficacy": 268,
|
||||
"lib/medications/intake-slot-dedup.ts :: dedupeUserIntakeSlots": 319,
|
||||
"lib/medications/list-read.ts :: buildMedicationsList": 310,
|
||||
|
||||
@@ -198,7 +198,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
|
||||
expect(mocks.importJobCreate).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.importJobCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
data: expect.objectContaining({ parserRevision: 3 }),
|
||||
data: expect.objectContaining({ parserRevision: 4 }),
|
||||
}),
|
||||
);
|
||||
expect(mocks.bossSend).toHaveBeenCalledWith(
|
||||
@@ -217,7 +217,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({
|
||||
status: { not: "failed" },
|
||||
parserRevision: 3,
|
||||
parserRevision: 4,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
@@ -241,7 +241,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
|
||||
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
|
||||
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({ parserRevision: 3 }),
|
||||
where: expect.objectContaining({ parserRevision: 4 }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -93,6 +93,54 @@ describe("GET /api/import/apple-health-export/[jobId]/status", () => {
|
||||
expect(body.data.status).toBe("parsing");
|
||||
});
|
||||
|
||||
it("bounds the lab counts and drops anything else in the block", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(OWNER_SESSION as never);
|
||||
mocks.importJobFindUnique.mockResolvedValue(
|
||||
makeRow({
|
||||
status: "done",
|
||||
result: {
|
||||
labs: {
|
||||
discovered: 3,
|
||||
imported: 2,
|
||||
duplicate: -1,
|
||||
skipped: "x",
|
||||
failed: 1.5,
|
||||
reasons: { leaked: "content" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
const req = new NextRequest(
|
||||
"http://localhost/api/import/apple-health-export/ij-1/status",
|
||||
);
|
||||
const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) });
|
||||
const body = (await res.json()) as {
|
||||
data: { result: { labs: Record<string, unknown> } };
|
||||
};
|
||||
expect(body.data.result.labs).toEqual({
|
||||
discovered: 3,
|
||||
imported: 2,
|
||||
duplicate: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports zeroed lab counts for a job from before labs were read", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(OWNER_SESSION as never);
|
||||
mocks.importJobFindUnique.mockResolvedValue(
|
||||
makeRow({ status: "done", result: { totals: { recordsRead: 1 } } }),
|
||||
);
|
||||
const req = new NextRequest(
|
||||
"http://localhost/api/import/apple-health-export/ij-1/status",
|
||||
);
|
||||
const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) });
|
||||
const body = (await res.json()) as {
|
||||
data: { result: { labs: Record<string, number> } };
|
||||
};
|
||||
expect(body.data.result.labs.imported).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 404 to a non-owner non-admin user", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(OTHER_SESSION as never);
|
||||
mocks.importJobFindUnique.mockResolvedValue(makeRow());
|
||||
|
||||
@@ -46,10 +46,12 @@ function privacySafeResult(
|
||||
value: Record<string, unknown> | null,
|
||||
): Record<string, unknown> | null {
|
||||
if (!value) return null;
|
||||
const ecg =
|
||||
typeof value.ecg === "object" && value.ecg !== null
|
||||
? (value.ecg as Record<string, unknown>)
|
||||
const block = (key: string): Record<string, unknown> =>
|
||||
typeof value[key] === "object" && value[key] !== null
|
||||
? (value[key] as Record<string, unknown>)
|
||||
: {};
|
||||
const ecg = block("ecg");
|
||||
const labs = block("labs");
|
||||
return {
|
||||
...value,
|
||||
ecg: {
|
||||
@@ -59,6 +61,13 @@ function privacySafeResult(
|
||||
skipped: boundedCount(ecg.skipped),
|
||||
failed: boundedCount(ecg.failed),
|
||||
},
|
||||
labs: {
|
||||
discovered: boundedCount(labs.discovered),
|
||||
imported: boundedCount(labs.imported),
|
||||
duplicate: boundedCount(labs.duplicate),
|
||||
skipped: boundedCount(labs.skipped),
|
||||
failed: boundedCount(labs.failed),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -200,7 +200,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
|
||||
expect(mocks.importJobCreate).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.importJobCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
data: expect.objectContaining({ parserRevision: 3 }),
|
||||
data: expect.objectContaining({ parserRevision: 4 }),
|
||||
}),
|
||||
);
|
||||
expect(mocks.bossSend).toHaveBeenCalledWith(
|
||||
@@ -217,7 +217,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({
|
||||
status: { not: "failed" },
|
||||
parserRevision: 3,
|
||||
parserRevision: 4,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
@@ -243,7 +243,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
|
||||
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
|
||||
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({ parserRevision: 3 }),
|
||||
where: expect.objectContaining({ parserRevision: 4 }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { strToU8, zipSync } from "fflate";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
write: vi.fn(),
|
||||
after: vi.fn(),
|
||||
annotate: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/labs/fhir-import/write", () => ({
|
||||
writeFhirLabReadings: mocks.write,
|
||||
}));
|
||||
vi.mock("@/lib/labs/fhir-import/after-write", () => ({
|
||||
afterLabReadingsWritten: mocks.after,
|
||||
}));
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: mocks.annotate }));
|
||||
|
||||
import { importAppleHealthClinicalLabs } from "../clinical-labs-import";
|
||||
import type { ClinicalLabReference } from "../clinical-record-ref";
|
||||
import { zip64Archive } from "./helpers/zip64";
|
||||
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0))
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.write.mockImplementation(
|
||||
async ({ readings }: { readings: unknown[] }) => ({
|
||||
statuses: readings.map(() => ({ status: "inserted" })),
|
||||
inserted: readings.length,
|
||||
duplicate: 0,
|
||||
failed: 0,
|
||||
newestTakenAt: new Date("2025-01-01T00:00:00Z"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
const ROOT = "apple_health_export/clinical-records/";
|
||||
|
||||
function observation(id: string, extra: Record<string, unknown> = {}) {
|
||||
return JSON.stringify({
|
||||
resourceType: "Observation",
|
||||
id,
|
||||
status: "final",
|
||||
category: [{ coding: [{ code: "laboratory" }] }],
|
||||
code: { text: `Analyte ${id}` },
|
||||
effectiveDateTime: "2025-01-01T08:00:00Z",
|
||||
valueQuantity: { value: 1, unit: "mmol/L" },
|
||||
...extra,
|
||||
});
|
||||
}
|
||||
|
||||
function archive(files: Record<string, string>, zip64 = false): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "healthlog-clinical-"));
|
||||
dirs.push(dir);
|
||||
const path = join(dir, "export.zip");
|
||||
writeFileSync(
|
||||
path,
|
||||
zip64
|
||||
? zip64Archive(files)
|
||||
: zipSync(
|
||||
Object.fromEntries(
|
||||
Object.entries(files).map(([name, body]) => [name, strToU8(body)]),
|
||||
),
|
||||
),
|
||||
);
|
||||
return path;
|
||||
}
|
||||
|
||||
function ref(
|
||||
path: string | null,
|
||||
identifier: string | null = null,
|
||||
): ClinicalLabReference {
|
||||
return {
|
||||
path,
|
||||
identifier,
|
||||
sourceURL: "https://fhir.example.test",
|
||||
sourceName: "Example",
|
||||
fhirVersion: "4.0.1",
|
||||
};
|
||||
}
|
||||
|
||||
function run(path: string, references: ClinicalLabReference[], limits = {}) {
|
||||
return importAppleHealthClinicalLabs({
|
||||
archivePath: path,
|
||||
userId: "user-1",
|
||||
tz: "Europe/Berlin",
|
||||
references,
|
||||
limits: {
|
||||
maxMembers: 50,
|
||||
maxClinicalMembers: 20,
|
||||
maxMemberBytes: 64 * 1024,
|
||||
maxTotalClinicalBytes: 1024 * 1024,
|
||||
maxCompressionRatio: 200,
|
||||
...limits,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("importAppleHealthClinicalLabs", () => {
|
||||
it("reads only referenced members and writes their readings with panels", async () => {
|
||||
const path = archive({
|
||||
"apple_health_export/export.xml": "<HealthData/>",
|
||||
[`${ROOT}Observation-a.json`]: observation("a"),
|
||||
[`${ROOT}Observation-panel.json`]: observation("panel", {
|
||||
code: { text: "Electrolytes" },
|
||||
valueQuantity: undefined,
|
||||
hasMember: [{ reference: "Observation/a" }],
|
||||
}),
|
||||
[`${ROOT}Condition-c.json`]: '{"resourceType":"Condition"}',
|
||||
});
|
||||
const counts = await run(path, [
|
||||
ref("/clinical-records/Observation-a.json", "a"),
|
||||
ref("/clinical-records/Observation-panel.json", "panel"),
|
||||
]);
|
||||
expect(counts).toEqual({
|
||||
discovered: 2,
|
||||
imported: 1,
|
||||
duplicate: 0,
|
||||
skipped: 1,
|
||||
failed: 0,
|
||||
});
|
||||
const [{ readings, tz, userId }] = mocks.write.mock.calls[0];
|
||||
expect(userId).toBe("user-1");
|
||||
expect(tz).toBe("Europe/Berlin");
|
||||
expect(readings).toHaveLength(1);
|
||||
expect(readings[0]).toMatchObject({
|
||||
analyte: "Analyte a",
|
||||
panel: "Electrolytes",
|
||||
});
|
||||
expect(mocks.after).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ userId: "user-1", inserted: 1 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("works on a ZIP64 archive", async () => {
|
||||
const path = archive(
|
||||
{ [`${ROOT}Observation-a.json`]: observation("a") },
|
||||
true,
|
||||
);
|
||||
const counts = await run(path, [
|
||||
ref("/clinical-records/Observation-a.json"),
|
||||
]);
|
||||
expect(counts.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("counts a missing member, malformed JSON and an unsafe path as failed", async () => {
|
||||
const path = archive({ [`${ROOT}Observation-bad.json`]: "{not json" });
|
||||
const counts = await run(path, [
|
||||
ref("/clinical-records/Observation-bad.json"),
|
||||
ref("/clinical-records/Observation-gone.json"),
|
||||
ref("/clinical-records/../export.xml"),
|
||||
]);
|
||||
expect(counts).toMatchObject({ discovered: 3, imported: 0, failed: 3 });
|
||||
expect(mocks.write).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("finds a reference without a path by the resource id", async () => {
|
||||
const path = archive({
|
||||
[`${ROOT}one.json`]: observation("a"),
|
||||
[`${ROOT}two.json`]: observation("unreferenced"),
|
||||
});
|
||||
const counts = await run(path, [ref(null, "a")]);
|
||||
expect(counts).toMatchObject({ discovered: 1, imported: 1, failed: 0 });
|
||||
expect(mocks.write.mock.calls[0][0].readings).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("refuses references past the clinical member ceiling", async () => {
|
||||
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
|
||||
const counts = await run(
|
||||
path,
|
||||
[
|
||||
ref("/clinical-records/Observation-a.json"),
|
||||
ref("/clinical-records/Observation-b.json"),
|
||||
],
|
||||
{ maxClinicalMembers: 1 },
|
||||
);
|
||||
expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 });
|
||||
});
|
||||
|
||||
it("fails soft when the archive breaks a limit", async () => {
|
||||
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
|
||||
const counts = await run(
|
||||
path,
|
||||
[ref("/clinical-records/Observation-a.json")],
|
||||
{
|
||||
maxMemberBytes: 10,
|
||||
},
|
||||
);
|
||||
expect(counts).toMatchObject({ discovered: 1, imported: 0, failed: 1 });
|
||||
});
|
||||
|
||||
it("annotates counts and closed reasons only", async () => {
|
||||
const path = archive({
|
||||
[`${ROOT}Observation-a.json`]: observation("a", {
|
||||
code: { text: "Secret analyte name" },
|
||||
status: "preliminary",
|
||||
}),
|
||||
});
|
||||
await run(path, [ref("/clinical-records/Observation-a.json")]);
|
||||
const serialised = JSON.stringify(mocks.annotate.mock.calls);
|
||||
expect(serialised).not.toContain("Secret analyte name");
|
||||
expect(serialised).not.toContain("fhir.example.test");
|
||||
expect(mocks.annotate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: { name: "apple_health.import.clinical_labs" },
|
||||
meta: expect.objectContaining({
|
||||
skipped: 1,
|
||||
reasons: { not_final: 1 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does nothing for no references", async () => {
|
||||
const counts = await run("/nonexistent.zip", []);
|
||||
expect(counts.discovered).toBe(0);
|
||||
expect(mocks.write).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,220 @@
|
||||
/**
|
||||
* Lab results from Apple Health's Health Records, out of an `export.zip`
|
||||
* (#1400).
|
||||
*
|
||||
* `export.xml` lists each Health Records resource as a `<ClinicalRecord>`
|
||||
* whose `resourceFilePath` names a FHIR JSON file under `clinical-records/`.
|
||||
* The XML pass hands the lab-result references here
|
||||
* (`routeClinicalRecord`); this module then reads the referenced members one
|
||||
* at a time, maps each with the shared FHIR mapper, gives members their
|
||||
* panel's name, and writes through the shared writer, so a result lands the
|
||||
* same way it would from the iOS app (`POST /api/labs/fhir-batch`).
|
||||
*
|
||||
* A reference with no path (an export from a Health app that did not write
|
||||
* one) is matched by the resource's own id among the clinical files instead.
|
||||
*
|
||||
* Memory: one member is held at a time, capped by the archive reader; what is
|
||||
* kept across members is the mapped readings (a few hundred bytes each) and
|
||||
* the panel headers' member lists.
|
||||
*
|
||||
* A failure here never fails the import: the measurements are committed. A
|
||||
* missing, oversized or malformed file is counted as failed. Nothing about a
|
||||
* resource reaches the result, the logs or a wide event: only counts and
|
||||
* closed reasons.
|
||||
*/
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import {
|
||||
clinicalMemberNameFromReference,
|
||||
isClinicalRecordJson,
|
||||
streamAppleHealthClinicalMembers,
|
||||
type AppleHealthClinicalArchiveLimits,
|
||||
} from "@/lib/apple-health/archive-stream";
|
||||
import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref";
|
||||
import { afterLabReadingsWritten } from "@/lib/labs/fhir-import/after-write";
|
||||
import {
|
||||
assignPanels,
|
||||
mapFhirLabRecord,
|
||||
type FhirLabMapping,
|
||||
} from "@/lib/labs/fhir-import/map-observation";
|
||||
import {
|
||||
emptyFhirLabCounts,
|
||||
type FhirLabCounts,
|
||||
} from "@/lib/labs/fhir-import/types";
|
||||
import { writeFhirLabReadings } from "@/lib/labs/fhir-import/write";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import type { ImportJobResult } from "@/lib/measurements/import-apple-health-export";
|
||||
|
||||
/** The terminal result of an Apple Health import job, labs included. */
|
||||
export type AppleHealthImportJobResult = ImportJobResult & {
|
||||
labs: FhirLabCounts;
|
||||
};
|
||||
|
||||
/** Per-archive ceilings for the Health Records members. */
|
||||
export const APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS: AppleHealthClinicalArchiveLimits =
|
||||
{
|
||||
maxMembers: 200_000,
|
||||
maxClinicalMembers: 50_000,
|
||||
// A single Observation is a few KB; 2 MiB leaves room for a long note.
|
||||
maxMemberBytes: 2 * 1024 * 1024,
|
||||
maxTotalClinicalBytes: 256 * 1024 * 1024,
|
||||
maxCompressionRatio: 200,
|
||||
};
|
||||
|
||||
async function readJson(stream: AsyncIterable<unknown>): Promise<unknown> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString("utf8");
|
||||
for (const chunk of chunks) chunk.fill(0);
|
||||
return JSON.parse(text) as unknown;
|
||||
}
|
||||
|
||||
function resourceId(resource: unknown): string | null {
|
||||
if (typeof resource !== "object" || resource === null) return null;
|
||||
const id = (resource as { id?: unknown }).id;
|
||||
return typeof id === "string" && id.trim() !== "" ? id.trim() : null;
|
||||
}
|
||||
|
||||
export async function importAppleHealthClinicalLabs(input: {
|
||||
archivePath: string;
|
||||
userId: string;
|
||||
tz: string;
|
||||
references: readonly ClinicalLabReference[];
|
||||
prisma?: Pick<PrismaClient, "labResult" | "biomarker" | "biomarkerAlias">;
|
||||
limits?: AppleHealthClinicalArchiveLimits;
|
||||
}): Promise<FhirLabCounts> {
|
||||
const counts = emptyFhirLabCounts();
|
||||
const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS;
|
||||
counts.discovered = input.references.length;
|
||||
if (input.references.length === 0) return counts;
|
||||
|
||||
const tally: Record<string, number> = {};
|
||||
const note = (reason: string) => {
|
||||
tally[reason] = (tally[reason] ?? 0) + 1;
|
||||
};
|
||||
|
||||
// More references than the archive may hold clinical files: the overflow
|
||||
// is refused, the rest is read.
|
||||
const usable = input.references.slice(0, limits.maxClinicalMembers);
|
||||
const overflow = input.references.length - usable.length;
|
||||
counts.failed += overflow;
|
||||
if (overflow > 0) tally.over_limit = overflow;
|
||||
|
||||
const byMember = new Map<string, ClinicalLabReference>();
|
||||
const byId = new Map<string, ClinicalLabReference>();
|
||||
for (const ref of usable) {
|
||||
const member = ref.path ? clinicalMemberNameFromReference(ref.path) : null;
|
||||
if (member && !byMember.has(member)) byMember.set(member, ref);
|
||||
else if (!ref.path && ref.identifier && !byId.has(ref.identifier)) {
|
||||
byId.set(ref.identifier, ref);
|
||||
} else {
|
||||
// An unsafe path, or a second reference to one file.
|
||||
counts.failed += 1;
|
||||
note("bad_reference");
|
||||
}
|
||||
}
|
||||
const expected = byMember.size + byId.size;
|
||||
let handled = 0;
|
||||
|
||||
const mapped: FhirLabMapping[] = [];
|
||||
try {
|
||||
for await (const member of streamAppleHealthClinicalMembers({
|
||||
archivePath: input.archivePath,
|
||||
limits,
|
||||
wanted: (name) =>
|
||||
byMember.has(name) || (byId.size > 0 && isClinicalRecordJson(name)),
|
||||
})) {
|
||||
const byPath = byMember.get(member.name);
|
||||
let resource: unknown;
|
||||
try {
|
||||
resource = await readJson(member.stream);
|
||||
} catch {
|
||||
if (byPath) {
|
||||
handled += 1;
|
||||
counts.failed += 1;
|
||||
note("malformed_json");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let ref = byPath;
|
||||
if (!ref) {
|
||||
const id = resourceId(resource);
|
||||
ref = id ? byId.get(id) : undefined;
|
||||
if (!ref || !id) continue;
|
||||
byId.delete(id);
|
||||
}
|
||||
handled += 1;
|
||||
const result = mapFhirLabRecord({
|
||||
resource,
|
||||
fhirVersion: ref.fhirVersion,
|
||||
identifier: ref.identifier,
|
||||
sourceURL: ref.sourceURL,
|
||||
sourceName: ref.sourceName,
|
||||
});
|
||||
resource = null;
|
||||
if (result.kind === "reading") {
|
||||
mapped.push(result);
|
||||
} else if (result.kind === "skip") {
|
||||
counts.skipped += 1;
|
||||
note(result.reason);
|
||||
if (result.reason === "panel_header") mapped.push(result);
|
||||
} else {
|
||||
counts.failed += 1;
|
||||
note(result.reason);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// The archive itself is unreadable for this pass (a limit hit in the
|
||||
// directory, an inflate failure): the files not yet read are counted
|
||||
// below as missing. What was read is still written.
|
||||
note("archive_unreadable");
|
||||
}
|
||||
const missing = Math.max(0, expected - handled);
|
||||
counts.failed += missing;
|
||||
if (missing > 0) tally.missing_member = (tally.missing_member ?? 0) + missing;
|
||||
|
||||
assignPanels(mapped);
|
||||
const readings = mapped.flatMap((m) =>
|
||||
m.kind === "reading" ? [m.reading] : [],
|
||||
);
|
||||
mapped.length = 0;
|
||||
if (readings.length > 0) {
|
||||
try {
|
||||
const written = await writeFhirLabReadings({
|
||||
userId: input.userId,
|
||||
tz: input.tz,
|
||||
readings,
|
||||
prisma: input.prisma,
|
||||
});
|
||||
counts.imported += written.inserted;
|
||||
counts.duplicate += written.duplicate;
|
||||
counts.failed += written.failed;
|
||||
for (const status of written.statuses) {
|
||||
if (status.status === "failed") note(status.reason);
|
||||
}
|
||||
afterLabReadingsWritten({
|
||||
userId: input.userId,
|
||||
inserted: written.inserted,
|
||||
newestTakenAt: written.newestTakenAt,
|
||||
source: "apple_health_export",
|
||||
});
|
||||
} catch {
|
||||
counts.failed += readings.length;
|
||||
note("write_failed");
|
||||
}
|
||||
}
|
||||
|
||||
annotate({
|
||||
action: { name: "apple_health.import.clinical_labs" },
|
||||
meta: {
|
||||
discovered: counts.discovered,
|
||||
imported: counts.imported,
|
||||
duplicate: counts.duplicate,
|
||||
skipped: counts.skipped,
|
||||
failed: counts.failed,
|
||||
reasons: tally,
|
||||
},
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* A `<ClinicalRecord>` in `export.xml`, as a reference to the lab result it
|
||||
* describes (#1400).
|
||||
*
|
||||
* Apple writes one element per Health Records resource:
|
||||
*
|
||||
* <ClinicalRecord type="HKClinicalTypeIdentifierLabResultRecord"
|
||||
* identifier="…" sourceName="…" sourceURL="…" fhirVersion="4.0.1"
|
||||
* receivedDate="…" resourceFilePath="/clinical-records/….json"/>
|
||||
*
|
||||
* The resource itself is the JSON file the path names. Only lab results are
|
||||
* handed on; every other clinical type (conditions, medications, vital signs,
|
||||
* immunisations) is left to the caller's skipped count. The attributes are
|
||||
* capped here so a hostile export cannot make the parser hold long strings.
|
||||
*/
|
||||
|
||||
export const HK_LAB_RESULT_RECORD = "HKClinicalTypeIdentifierLabResultRecord";
|
||||
|
||||
export interface ClinicalLabReference {
|
||||
/** `resourceFilePath`, from the export root. Null when Apple wrote none. */
|
||||
path: string | null;
|
||||
identifier: string | null;
|
||||
sourceURL: string | null;
|
||||
sourceName: string | null;
|
||||
fhirVersion: string | null;
|
||||
}
|
||||
|
||||
const CAPS = {
|
||||
path: 512,
|
||||
identifier: 256,
|
||||
sourceURL: 2048,
|
||||
sourceName: 200,
|
||||
fhirVersion: 16,
|
||||
} as const;
|
||||
|
||||
function capped(raw: string | undefined, max: number): string | null {
|
||||
if (raw === undefined) return null;
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed === "" || trimmed.length > max) return null;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand a lab-result record to `onLab` and return true; return false for any
|
||||
* other record, or when there is no callback, so the caller counts it as
|
||||
* skipped.
|
||||
*/
|
||||
export function routeClinicalRecord(
|
||||
attrs: Record<string, string | undefined>,
|
||||
onLab?: (ref: ClinicalLabReference) => void,
|
||||
): boolean {
|
||||
if (!onLab || attrs.type !== HK_LAB_RESULT_RECORD) return false;
|
||||
onLab({
|
||||
path: capped(attrs.resourceFilePath, CAPS.path),
|
||||
identifier: capped(attrs.identifier, CAPS.identifier),
|
||||
sourceURL: capped(attrs.sourceURL, CAPS.sourceURL),
|
||||
sourceName: capped(attrs.sourceName, CAPS.sourceName),
|
||||
fhirVersion: capped(attrs.fhirVersion, CAPS.fhirVersion),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -17,7 +17,11 @@
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import type { NormalizedAppleHealthEcg } from "@/lib/apple-health/ecg-csv";
|
||||
import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream";
|
||||
import {
|
||||
parseAppleHealthEcgCsv,
|
||||
type NormalizedAppleHealthEcg,
|
||||
} from "@/lib/apple-health/ecg-csv";
|
||||
import { persistEcgRecording } from "@/lib/ecg/persist-recording";
|
||||
|
||||
export type AppleHealthEcgImportOutcome = "imported" | "updated" | "skipped";
|
||||
@@ -68,3 +72,72 @@ export async function importAppleHealthEcg(input: {
|
||||
if (result.outcome === "duplicate") return "skipped";
|
||||
return "updated";
|
||||
}
|
||||
|
||||
const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = {
|
||||
maxMembers: 20_000,
|
||||
maxEcgMembers: 2_000,
|
||||
maxMemberBytes: 16 * 1024 * 1024,
|
||||
maxTotalEcgBytes: 512 * 1024 * 1024,
|
||||
maxCompressionRatio: 200,
|
||||
} as const;
|
||||
const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
|
||||
|
||||
export interface AppleHealthEcgCounts {
|
||||
discovered: number;
|
||||
imported: number;
|
||||
updated: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every ECG recording in an export archive. Intentionally fail-soft: the
|
||||
* canonical export.xml pass has already completed, and malformed, oversized
|
||||
* or unwritable ECG members affect only bounded scalar counters in the
|
||||
* terminal result. `memberNames` are the archive's other members, which the
|
||||
* discovered count is taken from.
|
||||
*/
|
||||
export async function importAppleHealthArchiveEcgs(input: {
|
||||
archivePath: string;
|
||||
userId: string;
|
||||
memberNames: readonly string[];
|
||||
prisma: Pick<PrismaClient, "$transaction">;
|
||||
}): Promise<AppleHealthEcgCounts> {
|
||||
const ecg: AppleHealthEcgCounts = {
|
||||
discovered: 0,
|
||||
imported: 0,
|
||||
updated: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
};
|
||||
ecg.discovered = input.memberNames.filter((name) => {
|
||||
const lower = name.toLowerCase();
|
||||
return lower.includes("electrocardiograms/") && lower.endsWith(".csv");
|
||||
}).length;
|
||||
try {
|
||||
for await (const member of streamAppleHealthEcgMembers({
|
||||
archivePath: input.archivePath,
|
||||
limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS,
|
||||
})) {
|
||||
try {
|
||||
const parsed = await parseAppleHealthEcgCsv({
|
||||
memberName: member.name,
|
||||
stream: member.stream,
|
||||
maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES,
|
||||
});
|
||||
const outcome = await importAppleHealthEcg({
|
||||
userId: input.userId,
|
||||
ecg: parsed,
|
||||
prisma: input.prisma,
|
||||
});
|
||||
ecg[outcome] += 1;
|
||||
} catch {
|
||||
ecg.failed += 1;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed;
|
||||
ecg.failed += Math.max(0, ecg.discovered - completed);
|
||||
}
|
||||
return ecg;
|
||||
}
|
||||
@@ -8,6 +8,10 @@ const workerSource = readFileSync(
|
||||
join(process.cwd(), "src/lib/jobs/apple-health-import-worker.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const ecgSource = readFileSync(
|
||||
join(process.cwd(), "src/lib/apple-health/ecg-import.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const parserSource = readFileSync(
|
||||
join(process.cwd(), "src/lib/measurements/import-apple-health-export.ts"),
|
||||
"utf8",
|
||||
@@ -19,18 +23,22 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
|
||||
});
|
||||
|
||||
it("runs bounded ECG archive ingestion beside the established XML parser", () => {
|
||||
expect(workerSource).toMatch(/streamAppleHealthEcgMembers/);
|
||||
expect(workerSource).toMatch(/parseAppleHealthEcgCsv/);
|
||||
expect(workerSource).toMatch(/importAppleHealthEcg/);
|
||||
expect(workerSource).toMatch(/streamParseExportXml/);
|
||||
expect(workerSource).toMatch(/importAppleHealthArchiveEcgs/);
|
||||
expect(ecgSource).toMatch(/streamAppleHealthEcgMembers/);
|
||||
expect(ecgSource).toMatch(/parseAppleHealthEcgCsv/);
|
||||
expect(ecgSource).toMatch(/importAppleHealthEcg\(/);
|
||||
});
|
||||
|
||||
it("catches per-ECG failures without routing the valid XML result to the fatal job catch", () => {
|
||||
expect(workerSource).toMatch(
|
||||
expect(ecgSource).toMatch(
|
||||
/for await[\s\S]*streamAppleHealthEcgMembers[\s\S]*try[\s\S]*parseAppleHealthEcgCsv[\s\S]*catch[\s\S]*ecg\.failed/,
|
||||
);
|
||||
expect(ecgSource).toMatch(
|
||||
/ecg:\s*AppleHealthEcgCounts\s*=\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/,
|
||||
);
|
||||
expect(workerSource).toMatch(
|
||||
/result[\s\S]*ecg:\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/,
|
||||
/result:\s*AppleHealthImportJobResult\s*=\s*\{[\s\S]*ecg,[\s\S]*labs:/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -45,7 +53,9 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
|
||||
|
||||
it("does not log ECG filenames, source labels, samples, or raw parser errors", () => {
|
||||
const loggingCalls =
|
||||
workerSource.match(/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g) ?? [];
|
||||
(workerSource + ecgSource).match(
|
||||
/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g,
|
||||
) ?? [];
|
||||
const ecgLogging = loggingCalls.filter((call) =>
|
||||
/ecg|electrocardiogram/i.test(call),
|
||||
);
|
||||
@@ -56,7 +66,7 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
|
||||
});
|
||||
|
||||
it("does not claim to diagnose or reclassify a waveform", () => {
|
||||
expect(workerSource).not.toMatch(
|
||||
expect(workerSource + ecgSource).not.toMatch(
|
||||
/(?:diagnose|infer|classify)Ecg|analy[sz]eWaveform|detectA(?:fib|trial)/i,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -80,8 +80,8 @@ describe("apple health import — pg-boss send policy", () => {
|
||||
});
|
||||
|
||||
describe("apple health import — parser revision boundary", () => {
|
||||
it("isolates revision-3 sends from legacy workers", () => {
|
||||
expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(3);
|
||||
it("isolates revision-4 sends from legacy workers", () => {
|
||||
expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(4);
|
||||
expect(APPLE_HEALTH_IMPORT_V2_QUEUE).toBe("apple-health-import-v2");
|
||||
expect(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).toBe("apple-health-import");
|
||||
});
|
||||
|
||||
@@ -30,9 +30,12 @@ import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream";
|
||||
import { parseAppleHealthEcgCsv } from "@/lib/apple-health/ecg-csv";
|
||||
import { importAppleHealthEcg } from "@/lib/apple-health/ecg-import";
|
||||
import {
|
||||
importAppleHealthClinicalLabs,
|
||||
type AppleHealthImportJobResult,
|
||||
} from "@/lib/apple-health/clinical-labs-import";
|
||||
import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref";
|
||||
import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import";
|
||||
import { extractExportXml } from "@/lib/import/unzip-export-xml";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
@@ -45,7 +48,6 @@ import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import {
|
||||
streamParseExportXml,
|
||||
type ImportJobProgress,
|
||||
type ImportJobResult,
|
||||
} from "@/lib/measurements/import-apple-health-export";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import { DEFAULT_TIMEZONE } from "@/lib/tz/format";
|
||||
@@ -125,17 +127,12 @@ export async function handleImportJobReconcileTick(
|
||||
);
|
||||
}
|
||||
|
||||
/** Parser semantics carried by every newly-created ImportJob. */
|
||||
export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 3;
|
||||
|
||||
const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = {
|
||||
maxMembers: 20_000,
|
||||
maxEcgMembers: 2_000,
|
||||
maxMemberBytes: 16 * 1024 * 1024,
|
||||
maxTotalEcgBytes: 512 * 1024 * 1024,
|
||||
maxCompressionRatio: 200,
|
||||
} as const;
|
||||
const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
|
||||
/**
|
||||
* Parser semantics carried by every newly-created ImportJob. 4: lab results
|
||||
* from Health Records are read (#1400), so an archive imported before is
|
||||
* worth uploading again.
|
||||
*/
|
||||
export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 4;
|
||||
|
||||
/**
|
||||
* (issue #775) Memory preflight for the parse. The streaming pipeline
|
||||
@@ -404,6 +401,7 @@ export async function handleAppleHealthImport(
|
||||
// GPS tracks are read from the archive only when the person keeps them.
|
||||
const keepRoutes = userRow?.storeWorkoutRoutes === true;
|
||||
const routeReferences: WorkoutRouteReference[] = [];
|
||||
const labReferences: ClinicalLabReference[] = [];
|
||||
const userTimezone =
|
||||
userRow?.timezone && userRow.timezone.length > 0
|
||||
? userRow.timezone
|
||||
@@ -439,80 +437,58 @@ export async function handleAppleHealthImport(
|
||||
elapsedMs: 0,
|
||||
});
|
||||
|
||||
const result: ImportJobResult = {
|
||||
...(await streamParseExportXml({
|
||||
xmlPath: unzip.xmlPath,
|
||||
userId,
|
||||
userTimezone,
|
||||
prisma,
|
||||
...(keepRoutes
|
||||
? { onWorkoutRoute: (ref) => void routeReferences.push(ref) }
|
||||
: {}),
|
||||
onProgress: async (snapshot) => {
|
||||
// The phase label here is what the polling endpoint surfaces;
|
||||
// map "parsing" / "upserting" through verbatim.
|
||||
await prisma.importJob.update({
|
||||
where: { id: importJobId },
|
||||
data: {
|
||||
status: snapshot.currentPhase,
|
||||
progress: toJson(snapshot),
|
||||
},
|
||||
});
|
||||
},
|
||||
// The instant the Health app stamped on the archive. Persisted as
|
||||
// soon as the parser reads it, so the status endpoint answers with
|
||||
// it for the rest of the run and a job that later fails still
|
||||
// records which export it was working from. Best-effort: the
|
||||
// stamp is provenance, and losing it must not fail an import.
|
||||
onExportDate: async (exportedAt) => {
|
||||
await prisma.importJob
|
||||
.update({ where: { id: importJobId }, data: { exportedAt } })
|
||||
.catch(caughtAs("apple_health.import.export_date_failed"));
|
||||
},
|
||||
})),
|
||||
ecg: {
|
||||
const parsed = await streamParseExportXml({
|
||||
xmlPath: unzip.xmlPath,
|
||||
userId,
|
||||
userTimezone,
|
||||
prisma,
|
||||
...(keepRoutes
|
||||
? { onWorkoutRoute: (ref) => void routeReferences.push(ref) }
|
||||
: {}),
|
||||
onClinicalLabRecord: (ref) => void labReferences.push(ref),
|
||||
onProgress: async (snapshot) => {
|
||||
// The phase label here is what the polling endpoint surfaces;
|
||||
// map "parsing" / "upserting" through verbatim.
|
||||
await prisma.importJob.update({
|
||||
where: { id: importJobId },
|
||||
data: {
|
||||
status: snapshot.currentPhase,
|
||||
progress: toJson(snapshot),
|
||||
},
|
||||
});
|
||||
},
|
||||
// The instant the Health app stamped on the archive. Persisted as
|
||||
// soon as the parser reads it, so the status endpoint answers with
|
||||
// it for the rest of the run and a job that later fails still
|
||||
// records which export it was working from. Best-effort: the
|
||||
// stamp is provenance, and losing it must not fail an import.
|
||||
onExportDate: async (exportedAt) => {
|
||||
await prisma.importJob
|
||||
.update({ where: { id: importJobId }, data: { exportedAt } })
|
||||
.catch(caughtAs("apple_health.import.export_date_failed"));
|
||||
},
|
||||
});
|
||||
|
||||
// The auxiliary passes below are fail-soft: the export.xml rows are
|
||||
// committed, and a member that cannot be read is only counted.
|
||||
const ecg = await importAppleHealthArchiveEcgs({
|
||||
archivePath: uploadPath,
|
||||
userId,
|
||||
memberNames: unzip.otherMembers.map((member) => member.name),
|
||||
prisma,
|
||||
});
|
||||
const result: AppleHealthImportJobResult = {
|
||||
...parsed,
|
||||
ecg,
|
||||
labs: {
|
||||
discovered: 0,
|
||||
imported: 0,
|
||||
updated: 0,
|
||||
duplicate: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
},
|
||||
};
|
||||
|
||||
// Auxiliary ECGs are intentionally fail-soft. The canonical export.xml
|
||||
// transaction has already completed; malformed, oversized, or unwritable
|
||||
// ECG members affect only bounded scalar counters in the terminal result.
|
||||
const ecg = result.ecg;
|
||||
ecg.discovered = unzip.otherMembers.filter((member) => {
|
||||
const lower = member.name.toLowerCase();
|
||||
return lower.includes("electrocardiograms/") && lower.endsWith(".csv");
|
||||
}).length;
|
||||
try {
|
||||
for await (const member of streamAppleHealthEcgMembers({
|
||||
archivePath: uploadPath,
|
||||
limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS,
|
||||
})) {
|
||||
try {
|
||||
const parsed = await parseAppleHealthEcgCsv({
|
||||
memberName: member.name,
|
||||
stream: member.stream,
|
||||
maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES,
|
||||
});
|
||||
const outcome = await importAppleHealthEcg({
|
||||
userId,
|
||||
ecg: parsed,
|
||||
prisma,
|
||||
});
|
||||
ecg[outcome] += 1;
|
||||
} catch {
|
||||
ecg.failed += 1;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed;
|
||||
ecg.failed += Math.max(0, ecg.discovered - completed);
|
||||
}
|
||||
|
||||
// Workout GPS tracks, on opt-in only. Fail-soft like the ECGs: the
|
||||
// workouts are committed, a track that cannot be read is only counted.
|
||||
// Read one member at a time from the archive; nothing about a track
|
||||
@@ -538,6 +514,19 @@ export async function handleAppleHealthImport(
|
||||
routeReferences.length = 0;
|
||||
}
|
||||
|
||||
// Lab results from Health Records, one FHIR file at a time; only counts
|
||||
// reach the result.
|
||||
if (labReferences.length > 0) {
|
||||
result.labs = await importAppleHealthClinicalLabs({
|
||||
archivePath: uploadPath,
|
||||
userId,
|
||||
tz: userTimezone,
|
||||
references: labReferences,
|
||||
prisma,
|
||||
});
|
||||
labReferences.length = 0;
|
||||
}
|
||||
|
||||
// Done. Persist the terminal envelope.
|
||||
await prisma.importJob.update({
|
||||
where: { id: importJobId },
|
||||
|
||||
+132
-104
@@ -70,7 +70,6 @@ interface Pending {
|
||||
at: number;
|
||||
reading: MappedFhirLab;
|
||||
markerId: string | null;
|
||||
keys: string[];
|
||||
}
|
||||
|
||||
export async function writeFhirLabReadings(args: {
|
||||
@@ -168,51 +167,11 @@ export async function writeFhirLabReadings(args: {
|
||||
at,
|
||||
reading,
|
||||
markerId: hit?.marker.id ?? null,
|
||||
keys: [],
|
||||
});
|
||||
}
|
||||
|
||||
// 3. The reading itself, against the live record.
|
||||
const existingKeys = new Set<string>();
|
||||
if (planned.length > 0) {
|
||||
const days = planned
|
||||
.map((p) => labReadingDay(p.reading.takenAt, tz))
|
||||
.sort();
|
||||
const ids = [
|
||||
...new Set(
|
||||
planned.map((p) => p.markerId).filter((id): id is string => !!id),
|
||||
),
|
||||
];
|
||||
const names = [...new Set(planned.map((p) => p.reading.analyte))];
|
||||
const existing = await client.labResult.findMany({
|
||||
where: {
|
||||
userId,
|
||||
deletedAt: null,
|
||||
takenAt: {
|
||||
gte: labReadingDaySearchRange(days[0]).gte,
|
||||
lt: labReadingDaySearchRange(days[days.length - 1]).lt,
|
||||
},
|
||||
OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }],
|
||||
},
|
||||
select: {
|
||||
biomarkerId: true,
|
||||
analyte: true,
|
||||
takenAt: true,
|
||||
value: true,
|
||||
valueText: true,
|
||||
sourceValue: true,
|
||||
sourceUnit: true,
|
||||
},
|
||||
});
|
||||
for (const e of existing) {
|
||||
for (const k of readingKeys(e, tz)) existingKeys.add(k);
|
||||
if (e.biomarkerId) {
|
||||
for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) {
|
||||
existingKeys.add(k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const existingKeys = await liveReadingKeys(client, userId, tz, planned);
|
||||
|
||||
// 4. Build the rows, field by field.
|
||||
const rows: Prisma.LabResultCreateManyInput[] = [];
|
||||
@@ -237,72 +196,16 @@ export async function writeFhirLabReadings(args: {
|
||||
continue;
|
||||
}
|
||||
|
||||
const marker = await resolveOrMintBiomarker(
|
||||
userId,
|
||||
{
|
||||
analyte: reading.analyte,
|
||||
unit: reading.unit ?? "",
|
||||
referenceLow: reading.sourceReferenceLow,
|
||||
referenceHigh: reading.sourceReferenceHigh,
|
||||
panel: reading.panel,
|
||||
},
|
||||
{ index, catalogKey: reading.catalogKey },
|
||||
);
|
||||
|
||||
let storedValue = reading.value;
|
||||
let storedLow =
|
||||
reading.valueText === null ? reading.sourceReferenceLow : null;
|
||||
let storedHigh =
|
||||
reading.valueText === null ? reading.sourceReferenceHigh : null;
|
||||
let convertedFrom: { sourceValue: number; sourceUnit: string } | null =
|
||||
null;
|
||||
if (reading.value !== null) {
|
||||
const converted = await readingInMarkerUnit(client, {
|
||||
marker,
|
||||
value: reading.value,
|
||||
unit: reading.unit ?? undefined,
|
||||
sourceReferenceLow: storedLow,
|
||||
sourceReferenceHigh: storedHigh,
|
||||
});
|
||||
if (!converted.ok) {
|
||||
failed(plan.at, "unit_mismatch");
|
||||
continue;
|
||||
}
|
||||
storedValue = converted.value;
|
||||
storedLow = converted.sourceReferenceLow;
|
||||
storedHigh = converted.sourceReferenceHigh;
|
||||
if (converted.converted) {
|
||||
convertedFrom = {
|
||||
sourceValue: converted.sourceValue,
|
||||
sourceUnit: converted.sourceUnit,
|
||||
};
|
||||
}
|
||||
const built = await buildRow(client, userId, reading, index);
|
||||
if (!built.ok) {
|
||||
failed(plan.at, "unit_mismatch");
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.push({
|
||||
userId,
|
||||
biomarkerId: marker.id,
|
||||
panel: reading.panel ?? marker.panel,
|
||||
analyte: marker.name,
|
||||
value: storedValue,
|
||||
valueText: reading.valueText,
|
||||
unit: marker.unit,
|
||||
sourceValue: convertedFrom?.sourceValue ?? null,
|
||||
sourceUnit: convertedFrom?.sourceUnit ?? null,
|
||||
referenceLow: marker.lowerBound,
|
||||
referenceHigh: marker.upperBound,
|
||||
sourceReferenceLow: storedLow,
|
||||
sourceReferenceHigh: storedHigh,
|
||||
sourceReferenceText: reading.sourceReferenceText,
|
||||
takenAt: reading.takenAt,
|
||||
source: APPLE_HEALTH_LAB_SOURCE,
|
||||
externalId: reading.externalId,
|
||||
noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null,
|
||||
});
|
||||
rows.push(built.row);
|
||||
rowAt.set(reading.externalId, plan);
|
||||
for (const k of [
|
||||
...keys,
|
||||
...readingKeys({ ...probe, biomarkerId: marker.id }, tz),
|
||||
...readingKeys({ ...probe, biomarkerId: built.markerId }, tz),
|
||||
]) {
|
||||
seenKeys.add(k);
|
||||
}
|
||||
@@ -335,3 +238,128 @@ export async function writeFhirLabReadings(args: {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Duplicate keys of the live readings a chunk could collide with. */
|
||||
async function liveReadingKeys(
|
||||
client: LabWriteClient,
|
||||
userId: string,
|
||||
tz: string,
|
||||
planned: readonly Pending[],
|
||||
): Promise<Set<string>> {
|
||||
const keys = new Set<string>();
|
||||
if (planned.length === 0) return keys;
|
||||
const days = planned.map((p) => labReadingDay(p.reading.takenAt, tz)).sort();
|
||||
const ids = [
|
||||
...new Set(
|
||||
planned.map((p) => p.markerId).filter((id): id is string => !!id),
|
||||
),
|
||||
];
|
||||
const names = [...new Set(planned.map((p) => p.reading.analyte))];
|
||||
const existing = await client.labResult.findMany({
|
||||
where: {
|
||||
userId,
|
||||
deletedAt: null,
|
||||
takenAt: {
|
||||
gte: labReadingDaySearchRange(days[0]).gte,
|
||||
lt: labReadingDaySearchRange(days[days.length - 1]).lt,
|
||||
},
|
||||
OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }],
|
||||
},
|
||||
select: {
|
||||
biomarkerId: true,
|
||||
analyte: true,
|
||||
takenAt: true,
|
||||
value: true,
|
||||
valueText: true,
|
||||
sourceValue: true,
|
||||
sourceUnit: true,
|
||||
},
|
||||
});
|
||||
for (const e of existing) {
|
||||
for (const k of readingKeys(e, tz)) keys.add(k);
|
||||
// A reading filed under a marker also answers to its printed name.
|
||||
if (e.biomarkerId) {
|
||||
for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) keys.add(k);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* The row for one reading: its marker resolved or minted, the value moved
|
||||
* into the marker's unit when it states another. Built field by field.
|
||||
*/
|
||||
async function buildRow(
|
||||
client: LabWriteClient,
|
||||
userId: string,
|
||||
reading: MappedFhirLab,
|
||||
index: MarkerIndex,
|
||||
): Promise<
|
||||
| { ok: true; row: Prisma.LabResultCreateManyInput; markerId: string }
|
||||
| { ok: false }
|
||||
> {
|
||||
const marker = await resolveOrMintBiomarker(
|
||||
userId,
|
||||
{
|
||||
analyte: reading.analyte,
|
||||
unit: reading.unit ?? "",
|
||||
referenceLow: reading.sourceReferenceLow,
|
||||
referenceHigh: reading.sourceReferenceHigh,
|
||||
panel: reading.panel,
|
||||
},
|
||||
{ index, catalogKey: reading.catalogKey },
|
||||
);
|
||||
|
||||
// Like the CSV import: on a text result the window only seeds a new
|
||||
// marker; the reading itself records none.
|
||||
let storedValue = reading.value;
|
||||
let storedLow =
|
||||
reading.valueText === null ? reading.sourceReferenceLow : null;
|
||||
let storedHigh =
|
||||
reading.valueText === null ? reading.sourceReferenceHigh : null;
|
||||
let convertedFrom: { sourceValue: number; sourceUnit: string } | null = null;
|
||||
if (reading.value !== null) {
|
||||
const converted = await readingInMarkerUnit(client, {
|
||||
marker,
|
||||
value: reading.value,
|
||||
unit: reading.unit ?? undefined,
|
||||
sourceReferenceLow: storedLow,
|
||||
sourceReferenceHigh: storedHigh,
|
||||
});
|
||||
if (!converted.ok) return { ok: false };
|
||||
storedValue = converted.value;
|
||||
storedLow = converted.sourceReferenceLow;
|
||||
storedHigh = converted.sourceReferenceHigh;
|
||||
if (converted.converted) {
|
||||
convertedFrom = {
|
||||
sourceValue: converted.sourceValue,
|
||||
sourceUnit: converted.sourceUnit,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
markerId: marker.id,
|
||||
row: {
|
||||
userId,
|
||||
biomarkerId: marker.id,
|
||||
panel: reading.panel ?? marker.panel,
|
||||
analyte: marker.name,
|
||||
value: storedValue,
|
||||
valueText: reading.valueText,
|
||||
unit: marker.unit,
|
||||
sourceValue: convertedFrom?.sourceValue ?? null,
|
||||
sourceUnit: convertedFrom?.sourceUnit ?? null,
|
||||
referenceLow: marker.lowerBound,
|
||||
referenceHigh: marker.upperBound,
|
||||
sourceReferenceLow: storedLow,
|
||||
sourceReferenceHigh: storedHigh,
|
||||
sourceReferenceText: reading.sourceReferenceText,
|
||||
takenAt: reading.takenAt,
|
||||
source: APPLE_HEALTH_LAB_SOURCE,
|
||||
externalId: reading.externalId,
|
||||
noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -952,6 +952,44 @@ describe("streamParseExportXml — end-to-end", () => {
|
||||
expect(result.totals.recordsRead).toBeGreaterThanOrEqual(7);
|
||||
});
|
||||
|
||||
it("hands lab-result clinical records to the callback and skips the rest", async () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
|
||||
const xmlPath = join(tmp, "export.xml");
|
||||
writeFileSync(
|
||||
xmlPath,
|
||||
`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<HealthData locale="en_US">
|
||||
<ClinicalRecord type="HKClinicalTypeIdentifierLabResultRecord"
|
||||
identifier="obs-1" sourceName="Example Clinic"
|
||||
sourceURL="https://fhir.example-clinic.test/R4" fhirVersion="4.0.1"
|
||||
receivedDate="2025-03-05 10:00:00 +0100"
|
||||
resourceFilePath="/clinical-records/Observation-obs-1.json"/>
|
||||
<ClinicalRecord type="HKClinicalTypeIdentifierConditionRecord"
|
||||
identifier="cond-1" sourceName="Example Clinic" fhirVersion="4.0.1"
|
||||
resourceFilePath="/clinical-records/Condition-cond-1.json"/>
|
||||
</HealthData>`,
|
||||
);
|
||||
const refs: unknown[] = [];
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath,
|
||||
userId: "user-1",
|
||||
userTimezone: "Europe/Berlin",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
prisma: makeFakePrisma() as any,
|
||||
onClinicalLabRecord: (ref) => void refs.push(ref),
|
||||
});
|
||||
expect(refs).toEqual([
|
||||
{
|
||||
path: "/clinical-records/Observation-obs-1.json",
|
||||
identifier: "obs-1",
|
||||
sourceURL: "https://fhir.example-clinic.test/R4",
|
||||
sourceName: "Example Clinic",
|
||||
fhirVersion: "4.0.1",
|
||||
},
|
||||
]);
|
||||
expect(result.clinical.skipped).toBe(1);
|
||||
});
|
||||
|
||||
it("re-importing the same export reports 0 inserts and N updates", async () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
|
||||
const xmlPath = join(tmp, "export.xml");
|
||||
|
||||
@@ -45,6 +45,10 @@ import {
|
||||
canonicalDailyTimestamp,
|
||||
} from "@/lib/measurements/drain-per-sample-cumulative";
|
||||
import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin";
|
||||
import {
|
||||
routeClinicalRecord,
|
||||
type ClinicalLabReference,
|
||||
} from "@/lib/apple-health/clinical-record-ref";
|
||||
import {
|
||||
isSameReadingAcrossSource,
|
||||
MEASURED_AT_TOLERANCE_MS,
|
||||
@@ -460,6 +464,9 @@ export interface StreamParseInput {
|
||||
* pass, from the archive (`src/lib/apple-health/route-import.ts`).
|
||||
*/
|
||||
onWorkoutRoute?: (ref: { externalId: string; path: string }) => void;
|
||||
/** Called once per lab-result `<ClinicalRecord>`; without it every clinical
|
||||
* record is counted as skipped (`src/lib/apple-health/clinical-labs-import.ts`). */
|
||||
onClinicalLabRecord?: (ref: ClinicalLabReference) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1449,7 +1456,9 @@ export async function streamParseExportXml(
|
||||
|
||||
if (name === "ClinicalRecord") {
|
||||
recordsRead += 1;
|
||||
clinical.skipped += 1;
|
||||
if (!routeClinicalRecord(attrs, input.onClinicalLabRecord)) {
|
||||
clinical.skipped += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user