Merge branch 'feat/apple-clinical-labs' into release/v1.47.7

# Conflicts:
#	CHANGELOG.md
#	src/__tests__/baselines/source-file-lines.json
#	src/__tests__/baselines/source-function-lines.json
#	src/__tests__/write-rate-limit-guard.test.ts
This commit is contained in:
Marc-André Bombeck committed 2026-11-17 19:42:40 +01:00
commit 5bb77b975a
84 files changed
+4668 -252

No files matched your search

@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 84,
"total": 93804,
"total": 93813,
"entries": {
"app/medications/page-client.tsx": 842,
"components/charts/mood-chart.tsx": 1166,
@@ -72,7 +72,7 @@
"lib/mcp/record-search.ts": 1397,
"lib/mcp/rich-reads.ts": 1471,
"lib/measurements/dense-intraday-retention.ts": 1016,
"lib/measurements/import-apple-health-export.ts": 1722,
"lib/measurements/import-apple-health-export.ts": 1731,
"lib/medications/scheduling/cadence.ts": 976,
"lib/medications/scheduling/recurrence.ts": 1100,
"lib/medications/scheduling/slot-upsert.ts": 912,
@@ -1,7 +1,7 @@
{
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 335,
"total": 108911,
"total": 108902,
"entries": {
"app/achievements/page.tsx :: AchievementsPage": 206,
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 275,
@@ -192,7 +192,7 @@
"components/settings/coach-nudge-card.tsx :: CoachNudgeCard": 251,
"components/settings/coach-prefs-section.tsx :: CoachPrefsSection": 350,
"components/settings/environment-section.tsx :: EnvironmentSection": 666,
"components/settings/import-panel/apple-health-import-card.tsx :: AppleHealthImportCard": 298,
"components/settings/import-panel/apple-health-import-card.tsx :: AppleHealthImportCard": 293,
"components/settings/import-panel/csv-import-card.tsx :: CsvImportCard": 212,
"components/settings/import-panel/dose-history-import-card.tsx :: DoseHistoryImportCard": 221,
"components/settings/import-panel/health-connect-import-card.tsx :: HealthConnectImportCard": 293,
@@ -285,7 +285,7 @@
"lib/insights/status-provider.ts :: runStatusCompletion": 215,
"lib/integrations/connection-health-load.ts :: loadConnectionFacts": 229,
"lib/integrations/status.ts :: recordSyncFailure": 292,
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 300,
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 292,
"lib/jobs/coach-nudge.ts :: findTriggerForUser": 279,
"lib/jobs/coach-nudge.ts :: runCoachNudgeTick": 292,
"lib/jobs/coach-reminder-sweep.ts :: runCoachReminderSweep": 222,
@@ -314,8 +314,8 @@
"lib/measurements/dense-intraday-retention.ts :: runDenseIntradayRetention > walkSource > writeDay > foldOnce": 212,
"lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative": 344,
"lib/measurements/drain-per-sample-cumulative.ts :: drainPerSampleCumulative > writeDay": 205,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1245,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 323,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml": 1247,
"lib/measurements/import-apple-health-export.ts :: streamParseExportXml > (anonymous) #2": 325,
"lib/medications/efficacy/build-efficacy.ts :: buildMedicationEfficacy": 268,
"lib/medications/intake-slot-dedup.ts :: dedupeUserIntakeSlots": 319,
"lib/medications/list-read.ts :: buildMedicationsList": 310,
@@ -82,6 +82,8 @@ const OWN_BUCKET: Record<string, string> = {
"The cycle HealthKit sweep sends 500-day slices in sequence; the route's own bulk bucket meters it.",
"app/api/nutrients/batch/route.ts":
"The nutrient sync sends 500-row batches in sequence; the route's own batch bucket meters it, with a separate bucket for bridge tokens.",
"app/api/labs/fhir-batch/route.ts":
"The Health Records lab sync sends 200-record batches in sequence on a first sync; the route's own 60-a-minute batch bucket meters it.",
"app/api/insights/chat/route.ts":
"A Coach turn is metered by the Coach's own buckets: 20 turns a minute (`coach-chat`) and one turn slot at a time.",
"app/api/insights/chat/fenced/route.ts":
@@ -274,6 +274,9 @@ const EXEMPT_ROUTES: ReadonlyArray<string> = [
// capability, which carries the labs module, so it counts as gated.
"src/app/api/labs/ocr/capability/route.ts",
"src/app/api/labs/ocr/commit/route.ts",
// #1400 — Health Records lab results from the iOS app write the same row
// store as the export import, which runs whatever the module switch says.
"src/app/api/labs/fhir-batch/route.ts",
// ── DATA LAYER (medications) ──────────────────────────────────────
// v1.18.1 — medications graduated from CORE to a toggleable module,
// but it is SURFACE-gated (nav / dashboard widget / settings entry), not
@@ -198,7 +198,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
expect(mocks.importJobCreate).toHaveBeenCalledTimes(1);
expect(mocks.importJobCreate).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ parserRevision: 3 }),
data: expect.objectContaining({ parserRevision: 4 }),
}),
);
expect(mocks.bossSend).toHaveBeenCalledWith(
@@ -217,7 +217,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
expect.objectContaining({
where: expect.objectContaining({
status: { not: "failed" },
parserRevision: 3,
parserRevision: 4,
}),
}),
);
@@ -241,7 +241,7 @@ describe("POST /api/admin/import-apple-health-export", () => {
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({ parserRevision: 3 }),
where: expect.objectContaining({ parserRevision: 4 }),
}),
);
});
@@ -47,6 +47,10 @@ const ADMIN_SESSION = {
const NOW = new Date("2026-05-15T10:00:00.000Z");
function ownerSession(): Awaited<ReturnType<typeof getSession>> {
return OWNER_SESSION as unknown as Awaited<ReturnType<typeof getSession>>;
}
function makeRow(overrides: Partial<Record<string, unknown>> = {}) {
return {
id: "ij-1",
@@ -93,6 +97,54 @@ describe("GET /api/import/apple-health-export/[jobId]/status", () => {
expect(body.data.status).toBe("parsing");
});
it("bounds the lab counts and drops anything else in the block", async () => {
vi.mocked(getSession).mockResolvedValue(ownerSession());
mocks.importJobFindUnique.mockResolvedValue(
makeRow({
status: "done",
result: {
labs: {
discovered: 3,
imported: 2,
duplicate: -1,
skipped: "x",
failed: 1.5,
reasons: { leaked: "content" },
},
},
}),
);
const req = new NextRequest(
"http://localhost/api/import/apple-health-export/ij-1/status",
);
const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) });
const body = (await res.json()) as {
data: { result: { labs: Record<string, unknown> } };
};
expect(body.data.result.labs).toEqual({
discovered: 3,
imported: 2,
duplicate: 0,
skipped: 0,
failed: 0,
});
});
it("reports zeroed lab counts for a job from before labs were read", async () => {
vi.mocked(getSession).mockResolvedValue(ownerSession());
mocks.importJobFindUnique.mockResolvedValue(
makeRow({ status: "done", result: { totals: { recordsRead: 1 } } }),
);
const req = new NextRequest(
"http://localhost/api/import/apple-health-export/ij-1/status",
);
const res = await GET(req, { params: Promise.resolve({ jobId: "ij-1" }) });
const body = (await res.json()) as {
data: { result: { labs: Record<string, number> } };
};
expect(body.data.result.labs.imported).toBe(0);
});
it("returns 404 to a non-owner non-admin user", async () => {
vi.mocked(getSession).mockResolvedValue(OTHER_SESSION as never);
mocks.importJobFindUnique.mockResolvedValue(makeRow());
@@ -46,10 +46,12 @@ function privacySafeResult(
value: Record<string, unknown> | null,
): Record<string, unknown> | null {
if (!value) return null;
const ecg =
typeof value.ecg === "object" && value.ecg !== null
? (value.ecg as Record<string, unknown>)
const block = (key: string): Record<string, unknown> =>
typeof value[key] === "object" && value[key] !== null
? (value[key] as Record<string, unknown>)
: {};
const ecg = block("ecg");
const labs = block("labs");
return {
...value,
ecg: {
@@ -59,6 +61,13 @@ function privacySafeResult(
skipped: boundedCount(ecg.skipped),
failed: boundedCount(ecg.failed),
},
labs: {
discovered: boundedCount(labs.discovered),
imported: boundedCount(labs.imported),
duplicate: boundedCount(labs.duplicate),
skipped: boundedCount(labs.skipped),
failed: boundedCount(labs.failed),
},
};
}
@@ -200,7 +200,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
expect(mocks.importJobCreate).toHaveBeenCalledTimes(1);
expect(mocks.importJobCreate).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ parserRevision: 3 }),
data: expect.objectContaining({ parserRevision: 4 }),
}),
);
expect(mocks.bossSend).toHaveBeenCalledWith(
@@ -217,7 +217,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
expect.objectContaining({
where: expect.objectContaining({
status: { not: "failed" },
parserRevision: 3,
parserRevision: 4,
}),
}),
);
@@ -243,7 +243,7 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({ parserRevision: 3 }),
where: expect.objectContaining({ parserRevision: 4 }),
}),
);
});
+183
View File
@@ -0,0 +1,183 @@
/**
* `POST /api/labs/fhir-batch` — lab results from Apple Health's Health
* Records, sent by the iOS app (#1400).
*
* Each record is one FHIR resource as HealthKit hands it over. The batch is
* mapped with the same mapper the export import uses
* (`src/lib/labs/fhir-import/map-observation.ts`), panel members are given
* their panel's name within the batch, and the readings are written with the
* shared writer, so a result reads the same whichever way it came in and a
* result that already arrived through an export is a duplicate here.
*
* Sending a batch again is safe: a result already held under its identity is
* `duplicate`, a result the person deleted stays deleted, and an amended
* result arriving under the same identity is a duplicate (first write wins).
*
* Owner-only: a cookie session or a full-scope login token; a narrow token is
* refused by `requireAuth()`'s default arm. No module gate: labs is a data
* layer (see `src/app/api/labs/route.ts`). Metered by its own bulk bucket.
* Only counts and closed reasons reach the response, the audit row and the
* wide event; nothing from a resource does.
*/
import { NextRequest } from "next/server";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
import { parseJsonBody } from "@/lib/api/body";
import { auditLog } from "@/lib/auth/audit";
import { withIdempotency } from "@/lib/idempotency";
import { afterLabReadingsWritten } from "@/lib/labs/fhir-import/after-write";
import {
assignPanels,
mapFhirLabRecord,
type FhirLabMapping,
} from "@/lib/labs/fhir-import/map-observation";
import type {
FhirLabFailureReason,
FhirLabSkipReason,
MappedFhirLab,
} from "@/lib/labs/fhir-import/types";
import { writeFhirLabReadings } from "@/lib/labs/fhir-import/write";
import { annotate } from "@/lib/logging/context";
import { classifyWrittenOutcome } from "@/lib/outcome/written-outcome";
import { checkRateLimit } from "@/lib/rate-limit";
import { resolveUserTimezone } from "@/lib/tz/resolver";
import {
fhirLabBatchSchema,
MAX_FHIR_LAB_RESOURCE_BYTES,
} from "@/lib/validations/labs-fhir";
const BATCH_RATE_LIMIT_MAX = 60;
const BATCH_RATE_LIMIT_WINDOW_MS = 60 * 1000;
/** The proxy's body ceiling: the route stays inside the proxy matcher. */
const MAX_BODY_BYTES = 1024 * 1024;
type RecordResult =
| { index: number; status: "inserted" | "duplicate" }
| { index: number; status: "skipped"; reason: FhirLabSkipReason }
| { index: number; status: "failed"; reason: FhirLabFailureReason };
export const POST = apiHandler(withIdempotency<[NextRequest]>(postBatch));
async function postBatch(request: NextRequest): Promise<Response> {
const { user } = await requireAuth(undefined, { writeBucket: "own" });
annotate({ action: { name: "labs.fhir_batch.ingest" } });
const rl = await checkRateLimit(
`labs:fhir-batch:${user.id}`,
BATCH_RATE_LIMIT_MAX,
BATCH_RATE_LIMIT_WINDOW_MS,
);
if (!rl.allowed) {
return apiError("Too many batch submissions, try again later", 429);
}
const parsed = await parseJsonBody(request, fhirLabBatchSchema, {
maxBytes: MAX_BODY_BYTES,
action: "labs.fhir_batch.validation-failed",
countIssues: true,
errorCode: "labs.fhir_batch.invalid",
});
if (!parsed.ok) return parsed.error;
const { records } = parsed.data;
// 1. Map every record; nothing is written yet.
const results: RecordResult[] = new Array(records.length);
const mapped: Array<{ index: number; mapping: FhirLabMapping }> = [];
const now = new Date();
for (let index = 0; index < records.length; index += 1) {
const record = records[index];
if (
Buffer.byteLength(JSON.stringify(record.resource)) >
MAX_FHIR_LAB_RESOURCE_BYTES
) {
results[index] = { index, status: "skipped", reason: "too_large" };
continue;
}
const mapping = mapFhirLabRecord(
{
resource: record.resource,
fhirVersion: record.fhirVersion,
identifier: record.identifier,
sourceURL: record.sourceURL,
sourceName: record.sourceName,
},
now,
);
mapped.push({ index, mapping });
if (mapping.kind === "skip") {
results[index] = { index, status: "skipped", reason: mapping.reason };
} else if (mapping.kind === "fail") {
results[index] = { index, status: "failed", reason: mapping.reason };
}
}
// 2. A panel and its members arrive in one batch.
assignPanels(mapped.map((m) => m.mapping));
const readings: MappedFhirLab[] = [];
const readingIndex: number[] = [];
for (const { index, mapping } of mapped) {
if (mapping.kind !== "reading") continue;
readings.push(mapping.reading);
readingIndex.push(index);
}
// 3. Write.
const tz = await resolveUserTimezone(user.id);
const written = await writeFhirLabReadings({
userId: user.id,
tz,
readings,
});
written.statuses.forEach((status, i) => {
const index = readingIndex[i];
results[index] =
status.status === "failed"
? { index, status: "failed", reason: status.reason }
: { index, status: status.status };
});
afterLabReadingsWritten({
userId: user.id,
inserted: written.inserted,
newestTakenAt: written.newestTakenAt,
source: "fhir_batch",
});
const counts = { inserted: 0, duplicate: 0, skipped: 0, failed: 0 };
let writeFailed = false;
for (const result of results) {
counts[result.status] += 1;
if (result.status === "failed" && result.reason === "write_failed") {
writeFailed = true;
}
}
await auditLog("labs.fhir.batch", {
userId: user.id,
ipAddress: getClientIp(request),
details: { records: records.length, ...counts },
});
annotate({
meta: {
records: records.length,
inserted: counts.inserted,
duplicate: counts.duplicate,
skipped: counts.skipped,
failed: counts.failed,
},
});
const response = apiSuccess({
results,
...counts,
outcome: classifyWrittenOutcome({
written: counts.inserted,
skipped: counts.failed,
}),
});
// `write_failed` is transient: nothing was stored for those records and the
// client sends them again. `no-store` keeps the idempotency layer from
// replaying this answer to the retry.
if (writeFailed) response.headers.set("Cache-Control", "private, no-store");
return response;
}
@@ -0,0 +1,61 @@
/**
* The ECG and lab-result lines under a finished Apple Health import.
*/
import { describe, expect, it } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
import { I18nProvider } from "@/lib/i18n/context";
import type { MessageBundle } from "@/lib/i18n/load-locale";
import en from "../../../../../messages/en.json";
import de from "../../../../../messages/de.json";
import { AppleHealthAuxiliaryLines } from "../apple-health-auxiliary-lines";
function render(
props: Parameters<typeof AppleHealthAuxiliaryLines>[0],
locale: "en" | "de" = "en",
): string {
return renderToStaticMarkup(
<I18nProvider
initialLocale={locale}
initialMessages={(locale === "en" ? en : de) as unknown as MessageBundle}
>
<AppleHealthAuxiliaryLines {...props} />
</I18nProvider>,
);
}
describe("<AppleHealthAuxiliaryLines>", () => {
it("renders nothing for an archive with neither", () => {
expect(render({})).toBe("");
expect(render({ ecg: { discovered: 0 }, labs: { discovered: 0 } })).toBe(
"",
);
});
it("renders the lab line with every count", () => {
const html = render({
labs: { discovered: 7, imported: 4, duplicate: 1, skipped: 1, failed: 1 },
});
expect(html).toContain('data-testid="import-apple-health-labs-result"');
expect(html).toContain(
"Lab results from health records: 7 found, 4 imported, 1 already in your record, 1 skipped, 1 failed.",
);
expect(html).not.toContain("import-apple-health-ecg-result");
});
it("renders the ECG line beside it", () => {
const html = render({
ecg: { discovered: 2, imported: 2 },
labs: { discovered: 1, imported: 1 },
});
expect(html).toContain("import-apple-health-ecg-result");
expect(html).toContain("import-apple-health-labs-result");
});
it("speaks the active locale", () => {
expect(render({ labs: { discovered: 2, imported: 2 } }, "de")).toContain(
"Laborwerte aus Gesundheitsakten: 2 gefunden, 2 importiert",
);
});
});
@@ -0,0 +1,66 @@
"use client";
import { useTranslations } from "@/lib/i18n/context";
export interface AppleHealthEcgCounts {
discovered?: number;
imported?: number;
updated?: number;
skipped?: number;
failed?: number;
}
export interface AppleHealthLabCounts {
discovered?: number;
imported?: number;
duplicate?: number;
skipped?: number;
failed?: number;
}
/**
* The outcome lines of the passes that run after export.xml: the ECG
* recordings and the lab results from health records. A line shows only when
* the archive held something of its kind.
*/
export function AppleHealthAuxiliaryLines({
ecg,
labs,
}: {
ecg?: AppleHealthEcgCounts;
labs?: AppleHealthLabCounts;
}) {
const { t } = useTranslations();
return (
<>
{(ecg?.discovered ?? 0) > 0 && (
<p
data-testid="import-apple-health-ecg-result"
className="text-muted-foreground text-xs"
>
{t("settings.sections.export.import.appleHealth.ecgResult", {
discovered: ecg?.discovered ?? 0,
imported: ecg?.imported ?? 0,
updated: ecg?.updated ?? 0,
skipped: ecg?.skipped ?? 0,
failed: ecg?.failed ?? 0,
})}
</p>
)}
{(labs?.discovered ?? 0) > 0 && (
<p
data-testid="import-apple-health-labs-result"
className="text-muted-foreground text-xs"
>
{t("settings.sections.export.import.appleHealth.labsResult", {
discovered: labs?.discovered ?? 0,
imported: labs?.imported ?? 0,
duplicate: labs?.duplicate ?? 0,
skipped: labs?.skipped ?? 0,
failed: labs?.failed ?? 0,
})}
</p>
)}
</>
);
}
@@ -11,6 +11,11 @@ import { queryKeys } from "@/lib/query-keys";
import { useTranslations } from "@/lib/i18n/context";
import { cn } from "@/lib/utils";
import { apiFetchRaw, apiGet } from "@/lib/api/api-fetch";
import {
AppleHealthAuxiliaryLines,
type AppleHealthEcgCounts,
type AppleHealthLabCounts,
} from "./apple-health-auxiliary-lines";
import { ImportCardShell } from "./import-card-shell";
import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line";
import { classifyWrittenOutcome } from "@/lib/outcome/written-outcome";
@@ -49,13 +54,8 @@ interface JobStatus {
daysFailed?: number;
firstFailureReason?: string | null;
};
ecg?: {
discovered?: number;
imported?: number;
updated?: number;
skipped?: number;
failed?: number;
};
ecg?: AppleHealthEcgCounts;
labs?: AppleHealthLabCounts;
} | null;
failureReason: string | null;
}
@@ -389,7 +389,10 @@ export function AppleHealthImportCard() {
}
})();
const ecg = status?.result?.ecg;
const ecgWritten = (ecg?.imported ?? 0) + (ecg?.updated ?? 0);
const labs = status?.result?.labs;
const rowsWritten = status?.result?.totals?.rowsUpserted ?? 0;
const auxWritten =
(ecg?.imported ?? 0) + (ecg?.updated ?? 0) + (labs?.imported ?? 0);
const skips = summarizeAppleHealthSkips(status?.result ?? null);
const cycleStats = status?.result?.cycle;
@@ -482,18 +485,23 @@ export function AppleHealthImportCard() {
{isDone && (
<WrittenOutcomeLine
outcome={classifyWrittenOutcome({
written: (status?.result?.totals?.rowsUpserted ?? 0) + ecgWritten,
written: rowsWritten + auxWritten,
// Deliberate exclusions (clinical records, unsupported /
// deferred types, a disabled cycle module) are information
// and must not tip the archive into a warning. Genuine
// refusals — samples that SHOULD have landed (unparseable,
// out of range, upsert failure) plus failed cycle days — do.
skipped: skips.refused + (cycleStats?.daysFailed ?? 0),
skipped:
skips.refused +
(cycleStats?.daysFailed ?? 0) +
(labs?.failed ?? 0),
})}
message={
(status?.result?.totals?.rowsUpserted ?? 0) > 0
// An archive whose only new data is lab results or ECGs still
// imported something.
rowsWritten + auxWritten > 0
? t("settings.sections.export.import.appleHealth.doneSummary", {
imported: status?.result?.totals?.rowsUpserted ?? 0,
imported: rowsWritten,
read: status?.result?.totals?.recordsRead ?? 0,
skipped: status?.result?.clinical?.skipped ?? 0,
})
@@ -505,20 +513,7 @@ export function AppleHealthImportCard() {
/>
)}
{isDone && <AppleHealthSkipLines skips={skips} cycle={cycleStats} />}
{isDone && (ecg?.discovered ?? 0) > 0 && (
<p
data-testid="import-apple-health-ecg-result"
className="text-muted-foreground text-xs"
>
{t("settings.sections.export.import.appleHealth.ecgResult", {
discovered: ecg?.discovered ?? 0,
imported: ecg?.imported ?? 0,
updated: ecg?.updated ?? 0,
skipped: ecg?.skipped ?? 0,
failed: ecg?.failed ?? 0,
})}
</p>
)}
{isDone && <AppleHealthAuxiliaryLines ecg={ecg} labs={labs} />}
{isDone && ownRecordsSkipped > 0 && (
<AppleHealthOwnRecordsNote count={ownRecordsSkipped} />
)}
@@ -0,0 +1,155 @@
import { afterEach, describe, expect, it } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
clinicalMemberNameFromReference,
isClinicalRecordJson,
streamAppleHealthClinicalMembers,
streamAppleHealthEcgMembers,
streamAppleHealthRouteMembers,
} from "../archive-stream";
import { zip64Archive } from "./helpers/zip64";
const dirs: string[] = [];
afterEach(() => {
for (const dir of dirs.splice(0))
rmSync(dir, { recursive: true, force: true });
});
function write(bytes: Buffer): string {
const dir = mkdtempSync(join(tmpdir(), "healthlog-zip64-"));
dirs.push(dir);
const path = join(dir, "export.zip");
writeFileSync(path, bytes);
return path;
}
async function text(stream: AsyncIterable<unknown>): Promise<string> {
let out = "";
for await (const chunk of stream) out += String(chunk);
return out;
}
const LIMITS = {
maxMembers: 10,
maxMemberBytes: 4096,
maxCompressionRatio: 100,
};
const FILES = {
"apple_health_export/export.xml": "<HealthData/>",
"apple_health_export/electrocardiograms/ecg_2025-01-01.csv": "Name,x\n",
"apple_health_export/workout-routes/route_1.gpx": "<gpx/>",
"apple_health_export/clinical-records/Observation-1.json": '{"id":"1"}',
};
describe("archive-stream on a ZIP64 archive", () => {
it("streams ECG, route and clinical members", async () => {
const path = write(zip64Archive(FILES));
const ecg: string[] = [];
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 },
})) {
ecg.push(`${m.name}=${await text(m.stream)}`);
}
expect(ecg).toEqual([
"apple_health_export/electrocardiograms/ecg_2025-01-01.csv=Name,x\n",
]);
const routes: string[] = [];
for await (const m of streamAppleHealthRouteMembers({
archivePath: path,
limits: { ...LIMITS, maxRouteMembers: 5, maxTotalRouteBytes: 8192 },
wanted: () => true,
})) {
routes.push(await text(m.stream));
}
expect(routes).toEqual(["<gpx/>"]);
const clinical: string[] = [];
for await (const m of streamAppleHealthClinicalMembers({
archivePath: path,
limits: { ...LIMITS, maxClinicalMembers: 5, maxTotalClinicalBytes: 8192 },
wanted: () => true,
})) {
clinical.push(await text(m.stream));
}
expect(clinical).toEqual(['{"id":"1"}']);
});
it("refuses ZIP64 sentinels without the locator", async () => {
const path = write(zip64Archive(FILES, { omitLocator: true }));
const run = async () => {
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 },
})) {
void m;
}
};
await expect(run()).rejects.toThrow(/ZIP64 locator/);
});
it("refuses a ZIP64 member without its extra field", async () => {
const path = write(zip64Archive(FILES, { omitExtra: true }));
const run = async () => {
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 },
})) {
void m;
}
};
await expect(run()).rejects.toThrow(/malformed ZIP64 member/);
});
it("still enforces the member count from the ZIP64 record", async () => {
const path = write(zip64Archive(FILES));
const run = async () => {
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: {
...LIMITS,
maxMembers: 3,
maxEcgMembers: 5,
maxTotalEcgBytes: 8192,
},
})) {
void m;
}
};
await expect(run()).rejects.toThrow(/member count limit/);
});
});
describe("clinical member names", () => {
it("maps Apple's reference path onto the archive member", () => {
expect(
clinicalMemberNameFromReference("/clinical-records/Observation-A.json"),
).toBe("apple_health_export/clinical-records/Observation-A.json");
expect(
clinicalMemberNameFromReference("clinical-records/Observation-A.json"),
).toBe("apple_health_export/clinical-records/Observation-A.json");
});
it.each([
"/clinical-records/../export.xml",
"/clinical-records/sub/Observation.json",
"/workout-routes/route.gpx",
"/clinical-records/Observation.txt",
])("refuses %s", (path) => {
expect(clinicalMemberNameFromReference(path)).toBeNull();
});
it("recognises only direct JSON children", () => {
expect(
isClinicalRecordJson("apple_health_export/clinical-records/a.JSON"),
).toBe(true);
expect(isClinicalRecordJson("apple_health_export/clinical-records/")).toBe(
false,
);
});
});
@@ -0,0 +1,225 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { strToU8, zipSync } from "fflate";
const mocks = vi.hoisted(() => ({
write: vi.fn(),
after: vi.fn(),
annotate: vi.fn(),
}));
vi.mock("@/lib/labs/fhir-import/write", () => ({
writeFhirLabReadings: mocks.write,
}));
vi.mock("@/lib/labs/fhir-import/after-write", () => ({
afterLabReadingsWritten: mocks.after,
}));
vi.mock("@/lib/logging/context", () => ({ annotate: mocks.annotate }));
import { importAppleHealthClinicalLabs } from "../clinical-labs-import";
import type { ClinicalLabReference } from "../clinical-record-ref";
import { zip64Archive } from "./helpers/zip64";
const dirs: string[] = [];
afterEach(() => {
for (const dir of dirs.splice(0))
rmSync(dir, { recursive: true, force: true });
});
beforeEach(() => {
vi.clearAllMocks();
mocks.write.mockImplementation(
async ({ readings }: { readings: unknown[] }) => ({
statuses: readings.map(() => ({ status: "inserted" })),
inserted: readings.length,
duplicate: 0,
failed: 0,
newestTakenAt: new Date("2025-01-01T00:00:00Z"),
}),
);
});
const ROOT = "apple_health_export/clinical-records/";
function observation(id: string, extra: Record<string, unknown> = {}) {
return JSON.stringify({
resourceType: "Observation",
id,
status: "final",
category: [{ coding: [{ code: "laboratory" }] }],
code: { text: `Analyte ${id}` },
effectiveDateTime: "2025-01-01T08:00:00Z",
valueQuantity: { value: 1, unit: "mmol/L" },
...extra,
});
}
function archive(files: Record<string, string>, zip64 = false): string {
const dir = mkdtempSync(join(tmpdir(), "healthlog-clinical-"));
dirs.push(dir);
const path = join(dir, "export.zip");
writeFileSync(
path,
zip64
? zip64Archive(files)
: zipSync(
Object.fromEntries(
Object.entries(files).map(([name, body]) => [name, strToU8(body)]),
),
),
);
return path;
}
function ref(
path: string | null,
identifier: string | null = null,
): ClinicalLabReference {
return {
path,
identifier,
sourceURL: "https://fhir.example.test",
sourceName: "Example",
fhirVersion: "4.0.1",
};
}
function run(path: string, references: ClinicalLabReference[], limits = {}) {
return importAppleHealthClinicalLabs({
archivePath: path,
userId: "user-1",
tz: "Europe/Berlin",
references,
limits: {
maxMembers: 50,
maxClinicalMembers: 20,
maxMemberBytes: 64 * 1024,
maxTotalClinicalBytes: 1024 * 1024,
maxCompressionRatio: 200,
...limits,
},
});
}
describe("importAppleHealthClinicalLabs", () => {
it("reads only referenced members and writes their readings with panels", async () => {
const path = archive({
"apple_health_export/export.xml": "<HealthData/>",
[`${ROOT}Observation-a.json`]: observation("a"),
[`${ROOT}Observation-panel.json`]: observation("panel", {
code: { text: "Electrolytes" },
valueQuantity: undefined,
hasMember: [{ reference: "Observation/a" }],
}),
[`${ROOT}Condition-c.json`]: '{"resourceType":"Condition"}',
});
const counts = await run(path, [
ref("/clinical-records/Observation-a.json", "a"),
ref("/clinical-records/Observation-panel.json", "panel"),
]);
expect(counts).toEqual({
discovered: 2,
imported: 1,
duplicate: 0,
skipped: 1,
failed: 0,
});
const [{ readings, tz, userId }] = mocks.write.mock.calls[0];
expect(userId).toBe("user-1");
expect(tz).toBe("Europe/Berlin");
expect(readings).toHaveLength(1);
expect(readings[0]).toMatchObject({
analyte: "Analyte a",
panel: "Electrolytes",
});
expect(mocks.after).toHaveBeenCalledWith(
expect.objectContaining({ userId: "user-1", inserted: 1 }),
);
});
it("works on a ZIP64 archive", async () => {
const path = archive(
{ [`${ROOT}Observation-a.json`]: observation("a") },
true,
);
const counts = await run(path, [
ref("/clinical-records/Observation-a.json"),
]);
expect(counts.imported).toBe(1);
});
it("counts a missing member, malformed JSON and an unsafe path as failed", async () => {
const path = archive({ [`${ROOT}Observation-bad.json`]: "{not json" });
const counts = await run(path, [
ref("/clinical-records/Observation-bad.json"),
ref("/clinical-records/Observation-gone.json"),
ref("/clinical-records/../export.xml"),
]);
expect(counts).toMatchObject({ discovered: 3, imported: 0, failed: 3 });
expect(mocks.write).not.toHaveBeenCalled();
});
it("finds a reference without a path by the resource id", async () => {
const path = archive({
[`${ROOT}one.json`]: observation("a"),
[`${ROOT}two.json`]: observation("unreferenced"),
});
const counts = await run(path, [ref(null, "a")]);
expect(counts).toMatchObject({ discovered: 1, imported: 1, failed: 0 });
expect(mocks.write.mock.calls[0][0].readings).toHaveLength(1);
});
it("refuses references past the clinical member ceiling", async () => {
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
const counts = await run(
path,
[
ref("/clinical-records/Observation-a.json"),
ref("/clinical-records/Observation-b.json"),
],
{ maxClinicalMembers: 1 },
);
expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 });
});
it("fails soft when the archive breaks a limit", async () => {
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
const counts = await run(
path,
[ref("/clinical-records/Observation-a.json")],
{
maxMemberBytes: 10,
},
);
expect(counts).toMatchObject({ discovered: 1, imported: 0, failed: 1 });
});
it("annotates counts and closed reasons only", async () => {
const path = archive({
[`${ROOT}Observation-a.json`]: observation("a", {
code: { text: "Secret analyte name" },
status: "preliminary",
}),
});
await run(path, [ref("/clinical-records/Observation-a.json")]);
const serialised = JSON.stringify(mocks.annotate.mock.calls);
expect(serialised).not.toContain("Secret analyte name");
expect(serialised).not.toContain("fhir.example.test");
expect(mocks.annotate).toHaveBeenCalledWith(
expect.objectContaining({
action: { name: "apple_health.import.clinical_labs" },
meta: expect.objectContaining({
skipped: 1,
reasons: { not_final: 1 },
}),
}),
);
});
it("does nothing for no references", async () => {
const counts = await run("/nonexistent.zip", []);
expect(counts.discovered).toBe(0);
expect(mocks.write).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,103 @@
/**
* A minimal ZIP64 writer for tests: stored members, every size and offset
* moved into the ZIP64 extra field and the ZIP64 end record, the way a large
* Health app export is written. fflate writes classic ZIP only.
*/
export function zip64Archive(
files: Record<string, string>,
options: { omitLocator?: boolean; omitExtra?: boolean } = {},
): Buffer {
const locals: Buffer[] = [];
const centrals: Buffer[] = [];
let offset = 0;
for (const [name, body] of Object.entries(files)) {
const nameBytes = Buffer.from(name, "utf8");
const data = Buffer.from(body, "utf8");
const crc = crc32(data);
const localExtra = Buffer.alloc(20);
localExtra.writeUInt16LE(0x0001, 0);
localExtra.writeUInt16LE(16, 2);
localExtra.writeBigUInt64LE(BigInt(data.length), 4);
localExtra.writeBigUInt64LE(BigInt(data.length), 12);
const local = Buffer.alloc(30);
local.writeUInt32LE(0x04034b50, 0);
local.writeUInt16LE(45, 4);
local.writeUInt16LE(0, 6);
local.writeUInt16LE(0, 8);
local.writeUInt32LE(crc, 14);
local.writeUInt32LE(0xffffffff, 18);
local.writeUInt32LE(0xffffffff, 22);
local.writeUInt16LE(nameBytes.length, 26);
local.writeUInt16LE(localExtra.length, 28);
const localRecord = Buffer.concat([local, nameBytes, localExtra, data]);
const centralExtra = Buffer.alloc(28);
centralExtra.writeUInt16LE(0x0001, 0);
centralExtra.writeUInt16LE(24, 2);
centralExtra.writeBigUInt64LE(BigInt(data.length), 4);
centralExtra.writeBigUInt64LE(BigInt(data.length), 12);
centralExtra.writeBigUInt64LE(BigInt(offset), 20);
const extra = options.omitExtra ? Buffer.alloc(0) : centralExtra;
const central = Buffer.alloc(46);
central.writeUInt32LE(0x02014b50, 0);
central.writeUInt16LE(45, 4);
central.writeUInt16LE(45, 6);
central.writeUInt16LE(0, 8);
central.writeUInt16LE(0, 10);
central.writeUInt32LE(crc, 16);
central.writeUInt32LE(0xffffffff, 20);
central.writeUInt32LE(0xffffffff, 24);
central.writeUInt16LE(nameBytes.length, 28);
central.writeUInt16LE(extra.length, 30);
central.writeUInt32LE(0xffffffff, 42);
centrals.push(Buffer.concat([central, nameBytes, extra]));
locals.push(localRecord);
offset += localRecord.length;
}
const centralDirectory = Buffer.concat(centrals);
const centralOffset = offset;
const count = Object.keys(files).length;
const record = Buffer.alloc(56);
record.writeUInt32LE(0x06064b50, 0);
record.writeBigUInt64LE(BigInt(44), 4);
record.writeUInt16LE(45, 12);
record.writeUInt16LE(45, 14);
record.writeBigUInt64LE(BigInt(count), 24);
record.writeBigUInt64LE(BigInt(count), 32);
record.writeBigUInt64LE(BigInt(centralDirectory.length), 40);
record.writeBigUInt64LE(BigInt(centralOffset), 48);
const locator = Buffer.alloc(20);
locator.writeUInt32LE(0x07064b50, 0);
locator.writeBigUInt64LE(BigInt(centralOffset + centralDirectory.length), 8);
locator.writeUInt32LE(1, 16);
const eocd = Buffer.alloc(22);
eocd.writeUInt32LE(0x06054b50, 0);
eocd.writeUInt16LE(0xffff, 8);
eocd.writeUInt16LE(0xffff, 10);
eocd.writeUInt32LE(0xffffffff, 12);
eocd.writeUInt32LE(0xffffffff, 16);
return Buffer.concat([
...locals,
centralDirectory,
record,
...(options.omitLocator ? [] : [locator]),
eocd,
]);
}
function crc32(data: Buffer): number {
let crc = 0xffffffff;
for (const byte of data) {
crc ^= byte;
for (let bit = 0; bit < 8; bit += 1) {
crc = crc & 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
}
}
return (crc ^ 0xffffffff) >>> 0;
}
+165 -16
View File
@@ -1,6 +1,7 @@
/**
* Bounded ZIP reader for Apple Health's auxiliary members: the ECG CSVs and
* the workout-route GPX files.
* Bounded ZIP reader for Apple Health's auxiliary members: the ECG CSVs, the
* workout-route GPX files and the Health Records FHIR JSON files. Reads ZIP64
* archives too, which is what the Health app writes for a large export.
*
* ZIP entry names are treated as untrusted identifiers only: no archive member
* is ever materialised on disk. Central-directory metadata is checked before
@@ -36,9 +37,13 @@ interface ZipEntry {
const EOCD_SIGNATURE = 0x06054b50;
const CENTRAL_SIGNATURE = 0x02014b50;
const LOCAL_SIGNATURE = 0x04034b50;
const ZIP64_EOCD_SIGNATURE = 0x06064b50;
const ZIP64_LOCATOR_SIGNATURE = 0x07064b50;
const ZIP64_EXTRA_ID = 0x0001;
const MAX_EOCD_SEARCH = 65_535 + 22;
const ECG_PREFIX = "apple_health_export/electrocardiograms/";
const ROUTE_PREFIX = "apple_health_export/workout-routes/";
const CLINICAL_PREFIX = "apple_health_export/clinical-records/";
/**
* One family of auxiliary members: which names belong to it, the label its
@@ -93,6 +98,48 @@ async function readExactly(
return buffer;
}
/** A 64-bit ZIP field as a number, refused past the safe-integer range. */
function safeNumber(value: bigint): number {
if (value > BigInt(Number.MAX_SAFE_INTEGER)) {
throw archiveError("ZIP64 value is too large to address safely");
}
return Number(value);
}
/**
* The ZIP64 extended-information values a central-directory entry asked for.
* A sentinel without its value in the extra field is a malformed archive.
*/
function zip64ExtraValues(
extra: Buffer,
wanted: { uncompressed: boolean; compressed: boolean; offset: boolean },
): { uncompressed?: number; compressed?: number; offset?: number } {
let cursor = 0;
while (cursor + 4 <= extra.length) {
const id = extra.readUInt16LE(cursor);
const size = extra.readUInt16LE(cursor + 2);
const end = cursor + 4 + size;
if (end > extra.length) break;
if (id === ZIP64_EXTRA_ID) {
const out: {
uncompressed?: number;
compressed?: number;
offset?: number;
} = {};
let at = cursor + 4;
for (const field of ["uncompressed", "compressed", "offset"] as const) {
if (!wanted[field]) continue;
if (at + 8 > end) throw archiveError("malformed ZIP64 member");
out[field] = safeNumber(extra.readBigUInt64LE(at));
at += 8;
}
return out;
}
cursor = end;
}
throw archiveError("malformed ZIP64 member");
}
function isUnsafeName(name: string): boolean {
if (
name.length === 0 ||
@@ -138,6 +185,23 @@ export function routeMemberNameFromReference(path: string): string | null {
return isWorkoutRouteGpx(name) ? name : null;
}
/** A Health Records resource as the archive names it: `clinical-records/<file>.json`. */
export function isClinicalRecordJson(name: string): boolean {
return isDirectChild(name, CLINICAL_PREFIX, ".json");
}
/**
* The archive member a `<ClinicalRecord resourceFilePath="…">` points at.
* Apple writes the path from the export root
* (`/clinical-records/Observation-….json`); the member carries the export
* folder in front. Null for anything that is not a direct clinical file.
*/
export function clinicalMemberNameFromReference(path: string): string | null {
const trimmed = path.startsWith("/") ? path.slice(1) : path;
const name = `apple_health_export/${trimmed}`;
return isClinicalRecordJson(name) ? name : null;
}
async function readCentralDirectory(
handle: FileHandle,
fileSize: number,
@@ -160,18 +224,47 @@ async function readCentralDirectory(
const disk = tail.readUInt16LE(eocdOffset + 4);
const centralDisk = tail.readUInt16LE(eocdOffset + 6);
const diskEntries = tail.readUInt16LE(eocdOffset + 8);
const totalEntries = tail.readUInt16LE(eocdOffset + 10);
const centralSize = tail.readUInt32LE(eocdOffset + 12);
const centralOffset = tail.readUInt32LE(eocdOffset + 16);
let totalEntries = tail.readUInt16LE(eocdOffset + 10);
let centralSize = tail.readUInt32LE(eocdOffset + 12);
let centralOffset = tail.readUInt32LE(eocdOffset + 16);
if (disk !== 0 || centralDisk !== 0 || diskEntries !== totalEntries) {
throw archiveError("unsupported multi-disk archive");
}
// ZIP64: a sentinel in any of the three fields means the real values live
// in the ZIP64 end record, found through the locator just before the end
// record. The Health app writes one for a large export.
if (
disk !== 0 ||
centralDisk !== 0 ||
diskEntries !== totalEntries ||
totalEntries === 0xffff ||
centralSize === 0xffffffff ||
centralOffset === 0xffffffff
) {
throw archiveError("unsupported multi-disk or ZIP64 archive");
const locatorOffset = fileSize - tailLength + eocdOffset - 20;
if (locatorOffset < 0) throw archiveError("missing ZIP64 locator");
const locator = await readExactly(handle, 20, locatorOffset);
if (locator.readUInt32LE(0) !== ZIP64_LOCATOR_SIGNATURE) {
throw archiveError("missing ZIP64 locator");
}
if (locator.readUInt32LE(4) !== 0 || locator.readUInt32LE(16) > 1) {
throw archiveError("unsupported multi-disk archive");
}
const recordOffset = safeNumber(locator.readBigUInt64LE(8));
if (recordOffset + 56 > fileSize) {
throw archiveError("invalid ZIP64 directory bounds");
}
const record = await readExactly(handle, 56, recordOffset);
if (record.readUInt32LE(0) !== ZIP64_EOCD_SIGNATURE) {
throw archiveError("malformed ZIP64 directory");
}
if (record.readUInt32LE(16) !== 0 || record.readUInt32LE(20) !== 0) {
throw archiveError("unsupported multi-disk archive");
}
const recordDiskEntries = safeNumber(record.readBigUInt64LE(24));
totalEntries = safeNumber(record.readBigUInt64LE(32));
centralSize = safeNumber(record.readBigUInt64LE(40));
centralOffset = safeNumber(record.readBigUInt64LE(48));
if (recordDiskEntries !== totalEntries) {
throw archiveError("unsupported multi-disk archive");
}
}
if (totalEntries > limits.maxMembers) {
throw archiveError("member count limit exceeded");
@@ -192,21 +285,37 @@ async function readCentralDirectory(
}
const flags = header.readUInt16LE(8);
const method = header.readUInt16LE(10);
const compressedSize = header.readUInt32LE(20);
const uncompressedSize = header.readUInt32LE(24);
let compressedSize = header.readUInt32LE(20);
let uncompressedSize = header.readUInt32LE(24);
const nameLength = header.readUInt16LE(28);
const extraLength = header.readUInt16LE(30);
const commentLength = header.readUInt16LE(32);
const localHeaderOffset = header.readUInt32LE(42);
let localHeaderOffset = header.readUInt32LE(42);
const nameBuffer = await readExactly(handle, nameLength, cursor + 46);
if (
compressedSize === 0xffffffff ||
uncompressedSize === 0xffffffff ||
localHeaderOffset === 0xffffffff
) {
throw archiveError("unsupported ZIP64 member");
// ZIP64 member: the real values sit in the extended-information extra
// field (id 0x0001), in this order, each present only when its 32-bit
// field holds the sentinel.
const extra = await readExactly(
handle,
extraLength,
cursor + 46 + nameLength,
);
const sizes = zip64ExtraValues(extra, {
uncompressed: uncompressedSize === 0xffffffff,
compressed: compressedSize === 0xffffffff,
offset: localHeaderOffset === 0xffffffff,
});
if (sizes.uncompressed !== undefined)
uncompressedSize = sizes.uncompressed;
if (sizes.compressed !== undefined) compressedSize = sizes.compressed;
if (sizes.offset !== undefined) localHeaderOffset = sizes.offset;
}
const nameBuffer = await readExactly(handle, nameLength, cursor + 46);
const name = nameBuffer.toString("utf8");
if (isUnsafeName(name)) throw archiveError("unsafe member path");
if (names.has(name)) throw archiveError("duplicate member name");
@@ -284,9 +393,11 @@ async function decompressEntry(
entry.compressedSize,
dataOffset,
);
// A stored member is read through a copy: the read buffer is wiped below,
// and the stored bytes are the member's content.
const source =
entry.method === 0
? Readable.from([compressed])
? Readable.from([Buffer.from(compressed)])
: Readable.from([compressed]).pipe(createInflateRaw());
const chunks: Buffer[] = [];
let memberBytes = 0;
@@ -403,3 +514,41 @@ export async function* streamAppleHealthRouteMembers(input: {
input.wanted,
);
}
/** Ceilings for the Health Records members; the shared ones keep their meaning. */
export interface AppleHealthClinicalArchiveLimits {
maxMembers: number;
maxClinicalMembers: number;
maxMemberBytes: number;
maxTotalClinicalBytes: number;
maxCompressionRatio: number;
}
/**
* Yield the Health Records JSON members `wanted` names, one bounded buffer at
* a time: the caller parses and drops each before the next is inflated.
*/
export async function* streamAppleHealthClinicalMembers(input: {
archivePath: string;
limits: AppleHealthClinicalArchiveLimits;
wanted: (name: string) => boolean;
}): AsyncGenerator<AppleHealthArchiveMember> {
const { limits } = input;
yield* streamMembers(
input.archivePath,
{
maxMembers: limits.maxMembers,
maxEcgMembers: limits.maxClinicalMembers,
maxMemberBytes: limits.maxMemberBytes,
maxTotalEcgBytes: limits.maxTotalClinicalBytes,
maxCompressionRatio: limits.maxCompressionRatio,
},
{
label: "clinical record",
matches: isClinicalRecordJson,
maxMembers: limits.maxClinicalMembers,
maxTotalBytes: limits.maxTotalClinicalBytes,
},
input.wanted,
);
}
@@ -0,0 +1,220 @@
/**
* Lab results from Apple Health's Health Records, out of an `export.zip`
* (#1400).
*
* `export.xml` lists each Health Records resource as a `<ClinicalRecord>`
* whose `resourceFilePath` names a FHIR JSON file under `clinical-records/`.
* The XML pass hands the lab-result references here
* (`routeClinicalRecord`); this module then reads the referenced members one
* at a time, maps each with the shared FHIR mapper, gives members their
* panel's name, and writes through the shared writer, so a result lands the
* same way it would from the iOS app (`POST /api/labs/fhir-batch`).
*
* A reference with no path (an export from a Health app that did not write
* one) is matched by the resource's own id among the clinical files instead.
*
* Memory: one member is held at a time, capped by the archive reader; what is
* kept across members is the mapped readings (a few hundred bytes each) and
* the panel headers' member lists.
*
* A failure here never fails the import: the measurements are committed. A
* missing, oversized or malformed file is counted as failed. Nothing about a
* resource reaches the result, the logs or a wide event: only counts and
* closed reasons.
*/
import type { PrismaClient } from "@/generated/prisma/client";
import {
clinicalMemberNameFromReference,
isClinicalRecordJson,
streamAppleHealthClinicalMembers,
type AppleHealthClinicalArchiveLimits,
} from "@/lib/apple-health/archive-stream";
import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref";
import { afterLabReadingsWritten } from "@/lib/labs/fhir-import/after-write";
import {
assignPanels,
mapFhirLabRecord,
type FhirLabMapping,
} from "@/lib/labs/fhir-import/map-observation";
import {
emptyFhirLabCounts,
type FhirLabCounts,
} from "@/lib/labs/fhir-import/types";
import { writeFhirLabReadings } from "@/lib/labs/fhir-import/write";
import { annotate } from "@/lib/logging/context";
import type { ImportJobResult } from "@/lib/measurements/import-apple-health-export";
/** The terminal result of an Apple Health import job, labs included. */
export type AppleHealthImportJobResult = ImportJobResult & {
labs: FhirLabCounts;
};
/** Per-archive ceilings for the Health Records members. */
export const APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS: AppleHealthClinicalArchiveLimits =
{
maxMembers: 200_000,
maxClinicalMembers: 50_000,
// A single Observation is a few KB; 2 MiB leaves room for a long note.
maxMemberBytes: 2 * 1024 * 1024,
maxTotalClinicalBytes: 256 * 1024 * 1024,
maxCompressionRatio: 200,
};
async function readJson(stream: AsyncIterable<unknown>): Promise<unknown> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
}
const text = Buffer.concat(chunks).toString("utf8");
for (const chunk of chunks) chunk.fill(0);
return JSON.parse(text) as unknown;
}
function resourceId(resource: unknown): string | null {
if (typeof resource !== "object" || resource === null) return null;
const id = (resource as { id?: unknown }).id;
return typeof id === "string" && id.trim() !== "" ? id.trim() : null;
}
export async function importAppleHealthClinicalLabs(input: {
archivePath: string;
userId: string;
tz: string;
references: readonly ClinicalLabReference[];
prisma?: Pick<PrismaClient, "labResult" | "biomarker" | "biomarkerAlias">;
limits?: AppleHealthClinicalArchiveLimits;
}): Promise<FhirLabCounts> {
const counts = emptyFhirLabCounts();
const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS;
counts.discovered = input.references.length;
if (input.references.length === 0) return counts;
const tally: Record<string, number> = {};
const note = (reason: string) => {
tally[reason] = (tally[reason] ?? 0) + 1;
};
// More references than the archive may hold clinical files: the overflow
// is refused, the rest is read.
const usable = input.references.slice(0, limits.maxClinicalMembers);
const overflow = input.references.length - usable.length;
counts.failed += overflow;
if (overflow > 0) tally.over_limit = overflow;
const byMember = new Map<string, ClinicalLabReference>();
const byId = new Map<string, ClinicalLabReference>();
for (const ref of usable) {
const member = ref.path ? clinicalMemberNameFromReference(ref.path) : null;
if (member && !byMember.has(member)) byMember.set(member, ref);
else if (!ref.path && ref.identifier && !byId.has(ref.identifier)) {
byId.set(ref.identifier, ref);
} else {
// An unsafe path, or a second reference to one file.
counts.failed += 1;
note("bad_reference");
}
}
const expected = byMember.size + byId.size;
let handled = 0;
const mapped: FhirLabMapping[] = [];
try {
for await (const member of streamAppleHealthClinicalMembers({
archivePath: input.archivePath,
limits,
wanted: (name) =>
byMember.has(name) || (byId.size > 0 && isClinicalRecordJson(name)),
})) {
const byPath = byMember.get(member.name);
let resource: unknown;
try {
resource = await readJson(member.stream);
} catch {
if (byPath) {
handled += 1;
counts.failed += 1;
note("malformed_json");
}
continue;
}
let ref = byPath;
if (!ref) {
const id = resourceId(resource);
ref = id ? byId.get(id) : undefined;
if (!ref || !id) continue;
byId.delete(id);
}
handled += 1;
const result = mapFhirLabRecord({
resource,
fhirVersion: ref.fhirVersion,
identifier: ref.identifier,
sourceURL: ref.sourceURL,
sourceName: ref.sourceName,
});
resource = null;
if (result.kind === "reading") {
mapped.push(result);
} else if (result.kind === "skip") {
counts.skipped += 1;
note(result.reason);
if (result.reason === "panel_header") mapped.push(result);
} else {
counts.failed += 1;
note(result.reason);
}
}
} catch {
// The archive itself is unreadable for this pass (a limit hit in the
// directory, an inflate failure): the files not yet read are counted
// below as missing. What was read is still written.
note("archive_unreadable");
}
const missing = Math.max(0, expected - handled);
counts.failed += missing;
if (missing > 0) tally.missing_member = (tally.missing_member ?? 0) + missing;
assignPanels(mapped);
const readings = mapped.flatMap((m) =>
m.kind === "reading" ? [m.reading] : [],
);
mapped.length = 0;
if (readings.length > 0) {
try {
const written = await writeFhirLabReadings({
userId: input.userId,
tz: input.tz,
readings,
prisma: input.prisma,
});
counts.imported += written.inserted;
counts.duplicate += written.duplicate;
counts.failed += written.failed;
for (const status of written.statuses) {
if (status.status === "failed") note(status.reason);
}
afterLabReadingsWritten({
userId: input.userId,
inserted: written.inserted,
newestTakenAt: written.newestTakenAt,
source: "apple_health_export",
});
} catch {
counts.failed += readings.length;
note("write_failed");
}
}
annotate({
action: { name: "apple_health.import.clinical_labs" },
meta: {
discovered: counts.discovered,
imported: counts.imported,
duplicate: counts.duplicate,
skipped: counts.skipped,
failed: counts.failed,
reasons: tally,
},
});
return counts;
}
@@ -0,0 +1,61 @@
/**
* A `<ClinicalRecord>` in `export.xml`, as a reference to the lab result it
* describes (#1400).
*
* Apple writes one element per Health Records resource:
*
* <ClinicalRecord type="HKClinicalTypeIdentifierLabResultRecord"
* identifier="…" sourceName="…" sourceURL="…" fhirVersion="4.0.1"
* receivedDate="…" resourceFilePath="/clinical-records/….json"/>
*
* The resource itself is the JSON file the path names. Only lab results are
* handed on; every other clinical type (conditions, medications, vital signs,
* immunisations) is left to the caller's skipped count. The attributes are
* capped here so a hostile export cannot make the parser hold long strings.
*/
export const HK_LAB_RESULT_RECORD = "HKClinicalTypeIdentifierLabResultRecord";
export interface ClinicalLabReference {
/** `resourceFilePath`, from the export root. Null when Apple wrote none. */
path: string | null;
identifier: string | null;
sourceURL: string | null;
sourceName: string | null;
fhirVersion: string | null;
}
const CAPS = {
path: 512,
identifier: 256,
sourceURL: 2048,
sourceName: 200,
fhirVersion: 16,
} as const;
function withinCap(raw: string | undefined, max: number): string | null {
if (raw === undefined) return null;
const trimmed = raw.trim();
if (trimmed === "" || trimmed.length > max) return null;
return trimmed;
}
/**
* Hand a lab-result record to `onLab` and return true; return false for any
* other record, or when there is no callback, so the caller counts it as
* skipped.
*/
export function routeClinicalRecord(
attrs: Record<string, string | undefined>,
onLab?: (ref: ClinicalLabReference) => void,
): boolean {
if (!onLab || attrs.type !== HK_LAB_RESULT_RECORD) return false;
onLab({
path: withinCap(attrs.resourceFilePath, CAPS.path),
identifier: withinCap(attrs.identifier, CAPS.identifier),
sourceURL: withinCap(attrs.sourceURL, CAPS.sourceURL),
sourceName: withinCap(attrs.sourceName, CAPS.sourceName),
fhirVersion: withinCap(attrs.fhirVersion, CAPS.fhirVersion),
});
return true;
}
+74 -1
View File
@@ -17,7 +17,11 @@
*/
import { createHash } from "node:crypto";
import type { PrismaClient } from "@/generated/prisma/client";
import type { NormalizedAppleHealthEcg } from "@/lib/apple-health/ecg-csv";
import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream";
import {
parseAppleHealthEcgCsv,
type NormalizedAppleHealthEcg,
} from "@/lib/apple-health/ecg-csv";
import { persistEcgRecording } from "@/lib/ecg/persist-recording";
export type AppleHealthEcgImportOutcome = "imported" | "updated" | "skipped";
@@ -68,3 +72,72 @@ export async function importAppleHealthEcg(input: {
if (result.outcome === "duplicate") return "skipped";
return "updated";
}
const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = {
maxMembers: 20_000,
maxEcgMembers: 2_000,
maxMemberBytes: 16 * 1024 * 1024,
maxTotalEcgBytes: 512 * 1024 * 1024,
maxCompressionRatio: 200,
} as const;
const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
export interface AppleHealthEcgCounts {
discovered: number;
imported: number;
updated: number;
skipped: number;
failed: number;
}
/**
* Every ECG recording in an export archive. Intentionally fail-soft: the
* canonical export.xml pass has already completed, and malformed, oversized
* or unwritable ECG members affect only bounded scalar counters in the
* terminal result. `memberNames` are the archive's other members, which the
* discovered count is taken from.
*/
export async function importAppleHealthArchiveEcgs(input: {
archivePath: string;
userId: string;
memberNames: readonly string[];
prisma: Pick<PrismaClient, "$transaction">;
}): Promise<AppleHealthEcgCounts> {
const ecg: AppleHealthEcgCounts = {
discovered: 0,
imported: 0,
updated: 0,
skipped: 0,
failed: 0,
};
ecg.discovered = input.memberNames.filter((name) => {
const lower = name.toLowerCase();
return lower.includes("electrocardiograms/") && lower.endsWith(".csv");
}).length;
try {
for await (const member of streamAppleHealthEcgMembers({
archivePath: input.archivePath,
limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS,
})) {
try {
const parsed = await parseAppleHealthEcgCsv({
memberName: member.name,
stream: member.stream,
maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES,
});
const outcome = await importAppleHealthEcg({
userId: input.userId,
ecg: parsed,
prisma: input.prisma,
});
ecg[outcome] += 1;
} catch {
ecg.failed += 1;
}
}
} catch {
const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed;
ecg.failed += Math.max(0, ecg.discovered - completed);
}
return ecg;
}
+2 -2
View File
@@ -256,8 +256,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
}
}
} catch {
// The archive itself is unreadable for this pass (a ZIP64 export, a
// limit hit in the directory). Every referenced track not yet handled
// The archive itself is unreadable for this pass (a malformed directory,
// a limit hit in it). Every referenced track not yet handled
// counts as failed; the import stays done.
const handled = counts.attached + counts.skipped + counts.failed;
counts.failed += Math.max(0, byMember.size - handled);
+3
View File
@@ -44,6 +44,9 @@ export const labResultBackupSchema = z
sourceUnit: z.string().nullable().optional(),
takenAt: isoDateTime,
source: z.string().min(1),
// The identity a reading arrived under from Health Records (a digest).
// Optional: absent on every file written before the column.
externalId: z.string().min(1).max(128).nullable().optional(),
biomarkerName: z.string().nullable().optional(),
biomarkerId: z.string().nullable().optional(),
note: z.string().nullable().optional(),
+5
View File
@@ -81,6 +81,9 @@ export interface LabResultBackupEntry {
sourceUnit?: string | null;
takenAt: string;
source: string;
/** The identity a Health Records reading arrived under (a digest), so a
* restored account still recognises it when it arrives again. */
externalId?: string | null;
/** Human-readable cross-reference into `biomarkers` below, not an id. */
/** Stable canonical FK; legacy portable payloads use biomarkerName. */
biomarkerId?: string | null;
@@ -221,6 +224,7 @@ export async function buildLabsBackupSlice(
sourceUnit: r.sourceUnit,
takenAt: r.takenAt.toISOString(),
source: r.source,
externalId: r.externalId,
biomarkerName: r.biomarker?.name ?? null,
}));
@@ -464,6 +468,7 @@ export async function restoreLabResults(ctx: RestoreContext): Promise<void> {
sourceUnit: lab.sourceUnit ?? null,
takenAt: new Date(lab.takenAt),
source: lab.source,
externalId: lab.externalId ?? null,
noteEncrypted:
lab.noteEncrypted !== undefined
? lab.noteEncrypted === null
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog";
import { CATALOG_SLUG_TO_LOINC_KEY, catalogSlugForLoinc } from "../lab-loinc";
describe("catalogSlugForLoinc", () => {
it("inverts the export map: each slug's own code answers with the slug", () => {
const exported: Record<string, string> = {
"2093-3": "total-cholesterol",
"2089-1": "ldl",
"2085-9": "hdl",
"2571-8": "triglycerides",
"1558-6": "fasting-glucose",
"4548-4": "hba1c",
"3016-3": "tsh",
"2276-4": "ferritin",
"62292-8": "vitamin-d",
"30522-7": "hs-crp",
"2160-0": "creatinine",
"69405-9": "egfr",
"1742-6": "alt",
"1920-8": "ast",
"2324-2": "ggt",
"718-7": "hemoglobin",
"1884-6": "apob",
"43583-4": "lp-a",
"27873-9": "fasting-insulin",
};
expect(Object.keys(exported)).toHaveLength(
Object.keys(CATALOG_SLUG_TO_LOINC_KEY).length,
);
for (const [loinc, slug] of Object.entries(exported)) {
expect(catalogSlugForLoinc(loinc)).toBe(slug);
}
});
it.each([
["13457-7", "ldl"],
["18262-6", "ldl"],
["33914-3", "egfr"],
["48642-3", "egfr"],
["48643-1", "egfr"],
["50044-7", "egfr"],
["62238-1", "egfr"],
["98979-8", "egfr"],
["17856-6", "hba1c"],
["11579-0", "tsh"],
])("pins the import-only alias %s -> %s", (loinc, slug) => {
expect(catalogSlugForLoinc(loinc)).toBe(slug);
});
it("answers only with slugs the catalog has", () => {
const slugs = new Set(BIOMARKER_CATALOG.map((seed) => seed.slug));
for (const loinc of [
"2089-1",
"13457-7",
"62238-1",
"17856-6",
"11579-0",
]) {
expect(slugs.has(catalogSlugForLoinc(loinc)!)).toBe(true);
}
});
it("returns null for a code it does not know", () => {
expect(catalogSlugForLoinc("1988-5")).toBeNull();
expect(catalogSlugForLoinc("")).toBeNull();
});
});
+46
View File
@@ -479,3 +479,49 @@ export function resolveLabCoding(
const ucum = normaliseUnit(unit) === mapping.ucum ? mapping.ucum : null;
return { loinc: mapping.loinc, display: mapping.display, ucum };
}
/**
* LOINC codes a FHIR import meets that this module does not export under, by
* the catalog slug they belong to. Each names the same analyte in the same
* kind of unit as the slug's own code, differing only in method or formula:
* LDL by calculation (13457-7) or by direct assay (18262-6); eGFR by MDRD
* (33914-3, 48642-3, 48643-1, 50044-7) or CKD-EPI (62238-1, 98979-8); HbA1c by
* HPLC (17856-6); TSH by a detection-limit method (11579-0). Import only: the
* exporter keeps its own method-less code.
*/
const IMPORT_ONLY_LOINC_TO_SLUG: Readonly<Record<string, string>> = {
"13457-7": "ldl",
"18262-6": "ldl",
"33914-3": "egfr",
"48642-3": "egfr",
"48643-1": "egfr",
"50044-7": "egfr",
"62238-1": "egfr",
"98979-8": "egfr",
"17856-6": "hba1c",
"11579-0": "tsh",
};
let loincToSlug: ReadonlyMap<string, string> | null = null;
/**
* The labs-catalog slug a LOINC code names, or null. The inverse of
* {@link CATALOG_SLUG_TO_LOINC_KEY} over {@link LAB_LOINC_BY_KEY}, plus the
* import-only method variants above. A FHIR lab result carrying one of these
* codes is matched to the account's marker for that analyte whatever the
* sending system called it.
*/
export function catalogSlugForLoinc(code: string): string | null {
if (!loincToSlug) {
const map = new Map<string, string>();
for (const [slug, key] of Object.entries(CATALOG_SLUG_TO_LOINC_KEY)) {
const mapping = LAB_LOINC_BY_KEY[key];
if (mapping) map.set(mapping.loinc, slug);
}
for (const [loinc, slug] of Object.entries(IMPORT_ONLY_LOINC_TO_SLUG)) {
map.set(loinc, slug);
}
loincToSlug = map;
}
return loincToSlug.get(code.trim()) ?? null;
}
@@ -14,7 +14,8 @@ import {
medicationsCsvQuerySchema,
} from "@/lib/validations/record-import";
import { normaliseLabRow, readingKeys } from "../labs-csv";
import { readingKeys } from "@/lib/labs/reading-keys";
import { normaliseLabRow } from "../labs-csv";
import {
medicationCreateBody,
normaliseMedicationRow,
+1 -47
View File
@@ -46,6 +46,7 @@ import {
labReadingDay,
labReadingDaySearchRange,
} from "@/lib/labs/reading-day";
import { LAB_TEXT_LIMITS, readingKeys } from "@/lib/labs/reading-keys";
import { encryptNoteToBytes } from "@/lib/labs/store";
import type {
LabsCsvQuery,
@@ -60,14 +61,6 @@ const MAX_LISTED_MARKERS = 200;
const FUTURE_SLACK_MS = 86_400_000;
const EARLIEST = Date.UTC(1900, 0, 1);
export const LAB_TEXT_LIMITS = {
analyte: 200,
valueText: 200,
unit: 80,
panel: 100,
note: 1000,
} as const;
export interface LabCsvRow {
line: number;
analyte: string;
@@ -212,45 +205,6 @@ export function normaliseLabRow(
};
}
/**
* The keys a reading is a duplicate under, mirroring the scan commit's
* `isDuplicate`: the marker (or the name, for a reading filed without one),
* the reading's calendar day, and the value as stored, as printed, or as text.
*/
export function readingKeys(
reading: {
biomarkerId: string | null;
analyte: string;
takenAt: Date;
value: number | null;
valueText: string | null;
sourceValue?: number | null;
sourceUnit?: string | null;
unit?: string | null;
},
tz: string,
): string[] {
const who = reading.biomarkerId
? `bm:${reading.biomarkerId}`
: `an:${reading.analyte.trim().toLowerCase()}`;
const day = labReadingDay(reading.takenAt, tz);
const keys: string[] = [];
if (reading.value !== null) keys.push(`${who}|${day}|n:${reading.value}`);
if (
reading.sourceValue !== null &&
reading.sourceValue !== undefined &&
reading.sourceUnit
) {
keys.push(
`${who}|${day}|s:${reading.sourceValue}:${reading.sourceUnit.trim().toLowerCase()}`,
);
}
if (reading.valueText !== null) {
keys.push(`${who}|${day}|t:${reading.valueText.trim().toLowerCase()}`);
}
return keys;
}
interface Planned {
row: LabCsvRow;
marker: IndexedMarker | null;
@@ -8,6 +8,10 @@ const workerSource = readFileSync(
join(process.cwd(), "src/lib/jobs/apple-health-import-worker.ts"),
"utf8",
);
const ecgSource = readFileSync(
join(process.cwd(), "src/lib/apple-health/ecg-import.ts"),
"utf8",
);
const parserSource = readFileSync(
join(process.cwd(), "src/lib/measurements/import-apple-health-export.ts"),
"utf8",
@@ -19,18 +23,22 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
});
it("runs bounded ECG archive ingestion beside the established XML parser", () => {
expect(workerSource).toMatch(/streamAppleHealthEcgMembers/);
expect(workerSource).toMatch(/parseAppleHealthEcgCsv/);
expect(workerSource).toMatch(/importAppleHealthEcg/);
expect(workerSource).toMatch(/streamParseExportXml/);
expect(workerSource).toMatch(/importAppleHealthArchiveEcgs/);
expect(ecgSource).toMatch(/streamAppleHealthEcgMembers/);
expect(ecgSource).toMatch(/parseAppleHealthEcgCsv/);
expect(ecgSource).toMatch(/importAppleHealthEcg\(/);
});
it("catches per-ECG failures without routing the valid XML result to the fatal job catch", () => {
expect(workerSource).toMatch(
expect(ecgSource).toMatch(
/for await[\s\S]*streamAppleHealthEcgMembers[\s\S]*try[\s\S]*parseAppleHealthEcgCsv[\s\S]*catch[\s\S]*ecg\.failed/,
);
expect(ecgSource).toMatch(
/ecg:\s*AppleHealthEcgCounts\s*=\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/,
);
expect(workerSource).toMatch(
/result[\s\S]*ecg:\s*\{[\s\S]*discovered[\s\S]*imported[\s\S]*updated[\s\S]*skipped[\s\S]*failed/,
/result:\s*AppleHealthImportJobResult\s*=\s*\{[\s\S]*ecg,[\s\S]*labs:/,
);
});
@@ -45,7 +53,9 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
it("does not log ECG filenames, source labels, samples, or raw parser errors", () => {
const loggingCalls =
workerSource.match(/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g) ?? [];
(workerSource + ecgSource).match(
/console\.(?:log|warn|error)\([\s\S]{0,250}\)/g,
) ?? [];
const ecgLogging = loggingCalls.filter((call) =>
/ecg|electrocardiogram/i.test(call),
);
@@ -56,7 +66,7 @@ describe("Apple Health import worker — ECG auxiliary branch contract", () => {
});
it("does not claim to diagnose or reclassify a waveform", () => {
expect(workerSource).not.toMatch(
expect(workerSource + ecgSource).not.toMatch(
/(?:diagnose|infer|classify)Ecg|analy[sz]eWaveform|detectA(?:fib|trial)/i,
);
});
@@ -80,8 +80,8 @@ describe("apple health import — pg-boss send policy", () => {
});
describe("apple health import — parser revision boundary", () => {
it("isolates revision-3 sends from legacy workers", () => {
expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(3);
it("isolates revision-4 sends from legacy workers", () => {
expect(APPLE_HEALTH_IMPORT_PARSER_REVISION).toBe(4);
expect(APPLE_HEALTH_IMPORT_V2_QUEUE).toBe("apple-health-import-v2");
expect(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).toBe("apple-health-import");
});
+71 -82
View File
@@ -30,9 +30,12 @@ import type { PrismaClient } from "@/generated/prisma/client";
import { prisma, toJson } from "@/lib/db";
import type { Job } from "pg-boss";
import { streamAppleHealthEcgMembers } from "@/lib/apple-health/archive-stream";
import { parseAppleHealthEcgCsv } from "@/lib/apple-health/ecg-csv";
import { importAppleHealthEcg } from "@/lib/apple-health/ecg-import";
import {
importAppleHealthClinicalLabs,
type AppleHealthImportJobResult,
} from "@/lib/apple-health/clinical-labs-import";
import type { ClinicalLabReference } from "@/lib/apple-health/clinical-record-ref";
import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import";
import { extractExportXml } from "@/lib/import/unzip-export-xml";
import { annotate } from "@/lib/logging/context";
import {
@@ -45,7 +48,6 @@ import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
streamParseExportXml,
type ImportJobProgress,
type ImportJobResult,
} from "@/lib/measurements/import-apple-health-export";
import { withBackgroundEvent } from "@/lib/logging/background";
import { DEFAULT_TIMEZONE } from "@/lib/tz/format";
@@ -125,17 +127,12 @@ export async function handleImportJobReconcileTick(
);
}
/** Parser semantics carried by every newly-created ImportJob. */
export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 3;
const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = {
maxMembers: 20_000,
maxEcgMembers: 2_000,
maxMemberBytes: 16 * 1024 * 1024,
maxTotalEcgBytes: 512 * 1024 * 1024,
maxCompressionRatio: 200,
} as const;
const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000;
/**
* Parser semantics carried by every newly-created ImportJob. 4: lab results
* from Health Records are read (#1400), so an archive imported before is
* worth uploading again.
*/
export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 4;
/**
* (issue #775) Memory preflight for the parse. The streaming pipeline
@@ -404,6 +401,7 @@ export async function handleAppleHealthImport(
// GPS tracks are read from the archive only when the person keeps them.
const keepRoutes = userRow?.storeWorkoutRoutes === true;
const routeReferences: WorkoutRouteReference[] = [];
const labReferences: ClinicalLabReference[] = [];
const userTimezone =
userRow?.timezone && userRow.timezone.length > 0
? userRow.timezone
@@ -439,80 +437,58 @@ export async function handleAppleHealthImport(
elapsedMs: 0,
});
const result: ImportJobResult = {
...(await streamParseExportXml({
xmlPath: unzip.xmlPath,
userId,
userTimezone,
prisma,
...(keepRoutes
? { onWorkoutRoute: (ref) => void routeReferences.push(ref) }
: {}),
onProgress: async (snapshot) => {
// The phase label here is what the polling endpoint surfaces;
// map "parsing" / "upserting" through verbatim.
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: snapshot.currentPhase,
progress: toJson(snapshot),
},
});
},
// The instant the Health app stamped on the archive. Persisted as
// soon as the parser reads it, so the status endpoint answers with
// it for the rest of the run and a job that later fails still
// records which export it was working from. Best-effort: the
// stamp is provenance, and losing it must not fail an import.
onExportDate: async (exportedAt) => {
await prisma.importJob
.update({ where: { id: importJobId }, data: { exportedAt } })
.catch(caughtAs("apple_health.import.export_date_failed"));
},
})),
ecg: {
const parsed = await streamParseExportXml({
xmlPath: unzip.xmlPath,
userId,
userTimezone,
prisma,
...(keepRoutes
? { onWorkoutRoute: (ref) => void routeReferences.push(ref) }
: {}),
onClinicalLabRecord: (ref) => void labReferences.push(ref),
onProgress: async (snapshot) => {
// The phase label here is what the polling endpoint surfaces;
// map "parsing" / "upserting" through verbatim.
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: snapshot.currentPhase,
progress: toJson(snapshot),
},
});
},
// The instant the Health app stamped on the archive. Persisted as
// soon as the parser reads it, so the status endpoint answers with
// it for the rest of the run and a job that later fails still
// records which export it was working from. Best-effort: the
// stamp is provenance, and losing it must not fail an import.
onExportDate: async (exportedAt) => {
await prisma.importJob
.update({ where: { id: importJobId }, data: { exportedAt } })
.catch(caughtAs("apple_health.import.export_date_failed"));
},
});
// The auxiliary passes below are fail-soft: the export.xml rows are
// committed, and a member that cannot be read is only counted.
const ecg = await importAppleHealthArchiveEcgs({
archivePath: uploadPath,
userId,
memberNames: unzip.otherMembers.map((member) => member.name),
prisma,
});
const result: AppleHealthImportJobResult = {
...parsed,
ecg,
labs: {
discovered: 0,
imported: 0,
updated: 0,
duplicate: 0,
skipped: 0,
failed: 0,
},
};
// Auxiliary ECGs are intentionally fail-soft. The canonical export.xml
// transaction has already completed; malformed, oversized, or unwritable
// ECG members affect only bounded scalar counters in the terminal result.
const ecg = result.ecg;
ecg.discovered = unzip.otherMembers.filter((member) => {
const lower = member.name.toLowerCase();
return lower.includes("electrocardiograms/") && lower.endsWith(".csv");
}).length;
try {
for await (const member of streamAppleHealthEcgMembers({
archivePath: uploadPath,
limits: APPLE_HEALTH_ECG_ARCHIVE_LIMITS,
})) {
try {
const parsed = await parseAppleHealthEcgCsv({
memberName: member.name,
stream: member.stream,
maxSamples: APPLE_HEALTH_ECG_MAX_SAMPLES,
});
const outcome = await importAppleHealthEcg({
userId,
ecg: parsed,
prisma,
});
ecg[outcome] += 1;
} catch {
ecg.failed += 1;
}
}
} catch {
const completed = ecg.imported + ecg.updated + ecg.skipped + ecg.failed;
ecg.failed += Math.max(0, ecg.discovered - completed);
}
// Workout GPS tracks, on opt-in only. Fail-soft like the ECGs: the
// workouts are committed, a track that cannot be read is only counted.
// Read one member at a time from the archive; nothing about a track
@@ -538,6 +514,19 @@ export async function handleAppleHealthImport(
routeReferences.length = 0;
}
// Lab results from Health Records, one FHIR file at a time; only counts
// reach the result.
if (labReferences.length > 0) {
result.labs = await importAppleHealthClinicalLabs({
archivePath: uploadPath,
userId,
tz: userTimezone,
references: labReferences,
prisma,
});
labReferences.length = 0;
}
// Done. Persist the terminal envelope.
await prisma.importJob.update({
where: { id: importJobId },
@@ -284,3 +284,33 @@ describe("suggestions", () => {
expect(nameSimilarity("LDL", "Ferritin")).toBeLessThan(0.5);
});
});
describe("a catalog hint from the source (#1400)", () => {
it("matches a marker by the hinted analyte when the name says nothing", () => {
const ldl = marker("LDL", "mg/dL");
const idx = index([ldl]);
const name =
"Cholesterol in LDL [Mass/volume] in Serum or Plasma by calculation";
expect(resolveInIndex(idx, name, "mmol/L")).toBeNull();
expect(
resolveInIndex(idx, name, "mmol/L", { catalogKey: "ldl" }),
).toMatchObject({ marker: { id: ldl.id }, matchKind: "catalog" });
});
it("still steps aside for a unit the marker cannot take", () => {
const ldl = marker("LDL", "mg/dL");
expect(
resolveInIndex(index([ldl]), "LDL calc", "U/L", { catalogKey: "ldl" }),
).toBeNull();
});
it("lets the account's own name win over the hint", () => {
const own = marker("Odd name", "mmol/L");
const ldl = marker("LDL", "mg/dL");
expect(
resolveInIndex(index([own, ldl]), "odd name", "mmol/L", {
catalogKey: "ldl",
}),
).toMatchObject({ marker: { id: own.id }, matchKind: "name" });
});
});
+6 -1
View File
@@ -172,11 +172,16 @@ export function unitFitsMarker(
/**
* The marker a printed name resolves to, and how; null when nothing does.
* `unit` is the unit the reading states, used only by the catalog step.
* `options.catalogKey` names the catalog analyte when the source says which
* one it is (a LOINC code on a FHIR result): the catalog step then uses it in
* place of the analyte the name alone would suggest. The account's own names
* and aliases still win first.
*/
export function resolveInIndex(
index: MarkerIndex,
rawName: string,
unit?: string | null,
options: { catalogKey?: string | null } = {},
): MarkerMatch | null {
const name = rawName.trim();
if (!name) return null;
@@ -203,7 +208,7 @@ export function resolveInIndex(
}
// 3. The catalog: a marker of the same analyte that can take the unit.
const key = catalogKeyForName(name);
const key = options.catalogKey ?? catalogKeyForName(name);
if (key) {
const candidates = index.markers.filter(
(m) =>
+11 -3
View File
@@ -22,6 +22,7 @@ import {
type MarkerIndex,
} from "@/lib/labs/biomarker-resolve";
import { mintedBounds } from "@/lib/labs/minted-bounds";
import { analyteAcceptsUnit } from "@/lib/labs/unit-conversion/convert";
import { getEvent } from "@/lib/logging/context";
/** Encrypt a UTF-8 context note into the `Bytes` payload the schema stores. */
@@ -110,11 +111,12 @@ export async function resolveOrMintBiomarker(
referenceHigh: number | null;
panel: string | null;
},
options: { index?: MarkerIndex } = {},
options: { index?: MarkerIndex; catalogKey?: string | null } = {},
): Promise<MintedBiomarker> {
const name = input.analyte.trim();
const index = options.index ?? (await loadMarkerIndex(prisma, userId));
const hit = resolveInIndex(index, name, input.unit || null);
const catalogKey = options.catalogKey ?? null;
const hit = resolveInIndex(index, name, input.unit || null, { catalogKey });
if (hit) return toMinted(hit.marker);
const { lowerBound, upperBound } = mintedBounds(
@@ -130,7 +132,13 @@ export async function resolveOrMintBiomarker(
lowerBound,
upperBound,
panel: input.panel,
analyteKey: matchAnalyteKey({ name, unit: input.unit }),
// A name the catalog does not know still links when the source named
// the analyte (a LOINC code), provided the unit is one it reaches.
analyteKey:
matchAnalyteKey({ name, unit: input.unit }) ??
(catalogKey && analyteAcceptsUnit(catalogKey, input.unit)
? catalogKey
: null),
},
select: INDEXED_SELECT,
});
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import { fhirLabExternalId, fhirLabIssuer } from "../external-id";
const AT = new Date("2025-03-04T07:15:00Z");
describe("fhirLabExternalId", () => {
it("gives the export and the HealthKit record the same id", () => {
const fromExport = fhirLabExternalId({
sourceURL: "https://FHIR.example-health.test/api/R4/",
sourceName: "Example Health",
identifier: "obs-1",
resourceId: "obs-1",
analyte: "LDL",
takenAt: AT,
value: 3.1,
valueText: null,
});
const fromPhone = fhirLabExternalId({
sourceURL: "https://fhir.example-health.test/api/R4",
sourceName: "Example Health (renamed)",
identifier: "obs-1",
analyte: "LDL Cholesterol",
takenAt: new Date("2025-03-05T00:00:00Z"),
value: 80,
valueText: null,
});
expect(fromExport).toBe(fromPhone);
expect(fromExport).toMatch(/^fhir:[0-9a-f]{40}$/);
});
it("keeps two institutions apart", () => {
const a = fhirLabExternalId({
sourceURL: "https://a.test",
identifier: "obs-1",
analyte: "x",
takenAt: AT,
value: 1,
valueText: null,
});
const b = fhirLabExternalId({
sourceURL: "https://b.test",
identifier: "obs-1",
analyte: "x",
takenAt: AT,
value: 1,
valueText: null,
});
expect(a).not.toBe(b);
});
it("falls back to a stable content identity", () => {
const input = {
sourceName: "Clinic",
loinc: "2345-7",
analyte: " Glucose ",
takenAt: AT,
value: 92,
valueText: null,
};
const first = fhirLabExternalId(input);
expect(first).toMatch(/^fhirc:[0-9a-f]{40}$/);
expect(fhirLabExternalId({ ...input, analyte: "glucose" })).toBe(first);
expect(fhirLabExternalId({ ...input, value: 93 })).not.toBe(first);
});
it("names the issuer by URL, then name, then unknown", () => {
expect(fhirLabIssuer({ sourceURL: " https://X.test/// " })).toBe(
"https://x.test",
);
expect(fhirLabIssuer({ sourceName: " Clinic " })).toBe("Clinic");
expect(fhirLabIssuer({})).toBe("unknown");
});
});
@@ -0,0 +1,17 @@
{
"resourceType": "Observation",
"id": "dstu2-blood-group",
"status": "final",
"category": { "coding": [{ "code": "laboratory" }] },
"code": { "text": "ABO group" },
"effectiveDateTime": "2019-06-11T07:05:00Z",
"valueCodeableConcept": {
"coding": [
{
"system": "http://snomed.info/sct",
"code": "112144000",
"display": "Blood group A"
}
]
}
}
@@ -0,0 +1,37 @@
{
"resourceType": "Observation",
"id": "dstu2-glu",
"status": "final",
"category": {
"coding": [
{
"system": "http://hl7.org/fhir/observation-category",
"code": "laboratory"
}
]
},
"code": {
"coding": [
{
"system": "http://loinc.org",
"code": "2345-7",
"display": "Glucose [Mass/volume] in Serum or Plasma"
}
]
},
"subject": { "reference": "Patient/example-patient" },
"effectiveDateTime": "2019-06-11T07:05:00-07:00",
"valueQuantity": {
"value": 92,
"unit": "mg/dL",
"system": "http://unitsofmeasure.org",
"code": "mg/dL"
},
"referenceRange": [
{
"low": { "value": 70, "unit": "mg/dL" },
"high": { "value": 99, "unit": "mg/dL" }
}
],
"comments": "Specimen slightly haemolysed."
}
@@ -0,0 +1,37 @@
[
{
"resourceType": "Observation",
"id": "dstu2-lipid",
"status": "final",
"category": { "coding": [{ "code": "laboratory" }] },
"code": { "text": "Lipid panel" },
"effectiveDateTime": "2019-06-11T07:05:00Z",
"related": [
{
"type": "has-member",
"target": { "reference": "Observation/dstu2-hdl" }
},
{
"type": "derived-from",
"target": { "reference": "Observation/dstu2-other" }
}
]
},
{
"resourceType": "Observation",
"id": "dstu2-hdl",
"status": "final",
"category": { "coding": [{ "code": "laboratory" }] },
"code": {
"coding": [
{
"system": "http://loinc.org",
"code": "2085-9",
"display": "HDL Cholesterol"
}
]
},
"effectiveDateTime": "2019-06-11T07:05:00Z",
"valueQuantity": { "value": 55, "unit": "mg/dL" }
}
]
@@ -0,0 +1,25 @@
{
"resourceType": "Observation",
"id": "obs-crp-1",
"status": "final",
"category": [{ "coding": [{ "code": "laboratory" }] }],
"code": {
"coding": [
{
"system": "http://loinc.org",
"code": "1988-5",
"display": "C reactive protein [Mass/volume] in Serum or Plasma"
}
]
},
"subject": { "reference": "Patient/example-patient" },
"effectiveDateTime": "2025-03-04T08:15:00Z",
"valueQuantity": {
"value": 0.5,
"comparator": "<",
"unit": "mg/L",
"system": "http://unitsofmeasure.org",
"code": "mg/L"
},
"referenceRange": [{ "high": { "value": 5, "unit": "mg/L" } }]
}
@@ -0,0 +1,17 @@
{
"resourceType": "Observation",
"id": "obs-tsh",
"status": "corrected",
"code": {
"coding": [
{
"system": "urn:oid:2.16.840.1.113883.6.1",
"code": "11579-0",
"display": "Thyrotropin [Units/volume] in Serum or Plasma by Detection limit <= 0.05 mIU/L"
}
]
},
"effectiveDateTime": "2024-11-20",
"valueQuantity": { "value": 2.3, "code": "m[IU]/L" },
"referenceRange": [{ "text": "0.4 - 4.0" }]
}
@@ -0,0 +1,18 @@
{
"resourceType": "Observation",
"id": "obs-covid",
"status": "final",
"category": [{ "coding": [{ "code": "laboratory" }] }],
"code": { "text": "SARS-CoV-2 RNA" },
"effectiveDateTime": "2024-12-01T10:00:00Z",
"interpretation": [
{
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation",
"code": "POS"
}
]
}
]
}
@@ -0,0 +1,56 @@
[
{
"resourceType": "Observation",
"id": "obs-cbc",
"status": "final",
"category": [{ "coding": [{ "code": "laboratory" }] }],
"code": {
"coding": [
{
"system": "http://loinc.org",
"code": "58410-2",
"display": "CBC panel - Blood by Automated count"
}
],
"text": "Complete blood count"
},
"effectiveDateTime": "2025-02-01T07:30:00Z",
"hasMember": [
{ "reference": "Observation/obs-hgb" },
{ "reference": "https://fhir.example.org/api/Observation/obs-wbc" }
]
},
{
"resourceType": "Observation",
"id": "obs-hgb",
"status": "final",
"category": [{ "coding": [{ "code": "laboratory" }] }],
"code": {
"coding": [
{
"system": "http://loinc.org",
"code": "718-7",
"display": "Hemoglobin [Mass/volume] in Blood"
}
],
"text": "Hemoglobin"
},
"effectiveDateTime": "2025-02-01T07:30:00Z",
"valueQuantity": { "value": 14.2, "unit": "g/dL", "code": "g/dL" },
"referenceRange": [
{
"low": { "value": 13.5, "unit": "g/dL" },
"high": { "value": 17.5, "unit": "g/dL" }
}
]
},
{
"resourceType": "Observation",
"id": "obs-wbc",
"status": "final",
"category": [{ "coding": [{ "code": "laboratory" }] }],
"code": { "text": "WBC" },
"effectiveDateTime": "2025-02-01T07:30:00Z",
"valueQuantity": { "value": 6.4, "unit": "x10^3/uL", "code": "10*3/uL" }
}
]
@@ -0,0 +1,9 @@
{
"resourceType": "Observation",
"id": "obs-prelim",
"status": "preliminary",
"category": [{ "coding": [{ "code": "laboratory" }] }],
"code": { "text": "Potassium" },
"effectiveDateTime": "2025-02-01T07:30:00Z",
"valueQuantity": { "value": 4.1, "unit": "mmol/L" }
}
@@ -0,0 +1,46 @@
{
"resourceType": "Observation",
"id": "obs-ldl-1",
"status": "final",
"category": [
{
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/observation-category",
"code": "laboratory",
"display": "Laboratory"
}
]
}
],
"code": {
"coding": [
{
"system": "http://loinc.org",
"code": "13457-7",
"display": "Cholesterol in LDL [Mass/volume] in Serum or Plasma by calculation"
}
],
"text": "LDL Cholesterol (calc)"
},
"subject": {
"reference": "Patient/example-patient",
"display": "Example Patient"
},
"performer": [{ "display": "Example Laboratory" }],
"effectiveDateTime": "2025-03-04T08:15:00+01:00",
"issued": "2025-03-05T10:00:00+01:00",
"valueQuantity": {
"value": 3.1,
"unit": "mmol/L",
"system": "http://unitsofmeasure.org",
"code": "mmol/L"
},
"referenceRange": [
{
"high": { "value": 3.0, "unit": "mmol/L", "code": "mmol/L" },
"text": "< 3.0 mmol/L"
}
],
"note": [{ "text": "Fasting sample." }]
}
@@ -0,0 +1,9 @@
{
"resourceType": "Observation",
"id": "obs-strep-1",
"status": "amended",
"category": [{ "text": "Laboratory" }],
"code": { "text": "Strep A antigen" },
"effectiveDateTime": "2025-01-10T09:00:00-05:00",
"valueString": " Positive "
}
@@ -0,0 +1,18 @@
{
"resourceType": "Observation",
"id": "obs-bp",
"status": "final",
"category": [
{
"coding": [
{
"system": "http://terminology.hl7.org/CodeSystem/observation-category",
"code": "vital-signs"
}
]
}
],
"code": { "text": "Body weight" },
"effectiveDateTime": "2025-02-01T07:30:00Z",
"valueQuantity": { "value": 72, "unit": "kg" }
}
@@ -0,0 +1,319 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
assignPanels,
fhirInstant,
mapFhirLabRecord,
normaliseObservationRef,
type FhirLabMapping,
} from "../map-observation";
const NOW = new Date("2026-01-01T00:00:00Z");
const SOURCE = {
sourceURL: "https://fhir.example-health.test/api/FHIR/R4/",
sourceName: "Example Health",
fhirVersion: "4.0.1",
};
function fixture(name: string): unknown {
return JSON.parse(
readFileSync(join(__dirname, "fixtures", `${name}.json`), "utf8"),
);
}
function map(resource: unknown, extra: Record<string, unknown> = {}) {
return mapFhirLabRecord({ resource, ...SOURCE, ...extra }, NOW);
}
function reading(result: FhirLabMapping) {
if (result.kind !== "reading") {
throw new Error(`expected a reading, got ${result.kind}`);
}
return result.reading;
}
function base(overrides: Record<string, unknown> = {}) {
return {
resourceType: "Observation",
id: "obs-x",
status: "final",
category: [{ coding: [{ code: "laboratory" }] }],
code: { text: "Sodium" },
effectiveDateTime: "2025-01-01T08:00:00Z",
valueQuantity: { value: 140, unit: "mmol/L" },
...overrides,
};
}
describe("mapFhirLabRecord — R4", () => {
it("maps a quantity with its LOINC hint, range and note", () => {
const r = reading(
map(fixture("r4-quantity-ldl"), { identifier: "obs-ldl-1" }),
);
expect(r.analyte).toBe("LDL Cholesterol (calc)");
expect(r.loinc).toBe("13457-7");
expect(r.catalogKey).toBe("ldl");
expect(r.value).toBe(3.1);
expect(r.valueText).toBeNull();
expect(r.unit).toBe("mmol/L");
expect(r.takenAt.toISOString()).toBe("2025-03-04T07:15:00.000Z");
expect(r.sourceReferenceLow).toBeNull();
expect(r.sourceReferenceHigh).toBe(3);
expect(r.sourceReferenceText).toBe("< 3.0 mmol/L");
expect(r.note).toBe("Fasting sample.");
expect(r.panel).toBeNull();
expect(r.externalId).toMatch(/^fhir:[0-9a-f]{40}$/);
});
it("never carries the subject or performer anywhere", () => {
const serialised = JSON.stringify(map(fixture("r4-quantity-ldl")));
expect(serialised).not.toContain("Example Patient");
expect(serialised).not.toContain("example-patient");
expect(serialised).not.toContain("Example Laboratory");
});
it("keeps a comparator result as the printed text, unit included", () => {
const r = reading(map(fixture("r4-comparator")));
expect(r.value).toBeNull();
expect(r.valueText).toBe("<0.5 mg/L");
expect(r.unit).toBe("mg/L");
// A text result records no window of its own.
expect(r.sourceReferenceText).toBeNull();
expect(r.analyte).toBe(
"C reactive protein [Mass/volume] in Serum or Plasma",
);
});
it("drops the unity from a comparator text", () => {
const r = reading(
map(
base({ valueQuantity: { value: 1.1, comparator: ">", code: "{INR}" } }),
),
);
expect(r.valueText).toBe(">1.1");
});
it("trims a string result", () => {
const r = reading(map(fixture("r4-valuestring-positive")));
expect(r.valueText).toBe("Positive");
expect(r.value).toBeNull();
expect(r.unit).toBeNull();
});
it("anchors a date alone, and a time without offset, at noon UTC", () => {
const r = reading(map(fixture("r4-date-only")));
expect(r.takenAt.toISOString()).toBe("2024-11-20T12:00:00.000Z");
expect(r.catalogKey).toBe("tsh");
expect(r.unit).toBe("mIU/L");
// A text-only range is read.
expect(r.sourceReferenceLow).toBe(0.4);
expect(r.sourceReferenceHigh).toBe(4);
expect(fhirInstant("2024-11-20T08:00:00")?.toISOString()).toBe(
"2024-11-20T12:00:00.000Z",
);
expect(fhirInstant("2024-11")).toBeNull();
expect(fhirInstant("2024")).toBeNull();
expect(fhirInstant("2024-02-30")).toBeNull();
});
it("falls back through the period start and issued", () => {
const period = reading(
map(
base({
effectiveDateTime: undefined,
effectivePeriod: { start: "2025-01-02T09:00:00Z" },
}),
),
);
expect(period.takenAt.toISOString()).toBe("2025-01-02T09:00:00.000Z");
const issued = reading(
map(
base({ effectiveDateTime: undefined, issued: "2025-01-03T09:00:00Z" }),
),
);
expect(issued.takenAt.toISOString()).toBe("2025-01-03T09:00:00.000Z");
});
it("keeps an interpretation as text when there is no value", () => {
const r = reading(map(fixture("r4-no-value-interpretation-pos")));
expect(r.valueText).toBe("Positive");
});
it("maps the panel header to its members' panel", () => {
const results = (fixture("r4-panel-cbc") as unknown[]).map((r) => map(r));
expect(results[0]).toMatchObject({ kind: "skip", reason: "panel_header" });
assignPanels(results);
expect(reading(results[1]).panel).toBe("Complete blood count");
expect(reading(results[2]).panel).toBe("Complete blood count");
expect(reading(results[2]).unit).toBe("10^3/µL");
});
it.each([
["r4-preliminary", "not_final"],
["r4-vital-signs", "not_lab"],
])("skips %s as %s", (name, reason) => {
expect(map(fixture(name))).toMatchObject({ kind: "skip", reason });
});
it.each([
[{ resourceType: "Condition" }, "not_observation"],
["not an object", "not_observation"],
[base({ status: "entered-in-error" }), "withdrawn"],
[base({ status: "cancelled" }), "withdrawn"],
[base({ status: "registered" }), "not_final"],
[base({ status: undefined }), "not_final"],
[
base({ valueQuantity: undefined, valueBoolean: true }),
"unsupported_value",
],
[base({ valueQuantity: undefined, valueRange: {} }), "unsupported_value"],
[base({ valueQuantity: undefined, component: [] }), "unsupported_value"],
[
base({ valueQuantity: undefined, dataAbsentReason: { text: "x" } }),
"no_value",
],
[base({ valueQuantity: undefined }), "no_value"],
[base({ code: {} }), "no_value"],
[base({ effectiveDateTime: undefined }), "no_date"],
[base({ effectiveDateTime: "2026-01-03T00:00:00Z" }), "future_date"],
[base({ effectiveDateTime: "1899-12-31" }), "invalid_date"],
])("skips %j as %s", (resource, reason) => {
expect(map(resource)).toMatchObject({ kind: "skip", reason });
});
it.each([
[base({ valueQuantity: { value: 5 } }), "missing_unit"],
[base({ valueQuantity: undefined, valueInteger: 5 }), "missing_unit"],
[base({ code: { text: "x".repeat(201) } }), "text_too_long"],
[
base({ valueQuantity: undefined, valueString: "y".repeat(201) }),
"text_too_long",
],
])("refuses %j as %s", (resource, reason) => {
expect(map(resource)).toMatchObject({ kind: "fail", reason });
});
it("accepts a resource with no category", () => {
expect(map(base({ category: undefined })).kind).toBe("reading");
});
it("keeps a range only in the reading's unit", () => {
const r = reading(
map(
base({
referenceRange: [
{
low: { value: 3.5, unit: "mg/dL" },
high: { value: 145, unit: "mmol/L" },
},
],
}),
),
);
expect(r.sourceReferenceLow).toBeNull();
expect(r.sourceReferenceHigh).toBe(145);
});
it("prefers the general range over an age-specific one, and drops low > high", () => {
const chosen = reading(
map(
base({
referenceRange: [
{
low: { value: 1 },
high: { value: 2 },
age: { low: { value: 1 } },
},
{ low: { value: 135 }, high: { value: 145 } },
],
}),
),
);
expect([chosen.sourceReferenceLow, chosen.sourceReferenceHigh]).toEqual([
135, 145,
]);
const inverted = reading(
map(
base({ referenceRange: [{ low: { value: 9 }, high: { value: 1 } }] }),
),
);
expect(inverted.sourceReferenceLow).toBeNull();
expect(inverted.sourceReferenceHigh).toBeNull();
});
it("caps a long note", () => {
const r = reading(map(base({ note: [{ text: "n".repeat(1500) }] })));
expect(r.note).toHaveLength(1000);
expect(r.note?.endsWith("…")).toBe(true);
});
it("reads a code display when the code carries no text", () => {
const r = reading(
map(
base({
code: {
coding: [
{ system: "http://example.org", display: "Local name" },
{
system: "http://loinc.org",
code: "2951-2",
display: "Sodium [Moles/volume] in Serum or Plasma",
},
],
},
}),
),
);
expect(r.analyte).toBe("Sodium [Moles/volume] in Serum or Plasma");
expect(r.loinc).toBe("2951-2");
expect(r.catalogKey).toBeNull();
});
});
describe("mapFhirLabRecord — DSTU2", () => {
it("reads a single category object, comments and a glucose quantity", () => {
const r = reading(map(fixture("dstu2-quantity-glucose")));
expect(r.analyte).toBe("Glucose [Mass/volume] in Serum or Plasma");
expect(r.value).toBe(92);
expect(r.unit).toBe("mg/dL");
expect([r.sourceReferenceLow, r.sourceReferenceHigh]).toEqual([70, 99]);
expect(r.note).toBe("Specimen slightly haemolysed.");
expect(JSON.stringify(r)).not.toContain("example-patient");
});
it("reads a coded value as text", () => {
expect(reading(map(fixture("dstu2-codeableconcept"))).valueText).toBe(
"Blood group A",
);
});
it("follows related has-member links only", () => {
const results = (fixture("dstu2-related-has-member") as unknown[]).map(
(r) => map(r),
);
expect(results[0]).toMatchObject({
kind: "skip",
reason: "panel_header",
memberRefs: ["Observation/dstu2-hdl"],
});
assignPanels(results);
expect(reading(results[1]).panel).toBe("Lipid panel");
expect(reading(results[1]).catalogKey).toBe("hdl");
});
});
describe("normaliseObservationRef", () => {
it.each([
["Observation/abc", "Observation/abc"],
["https://x.test/fhir/Observation/abc", "Observation/abc"],
["https://x.test/fhir/Observation/abc/_history/2", "Observation/abc"],
["Patient/abc", null],
["", null],
[42, null],
])("%j -> %j", (raw, expected) => {
expect(normaliseObservationRef(raw)).toBe(expected);
});
});
@@ -0,0 +1,66 @@
import { describe, expect, it } from "vitest";
import { convertLabValue } from "@/lib/labs/unit-conversion/convert";
import { scaleUnitOf } from "@/lib/labs/unit-conversion/scale";
import { sameLabUnit } from "@/lib/labs/unit-normalise";
import { labUnitFromFhirQuantity } from "../units";
describe("labUnitFromFhirQuantity", () => {
it.each([
[{ code: "10*3/uL" }, "10^3/µL"],
[{ code: "10*6/uL" }, "10^6/µL"],
[{ code: "10*9/L" }, "10^9/L"],
[{ code: "10*12/L" }, "10^12/L"],
[{ code: "[IU]/L" }, "IU/L"],
[{ code: "m[IU]/L" }, "mIU/L"],
[{ code: "m[IU]/mL" }, "mIU/mL"],
[{ code: "u[IU]/mL" }, "µIU/mL"],
[{ code: "meq/L" }, "mEq/L"],
[{ code: "/uL" }, "/µL"],
[{ code: "mL/min/{1.73_m2}" }, "mL/min/1.73m²"],
[{ code: "1" }, "1"],
[{ code: "{INR}" }, "1"],
[{ unit: "mg/dL", code: "mg/dL" }, "mg/dL"],
[{ unit: "x10^3/uL", code: "10*3/uL" }, "10^3/µL"],
[{ unit: "mmol/L" }, "mmol/L"],
[{ unit: " ", code: "g/L" }, "g/L"],
[{}, null],
[{ unit: 4 }, null],
])("%j -> %j", (quantity, expected) => {
expect(labUnitFromFhirQuantity(quantity)).toBe(expected);
});
it("produces spellings the normaliser and conversion understand", () => {
for (const code of [
"10*3/uL",
"10*6/uL",
"10*9/L",
"10*12/L",
"m[IU]/L",
"u[IU]/mL",
"[IU]/L",
]) {
expect(scaleUnitOf(labUnitFromFhirQuantity({ code })!)).not.toBeNull();
}
expect(
sameLabUnit(labUnitFromFhirQuantity({ code: "u[IU]/mL" })!, "µIU/ml"),
).toBe(true);
expect(
convertLabValue({
analyteKey: "tsh",
value: 2,
from: labUnitFromFhirQuantity({ code: "u[IU]/mL" })!,
to: "mIU/L",
}),
).toMatchObject({ ok: true, value: 2 });
expect(
convertLabValue({
analyteKey: "ldl",
value: 3.1,
from: labUnitFromFhirQuantity({ code: "mmol/L" })!,
to: "mg/dL",
}).ok,
).toBe(true);
});
});
+34
View File
@@ -0,0 +1,34 @@
/**
* What follows a batch of lab readings landing, for the FHIR import paths:
* the health score is recomputed, a "blood panel due" reminder is resolved
* now rather than at the next cron, and the labs arm of the data-arrival
* spine is told once, with the newest draw and the true inserted count (the
* same three effects the scan commit runs). Nothing here throws.
*/
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
import { invalidateUserHealthScore } from "@/lib/cache/invalidate";
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
import { fireAndForget } from "@/lib/logging/fire-and-forget";
export function afterLabReadingsWritten(input: {
userId: string;
inserted: number;
newestTakenAt: Date | null;
/** Transport token, for annotations only. */
source: string;
}): void {
if (input.inserted <= 0) return;
invalidateUserHealthScore(input.userId);
fireAndForget(() => enqueueReminderSatisfy(input.userId), {
action: "reminder.satisfy.enqueue",
});
if (input.newestTakenAt) {
void emitDataArrival({
userId: input.userId,
kind: "labs_panel",
newestSampleAt: input.newestTakenAt,
insertedCount: input.inserted,
source: input.source,
}).catch(() => {});
}
}
+66
View File
@@ -0,0 +1,66 @@
/**
* The identity a FHIR lab result is stored under (`LabResult.externalId`).
*
* The same result reaches HealthLog from two directions: inside an Apple
* Health export (`export.xml` names the issuing institution in `sourceURL`
* and the record in `identifier`, the JSON file carries the resource) and
* from the iOS app (`HKFHIRResource` carries the same `sourceURL`,
* `identifier` and data). The identity is built from those, so both paths
* land on one id and the second sees a duplicate.
*
* Only a digest is stored: the issuer's URL and the record identifier are
* hashed together and cut to 40 hex characters, which is enough to keep two
* results of one account apart and says nothing about either.
*
* A record with neither an identifier nor a resource id falls back to a
* content identity: the issuer, the LOINC code, the name, the instant and the
* value. Stable across re-imports of the same record; a corrected result
* whose value changed reads as a new one.
*/
import { createHash } from "node:crypto";
const DIGEST_HEX = 40;
function digest(parts: readonly string[]): string {
return createHash("sha256")
.update(parts.join("\0"))
.digest("hex")
.slice(0, DIGEST_HEX);
}
/** The institution that issued a record, as one stable string. */
export function fhirLabIssuer(input: {
sourceURL?: string | null;
sourceName?: string | null;
}): string {
const url = input.sourceURL?.trim().toLowerCase().replace(/\/+$/u, "");
if (url) return url;
const name = input.sourceName?.trim();
if (name) return name;
return "unknown";
}
export function fhirLabExternalId(input: {
sourceURL?: string | null;
sourceName?: string | null;
/** The record identifier, else the resource's own `id`. */
identifier?: string | null;
resourceId?: string | null;
/** For the content fallback only. */
loinc?: string | null;
analyte: string;
takenAt: Date;
value: number | null;
valueText: string | null;
}): string {
const issuer = fhirLabIssuer(input);
const id = input.identifier?.trim() || input.resourceId?.trim();
if (id) return `fhir:${digest([issuer, "Observation", id])}`;
return `fhirc:${digest([
issuer,
input.loinc ?? "",
input.analyte.trim().toLowerCase(),
input.takenAt.toISOString(),
input.value !== null ? String(input.value) : (input.valueText ?? ""),
])}`;
}
+463
View File
@@ -0,0 +1,463 @@
/**
* One FHIR lab result (an `Observation`, DSTU2 or R4) → one lab reading, or
* the closed reason it is not one. Pure.
*
* Only whitelisted fields are read: `resourceType`, `id`, `status`,
* `category`, `code`, `effective*`, `issued`, `value*`, `dataAbsentReason`,
* `interpretation`, `referenceRange`, `hasMember` / `related`, `note` /
* `comments`. The patient, the performer and any contained resource are never
* looked at, and nothing from the resource ever reaches an error message: a
* malformed field is a reason, not a quote.
*
* The rules, in brief:
* - a result counts when it is final (`final`, `amended`, `corrected`);
* a preliminary one is skipped and a withdrawn one never written;
* - a category other than laboratory (a vital sign, a survey) is skipped;
* a resource with no category at all is taken, as Health Records only
* hands lab-result records to this path;
* - the name is the code's text, else the LOINC coding's display, else any
* coding's display; the LOINC code is a matching hint;
* - a number with a comparator ("<0.5 mg/L") is kept as text, as the report
* printed it, since a bound is not a value;
* - the report's own reference range is kept, in the reading's unit only;
* - a result with no value whose interpretation says positive, negative,
* detected, not detected or indeterminate is kept as that text;
* - a result with no value but members is a panel header: not a reading,
* but its name becomes its members' panel (`assignPanels`).
*/
import { catalogSlugForLoinc } from "@/lib/fhir/lab-loinc";
import {
parseReferenceRange,
SOURCE_REFERENCE_TEXT_MAX,
} from "@/lib/labs/parse-reference-range";
import { LAB_TEXT_LIMITS } from "@/lib/labs/reading-keys";
import { sameLabUnit } from "@/lib/labs/unit-normalise";
import { fhirLabExternalId } from "./external-id";
import type {
FhirLabFailureReason,
FhirLabRecordInput,
FhirLabSkipReason,
MappedFhirLab,
} from "./types";
import { labUnitFromFhirQuantity } from "./units";
/** A reading dated more than a day ahead is a mistake, not a result. */
const FUTURE_SLACK_MS = 86_400_000;
const EARLIEST = Date.UTC(1900, 0, 1);
const LOINC_SYSTEMS = new Set([
"http://loinc.org",
"urn:oid:2.16.840.1.113883.6.1",
]);
export type FhirLabMapping =
| {
kind: "reading";
reading: MappedFhirLab;
selfRef: string | null;
memberRefs: string[];
}
| {
kind: "skip";
reason: FhirLabSkipReason;
selfRef: string | null;
memberRefs: string[];
/** A panel header's name, for its members. */
panelName: string | null;
}
| { kind: "fail"; reason: FhirLabFailureReason; selfRef: string | null };
type Json = Record<string, unknown>;
function obj(value: unknown): Json | null {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Json)
: null;
}
/** A field DSTU2 writes as one object and R4 as a list, read as a list. */
function list(value: unknown): Json[] {
if (Array.isArray(value)) {
return value.map(obj).filter((v): v is Json => v !== null);
}
const one = obj(value);
return one ? [one] : [];
}
function str(value: unknown): string | null {
if (typeof value !== "string") return null;
const collapsed = value.replace(/\s+/gu, " ").trim();
return collapsed === "" ? null : collapsed;
}
function num(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function codings(concept: unknown): Json[] {
return list(obj(concept)?.coding);
}
/** `Observation/<id>` for a reference however it is written. */
export function normaliseObservationRef(raw: unknown): string | null {
const reference = str(raw);
if (!reference) return null;
const parts = reference.split(/[/?#]/u).filter(Boolean);
const at = parts.lastIndexOf("Observation");
if (at < 0 || at + 1 >= parts.length) return null;
return `Observation/${parts[at + 1]}`;
}
function memberRefs(resource: Json): string[] {
const refs: string[] = [];
for (const member of list(resource.hasMember)) {
const ref = normaliseObservationRef(member.reference);
if (ref) refs.push(ref);
}
for (const related of list(resource.related)) {
if (str(related.type) !== "has-member") continue;
const ref = normaliseObservationRef(obj(related.target)?.reference);
if (ref) refs.push(ref);
}
return refs;
}
function isLaboratory(concept: Json): boolean {
if (str(concept.text)?.toLowerCase() === "laboratory") return true;
return codings(concept).some(
(c) => str(c.code)?.toLowerCase() === "laboratory",
);
}
function analyteName(code: Json | null): string | null {
if (!code) return null;
const text = str(code.text);
if (text) return text;
const all = codings(code);
const loinc = all.find((c) => LOINC_SYSTEMS.has(str(c.system) ?? ""));
return (
str(loinc?.display) ?? all.map((c) => str(c.display)).find(Boolean) ?? null
);
}
function loincCode(code: Json | null): string | null {
if (!code) return null;
for (const c of codings(code)) {
if (LOINC_SYSTEMS.has(str(c.system) ?? "")) {
const value = str(c.code);
if (value) return value;
}
}
return null;
}
const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/u;
const LOCAL_DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})T[\d:.]+$/u;
const WITH_OFFSET = /^\d{4}-\d{2}-\d{2}T[\d:.]+(Z|[+-]\d{2}:?\d{2})$/u;
/**
* A FHIR date or dateTime as an instant. A date alone, or a time with no
* offset, is the day at the noon-UTC anchor the labs feature stores a date
* at; a year or a year and month alone does not say which day and is
* unusable.
*/
export function fhirInstant(raw: unknown): Date | null {
const text = str(raw);
if (!text) return null;
const day = DATE_ONLY.exec(text) ?? LOCAL_DATE_TIME.exec(text);
if (day) {
const [y, m, d] = [Number(day[1]), Number(day[2]), Number(day[3])];
const at = new Date(Date.UTC(y, m - 1, d, 12));
return at.getUTCFullYear() === y &&
at.getUTCMonth() === m - 1 &&
at.getUTCDate() === d
? at
: null;
}
if (!WITH_OFFSET.test(text)) return null;
const ms = Date.parse(text);
return Number.isFinite(ms) ? new Date(ms) : null;
}
function takenAtOf(resource: Json): Date | null {
return (
fhirInstant(resource.effectiveDateTime) ??
fhirInstant(obj(resource.effectivePeriod)?.start) ??
fhirInstant(resource.effectiveInstant) ??
fhirInstant(resource.issued)
);
}
const INTERPRETATION_TEXT: Readonly<Record<string, string>> = {
POS: "Positive",
NEG: "Negative",
DET: "Detected",
ND: "Not detected",
IND: "Indeterminate",
};
function interpretationText(resource: Json): string | null {
for (const concept of list(resource.interpretation)) {
for (const c of codings(concept)) {
const code = str(c.code)?.toUpperCase();
if (code && INTERPRETATION_TEXT[code]) {
return str(c.display) ?? str(concept.text) ?? INTERPRETATION_TEXT[code];
}
}
}
return null;
}
const UNSUPPORTED_VALUES = [
"valueBoolean",
"valueRange",
"valueRatio",
"valueSampledData",
"valueAttachment",
"valueTime",
"valuePeriod",
"valueDateTime",
];
type ValueRead =
| {
ok: true;
value: number | null;
valueText: string | null;
unit: string | null;
}
| { ok: false; skip?: FhirLabSkipReason; fail?: FhirLabFailureReason }
| null;
function readValue(resource: Json): ValueRead {
const quantity = obj(resource.valueQuantity);
if (quantity) {
const value = num(quantity.value);
if (value === null) return { ok: false, skip: "no_value" };
const unit = labUnitFromFhirQuantity(quantity);
const comparator = str(quantity.comparator);
if (comparator) {
const printed =
unit && unit !== "1"
? `${comparator}${value} ${unit}`
: `${comparator}${value}`;
return { ok: true, value: null, valueText: printed, unit };
}
if (!unit) return { ok: false, fail: "missing_unit" };
return { ok: true, value, valueText: null, unit };
}
if (Number.isSafeInteger(resource.valueInteger)) {
return { ok: false, fail: "missing_unit" };
}
const text = str(resource.valueString);
if (text) return { ok: true, value: null, valueText: text, unit: null };
const concept = obj(resource.valueCodeableConcept);
if (concept) {
const first = codings(concept)[0];
const label = str(concept.text) ?? str(first?.display) ?? str(first?.code);
if (label) return { ok: true, value: null, valueText: label, unit: null };
return { ok: false, skip: "no_value" };
}
if (UNSUPPORTED_VALUES.some((key) => resource[key] !== undefined)) {
return { ok: false, skip: "unsupported_value" };
}
return null;
}
function referenceRangeOf(
resource: Json,
unit: string | null,
): { low: number | null; high: number | null; text: string | null } {
const ranges = list(resource.referenceRange);
const chosen =
ranges.find((r) => r.appliesTo === undefined && r.age === undefined) ??
ranges.find((r) =>
codings(r.type).some((c) => str(c.code)?.toLowerCase() === "normal"),
);
if (!chosen) return { low: null, high: null, text: null };
const bound = (q: unknown): number | null => {
const quantity = obj(q);
if (!quantity) return null;
const value = num(quantity.value);
if (value === null) return null;
const boundUnit = labUnitFromFhirQuantity(quantity);
if (boundUnit && unit && !sameLabUnit(boundUnit, unit)) return null;
if (boundUnit && !unit) return null;
return value;
};
let low = bound(chosen.low);
let high = bound(chosen.high);
const text = str(chosen.text)?.slice(0, SOURCE_REFERENCE_TEXT_MAX) ?? null;
if (low === null && high === null && text) {
const parsed = parseReferenceRange(text, unit);
low = parsed?.low ?? null;
high = parsed?.high ?? null;
}
if (low !== null && high !== null && low > high) {
low = null;
high = null;
}
return { low, high, text };
}
function noteOf(resource: Json): string | null {
const parts = list(resource.note)
.map((n) => str(n.text))
.filter((t): t is string => t !== null);
const comments = str(resource.comments);
if (comments) parts.push(comments);
if (parts.length === 0) return null;
const joined = parts.join("\n");
return joined.length > LAB_TEXT_LIMITS.note
? `${joined.slice(0, LAB_TEXT_LIMITS.note - 1)}…`
: joined;
}
const FINAL = new Set(["final", "amended", "corrected", "missing"]);
const NOT_YET = new Set(["preliminary", "registered", "unknown"]);
const WITHDRAWN = new Set(["entered-in-error", "cancelled"]);
export function mapFhirLabRecord(
input: FhirLabRecordInput,
now: Date = new Date(),
): FhirLabMapping {
const resource = obj(input.resource);
const id = resource ? str(resource.id) : null;
const selfRef = id ? `Observation/${id}` : null;
const skip = (
reason: FhirLabSkipReason,
members: string[] = [],
panelName: string | null = null,
): FhirLabMapping => ({
kind: "skip",
reason,
selfRef,
memberRefs: members,
panelName,
});
const fail = (reason: FhirLabFailureReason): FhirLabMapping => ({
kind: "fail",
reason,
selfRef,
});
if (!resource || resource.resourceType !== "Observation") {
return skip("not_observation");
}
const status = str(resource.status)?.toLowerCase() ?? "";
if (WITHDRAWN.has(status)) return skip("withdrawn");
if (NOT_YET.has(status) || !FINAL.has(status)) return skip("not_final");
const categories = list(resource.category);
if (categories.length > 0 && !categories.some(isLaboratory)) {
return skip("not_lab");
}
const code = obj(resource.code);
const analyte = analyteName(code);
const members = memberRefs(resource);
if (analyte && analyte.length > LAB_TEXT_LIMITS.analyte) {
return fail("text_too_long");
}
let read = readValue(resource);
if (read === null && resource.dataAbsentReason !== undefined) {
read = { ok: false, skip: "no_value" };
}
if (read === null) {
const interpreted = interpretationText(resource);
if (interpreted) {
read = { ok: true, value: null, valueText: interpreted, unit: null };
} else if (members.length > 0) {
return skip(
"panel_header",
members,
analyte ? analyte.slice(0, LAB_TEXT_LIMITS.panel) : null,
);
} else if (resource.component !== undefined) {
return skip("unsupported_value");
} else {
return skip("no_value");
}
}
if (!read.ok) {
return read.fail ? fail(read.fail) : skip(read.skip ?? "no_value", members);
}
if (!analyte) return skip("no_value", members);
if (
read.valueText !== null &&
read.valueText.length > LAB_TEXT_LIMITS.valueText
) {
return fail("text_too_long");
}
if (read.unit !== null && read.unit.length > LAB_TEXT_LIMITS.unit) {
return fail("text_too_long");
}
const takenAt = takenAtOf(resource);
if (!takenAt) return skip("no_date", members);
if (takenAt.getTime() > now.getTime() + FUTURE_SLACK_MS) {
return skip("future_date", members);
}
if (takenAt.getTime() < EARLIEST) return skip("invalid_date", members);
const loinc = loincCode(code);
const range = referenceRangeOf(resource, read.unit);
return {
kind: "reading",
selfRef,
memberRefs: members,
reading: {
externalId: fhirLabExternalId({
sourceURL: input.sourceURL,
sourceName: input.sourceName,
identifier: input.identifier,
resourceId: id,
loinc,
analyte,
takenAt,
value: read.value,
valueText: read.valueText,
}),
analyte,
loinc,
catalogKey: loinc ? catalogSlugForLoinc(loinc) : null,
value: read.value,
valueText: read.valueText,
unit: read.unit,
takenAt,
// Like the CSV import: on a text result the window only seeds a new
// marker; the reading itself records none.
sourceReferenceLow: range.low,
sourceReferenceHigh: range.high,
sourceReferenceText: read.valueText === null ? range.text : null,
panel: null,
note: noteOf(resource),
},
};
}
/**
* Give each reading the name of the panel that lists it as a member (R4
* `hasMember`, DSTU2 `related` of type `has-member`). Mutates the readings in
* place; a reading already carrying a panel keeps it.
*/
export function assignPanels(results: readonly FhirLabMapping[]): void {
const panelOf = new Map<string, string>();
for (const result of results) {
if (result.kind !== "skip" || result.reason !== "panel_header") continue;
if (!result.panelName) continue;
for (const ref of result.memberRefs) {
if (!panelOf.has(ref)) panelOf.set(ref, result.panelName);
}
}
if (panelOf.size === 0) return;
for (const result of results) {
if (result.kind !== "reading" || !result.selfRef) continue;
if (result.reading.panel !== null) continue;
result.reading.panel = panelOf.get(result.selfRef) ?? null;
}
}
+96
View File
@@ -0,0 +1,96 @@
/**
* Shared shapes of the FHIR lab import (#1400): lab results from Apple
* Health's Health Records, arriving either inside an `export.zip`
* (`src/lib/apple-health/clinical-labs-import.ts`) or from the iOS app
* (`POST /api/labs/fhir-batch`). Both paths map with `mapFhirLabRecord` and
* write with `writeFhirLabReadings`, so a result reads the same however it
* came in.
*
* Nothing about a resource's content leaves this module family except the
* reading it maps to: reasons are closed enums, counts are numbers.
*/
/** `LabResult.source` of every reading this import writes. */
export const APPLE_HEALTH_LAB_SOURCE = "APPLE_HEALTH";
/** One clinical record as the export or HealthKit hands it over. */
export interface FhirLabRecordInput {
/** The parsed FHIR resource. Untrusted: every field is checked. */
resource: unknown;
/** `HKFHIRResource.fhirVersion` / `fhirVersion` attribute, e.g. "4.0.1". */
fhirVersion?: string | null;
/** `HKFHIRResource.identifier` / `identifier` attribute. */
identifier?: string | null;
/** `HKFHIRResource.sourceURL` / `sourceURL` attribute. */
sourceURL?: string | null;
/** The institution as the Health app names it. */
sourceName?: string | null;
}
/** A lab reading a FHIR Observation maps to, before any marker is chosen. */
export interface MappedFhirLab {
externalId: string;
analyte: string;
/** The LOINC code the resource carries, when it carries one. */
loinc: string | null;
/** The catalog slug the LOINC code names, used as a matching hint. */
catalogKey: string | null;
value: number | null;
valueText: string | null;
/** Unit in the spelling the labs feature uses; null for a text result. */
unit: string | null;
takenAt: Date;
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
panel: string | null;
note: string | null;
}
export const FHIR_LAB_SKIP_REASONS = [
"not_observation",
"not_lab",
"not_final",
"withdrawn",
"panel_header",
"no_value",
"unsupported_value",
"no_date",
"future_date",
"invalid_date",
"too_large",
] as const;
export type FhirLabSkipReason = (typeof FHIR_LAB_SKIP_REASONS)[number];
export const FHIR_LAB_FAILURE_REASONS = [
"missing_unit",
"unit_mismatch",
"text_too_long",
"write_failed",
] as const;
export type FhirLabFailureReason = (typeof FHIR_LAB_FAILURE_REASONS)[number];
/** Bounded outcome counts, the only thing an import reports. */
export interface FhirLabCounts {
/** Lab result records the source offered. */
discovered: number;
/** Readings written. */
imported: number;
/** Readings the record already held (by identity or by value). */
duplicate: number;
/** Records that are not a reading HealthLog keeps (a panel header, a
* preliminary result, a vital sign). */
skipped: number;
/** Readings refused (no unit, a unit the marker cannot take) or unreadable. */
failed: number;
}
export function emptyFhirLabCounts(): FhirLabCounts {
return { discovered: 0, imported: 0, duplicate: 0, skipped: 0, failed: 0 };
}
/** Per-input outcome of a write. */
export type FhirLabWriteStatus =
| { status: "inserted" }
| { status: "duplicate" }
| { status: "failed"; reason: FhirLabFailureReason };
+56
View File
@@ -0,0 +1,56 @@
/**
* A FHIR `Quantity` unit in the spelling the labs feature uses.
*
* FHIR systems write UCUM codes (`10*3/uL`, `m[IU]/L`, `mL/min/{1.73_m2}`);
* the labs catalog, the unit normaliser and the conversion registry speak the
* printed spellings a report uses (`10^3/µL`, `mIU/L`, `mL/min/1.73m²`). The
* table below covers the UCUM forms lab results actually carry that the
* normaliser would not read on its own; anything else passes through as
* written, which is already the printed form for `mg/dL`, `mmol/L`, `g/L`,
* `U/L`, `%` and the like.
*/
const UCUM_TO_LAB: ReadonlyMap<string, string> = new Map([
["10*3/uL", "10^3/µL"],
["10*3/ul", "10^3/µL"],
["10*6/uL", "10^6/µL"],
["10*6/ul", "10^6/µL"],
["10*9/L", "10^9/L"],
["10*9/l", "10^9/L"],
["10*12/L", "10^12/L"],
["10*12/l", "10^12/L"],
["[IU]/L", "IU/L"],
["m[IU]/L", "mIU/L"],
["m[IU]/mL", "mIU/mL"],
["u[IU]/mL", "µIU/mL"],
["meq/L", "mEq/L"],
["/uL", "/µL"],
["mL/min/{1.73_m2}", "mL/min/1.73m²"],
["mL/min/1.73.m2", "mL/min/1.73m²"],
]);
/** A UCUM annotation alone (`{INR}`, `{ratio}`) is the unity. */
const ANNOTATION_ONLY = /^\{[^{}]*\}$/u;
/**
* The lab spelling of a FHIR quantity's unit: a UCUM form the table knows
* (the `code` first, then the human `unit`), else the human `unit` as written,
* else the `code`. Null when neither says anything. The unity (`1`, or an
* annotation alone such as `{INR}`) becomes "1".
*/
export function labUnitFromFhirQuantity(quantity: {
unit?: unknown;
code?: unknown;
}): string | null {
const text = (raw: unknown): string | null =>
typeof raw === "string" && raw.trim() !== "" ? raw.trim() : null;
const code = text(quantity.code);
const unit = text(quantity.unit);
for (const candidate of [code, unit]) {
if (candidate === null) continue;
const mapped = UCUM_TO_LAB.get(candidate);
if (mapped) return mapped;
if (candidate === "1" || ANNOTATION_ONLY.test(candidate)) return "1";
}
return unit ?? code;
}
+365
View File
@@ -0,0 +1,365 @@
/**
* Write mapped FHIR lab readings (#1400), shared by the Apple Health export
* job and `POST /api/labs/fhir-batch`.
*
* A reading becomes a `LabResult` the way a CSV row does
* (`src/lib/import/labs-csv.ts`): the name resolves to the account's marker
* through the shared index (with the LOINC code's catalog analyte as a hint),
* a marker is minted for a new analyte, a reading in another unit is
* converted when the conversion is known and kept beside what the report
* said, and a unit that does not convert is refused.
*
* Two duplicate checks, in this order:
* 1. the identity (`externalId`) under `source = APPLE_HEALTH`, against
* every row including the deleted ones: a result the person deleted
* stays deleted however often the export or the phone sends it again;
* 2. the reading itself (marker or name, day, value) against the live
* record, so a result already typed, scanned or imported from a CSV is
* not written a second time.
*
* First write wins: an amended or corrected result arriving under an
* identity already held is a duplicate, not an update.
*
* Writes go in chunks of `CHUNK` through `createManyAndReturn` with
* `skipDuplicates`, so a concurrent writer of the same identity loses
* quietly and is reported as a duplicate. A chunk that fails to write is
* reported as `write_failed` and the next chunk still runs.
*/
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
import { prisma as defaultPrisma } from "@/lib/db";
import { matchAnalyteKey } from "@/lib/labs/analyte-match";
import {
loadMarkerIndex,
resolveInIndex,
unitFitsMarker,
type MarkerIndex,
} from "@/lib/labs/biomarker-resolve";
import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store";
import { readingInMarkerUnit } from "@/lib/labs/reading-conversion";
import {
labReadingDay,
labReadingDaySearchRange,
} from "@/lib/labs/reading-day";
import { readingKeys } from "@/lib/labs/reading-keys";
import { encryptNoteToBytes } from "@/lib/labs/store";
import {
APPLE_HEALTH_LAB_SOURCE,
type FhirLabWriteStatus,
type MappedFhirLab,
} from "./types";
const CHUNK = 200;
type LabWriteClient = Pick<
PrismaClient,
"labResult" | "biomarker" | "biomarkerAlias"
>;
export interface FhirLabWriteResult {
/** One per input reading, in input order. */
statuses: FhirLabWriteStatus[];
inserted: number;
duplicate: number;
failed: number;
/** The newest instant among the readings written, or null. */
newestTakenAt: Date | null;
}
interface Pending {
at: number;
reading: MappedFhirLab;
markerId: string | null;
}
export async function writeFhirLabReadings(args: {
userId: string;
tz: string;
readings: readonly MappedFhirLab[];
prisma?: LabWriteClient;
}): Promise<FhirLabWriteResult> {
const client = args.prisma ?? defaultPrisma;
const { userId, tz, readings } = args;
const statuses: FhirLabWriteStatus[] = new Array(readings.length);
const result: FhirLabWriteResult = {
statuses,
inserted: 0,
duplicate: 0,
failed: 0,
newestTakenAt: null,
};
if (readings.length === 0) return result;
const index: MarkerIndex = await loadMarkerIndex(client, userId);
const seenIds = new Set<string>();
const seenKeys = new Set<string>();
const duplicate = (at: number) => {
statuses[at] = { status: "duplicate" };
result.duplicate += 1;
};
const failed = (at: number, reason: "unit_mismatch" | "write_failed") => {
statuses[at] = { status: "failed", reason };
result.failed += 1;
};
for (let start = 0; start < readings.length; start += CHUNK) {
const slice = readings.slice(start, start + CHUNK);
// 1. Identity: in the batch, then against every stored row.
const fresh: Array<{ at: number; reading: MappedFhirLab }> = [];
for (let i = 0; i < slice.length; i += 1) {
const at = start + i;
const reading = slice[i];
if (seenIds.has(reading.externalId)) {
duplicate(at);
continue;
}
seenIds.add(reading.externalId);
fresh.push({ at, reading });
}
const held = new Set(
(
await client.labResult.findMany({
where: {
userId,
source: APPLE_HEALTH_LAB_SOURCE,
externalId: { in: fresh.map((f) => f.reading.externalId) },
},
select: { externalId: true },
})
).map((row) => row.externalId),
);
// 2. The marker, and whether the unit fits it.
const planned: Pending[] = [];
for (const { at, reading } of fresh) {
if (held.has(reading.externalId)) {
duplicate(at);
continue;
}
const hit = resolveInIndex(
index,
reading.analyte,
reading.valueText === null ? reading.unit : null,
{ catalogKey: reading.catalogKey },
);
if (
hit &&
reading.value !== null &&
!unitFitsMarker(
{
unit: hit.marker.unit,
analyteKey:
matchAnalyteKey({
name: hit.marker.name,
unit: hit.marker.unit,
}) ?? hit.marker.analyteKey,
},
reading.unit,
reading.catalogKey,
)
) {
failed(at, "unit_mismatch");
continue;
}
planned.push({
at,
reading,
markerId: hit?.marker.id ?? null,
});
}
// 3. The reading itself, against the live record.
const existingKeys = await liveReadingKeys(client, userId, tz, planned);
// 4. Build the rows, field by field.
const rows: Prisma.LabResultCreateManyInput[] = [];
const rowAt = new Map<string, Pending>();
for (const plan of planned) {
const { reading } = plan;
const probe = {
biomarkerId: plan.markerId,
analyte: reading.analyte,
takenAt: reading.takenAt,
value: reading.value,
valueText: reading.valueText,
sourceValue: reading.value,
sourceUnit: reading.unit,
};
const keys = [
...readingKeys(probe, tz),
...readingKeys({ ...probe, biomarkerId: null }, tz),
];
if (keys.some((k) => existingKeys.has(k) || seenKeys.has(k))) {
duplicate(plan.at);
continue;
}
const built = await buildRow(client, userId, reading, index);
if (!built.ok) {
failed(plan.at, "unit_mismatch");
continue;
}
rows.push(built.row);
rowAt.set(reading.externalId, plan);
for (const k of [
...keys,
...readingKeys({ ...probe, biomarkerId: built.markerId }, tz),
]) {
seenKeys.add(k);
}
}
if (rows.length === 0) continue;
try {
const written = await client.labResult.createManyAndReturn({
data: rows,
skipDuplicates: true,
select: { externalId: true },
});
const landed = new Set(written.map((w) => w.externalId));
for (const [externalId, plan] of rowAt) {
if (landed.has(externalId)) {
statuses[plan.at] = { status: "inserted" };
result.inserted += 1;
const t = plan.reading.takenAt;
if (!result.newestTakenAt || t > result.newestTakenAt) {
result.newestTakenAt = t;
}
} else {
// A concurrent write of the same identity won.
duplicate(plan.at);
}
}
} catch {
for (const plan of rowAt.values()) failed(plan.at, "write_failed");
}
}
return result;
}
/** Duplicate keys of the live readings a chunk could collide with. */
async function liveReadingKeys(
client: LabWriteClient,
userId: string,
tz: string,
planned: readonly Pending[],
): Promise<Set<string>> {
const keys = new Set<string>();
if (planned.length === 0) return keys;
const days = planned.map((p) => labReadingDay(p.reading.takenAt, tz)).sort();
const ids = [
...new Set(
planned.map((p) => p.markerId).filter((id): id is string => !!id),
),
];
const names = [...new Set(planned.map((p) => p.reading.analyte))];
const existing = await client.labResult.findMany({
where: {
userId,
deletedAt: null,
takenAt: {
gte: labReadingDaySearchRange(days[0]).gte,
lt: labReadingDaySearchRange(days[days.length - 1]).lt,
},
OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }],
},
select: {
biomarkerId: true,
analyte: true,
takenAt: true,
value: true,
valueText: true,
sourceValue: true,
sourceUnit: true,
},
});
for (const e of existing) {
for (const k of readingKeys(e, tz)) keys.add(k);
// A reading filed under a marker also answers to its printed name.
if (e.biomarkerId) {
for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) keys.add(k);
}
}
return keys;
}
/**
* The row for one reading: its marker resolved or minted, the value moved
* into the marker's unit when it states another. Built field by field.
*/
async function buildRow(
client: LabWriteClient,
userId: string,
reading: MappedFhirLab,
index: MarkerIndex,
): Promise<
| { ok: true; row: Prisma.LabResultCreateManyInput; markerId: string }
| { ok: false }
> {
const marker = await resolveOrMintBiomarker(
userId,
{
analyte: reading.analyte,
unit: reading.unit ?? "",
referenceLow: reading.sourceReferenceLow,
referenceHigh: reading.sourceReferenceHigh,
panel: reading.panel,
},
{ index, catalogKey: reading.catalogKey },
);
// Like the CSV import: on a text result the window only seeds a new
// marker; the reading itself records none.
let storedValue = reading.value;
let storedLow =
reading.valueText === null ? reading.sourceReferenceLow : null;
let storedHigh =
reading.valueText === null ? reading.sourceReferenceHigh : null;
let convertedFrom: { sourceValue: number; sourceUnit: string } | null = null;
if (reading.value !== null) {
const converted = await readingInMarkerUnit(client, {
marker,
value: reading.value,
unit: reading.unit ?? undefined,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
});
if (!converted.ok) return { ok: false };
storedValue = converted.value;
storedLow = converted.sourceReferenceLow;
storedHigh = converted.sourceReferenceHigh;
if (converted.converted) {
convertedFrom = {
sourceValue: converted.sourceValue,
sourceUnit: converted.sourceUnit,
};
}
}
return {
ok: true,
markerId: marker.id,
row: {
userId,
biomarkerId: marker.id,
panel: reading.panel ?? marker.panel,
analyte: marker.name,
value: storedValue,
valueText: reading.valueText,
unit: marker.unit,
sourceValue: convertedFrom?.sourceValue ?? null,
sourceUnit: convertedFrom?.sourceUnit ?? null,
referenceLow: marker.lowerBound,
referenceHigh: marker.upperBound,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
sourceReferenceText: reading.sourceReferenceText,
takenAt: reading.takenAt,
source: APPLE_HEALTH_LAB_SOURCE,
externalId: reading.externalId,
noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null,
},
};
}
+57
View File
@@ -0,0 +1,57 @@
/**
* What makes two lab readings the same reading, and the text ceilings every
* bulk lab write shares.
*
* Used by the CSV import (`src/lib/import/labs-csv.ts`) and the FHIR import
* (`src/lib/labs/fhir-import/write.ts`), so a reading typed by hand, scanned,
* imported from a spreadsheet or arriving from Apple Health is recognised as
* one reading whichever way it came in first.
*/
import { labReadingDay } from "@/lib/labs/reading-day";
export const LAB_TEXT_LIMITS = {
analyte: 200,
valueText: 200,
unit: 80,
panel: 100,
note: 1000,
} as const;
/**
* The keys a reading is a duplicate under, mirroring the scan commit's
* `isDuplicate`: the marker (or the name, for a reading filed without one),
* the reading's calendar day, and the value as stored, as printed, or as text.
*/
export function readingKeys(
reading: {
biomarkerId: string | null;
analyte: string;
takenAt: Date;
value: number | null;
valueText: string | null;
sourceValue?: number | null;
sourceUnit?: string | null;
unit?: string | null;
},
tz: string,
): string[] {
const who = reading.biomarkerId
? `bm:${reading.biomarkerId}`
: `an:${reading.analyte.trim().toLowerCase()}`;
const day = labReadingDay(reading.takenAt, tz);
const keys: string[] = [];
if (reading.value !== null) keys.push(`${who}|${day}|n:${reading.value}`);
if (
reading.sourceValue !== null &&
reading.sourceValue !== undefined &&
reading.sourceUnit
) {
keys.push(
`${who}|${day}|s:${reading.sourceValue}:${reading.sourceUnit.trim().toLowerCase()}`,
);
}
if (reading.valueText !== null) {
keys.push(`${who}|${day}|t:${reading.valueText.trim().toLowerCase()}`);
}
return keys;
}
@@ -952,6 +952,44 @@ describe("streamParseExportXml — end-to-end", () => {
expect(result.totals.recordsRead).toBeGreaterThanOrEqual(7);
});
it("hands lab-result clinical records to the callback and skips the rest", async () => {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
const xmlPath = join(tmp, "export.xml");
writeFileSync(
xmlPath,
`<?xml version="1.0" encoding="UTF-8"?>
<HealthData locale="en_US">
<ClinicalRecord type="HKClinicalTypeIdentifierLabResultRecord"
identifier="obs-1" sourceName="Example Clinic"
sourceURL="https://fhir.example-clinic.test/R4" fhirVersion="4.0.1"
receivedDate="2025-03-05 10:00:00 +0100"
resourceFilePath="/clinical-records/Observation-obs-1.json"/>
<ClinicalRecord type="HKClinicalTypeIdentifierConditionRecord"
identifier="cond-1" sourceName="Example Clinic" fhirVersion="4.0.1"
resourceFilePath="/clinical-records/Condition-cond-1.json"/>
</HealthData>`,
);
const refs: unknown[] = [];
const result = await streamParseExportXml({
xmlPath,
userId: "user-1",
userTimezone: "Europe/Berlin",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
prisma: makeFakePrisma() as any,
onClinicalLabRecord: (ref) => void refs.push(ref),
});
expect(refs).toEqual([
{
path: "/clinical-records/Observation-obs-1.json",
identifier: "obs-1",
sourceURL: "https://fhir.example-clinic.test/R4",
sourceName: "Example Clinic",
fhirVersion: "4.0.1",
},
]);
expect(result.clinical.skipped).toBe(1);
});
it("re-importing the same export reports 0 inserts and N updates", async () => {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
const xmlPath = join(tmp, "export.xml");
+4 -1
View File
@@ -1262,7 +1262,10 @@ export const HK_QUANTITY_TYPE_DEFERRED = new Set<string>([
// branch runs.
"HKCategoryTypeIdentifierBleedingAfterPregnancy",
"HKCategoryTypeIdentifierBleedingDuringPregnancy",
// Clinical (FHIR) — v1.6+
// Clinical (FHIR). These never arrive as a `<Record>`; the export writes
// `<ClinicalRecord>` elements, which the parser handles itself: lab results
// are imported as lab readings (`src/lib/apple-health/clinical-labs-import.ts`),
// every other clinical type is counted in `clinical.skipped`.
"HKClinicalTypeIdentifierAllergyRecord",
"HKClinicalTypeIdentifierConditionRecord",
"HKClinicalTypeIdentifierImmunizationRecord",
@@ -45,6 +45,10 @@ import {
canonicalDailyTimestamp,
} from "@/lib/measurements/drain-per-sample-cumulative";
import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin";
import {
routeClinicalRecord,
type ClinicalLabReference,
} from "@/lib/apple-health/clinical-record-ref";
import {
isSameReadingAcrossSource,
MEASURED_AT_TOLERANCE_MS,
@@ -460,6 +464,9 @@ export interface StreamParseInput {
* pass, from the archive (`src/lib/apple-health/route-import.ts`).
*/
onWorkoutRoute?: (ref: { externalId: string; path: string }) => void;
/** Called once per lab-result `<ClinicalRecord>`; without it every clinical
* record is counted as skipped (`src/lib/apple-health/clinical-labs-import.ts`). */
onClinicalLabRecord?: (ref: ClinicalLabReference) => void;
}
/**
@@ -1449,7 +1456,9 @@ export async function streamParseExportXml(
if (name === "ClinicalRecord") {
recordsRead += 1;
clinical.skipped += 1;
if (!routeClinicalRecord(attrs, input.onClinicalLabRecord)) {
clinical.skipped += 1;
}
return;
}
+1
View File
@@ -289,6 +289,7 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
"labs.biomarker.update.invalid",
"labs.biomarker.update.referenceRangeInvalid",
"labs.create.invalid",
"labs.fhir_batch.invalid",
"labs.series.invalid",
"labs.link.invalid",
"labs.ocr.budgetExceeded",
+1 -1
View File
@@ -748,7 +748,7 @@ export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
.record(z.string(), z.unknown())
.nullable()
.describe(
"The worker's outcome summary, null until it finishes. Free-form except for `ecg`, whose five counters (`discovered` / `imported` / `updated` / `skipped` / `failed`) are normalised to bounded non-negative integers before serving.",
"The worker's outcome summary, null until it finishes. Free-form except for `ecg` and `labs`, whose five counters each (`ecg`: `discovered` / `imported` / `updated` / `skipped` / `failed`; `labs`, the lab results from Health Records: `discovered` / `imported` / `duplicate` / `skipped` / `failed`) are normalised to bounded non-negative integers before serving.",
),
failureReason: z
.string()
+74 -1
View File
@@ -19,6 +19,11 @@ import {
listLabResultsSchema,
updateLabResultSchema,
} from "@/lib/validations/labs";
import {
FHIR_LAB_FAILURE_REASONS,
FHIR_LAB_SKIP_REASONS,
} from "@/lib/labs/fhir-import/types";
import { fhirLabBatchSchema } from "@/lib/validations/labs-fhir";
import {
dataEnvelope,
@@ -88,7 +93,10 @@ const labResultRow = z
referenceOrigin: referenceOriginEnum,
referenceDivergesFromCatalog: z.boolean(),
takenAt: z.string(),
source: z.string(),
source: z.string().meta({
description:
"Where the reading came from: `MANUAL` (typed), `OCR` (a scanned report), `DOCUMENT` (a document import), `IMPORT` (a lab CSV) or `APPLE_HEALTH` (Apple Health's Health Records, from an export or `POST /api/labs/fhir-batch`). Open-ended: render an unknown value as-is.",
}),
hasNote: z.boolean(),
rangeStatus: rangeStatusEnum,
createdAt: z.string(),
@@ -180,6 +188,37 @@ export const labOverviewResponse = z
})
.meta({ id: "LabOverviewResponse" });
const fhirLabBatchRequest = fhirLabBatchSchema.meta({
id: "FhirLabBatchRequest",
description:
"1 to 200 Health Records lab results (`HKClinicalTypeIdentifierLabResultRecord`), each the FHIR resource as JSON plus the HealthKit fields its identity is built from. Send a panel and its members in the same batch so the members carry the panel's name. No `userId`: the owner is the caller.",
});
const fhirLabRecordResult = z
.object({
index: z.number().int(),
status: z.enum(["inserted", "duplicate", "skipped", "failed"]),
reason: z
.enum([...FHIR_LAB_SKIP_REASONS, ...FHIR_LAB_FAILURE_REASONS])
.optional(),
})
.meta({
id: "FhirLabRecordResult",
description:
"One record's outcome, by its position in the request. `inserted`: written. `duplicate`: the record already holds it, under the same identity (deleted readings included, so a deleted result stays deleted) or as the same reading on the same day from another source. `skipped`: not a reading HealthLog keeps, with `reason` `not_observation` | `not_lab` | `not_final` (preliminary; it is taken once final) | `withdrawn` | `panel_header` | `no_value` | `unsupported_value` | `no_date` | `future_date` | `invalid_date` | `too_large` (a resource over 256 KiB). `failed`: a reading that could not be stored, with `reason` `missing_unit` | `unit_mismatch` (the account's marker is in a unit this one does not convert into) | `text_too_long` | `write_failed` (transient: send it again).",
});
const fhirLabBatchResponse = z
.object({
results: z.array(fhirLabRecordResult),
inserted: z.number().int(),
duplicate: z.number().int(),
skipped: z.number().int(),
failed: z.number().int(),
outcome: z.enum(["empty", "failed", "partial", "success"]),
})
.meta({ id: "FhirLabBatchResponse" });
const notFound = {
"404": {
description: "Lab result not found (or owned by another user).",
@@ -239,6 +278,40 @@ export const labsPaths: NonNullable<ZodOpenApiObject["paths"]> = {
},
},
},
"/api/labs/fhir-batch": {
post: {
tags: ["Labs"],
summary: "Ingest lab results from Apple Health's Health Records",
description:
"Maps each FHIR lab result (DSTU2 or R4 Observation) to a lab reading and writes it: final results only (`final`, `amended`, `corrected`), the laboratory category, numeric values in their unit (UCUM codes are read), a value with a comparator (`<0.5 mg/L`) kept as text as printed, string and coded values as text, the report's own reference range, and the panel's name for its members. The name resolves to the caller's marker by its own name, an alias, or the catalog analyte its LOINC code names; a new analyte gets a new marker; a reading in another unit is converted when the conversion is known and refused otherwise. The identity is a digest of `sourceURL` and `identifier`, the same one an Apple Health export import builds, so a result already imported from an export is a `duplicate` here, and sending a batch again is safe. First write wins: an amended result under a held identity is a duplicate. Owner-only: a cookie session or a full-scope login token. The body is capped at 1 MiB (413 past it): a client splits a large sync by size as well as by count. Rate limit 60 batches a minute; Idempotency-Key supported. A response carrying `write_failed` is `Cache-Control: no-store` and is not replayed for its key. Audits as `labs.fhir.batch` with counts only.",
parameters: [idempotencyKeyParameter],
requestBody: {
required: true,
content: { "application/json": { schema: fhirLabBatchRequest } },
},
responses: {
...idempotentWrite(),
"200": {
description:
"Batch processed; one result per record. `outcome` reads `inserted` against `failed` (skips and duplicates are information, not failure).",
content: {
"application/json": {
schema: dataEnvelope(
fhirLabBatchResponse,
"FhirLabBatchEnvelope",
),
},
},
},
...stdResponses,
"422": {
description:
"Nothing was written. `meta.errorCode` = `labs.fhir_batch.invalid`: no records, more than 200, a record without a `resource` object, an over-long HealthKit field, or a field the request does not define (every issue under `details.issues`).",
content: { "application/json": { schema: errorEnvelope } },
},
},
},
},
"/api/labs/overview": {
get: {
tags: ["Labs"],
+48
View File
@@ -0,0 +1,48 @@
/**
* `POST /api/labs/fhir-batch` (#1400): lab results from Apple Health's Health
* Records, as the iOS app reads them from HealthKit (`HKClinicalRecord` →
* `HKFHIRResource`). Each record carries the FHIR resource as parsed JSON and
* the four HealthKit fields the identity is built from. The resource itself is
* checked by the mapper, not here: an unknown shape is a per-record skip, not
* a refusal of the batch.
*
* No `userId` field: the owner is the authenticated caller.
*/
import { z } from "zod/v4";
export const MAX_FHIR_LAB_RECORDS_PER_BATCH = 200;
/** A serialised resource past this is skipped as `too_large`. */
export const MAX_FHIR_LAB_RESOURCE_BYTES = 256 * 1024;
export const fhirLabRecordSchema = z.object({
resource: z.record(z.string(), z.unknown()).meta({
description:
"The FHIR resource as JSON (`HKFHIRResource.data`), DSTU2 or R4. Only Observations of the laboratory category are kept.",
}),
fhirVersion: z.string().max(16).optional().meta({
description: "`HKFHIRResource.fhirVersion` (e.g. `4.0.1`).",
}),
identifier: z.string().max(256).optional().meta({
description:
"`HKFHIRResource.identifier`, the resource's id at its source.",
}),
sourceURL: z.string().max(2048).optional().meta({
description:
"`HKFHIRResource.sourceURL`, the issuing institution's FHIR base.",
}),
sourceName: z.string().max(200).optional().meta({
description:
"The institution as the Health app names it (`sourceRevision.source.name`).",
}),
});
export const fhirLabBatchSchema = z
.object({
records: z
.array(fhirLabRecordSchema)
.min(1)
.max(MAX_FHIR_LAB_RECORDS_PER_BATCH),
})
.strict();
export type FhirLabBatchBody = z.infer<typeof fhirLabBatchSchema>;