diff --git a/prisma/migrations/0500_lab_result_external_id/migration.sql b/prisma/migrations/0500_lab_result_external_id/migration.sql new file mode 100644 index 000000000..30f1f3bcb --- /dev/null +++ b/prisma/migrations/0500_lab_result_external_id/migration.sql @@ -0,0 +1,25 @@ +-- The identity a lab reading arrived under from Apple Health's Health Records +-- (#1400): a SHA-256 digest over the issuing institution and the FHIR +-- Observation's identifier, never the identifier itself. It lets the same +-- result, arriving again from the export or from the iOS app, be recognised +-- and left alone, and a result the person deleted stay deleted. +-- +-- Nullable, no default: every existing reading reads as "no external +-- identity", and adding the column rewrites nothing. The unique index is on +-- (user_id, source, external_id); NULLs never collide, so the readings typed, +-- scanned or imported from CSV are unaffected. IF NOT EXISTS keeps a replay +-- on a database that already ran a build carrying the column harmless. +-- Reversibility: dropping the index and the column restores the previous +-- schema. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + ALTER TABLE "lab_results" + ADD COLUMN IF NOT EXISTS "external_id" TEXT; + CREATE UNIQUE INDEX IF NOT EXISTS "lab_results_user_id_source_external_id_key" + ON "lab_results" ("user_id", "source", "external_id"); +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c1077c52d..7f3e6bee0 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -6653,6 +6653,14 @@ model LabResult { biomarkerId String? @map("biomarker_id") /// Provenance: "MANUAL" today; import paths set their own source string. source String @default("MANUAL") + /// #1400 — the identity a reading arrived under from Apple Health's Health + /// Records: "fhir:" + a SHA-256 digest of the issuing institution and the + /// FHIR Observation's identifier (or "fhirc:" + a digest of its content when + /// it has none). A digest, not the identifier and not ciphertext. NULL for + /// every reading typed, scanned or imported from CSV. Read by + /// `writeFhirLabReadings` (`src/lib/labs/fhir-import/write.ts`), which treats + /// a reading it already holds under this id, deleted or not, as a duplicate. + externalId String? @map("external_id") /// Optional free-text note, AES-256-GCM encrypted at rest via /// `src/lib/crypto.ts` (Bytes column, `*Encrypted` suffix per convention). noteEncrypted Bytes? @map("note_encrypted") @@ -6666,6 +6674,7 @@ model LabResult { encounterLinks EncounterLabLink[] documentLinks LabResultDocumentLink[] + @@unique([userId, source, externalId]) @@index([userId, analyte, takenAt]) @@index([userId, deletedAt]) @@index([biomarkerId]) diff --git a/src/lib/fhir/__tests__/lab-loinc-reverse.test.ts b/src/lib/fhir/__tests__/lab-loinc-reverse.test.ts new file mode 100644 index 000000000..247115ef7 --- /dev/null +++ b/src/lib/fhir/__tests__/lab-loinc-reverse.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog"; + +import { CATALOG_SLUG_TO_LOINC_KEY, catalogSlugForLoinc } from "../lab-loinc"; + +describe("catalogSlugForLoinc", () => { + it("inverts the export map: each slug's own code answers with the slug", () => { + const exported: Record = { + "2093-3": "total-cholesterol", + "2089-1": "ldl", + "2085-9": "hdl", + "2571-8": "triglycerides", + "1558-6": "fasting-glucose", + "4548-4": "hba1c", + "3016-3": "tsh", + "2276-4": "ferritin", + "62292-8": "vitamin-d", + "30522-7": "hs-crp", + "2160-0": "creatinine", + "69405-9": "egfr", + "1742-6": "alt", + "1920-8": "ast", + "2324-2": "ggt", + "718-7": "hemoglobin", + "1884-6": "apob", + "43583-4": "lp-a", + "27873-9": "fasting-insulin", + }; + expect(Object.keys(exported)).toHaveLength( + Object.keys(CATALOG_SLUG_TO_LOINC_KEY).length, + ); + for (const [loinc, slug] of Object.entries(exported)) { + expect(catalogSlugForLoinc(loinc)).toBe(slug); + } + }); + + it.each([ + ["13457-7", "ldl"], + ["18262-6", "ldl"], + ["33914-3", "egfr"], + ["48642-3", "egfr"], + ["48643-1", "egfr"], + ["50044-7", "egfr"], + ["62238-1", "egfr"], + ["98979-8", "egfr"], + ["17856-6", "hba1c"], + ["11579-0", "tsh"], + ])("pins the import-only alias %s -> %s", (loinc, slug) => { + expect(catalogSlugForLoinc(loinc)).toBe(slug); + }); + + it("answers only with slugs the catalog has", () => { + const slugs = new Set(BIOMARKER_CATALOG.map((seed) => seed.slug)); + for (const loinc of [ + "2089-1", + "13457-7", + "62238-1", + "17856-6", + "11579-0", + ]) { + expect(slugs.has(catalogSlugForLoinc(loinc)!)).toBe(true); + } + }); + + it("returns null for a code it does not know", () => { + expect(catalogSlugForLoinc("1988-5")).toBeNull(); + expect(catalogSlugForLoinc("")).toBeNull(); + }); +}); diff --git a/src/lib/fhir/lab-loinc.ts b/src/lib/fhir/lab-loinc.ts index 03699feae..36c364ae9 100644 --- a/src/lib/fhir/lab-loinc.ts +++ b/src/lib/fhir/lab-loinc.ts @@ -479,3 +479,49 @@ export function resolveLabCoding( const ucum = normaliseUnit(unit) === mapping.ucum ? mapping.ucum : null; return { loinc: mapping.loinc, display: mapping.display, ucum }; } + +/** + * LOINC codes a FHIR import meets that this module does not export under, by + * the catalog slug they belong to. Each names the same analyte in the same + * kind of unit as the slug's own code, differing only in method or formula: + * LDL by calculation (13457-7) or by direct assay (18262-6); eGFR by MDRD + * (33914-3, 48642-3, 48643-1, 50044-7) or CKD-EPI (62238-1, 98979-8); HbA1c by + * HPLC (17856-6); TSH by a detection-limit method (11579-0). Import only: the + * exporter keeps its own method-less code. + */ +const IMPORT_ONLY_LOINC_TO_SLUG: Readonly> = { + "13457-7": "ldl", + "18262-6": "ldl", + "33914-3": "egfr", + "48642-3": "egfr", + "48643-1": "egfr", + "50044-7": "egfr", + "62238-1": "egfr", + "98979-8": "egfr", + "17856-6": "hba1c", + "11579-0": "tsh", +}; + +let loincToSlug: ReadonlyMap | null = null; + +/** + * The labs-catalog slug a LOINC code names, or null. The inverse of + * {@link CATALOG_SLUG_TO_LOINC_KEY} over {@link LAB_LOINC_BY_KEY}, plus the + * import-only method variants above. A FHIR lab result carrying one of these + * codes is matched to the account's marker for that analyte whatever the + * sending system called it. + */ +export function catalogSlugForLoinc(code: string): string | null { + if (!loincToSlug) { + const map = new Map(); + for (const [slug, key] of Object.entries(CATALOG_SLUG_TO_LOINC_KEY)) { + const mapping = LAB_LOINC_BY_KEY[key]; + if (mapping) map.set(mapping.loinc, slug); + } + for (const [loinc, slug] of Object.entries(IMPORT_ONLY_LOINC_TO_SLUG)) { + map.set(loinc, slug); + } + loincToSlug = map; + } + return loincToSlug.get(code.trim()) ?? null; +} diff --git a/src/lib/labs/__tests__/biomarker-resolve.test.ts b/src/lib/labs/__tests__/biomarker-resolve.test.ts index 6754c88b6..92a431b3a 100644 --- a/src/lib/labs/__tests__/biomarker-resolve.test.ts +++ b/src/lib/labs/__tests__/biomarker-resolve.test.ts @@ -284,3 +284,33 @@ describe("suggestions", () => { expect(nameSimilarity("LDL", "Ferritin")).toBeLessThan(0.5); }); }); + +describe("a catalog hint from the source (#1400)", () => { + it("matches a marker by the hinted analyte when the name says nothing", () => { + const ldl = marker("LDL", "mg/dL"); + const idx = index([ldl]); + const name = + "Cholesterol in LDL [Mass/volume] in Serum or Plasma by calculation"; + expect(resolveInIndex(idx, name, "mmol/L")).toBeNull(); + expect( + resolveInIndex(idx, name, "mmol/L", { catalogKey: "ldl" }), + ).toMatchObject({ marker: { id: ldl.id }, matchKind: "catalog" }); + }); + + it("still steps aside for a unit the marker cannot take", () => { + const ldl = marker("LDL", "mg/dL"); + expect( + resolveInIndex(index([ldl]), "LDL calc", "U/L", { catalogKey: "ldl" }), + ).toBeNull(); + }); + + it("lets the account's own name win over the hint", () => { + const own = marker("Odd name", "mmol/L"); + const ldl = marker("LDL", "mg/dL"); + expect( + resolveInIndex(index([own, ldl]), "odd name", "mmol/L", { + catalogKey: "ldl", + }), + ).toMatchObject({ marker: { id: own.id }, matchKind: "name" }); + }); +}); diff --git a/src/lib/labs/biomarker-resolve.ts b/src/lib/labs/biomarker-resolve.ts index e32a56bea..2ed57d1d1 100644 --- a/src/lib/labs/biomarker-resolve.ts +++ b/src/lib/labs/biomarker-resolve.ts @@ -172,11 +172,16 @@ export function unitFitsMarker( /** * The marker a printed name resolves to, and how; null when nothing does. * `unit` is the unit the reading states, used only by the catalog step. + * `options.catalogKey` names the catalog analyte when the source says which + * one it is (a LOINC code on a FHIR result): the catalog step then uses it in + * place of the analyte the name alone would suggest. The account's own names + * and aliases still win first. */ export function resolveInIndex( index: MarkerIndex, rawName: string, unit?: string | null, + options: { catalogKey?: string | null } = {}, ): MarkerMatch | null { const name = rawName.trim(); if (!name) return null; @@ -203,7 +208,7 @@ export function resolveInIndex( } // 3. The catalog: a marker of the same analyte that can take the unit. - const key = catalogKeyForName(name); + const key = options.catalogKey ?? catalogKeyForName(name); if (key) { const candidates = index.markers.filter( (m) => diff --git a/src/lib/labs/biomarker-store.ts b/src/lib/labs/biomarker-store.ts index 3f4728eba..ad750e646 100644 --- a/src/lib/labs/biomarker-store.ts +++ b/src/lib/labs/biomarker-store.ts @@ -22,6 +22,7 @@ import { type MarkerIndex, } from "@/lib/labs/biomarker-resolve"; import { mintedBounds } from "@/lib/labs/minted-bounds"; +import { analyteAcceptsUnit } from "@/lib/labs/unit-conversion/convert"; import { getEvent } from "@/lib/logging/context"; /** Encrypt a UTF-8 context note into the `Bytes` payload the schema stores. */ @@ -110,11 +111,12 @@ export async function resolveOrMintBiomarker( referenceHigh: number | null; panel: string | null; }, - options: { index?: MarkerIndex } = {}, + options: { index?: MarkerIndex; catalogKey?: string | null } = {}, ): Promise { const name = input.analyte.trim(); const index = options.index ?? (await loadMarkerIndex(prisma, userId)); - const hit = resolveInIndex(index, name, input.unit || null); + const catalogKey = options.catalogKey ?? null; + const hit = resolveInIndex(index, name, input.unit || null, { catalogKey }); if (hit) return toMinted(hit.marker); const { lowerBound, upperBound } = mintedBounds( @@ -130,7 +132,13 @@ export async function resolveOrMintBiomarker( lowerBound, upperBound, panel: input.panel, - analyteKey: matchAnalyteKey({ name, unit: input.unit }), + // A name the catalog does not know still links when the source named + // the analyte (a LOINC code), provided the unit is one it reaches. + analyteKey: + matchAnalyteKey({ name, unit: input.unit }) ?? + (catalogKey && analyteAcceptsUnit(catalogKey, input.unit) + ? catalogKey + : null), }, select: INDEXED_SELECT, }); diff --git a/src/lib/labs/fhir-import/__tests__/external-id.test.ts b/src/lib/labs/fhir-import/__tests__/external-id.test.ts new file mode 100644 index 000000000..a31c41d74 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/external-id.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { fhirLabExternalId, fhirLabIssuer } from "../external-id"; + +const AT = new Date("2025-03-04T07:15:00Z"); + +describe("fhirLabExternalId", () => { + it("gives the export and the HealthKit record the same id", () => { + const fromExport = fhirLabExternalId({ + sourceURL: "https://FHIR.example-health.test/api/R4/", + sourceName: "Example Health", + identifier: "obs-1", + resourceId: "obs-1", + analyte: "LDL", + takenAt: AT, + value: 3.1, + valueText: null, + }); + const fromPhone = fhirLabExternalId({ + sourceURL: "https://fhir.example-health.test/api/R4", + sourceName: "Example Health (renamed)", + identifier: "obs-1", + analyte: "LDL Cholesterol", + takenAt: new Date("2025-03-05T00:00:00Z"), + value: 80, + valueText: null, + }); + expect(fromExport).toBe(fromPhone); + expect(fromExport).toMatch(/^fhir:[0-9a-f]{40}$/); + }); + + it("keeps two institutions apart", () => { + const a = fhirLabExternalId({ + sourceURL: "https://a.test", + identifier: "obs-1", + analyte: "x", + takenAt: AT, + value: 1, + valueText: null, + }); + const b = fhirLabExternalId({ + sourceURL: "https://b.test", + identifier: "obs-1", + analyte: "x", + takenAt: AT, + value: 1, + valueText: null, + }); + expect(a).not.toBe(b); + }); + + it("falls back to a stable content identity", () => { + const input = { + sourceName: "Clinic", + loinc: "2345-7", + analyte: " Glucose ", + takenAt: AT, + value: 92, + valueText: null, + }; + const first = fhirLabExternalId(input); + expect(first).toMatch(/^fhirc:[0-9a-f]{40}$/); + expect(fhirLabExternalId({ ...input, analyte: "glucose" })).toBe(first); + expect(fhirLabExternalId({ ...input, value: 93 })).not.toBe(first); + }); + + it("names the issuer by URL, then name, then unknown", () => { + expect(fhirLabIssuer({ sourceURL: " https://X.test/// " })).toBe( + "https://x.test", + ); + expect(fhirLabIssuer({ sourceName: " Clinic " })).toBe("Clinic"); + expect(fhirLabIssuer({})).toBe("unknown"); + }); +}); diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-codeableconcept.json b/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-codeableconcept.json new file mode 100644 index 000000000..95573c21e --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-codeableconcept.json @@ -0,0 +1,17 @@ +{ + "resourceType": "Observation", + "id": "dstu2-blood-group", + "status": "final", + "category": { "coding": [{ "code": "laboratory" }] }, + "code": { "text": "ABO group" }, + "effectiveDateTime": "2019-06-11T07:05:00Z", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "112144000", + "display": "Blood group A" + } + ] + } +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-quantity-glucose.json b/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-quantity-glucose.json new file mode 100644 index 000000000..68c838e65 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-quantity-glucose.json @@ -0,0 +1,37 @@ +{ + "resourceType": "Observation", + "id": "dstu2-glu", + "status": "final", + "category": { + "coding": [ + { + "system": "http://hl7.org/fhir/observation-category", + "code": "laboratory" + } + ] + }, + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "2345-7", + "display": "Glucose [Mass/volume] in Serum or Plasma" + } + ] + }, + "subject": { "reference": "Patient/example-patient" }, + "effectiveDateTime": "2019-06-11T07:05:00-07:00", + "valueQuantity": { + "value": 92, + "unit": "mg/dL", + "system": "http://unitsofmeasure.org", + "code": "mg/dL" + }, + "referenceRange": [ + { + "low": { "value": 70, "unit": "mg/dL" }, + "high": { "value": 99, "unit": "mg/dL" } + } + ], + "comments": "Specimen slightly haemolysed." +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-related-has-member.json b/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-related-has-member.json new file mode 100644 index 000000000..4382f1e12 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/dstu2-related-has-member.json @@ -0,0 +1,37 @@ +[ + { + "resourceType": "Observation", + "id": "dstu2-lipid", + "status": "final", + "category": { "coding": [{ "code": "laboratory" }] }, + "code": { "text": "Lipid panel" }, + "effectiveDateTime": "2019-06-11T07:05:00Z", + "related": [ + { + "type": "has-member", + "target": { "reference": "Observation/dstu2-hdl" } + }, + { + "type": "derived-from", + "target": { "reference": "Observation/dstu2-other" } + } + ] + }, + { + "resourceType": "Observation", + "id": "dstu2-hdl", + "status": "final", + "category": { "coding": [{ "code": "laboratory" }] }, + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "2085-9", + "display": "HDL Cholesterol" + } + ] + }, + "effectiveDateTime": "2019-06-11T07:05:00Z", + "valueQuantity": { "value": 55, "unit": "mg/dL" } + } +] diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-comparator.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-comparator.json new file mode 100644 index 000000000..b16d70510 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-comparator.json @@ -0,0 +1,25 @@ +{ + "resourceType": "Observation", + "id": "obs-crp-1", + "status": "final", + "category": [{ "coding": [{ "code": "laboratory" }] }], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "1988-5", + "display": "C reactive protein [Mass/volume] in Serum or Plasma" + } + ] + }, + "subject": { "reference": "Patient/example-patient" }, + "effectiveDateTime": "2025-03-04T08:15:00Z", + "valueQuantity": { + "value": 0.5, + "comparator": "<", + "unit": "mg/L", + "system": "http://unitsofmeasure.org", + "code": "mg/L" + }, + "referenceRange": [{ "high": { "value": 5, "unit": "mg/L" } }] +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-date-only.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-date-only.json new file mode 100644 index 000000000..68b13cf08 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-date-only.json @@ -0,0 +1,17 @@ +{ + "resourceType": "Observation", + "id": "obs-tsh", + "status": "corrected", + "code": { + "coding": [ + { + "system": "urn:oid:2.16.840.1.113883.6.1", + "code": "11579-0", + "display": "Thyrotropin [Units/volume] in Serum or Plasma by Detection limit <= 0.05 mIU/L" + } + ] + }, + "effectiveDateTime": "2024-11-20", + "valueQuantity": { "value": 2.3, "code": "m[IU]/L" }, + "referenceRange": [{ "text": "0.4 - 4.0" }] +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-no-value-interpretation-pos.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-no-value-interpretation-pos.json new file mode 100644 index 000000000..347964c74 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-no-value-interpretation-pos.json @@ -0,0 +1,18 @@ +{ + "resourceType": "Observation", + "id": "obs-covid", + "status": "final", + "category": [{ "coding": [{ "code": "laboratory" }] }], + "code": { "text": "SARS-CoV-2 RNA" }, + "effectiveDateTime": "2024-12-01T10:00:00Z", + "interpretation": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", + "code": "POS" + } + ] + } + ] +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-panel-cbc.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-panel-cbc.json new file mode 100644 index 000000000..983ae6d6b --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-panel-cbc.json @@ -0,0 +1,56 @@ +[ + { + "resourceType": "Observation", + "id": "obs-cbc", + "status": "final", + "category": [{ "coding": [{ "code": "laboratory" }] }], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "58410-2", + "display": "CBC panel - Blood by Automated count" + } + ], + "text": "Complete blood count" + }, + "effectiveDateTime": "2025-02-01T07:30:00Z", + "hasMember": [ + { "reference": "Observation/obs-hgb" }, + { "reference": "https://fhir.example.org/api/Observation/obs-wbc" } + ] + }, + { + "resourceType": "Observation", + "id": "obs-hgb", + "status": "final", + "category": [{ "coding": [{ "code": "laboratory" }] }], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "718-7", + "display": "Hemoglobin [Mass/volume] in Blood" + } + ], + "text": "Hemoglobin" + }, + "effectiveDateTime": "2025-02-01T07:30:00Z", + "valueQuantity": { "value": 14.2, "unit": "g/dL", "code": "g/dL" }, + "referenceRange": [ + { + "low": { "value": 13.5, "unit": "g/dL" }, + "high": { "value": 17.5, "unit": "g/dL" } + } + ] + }, + { + "resourceType": "Observation", + "id": "obs-wbc", + "status": "final", + "category": [{ "coding": [{ "code": "laboratory" }] }], + "code": { "text": "WBC" }, + "effectiveDateTime": "2025-02-01T07:30:00Z", + "valueQuantity": { "value": 6.4, "unit": "x10^3/uL", "code": "10*3/uL" } + } +] diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-preliminary.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-preliminary.json new file mode 100644 index 000000000..cd8fb3fe0 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-preliminary.json @@ -0,0 +1,9 @@ +{ + "resourceType": "Observation", + "id": "obs-prelim", + "status": "preliminary", + "category": [{ "coding": [{ "code": "laboratory" }] }], + "code": { "text": "Potassium" }, + "effectiveDateTime": "2025-02-01T07:30:00Z", + "valueQuantity": { "value": 4.1, "unit": "mmol/L" } +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-quantity-ldl.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-quantity-ldl.json new file mode 100644 index 000000000..59f4bb415 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-quantity-ldl.json @@ -0,0 +1,46 @@ +{ + "resourceType": "Observation", + "id": "obs-ldl-1", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display": "Laboratory" + } + ] + } + ], + "code": { + "coding": [ + { + "system": "http://loinc.org", + "code": "13457-7", + "display": "Cholesterol in LDL [Mass/volume] in Serum or Plasma by calculation" + } + ], + "text": "LDL Cholesterol (calc)" + }, + "subject": { + "reference": "Patient/example-patient", + "display": "Example Patient" + }, + "performer": [{ "display": "Example Laboratory" }], + "effectiveDateTime": "2025-03-04T08:15:00+01:00", + "issued": "2025-03-05T10:00:00+01:00", + "valueQuantity": { + "value": 3.1, + "unit": "mmol/L", + "system": "http://unitsofmeasure.org", + "code": "mmol/L" + }, + "referenceRange": [ + { + "high": { "value": 3.0, "unit": "mmol/L", "code": "mmol/L" }, + "text": "< 3.0 mmol/L" + } + ], + "note": [{ "text": "Fasting sample." }] +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-valuestring-positive.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-valuestring-positive.json new file mode 100644 index 000000000..c5c2e5ce5 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-valuestring-positive.json @@ -0,0 +1,9 @@ +{ + "resourceType": "Observation", + "id": "obs-strep-1", + "status": "amended", + "category": [{ "text": "Laboratory" }], + "code": { "text": "Strep A antigen" }, + "effectiveDateTime": "2025-01-10T09:00:00-05:00", + "valueString": " Positive " +} diff --git a/src/lib/labs/fhir-import/__tests__/fixtures/r4-vital-signs.json b/src/lib/labs/fhir-import/__tests__/fixtures/r4-vital-signs.json new file mode 100644 index 000000000..021ab2064 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/fixtures/r4-vital-signs.json @@ -0,0 +1,18 @@ +{ + "resourceType": "Observation", + "id": "obs-bp", + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs" + } + ] + } + ], + "code": { "text": "Body weight" }, + "effectiveDateTime": "2025-02-01T07:30:00Z", + "valueQuantity": { "value": 72, "unit": "kg" } +} diff --git a/src/lib/labs/fhir-import/__tests__/map-observation.test.ts b/src/lib/labs/fhir-import/__tests__/map-observation.test.ts new file mode 100644 index 000000000..d19094844 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/map-observation.test.ts @@ -0,0 +1,319 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + assignPanels, + fhirInstant, + mapFhirLabRecord, + normaliseObservationRef, + type FhirLabMapping, +} from "../map-observation"; + +const NOW = new Date("2026-01-01T00:00:00Z"); +const SOURCE = { + sourceURL: "https://fhir.example-health.test/api/FHIR/R4/", + sourceName: "Example Health", + fhirVersion: "4.0.1", +}; + +function fixture(name: string): unknown { + return JSON.parse( + readFileSync(join(__dirname, "fixtures", `${name}.json`), "utf8"), + ); +} + +function map(resource: unknown, extra: Record = {}) { + return mapFhirLabRecord({ resource, ...SOURCE, ...extra }, NOW); +} + +function reading(result: FhirLabMapping) { + if (result.kind !== "reading") { + throw new Error(`expected a reading, got ${result.kind}`); + } + return result.reading; +} + +function base(overrides: Record = {}) { + return { + resourceType: "Observation", + id: "obs-x", + status: "final", + category: [{ coding: [{ code: "laboratory" }] }], + code: { text: "Sodium" }, + effectiveDateTime: "2025-01-01T08:00:00Z", + valueQuantity: { value: 140, unit: "mmol/L" }, + ...overrides, + }; +} + +describe("mapFhirLabRecord — R4", () => { + it("maps a quantity with its LOINC hint, range and note", () => { + const r = reading( + map(fixture("r4-quantity-ldl"), { identifier: "obs-ldl-1" }), + ); + expect(r.analyte).toBe("LDL Cholesterol (calc)"); + expect(r.loinc).toBe("13457-7"); + expect(r.catalogKey).toBe("ldl"); + expect(r.value).toBe(3.1); + expect(r.valueText).toBeNull(); + expect(r.unit).toBe("mmol/L"); + expect(r.takenAt.toISOString()).toBe("2025-03-04T07:15:00.000Z"); + expect(r.sourceReferenceLow).toBeNull(); + expect(r.sourceReferenceHigh).toBe(3); + expect(r.sourceReferenceText).toBe("< 3.0 mmol/L"); + expect(r.note).toBe("Fasting sample."); + expect(r.panel).toBeNull(); + expect(r.externalId).toMatch(/^fhir:[0-9a-f]{40}$/); + }); + + it("never carries the subject or performer anywhere", () => { + const serialised = JSON.stringify(map(fixture("r4-quantity-ldl"))); + expect(serialised).not.toContain("Example Patient"); + expect(serialised).not.toContain("example-patient"); + expect(serialised).not.toContain("Example Laboratory"); + }); + + it("keeps a comparator result as the printed text, unit included", () => { + const r = reading(map(fixture("r4-comparator"))); + expect(r.value).toBeNull(); + expect(r.valueText).toBe("<0.5 mg/L"); + expect(r.unit).toBe("mg/L"); + // A text result records no window of its own. + expect(r.sourceReferenceText).toBeNull(); + expect(r.analyte).toBe( + "C reactive protein [Mass/volume] in Serum or Plasma", + ); + }); + + it("drops the unity from a comparator text", () => { + const r = reading( + map( + base({ valueQuantity: { value: 1.1, comparator: ">", code: "{INR}" } }), + ), + ); + expect(r.valueText).toBe(">1.1"); + }); + + it("trims a string result", () => { + const r = reading(map(fixture("r4-valuestring-positive"))); + expect(r.valueText).toBe("Positive"); + expect(r.value).toBeNull(); + expect(r.unit).toBeNull(); + }); + + it("anchors a date alone, and a time without offset, at noon UTC", () => { + const r = reading(map(fixture("r4-date-only"))); + expect(r.takenAt.toISOString()).toBe("2024-11-20T12:00:00.000Z"); + expect(r.catalogKey).toBe("tsh"); + expect(r.unit).toBe("mIU/L"); + // A text-only range is read. + expect(r.sourceReferenceLow).toBe(0.4); + expect(r.sourceReferenceHigh).toBe(4); + expect(fhirInstant("2024-11-20T08:00:00")?.toISOString()).toBe( + "2024-11-20T12:00:00.000Z", + ); + expect(fhirInstant("2024-11")).toBeNull(); + expect(fhirInstant("2024")).toBeNull(); + expect(fhirInstant("2024-02-30")).toBeNull(); + }); + + it("falls back through the period start and issued", () => { + const period = reading( + map( + base({ + effectiveDateTime: undefined, + effectivePeriod: { start: "2025-01-02T09:00:00Z" }, + }), + ), + ); + expect(period.takenAt.toISOString()).toBe("2025-01-02T09:00:00.000Z"); + const issued = reading( + map( + base({ effectiveDateTime: undefined, issued: "2025-01-03T09:00:00Z" }), + ), + ); + expect(issued.takenAt.toISOString()).toBe("2025-01-03T09:00:00.000Z"); + }); + + it("keeps an interpretation as text when there is no value", () => { + const r = reading(map(fixture("r4-no-value-interpretation-pos"))); + expect(r.valueText).toBe("Positive"); + }); + + it("maps the panel header to its members' panel", () => { + const results = (fixture("r4-panel-cbc") as unknown[]).map((r) => map(r)); + expect(results[0]).toMatchObject({ kind: "skip", reason: "panel_header" }); + assignPanels(results); + expect(reading(results[1]).panel).toBe("Complete blood count"); + expect(reading(results[2]).panel).toBe("Complete blood count"); + expect(reading(results[2]).unit).toBe("10^3/µL"); + }); + + it.each([ + ["r4-preliminary", "not_final"], + ["r4-vital-signs", "not_lab"], + ])("skips %s as %s", (name, reason) => { + expect(map(fixture(name))).toMatchObject({ kind: "skip", reason }); + }); + + it.each([ + [{ resourceType: "Condition" }, "not_observation"], + ["not an object", "not_observation"], + [base({ status: "entered-in-error" }), "withdrawn"], + [base({ status: "cancelled" }), "withdrawn"], + [base({ status: "registered" }), "not_final"], + [base({ status: undefined }), "not_final"], + [ + base({ valueQuantity: undefined, valueBoolean: true }), + "unsupported_value", + ], + [base({ valueQuantity: undefined, valueRange: {} }), "unsupported_value"], + [base({ valueQuantity: undefined, component: [] }), "unsupported_value"], + [ + base({ valueQuantity: undefined, dataAbsentReason: { text: "x" } }), + "no_value", + ], + [base({ valueQuantity: undefined }), "no_value"], + [base({ code: {} }), "no_value"], + [base({ effectiveDateTime: undefined }), "no_date"], + [base({ effectiveDateTime: "2026-01-03T00:00:00Z" }), "future_date"], + [base({ effectiveDateTime: "1899-12-31" }), "invalid_date"], + ])("skips %j as %s", (resource, reason) => { + expect(map(resource)).toMatchObject({ kind: "skip", reason }); + }); + + it.each([ + [base({ valueQuantity: { value: 5 } }), "missing_unit"], + [base({ valueQuantity: undefined, valueInteger: 5 }), "missing_unit"], + [base({ code: { text: "x".repeat(201) } }), "text_too_long"], + [ + base({ valueQuantity: undefined, valueString: "y".repeat(201) }), + "text_too_long", + ], + ])("refuses %j as %s", (resource, reason) => { + expect(map(resource)).toMatchObject({ kind: "fail", reason }); + }); + + it("accepts a resource with no category", () => { + expect(map(base({ category: undefined })).kind).toBe("reading"); + }); + + it("keeps a range only in the reading's unit", () => { + const r = reading( + map( + base({ + referenceRange: [ + { + low: { value: 3.5, unit: "mg/dL" }, + high: { value: 145, unit: "mmol/L" }, + }, + ], + }), + ), + ); + expect(r.sourceReferenceLow).toBeNull(); + expect(r.sourceReferenceHigh).toBe(145); + }); + + it("prefers the general range over an age-specific one, and drops low > high", () => { + const chosen = reading( + map( + base({ + referenceRange: [ + { + low: { value: 1 }, + high: { value: 2 }, + age: { low: { value: 1 } }, + }, + { low: { value: 135 }, high: { value: 145 } }, + ], + }), + ), + ); + expect([chosen.sourceReferenceLow, chosen.sourceReferenceHigh]).toEqual([ + 135, 145, + ]); + const inverted = reading( + map( + base({ referenceRange: [{ low: { value: 9 }, high: { value: 1 } }] }), + ), + ); + expect(inverted.sourceReferenceLow).toBeNull(); + expect(inverted.sourceReferenceHigh).toBeNull(); + }); + + it("caps a long note", () => { + const r = reading(map(base({ note: [{ text: "n".repeat(1500) }] }))); + expect(r.note).toHaveLength(1000); + expect(r.note?.endsWith("…")).toBe(true); + }); + + it("reads a code display when the code carries no text", () => { + const r = reading( + map( + base({ + code: { + coding: [ + { system: "http://example.org", display: "Local name" }, + { + system: "http://loinc.org", + code: "2951-2", + display: "Sodium [Moles/volume] in Serum or Plasma", + }, + ], + }, + }), + ), + ); + expect(r.analyte).toBe("Sodium [Moles/volume] in Serum or Plasma"); + expect(r.loinc).toBe("2951-2"); + expect(r.catalogKey).toBeNull(); + }); +}); + +describe("mapFhirLabRecord — DSTU2", () => { + it("reads a single category object, comments and a glucose quantity", () => { + const r = reading(map(fixture("dstu2-quantity-glucose"))); + expect(r.analyte).toBe("Glucose [Mass/volume] in Serum or Plasma"); + expect(r.value).toBe(92); + expect(r.unit).toBe("mg/dL"); + expect([r.sourceReferenceLow, r.sourceReferenceHigh]).toEqual([70, 99]); + expect(r.note).toBe("Specimen slightly haemolysed."); + expect(JSON.stringify(r)).not.toContain("example-patient"); + }); + + it("reads a coded value as text", () => { + expect(reading(map(fixture("dstu2-codeableconcept"))).valueText).toBe( + "Blood group A", + ); + }); + + it("follows related has-member links only", () => { + const results = (fixture("dstu2-related-has-member") as unknown[]).map( + (r) => map(r), + ); + expect(results[0]).toMatchObject({ + kind: "skip", + reason: "panel_header", + memberRefs: ["Observation/dstu2-hdl"], + }); + assignPanels(results); + expect(reading(results[1]).panel).toBe("Lipid panel"); + expect(reading(results[1]).catalogKey).toBe("hdl"); + }); +}); + +describe("normaliseObservationRef", () => { + it.each([ + ["Observation/abc", "Observation/abc"], + ["https://x.test/fhir/Observation/abc", "Observation/abc"], + ["https://x.test/fhir/Observation/abc/_history/2", "Observation/abc"], + ["Patient/abc", null], + ["", null], + [42, null], + ])("%j -> %j", (raw, expected) => { + expect(normaliseObservationRef(raw)).toBe(expected); + }); +}); diff --git a/src/lib/labs/fhir-import/__tests__/units.test.ts b/src/lib/labs/fhir-import/__tests__/units.test.ts new file mode 100644 index 000000000..af831c0b5 --- /dev/null +++ b/src/lib/labs/fhir-import/__tests__/units.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { convertLabValue } from "@/lib/labs/unit-conversion/convert"; +import { scaleUnitOf } from "@/lib/labs/unit-conversion/scale"; +import { sameLabUnit } from "@/lib/labs/unit-normalise"; + +import { labUnitFromFhirQuantity } from "../units"; + +describe("labUnitFromFhirQuantity", () => { + it.each([ + [{ code: "10*3/uL" }, "10^3/µL"], + [{ code: "10*6/uL" }, "10^6/µL"], + [{ code: "10*9/L" }, "10^9/L"], + [{ code: "10*12/L" }, "10^12/L"], + [{ code: "[IU]/L" }, "IU/L"], + [{ code: "m[IU]/L" }, "mIU/L"], + [{ code: "m[IU]/mL" }, "mIU/mL"], + [{ code: "u[IU]/mL" }, "µIU/mL"], + [{ code: "meq/L" }, "mEq/L"], + [{ code: "/uL" }, "/µL"], + [{ code: "mL/min/{1.73_m2}" }, "mL/min/1.73m²"], + [{ code: "1" }, "1"], + [{ code: "{INR}" }, "1"], + [{ unit: "mg/dL", code: "mg/dL" }, "mg/dL"], + [{ unit: "x10^3/uL", code: "10*3/uL" }, "10^3/µL"], + [{ unit: "mmol/L" }, "mmol/L"], + [{ unit: " ", code: "g/L" }, "g/L"], + [{}, null], + [{ unit: 4 }, null], + ])("%j -> %j", (quantity, expected) => { + expect(labUnitFromFhirQuantity(quantity)).toBe(expected); + }); + + it("produces spellings the normaliser and conversion understand", () => { + for (const code of [ + "10*3/uL", + "10*6/uL", + "10*9/L", + "10*12/L", + "m[IU]/L", + "u[IU]/mL", + "[IU]/L", + ]) { + expect(scaleUnitOf(labUnitFromFhirQuantity({ code })!)).not.toBeNull(); + } + expect( + sameLabUnit(labUnitFromFhirQuantity({ code: "u[IU]/mL" })!, "µIU/ml"), + ).toBe(true); + expect( + convertLabValue({ + analyteKey: "tsh", + value: 2, + from: labUnitFromFhirQuantity({ code: "u[IU]/mL" })!, + to: "mIU/L", + }), + ).toMatchObject({ ok: true, value: 2 }); + expect( + convertLabValue({ + analyteKey: "ldl", + value: 3.1, + from: labUnitFromFhirQuantity({ code: "mmol/L" })!, + to: "mg/dL", + }).ok, + ).toBe(true); + }); +}); diff --git a/src/lib/labs/fhir-import/after-write.ts b/src/lib/labs/fhir-import/after-write.ts new file mode 100644 index 000000000..7b66c5209 --- /dev/null +++ b/src/lib/labs/fhir-import/after-write.ts @@ -0,0 +1,34 @@ +/** + * What follows a batch of lab readings landing, for the FHIR import paths: + * the health score is recomputed, a "blood panel due" reminder is resolved + * now rather than at the next cron, and the labs arm of the data-arrival + * spine is told once, with the newest draw and the true inserted count (the + * same three effects the scan commit runs). Nothing here throws. + */ +import { emitDataArrival } from "@/lib/arrivals/emit-shared"; +import { invalidateUserHealthScore } from "@/lib/cache/invalidate"; +import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; +import { fireAndForget } from "@/lib/logging/fire-and-forget"; + +export function afterLabReadingsWritten(input: { + userId: string; + inserted: number; + newestTakenAt: Date | null; + /** Transport token, for annotations only. */ + source: string; +}): void { + if (input.inserted <= 0) return; + invalidateUserHealthScore(input.userId); + fireAndForget(() => enqueueReminderSatisfy(input.userId), { + action: "reminder.satisfy.enqueue", + }); + if (input.newestTakenAt) { + void emitDataArrival({ + userId: input.userId, + kind: "labs_panel", + newestSampleAt: input.newestTakenAt, + insertedCount: input.inserted, + source: input.source, + }).catch(() => {}); + } +} diff --git a/src/lib/labs/fhir-import/external-id.ts b/src/lib/labs/fhir-import/external-id.ts new file mode 100644 index 000000000..075e7a2e1 --- /dev/null +++ b/src/lib/labs/fhir-import/external-id.ts @@ -0,0 +1,66 @@ +/** + * The identity a FHIR lab result is stored under (`LabResult.externalId`). + * + * The same result reaches HealthLog from two directions: inside an Apple + * Health export (`export.xml` names the issuing institution in `sourceURL` + * and the record in `identifier`, the JSON file carries the resource) and + * from the iOS app (`HKFHIRResource` carries the same `sourceURL`, + * `identifier` and data). The identity is built from those, so both paths + * land on one id and the second sees a duplicate. + * + * Only a digest is stored: the issuer's URL and the record identifier are + * hashed together and cut to 40 hex characters, which is enough to keep two + * results of one account apart and says nothing about either. + * + * A record with neither an identifier nor a resource id falls back to a + * content identity: the issuer, the LOINC code, the name, the instant and the + * value. Stable across re-imports of the same record; a corrected result + * whose value changed reads as a new one. + */ +import { createHash } from "node:crypto"; + +const DIGEST_HEX = 40; + +function digest(parts: readonly string[]): string { + return createHash("sha256") + .update(parts.join("\0")) + .digest("hex") + .slice(0, DIGEST_HEX); +} + +/** The institution that issued a record, as one stable string. */ +export function fhirLabIssuer(input: { + sourceURL?: string | null; + sourceName?: string | null; +}): string { + const url = input.sourceURL?.trim().toLowerCase().replace(/\/+$/u, ""); + if (url) return url; + const name = input.sourceName?.trim(); + if (name) return name; + return "unknown"; +} + +export function fhirLabExternalId(input: { + sourceURL?: string | null; + sourceName?: string | null; + /** The record identifier, else the resource's own `id`. */ + identifier?: string | null; + resourceId?: string | null; + /** For the content fallback only. */ + loinc?: string | null; + analyte: string; + takenAt: Date; + value: number | null; + valueText: string | null; +}): string { + const issuer = fhirLabIssuer(input); + const id = input.identifier?.trim() || input.resourceId?.trim(); + if (id) return `fhir:${digest([issuer, "Observation", id])}`; + return `fhirc:${digest([ + issuer, + input.loinc ?? "", + input.analyte.trim().toLowerCase(), + input.takenAt.toISOString(), + input.value !== null ? String(input.value) : (input.valueText ?? ""), + ])}`; +} diff --git a/src/lib/labs/fhir-import/map-observation.ts b/src/lib/labs/fhir-import/map-observation.ts new file mode 100644 index 000000000..d873a1998 --- /dev/null +++ b/src/lib/labs/fhir-import/map-observation.ts @@ -0,0 +1,463 @@ +/** + * One FHIR lab result (an `Observation`, DSTU2 or R4) → one lab reading, or + * the closed reason it is not one. Pure. + * + * Only whitelisted fields are read: `resourceType`, `id`, `status`, + * `category`, `code`, `effective*`, `issued`, `value*`, `dataAbsentReason`, + * `interpretation`, `referenceRange`, `hasMember` / `related`, `note` / + * `comments`. The patient, the performer and any contained resource are never + * looked at, and nothing from the resource ever reaches an error message: a + * malformed field is a reason, not a quote. + * + * The rules, in brief: + * - a result counts when it is final (`final`, `amended`, `corrected`); + * a preliminary one is skipped and a withdrawn one never written; + * - a category other than laboratory (a vital sign, a survey) is skipped; + * a resource with no category at all is taken, as Health Records only + * hands lab-result records to this path; + * - the name is the code's text, else the LOINC coding's display, else any + * coding's display; the LOINC code is a matching hint; + * - a number with a comparator ("<0.5 mg/L") is kept as text, as the report + * printed it, since a bound is not a value; + * - the report's own reference range is kept, in the reading's unit only; + * - a result with no value whose interpretation says positive, negative, + * detected, not detected or indeterminate is kept as that text; + * - a result with no value but members is a panel header: not a reading, + * but its name becomes its members' panel (`assignPanels`). + */ +import { catalogSlugForLoinc } from "@/lib/fhir/lab-loinc"; +import { + parseReferenceRange, + SOURCE_REFERENCE_TEXT_MAX, +} from "@/lib/labs/parse-reference-range"; +import { LAB_TEXT_LIMITS } from "@/lib/labs/reading-keys"; +import { sameLabUnit } from "@/lib/labs/unit-normalise"; + +import { fhirLabExternalId } from "./external-id"; +import type { + FhirLabFailureReason, + FhirLabRecordInput, + FhirLabSkipReason, + MappedFhirLab, +} from "./types"; +import { labUnitFromFhirQuantity } from "./units"; + +/** A reading dated more than a day ahead is a mistake, not a result. */ +const FUTURE_SLACK_MS = 86_400_000; +const EARLIEST = Date.UTC(1900, 0, 1); +const LOINC_SYSTEMS = new Set([ + "http://loinc.org", + "urn:oid:2.16.840.1.113883.6.1", +]); + +export type FhirLabMapping = + | { + kind: "reading"; + reading: MappedFhirLab; + selfRef: string | null; + memberRefs: string[]; + } + | { + kind: "skip"; + reason: FhirLabSkipReason; + selfRef: string | null; + memberRefs: string[]; + /** A panel header's name, for its members. */ + panelName: string | null; + } + | { kind: "fail"; reason: FhirLabFailureReason; selfRef: string | null }; + +type Json = Record; + +function obj(value: unknown): Json | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Json) + : null; +} + +/** A field DSTU2 writes as one object and R4 as a list, read as a list. */ +function list(value: unknown): Json[] { + if (Array.isArray(value)) { + return value.map(obj).filter((v): v is Json => v !== null); + } + const one = obj(value); + return one ? [one] : []; +} + +function str(value: unknown): string | null { + if (typeof value !== "string") return null; + const collapsed = value.replace(/\s+/gu, " ").trim(); + return collapsed === "" ? null : collapsed; +} + +function num(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function codings(concept: unknown): Json[] { + return list(obj(concept)?.coding); +} + +/** `Observation/` for a reference however it is written. */ +export function normaliseObservationRef(raw: unknown): string | null { + const reference = str(raw); + if (!reference) return null; + const parts = reference.split(/[/?#]/u).filter(Boolean); + const at = parts.lastIndexOf("Observation"); + if (at < 0 || at + 1 >= parts.length) return null; + return `Observation/${parts[at + 1]}`; +} + +function memberRefs(resource: Json): string[] { + const refs: string[] = []; + for (const member of list(resource.hasMember)) { + const ref = normaliseObservationRef(member.reference); + if (ref) refs.push(ref); + } + for (const related of list(resource.related)) { + if (str(related.type) !== "has-member") continue; + const ref = normaliseObservationRef(obj(related.target)?.reference); + if (ref) refs.push(ref); + } + return refs; +} + +function isLaboratory(concept: Json): boolean { + if (str(concept.text)?.toLowerCase() === "laboratory") return true; + return codings(concept).some( + (c) => str(c.code)?.toLowerCase() === "laboratory", + ); +} + +function analyteName(code: Json | null): string | null { + if (!code) return null; + const text = str(code.text); + if (text) return text; + const all = codings(code); + const loinc = all.find((c) => LOINC_SYSTEMS.has(str(c.system) ?? "")); + return ( + str(loinc?.display) ?? all.map((c) => str(c.display)).find(Boolean) ?? null + ); +} + +function loincCode(code: Json | null): string | null { + if (!code) return null; + for (const c of codings(code)) { + if (LOINC_SYSTEMS.has(str(c.system) ?? "")) { + const value = str(c.code); + if (value) return value; + } + } + return null; +} + +const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/u; +const LOCAL_DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})T[\d:.]+$/u; +const WITH_OFFSET = /^\d{4}-\d{2}-\d{2}T[\d:.]+(Z|[+-]\d{2}:?\d{2})$/u; + +/** + * A FHIR date or dateTime as an instant. A date alone, or a time with no + * offset, is the day at the noon-UTC anchor the labs feature stores a date + * at; a year or a year and month alone does not say which day and is + * unusable. + */ +export function fhirInstant(raw: unknown): Date | null { + const text = str(raw); + if (!text) return null; + const day = DATE_ONLY.exec(text) ?? LOCAL_DATE_TIME.exec(text); + if (day) { + const [y, m, d] = [Number(day[1]), Number(day[2]), Number(day[3])]; + const at = new Date(Date.UTC(y, m - 1, d, 12)); + return at.getUTCFullYear() === y && + at.getUTCMonth() === m - 1 && + at.getUTCDate() === d + ? at + : null; + } + if (!WITH_OFFSET.test(text)) return null; + const ms = Date.parse(text); + return Number.isFinite(ms) ? new Date(ms) : null; +} + +function takenAtOf(resource: Json): Date | null { + return ( + fhirInstant(resource.effectiveDateTime) ?? + fhirInstant(obj(resource.effectivePeriod)?.start) ?? + fhirInstant(resource.effectiveInstant) ?? + fhirInstant(resource.issued) + ); +} + +const INTERPRETATION_TEXT: Readonly> = { + POS: "Positive", + NEG: "Negative", + DET: "Detected", + ND: "Not detected", + IND: "Indeterminate", +}; + +function interpretationText(resource: Json): string | null { + for (const concept of list(resource.interpretation)) { + for (const c of codings(concept)) { + const code = str(c.code)?.toUpperCase(); + if (code && INTERPRETATION_TEXT[code]) { + return str(c.display) ?? str(concept.text) ?? INTERPRETATION_TEXT[code]; + } + } + } + return null; +} + +const UNSUPPORTED_VALUES = [ + "valueBoolean", + "valueRange", + "valueRatio", + "valueSampledData", + "valueAttachment", + "valueTime", + "valuePeriod", + "valueDateTime", +]; + +type ValueRead = + | { + ok: true; + value: number | null; + valueText: string | null; + unit: string | null; + } + | { ok: false; skip?: FhirLabSkipReason; fail?: FhirLabFailureReason } + | null; + +function readValue(resource: Json): ValueRead { + const quantity = obj(resource.valueQuantity); + if (quantity) { + const value = num(quantity.value); + if (value === null) return { ok: false, skip: "no_value" }; + const unit = labUnitFromFhirQuantity(quantity); + const comparator = str(quantity.comparator); + if (comparator) { + const printed = + unit && unit !== "1" + ? `${comparator}${value} ${unit}` + : `${comparator}${value}`; + return { ok: true, value: null, valueText: printed, unit }; + } + if (!unit) return { ok: false, fail: "missing_unit" }; + return { ok: true, value, valueText: null, unit }; + } + if (Number.isSafeInteger(resource.valueInteger)) { + return { ok: false, fail: "missing_unit" }; + } + const text = str(resource.valueString); + if (text) return { ok: true, value: null, valueText: text, unit: null }; + const concept = obj(resource.valueCodeableConcept); + if (concept) { + const first = codings(concept)[0]; + const label = str(concept.text) ?? str(first?.display) ?? str(first?.code); + if (label) return { ok: true, value: null, valueText: label, unit: null }; + return { ok: false, skip: "no_value" }; + } + if (UNSUPPORTED_VALUES.some((key) => resource[key] !== undefined)) { + return { ok: false, skip: "unsupported_value" }; + } + return null; +} + +function referenceRangeOf( + resource: Json, + unit: string | null, +): { low: number | null; high: number | null; text: string | null } { + const ranges = list(resource.referenceRange); + const chosen = + ranges.find((r) => r.appliesTo === undefined && r.age === undefined) ?? + ranges.find((r) => + codings(r.type).some((c) => str(c.code)?.toLowerCase() === "normal"), + ); + if (!chosen) return { low: null, high: null, text: null }; + + const bound = (q: unknown): number | null => { + const quantity = obj(q); + if (!quantity) return null; + const value = num(quantity.value); + if (value === null) return null; + const boundUnit = labUnitFromFhirQuantity(quantity); + if (boundUnit && unit && !sameLabUnit(boundUnit, unit)) return null; + if (boundUnit && !unit) return null; + return value; + }; + let low = bound(chosen.low); + let high = bound(chosen.high); + const text = str(chosen.text)?.slice(0, SOURCE_REFERENCE_TEXT_MAX) ?? null; + if (low === null && high === null && text) { + const parsed = parseReferenceRange(text, unit); + low = parsed?.low ?? null; + high = parsed?.high ?? null; + } + if (low !== null && high !== null && low > high) { + low = null; + high = null; + } + return { low, high, text }; +} + +function noteOf(resource: Json): string | null { + const parts = list(resource.note) + .map((n) => str(n.text)) + .filter((t): t is string => t !== null); + const comments = str(resource.comments); + if (comments) parts.push(comments); + if (parts.length === 0) return null; + const joined = parts.join("\n"); + return joined.length > LAB_TEXT_LIMITS.note + ? `${joined.slice(0, LAB_TEXT_LIMITS.note - 1)}…` + : joined; +} + +const FINAL = new Set(["final", "amended", "corrected", "missing"]); +const NOT_YET = new Set(["preliminary", "registered", "unknown"]); +const WITHDRAWN = new Set(["entered-in-error", "cancelled"]); + +export function mapFhirLabRecord( + input: FhirLabRecordInput, + now: Date = new Date(), +): FhirLabMapping { + const resource = obj(input.resource); + const id = resource ? str(resource.id) : null; + const selfRef = id ? `Observation/${id}` : null; + const skip = ( + reason: FhirLabSkipReason, + members: string[] = [], + panelName: string | null = null, + ): FhirLabMapping => ({ + kind: "skip", + reason, + selfRef, + memberRefs: members, + panelName, + }); + const fail = (reason: FhirLabFailureReason): FhirLabMapping => ({ + kind: "fail", + reason, + selfRef, + }); + + if (!resource || resource.resourceType !== "Observation") { + return skip("not_observation"); + } + + const status = str(resource.status)?.toLowerCase() ?? ""; + if (WITHDRAWN.has(status)) return skip("withdrawn"); + if (NOT_YET.has(status) || !FINAL.has(status)) return skip("not_final"); + + const categories = list(resource.category); + if (categories.length > 0 && !categories.some(isLaboratory)) { + return skip("not_lab"); + } + + const code = obj(resource.code); + const analyte = analyteName(code); + const members = memberRefs(resource); + if (analyte && analyte.length > LAB_TEXT_LIMITS.analyte) { + return fail("text_too_long"); + } + + let read = readValue(resource); + if (read === null && resource.dataAbsentReason !== undefined) { + read = { ok: false, skip: "no_value" }; + } + if (read === null) { + const interpreted = interpretationText(resource); + if (interpreted) { + read = { ok: true, value: null, valueText: interpreted, unit: null }; + } else if (members.length > 0) { + return skip( + "panel_header", + members, + analyte ? analyte.slice(0, LAB_TEXT_LIMITS.panel) : null, + ); + } else if (resource.component !== undefined) { + return skip("unsupported_value"); + } else { + return skip("no_value"); + } + } + if (!read.ok) { + return read.fail ? fail(read.fail) : skip(read.skip ?? "no_value", members); + } + if (!analyte) return skip("no_value", members); + if ( + read.valueText !== null && + read.valueText.length > LAB_TEXT_LIMITS.valueText + ) { + return fail("text_too_long"); + } + if (read.unit !== null && read.unit.length > LAB_TEXT_LIMITS.unit) { + return fail("text_too_long"); + } + + const takenAt = takenAtOf(resource); + if (!takenAt) return skip("no_date", members); + if (takenAt.getTime() > now.getTime() + FUTURE_SLACK_MS) { + return skip("future_date", members); + } + if (takenAt.getTime() < EARLIEST) return skip("invalid_date", members); + + const loinc = loincCode(code); + const range = referenceRangeOf(resource, read.unit); + + return { + kind: "reading", + selfRef, + memberRefs: members, + reading: { + externalId: fhirLabExternalId({ + sourceURL: input.sourceURL, + sourceName: input.sourceName, + identifier: input.identifier, + resourceId: id, + loinc, + analyte, + takenAt, + value: read.value, + valueText: read.valueText, + }), + analyte, + loinc, + catalogKey: loinc ? catalogSlugForLoinc(loinc) : null, + value: read.value, + valueText: read.valueText, + unit: read.unit, + takenAt, + // Like the CSV import: on a text result the window only seeds a new + // marker; the reading itself records none. + sourceReferenceLow: range.low, + sourceReferenceHigh: range.high, + sourceReferenceText: read.valueText === null ? range.text : null, + panel: null, + note: noteOf(resource), + }, + }; +} + +/** + * Give each reading the name of the panel that lists it as a member (R4 + * `hasMember`, DSTU2 `related` of type `has-member`). Mutates the readings in + * place; a reading already carrying a panel keeps it. + */ +export function assignPanels(results: readonly FhirLabMapping[]): void { + const panelOf = new Map(); + for (const result of results) { + if (result.kind !== "skip" || result.reason !== "panel_header") continue; + if (!result.panelName) continue; + for (const ref of result.memberRefs) { + if (!panelOf.has(ref)) panelOf.set(ref, result.panelName); + } + } + if (panelOf.size === 0) return; + for (const result of results) { + if (result.kind !== "reading" || !result.selfRef) continue; + if (result.reading.panel !== null) continue; + result.reading.panel = panelOf.get(result.selfRef) ?? null; + } +} diff --git a/src/lib/labs/fhir-import/types.ts b/src/lib/labs/fhir-import/types.ts new file mode 100644 index 000000000..9d7a57f64 --- /dev/null +++ b/src/lib/labs/fhir-import/types.ts @@ -0,0 +1,96 @@ +/** + * Shared shapes of the FHIR lab import (#1400): lab results from Apple + * Health's Health Records, arriving either inside an `export.zip` + * (`src/lib/apple-health/clinical-labs-import.ts`) or from the iOS app + * (`POST /api/labs/fhir-batch`). Both paths map with `mapFhirLabRecord` and + * write with `writeFhirLabReadings`, so a result reads the same however it + * came in. + * + * Nothing about a resource's content leaves this module family except the + * reading it maps to: reasons are closed enums, counts are numbers. + */ + +/** `LabResult.source` of every reading this import writes. */ +export const APPLE_HEALTH_LAB_SOURCE = "APPLE_HEALTH"; + +/** One clinical record as the export or HealthKit hands it over. */ +export interface FhirLabRecordInput { + /** The parsed FHIR resource. Untrusted: every field is checked. */ + resource: unknown; + /** `HKFHIRResource.fhirVersion` / `fhirVersion` attribute, e.g. "4.0.1". */ + fhirVersion?: string | null; + /** `HKFHIRResource.identifier` / `identifier` attribute. */ + identifier?: string | null; + /** `HKFHIRResource.sourceURL` / `sourceURL` attribute. */ + sourceURL?: string | null; + /** The institution as the Health app names it. */ + sourceName?: string | null; +} + +/** A lab reading a FHIR Observation maps to, before any marker is chosen. */ +export interface MappedFhirLab { + externalId: string; + analyte: string; + /** The LOINC code the resource carries, when it carries one. */ + loinc: string | null; + /** The catalog slug the LOINC code names, used as a matching hint. */ + catalogKey: string | null; + value: number | null; + valueText: string | null; + /** Unit in the spelling the labs feature uses; null for a text result. */ + unit: string | null; + takenAt: Date; + sourceReferenceLow: number | null; + sourceReferenceHigh: number | null; + sourceReferenceText: string | null; + panel: string | null; + note: string | null; +} + +export const FHIR_LAB_SKIP_REASONS = [ + "not_observation", + "not_lab", + "not_final", + "withdrawn", + "panel_header", + "no_value", + "unsupported_value", + "no_date", + "future_date", + "invalid_date", + "too_large", +] as const; +export type FhirLabSkipReason = (typeof FHIR_LAB_SKIP_REASONS)[number]; + +export const FHIR_LAB_FAILURE_REASONS = [ + "missing_unit", + "unit_mismatch", + "text_too_long", + "write_failed", +] as const; +export type FhirLabFailureReason = (typeof FHIR_LAB_FAILURE_REASONS)[number]; + +/** Bounded outcome counts, the only thing an import reports. */ +export interface FhirLabCounts { + /** Lab result records the source offered. */ + discovered: number; + /** Readings written. */ + imported: number; + /** Readings the record already held (by identity or by value). */ + duplicate: number; + /** Records that are not a reading HealthLog keeps (a panel header, a + * preliminary result, a vital sign). */ + skipped: number; + /** Readings refused (no unit, a unit the marker cannot take) or unreadable. */ + failed: number; +} + +export function emptyFhirLabCounts(): FhirLabCounts { + return { discovered: 0, imported: 0, duplicate: 0, skipped: 0, failed: 0 }; +} + +/** Per-input outcome of a write. */ +export type FhirLabWriteStatus = + | { status: "inserted" } + | { status: "duplicate" } + | { status: "failed"; reason: FhirLabFailureReason }; diff --git a/src/lib/labs/fhir-import/units.ts b/src/lib/labs/fhir-import/units.ts new file mode 100644 index 000000000..0ad2e474f --- /dev/null +++ b/src/lib/labs/fhir-import/units.ts @@ -0,0 +1,56 @@ +/** + * A FHIR `Quantity` unit in the spelling the labs feature uses. + * + * FHIR systems write UCUM codes (`10*3/uL`, `m[IU]/L`, `mL/min/{1.73_m2}`); + * the labs catalog, the unit normaliser and the conversion registry speak the + * printed spellings a report uses (`10^3/µL`, `mIU/L`, `mL/min/1.73m²`). The + * table below covers the UCUM forms lab results actually carry that the + * normaliser would not read on its own; anything else passes through as + * written, which is already the printed form for `mg/dL`, `mmol/L`, `g/L`, + * `U/L`, `%` and the like. + */ + +const UCUM_TO_LAB: ReadonlyMap = new Map([ + ["10*3/uL", "10^3/µL"], + ["10*3/ul", "10^3/µL"], + ["10*6/uL", "10^6/µL"], + ["10*6/ul", "10^6/µL"], + ["10*9/L", "10^9/L"], + ["10*9/l", "10^9/L"], + ["10*12/L", "10^12/L"], + ["10*12/l", "10^12/L"], + ["[IU]/L", "IU/L"], + ["m[IU]/L", "mIU/L"], + ["m[IU]/mL", "mIU/mL"], + ["u[IU]/mL", "µIU/mL"], + ["meq/L", "mEq/L"], + ["/uL", "/µL"], + ["mL/min/{1.73_m2}", "mL/min/1.73m²"], + ["mL/min/1.73.m2", "mL/min/1.73m²"], +]); + +/** A UCUM annotation alone (`{INR}`, `{ratio}`) is the unity. */ +const ANNOTATION_ONLY = /^\{[^{}]*\}$/u; + +/** + * The lab spelling of a FHIR quantity's unit: a UCUM form the table knows + * (the `code` first, then the human `unit`), else the human `unit` as written, + * else the `code`. Null when neither says anything. The unity (`1`, or an + * annotation alone such as `{INR}`) becomes "1". + */ +export function labUnitFromFhirQuantity(quantity: { + unit?: unknown; + code?: unknown; +}): string | null { + const text = (raw: unknown): string | null => + typeof raw === "string" && raw.trim() !== "" ? raw.trim() : null; + const code = text(quantity.code); + const unit = text(quantity.unit); + for (const candidate of [code, unit]) { + if (candidate === null) continue; + const mapped = UCUM_TO_LAB.get(candidate); + if (mapped) return mapped; + if (candidate === "1" || ANNOTATION_ONLY.test(candidate)) return "1"; + } + return unit ?? code; +} diff --git a/src/lib/labs/fhir-import/write.ts b/src/lib/labs/fhir-import/write.ts new file mode 100644 index 000000000..5f3151569 --- /dev/null +++ b/src/lib/labs/fhir-import/write.ts @@ -0,0 +1,337 @@ +/** + * Write mapped FHIR lab readings (#1400), shared by the Apple Health export + * job and `POST /api/labs/fhir-batch`. + * + * A reading becomes a `LabResult` the way a CSV row does + * (`src/lib/import/labs-csv.ts`): the name resolves to the account's marker + * through the shared index (with the LOINC code's catalog analyte as a hint), + * a marker is minted for a new analyte, a reading in another unit is + * converted when the conversion is known and kept beside what the report + * said, and a unit that does not convert is refused. + * + * Two duplicate checks, in this order: + * 1. the identity (`externalId`) under `source = APPLE_HEALTH`, against + * every row including the deleted ones: a result the person deleted + * stays deleted however often the export or the phone sends it again; + * 2. the reading itself (marker or name, day, value) against the live + * record, so a result already typed, scanned or imported from a CSV is + * not written a second time. + * + * First write wins: an amended or corrected result arriving under an + * identity already held is a duplicate, not an update. + * + * Writes go in chunks of `CHUNK` through `createManyAndReturn` with + * `skipDuplicates`, so a concurrent writer of the same identity loses + * quietly and is reported as a duplicate. A chunk that fails to write is + * reported as `write_failed` and the next chunk still runs. + */ +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { prisma as defaultPrisma } from "@/lib/db"; +import { matchAnalyteKey } from "@/lib/labs/analyte-match"; +import { + loadMarkerIndex, + resolveInIndex, + unitFitsMarker, + type MarkerIndex, +} from "@/lib/labs/biomarker-resolve"; +import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store"; +import { readingInMarkerUnit } from "@/lib/labs/reading-conversion"; +import { + labReadingDay, + labReadingDaySearchRange, +} from "@/lib/labs/reading-day"; +import { readingKeys } from "@/lib/labs/reading-keys"; +import { encryptNoteToBytes } from "@/lib/labs/store"; + +import { + APPLE_HEALTH_LAB_SOURCE, + type FhirLabWriteStatus, + type MappedFhirLab, +} from "./types"; + +const CHUNK = 200; + +type LabWriteClient = Pick< + PrismaClient, + "labResult" | "biomarker" | "biomarkerAlias" +>; + +export interface FhirLabWriteResult { + /** One per input reading, in input order. */ + statuses: FhirLabWriteStatus[]; + inserted: number; + duplicate: number; + failed: number; + /** The newest instant among the readings written, or null. */ + newestTakenAt: Date | null; +} + +interface Pending { + at: number; + reading: MappedFhirLab; + markerId: string | null; + keys: string[]; +} + +export async function writeFhirLabReadings(args: { + userId: string; + tz: string; + readings: readonly MappedFhirLab[]; + prisma?: LabWriteClient; +}): Promise { + const client = args.prisma ?? defaultPrisma; + const { userId, tz, readings } = args; + const statuses: FhirLabWriteStatus[] = new Array(readings.length); + const result: FhirLabWriteResult = { + statuses, + inserted: 0, + duplicate: 0, + failed: 0, + newestTakenAt: null, + }; + if (readings.length === 0) return result; + + const index: MarkerIndex = await loadMarkerIndex(client, userId); + const seenIds = new Set(); + const seenKeys = new Set(); + + const duplicate = (at: number) => { + statuses[at] = { status: "duplicate" }; + result.duplicate += 1; + }; + const failed = (at: number, reason: "unit_mismatch" | "write_failed") => { + statuses[at] = { status: "failed", reason }; + result.failed += 1; + }; + + for (let start = 0; start < readings.length; start += CHUNK) { + const slice = readings.slice(start, start + CHUNK); + + // 1. Identity: in the batch, then against every stored row. + const fresh: Array<{ at: number; reading: MappedFhirLab }> = []; + for (let i = 0; i < slice.length; i += 1) { + const at = start + i; + const reading = slice[i]; + if (seenIds.has(reading.externalId)) { + duplicate(at); + continue; + } + seenIds.add(reading.externalId); + fresh.push({ at, reading }); + } + const held = new Set( + ( + await client.labResult.findMany({ + where: { + userId, + source: APPLE_HEALTH_LAB_SOURCE, + externalId: { in: fresh.map((f) => f.reading.externalId) }, + }, + select: { externalId: true }, + }) + ).map((row) => row.externalId), + ); + + // 2. The marker, and whether the unit fits it. + const planned: Pending[] = []; + for (const { at, reading } of fresh) { + if (held.has(reading.externalId)) { + duplicate(at); + continue; + } + const hit = resolveInIndex( + index, + reading.analyte, + reading.valueText === null ? reading.unit : null, + { catalogKey: reading.catalogKey }, + ); + if ( + hit && + reading.value !== null && + !unitFitsMarker( + { + unit: hit.marker.unit, + analyteKey: + matchAnalyteKey({ + name: hit.marker.name, + unit: hit.marker.unit, + }) ?? hit.marker.analyteKey, + }, + reading.unit, + reading.catalogKey, + ) + ) { + failed(at, "unit_mismatch"); + continue; + } + planned.push({ + at, + reading, + markerId: hit?.marker.id ?? null, + keys: [], + }); + } + + // 3. The reading itself, against the live record. + const existingKeys = new Set(); + if (planned.length > 0) { + const days = planned + .map((p) => labReadingDay(p.reading.takenAt, tz)) + .sort(); + const ids = [ + ...new Set( + planned.map((p) => p.markerId).filter((id): id is string => !!id), + ), + ]; + const names = [...new Set(planned.map((p) => p.reading.analyte))]; + const existing = await client.labResult.findMany({ + where: { + userId, + deletedAt: null, + takenAt: { + gte: labReadingDaySearchRange(days[0]).gte, + lt: labReadingDaySearchRange(days[days.length - 1]).lt, + }, + OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }], + }, + select: { + biomarkerId: true, + analyte: true, + takenAt: true, + value: true, + valueText: true, + sourceValue: true, + sourceUnit: true, + }, + }); + for (const e of existing) { + for (const k of readingKeys(e, tz)) existingKeys.add(k); + if (e.biomarkerId) { + for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) { + existingKeys.add(k); + } + } + } + } + + // 4. Build the rows, field by field. + const rows: Prisma.LabResultCreateManyInput[] = []; + const rowAt = new Map(); + for (const plan of planned) { + const { reading } = plan; + const probe = { + biomarkerId: plan.markerId, + analyte: reading.analyte, + takenAt: reading.takenAt, + value: reading.value, + valueText: reading.valueText, + sourceValue: reading.value, + sourceUnit: reading.unit, + }; + const keys = [ + ...readingKeys(probe, tz), + ...readingKeys({ ...probe, biomarkerId: null }, tz), + ]; + if (keys.some((k) => existingKeys.has(k) || seenKeys.has(k))) { + duplicate(plan.at); + continue; + } + + const marker = await resolveOrMintBiomarker( + userId, + { + analyte: reading.analyte, + unit: reading.unit ?? "", + referenceLow: reading.sourceReferenceLow, + referenceHigh: reading.sourceReferenceHigh, + panel: reading.panel, + }, + { index, catalogKey: reading.catalogKey }, + ); + + let storedValue = reading.value; + let storedLow = + reading.valueText === null ? reading.sourceReferenceLow : null; + let storedHigh = + reading.valueText === null ? reading.sourceReferenceHigh : null; + let convertedFrom: { sourceValue: number; sourceUnit: string } | null = + null; + if (reading.value !== null) { + const converted = await readingInMarkerUnit(client, { + marker, + value: reading.value, + unit: reading.unit ?? undefined, + sourceReferenceLow: storedLow, + sourceReferenceHigh: storedHigh, + }); + if (!converted.ok) { + failed(plan.at, "unit_mismatch"); + continue; + } + storedValue = converted.value; + storedLow = converted.sourceReferenceLow; + storedHigh = converted.sourceReferenceHigh; + if (converted.converted) { + convertedFrom = { + sourceValue: converted.sourceValue, + sourceUnit: converted.sourceUnit, + }; + } + } + + rows.push({ + userId, + biomarkerId: marker.id, + panel: reading.panel ?? marker.panel, + analyte: marker.name, + value: storedValue, + valueText: reading.valueText, + unit: marker.unit, + sourceValue: convertedFrom?.sourceValue ?? null, + sourceUnit: convertedFrom?.sourceUnit ?? null, + referenceLow: marker.lowerBound, + referenceHigh: marker.upperBound, + sourceReferenceLow: storedLow, + sourceReferenceHigh: storedHigh, + sourceReferenceText: reading.sourceReferenceText, + takenAt: reading.takenAt, + source: APPLE_HEALTH_LAB_SOURCE, + externalId: reading.externalId, + noteEncrypted: reading.note ? encryptNoteToBytes(reading.note) : null, + }); + rowAt.set(reading.externalId, plan); + for (const k of [ + ...keys, + ...readingKeys({ ...probe, biomarkerId: marker.id }, tz), + ]) { + seenKeys.add(k); + } + } + + if (rows.length === 0) continue; + try { + const written = await client.labResult.createManyAndReturn({ + data: rows, + skipDuplicates: true, + select: { externalId: true }, + }); + const landed = new Set(written.map((w) => w.externalId)); + for (const [externalId, plan] of rowAt) { + if (landed.has(externalId)) { + statuses[plan.at] = { status: "inserted" }; + result.inserted += 1; + const t = plan.reading.takenAt; + if (!result.newestTakenAt || t > result.newestTakenAt) { + result.newestTakenAt = t; + } + } else { + // A concurrent write of the same identity won. + duplicate(plan.at); + } + } + } catch { + for (const plan of rowAt.values()) failed(plan.at, "write_failed"); + } + } + return result; +}