diff --git a/src/components/labs/lab-form.tsx b/src/components/labs/lab-form.tsx index 6403444d8..f22d1efe8 100644 --- a/src/components/labs/lab-form.tsx +++ b/src/components/labs/lab-form.tsx @@ -22,7 +22,10 @@ import { import { Textarea } from "@/components/ui/textarea"; import { apiGet, apiPost } from "@/lib/api/api-fetch"; import { localizedApiError } from "@/lib/api/localized-error"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; import { formatReferenceRange } from "@/lib/labs/reference-range"; import { formatLabValue } from "@/lib/labs/format-value"; import { EncounterSuggestionField } from "@/components/encounters/encounter-suggestion-field"; @@ -300,7 +303,7 @@ export function LabForm({ // the hint says the text is kept as written rather than staying silent. const parsedSourceRange = parseReferenceRange(sourceRange, selected?.unit); const sourceRangeHint = parsedSourceRange - ? parsedSourceRange.low === null && parsedSourceRange.high === null + ? isUnreadableRange(parsedSourceRange) ? t("labs.form.sourceRangeUnparsed") : `${formatReferenceRange( parsedSourceRange.low, diff --git a/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts b/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts new file mode 100644 index 000000000..53fad3039 --- /dev/null +++ b/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "vitest"; + +import { + derivedAnalyteNamePattern, + looksLikeLabDocument, +} from "@/lib/documents/auto-stage-labs"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; + +/** + * The gate decides whether a transcribed document is worth an extraction call. + * A lab report that fails it is lost from auto-staging entirely, and nothing + * says so — the manual extract button stays available, so the flow reads as + * working. + * + * Before this, the four signal classes were English and German only, so a + * report printed anywhere else scored at most one hit (the units, which are + * the same everywhere) and never reached two. + */ +const FRENCH_REPORT = ` +Laboratoire d'analyses médicales +Compte rendu du 14 mars 2026 + +Cholestérol total 5,8 mmol/l Valeurs de référence: jusqu'à 5,0 +Créatinine 88 µmol/l Plage de référence: 62 - 106 +Hémoglobine 14,2 g/dl +`; + +const SPANISH_REPORT = ` +Laboratorio de análisis clínicos +Analítica de sangre + +Colesterol total 210 mg/dl Valores de referencia: hasta 200 +Glucosa en ayunas 92 mg/dl Rango de referencia: 70 - 100 +`; + +const ITALIAN_REPORT = ` +Referto di laboratorio — esami del sangue + +Colesterolo totale 5,4 mmol/l Valori di riferimento: fino a 5,0 +Emoglobina 13,8 g/dl Intervallo di riferimento: 12 - 16 +`; + +const POLISH_REPORT = ` +Laboratorium analiz lekarskich +Wyniki badań + +Cholesterol całkowity 5,2 mmol/l Zakres referencyjny: do 5,0 +Kreatynina 82 µmol/l Wartości referencyjne: 62 - 106 +`; + +describe("looksLikeLabDocument — a report printed outside English and German", () => { + it.each([ + ["French", FRENCH_REPORT], + ["Spanish", SPANISH_REPORT], + ["Italian", ITALIAN_REPORT], + ["Polish", POLISH_REPORT], + ])("recognises a %s lab report", (_language, text) => { + expect(looksLikeLabDocument(text)).toBe(true); + }); + + it("recognises a report on its analyte names and units alone", () => { + // No header, no reference-range label — the two classes left are the ones + // a stripped-down result table still carries. + expect( + looksLikeLabDocument("Triglycérides 1,4 mmol/l\nFerritine 120 ng/ml"), + ).toBe(true); + }); +}); + +/** + * The four reports above each carry several kinds of evidence at once, so any + * one of the additions could carry them alone. These cases isolate one + * addition each: remove the piece each names and the case drops back to a + * single class and fails the gate. + */ +describe("looksLikeLabDocument — each addition carries its own weight", () => { + it("needs the DERIVED analyte names for a panel of markers no stem covers", () => { + // The electrolytes are in the catalog and in no hand-written stem, so the + // derived names are the only analyte evidence here. Units are the second + // class. + expect( + looksLikeLabDocument("Sodium 140 mmol/l\nPotassium 4,2 mmol/l"), + ).toBe(true); + expect(looksLikeLabDocument("Sodio 140 mmol/l\nPotassio 4,2 mmol/l")).toBe( + true, + ); + }); + + it("needs the reference-range labels of the four locales", () => { + // "Płytki krwi" is a derived name; the label is the only other class here. + expect( + looksLikeLabDocument("Płytki krwi 250\nZakres referencyjny: 150 - 400"), + ).toBe(true); + expect( + looksLikeLabDocument("Plaquettes 250\nValeurs de référence: 150 - 400"), + ).toBe(true); + }); + + it("needs the report headers of the four locales", () => { + expect(looksLikeLabDocument("Referto di laboratorio\nPiastrine 250")).toBe( + true, + ); + expect(looksLikeLabDocument("Wyniki badań\nPłytki krwi 250")).toBe(true); + expect(looksLikeLabDocument("Analítica de sangre\nPlaquetas 250")).toBe( + true, + ); + }); +}); + +describe("looksLikeLabDocument — what must still fail the gate", () => { + it.each([ + [ + "English prose with one incidental unit", + "Please take 500 mg twice daily.", + ], + [ + "a French letter mentioning one analyte", + "Cher patient, votre taux de cholestérol sera revu lors du prochain rendez-vous.", + ], + [ + "a Spanish appointment letter", + "Estimado paciente, su cita en el hospital es el martes a las diez.", + ], + [ + "an Italian pharmacy note", + "Ritirare la confezione in farmacia entro venerdì.", + ], + ["an invoice with a currency amount", "Rechnung über 120,00 EUR. Danke."], + ])("rejects %s", (_case, text) => { + expect(looksLikeLabDocument(text)).toBe(false); + }); + + it("does not let one analyte named twice reach two classes", () => { + // The derived names extend the analyte class rather than forming their + // own. If they were a fifth class, a document saying "Cholestérol total" + // and "cholesterol" would score two and pass on one piece of evidence. + expect(looksLikeLabDocument("Cholestérol total et cholesterol")).toBe( + false, + ); + }); + + it("ignores a derived analyte name embedded in a longer word", () => { + // The hand-written half of the analyte class matches STEMS on purpose + // ("ferritin" catches "Ferritinwert"); the derived half is bounded, + // because a whole catalog name inside another word is noise. + expect(looksLikeLabDocument("Xhematocritx 5 mg/dl")).toBe(false); + }); +}); + +describe("looksLikeLabDocument — the English and German behaviour is unchanged", () => { + it("still recognises a German report", () => { + expect( + looksLikeLabDocument( + "Laborbefund. Hämoglobin 14.2 g/dl. Referenzbereich 13-17. LDL 120 mg/dl.", + ), + ).toBe(true); + }); + + it("still recognises an English report", () => { + expect( + looksLikeLabDocument( + "Laboratory report. Hemoglobin 14.2 g/dL. Reference range 13-17.", + ), + ).toBe(true); + }); + + it("still rejects the prose the previous guard rejected", () => { + expect( + looksLikeLabDocument("Dear patient, please take 500 mg twice daily."), + ).toBe(false); + }); +}); + +describe("the derived analyte-name pattern", () => { + it("carries a name from every shipped locale", () => { + // The property that matters. A transcribed list would satisfy the report + // cases above and still be frozen at the languages somebody typed. + for (const locale of locales) { + const name = resolveKey( + allMessages[locale], + "labs.catalog.total-cholesterol", + ); + expect(name, `${locale} has no catalog name`).toBeTruthy(); + expect( + derivedAnalyteNamePattern().test( + stripDiacritics((name as string).toLowerCase()), + ), + `${locale}: ${name}`, + ).toBe(true); + } + }); + + it("excludes the short acronyms that are ordinary words elsewhere", () => { + // "ALT" is German for "old". A three-character alternative in a gate this + // cheap would make every German letter a lab report. The acronyms worth + // having live in the hand-written class, each considered on its own. + const pattern = derivedAnalyteNamePattern(); + for (const acronym of ["alt", "ast", "ggt", "sod"]) { + expect(pattern.test(acronym), acronym).toBe(false); + } + }); + + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(derivedAnalyteNamePattern().test("triglycerides")).toBe(true); + // The Polish "ł" is a single codepoint NFD does not decompose, so it + // survives the fold on both sides and has to be written as the bundle + // writes it. A document that spelled it "plytki" would not match, which is + // the stated limit of folding rather than transliterating. + expect(derivedAnalyteNamePattern().test("płytki krwi")).toBe(true); + }); +}); diff --git a/src/lib/documents/auto-stage-labs.ts b/src/lib/documents/auto-stage-labs.ts index 1c815b7d7..52ccaca2c 100644 --- a/src/lib/documents/auto-stage-labs.ts +++ b/src/lib/documents/auto-stage-labs.ts @@ -42,28 +42,139 @@ import { } from "@/lib/documents/extract"; import { encryptFactData, encryptFactProvenance } from "@/lib/documents/store"; import { prisma } from "@/lib/db"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; +import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog"; import { annotate } from "@/lib/logging/context"; import { isModuleEnabled } from "@/lib/modules/gate"; /** * Lab-report signals used to decide whether a transcribed document is worth an - * extraction call. Each pattern is one class of evidence (a reference-range - * label, a lab unit, a common analyte name, a report header); requiring TWO - * distinct hits keeps a stray "mg" in a prose letter from tripping the auto - * extraction while still catching a real panel in German or English. + * extraction call. Each entry is one CLASS of evidence — a reference-range + * label, a lab unit, a common analyte name, a report header — and contributes + * at most one hit however many of its alternatives fire. Requiring TWO + * distinct classes keeps a stray "mg" in a prose letter from tripping the auto + * extraction while still catching a real panel. + * + * Every pattern is tested against the text LOWER-CASED AND ACCENT-FOLDED, not + * against the transcription as written, so the patterns are spelled without + * accents. That is also what lets one pattern serve "Hämoglobin", "haemoglobin" + * and "hemoglobin" instead of three. + * + * Until now the four classes were English and German only, so a French, + * Spanish, Italian or Polish lab report scored at most one hit — the units, + * which are the same everywhere — and auto-staging never fired for it. The + * whole document was lost from the flow, silently: the manual extract button + * stays available, so nothing anywhere reads as broken. + * + * WHICH WAY THIS ERRS. A false negative loses a real report from auto-staging; + * a false positive burns one extraction call on a document that is not a lab + * report, and stages facts a person then declines on the review screen — + * nothing commits either way. The cost is lopsided, so the vocabulary is + * widened rather than kept tight, and the two-distinct-class rule is what + * holds the other side: an incidental "sodium" in a prose letter is one class + * and still fails the gate. Terms whose only home is a lab report ("emocromo", + * "bilan sanguin", "zakres referencyjny") were preferred over generic ones, + * and short acronyms below four characters are excluded from the derived names + * for exactly this reason — the German word "alt" would otherwise make every + * German letter a lab report. */ +/** + * The analyte class, hand-written half: word stems and the short acronyms, + * which are too short to derive safely but were vetted one at a time long ago. + * Named rather than inlined so the derived names can be attached to THIS class + * by identity — an index would silently re-point them at another class the + * first time somebody reorders the list. + */ +const ANALYTE_SIGNAL = + /h(?:a|ae|e)moglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i; + const LAB_SIGNALS: readonly RegExp[] = [ - /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich/i, + // 1 — a reference-range label. Only a report that prints windows says this. + /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich|valeurs? de reference|intervalle de reference|plage de reference|valores? de referencia|rango de referencia|intervalo de referencia|valori di riferimento|intervallo di riferimento|zakres referencyjny|wartosci referencyjne/i, + // 2 — a lab unit. Language-independent already; unchanged. /\b(mg\/dl|mmol\/l|nmol\/l|µmol\/l|umol\/l|g\/dl|µg\/l|ug\/l|ng\/ml|pg\/ml|mmol\/mol|u\/l|iu\/l|mg\/l|\/µl|\/ul)\b/i, - /h(?:ä|ae)moglobin|hemoglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i, - /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b/i, + // 3 — an analyte name. + ANALYTE_SIGNAL, + // 4 — a report header. + /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b|laboratoire|bilan sanguin|analyses medicales|hemogramme|laboratorio|analitica|hemograma|analisis de sangre|esami del sangue|emocromo|referto|laboratorium|morfologia krwi|wyniki badan/i, ]; -/** True when the transcribed text carries at least two lab-report signals. */ +/** + * Class 3 again, DERIVED from `messages/.json` instead of transcribed. + * + * The names above are stems a maintainer typed. These are the names the app + * itself displays for the markers in its catalog, in every language it ships — + * so a French report's "Cholestérol total" or a Polish "Płytki krwi" counts as + * an analyte the day the bundle carries it, with no edit here. Transcribing + * thirty names times four more languages would have fixed today's six locales + * and reopened the hole at the seventh. + * + * Names shorter than four folded characters are dropped: "ALT", "AST", "GGT", + * "TSH" and Polish "Sód" are ordinary words or fragments in some language, and + * a three-letter alternative in a gate this cheap is a false-positive machine. + * The acronyms worth having are already in the hand-written class above, where + * each was considered one at a time. + */ +function buildAnalyteNamePattern(): RegExp { + const names = new Set(); + for (const seed of BIOMARKER_CATALOG) { + for (const name of localisedValues(`labs.catalog.${seed.slug}`)) { + const folded = stripDiacritics(name.toLowerCase()).trim(); + // Also the name without its parenthetical gloss. A report prints + // "Apolipoprotéine B", not "Apolipoprotéine B (ApoB)", and the gloss is + // the app's way of naming a marker two ways at once. + const bare = folded.replace(/\s*\([^)]*\)/gu, "").trim(); + for (const candidate of [folded, bare]) { + if (candidate.length >= 4) names.add(candidate); + } + } + } + const alternation = [...names] + .sort((a, b) => b.length - a.length) + .map((name) => + name + .split(/\s+/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + // Any run of whitespace, so a name broken across a transcribed line + // still reads as one name. + .join(String.raw`\s+`), + ) + .join("|"); + // Bounded by "not a letter or digit" rather than by \b, because several + // names end in punctuation ("Lipoprotéine(a)") where \b cannot match, and + // because the Polish "ł" is not a word character to \b. + return new RegExp( + `(?= 2) return true; + const hit = + signal.test(folded) || + (signal === ANALYTE_SIGNAL && DERIVED_ANALYTE_NAMES.test(folded)); + if (hit && ++hits >= 2) return true; } return false; } diff --git a/src/lib/documents/commit.ts b/src/lib/documents/commit.ts index b960b39b9..0cecca25a 100644 --- a/src/lib/documents/commit.ts +++ b/src/lib/documents/commit.ts @@ -28,7 +28,11 @@ import { invalidateUserHealthScore } from "@/lib/cache/invalidate"; import { emitDataArrival } from "@/lib/arrivals/emit-shared"; import { decryptFactData } from "@/lib/documents/store"; import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; +import { annotate } from "@/lib/logging/context"; import type { ExtractedFact } from "@/generated/prisma/client"; import type { ConditionFactData, @@ -82,6 +86,19 @@ async function commitObservation( const sourceLow = printed?.low ?? data.referenceLow; const sourceHigh = printed?.high ?? data.referenceHigh; const sourceText = printed?.text ?? null; + // A window the report printed and the parser could not read is the case that + // used to pass without a trace: the reading is then judged against the + // catalog band, or against nothing, and no surface says the lab had stated + // something else. Count it here so an unreadable notation — a language the + // prose table does not carry, a layout nobody anticipated — shows up as a + // number on a dashboard instead of as silence. The string itself is the + // user's document and stays out of the event. + if (isUnreadableRange(printed)) { + annotate({ + action: { name: "labs.referenceRange.unreadable" }, + meta: { surface: "document.commit", length: printed?.text.length ?? 0 }, + }); + } const biomarker = await resolveOrMintBiomarker(userId, { analyte: data.label, diff --git a/src/lib/fhir/__tests__/lab-loinc-locales.test.ts b/src/lib/fhir/__tests__/lab-loinc-locales.test.ts new file mode 100644 index 000000000..2e4363e45 --- /dev/null +++ b/src/lib/fhir/__tests__/lab-loinc-locales.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import { + CATALOG_SLUG_TO_LOINC_KEY, + localisedLabAliases, + normaliseLabKey, + resolveLabCoding, +} from "@/lib/fhir/lab-loinc"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; +import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog"; + +/** + * A biomarker minted from the labs catalog stores `t("labs.catalog.")` + * — the name in the language of whoever minted it — and that stored name is + * what reaches the FHIR exporter as free text. The alias table it was matched + * against was written in English and German, so four of the six shipped + * locales exported every one of these analytes as a text-only concept with no + * LOINC code. + */ +describe("resolveLabCoding — the analyte name in every shipped locale", () => { + it.each([ + ["Cholestérol total", "2093-3"], + ["Colesterol total", "2093-3"], + ["Colesterolo totale", "2093-3"], + ["Cholesterol całkowity", "2093-3"], + ["Créatinine", "2160-0"], + ["Creatinina", "2160-0"], + ["Kreatynina", "2160-0"], + ["Glycémie à jeun", "1558-6"], + ["Glucosa en ayunas", "1558-6"], + ["Glicemia a digiuno", "1558-6"], + ["Glukoza na czczo", "1558-6"], + ["Ferritine", "2276-4"], + ["Ferrytyna", "2276-4"], + ["Hémoglobine", "718-7"], + ["Emoglobina", "718-7"], + ["Cholestérol LDL", "18262-6"], + ["Colesterol HDL", "2085-9"], + ["Triglycérides", "2571-8"], + ["Trójglicerydy", "2571-8"], + ["Insuline à jeun", "27873-9"], + ["Lipoprotéine(a)", "43583-4"], + ])("codes %s as LOINC %s", (analyte, loinc) => { + expect(resolveLabCoding(analyte, "")?.loinc).toBe(loinc); + }); + + it("derives from the bundles rather than from a transcribed list", () => { + // The property that matters: every locale contributes. A hand-written + // table would pass the cases above and still be frozen at the languages + // somebody happened to type. + for (const locale of locales) { + const name = resolveKey( + allMessages[locale], + "labs.catalog.total-cholesterol", + ); + expect(name, `${locale} has no catalog name`).toBeTruthy(); + expect( + resolveLabCoding(name as string, "")?.loinc, + `${locale}: ${name}`, + ).toBe("2093-3"); + } + }); + + it("stamps the canonical UCUM when the recorded unit matches", () => { + expect(resolveLabCoding("Cholestérol total", "mg/dL")).toEqual({ + loinc: "2093-3", + display: "Cholesterol [Mass/volume] in Serum or Plasma", + ucum: "mg/dL", + }); + }); + + it("still keeps an unmapped analyte text-only", () => { + // Conservative by design: a name the app does not ship and no alias names + // gets no fabricated code, in any language. + expect(resolveLabCoding("Analyse inconnue", "mg/dL")).toBeNull(); + }); +}); + +describe("resolveLabCoding — the existing English and German behaviour", () => { + it.each([ + ["HbA1c", "4548-4"], + ["LDL-C", "18262-6"], + ["LDL Cholesterol", "18262-6"], + ["Gesamtcholesterin", "2093-3"], + ["Kreatinin", "2160-0"], + ["GPT", "1742-6"], + ["ASAT", "1920-8"], + ["hgb", "718-7"], + ["Nüchternglucose", "1558-6"], + ["Thyrotropin", "3016-3"], + ["Apolipoprotein B", "1884-6"], + ])("still codes %s as LOINC %s", (analyte, loinc) => { + expect(resolveLabCoding(analyte, "")?.loinc).toBe(loinc); + }); + + it("resolves a German name whose umlaut the old key fold destroyed", () => { + // "Hämoglobin" used to fold to `hmoglobin`, which matched nothing — the + // hand-written `hamoglobin` alias only ever caught the un-umlauted typing. + expect(normaliseLabKey("Hämoglobin")).toBe("hamoglobin"); + expect(resolveLabCoding("Hämoglobin", "g/dL")?.loinc).toBe("718-7"); + }); +}); + +describe("the derived alias index", () => { + it("contradicts no hand-written alias", () => { + // A derived name that resolved to a DIFFERENT analyte than a verified + // alias would be a bundle bug re-pointing an existing mapping. The build + // refuses it; this asserts the refusal held rather than that it exists. + for (const [name, key] of Object.entries(localisedLabAliases())) { + const viaHand = resolveLabCoding(name, ""); + expect(viaHand, name).not.toBeNull(); + expect(viaHand?.loinc, name).toBe(resolveLabCoding(key, "")?.loinc); + } + }); + + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(Object.keys(localisedLabAliases()).length).toBeGreaterThan(20); + }); + + it("maps only slugs the labs catalog actually ships, in every locale", () => { + // A renamed slug or a bundle that dropped a name would silently stop + // deriving: the index would shrink and nothing else would say so. + const slugs = new Set(BIOMARKER_CATALOG.map((s) => s.slug)); + for (const slug of Object.keys(CATALOG_SLUG_TO_LOINC_KEY)) { + expect(slugs.has(slug), `catalog lost slug ${slug}`).toBe(true); + for (const locale of locales) { + expect( + resolveKey(allMessages[locale], `labs.catalog.${slug}`), + `${locale} has no name for ${slug}`, + ).toBeTruthy(); + } + } + }); +}); diff --git a/src/lib/fhir/__tests__/lab-qualitative-locales.test.ts b/src/lib/fhir/__tests__/lab-qualitative-locales.test.ts new file mode 100644 index 000000000..7ba526323 --- /dev/null +++ b/src/lib/fhir/__tests__/lab-qualitative-locales.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; + +import { + localisedQualitativeIndex, + qualitativeValueConcept, +} from "@/lib/fhir/lab-qualitative"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; + +const SNOMED = "http://snomed.info/sct"; + +function codeOf(valueText: string): string | undefined { + const concept = qualitativeValueConcept(valueText); + return concept.coding?.[0]?.code; +} + +/** + * The lab form offers "negative" and "positive" as quick picks and writes the + * LABEL it rendered, so the column holds the bundle's own string in the + * language the reading was recorded in. Matched against a transcribed + * English/German list, four of the six shipped locales exported a result the + * receiving system could not read. + */ +describe("qualitativeValueConcept — the quick-pick label in every locale", () => { + it.each([ + ["négatif", "260385009"], + ["negativo", "260385009"], + ["ujemny", "260385009"], + ["positif", "10828004"], + ["positivo", "10828004"], + ["dodatni", "10828004"], + ])("codes %s as SNOMED %s", (valueText, code) => { + expect(codeOf(valueText)).toBe(code); + }); + + it("derives from the bundles rather than from a transcribed list", () => { + for (const locale of locales) { + const negative = resolveKey( + allMessages[locale], + "labs.form.qualNegative", + ); + const positive = resolveKey( + allMessages[locale], + "labs.form.qualPositive", + ); + expect(negative, `${locale} has no negative label`).toBeTruthy(); + expect(positive, `${locale} has no positive label`).toBeTruthy(); + expect(codeOf(negative as string), `${locale}: ${negative}`).toBe( + "260385009", + ); + expect(codeOf(positive as string), `${locale}: ${positive}`).toBe( + "10828004", + ); + } + }); + + it("tolerates a term typed without its accents", () => { + expect(codeOf("Negatif")).toBe("260385009"); + expect(codeOf("NÉGATIF")).toBe("260385009"); + }); + + it("keeps the raw text on the concept whatever the language", () => { + expect(qualitativeValueConcept("négatif")).toEqual({ + coding: [{ system: SNOMED, code: "260385009", display: "Negative" }], + text: "négatif", + }); + }); +}); + +describe("qualitativeValueConcept — what stays text-only, and why", () => { + it("leaves borderline uncoded in every locale", () => { + // The candidate "Borderline" qualifier concept was never confidently + // verified, so `labs.form.qualBorderline` is read by nothing. This pins + // that the derivation did not sweep it up by accident. + for (const locale of locales) { + const label = resolveKey(allMessages[locale], "labs.form.qualBorderline"); + expect(label, `${locale} has no borderline label`).toBeTruthy(); + expect( + qualitativeValueConcept(label as string).coding, + `${locale}: ${label}`, + ).toBeUndefined(); + } + }); + + it("leaves the detected / not-detected pair uncoded outside English and German", () => { + // The honest limit of this change, written down rather than discovered + // later: the app ships no string for these, they are prose a lab printed, + // and inventing the four other languages would be inventing clinical + // vocabulary nobody here can check. + expect(codeOf("nicht nachweisbar")).toBe("260415000"); + expect(codeOf("not detected")).toBe("260415000"); + expect(codeOf("non détecté")).toBeUndefined(); + expect(codeOf("niewykrywalny")).toBeUndefined(); + expect(qualitativeValueConcept("non détecté")).toEqual({ + text: "non détecté", + }); + }); + + it("leaves free prose uncoded", () => { + expect(qualitativeValueConcept("siehe Befund")).toEqual({ + text: "siehe Befund", + }); + }); +}); + +describe("qualitativeValueConcept — the existing English and German terms", () => { + it.each([ + ["negativ", "260385009"], + ["negative", "260385009"], + ["Negativ", "260385009"], + ["neg", "260385009"], + ["positiv", "10828004"], + ["positive", "10828004"], + ["pos", "10828004"], + ["nachweisbar", "260373001"], + ["detected", "260373001"], + ])("still codes %s as SNOMED %s", (valueText, code) => { + expect(codeOf(valueText)).toBe(code); + }); +}); + +describe("the derived qualitative index", () => { + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(Object.keys(localisedQualitativeIndex()).length).toBeGreaterThan(3); + }); + + it("never puts one word on both arms", () => { + // Two locales spelling a term the same way ("negativo" in Spanish and + // Italian) is expected and folds onto one row. The same word meaning both + // negative and positive would be a bundle bug. + const negatives = new Set(); + const positives = new Set(); + for (const [word, concept] of Object.entries(localisedQualitativeIndex())) { + (concept.code === "260385009" ? negatives : positives).add(word); + } + for (const word of negatives) expect(positives.has(word)).toBe(false); + }); +}); diff --git a/src/lib/fhir/lab-loinc.ts b/src/lib/fhir/lab-loinc.ts index 17eaf927f..0e748d0f9 100644 --- a/src/lib/fhir/lab-loinc.ts +++ b/src/lib/fhir/lab-loinc.ts @@ -30,9 +30,16 @@ * yet carry that field, so the call site passes only `analyte` today. When the * DTO gains a `biomarkerCanonicalName` (or similar), thread it in — no other * change to this module is required. + * + * SERVER-ONLY since the localised alias index landed: the index reads every + * shipped message bundle through `@/lib/i18n/shared-resolve`. The single + * caller is the FHIR exporter, which runs on the server; a client that needs + * to show a biomarker's name resolves it through the labs catalog instead. */ import { LOINC_SYSTEM, UCUM_SYSTEM } from "@/lib/fhir/loinc-map"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; export { LOINC_SYSTEM, UCUM_SYSTEM }; @@ -45,13 +52,20 @@ export interface LabLoincMapping { } /** - * Normalise an analyte / unit string to a lookup key: lower-case, strip every - * character that is not a letter or digit. So "LDL-C", "LDL Cholesterol", - * "ldl_c" all fold to `ldlc`-ish keys; the alias table below pins the exact - * variants the app and German users actually type onto a single canonical key. + * Normalise an analyte / unit string to a lookup key: lower-case, drop + * accents, strip every character that is not a letter or digit. So "LDL-C", + * "LDL Cholesterol", "ldl_c" all fold to `ldlc`-ish keys. + * + * The accent fold is what lets a name carrying diacritics reach a key at all. + * Without it "Cholestérol total" lost its "é" entirely and folded to + * `cholestroltotal`, and "Hämoglobin" to `hmoglobin` — keys nothing was ever + * going to match, which is also why the hand-written German aliases below are + * spelled `hamoglobin` and `kreatinin` rather than as the language writes + * them. Every pre-existing alias is pure ASCII, so folding is a widening: what + * resolved before resolves to the same key now. */ export function normaliseLabKey(s: string): string { - return s.toLowerCase().replace(/[^a-z0-9]/g, ""); + return stripDiacritics(s.toLowerCase()).replace(/[^a-z0-9]/g, ""); } /** @@ -172,11 +186,17 @@ const LAB_LOINC_BY_KEY: Record = { }; /** - * Alias → canonical key. The LEFT side is the normalised form of a name a user - * (EN or DE) might type; the RIGHT side is a key in `LAB_LOINC_BY_KEY`. Every - * canonical key is also implicitly its own alias (looked up directly first). - * Keep the right side honest — only fold a synonym when it unambiguously names - * the same analyte. + * Alias → canonical key, hand-written. The LEFT side is the normalised form of + * a name a user might type; the RIGHT side is a key in `LAB_LOINC_BY_KEY`. + * Every canonical key is also implicitly its own alias (looked up directly + * first). Keep the right side honest — only fold a synonym when it + * unambiguously names the same analyte. + * + * This table covers spellings the app owns NO string for: clinical + * abbreviations ("GPT", "ASAT", "hgb"), and the ways a person shortens a name + * by hand. The NAMES the app itself displays are not transcribed here — see + * the derived index below — because that list exists in six languages already + * and copying it in would freeze it at today's six. */ const LAB_ALIASES: Record = { // HbA1c @@ -252,6 +272,94 @@ const LAB_ALIASES: Record = { nuchterninsulin: "fastinginsulin", }; +/** + * Labs-catalog slug → canonical LOINC key. + * + * Both sides are app-owned stable identities with no language in them, which + * is the whole point: this is the only edit a new locale does NOT need. A + * biomarker minted from the catalog stores whatever `labs.catalog.` said + * in the language of the person who minted it, and that name is what reaches + * the FHIR exporter as free text. Pinning the slug once means every + * translation of it resolves. + * + * Slugs absent here are catalog markers with no entry in `LAB_LOINC_BY_KEY` + * (T3, T4, B12, folate, the electrolytes, most of the blood count) — that map + * is conservative by design and text-only stays the honest default for them. + * Adding one is a decision about a LOINC term, not about a language. + */ +export const CATALOG_SLUG_TO_LOINC_KEY: Record = { + "total-cholesterol": "cholesterol", + ldl: "ldlc", + hdl: "hdlc", + triglycerides: "triglycerides", + "fasting-glucose": "glucosefasting", + hba1c: "hba1c", + tsh: "tsh", + ferritin: "ferritin", + "vitamin-d": "vitamind", + "hs-crp": "hscrp", + creatinine: "creatinine", + egfr: "egfr", + alt: "alt", + ast: "ast", + ggt: "ggt", + hemoglobin: "hemoglobin", + apob: "apob", + "lp-a": "lpa", + "fasting-insulin": "fastinginsulin", +}; + +/** + * Localised analyte name → canonical LOINC key, DERIVED from + * `messages/.json` rather than transcribed out of it. + * + * That is the load-bearing part. Before this index, the alias table above was + * the whole of the mapping and it was written in English and German, so a + * French account's "Cholestérol total", a Spanish "Colesterol total", an + * Italian "Colesterolo totale" and a Polish "Cholesterol całkowity" all + * exported as a text-only concept with no LOINC code, on a document a + * clinician or another system reads. Transcribing four more languages here + * would have fixed today's six locales and reopened the same hole on the day a + * seventh landed. Reading the bundles means a new locale needs no edit in this + * file: `locales` grows, `messages/xx.json` lands, and the names it ships + * resolve the same day. + * + * The hand-written aliases WIN on a conflict. They were verified one by one + * over several releases; a derived name that folds onto the same key as one of + * them is a duplicate and harmless, and one that folds onto a DIFFERENT key + * would be a bundle bug, so it is refused here and the guard suite fails on + * it — a translation typo must not quietly re-point an existing alias at + * another analyte. + * + * Known limit: the index is built from the names the bundles carry TODAY, + * while the analytes it resolves were typed in the past. Editing a shipped + * `labs.catalog.` string orphans the biomarkers minted under the old + * wording; the fix for that is a legacy row in `LAB_ALIASES`, not a migration. + */ +function buildLocalisedAliases(): Record { + const index: Record = {}; + for (const [slug, loincKey] of Object.entries(CATALOG_SLUG_TO_LOINC_KEY)) { + for (const name of localisedValues(`labs.catalog.${slug}`)) { + const key = normaliseLabKey(name); + if (!key) continue; + if (LAB_LOINC_BY_KEY[key] || LAB_ALIASES[key]) continue; + if (index[key] && index[key] !== loincKey) continue; + index[key] = loincKey; + } + } + return index; +} + +const LOCALISED_LAB_ALIASES = buildLocalisedAliases(); + +/** + * Test seam: the guard suite asserts the derived index covers every shipped + * locale and contradicts no hand-written alias. Production reads the constant. + */ +export function localisedLabAliases(): Readonly> { + return LOCALISED_LAB_ALIASES; +} + /** * Canonical UCUM symbols that a recorded free-text unit may normalise to. The * LEFT is the normalised (lower-cased, punctuation-stripped) recorded unit; @@ -344,7 +452,12 @@ export function resolveLabCoding( const source = canonicalName && canonicalName.length > 0 ? canonicalName : analyte; const key = normaliseLabKey(source); - const resolvedKey = LAB_LOINC_BY_KEY[key] ? key : LAB_ALIASES[key]; + // Canonical key first, then the hand-verified aliases, then the names the + // shipped bundles carry. The order is the confidence order, and it keeps + // every resolution that worked before working the same way. + const resolvedKey = LAB_LOINC_BY_KEY[key] + ? key + : (LAB_ALIASES[key] ?? LOCALISED_LAB_ALIASES[key]); if (!resolvedKey) return null; const mapping = LAB_LOINC_BY_KEY[resolvedKey]; if (!mapping) return null; diff --git a/src/lib/fhir/lab-qualitative.ts b/src/lib/fhir/lab-qualitative.ts index fb614e39c..7b5448ddc 100644 --- a/src/lib/fhir/lab-qualitative.ts +++ b/src/lib/fhir/lab-qualitative.ts @@ -3,29 +3,52 @@ * * A qualitative serology result ("negativ" / "positiv" / "nicht nachweisbar") * has no number, so its FHIR Observation carries a `valueCodeableConcept` - * instead of a `valueQuantity`. This module maps the common German / English - * result terms onto a small set of WELL-ESTABLISHED SNOMED CT qualifier-value - * concepts. The raw recorded text ALWAYS rides `.text`; a coded `coding` is - * added only when the term resolves CONFIDENTLY. When in doubt the result stays - * text-only — never a fabricated code — mirroring the conservative stance of - * the `lab-loinc` and `illness-snomed` mappers. + * instead of a `valueQuantity`. This module maps common result terms onto a + * small set of WELL-ESTABLISHED SNOMED CT qualifier-value concepts. The raw + * recorded text ALWAYS rides `.text`; a coded `coding` is added only when the + * term resolves CONFIDENTLY. When in doubt the result stays text-only — never + * a fabricated code — mirroring the conservative stance of the `lab-loinc` and + * `illness-snomed` mappers. * * Concepts (SNOMED CT International, qualifier value, REFERENCED not * redistributed per the licence): - * - 260385009 "Negative" — negativ / negative - * - 10828004 "Positive" — positiv / positive + * - 260385009 "Negative" — every locale's `labs.form.qualNegative` + * - 10828004 "Positive" — every locale's `labs.form.qualPositive` * - 260415000 "Not detected" — nicht nachweisbar / not detected * - 260373001 "Detected" — nachweisbar / detected * + * The negative and positive arms are DERIVED from `messages/.json`. + * The lab form offers those two as quick picks and writes the label it + * rendered, so what lands in the column is the bundle's own string — + * "négatif", "negativo", "ujemny" — and matching it against a transcribed + * English/German list meant four of the six shipped locales recorded a result + * the export then could not code. Deriving means a new locale is coded the day + * its bundle lands, with no edit here. + * + * The detected / not-detected arm is NOT derived, because the app ships no + * string for it: those are terms a LAB printed and a person copied in. It + * stays the hand-written English and German pair it has always been, and the + * honest cost is that "non détecté" or "niewykrywalny" stay text-only. Coding + * them would mean inventing clinical vocabulary in four languages nobody here + * can check, and this module's own rule is that an unverified code is worse + * than an honest `.text`. When those terms need coding, they need a quick pick + * in the form and a bundle key — the same route the negative and positive arms + * took. + * * Borderline / grenzwertig is DELIBERATELY left text-only: the candidate - * "Borderline" qualifier concept could not be confidently verified, and an - * unverified code is worse than an honest `.text`. Any unrecognised term + * "Borderline" qualifier concept could not be confidently verified. Its bundle + * key is therefore read by nothing here, on purpose. Any unrecognised term * likewise stays text-only. * + * SERVER-ONLY: the derived index reads every shipped message bundle. Its one + * caller is the FHIR exporter, which runs on the server. + * * Systems: SNOMED CT — http://snomed.info/sct */ import type { FhirCodeableConcept } from "@/lib/fhir/types"; +import { foldForMatch } from "@/lib/i18n/fold-for-match"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; /** * SNOMED CT system URI. Inlined (rather than imported from `resources.ts`) to @@ -39,29 +62,76 @@ interface QualitativeConcept { display: string; } +const NEGATIVE: QualitativeConcept = { code: "260385009", display: "Negative" }; +const POSITIVE: QualitativeConcept = { code: "10828004", display: "Positive" }; + /** - * Normalised result term → SNOMED qualifier-value concept. The left side is the - * lower-cased, whitespace-collapsed recorded text. Only confidently-known terms - * appear; everything else falls through to text-only. + * Hand-written result terms → SNOMED qualifier-value concept. The left side is + * the folded recorded text. + * + * What belongs here is what the app ships no string for: the clinical + * shorthand a person types ("neg", "pos") and the detected / not-detected pair + * a lab prints. The negative and positive WORDS are not transcribed here — see + * the derived index below. */ -const QUALITATIVE_SNOMED: Record = { - // Negative-like - negativ: { code: "260385009", display: "Negative" }, - negative: { code: "260385009", display: "Negative" }, - neg: { code: "260385009", display: "Negative" }, +const HAND_WRITTEN_SNOMED: Record = { + neg: NEGATIVE, + pos: POSITIVE, "nicht nachweisbar": { code: "260415000", display: "Not detected" }, "not detected": { code: "260415000", display: "Not detected" }, - // Positive-like - positiv: { code: "10828004", display: "Positive" }, - positive: { code: "10828004", display: "Positive" }, - pos: { code: "10828004", display: "Positive" }, nachweisbar: { code: "260373001", display: "Detected" }, detected: { code: "260373001", display: "Detected" }, }; -/** Collapse a recorded qualitative term to a lookup key. */ +/** + * The quick-pick labels, in every shipped locale, read out of the bundles. + * + * The form writes the label it rendered, so this index and the write side + * cannot drift: whatever `labs.form.qualNegative` says in a language is + * exactly what a reading recorded in that language holds. `qualBorderline` is + * deliberately absent — see the module comment. + */ +function buildLocalisedIndex(): Record { + const index: Record = {}; + const arms: [string, QualitativeConcept][] = [ + ["labs.form.qualNegative", NEGATIVE], + ["labs.form.qualPositive", POSITIVE], + ]; + for (const [key, concept] of arms) { + for (const label of localisedValues(key)) { + const folded = foldForMatch(label); + // A folded label already claimed by the OTHER arm would mean a bundle + // says the same word means both negative and positive. First wins so a + // typo cannot take the export down at boot; the guard suite fails on it. + if (!folded || index[folded]) continue; + index[folded] = concept; + } + } + return index; +} + +const LOCALISED_SNOMED = buildLocalisedIndex(); + +/** + * Test seam: the guard suite asserts the derived index covers every shipped + * locale and disagrees with no hand-written term. Production reads the + * constant. + */ +export function localisedQualitativeIndex(): Readonly< + Record +> { + return LOCALISED_SNOMED; +} + +/** + * Collapse a recorded qualitative term to a lookup key. + * + * Folds case, accents and separators, so a reading typed "Negatif" on a + * keyboard without accents meets the bundle's "négatif", and a stored + * "negativ" resolves exactly as it did before the fold widened. + */ function normaliseQualitativeKey(text: string): string { - return text.trim().toLowerCase().replace(/\s+/g, " "); + return foldForMatch(text); } /** @@ -73,7 +143,8 @@ function normaliseQualitativeKey(text: string): string { export function qualitativeValueConcept( valueText: string, ): FhirCodeableConcept { - const concept = QUALITATIVE_SNOMED[normaliseQualitativeKey(valueText)]; + const key = normaliseQualitativeKey(valueText); + const concept = HAND_WRITTEN_SNOMED[key] ?? LOCALISED_SNOMED[key]; if (!concept) { return { text: valueText }; } diff --git a/src/lib/i18n/fold-for-match.ts b/src/lib/i18n/fold-for-match.ts new file mode 100644 index 000000000..db0815e91 --- /dev/null +++ b/src/lib/i18n/fold-for-match.ts @@ -0,0 +1,59 @@ +/** + * Character folds used when app text has to be matched against text a PERSON + * or a LAB typed, rather than compared byte-for-byte. + * + * Every matcher that reads a shipped translation and looks for it in stored + * or transcribed text needs the same three tolerances — case, accents, and the + * apostrophe a word processor curled — and every matcher that grows its own + * copy of them drifts from the others. One home, so "Perte d'appétit" typed + * with a straight quote meets the bundle's "Perte d’appétit", and a French + * report's "jusqu'à" meets a table written "jusqu'a". + * + * The folds are one-directional and deliberately NOT transliteration: German + * "ue" does not fold onto "ü", "ß" is left alone (it has no canonical + * decomposition), and the Polish "ł" is a single codepoint NFD does not touch. + * That is fine — what has to agree is a shipped string with itself as a person + * retyped it, not two spellings of the same word. + * + * Client-safe: no message bundles are imported here, so a client component may + * use these. Building an index OVER the bundles is a server concern and lives + * in `./shared-resolve`. + */ + +/** Curly, modifier and backtick apostrophes a keyboard or word processor emits. */ +const APOSTROPHES = /[\u2018\u2019\u02bc\u00b4`']/gu; + +/** Combining marks left behind by an NFD decomposition. */ +const COMBINING_MARKS = /[\u0300-\u036f]/gu; + +/** + * Drop accents by decomposing and removing the combining marks: "é" → "e", + * "ż" → "z", "Ü" → "U". Punctuation, case and spacing survive untouched, so + * this is the fold to use where the surrounding characters carry meaning — a + * printed range's "3,5-5,0", a unit's "mg/dL". + */ +export function stripDiacritics(raw: string): string { + return raw.normalize("NFD").replace(COMBINING_MARKS, ""); +} + +/** Collapse every apostrophe variant onto the straight ASCII one. */ +export function normaliseApostrophes(raw: string): string { + return raw.replace(APOSTROPHES, "'"); +} + +/** + * Fold a label to its comparison form: lower case, no accents, and every + * apostrophe / hyphen / underscore / slash reduced to a single space. + * + * This is the fold for whole LABELS — a tag, an analyte name, a qualitative + * result term — where internal punctuation is noise. Apply it identically to + * both sides of a comparison; a fold applied to only one side is an + * exact-string match wearing a disguise. + */ +export function foldForMatch(raw: string): string { + return stripDiacritics(raw.toLowerCase()) + .replace(APOSTROPHES, " ") + .replace(/[-_/]/gu, " ") + .replace(/\s+/gu, " ") + .trim(); +} diff --git a/src/lib/i18n/shared-resolve.ts b/src/lib/i18n/shared-resolve.ts index f08790d05..fba5d019f 100644 --- a/src/lib/i18n/shared-resolve.ts +++ b/src/lib/i18n/shared-resolve.ts @@ -20,9 +20,10 @@ import frMessages from "../../../messages/fr.json"; import esMessages from "../../../messages/es.json"; import itMessages from "../../../messages/it.json"; import plMessages from "../../../messages/pl.json"; -import { type Locale } from "./config"; +import { locales, type Locale } from "./config"; +import { resolveKey } from "./resolve-key"; -export { resolveKey } from "./resolve-key"; +export { resolveKey }; /** * Single source of truth for the per-locale message bundle on the @@ -36,3 +37,29 @@ export const allMessages: Record> = { it: itMessages, pl: plMessages, }; + +/** + * Every shipped translation of one key, in `locales` order, deduplicated and + * with missing entries dropped. + * + * This is the primitive a DERIVED match index is built from. A matcher that + * has to recognise an app-owned label in text a person typed must not + * transcribe the label into its own source: transcribing fixes the locales + * shipped on the day it was written and reopens the same hole when a new + * bundle lands. Reading the bundles means a new locale is matchable the day + * `messages/.json` arrives, with no edit in the matcher. + * + * Known limit, worth stating where it bites: the values returned are what the + * bundles carry TODAY, while a matcher applies them to text written in the + * past. Editing a shipped translation therefore orphans whatever was written + * under the old wording — the fix for that is a legacy alias at the call site, + * not a data migration. + */ +export function localisedValues(key: string): string[] { + const seen = new Set(); + for (const locale of locales) { + const value = resolveKey(allMessages[locale], key); + if (value && value.trim()) seen.add(value.trim()); + } + return [...seen]; +} diff --git a/src/lib/labs/__tests__/parse-reference-range-locales.test.ts b/src/lib/labs/__tests__/parse-reference-range-locales.test.ts new file mode 100644 index 000000000..933bbce75 --- /dev/null +++ b/src/lib/labs/__tests__/parse-reference-range-locales.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from "vitest"; + +import { locales } from "@/lib/i18n/config"; +import { + isUnreadableRange, + parseReferenceRange, + RANGE_PROSE, +} from "@/lib/labs/parse-reference-range"; + +/** + * The notation a lab prints is the LAB's prose, not the app's, so this parser + * cannot derive its vocabulary from `messages/.json` the way a matcher + * over app-owned labels does. The table is hand-written, and these are the + * strings it has to read: real reference-range notation as French, Spanish, + * Italian and Polish reports print it, decimal comma included. + * + * Before the table existed, every case below returned `low: null, high: null` + * — the string was stored and the reading was then judged against a band the + * lab never printed, or against no band at all, with nothing anywhere saying + * so. + */ +describe("parseReferenceRange — French notation", () => { + it.each([ + ["jusqu'à 5,0", null, 5], + ["jusqu’à 5,0", null, 5], + ["JUSQU'À 5,0", null, 5], + ["inférieur à 5,0", null, 5], + ["moins de 5,0", null, 5], + ["à partir de 3,5", 3.5, null], + ["supérieur à 3,5", 3.5, null], + ["au moins 3,5", 3.5, null], + ["3,5 à 5,0", 3.5, 5], + ["de 3,5 à 5,0", 3.5, 5], + ["entre 3,5 et 5,0", 3.5, 5], + ["valeurs de référence 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — Spanish notation", () => { + it.each([ + ["hasta 5,0", null, 5], + ["menor de 5,0", null, 5], + ["inferior a 5,0", null, 5], + ["como máximo 5,0", null, 5], + ["desde 3,5", 3.5, null], + ["mayor que 3,5", 3.5, null], + ["superior a 3,5", 3.5, null], + ["más de 3,5", 3.5, null], + ["3,5 a 5,0", 3.5, 5], + ["de 3,5 a 5,0", 3.5, 5], + ["entre 3,5 y 5,0", 3.5, 5], + ["valores de referencia 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — Italian notation", () => { + it.each([ + ["fino a 5,0", null, 5], + ["inferiore a 5,0", null, 5], + ["meno di 5,0", null, 5], + ["oltre 3,5", 3.5, null], + ["superiore a 3,5", 3.5, null], + ["più di 3,5", 3.5, null], + ["almeno 3,5", 3.5, null], + ["3,5 a 5,0", 3.5, 5], + ["da 3,5 a 5,0", 3.5, 5], + ["tra 3,5 e 5,0", 3.5, 5], + ["valori di riferimento 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); + + it("reads a bare Italian floor rather than swallowing its introducing word", () => { + // "da" both introduces a two-sided window ("da 3,5 a 5,0") and states a + // floor on its own. Stripping introducing words before matching — which is + // what the parser used to do unconditionally — turns the second form into + // a bare number with no bound at all. + expect(parseReferenceRange("da 3,5")).toEqual({ + low: 3.5, + high: null, + text: "da 3,5", + }); + }); +}); + +describe("parseReferenceRange — Polish notation", () => { + it.each([ + ["do 5,0", null, 5], + ["poniżej 5,0", null, 5], + ["mniej niż 5,0", null, 5], + ["maks. 5,0", null, 5], + ["od 3,5", 3.5, null], + ["powyżej 3,5", 3.5, null], + ["co najmniej 3,5", 3.5, null], + ["3,5 do 5,0", 3.5, 5], + ["od 3,5 do 5,0", 3.5, 5], + ["zakres referencyjny 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — the new prose does not loosen the old rules", () => { + it("still refuses a bare number in any language", () => { + expect(parseReferenceRange("5,0")).toEqual({ + low: null, + high: null, + text: "5,0", + }); + }); + + it("does not read scientific notation as a two-sided window", () => { + // "e" is an Italian separator ("tra 3,5 e 5,0") and "y" a Spanish one. + // Both are one letter, so they are only accepted with whitespace on both + // sides — otherwise "3.5e5" would parse as a 3.5–5 window. + expect(parseReferenceRange("3.5e5")).toEqual({ + low: null, + high: null, + text: "3.5e5", + }); + }); + + it("still refuses a transposed window printed in French", () => { + expect(parseReferenceRange("de 5,0 à 3,5")).toEqual({ + low: null, + high: null, + text: "de 5,0 à 3,5", + }); + }); + + it("still refuses bounds when the printed unit contradicts the reading", () => { + expect(parseReferenceRange("jusqu'à 5,0 mmol/l", "mg/dL")).toEqual({ + low: null, + high: null, + text: "jusqu'à 5,0 mmol/l", + }); + }); +}); + +describe("parseReferenceRange — an unreadable window is named, not silent", () => { + it("separates 'nothing printed' from 'printed and not read'", () => { + expect(parseReferenceRange(null)).toBeNull(); + expect(isUnreadableRange(parseReferenceRange(null))).toBe(false); + + const prose = parseReferenceRange("voir le compte rendu"); + expect(prose).toEqual({ + low: null, + high: null, + text: "voir le compte rendu", + }); + expect(isUnreadableRange(prose)).toBe(true); + }); + + it("reports a window it DID read as readable", () => { + expect(isUnreadableRange(parseReferenceRange("jusqu'à 5,0"))).toBe(false); + }); +}); + +describe("RANGE_PROSE coverage", () => { + it("carries an entry for every shipped locale", () => { + // Keyed by the `Locale` union on purpose: a seventh locale cannot be added + // to `locales` without someone answering how that language prints a range. + // This asserts the runtime side of the same promise. + expect(Object.keys(RANGE_PROSE).sort()).toEqual([...locales].sort()); + }); + + it("states at least one word for each notation in each locale", () => { + for (const locale of locales) { + const prose = RANGE_PROSE[locale]; + expect(prose.twoSided.length, `${locale}.twoSided`).toBeGreaterThan(0); + expect(prose.upper.length, `${locale}.upper`).toBeGreaterThan(0); + expect(prose.lower.length, `${locale}.lower`).toBeGreaterThan(0); + } + }); + + it("carries the table already folded, so both sides of a match agree", () => { + // A row written with an accent would never match: the input is + // diacritic-folded before it reaches the alternation, so an accented row + // is a row that can only ever miss. + for (const locale of locales) { + const prose = RANGE_PROSE[locale]; + for (const word of [ + ...prose.twoSided, + ...prose.upper, + ...prose.lower, + ...prose.filler, + ]) { + expect(word.normalize("NFD"), `${locale}: ${word}`).not.toMatch(/[̀-ͯ]/u); + expect(word, `${locale}: ${word}`).toBe(word.toLowerCase()); + } + } + }); +}); diff --git a/src/lib/labs/ocr-extract.ts b/src/lib/labs/ocr-extract.ts index fcdb01563..4329211b8 100644 --- a/src/lib/labs/ocr-extract.ts +++ b/src/lib/labs/ocr-extract.ts @@ -24,7 +24,10 @@ import { } from "@/lib/ai/types"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { prisma } from "@/lib/db"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; import { annotate } from "@/lib/logging/context"; import { extractedLabsSchema, @@ -270,7 +273,16 @@ export async function runOcrExtraction( // Cap defensively (the schema already bounds it) and annotate each row. const rows: OcrExtractedRowDto[] = []; + // How many rows printed a window the parser could not read. Those rows fall + // back to whatever bounds the model itself reported, or to none — either way + // the notation went unread, and a count of it is the difference between + // noticing that a report layout or a language is unsupported and never + // hearing about it. A count only: the string is the user's document. + let unreadableRanges = 0; for (const row of envelope.rows.slice(0, OCR_MAX_ROWS)) { + if (isUnreadableRange(parseReferenceRange(row.referenceText, row.unit))) { + unreadableRanges += 1; + } rows.push(await annotateRow(args.userId, row, reportDate)); } @@ -282,6 +294,7 @@ export async function runOcrExtraction( rows: rows.length, duplicates: rows.filter((r) => r.duplicateOf !== null).length, newBiomarkers: rows.filter((r) => r.biomarkerMatch === "new").length, + unreadableRanges, }, }); diff --git a/src/lib/labs/parse-reference-range.ts b/src/lib/labs/parse-reference-range.ts index 24e413f37..f16765ff6 100644 --- a/src/lib/labs/parse-reference-range.ts +++ b/src/lib/labs/parse-reference-range.ts @@ -19,14 +19,31 @@ * plausible bound out of a string it did not understand, because a fake * window silently reclassifies a reading as normal or abnormal. * - * Notations read (German and English, comma- or dot-decimal): + * Notations read (comma- or dot-decimal): * * two-sided 3.5-5.0 · 3,5 – 5,0 · 3,5 bis 5,0 · 3.5 to 5.0 · 3,5 ... 5,0 - * von 3,5 bis 5,0 · [3,5 - 5,0] · (3.5-5.0) + * von 3,5 bis 5,0 · 3,5 à 5,0 · de 3,5 a 5,0 · da 3,5 a 5,0 · + * od 3,5 do 5,0 · [3,5 - 5,0] · (3.5-5.0) * upper only < 5 · ≤ 5 · <= 5 · bis 5,0 · bis zu 5,0 · max. 5,0 · - * kleiner 5 · unter 5 · up to 5.0 · less than 5 + * kleiner 5 · unter 5 · up to 5.0 · less than 5 · jusqu'à 5,0 · + * hasta 5,0 · fino a 5,0 · do 5,0 · poniżej 5,0 * lower only > 3.5 · ≥ 3,5 · >= 3.5 · ab 3,5 · min. 3,5 · mindestens 3,5 · - * größer 3,5 · über 3,5 · at least 3.5 + * größer 3,5 · über 3,5 · at least 3.5 · à partir de 3,5 · + * desde 3,5 · da 3,5 · od 3,5 · powyżej 3,5 + * + * WHY THE VOCABULARY IS TABULATED AND NOT DERIVED. Elsewhere in the app, a + * matcher that has to recognise a word list in six languages builds its index + * out of `messages/.json`, so a seventh locale works the day its + * bundle lands. That does not apply here. "jusqu'à", "hasta", "fino a" and + * "do" are prose a LAB printed on a sheet of paper; the app owns no string + * they could be derived from, and the language of the report has nothing to do + * with the language of the UI — a French self-hoster running the app in + * English still receives French reports. So the table below is hand-written, + * every locale's prose is tried against every input at once, and the honest + * cost is written down: a report printed in a language absent from the table + * yields text-only. `RANGE_PROSE` is keyed by the shipped `Locale` union + * precisely so that adding a locale to `locales` fails to compile until + * someone has answered "and how does that language print a range?". * * Units are read, never converted. A window may carry the unit the report * printed it in; when the caller states the reading's own unit and the two @@ -34,6 +51,11 @@ * mmol/L into mg/dL needs a per-analyte factor, so doing it here would be the * parser deciding what the report meant. */ +import { type Locale } from "@/lib/i18n/config"; +import { + normaliseApostrophes, + stripDiacritics, +} from "@/lib/i18n/fold-for-match"; /** What a printed range yielded. `text` is set whenever the input was non-empty. */ export interface ParsedReferenceRange { @@ -58,14 +80,224 @@ const MAX_ABS_BOUND = 1e12; /** One number token: optional sign, digits, any number of `.`/`,` groups. */ const NUM = String.raw`[+-]?\d+(?:[.,]\d+)*`; -/** Infix separators for a two-sided window. */ -const TWO_SIDED_SEP = String.raw`(?:-|–|—|‒|…|\.{2,3}|bis\s+zu|bis|to|until)`; +/** How one language prints a range. Every field is a list of literal words. */ +export interface RangeProse { + /** Infix words joining two bounds: "3,5 5,0". */ + twoSided: string[]; + /** Words that introduce a ceiling: " 5,0". */ + upper: string[]; + /** Words that introduce a floor: " 3,5". */ + lower: string[]; + /** + * Words that only announce that a window follows and carry no bound of + * their own ("von 3,5 bis 5,0"). Stripped and the input retried — never on + * the first pass, because several of them ("da", "od", "de") are also a + * floor prefix in their own language, and stripping first would read + * "da 3,5" as a bare number. + */ + filler: string[]; +} + +/** + * The prose, per shipped locale. Written ALREADY FOLDED — lower case and + * accent-free — because the input is diacritic-folded before matching, so + * "jusqu'à", "jusqu'a" and "JUSQU'À" are one row. The German "ß" survives the + * fold (it has no canonical decomposition), which is why "größer" appears here + * as "großer"; its "oe"/"ss" twins are listed beside it, exactly as the + * hand-written alternation carried them before. + * + * The `de` and `en` lists are the pre-existing alternation, split apart word + * for word and nothing more — this change is about the four locales that had + * no entry at all, and a German or English range must read the same afterwards + * as it did before. + * + * Every list is applied to every input regardless of the reader's locale — see + * the module comment for why. Multi-word entries match across any run of + * whitespace, so a report that printed a non-breaking space still reads. + */ +export const RANGE_PROSE: Record = { + de: { + twoSided: ["bis zu", "bis"], + upper: [ + "bis zu", + "bis", + "max.", + "maximal", + "max", + "kleiner als", + "kleiner", + "unter", + "hochstens", + "hoechstens", + ], + lower: [ + "ab", + "mind.", + "mindestens", + "minimal", + "min.", + "min", + "großer als", + "großer", + "groesser als", + "groesser", + "uber", + "ueber", + ], + filler: ["von", "referenz", "ref.", "ref"], + }, + en: { + twoSided: ["to", "until"], + upper: ["up to", "less than", "below"], + lower: ["at least", "greater than", "above"], + filler: ["from", "reference", "ref.", "ref"], + }, + fr: { + twoSided: ["a", "et"], + upper: [ + "jusqu'a", + "jusque", + "inferieur a", + "inferieure a", + "moins de", + "au plus", + "au maximum", + ], + lower: [ + "a partir de", + "superieur a", + "superieure a", + "plus de", + "au moins", + "au minimum", + ], + filler: [ + "de", + "entre", + "valeurs de reference", + "valeur de reference", + "intervalle de reference", + "plage de reference", + ], + }, + es: { + twoSided: ["a", "y"], + upper: [ + "hasta", + "menor que", + "menor de", + "menor a", + "inferior a", + "menos de", + "como maximo", + "maximo", + ], + lower: [ + "desde", + "a partir de", + "mayor que", + "mayor de", + "mayor a", + "superior a", + "mas de", + "al menos", + "como minimo", + "minimo", + ], + filler: [ + "de", + "entre", + "valores de referencia", + "valor de referencia", + "rango de referencia", + "intervalo de referencia", + ], + }, + it: { + twoSided: ["a", "e"], + upper: [ + "fino a", + "inferiore a", + "minore di", + "meno di", + "al massimo", + "massimo", + ], + lower: [ + "da", + "oltre", + "superiore a", + "maggiore di", + "piu di", + "almeno", + "al minimo", + "minimo", + ], + filler: [ + "da", + "tra", + "fra", + "valori di riferimento", + "valore di riferimento", + "intervallo di riferimento", + ], + }, + pl: { + twoSided: ["do"], + upper: ["do", "ponizej", "mniej niz", "nie wiecej niz", "maks.", "maks"], + lower: [ + "od", + "powyzej", + "wiecej niz", + "co najmniej", + "nie mniej niz", + "minimum", + ], + filler: [ + "od", + "zakres referencyjny", + "wartosci referencyjne", + "normy laboratoryjne", + ], + }, +}; + +/** Escape a literal for regex use and let any run of whitespace join its words. */ +function literalToPattern(word: string): string { + return word + .split(/\s+/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join(String.raw`\s+`); +} + +/** + * Alternation over every locale's list for one field, longest literal first so + * the reader sees the specific form ahead of the prefix it contains. + */ +function proseAlternation(field: keyof RangeProse): string { + const words = new Set(); + for (const prose of Object.values(RANGE_PROSE)) { + for (const word of prose[field]) words.add(word); + } + return [...words] + .sort((a, b) => b.length - a.length) + .map(literalToPattern) + .join("|"); +} + +/** + * Symbolic infix separators. Kept apart from the word separators because they + * need no surrounding whitespace, while a word separator must have it: "a", + * "e" and "y" are one letter long, and a bare `e` that could sit between two + * digits would read "3.5e5" as a window. + */ +const SYMBOL_SEP = String.raw`-|–|—|‒|…|\.{2,3}`; /** Prefixes that state a ceiling only. */ -const UPPER_PREFIX = String.raw`(?:<=|<|≤|bis\s+zu|bis|max\.|maximal|max|kleiner\s+als|kleiner|unter|höchstens|hoechstens|up\s+to|less\s+than|below)`; +const UPPER_PREFIX = `(?:<=|<|≤|${proseAlternation("upper")})`; /** Prefixes that state a floor only. */ -const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|min|größer\s+als|größer|groesser\s+als|groesser|über|ueber|at\s+least|greater\s+than|above)`; +const LOWER_PREFIX = `(?:>=|>|≥|${proseAlternation("lower")})`; /** * Trailing unit tail. Deliberately whitespace-free and comma-free: a real unit @@ -76,14 +308,19 @@ const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|m const UNIT_TAIL = String.raw`(?:\s*([^\s,]{1,24}))?`; const TWO_SIDED_RE = new RegExp( - `^(${NUM})\\s*${TWO_SIDED_SEP}\\s*(${NUM})${UNIT_TAIL}$`, + `^(${NUM})(?:\\s*(?:${SYMBOL_SEP})\\s*|\\s+(?:${proseAlternation( + "twoSided", + )})\\s+)(${NUM})${UNIT_TAIL}$`, "iu", ); const UPPER_RE = new RegExp(`^${UPPER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu"); const LOWER_RE = new RegExp(`^${LOWER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu"); /** Leading words that only introduce the window ("von 3,5 bis 5,0"). */ -const LEADING_FILLER_RE = /^(?:von|from|ref\.?|referenz|reference)\s+/iu; +const LEADING_FILLER_RE = new RegExp( + `^(?:${proseAlternation("filler")})\\s+`, + "iu", +); /** * Read one number token into a JS number, or null when its separators are @@ -148,33 +385,18 @@ function unitsAgree(printed: string | undefined, reading: string | undefined) { } /** - * Parse the reference range a lab report printed for one reading. - * - * @param raw the printed string, exactly as transcribed. - * @param unit the reading's own unit, when known. A printed window in a - * different unit yields text-only — never a converted bound. - * @returns null when there was nothing printed at all. + * Try the three notations against one candidate string. Null means none of + * them matched; a result with both bounds null means one matched but the + * numbers or the unit refused it, which is a final answer and not a reason to + * retry a differently-trimmed candidate. */ -export function parseReferenceRange( - raw: string | null | undefined, - unit?: string | null, +function matchNotation( + candidate: string, + text: string, + unit: string | null | undefined, ): ParsedReferenceRange | null { - if (raw === null || raw === undefined) return null; - const text = raw - .replace(/\s+/gu, " ") - .trim() - .slice(0, SOURCE_REFERENCE_TEXT_MAX); - if (text === "") return null; - const noBounds: ParsedReferenceRange = { low: null, high: null, text }; - // Strip a bracket wrapper and an introducing word for MATCHING only — the - // stored text stays exactly what the report printed. - let candidate = text; - const bracketed = /^[[(](.*)[\])]$/u.exec(candidate); - if (bracketed) candidate = bracketed[1].trim(); - candidate = candidate.replace(LEADING_FILLER_RE, "").trim(); - const two = TWO_SIDED_RE.exec(candidate); if (two) { if (!unitsAgree(two[3], unit ?? undefined)) return noBounds; @@ -201,7 +423,76 @@ export function parseReferenceRange( return low === null ? noBounds : { low, high: null, text }; } + return null; +} + +/** + * Parse the reference range a lab report printed for one reading. + * + * @param raw the printed string, exactly as transcribed. + * @param unit the reading's own unit, when known. A printed window in a + * different unit yields text-only — never a converted bound. + * @returns null when there was nothing printed at all. A non-null result with + * both bounds null means the report DID print something the parser + * could not read — see `isUnreadableRange`, which is the difference + * between "no window" and "a window nobody checked". + */ +export function parseReferenceRange( + raw: string | null | undefined, + unit?: string | null, +): ParsedReferenceRange | null { + if (raw === null || raw === undefined) return null; + const text = raw + .replace(/\s+/gu, " ") + .trim() + .slice(0, SOURCE_REFERENCE_TEXT_MAX); + if (text === "") return null; + + // Fold for MATCHING only — the stored text stays exactly what the report + // printed. Accents go so a table written "jusqu'a" meets a report's + // "jusqu'à"; the curled apostrophe a word processor emits becomes the + // straight one the table carries. Everything else, notably the digits and + // their separators, is left alone. + let candidate = normaliseApostrophes(stripDiacritics(text)); + const bracketed = /^[[(](.*)[\])]$/u.exec(candidate); + if (bracketed) candidate = bracketed[1].trim(); + + // Untrimmed first. Several introducing words are also a bound prefix in + // their own language — Italian "da", Polish "od", French "de" — so stripping + // before matching would read "da 3,5" as a bare number with no floor. + const direct = matchNotation(candidate, text, unit); + if (direct) return direct; + + const trimmed = candidate.replace(LEADING_FILLER_RE, "").trim(); + if (trimmed !== candidate) { + const viaFiller = matchNotation(trimmed, text, unit); + if (viaFiller) return viaFiller; + } + // Anything else — "negativ", "siehe Befund", a bare number, prose — keeps its // text and states no window. - return noBounds; + return { low: null, high: null, text }; +} + +/** + * True when the report printed a range and the parser could not derive a + * single bound from it. + * + * This is the case worth naming. `null` from the parser means the report + * stated nothing, and the reading falls back to the biomarker's own band as it + * always did. A parsed result with two null bounds means something else + * entirely: the lab DID state a window, on this reading, and the app is about + * to judge the value against a different band or against none at all. Left + * unnamed it reads identically to "no range" at every call site, which is how + * a French account's `jusqu'à 5,0` became a reading nobody ever flagged. + * + * Callers are expected to SAY so rather than to guess a bound — the manual + * form shows the reader that the string was kept but not read, and the ingest + * paths annotate it so an unreadable notation shows up as a number in the wide + * event instead of as silence. + */ +export function isUnreadableRange( + parsed: ParsedReferenceRange | null | undefined, +): boolean { + return !!parsed && parsed.low === null && parsed.high === null; }