diff --git a/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts b/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts new file mode 100644 index 000000000..53fad3039 --- /dev/null +++ b/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "vitest"; + +import { + derivedAnalyteNamePattern, + looksLikeLabDocument, +} from "@/lib/documents/auto-stage-labs"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; + +/** + * The gate decides whether a transcribed document is worth an extraction call. + * A lab report that fails it is lost from auto-staging entirely, and nothing + * says so — the manual extract button stays available, so the flow reads as + * working. + * + * Before this, the four signal classes were English and German only, so a + * report printed anywhere else scored at most one hit (the units, which are + * the same everywhere) and never reached two. + */ +const FRENCH_REPORT = ` +Laboratoire d'analyses médicales +Compte rendu du 14 mars 2026 + +Cholestérol total 5,8 mmol/l Valeurs de référence: jusqu'à 5,0 +Créatinine 88 µmol/l Plage de référence: 62 - 106 +Hémoglobine 14,2 g/dl +`; + +const SPANISH_REPORT = ` +Laboratorio de análisis clínicos +Analítica de sangre + +Colesterol total 210 mg/dl Valores de referencia: hasta 200 +Glucosa en ayunas 92 mg/dl Rango de referencia: 70 - 100 +`; + +const ITALIAN_REPORT = ` +Referto di laboratorio — esami del sangue + +Colesterolo totale 5,4 mmol/l Valori di riferimento: fino a 5,0 +Emoglobina 13,8 g/dl Intervallo di riferimento: 12 - 16 +`; + +const POLISH_REPORT = ` +Laboratorium analiz lekarskich +Wyniki badań + +Cholesterol całkowity 5,2 mmol/l Zakres referencyjny: do 5,0 +Kreatynina 82 µmol/l Wartości referencyjne: 62 - 106 +`; + +describe("looksLikeLabDocument — a report printed outside English and German", () => { + it.each([ + ["French", FRENCH_REPORT], + ["Spanish", SPANISH_REPORT], + ["Italian", ITALIAN_REPORT], + ["Polish", POLISH_REPORT], + ])("recognises a %s lab report", (_language, text) => { + expect(looksLikeLabDocument(text)).toBe(true); + }); + + it("recognises a report on its analyte names and units alone", () => { + // No header, no reference-range label — the two classes left are the ones + // a stripped-down result table still carries. + expect( + looksLikeLabDocument("Triglycérides 1,4 mmol/l\nFerritine 120 ng/ml"), + ).toBe(true); + }); +}); + +/** + * The four reports above each carry several kinds of evidence at once, so any + * one of the additions could carry them alone. These cases isolate one + * addition each: remove the piece each names and the case drops back to a + * single class and fails the gate. + */ +describe("looksLikeLabDocument — each addition carries its own weight", () => { + it("needs the DERIVED analyte names for a panel of markers no stem covers", () => { + // The electrolytes are in the catalog and in no hand-written stem, so the + // derived names are the only analyte evidence here. Units are the second + // class. + expect( + looksLikeLabDocument("Sodium 140 mmol/l\nPotassium 4,2 mmol/l"), + ).toBe(true); + expect(looksLikeLabDocument("Sodio 140 mmol/l\nPotassio 4,2 mmol/l")).toBe( + true, + ); + }); + + it("needs the reference-range labels of the four locales", () => { + // "Płytki krwi" is a derived name; the label is the only other class here. + expect( + looksLikeLabDocument("Płytki krwi 250\nZakres referencyjny: 150 - 400"), + ).toBe(true); + expect( + looksLikeLabDocument("Plaquettes 250\nValeurs de référence: 150 - 400"), + ).toBe(true); + }); + + it("needs the report headers of the four locales", () => { + expect(looksLikeLabDocument("Referto di laboratorio\nPiastrine 250")).toBe( + true, + ); + expect(looksLikeLabDocument("Wyniki badań\nPłytki krwi 250")).toBe(true); + expect(looksLikeLabDocument("Analítica de sangre\nPlaquetas 250")).toBe( + true, + ); + }); +}); + +describe("looksLikeLabDocument — what must still fail the gate", () => { + it.each([ + [ + "English prose with one incidental unit", + "Please take 500 mg twice daily.", + ], + [ + "a French letter mentioning one analyte", + "Cher patient, votre taux de cholestérol sera revu lors du prochain rendez-vous.", + ], + [ + "a Spanish appointment letter", + "Estimado paciente, su cita en el hospital es el martes a las diez.", + ], + [ + "an Italian pharmacy note", + "Ritirare la confezione in farmacia entro venerdì.", + ], + ["an invoice with a currency amount", "Rechnung über 120,00 EUR. Danke."], + ])("rejects %s", (_case, text) => { + expect(looksLikeLabDocument(text)).toBe(false); + }); + + it("does not let one analyte named twice reach two classes", () => { + // The derived names extend the analyte class rather than forming their + // own. If they were a fifth class, a document saying "Cholestérol total" + // and "cholesterol" would score two and pass on one piece of evidence. + expect(looksLikeLabDocument("Cholestérol total et cholesterol")).toBe( + false, + ); + }); + + it("ignores a derived analyte name embedded in a longer word", () => { + // The hand-written half of the analyte class matches STEMS on purpose + // ("ferritin" catches "Ferritinwert"); the derived half is bounded, + // because a whole catalog name inside another word is noise. + expect(looksLikeLabDocument("Xhematocritx 5 mg/dl")).toBe(false); + }); +}); + +describe("looksLikeLabDocument — the English and German behaviour is unchanged", () => { + it("still recognises a German report", () => { + expect( + looksLikeLabDocument( + "Laborbefund. Hämoglobin 14.2 g/dl. Referenzbereich 13-17. LDL 120 mg/dl.", + ), + ).toBe(true); + }); + + it("still recognises an English report", () => { + expect( + looksLikeLabDocument( + "Laboratory report. Hemoglobin 14.2 g/dL. Reference range 13-17.", + ), + ).toBe(true); + }); + + it("still rejects the prose the previous guard rejected", () => { + expect( + looksLikeLabDocument("Dear patient, please take 500 mg twice daily."), + ).toBe(false); + }); +}); + +describe("the derived analyte-name pattern", () => { + it("carries a name from every shipped locale", () => { + // The property that matters. A transcribed list would satisfy the report + // cases above and still be frozen at the languages somebody typed. + for (const locale of locales) { + const name = resolveKey( + allMessages[locale], + "labs.catalog.total-cholesterol", + ); + expect(name, `${locale} has no catalog name`).toBeTruthy(); + expect( + derivedAnalyteNamePattern().test( + stripDiacritics((name as string).toLowerCase()), + ), + `${locale}: ${name}`, + ).toBe(true); + } + }); + + it("excludes the short acronyms that are ordinary words elsewhere", () => { + // "ALT" is German for "old". A three-character alternative in a gate this + // cheap would make every German letter a lab report. The acronyms worth + // having live in the hand-written class, each considered on its own. + const pattern = derivedAnalyteNamePattern(); + for (const acronym of ["alt", "ast", "ggt", "sod"]) { + expect(pattern.test(acronym), acronym).toBe(false); + } + }); + + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(derivedAnalyteNamePattern().test("triglycerides")).toBe(true); + // The Polish "ł" is a single codepoint NFD does not decompose, so it + // survives the fold on both sides and has to be written as the bundle + // writes it. A document that spelled it "plytki" would not match, which is + // the stated limit of folding rather than transliterating. + expect(derivedAnalyteNamePattern().test("płytki krwi")).toBe(true); + }); +}); diff --git a/src/lib/documents/auto-stage-labs.ts b/src/lib/documents/auto-stage-labs.ts index 1c815b7d7..b001471d7 100644 --- a/src/lib/documents/auto-stage-labs.ts +++ b/src/lib/documents/auto-stage-labs.ts @@ -42,28 +42,131 @@ import { } from "@/lib/documents/extract"; import { encryptFactData, encryptFactProvenance } from "@/lib/documents/store"; import { prisma } from "@/lib/db"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; +import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog"; import { annotate } from "@/lib/logging/context"; import { isModuleEnabled } from "@/lib/modules/gate"; /** * Lab-report signals used to decide whether a transcribed document is worth an - * extraction call. Each pattern is one class of evidence (a reference-range - * label, a lab unit, a common analyte name, a report header); requiring TWO - * distinct hits keeps a stray "mg" in a prose letter from tripping the auto - * extraction while still catching a real panel in German or English. + * extraction call. Each entry is one CLASS of evidence — a reference-range + * label, a lab unit, a common analyte name, a report header — and contributes + * at most one hit however many of its alternatives fire. Requiring TWO + * distinct classes keeps a stray "mg" in a prose letter from tripping the auto + * extraction while still catching a real panel. + * + * Every pattern is tested against the text LOWER-CASED AND ACCENT-FOLDED, not + * against the transcription as written, so the patterns are spelled without + * accents. That is also what lets one pattern serve "Hämoglobin", "haemoglobin" + * and "hemoglobin" instead of three. + * + * Until now the four classes were English and German only, so a French, + * Spanish, Italian or Polish lab report scored at most one hit — the units, + * which are the same everywhere — and auto-staging never fired for it. The + * whole document was lost from the flow, silently: the manual extract button + * stays available, so nothing anywhere reads as broken. + * + * WHICH WAY THIS ERRS. A false negative loses a real report from auto-staging; + * a false positive burns one extraction call on a document that is not a lab + * report, and stages facts a person then declines on the review screen — + * nothing commits either way. The cost is lopsided, so the vocabulary is + * widened rather than kept tight, and the two-distinct-class rule is what + * holds the other side: an incidental "sodium" in a prose letter is one class + * and still fails the gate. Terms whose only home is a lab report ("emocromo", + * "bilan sanguin", "zakres referencyjny") were preferred over generic ones, + * and short acronyms below four characters are excluded from the derived names + * for exactly this reason — the German word "alt" would otherwise make every + * German letter a lab report. */ const LAB_SIGNALS: readonly RegExp[] = [ - /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich/i, + // 1 — a reference-range label. Only a report that prints windows says this. + /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich|valeurs? de reference|intervalle de reference|plage de reference|valores? de referencia|rango de referencia|intervalo de referencia|valori di riferimento|intervallo di riferimento|zakres referencyjny|wartosci referencyjne/i, + // 2 — a lab unit. Language-independent already; unchanged. /\b(mg\/dl|mmol\/l|nmol\/l|µmol\/l|umol\/l|g\/dl|µg\/l|ug\/l|ng\/ml|pg\/ml|mmol\/mol|u\/l|iu\/l|mg\/l|\/µl|\/ul)\b/i, - /h(?:ä|ae)moglobin|hemoglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i, - /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b/i, + // 3 — an analyte name. The hand-written half: word stems and the short + // acronyms, which are too short to derive safely but were vetted long ago. + /h(?:a|ae|e)moglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i, + // 4 — a report header. + /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b|laboratoire|bilan sanguin|analyses medicales|hemogramme|laboratorio|analitica|hemograma|analisis de sangre|esami del sangue|emocromo|referto|laboratorium|morfologia krwi|wyniki badan/i, ]; -/** True when the transcribed text carries at least two lab-report signals. */ +/** + * Class 3 again, DERIVED from `messages/.json` instead of transcribed. + * + * The names above are stems a maintainer typed. These are the names the app + * itself displays for the markers in its catalog, in every language it ships — + * so a French report's "Cholestérol total" or a Polish "Płytki krwi" counts as + * an analyte the day the bundle carries it, with no edit here. Transcribing + * thirty names times four more languages would have fixed today's six locales + * and reopened the hole at the seventh. + * + * Names shorter than four folded characters are dropped: "ALT", "AST", "GGT", + * "TSH" and Polish "Sód" are ordinary words or fragments in some language, and + * a three-letter alternative in a gate this cheap is a false-positive machine. + * The acronyms worth having are already in the hand-written class above, where + * each was considered one at a time. + */ +function buildAnalyteNamePattern(): RegExp { + const names = new Set(); + for (const seed of BIOMARKER_CATALOG) { + for (const name of localisedValues(`labs.catalog.${seed.slug}`)) { + const folded = stripDiacritics(name.toLowerCase()).trim(); + // Also the name without its parenthetical gloss. A report prints + // "Apolipoprotéine B", not "Apolipoprotéine B (ApoB)", and the gloss is + // the app's way of naming a marker two ways at once. + const bare = folded.replace(/\s*\([^)]*\)/gu, "").trim(); + for (const candidate of [folded, bare]) { + if (candidate.length >= 4) names.add(candidate); + } + } + } + const alternation = [...names] + .sort((a, b) => b.length - a.length) + .map((name) => + name + .split(/\s+/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + // Any run of whitespace, so a name broken across a transcribed line + // still reads as one name. + .join(String.raw`\s+`), + ) + .join("|"); + // Bounded by "not a letter or digit" rather than by \b, because several + // names end in punctuation ("Lipoprotéine(a)") where \b cannot match, and + // because the Polish "ł" is not a word character to \b. + return new RegExp( + `(?= 2) return true; + for (const [index, signal] of LAB_SIGNALS.entries()) { + const analyteClass = index === 2; + const hit = + signal.test(folded) || + (analyteClass && DERIVED_ANALYTE_NAMES.test(folded)); + if (hit && ++hits >= 2) return true; } return false; }