From 2dc553057d20118dc72e6db365c824d851b9a5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 23 Aug 2026 14:16:15 +0200 Subject: [PATCH 1/5] fix(labs): read a reference range printed in French, Spanish, Italian or Polish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The free-text reference-range parser decided what a printed window meant from an alternation of German and English words. The app ships six locales, and a lab prints in the language of the country it stands in, so `jusqu'à 5,0`, `hasta 5,0`, `fino a 5,0` and `do 5,0` all fell through to `low: null, high: null`. The string was stored, the reading was then judged against the biomarker's own band or against nothing at all, and no surface anywhere said that the lab had printed a window the app could not read. The vocabulary is TABULATED here, not derived from `messages/.json` the way an app-owned word list would be. These words are prose a lab printed on paper; the app owns no string they could be read out of, and the language of a report has nothing to do with the language of the UI. So the table is hand-written, keyed by the shipped `Locale` union so that adding a locale fails to compile until someone has answered how that language prints a range, and every locale's words are tried against every input at once. The cost is written down at the table: a report in a language absent from it stays text-only. Three things had to change beside the word lists. Introducing words are now stripped only on a second pass, because Italian "da", Polish "od" and French "de" also state a floor on their own and stripping first read "da 3,5" as a bare number. Input and table are diacritic-folded and their apostrophes normalised, so a table row written "jusqu'a" meets a report's "jusqu'à" and the curled apostrophe a word processor emits. And one-letter separators ("a", "e", "y") are only accepted with whitespace on both sides, so "3.5e5" is not read as a window. Decimal commas already worked and are pinned by the new cases. An unreadable window now has a name. `isUnreadableRange` separates "the report stated nothing" from "the report stated something nobody could read" — the two were indistinguishable at every call site, which is what let the French case pass unnoticed. The manual form already showed the reader that the text was kept but not read; the two ingest paths now count it into their wide event, so an unsupported notation surfaces as a number rather than as silence. The German and English lists are the previous alternation split word for word, and the existing suite covering them passes unchanged. --- src/components/labs/lab-form.tsx | 7 +- src/lib/documents/commit.ts | 19 +- src/lib/i18n/fold-for-match.ts | 59 +++ src/lib/i18n/shared-resolve.ts | 31 +- .../parse-reference-range-locales.test.ts | 197 ++++++++++ src/lib/labs/ocr-extract.ts | 15 +- src/lib/labs/parse-reference-range.ts | 359 ++++++++++++++++-- 7 files changed, 647 insertions(+), 40 deletions(-) create mode 100644 src/lib/i18n/fold-for-match.ts create mode 100644 src/lib/labs/__tests__/parse-reference-range-locales.test.ts diff --git a/src/components/labs/lab-form.tsx b/src/components/labs/lab-form.tsx index 6403444d8..f22d1efe8 100644 --- a/src/components/labs/lab-form.tsx +++ b/src/components/labs/lab-form.tsx @@ -22,7 +22,10 @@ import { import { Textarea } from "@/components/ui/textarea"; import { apiGet, apiPost } from "@/lib/api/api-fetch"; import { localizedApiError } from "@/lib/api/localized-error"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; import { formatReferenceRange } from "@/lib/labs/reference-range"; import { formatLabValue } from "@/lib/labs/format-value"; import { EncounterSuggestionField } from "@/components/encounters/encounter-suggestion-field"; @@ -300,7 +303,7 @@ export function LabForm({ // the hint says the text is kept as written rather than staying silent. const parsedSourceRange = parseReferenceRange(sourceRange, selected?.unit); const sourceRangeHint = parsedSourceRange - ? parsedSourceRange.low === null && parsedSourceRange.high === null + ? isUnreadableRange(parsedSourceRange) ? t("labs.form.sourceRangeUnparsed") : `${formatReferenceRange( parsedSourceRange.low, diff --git a/src/lib/documents/commit.ts b/src/lib/documents/commit.ts index b960b39b9..0cecca25a 100644 --- a/src/lib/documents/commit.ts +++ b/src/lib/documents/commit.ts @@ -28,7 +28,11 @@ import { invalidateUserHealthScore } from "@/lib/cache/invalidate"; import { emitDataArrival } from "@/lib/arrivals/emit-shared"; import { decryptFactData } from "@/lib/documents/store"; import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; +import { annotate } from "@/lib/logging/context"; import type { ExtractedFact } from "@/generated/prisma/client"; import type { ConditionFactData, @@ -82,6 +86,19 @@ async function commitObservation( const sourceLow = printed?.low ?? data.referenceLow; const sourceHigh = printed?.high ?? data.referenceHigh; const sourceText = printed?.text ?? null; + // A window the report printed and the parser could not read is the case that + // used to pass without a trace: the reading is then judged against the + // catalog band, or against nothing, and no surface says the lab had stated + // something else. Count it here so an unreadable notation — a language the + // prose table does not carry, a layout nobody anticipated — shows up as a + // number on a dashboard instead of as silence. The string itself is the + // user's document and stays out of the event. + if (isUnreadableRange(printed)) { + annotate({ + action: { name: "labs.referenceRange.unreadable" }, + meta: { surface: "document.commit", length: printed?.text.length ?? 0 }, + }); + } const biomarker = await resolveOrMintBiomarker(userId, { analyte: data.label, diff --git a/src/lib/i18n/fold-for-match.ts b/src/lib/i18n/fold-for-match.ts new file mode 100644 index 000000000..db0815e91 --- /dev/null +++ b/src/lib/i18n/fold-for-match.ts @@ -0,0 +1,59 @@ +/** + * Character folds used when app text has to be matched against text a PERSON + * or a LAB typed, rather than compared byte-for-byte. + * + * Every matcher that reads a shipped translation and looks for it in stored + * or transcribed text needs the same three tolerances — case, accents, and the + * apostrophe a word processor curled — and every matcher that grows its own + * copy of them drifts from the others. One home, so "Perte d'appétit" typed + * with a straight quote meets the bundle's "Perte d’appétit", and a French + * report's "jusqu'à" meets a table written "jusqu'a". + * + * The folds are one-directional and deliberately NOT transliteration: German + * "ue" does not fold onto "ü", "ß" is left alone (it has no canonical + * decomposition), and the Polish "ł" is a single codepoint NFD does not touch. + * That is fine — what has to agree is a shipped string with itself as a person + * retyped it, not two spellings of the same word. + * + * Client-safe: no message bundles are imported here, so a client component may + * use these. Building an index OVER the bundles is a server concern and lives + * in `./shared-resolve`. + */ + +/** Curly, modifier and backtick apostrophes a keyboard or word processor emits. */ +const APOSTROPHES = /[\u2018\u2019\u02bc\u00b4`']/gu; + +/** Combining marks left behind by an NFD decomposition. */ +const COMBINING_MARKS = /[\u0300-\u036f]/gu; + +/** + * Drop accents by decomposing and removing the combining marks: "é" → "e", + * "ż" → "z", "Ü" → "U". Punctuation, case and spacing survive untouched, so + * this is the fold to use where the surrounding characters carry meaning — a + * printed range's "3,5-5,0", a unit's "mg/dL". + */ +export function stripDiacritics(raw: string): string { + return raw.normalize("NFD").replace(COMBINING_MARKS, ""); +} + +/** Collapse every apostrophe variant onto the straight ASCII one. */ +export function normaliseApostrophes(raw: string): string { + return raw.replace(APOSTROPHES, "'"); +} + +/** + * Fold a label to its comparison form: lower case, no accents, and every + * apostrophe / hyphen / underscore / slash reduced to a single space. + * + * This is the fold for whole LABELS — a tag, an analyte name, a qualitative + * result term — where internal punctuation is noise. Apply it identically to + * both sides of a comparison; a fold applied to only one side is an + * exact-string match wearing a disguise. + */ +export function foldForMatch(raw: string): string { + return stripDiacritics(raw.toLowerCase()) + .replace(APOSTROPHES, " ") + .replace(/[-_/]/gu, " ") + .replace(/\s+/gu, " ") + .trim(); +} diff --git a/src/lib/i18n/shared-resolve.ts b/src/lib/i18n/shared-resolve.ts index f08790d05..fba5d019f 100644 --- a/src/lib/i18n/shared-resolve.ts +++ b/src/lib/i18n/shared-resolve.ts @@ -20,9 +20,10 @@ import frMessages from "../../../messages/fr.json"; import esMessages from "../../../messages/es.json"; import itMessages from "../../../messages/it.json"; import plMessages from "../../../messages/pl.json"; -import { type Locale } from "./config"; +import { locales, type Locale } from "./config"; +import { resolveKey } from "./resolve-key"; -export { resolveKey } from "./resolve-key"; +export { resolveKey }; /** * Single source of truth for the per-locale message bundle on the @@ -36,3 +37,29 @@ export const allMessages: Record> = { it: itMessages, pl: plMessages, }; + +/** + * Every shipped translation of one key, in `locales` order, deduplicated and + * with missing entries dropped. + * + * This is the primitive a DERIVED match index is built from. A matcher that + * has to recognise an app-owned label in text a person typed must not + * transcribe the label into its own source: transcribing fixes the locales + * shipped on the day it was written and reopens the same hole when a new + * bundle lands. Reading the bundles means a new locale is matchable the day + * `messages/.json` arrives, with no edit in the matcher. + * + * Known limit, worth stating where it bites: the values returned are what the + * bundles carry TODAY, while a matcher applies them to text written in the + * past. Editing a shipped translation therefore orphans whatever was written + * under the old wording — the fix for that is a legacy alias at the call site, + * not a data migration. + */ +export function localisedValues(key: string): string[] { + const seen = new Set(); + for (const locale of locales) { + const value = resolveKey(allMessages[locale], key); + if (value && value.trim()) seen.add(value.trim()); + } + return [...seen]; +} diff --git a/src/lib/labs/__tests__/parse-reference-range-locales.test.ts b/src/lib/labs/__tests__/parse-reference-range-locales.test.ts new file mode 100644 index 000000000..933bbce75 --- /dev/null +++ b/src/lib/labs/__tests__/parse-reference-range-locales.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from "vitest"; + +import { locales } from "@/lib/i18n/config"; +import { + isUnreadableRange, + parseReferenceRange, + RANGE_PROSE, +} from "@/lib/labs/parse-reference-range"; + +/** + * The notation a lab prints is the LAB's prose, not the app's, so this parser + * cannot derive its vocabulary from `messages/.json` the way a matcher + * over app-owned labels does. The table is hand-written, and these are the + * strings it has to read: real reference-range notation as French, Spanish, + * Italian and Polish reports print it, decimal comma included. + * + * Before the table existed, every case below returned `low: null, high: null` + * — the string was stored and the reading was then judged against a band the + * lab never printed, or against no band at all, with nothing anywhere saying + * so. + */ +describe("parseReferenceRange — French notation", () => { + it.each([ + ["jusqu'à 5,0", null, 5], + ["jusqu’à 5,0", null, 5], + ["JUSQU'À 5,0", null, 5], + ["inférieur à 5,0", null, 5], + ["moins de 5,0", null, 5], + ["à partir de 3,5", 3.5, null], + ["supérieur à 3,5", 3.5, null], + ["au moins 3,5", 3.5, null], + ["3,5 à 5,0", 3.5, 5], + ["de 3,5 à 5,0", 3.5, 5], + ["entre 3,5 et 5,0", 3.5, 5], + ["valeurs de référence 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — Spanish notation", () => { + it.each([ + ["hasta 5,0", null, 5], + ["menor de 5,0", null, 5], + ["inferior a 5,0", null, 5], + ["como máximo 5,0", null, 5], + ["desde 3,5", 3.5, null], + ["mayor que 3,5", 3.5, null], + ["superior a 3,5", 3.5, null], + ["más de 3,5", 3.5, null], + ["3,5 a 5,0", 3.5, 5], + ["de 3,5 a 5,0", 3.5, 5], + ["entre 3,5 y 5,0", 3.5, 5], + ["valores de referencia 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — Italian notation", () => { + it.each([ + ["fino a 5,0", null, 5], + ["inferiore a 5,0", null, 5], + ["meno di 5,0", null, 5], + ["oltre 3,5", 3.5, null], + ["superiore a 3,5", 3.5, null], + ["più di 3,5", 3.5, null], + ["almeno 3,5", 3.5, null], + ["3,5 a 5,0", 3.5, 5], + ["da 3,5 a 5,0", 3.5, 5], + ["tra 3,5 e 5,0", 3.5, 5], + ["valori di riferimento 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); + + it("reads a bare Italian floor rather than swallowing its introducing word", () => { + // "da" both introduces a two-sided window ("da 3,5 a 5,0") and states a + // floor on its own. Stripping introducing words before matching — which is + // what the parser used to do unconditionally — turns the second form into + // a bare number with no bound at all. + expect(parseReferenceRange("da 3,5")).toEqual({ + low: 3.5, + high: null, + text: "da 3,5", + }); + }); +}); + +describe("parseReferenceRange — Polish notation", () => { + it.each([ + ["do 5,0", null, 5], + ["poniżej 5,0", null, 5], + ["mniej niż 5,0", null, 5], + ["maks. 5,0", null, 5], + ["od 3,5", 3.5, null], + ["powyżej 3,5", 3.5, null], + ["co najmniej 3,5", 3.5, null], + ["3,5 do 5,0", 3.5, 5], + ["od 3,5 do 5,0", 3.5, 5], + ["zakres referencyjny 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — the new prose does not loosen the old rules", () => { + it("still refuses a bare number in any language", () => { + expect(parseReferenceRange("5,0")).toEqual({ + low: null, + high: null, + text: "5,0", + }); + }); + + it("does not read scientific notation as a two-sided window", () => { + // "e" is an Italian separator ("tra 3,5 e 5,0") and "y" a Spanish one. + // Both are one letter, so they are only accepted with whitespace on both + // sides — otherwise "3.5e5" would parse as a 3.5–5 window. + expect(parseReferenceRange("3.5e5")).toEqual({ + low: null, + high: null, + text: "3.5e5", + }); + }); + + it("still refuses a transposed window printed in French", () => { + expect(parseReferenceRange("de 5,0 à 3,5")).toEqual({ + low: null, + high: null, + text: "de 5,0 à 3,5", + }); + }); + + it("still refuses bounds when the printed unit contradicts the reading", () => { + expect(parseReferenceRange("jusqu'à 5,0 mmol/l", "mg/dL")).toEqual({ + low: null, + high: null, + text: "jusqu'à 5,0 mmol/l", + }); + }); +}); + +describe("parseReferenceRange — an unreadable window is named, not silent", () => { + it("separates 'nothing printed' from 'printed and not read'", () => { + expect(parseReferenceRange(null)).toBeNull(); + expect(isUnreadableRange(parseReferenceRange(null))).toBe(false); + + const prose = parseReferenceRange("voir le compte rendu"); + expect(prose).toEqual({ + low: null, + high: null, + text: "voir le compte rendu", + }); + expect(isUnreadableRange(prose)).toBe(true); + }); + + it("reports a window it DID read as readable", () => { + expect(isUnreadableRange(parseReferenceRange("jusqu'à 5,0"))).toBe(false); + }); +}); + +describe("RANGE_PROSE coverage", () => { + it("carries an entry for every shipped locale", () => { + // Keyed by the `Locale` union on purpose: a seventh locale cannot be added + // to `locales` without someone answering how that language prints a range. + // This asserts the runtime side of the same promise. + expect(Object.keys(RANGE_PROSE).sort()).toEqual([...locales].sort()); + }); + + it("states at least one word for each notation in each locale", () => { + for (const locale of locales) { + const prose = RANGE_PROSE[locale]; + expect(prose.twoSided.length, `${locale}.twoSided`).toBeGreaterThan(0); + expect(prose.upper.length, `${locale}.upper`).toBeGreaterThan(0); + expect(prose.lower.length, `${locale}.lower`).toBeGreaterThan(0); + } + }); + + it("carries the table already folded, so both sides of a match agree", () => { + // A row written with an accent would never match: the input is + // diacritic-folded before it reaches the alternation, so an accented row + // is a row that can only ever miss. + for (const locale of locales) { + const prose = RANGE_PROSE[locale]; + for (const word of [ + ...prose.twoSided, + ...prose.upper, + ...prose.lower, + ...prose.filler, + ]) { + expect(word.normalize("NFD"), `${locale}: ${word}`).not.toMatch(/[̀-ͯ]/u); + expect(word, `${locale}: ${word}`).toBe(word.toLowerCase()); + } + } + }); +}); diff --git a/src/lib/labs/ocr-extract.ts b/src/lib/labs/ocr-extract.ts index fcdb01563..4329211b8 100644 --- a/src/lib/labs/ocr-extract.ts +++ b/src/lib/labs/ocr-extract.ts @@ -24,7 +24,10 @@ import { } from "@/lib/ai/types"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { prisma } from "@/lib/db"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; import { annotate } from "@/lib/logging/context"; import { extractedLabsSchema, @@ -270,7 +273,16 @@ export async function runOcrExtraction( // Cap defensively (the schema already bounds it) and annotate each row. const rows: OcrExtractedRowDto[] = []; + // How many rows printed a window the parser could not read. Those rows fall + // back to whatever bounds the model itself reported, or to none — either way + // the notation went unread, and a count of it is the difference between + // noticing that a report layout or a language is unsupported and never + // hearing about it. A count only: the string is the user's document. + let unreadableRanges = 0; for (const row of envelope.rows.slice(0, OCR_MAX_ROWS)) { + if (isUnreadableRange(parseReferenceRange(row.referenceText, row.unit))) { + unreadableRanges += 1; + } rows.push(await annotateRow(args.userId, row, reportDate)); } @@ -282,6 +294,7 @@ export async function runOcrExtraction( rows: rows.length, duplicates: rows.filter((r) => r.duplicateOf !== null).length, newBiomarkers: rows.filter((r) => r.biomarkerMatch === "new").length, + unreadableRanges, }, }); diff --git a/src/lib/labs/parse-reference-range.ts b/src/lib/labs/parse-reference-range.ts index 24e413f37..f16765ff6 100644 --- a/src/lib/labs/parse-reference-range.ts +++ b/src/lib/labs/parse-reference-range.ts @@ -19,14 +19,31 @@ * plausible bound out of a string it did not understand, because a fake * window silently reclassifies a reading as normal or abnormal. * - * Notations read (German and English, comma- or dot-decimal): + * Notations read (comma- or dot-decimal): * * two-sided 3.5-5.0 · 3,5 – 5,0 · 3,5 bis 5,0 · 3.5 to 5.0 · 3,5 ... 5,0 - * von 3,5 bis 5,0 · [3,5 - 5,0] · (3.5-5.0) + * von 3,5 bis 5,0 · 3,5 à 5,0 · de 3,5 a 5,0 · da 3,5 a 5,0 · + * od 3,5 do 5,0 · [3,5 - 5,0] · (3.5-5.0) * upper only < 5 · ≤ 5 · <= 5 · bis 5,0 · bis zu 5,0 · max. 5,0 · - * kleiner 5 · unter 5 · up to 5.0 · less than 5 + * kleiner 5 · unter 5 · up to 5.0 · less than 5 · jusqu'à 5,0 · + * hasta 5,0 · fino a 5,0 · do 5,0 · poniżej 5,0 * lower only > 3.5 · ≥ 3,5 · >= 3.5 · ab 3,5 · min. 3,5 · mindestens 3,5 · - * größer 3,5 · über 3,5 · at least 3.5 + * größer 3,5 · über 3,5 · at least 3.5 · à partir de 3,5 · + * desde 3,5 · da 3,5 · od 3,5 · powyżej 3,5 + * + * WHY THE VOCABULARY IS TABULATED AND NOT DERIVED. Elsewhere in the app, a + * matcher that has to recognise a word list in six languages builds its index + * out of `messages/.json`, so a seventh locale works the day its + * bundle lands. That does not apply here. "jusqu'à", "hasta", "fino a" and + * "do" are prose a LAB printed on a sheet of paper; the app owns no string + * they could be derived from, and the language of the report has nothing to do + * with the language of the UI — a French self-hoster running the app in + * English still receives French reports. So the table below is hand-written, + * every locale's prose is tried against every input at once, and the honest + * cost is written down: a report printed in a language absent from the table + * yields text-only. `RANGE_PROSE` is keyed by the shipped `Locale` union + * precisely so that adding a locale to `locales` fails to compile until + * someone has answered "and how does that language print a range?". * * Units are read, never converted. A window may carry the unit the report * printed it in; when the caller states the reading's own unit and the two @@ -34,6 +51,11 @@ * mmol/L into mg/dL needs a per-analyte factor, so doing it here would be the * parser deciding what the report meant. */ +import { type Locale } from "@/lib/i18n/config"; +import { + normaliseApostrophes, + stripDiacritics, +} from "@/lib/i18n/fold-for-match"; /** What a printed range yielded. `text` is set whenever the input was non-empty. */ export interface ParsedReferenceRange { @@ -58,14 +80,224 @@ const MAX_ABS_BOUND = 1e12; /** One number token: optional sign, digits, any number of `.`/`,` groups. */ const NUM = String.raw`[+-]?\d+(?:[.,]\d+)*`; -/** Infix separators for a two-sided window. */ -const TWO_SIDED_SEP = String.raw`(?:-|–|—|‒|…|\.{2,3}|bis\s+zu|bis|to|until)`; +/** How one language prints a range. Every field is a list of literal words. */ +export interface RangeProse { + /** Infix words joining two bounds: "3,5 5,0". */ + twoSided: string[]; + /** Words that introduce a ceiling: " 5,0". */ + upper: string[]; + /** Words that introduce a floor: " 3,5". */ + lower: string[]; + /** + * Words that only announce that a window follows and carry no bound of + * their own ("von 3,5 bis 5,0"). Stripped and the input retried — never on + * the first pass, because several of them ("da", "od", "de") are also a + * floor prefix in their own language, and stripping first would read + * "da 3,5" as a bare number. + */ + filler: string[]; +} + +/** + * The prose, per shipped locale. Written ALREADY FOLDED — lower case and + * accent-free — because the input is diacritic-folded before matching, so + * "jusqu'à", "jusqu'a" and "JUSQU'À" are one row. The German "ß" survives the + * fold (it has no canonical decomposition), which is why "größer" appears here + * as "großer"; its "oe"/"ss" twins are listed beside it, exactly as the + * hand-written alternation carried them before. + * + * The `de` and `en` lists are the pre-existing alternation, split apart word + * for word and nothing more — this change is about the four locales that had + * no entry at all, and a German or English range must read the same afterwards + * as it did before. + * + * Every list is applied to every input regardless of the reader's locale — see + * the module comment for why. Multi-word entries match across any run of + * whitespace, so a report that printed a non-breaking space still reads. + */ +export const RANGE_PROSE: Record = { + de: { + twoSided: ["bis zu", "bis"], + upper: [ + "bis zu", + "bis", + "max.", + "maximal", + "max", + "kleiner als", + "kleiner", + "unter", + "hochstens", + "hoechstens", + ], + lower: [ + "ab", + "mind.", + "mindestens", + "minimal", + "min.", + "min", + "großer als", + "großer", + "groesser als", + "groesser", + "uber", + "ueber", + ], + filler: ["von", "referenz", "ref.", "ref"], + }, + en: { + twoSided: ["to", "until"], + upper: ["up to", "less than", "below"], + lower: ["at least", "greater than", "above"], + filler: ["from", "reference", "ref.", "ref"], + }, + fr: { + twoSided: ["a", "et"], + upper: [ + "jusqu'a", + "jusque", + "inferieur a", + "inferieure a", + "moins de", + "au plus", + "au maximum", + ], + lower: [ + "a partir de", + "superieur a", + "superieure a", + "plus de", + "au moins", + "au minimum", + ], + filler: [ + "de", + "entre", + "valeurs de reference", + "valeur de reference", + "intervalle de reference", + "plage de reference", + ], + }, + es: { + twoSided: ["a", "y"], + upper: [ + "hasta", + "menor que", + "menor de", + "menor a", + "inferior a", + "menos de", + "como maximo", + "maximo", + ], + lower: [ + "desde", + "a partir de", + "mayor que", + "mayor de", + "mayor a", + "superior a", + "mas de", + "al menos", + "como minimo", + "minimo", + ], + filler: [ + "de", + "entre", + "valores de referencia", + "valor de referencia", + "rango de referencia", + "intervalo de referencia", + ], + }, + it: { + twoSided: ["a", "e"], + upper: [ + "fino a", + "inferiore a", + "minore di", + "meno di", + "al massimo", + "massimo", + ], + lower: [ + "da", + "oltre", + "superiore a", + "maggiore di", + "piu di", + "almeno", + "al minimo", + "minimo", + ], + filler: [ + "da", + "tra", + "fra", + "valori di riferimento", + "valore di riferimento", + "intervallo di riferimento", + ], + }, + pl: { + twoSided: ["do"], + upper: ["do", "ponizej", "mniej niz", "nie wiecej niz", "maks.", "maks"], + lower: [ + "od", + "powyzej", + "wiecej niz", + "co najmniej", + "nie mniej niz", + "minimum", + ], + filler: [ + "od", + "zakres referencyjny", + "wartosci referencyjne", + "normy laboratoryjne", + ], + }, +}; + +/** Escape a literal for regex use and let any run of whitespace join its words. */ +function literalToPattern(word: string): string { + return word + .split(/\s+/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join(String.raw`\s+`); +} + +/** + * Alternation over every locale's list for one field, longest literal first so + * the reader sees the specific form ahead of the prefix it contains. + */ +function proseAlternation(field: keyof RangeProse): string { + const words = new Set(); + for (const prose of Object.values(RANGE_PROSE)) { + for (const word of prose[field]) words.add(word); + } + return [...words] + .sort((a, b) => b.length - a.length) + .map(literalToPattern) + .join("|"); +} + +/** + * Symbolic infix separators. Kept apart from the word separators because they + * need no surrounding whitespace, while a word separator must have it: "a", + * "e" and "y" are one letter long, and a bare `e` that could sit between two + * digits would read "3.5e5" as a window. + */ +const SYMBOL_SEP = String.raw`-|–|—|‒|…|\.{2,3}`; /** Prefixes that state a ceiling only. */ -const UPPER_PREFIX = String.raw`(?:<=|<|≤|bis\s+zu|bis|max\.|maximal|max|kleiner\s+als|kleiner|unter|höchstens|hoechstens|up\s+to|less\s+than|below)`; +const UPPER_PREFIX = `(?:<=|<|≤|${proseAlternation("upper")})`; /** Prefixes that state a floor only. */ -const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|min|größer\s+als|größer|groesser\s+als|groesser|über|ueber|at\s+least|greater\s+than|above)`; +const LOWER_PREFIX = `(?:>=|>|≥|${proseAlternation("lower")})`; /** * Trailing unit tail. Deliberately whitespace-free and comma-free: a real unit @@ -76,14 +308,19 @@ const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|m const UNIT_TAIL = String.raw`(?:\s*([^\s,]{1,24}))?`; const TWO_SIDED_RE = new RegExp( - `^(${NUM})\\s*${TWO_SIDED_SEP}\\s*(${NUM})${UNIT_TAIL}$`, + `^(${NUM})(?:\\s*(?:${SYMBOL_SEP})\\s*|\\s+(?:${proseAlternation( + "twoSided", + )})\\s+)(${NUM})${UNIT_TAIL}$`, "iu", ); const UPPER_RE = new RegExp(`^${UPPER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu"); const LOWER_RE = new RegExp(`^${LOWER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu"); /** Leading words that only introduce the window ("von 3,5 bis 5,0"). */ -const LEADING_FILLER_RE = /^(?:von|from|ref\.?|referenz|reference)\s+/iu; +const LEADING_FILLER_RE = new RegExp( + `^(?:${proseAlternation("filler")})\\s+`, + "iu", +); /** * Read one number token into a JS number, or null when its separators are @@ -148,33 +385,18 @@ function unitsAgree(printed: string | undefined, reading: string | undefined) { } /** - * Parse the reference range a lab report printed for one reading. - * - * @param raw the printed string, exactly as transcribed. - * @param unit the reading's own unit, when known. A printed window in a - * different unit yields text-only — never a converted bound. - * @returns null when there was nothing printed at all. + * Try the three notations against one candidate string. Null means none of + * them matched; a result with both bounds null means one matched but the + * numbers or the unit refused it, which is a final answer and not a reason to + * retry a differently-trimmed candidate. */ -export function parseReferenceRange( - raw: string | null | undefined, - unit?: string | null, +function matchNotation( + candidate: string, + text: string, + unit: string | null | undefined, ): ParsedReferenceRange | null { - if (raw === null || raw === undefined) return null; - const text = raw - .replace(/\s+/gu, " ") - .trim() - .slice(0, SOURCE_REFERENCE_TEXT_MAX); - if (text === "") return null; - const noBounds: ParsedReferenceRange = { low: null, high: null, text }; - // Strip a bracket wrapper and an introducing word for MATCHING only — the - // stored text stays exactly what the report printed. - let candidate = text; - const bracketed = /^[[(](.*)[\])]$/u.exec(candidate); - if (bracketed) candidate = bracketed[1].trim(); - candidate = candidate.replace(LEADING_FILLER_RE, "").trim(); - const two = TWO_SIDED_RE.exec(candidate); if (two) { if (!unitsAgree(two[3], unit ?? undefined)) return noBounds; @@ -201,7 +423,76 @@ export function parseReferenceRange( return low === null ? noBounds : { low, high: null, text }; } + return null; +} + +/** + * Parse the reference range a lab report printed for one reading. + * + * @param raw the printed string, exactly as transcribed. + * @param unit the reading's own unit, when known. A printed window in a + * different unit yields text-only — never a converted bound. + * @returns null when there was nothing printed at all. A non-null result with + * both bounds null means the report DID print something the parser + * could not read — see `isUnreadableRange`, which is the difference + * between "no window" and "a window nobody checked". + */ +export function parseReferenceRange( + raw: string | null | undefined, + unit?: string | null, +): ParsedReferenceRange | null { + if (raw === null || raw === undefined) return null; + const text = raw + .replace(/\s+/gu, " ") + .trim() + .slice(0, SOURCE_REFERENCE_TEXT_MAX); + if (text === "") return null; + + // Fold for MATCHING only — the stored text stays exactly what the report + // printed. Accents go so a table written "jusqu'a" meets a report's + // "jusqu'à"; the curled apostrophe a word processor emits becomes the + // straight one the table carries. Everything else, notably the digits and + // their separators, is left alone. + let candidate = normaliseApostrophes(stripDiacritics(text)); + const bracketed = /^[[(](.*)[\])]$/u.exec(candidate); + if (bracketed) candidate = bracketed[1].trim(); + + // Untrimmed first. Several introducing words are also a bound prefix in + // their own language — Italian "da", Polish "od", French "de" — so stripping + // before matching would read "da 3,5" as a bare number with no floor. + const direct = matchNotation(candidate, text, unit); + if (direct) return direct; + + const trimmed = candidate.replace(LEADING_FILLER_RE, "").trim(); + if (trimmed !== candidate) { + const viaFiller = matchNotation(trimmed, text, unit); + if (viaFiller) return viaFiller; + } + // Anything else — "negativ", "siehe Befund", a bare number, prose — keeps its // text and states no window. - return noBounds; + return { low: null, high: null, text }; +} + +/** + * True when the report printed a range and the parser could not derive a + * single bound from it. + * + * This is the case worth naming. `null` from the parser means the report + * stated nothing, and the reading falls back to the biomarker's own band as it + * always did. A parsed result with two null bounds means something else + * entirely: the lab DID state a window, on this reading, and the app is about + * to judge the value against a different band or against none at all. Left + * unnamed it reads identically to "no range" at every call site, which is how + * a French account's `jusqu'à 5,0` became a reading nobody ever flagged. + * + * Callers are expected to SAY so rather than to guess a bound — the manual + * form shows the reader that the string was kept but not read, and the ingest + * paths annotate it so an unreadable notation shows up as a number in the wide + * event instead of as silence. + */ +export function isUnreadableRange( + parsed: ParsedReferenceRange | null | undefined, +): boolean { + return !!parsed && parsed.low === null && parsed.high === null; } From ce7372991661696c5f15578cbf06e88f4d917a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 23 Aug 2026 14:19:18 +0200 Subject: [PATCH 2/5] fix(fhir): resolve a lab analyte's LOINC code from its name in any shipped locale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A biomarker minted from the labs catalog stores `t("labs.catalog.")` — the name in the language of whoever minted it — and that stored name is what reaches the FHIR exporter as free text. The exporter matched it against an alias table written in English and German. So an account working in French, Spanish, Italian or Polish exported "Cholestérol total", "Glucosa en ayunas", "Glicemia a digiuno" and "Cholesterol całkowity" as text-only concepts with no LOINC code, on the one document whose whole purpose is being read by another system. The names are DERIVED from `messages/.json` now, not transcribed into this file. Both sides of the new slug map are app-owned identities with no language in them, which is what makes it the one edit a new locale does not need: `locales` grows, `messages/xx.json` lands, and the analyte names it ships resolve the same day. Transcribing four more languages would have fixed today's six and reopened the hole at the seventh. The hand-written aliases keep precedence. They cover what the app owns no string for — clinical abbreviations, the ways a person shortens a name — and they were verified one at a time over several releases. A derived name landing on the same key is a harmless duplicate; one landing on a different key would be a translation typo re-pointing a verified mapping at another analyte, so the build refuses it and the guard suite asserts the refusal held. The lookup fold now drops accents before stripping punctuation. Without it a name carrying diacritics lost them entirely rather than reducing to its base letters — "Cholestérol total" folded to `cholestroltotal`, and "Hämoglobin" to `hmoglobin`. That last one means the German umlaut spellings never resolved either: the hand-written `hamoglobin` and `nuchternglucose` aliases only ever caught the un-umlauted typing. Every pre-existing alias is ASCII, so the fold is a widening and what resolved before resolves to the same key. --- .../fhir/__tests__/lab-loinc-locales.test.ts | 135 ++++++++++++++++++ src/lib/fhir/lab-loinc.ts | 135 ++++++++++++++++-- 2 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 src/lib/fhir/__tests__/lab-loinc-locales.test.ts diff --git a/src/lib/fhir/__tests__/lab-loinc-locales.test.ts b/src/lib/fhir/__tests__/lab-loinc-locales.test.ts new file mode 100644 index 000000000..2e4363e45 --- /dev/null +++ b/src/lib/fhir/__tests__/lab-loinc-locales.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import { + CATALOG_SLUG_TO_LOINC_KEY, + localisedLabAliases, + normaliseLabKey, + resolveLabCoding, +} from "@/lib/fhir/lab-loinc"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; +import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog"; + +/** + * A biomarker minted from the labs catalog stores `t("labs.catalog.")` + * — the name in the language of whoever minted it — and that stored name is + * what reaches the FHIR exporter as free text. The alias table it was matched + * against was written in English and German, so four of the six shipped + * locales exported every one of these analytes as a text-only concept with no + * LOINC code. + */ +describe("resolveLabCoding — the analyte name in every shipped locale", () => { + it.each([ + ["Cholestérol total", "2093-3"], + ["Colesterol total", "2093-3"], + ["Colesterolo totale", "2093-3"], + ["Cholesterol całkowity", "2093-3"], + ["Créatinine", "2160-0"], + ["Creatinina", "2160-0"], + ["Kreatynina", "2160-0"], + ["Glycémie à jeun", "1558-6"], + ["Glucosa en ayunas", "1558-6"], + ["Glicemia a digiuno", "1558-6"], + ["Glukoza na czczo", "1558-6"], + ["Ferritine", "2276-4"], + ["Ferrytyna", "2276-4"], + ["Hémoglobine", "718-7"], + ["Emoglobina", "718-7"], + ["Cholestérol LDL", "18262-6"], + ["Colesterol HDL", "2085-9"], + ["Triglycérides", "2571-8"], + ["Trójglicerydy", "2571-8"], + ["Insuline à jeun", "27873-9"], + ["Lipoprotéine(a)", "43583-4"], + ])("codes %s as LOINC %s", (analyte, loinc) => { + expect(resolveLabCoding(analyte, "")?.loinc).toBe(loinc); + }); + + it("derives from the bundles rather than from a transcribed list", () => { + // The property that matters: every locale contributes. A hand-written + // table would pass the cases above and still be frozen at the languages + // somebody happened to type. + for (const locale of locales) { + const name = resolveKey( + allMessages[locale], + "labs.catalog.total-cholesterol", + ); + expect(name, `${locale} has no catalog name`).toBeTruthy(); + expect( + resolveLabCoding(name as string, "")?.loinc, + `${locale}: ${name}`, + ).toBe("2093-3"); + } + }); + + it("stamps the canonical UCUM when the recorded unit matches", () => { + expect(resolveLabCoding("Cholestérol total", "mg/dL")).toEqual({ + loinc: "2093-3", + display: "Cholesterol [Mass/volume] in Serum or Plasma", + ucum: "mg/dL", + }); + }); + + it("still keeps an unmapped analyte text-only", () => { + // Conservative by design: a name the app does not ship and no alias names + // gets no fabricated code, in any language. + expect(resolveLabCoding("Analyse inconnue", "mg/dL")).toBeNull(); + }); +}); + +describe("resolveLabCoding — the existing English and German behaviour", () => { + it.each([ + ["HbA1c", "4548-4"], + ["LDL-C", "18262-6"], + ["LDL Cholesterol", "18262-6"], + ["Gesamtcholesterin", "2093-3"], + ["Kreatinin", "2160-0"], + ["GPT", "1742-6"], + ["ASAT", "1920-8"], + ["hgb", "718-7"], + ["Nüchternglucose", "1558-6"], + ["Thyrotropin", "3016-3"], + ["Apolipoprotein B", "1884-6"], + ])("still codes %s as LOINC %s", (analyte, loinc) => { + expect(resolveLabCoding(analyte, "")?.loinc).toBe(loinc); + }); + + it("resolves a German name whose umlaut the old key fold destroyed", () => { + // "Hämoglobin" used to fold to `hmoglobin`, which matched nothing — the + // hand-written `hamoglobin` alias only ever caught the un-umlauted typing. + expect(normaliseLabKey("Hämoglobin")).toBe("hamoglobin"); + expect(resolveLabCoding("Hämoglobin", "g/dL")?.loinc).toBe("718-7"); + }); +}); + +describe("the derived alias index", () => { + it("contradicts no hand-written alias", () => { + // A derived name that resolved to a DIFFERENT analyte than a verified + // alias would be a bundle bug re-pointing an existing mapping. The build + // refuses it; this asserts the refusal held rather than that it exists. + for (const [name, key] of Object.entries(localisedLabAliases())) { + const viaHand = resolveLabCoding(name, ""); + expect(viaHand, name).not.toBeNull(); + expect(viaHand?.loinc, name).toBe(resolveLabCoding(key, "")?.loinc); + } + }); + + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(Object.keys(localisedLabAliases()).length).toBeGreaterThan(20); + }); + + it("maps only slugs the labs catalog actually ships, in every locale", () => { + // A renamed slug or a bundle that dropped a name would silently stop + // deriving: the index would shrink and nothing else would say so. + const slugs = new Set(BIOMARKER_CATALOG.map((s) => s.slug)); + for (const slug of Object.keys(CATALOG_SLUG_TO_LOINC_KEY)) { + expect(slugs.has(slug), `catalog lost slug ${slug}`).toBe(true); + for (const locale of locales) { + expect( + resolveKey(allMessages[locale], `labs.catalog.${slug}`), + `${locale} has no name for ${slug}`, + ).toBeTruthy(); + } + } + }); +}); diff --git a/src/lib/fhir/lab-loinc.ts b/src/lib/fhir/lab-loinc.ts index 17eaf927f..0e748d0f9 100644 --- a/src/lib/fhir/lab-loinc.ts +++ b/src/lib/fhir/lab-loinc.ts @@ -30,9 +30,16 @@ * yet carry that field, so the call site passes only `analyte` today. When the * DTO gains a `biomarkerCanonicalName` (or similar), thread it in — no other * change to this module is required. + * + * SERVER-ONLY since the localised alias index landed: the index reads every + * shipped message bundle through `@/lib/i18n/shared-resolve`. The single + * caller is the FHIR exporter, which runs on the server; a client that needs + * to show a biomarker's name resolves it through the labs catalog instead. */ import { LOINC_SYSTEM, UCUM_SYSTEM } from "@/lib/fhir/loinc-map"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; export { LOINC_SYSTEM, UCUM_SYSTEM }; @@ -45,13 +52,20 @@ export interface LabLoincMapping { } /** - * Normalise an analyte / unit string to a lookup key: lower-case, strip every - * character that is not a letter or digit. So "LDL-C", "LDL Cholesterol", - * "ldl_c" all fold to `ldlc`-ish keys; the alias table below pins the exact - * variants the app and German users actually type onto a single canonical key. + * Normalise an analyte / unit string to a lookup key: lower-case, drop + * accents, strip every character that is not a letter or digit. So "LDL-C", + * "LDL Cholesterol", "ldl_c" all fold to `ldlc`-ish keys. + * + * The accent fold is what lets a name carrying diacritics reach a key at all. + * Without it "Cholestérol total" lost its "é" entirely and folded to + * `cholestroltotal`, and "Hämoglobin" to `hmoglobin` — keys nothing was ever + * going to match, which is also why the hand-written German aliases below are + * spelled `hamoglobin` and `kreatinin` rather than as the language writes + * them. Every pre-existing alias is pure ASCII, so folding is a widening: what + * resolved before resolves to the same key now. */ export function normaliseLabKey(s: string): string { - return s.toLowerCase().replace(/[^a-z0-9]/g, ""); + return stripDiacritics(s.toLowerCase()).replace(/[^a-z0-9]/g, ""); } /** @@ -172,11 +186,17 @@ const LAB_LOINC_BY_KEY: Record = { }; /** - * Alias → canonical key. The LEFT side is the normalised form of a name a user - * (EN or DE) might type; the RIGHT side is a key in `LAB_LOINC_BY_KEY`. Every - * canonical key is also implicitly its own alias (looked up directly first). - * Keep the right side honest — only fold a synonym when it unambiguously names - * the same analyte. + * Alias → canonical key, hand-written. The LEFT side is the normalised form of + * a name a user might type; the RIGHT side is a key in `LAB_LOINC_BY_KEY`. + * Every canonical key is also implicitly its own alias (looked up directly + * first). Keep the right side honest — only fold a synonym when it + * unambiguously names the same analyte. + * + * This table covers spellings the app owns NO string for: clinical + * abbreviations ("GPT", "ASAT", "hgb"), and the ways a person shortens a name + * by hand. The NAMES the app itself displays are not transcribed here — see + * the derived index below — because that list exists in six languages already + * and copying it in would freeze it at today's six. */ const LAB_ALIASES: Record = { // HbA1c @@ -252,6 +272,94 @@ const LAB_ALIASES: Record = { nuchterninsulin: "fastinginsulin", }; +/** + * Labs-catalog slug → canonical LOINC key. + * + * Both sides are app-owned stable identities with no language in them, which + * is the whole point: this is the only edit a new locale does NOT need. A + * biomarker minted from the catalog stores whatever `labs.catalog.` said + * in the language of the person who minted it, and that name is what reaches + * the FHIR exporter as free text. Pinning the slug once means every + * translation of it resolves. + * + * Slugs absent here are catalog markers with no entry in `LAB_LOINC_BY_KEY` + * (T3, T4, B12, folate, the electrolytes, most of the blood count) — that map + * is conservative by design and text-only stays the honest default for them. + * Adding one is a decision about a LOINC term, not about a language. + */ +export const CATALOG_SLUG_TO_LOINC_KEY: Record = { + "total-cholesterol": "cholesterol", + ldl: "ldlc", + hdl: "hdlc", + triglycerides: "triglycerides", + "fasting-glucose": "glucosefasting", + hba1c: "hba1c", + tsh: "tsh", + ferritin: "ferritin", + "vitamin-d": "vitamind", + "hs-crp": "hscrp", + creatinine: "creatinine", + egfr: "egfr", + alt: "alt", + ast: "ast", + ggt: "ggt", + hemoglobin: "hemoglobin", + apob: "apob", + "lp-a": "lpa", + "fasting-insulin": "fastinginsulin", +}; + +/** + * Localised analyte name → canonical LOINC key, DERIVED from + * `messages/.json` rather than transcribed out of it. + * + * That is the load-bearing part. Before this index, the alias table above was + * the whole of the mapping and it was written in English and German, so a + * French account's "Cholestérol total", a Spanish "Colesterol total", an + * Italian "Colesterolo totale" and a Polish "Cholesterol całkowity" all + * exported as a text-only concept with no LOINC code, on a document a + * clinician or another system reads. Transcribing four more languages here + * would have fixed today's six locales and reopened the same hole on the day a + * seventh landed. Reading the bundles means a new locale needs no edit in this + * file: `locales` grows, `messages/xx.json` lands, and the names it ships + * resolve the same day. + * + * The hand-written aliases WIN on a conflict. They were verified one by one + * over several releases; a derived name that folds onto the same key as one of + * them is a duplicate and harmless, and one that folds onto a DIFFERENT key + * would be a bundle bug, so it is refused here and the guard suite fails on + * it — a translation typo must not quietly re-point an existing alias at + * another analyte. + * + * Known limit: the index is built from the names the bundles carry TODAY, + * while the analytes it resolves were typed in the past. Editing a shipped + * `labs.catalog.` string orphans the biomarkers minted under the old + * wording; the fix for that is a legacy row in `LAB_ALIASES`, not a migration. + */ +function buildLocalisedAliases(): Record { + const index: Record = {}; + for (const [slug, loincKey] of Object.entries(CATALOG_SLUG_TO_LOINC_KEY)) { + for (const name of localisedValues(`labs.catalog.${slug}`)) { + const key = normaliseLabKey(name); + if (!key) continue; + if (LAB_LOINC_BY_KEY[key] || LAB_ALIASES[key]) continue; + if (index[key] && index[key] !== loincKey) continue; + index[key] = loincKey; + } + } + return index; +} + +const LOCALISED_LAB_ALIASES = buildLocalisedAliases(); + +/** + * Test seam: the guard suite asserts the derived index covers every shipped + * locale and contradicts no hand-written alias. Production reads the constant. + */ +export function localisedLabAliases(): Readonly> { + return LOCALISED_LAB_ALIASES; +} + /** * Canonical UCUM symbols that a recorded free-text unit may normalise to. The * LEFT is the normalised (lower-cased, punctuation-stripped) recorded unit; @@ -344,7 +452,12 @@ export function resolveLabCoding( const source = canonicalName && canonicalName.length > 0 ? canonicalName : analyte; const key = normaliseLabKey(source); - const resolvedKey = LAB_LOINC_BY_KEY[key] ? key : LAB_ALIASES[key]; + // Canonical key first, then the hand-verified aliases, then the names the + // shipped bundles carry. The order is the confidence order, and it keeps + // every resolution that worked before working the same way. + const resolvedKey = LAB_LOINC_BY_KEY[key] + ? key + : (LAB_ALIASES[key] ?? LOCALISED_LAB_ALIASES[key]); if (!resolvedKey) return null; const mapping = LAB_LOINC_BY_KEY[resolvedKey]; if (!mapping) return null; From c5a01767aaf7fe83d1abf5faae7549d0ed14082c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 23 Aug 2026 14:21:34 +0200 Subject: [PATCH 3/5] fix(fhir): code a qualitative lab result recorded in any shipped locale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lab form offers "negative" and "positive" as quick picks and writes the label it rendered, so the column holds the bundle's own string in the language the reading was recorded in. The SNOMED mapper matched that string against a list transcribed in English and German, so "négatif", "negativo", "ujemny", "positif" and "dodatni" all exported without a code — text-only on the one document meant to be machine-readable. The negative and positive arms read `labs.form.qualNegative` and `labs.form.qualPositive` out of every bundle now. The form writes those exact keys, so the write side and the read side cannot drift, and a new locale is coded the day its bundle lands. The detected / not-detected pair stays hand-written English and German, and that is the honest limit of this change: the app ships no string for those terms, they are prose a lab printed and a person copied in, and coding "non détecté" or "niewykrywalny" would mean inventing clinical vocabulary in four languages nobody here can check. This module's own rule is that an unverified code is worse than an honest `.text`. When they need coding they need a quick pick and a bundle key, which is the route the other two arms took. A test pins that they stay uncoded rather than leaving it to be discovered. Borderline stays uncoded as before; a test now pins that the derivation did not sweep its bundle key up by accident. Comparison folds accents as well as case, so a result typed without them still resolves. --- .../__tests__/lab-qualitative-locales.test.ts | 138 ++++++++++++++++++ src/lib/fhir/lab-qualitative.ts | 121 +++++++++++---- 2 files changed, 234 insertions(+), 25 deletions(-) create mode 100644 src/lib/fhir/__tests__/lab-qualitative-locales.test.ts diff --git a/src/lib/fhir/__tests__/lab-qualitative-locales.test.ts b/src/lib/fhir/__tests__/lab-qualitative-locales.test.ts new file mode 100644 index 000000000..7ba526323 --- /dev/null +++ b/src/lib/fhir/__tests__/lab-qualitative-locales.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; + +import { + localisedQualitativeIndex, + qualitativeValueConcept, +} from "@/lib/fhir/lab-qualitative"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; + +const SNOMED = "http://snomed.info/sct"; + +function codeOf(valueText: string): string | undefined { + const concept = qualitativeValueConcept(valueText); + return concept.coding?.[0]?.code; +} + +/** + * The lab form offers "negative" and "positive" as quick picks and writes the + * LABEL it rendered, so the column holds the bundle's own string in the + * language the reading was recorded in. Matched against a transcribed + * English/German list, four of the six shipped locales exported a result the + * receiving system could not read. + */ +describe("qualitativeValueConcept — the quick-pick label in every locale", () => { + it.each([ + ["négatif", "260385009"], + ["negativo", "260385009"], + ["ujemny", "260385009"], + ["positif", "10828004"], + ["positivo", "10828004"], + ["dodatni", "10828004"], + ])("codes %s as SNOMED %s", (valueText, code) => { + expect(codeOf(valueText)).toBe(code); + }); + + it("derives from the bundles rather than from a transcribed list", () => { + for (const locale of locales) { + const negative = resolveKey( + allMessages[locale], + "labs.form.qualNegative", + ); + const positive = resolveKey( + allMessages[locale], + "labs.form.qualPositive", + ); + expect(negative, `${locale} has no negative label`).toBeTruthy(); + expect(positive, `${locale} has no positive label`).toBeTruthy(); + expect(codeOf(negative as string), `${locale}: ${negative}`).toBe( + "260385009", + ); + expect(codeOf(positive as string), `${locale}: ${positive}`).toBe( + "10828004", + ); + } + }); + + it("tolerates a term typed without its accents", () => { + expect(codeOf("Negatif")).toBe("260385009"); + expect(codeOf("NÉGATIF")).toBe("260385009"); + }); + + it("keeps the raw text on the concept whatever the language", () => { + expect(qualitativeValueConcept("négatif")).toEqual({ + coding: [{ system: SNOMED, code: "260385009", display: "Negative" }], + text: "négatif", + }); + }); +}); + +describe("qualitativeValueConcept — what stays text-only, and why", () => { + it("leaves borderline uncoded in every locale", () => { + // The candidate "Borderline" qualifier concept was never confidently + // verified, so `labs.form.qualBorderline` is read by nothing. This pins + // that the derivation did not sweep it up by accident. + for (const locale of locales) { + const label = resolveKey(allMessages[locale], "labs.form.qualBorderline"); + expect(label, `${locale} has no borderline label`).toBeTruthy(); + expect( + qualitativeValueConcept(label as string).coding, + `${locale}: ${label}`, + ).toBeUndefined(); + } + }); + + it("leaves the detected / not-detected pair uncoded outside English and German", () => { + // The honest limit of this change, written down rather than discovered + // later: the app ships no string for these, they are prose a lab printed, + // and inventing the four other languages would be inventing clinical + // vocabulary nobody here can check. + expect(codeOf("nicht nachweisbar")).toBe("260415000"); + expect(codeOf("not detected")).toBe("260415000"); + expect(codeOf("non détecté")).toBeUndefined(); + expect(codeOf("niewykrywalny")).toBeUndefined(); + expect(qualitativeValueConcept("non détecté")).toEqual({ + text: "non détecté", + }); + }); + + it("leaves free prose uncoded", () => { + expect(qualitativeValueConcept("siehe Befund")).toEqual({ + text: "siehe Befund", + }); + }); +}); + +describe("qualitativeValueConcept — the existing English and German terms", () => { + it.each([ + ["negativ", "260385009"], + ["negative", "260385009"], + ["Negativ", "260385009"], + ["neg", "260385009"], + ["positiv", "10828004"], + ["positive", "10828004"], + ["pos", "10828004"], + ["nachweisbar", "260373001"], + ["detected", "260373001"], + ])("still codes %s as SNOMED %s", (valueText, code) => { + expect(codeOf(valueText)).toBe(code); + }); +}); + +describe("the derived qualitative index", () => { + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(Object.keys(localisedQualitativeIndex()).length).toBeGreaterThan(3); + }); + + it("never puts one word on both arms", () => { + // Two locales spelling a term the same way ("negativo" in Spanish and + // Italian) is expected and folds onto one row. The same word meaning both + // negative and positive would be a bundle bug. + const negatives = new Set(); + const positives = new Set(); + for (const [word, concept] of Object.entries(localisedQualitativeIndex())) { + (concept.code === "260385009" ? negatives : positives).add(word); + } + for (const word of negatives) expect(positives.has(word)).toBe(false); + }); +}); diff --git a/src/lib/fhir/lab-qualitative.ts b/src/lib/fhir/lab-qualitative.ts index fb614e39c..7b5448ddc 100644 --- a/src/lib/fhir/lab-qualitative.ts +++ b/src/lib/fhir/lab-qualitative.ts @@ -3,29 +3,52 @@ * * A qualitative serology result ("negativ" / "positiv" / "nicht nachweisbar") * has no number, so its FHIR Observation carries a `valueCodeableConcept` - * instead of a `valueQuantity`. This module maps the common German / English - * result terms onto a small set of WELL-ESTABLISHED SNOMED CT qualifier-value - * concepts. The raw recorded text ALWAYS rides `.text`; a coded `coding` is - * added only when the term resolves CONFIDENTLY. When in doubt the result stays - * text-only — never a fabricated code — mirroring the conservative stance of - * the `lab-loinc` and `illness-snomed` mappers. + * instead of a `valueQuantity`. This module maps common result terms onto a + * small set of WELL-ESTABLISHED SNOMED CT qualifier-value concepts. The raw + * recorded text ALWAYS rides `.text`; a coded `coding` is added only when the + * term resolves CONFIDENTLY. When in doubt the result stays text-only — never + * a fabricated code — mirroring the conservative stance of the `lab-loinc` and + * `illness-snomed` mappers. * * Concepts (SNOMED CT International, qualifier value, REFERENCED not * redistributed per the licence): - * - 260385009 "Negative" — negativ / negative - * - 10828004 "Positive" — positiv / positive + * - 260385009 "Negative" — every locale's `labs.form.qualNegative` + * - 10828004 "Positive" — every locale's `labs.form.qualPositive` * - 260415000 "Not detected" — nicht nachweisbar / not detected * - 260373001 "Detected" — nachweisbar / detected * + * The negative and positive arms are DERIVED from `messages/.json`. + * The lab form offers those two as quick picks and writes the label it + * rendered, so what lands in the column is the bundle's own string — + * "négatif", "negativo", "ujemny" — and matching it against a transcribed + * English/German list meant four of the six shipped locales recorded a result + * the export then could not code. Deriving means a new locale is coded the day + * its bundle lands, with no edit here. + * + * The detected / not-detected arm is NOT derived, because the app ships no + * string for it: those are terms a LAB printed and a person copied in. It + * stays the hand-written English and German pair it has always been, and the + * honest cost is that "non détecté" or "niewykrywalny" stay text-only. Coding + * them would mean inventing clinical vocabulary in four languages nobody here + * can check, and this module's own rule is that an unverified code is worse + * than an honest `.text`. When those terms need coding, they need a quick pick + * in the form and a bundle key — the same route the negative and positive arms + * took. + * * Borderline / grenzwertig is DELIBERATELY left text-only: the candidate - * "Borderline" qualifier concept could not be confidently verified, and an - * unverified code is worse than an honest `.text`. Any unrecognised term + * "Borderline" qualifier concept could not be confidently verified. Its bundle + * key is therefore read by nothing here, on purpose. Any unrecognised term * likewise stays text-only. * + * SERVER-ONLY: the derived index reads every shipped message bundle. Its one + * caller is the FHIR exporter, which runs on the server. + * * Systems: SNOMED CT — http://snomed.info/sct */ import type { FhirCodeableConcept } from "@/lib/fhir/types"; +import { foldForMatch } from "@/lib/i18n/fold-for-match"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; /** * SNOMED CT system URI. Inlined (rather than imported from `resources.ts`) to @@ -39,29 +62,76 @@ interface QualitativeConcept { display: string; } +const NEGATIVE: QualitativeConcept = { code: "260385009", display: "Negative" }; +const POSITIVE: QualitativeConcept = { code: "10828004", display: "Positive" }; + /** - * Normalised result term → SNOMED qualifier-value concept. The left side is the - * lower-cased, whitespace-collapsed recorded text. Only confidently-known terms - * appear; everything else falls through to text-only. + * Hand-written result terms → SNOMED qualifier-value concept. The left side is + * the folded recorded text. + * + * What belongs here is what the app ships no string for: the clinical + * shorthand a person types ("neg", "pos") and the detected / not-detected pair + * a lab prints. The negative and positive WORDS are not transcribed here — see + * the derived index below. */ -const QUALITATIVE_SNOMED: Record = { - // Negative-like - negativ: { code: "260385009", display: "Negative" }, - negative: { code: "260385009", display: "Negative" }, - neg: { code: "260385009", display: "Negative" }, +const HAND_WRITTEN_SNOMED: Record = { + neg: NEGATIVE, + pos: POSITIVE, "nicht nachweisbar": { code: "260415000", display: "Not detected" }, "not detected": { code: "260415000", display: "Not detected" }, - // Positive-like - positiv: { code: "10828004", display: "Positive" }, - positive: { code: "10828004", display: "Positive" }, - pos: { code: "10828004", display: "Positive" }, nachweisbar: { code: "260373001", display: "Detected" }, detected: { code: "260373001", display: "Detected" }, }; -/** Collapse a recorded qualitative term to a lookup key. */ +/** + * The quick-pick labels, in every shipped locale, read out of the bundles. + * + * The form writes the label it rendered, so this index and the write side + * cannot drift: whatever `labs.form.qualNegative` says in a language is + * exactly what a reading recorded in that language holds. `qualBorderline` is + * deliberately absent — see the module comment. + */ +function buildLocalisedIndex(): Record { + const index: Record = {}; + const arms: [string, QualitativeConcept][] = [ + ["labs.form.qualNegative", NEGATIVE], + ["labs.form.qualPositive", POSITIVE], + ]; + for (const [key, concept] of arms) { + for (const label of localisedValues(key)) { + const folded = foldForMatch(label); + // A folded label already claimed by the OTHER arm would mean a bundle + // says the same word means both negative and positive. First wins so a + // typo cannot take the export down at boot; the guard suite fails on it. + if (!folded || index[folded]) continue; + index[folded] = concept; + } + } + return index; +} + +const LOCALISED_SNOMED = buildLocalisedIndex(); + +/** + * Test seam: the guard suite asserts the derived index covers every shipped + * locale and disagrees with no hand-written term. Production reads the + * constant. + */ +export function localisedQualitativeIndex(): Readonly< + Record +> { + return LOCALISED_SNOMED; +} + +/** + * Collapse a recorded qualitative term to a lookup key. + * + * Folds case, accents and separators, so a reading typed "Negatif" on a + * keyboard without accents meets the bundle's "négatif", and a stored + * "negativ" resolves exactly as it did before the fold widened. + */ function normaliseQualitativeKey(text: string): string { - return text.trim().toLowerCase().replace(/\s+/g, " "); + return foldForMatch(text); } /** @@ -73,7 +143,8 @@ function normaliseQualitativeKey(text: string): string { export function qualitativeValueConcept( valueText: string, ): FhirCodeableConcept { - const concept = QUALITATIVE_SNOMED[normaliseQualitativeKey(valueText)]; + const key = normaliseQualitativeKey(valueText); + const concept = HAND_WRITTEN_SNOMED[key] ?? LOCALISED_SNOMED[key]; if (!concept) { return { text: valueText }; } From b06d7dbd92f9c41f3ead587c313a7a606e83d93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 23 Aug 2026 14:28:53 +0200 Subject: [PATCH 4/5] fix(documents): recognise a lab report printed outside English and German MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate that decides whether a transcribed document is worth an extraction call scored four classes of evidence and needed two. Three of them — the reference-range label, the analyte names, the report header — were written in English and German, so a French, Spanish, Italian or Polish report scored at most one: the units, which are the same everywhere. Auto-staging never fired for it. The document was lost from the flow silently, because the manual extract button stays available and nothing reads as broken. The text is folded once — lower case, accents dropped — and every pattern is matched against that, which on its own recovers a good deal: "Cholestérol", "Créatinine" and "Emoglobina" reach the existing stems as soon as the accents are out of the way. The reference-range labels and report headers of the four locales are added by hand, because they are prose a lab printed and the app owns no string for them. The analyte names ARE derived, from `labs.catalog.` across every shipped bundle: those are names the app itself displays, and a new locale should not need an edit here to be recognised. Names under four folded characters are dropped — "ALT" is German for "old", and a three-letter alternative in a gate this cheap would make every German letter a lab report. The acronyms worth having are in the hand-written half, where each was considered on its own. Which way this errs, stated at the code: a false negative loses a whole document from auto-staging, while a false positive costs one extraction call and stages facts a person then declines — nothing commits either way. So the vocabulary is widened, and the two-distinct-class rule holds the other side. The derived names extend the analyte class rather than forming a fifth one, so a document naming one analyte in two languages still scores one. Tests pin each addition on a case that fails the gate without it, and pin the prose that must keep failing it. --- .../__tests__/auto-stage-labs-locales.test.ts | 213 ++++++++++++++++++ src/lib/documents/auto-stage-labs.ts | 123 +++++++++- 2 files changed, 326 insertions(+), 10 deletions(-) create mode 100644 src/lib/documents/__tests__/auto-stage-labs-locales.test.ts diff --git a/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts b/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts new file mode 100644 index 000000000..53fad3039 --- /dev/null +++ b/src/lib/documents/__tests__/auto-stage-labs-locales.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "vitest"; + +import { + derivedAnalyteNamePattern, + looksLikeLabDocument, +} from "@/lib/documents/auto-stage-labs"; +import { locales } from "@/lib/i18n/config"; +import { allMessages, resolveKey } from "@/lib/i18n/shared-resolve"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; + +/** + * The gate decides whether a transcribed document is worth an extraction call. + * A lab report that fails it is lost from auto-staging entirely, and nothing + * says so — the manual extract button stays available, so the flow reads as + * working. + * + * Before this, the four signal classes were English and German only, so a + * report printed anywhere else scored at most one hit (the units, which are + * the same everywhere) and never reached two. + */ +const FRENCH_REPORT = ` +Laboratoire d'analyses médicales +Compte rendu du 14 mars 2026 + +Cholestérol total 5,8 mmol/l Valeurs de référence: jusqu'à 5,0 +Créatinine 88 µmol/l Plage de référence: 62 - 106 +Hémoglobine 14,2 g/dl +`; + +const SPANISH_REPORT = ` +Laboratorio de análisis clínicos +Analítica de sangre + +Colesterol total 210 mg/dl Valores de referencia: hasta 200 +Glucosa en ayunas 92 mg/dl Rango de referencia: 70 - 100 +`; + +const ITALIAN_REPORT = ` +Referto di laboratorio — esami del sangue + +Colesterolo totale 5,4 mmol/l Valori di riferimento: fino a 5,0 +Emoglobina 13,8 g/dl Intervallo di riferimento: 12 - 16 +`; + +const POLISH_REPORT = ` +Laboratorium analiz lekarskich +Wyniki badań + +Cholesterol całkowity 5,2 mmol/l Zakres referencyjny: do 5,0 +Kreatynina 82 µmol/l Wartości referencyjne: 62 - 106 +`; + +describe("looksLikeLabDocument — a report printed outside English and German", () => { + it.each([ + ["French", FRENCH_REPORT], + ["Spanish", SPANISH_REPORT], + ["Italian", ITALIAN_REPORT], + ["Polish", POLISH_REPORT], + ])("recognises a %s lab report", (_language, text) => { + expect(looksLikeLabDocument(text)).toBe(true); + }); + + it("recognises a report on its analyte names and units alone", () => { + // No header, no reference-range label — the two classes left are the ones + // a stripped-down result table still carries. + expect( + looksLikeLabDocument("Triglycérides 1,4 mmol/l\nFerritine 120 ng/ml"), + ).toBe(true); + }); +}); + +/** + * The four reports above each carry several kinds of evidence at once, so any + * one of the additions could carry them alone. These cases isolate one + * addition each: remove the piece each names and the case drops back to a + * single class and fails the gate. + */ +describe("looksLikeLabDocument — each addition carries its own weight", () => { + it("needs the DERIVED analyte names for a panel of markers no stem covers", () => { + // The electrolytes are in the catalog and in no hand-written stem, so the + // derived names are the only analyte evidence here. Units are the second + // class. + expect( + looksLikeLabDocument("Sodium 140 mmol/l\nPotassium 4,2 mmol/l"), + ).toBe(true); + expect(looksLikeLabDocument("Sodio 140 mmol/l\nPotassio 4,2 mmol/l")).toBe( + true, + ); + }); + + it("needs the reference-range labels of the four locales", () => { + // "Płytki krwi" is a derived name; the label is the only other class here. + expect( + looksLikeLabDocument("Płytki krwi 250\nZakres referencyjny: 150 - 400"), + ).toBe(true); + expect( + looksLikeLabDocument("Plaquettes 250\nValeurs de référence: 150 - 400"), + ).toBe(true); + }); + + it("needs the report headers of the four locales", () => { + expect(looksLikeLabDocument("Referto di laboratorio\nPiastrine 250")).toBe( + true, + ); + expect(looksLikeLabDocument("Wyniki badań\nPłytki krwi 250")).toBe(true); + expect(looksLikeLabDocument("Analítica de sangre\nPlaquetas 250")).toBe( + true, + ); + }); +}); + +describe("looksLikeLabDocument — what must still fail the gate", () => { + it.each([ + [ + "English prose with one incidental unit", + "Please take 500 mg twice daily.", + ], + [ + "a French letter mentioning one analyte", + "Cher patient, votre taux de cholestérol sera revu lors du prochain rendez-vous.", + ], + [ + "a Spanish appointment letter", + "Estimado paciente, su cita en el hospital es el martes a las diez.", + ], + [ + "an Italian pharmacy note", + "Ritirare la confezione in farmacia entro venerdì.", + ], + ["an invoice with a currency amount", "Rechnung über 120,00 EUR. Danke."], + ])("rejects %s", (_case, text) => { + expect(looksLikeLabDocument(text)).toBe(false); + }); + + it("does not let one analyte named twice reach two classes", () => { + // The derived names extend the analyte class rather than forming their + // own. If they were a fifth class, a document saying "Cholestérol total" + // and "cholesterol" would score two and pass on one piece of evidence. + expect(looksLikeLabDocument("Cholestérol total et cholesterol")).toBe( + false, + ); + }); + + it("ignores a derived analyte name embedded in a longer word", () => { + // The hand-written half of the analyte class matches STEMS on purpose + // ("ferritin" catches "Ferritinwert"); the derived half is bounded, + // because a whole catalog name inside another word is noise. + expect(looksLikeLabDocument("Xhematocritx 5 mg/dl")).toBe(false); + }); +}); + +describe("looksLikeLabDocument — the English and German behaviour is unchanged", () => { + it("still recognises a German report", () => { + expect( + looksLikeLabDocument( + "Laborbefund. Hämoglobin 14.2 g/dl. Referenzbereich 13-17. LDL 120 mg/dl.", + ), + ).toBe(true); + }); + + it("still recognises an English report", () => { + expect( + looksLikeLabDocument( + "Laboratory report. Hemoglobin 14.2 g/dL. Reference range 13-17.", + ), + ).toBe(true); + }); + + it("still rejects the prose the previous guard rejected", () => { + expect( + looksLikeLabDocument("Dear patient, please take 500 mg twice daily."), + ).toBe(false); + }); +}); + +describe("the derived analyte-name pattern", () => { + it("carries a name from every shipped locale", () => { + // The property that matters. A transcribed list would satisfy the report + // cases above and still be frozen at the languages somebody typed. + for (const locale of locales) { + const name = resolveKey( + allMessages[locale], + "labs.catalog.total-cholesterol", + ); + expect(name, `${locale} has no catalog name`).toBeTruthy(); + expect( + derivedAnalyteNamePattern().test( + stripDiacritics((name as string).toLowerCase()), + ), + `${locale}: ${name}`, + ).toBe(true); + } + }); + + it("excludes the short acronyms that are ordinary words elsewhere", () => { + // "ALT" is German for "old". A three-character alternative in a gate this + // cheap would make every German letter a lab report. The acronyms worth + // having live in the hand-written class, each considered on its own. + const pattern = derivedAnalyteNamePattern(); + for (const acronym of ["alt", "ast", "ggt", "sod"]) { + expect(pattern.test(acronym), acronym).toBe(false); + } + }); + + it("is not empty, so a matcher that matches nothing cannot pass", () => { + expect(derivedAnalyteNamePattern().test("triglycerides")).toBe(true); + // The Polish "ł" is a single codepoint NFD does not decompose, so it + // survives the fold on both sides and has to be written as the bundle + // writes it. A document that spelled it "plytki" would not match, which is + // the stated limit of folding rather than transliterating. + expect(derivedAnalyteNamePattern().test("płytki krwi")).toBe(true); + }); +}); diff --git a/src/lib/documents/auto-stage-labs.ts b/src/lib/documents/auto-stage-labs.ts index 1c815b7d7..b001471d7 100644 --- a/src/lib/documents/auto-stage-labs.ts +++ b/src/lib/documents/auto-stage-labs.ts @@ -42,28 +42,131 @@ import { } from "@/lib/documents/extract"; import { encryptFactData, encryptFactProvenance } from "@/lib/documents/store"; import { prisma } from "@/lib/db"; +import { stripDiacritics } from "@/lib/i18n/fold-for-match"; +import { localisedValues } from "@/lib/i18n/shared-resolve"; +import { BIOMARKER_CATALOG } from "@/lib/labs/biomarker-catalog"; import { annotate } from "@/lib/logging/context"; import { isModuleEnabled } from "@/lib/modules/gate"; /** * Lab-report signals used to decide whether a transcribed document is worth an - * extraction call. Each pattern is one class of evidence (a reference-range - * label, a lab unit, a common analyte name, a report header); requiring TWO - * distinct hits keeps a stray "mg" in a prose letter from tripping the auto - * extraction while still catching a real panel in German or English. + * extraction call. Each entry is one CLASS of evidence — a reference-range + * label, a lab unit, a common analyte name, a report header — and contributes + * at most one hit however many of its alternatives fire. Requiring TWO + * distinct classes keeps a stray "mg" in a prose letter from tripping the auto + * extraction while still catching a real panel. + * + * Every pattern is tested against the text LOWER-CASED AND ACCENT-FOLDED, not + * against the transcription as written, so the patterns are spelled without + * accents. That is also what lets one pattern serve "Hämoglobin", "haemoglobin" + * and "hemoglobin" instead of three. + * + * Until now the four classes were English and German only, so a French, + * Spanish, Italian or Polish lab report scored at most one hit — the units, + * which are the same everywhere — and auto-staging never fired for it. The + * whole document was lost from the flow, silently: the manual extract button + * stays available, so nothing anywhere reads as broken. + * + * WHICH WAY THIS ERRS. A false negative loses a real report from auto-staging; + * a false positive burns one extraction call on a document that is not a lab + * report, and stages facts a person then declines on the review screen — + * nothing commits either way. The cost is lopsided, so the vocabulary is + * widened rather than kept tight, and the two-distinct-class rule is what + * holds the other side: an incidental "sodium" in a prose letter is one class + * and still fails the gate. Terms whose only home is a lab report ("emocromo", + * "bilan sanguin", "zakres referencyjny") were preferred over generic ones, + * and short acronyms below four characters are excluded from the derived names + * for exactly this reason — the German word "alt" would otherwise make every + * German letter a lab report. */ const LAB_SIGNALS: readonly RegExp[] = [ - /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich/i, + // 1 — a reference-range label. Only a report that prints windows says this. + /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich|valeurs? de reference|intervalle de reference|plage de reference|valores? de referencia|rango de referencia|intervalo de referencia|valori di riferimento|intervallo di riferimento|zakres referencyjny|wartosci referencyjne/i, + // 2 — a lab unit. Language-independent already; unchanged. /\b(mg\/dl|mmol\/l|nmol\/l|µmol\/l|umol\/l|g\/dl|µg\/l|ug\/l|ng\/ml|pg\/ml|mmol\/mol|u\/l|iu\/l|mg\/l|\/µl|\/ul)\b/i, - /h(?:ä|ae)moglobin|hemoglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i, - /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b/i, + // 3 — an analyte name. The hand-written half: word stems and the short + // acronyms, which are too short to derive safely but were vetted long ago. + /h(?:a|ae|e)moglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i, + // 4 — a report header. + /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b|laboratoire|bilan sanguin|analyses medicales|hemogramme|laboratorio|analitica|hemograma|analisis de sangre|esami del sangue|emocromo|referto|laboratorium|morfologia krwi|wyniki badan/i, ]; -/** True when the transcribed text carries at least two lab-report signals. */ +/** + * Class 3 again, DERIVED from `messages/.json` instead of transcribed. + * + * The names above are stems a maintainer typed. These are the names the app + * itself displays for the markers in its catalog, in every language it ships — + * so a French report's "Cholestérol total" or a Polish "Płytki krwi" counts as + * an analyte the day the bundle carries it, with no edit here. Transcribing + * thirty names times four more languages would have fixed today's six locales + * and reopened the hole at the seventh. + * + * Names shorter than four folded characters are dropped: "ALT", "AST", "GGT", + * "TSH" and Polish "Sód" are ordinary words or fragments in some language, and + * a three-letter alternative in a gate this cheap is a false-positive machine. + * The acronyms worth having are already in the hand-written class above, where + * each was considered one at a time. + */ +function buildAnalyteNamePattern(): RegExp { + const names = new Set(); + for (const seed of BIOMARKER_CATALOG) { + for (const name of localisedValues(`labs.catalog.${seed.slug}`)) { + const folded = stripDiacritics(name.toLowerCase()).trim(); + // Also the name without its parenthetical gloss. A report prints + // "Apolipoprotéine B", not "Apolipoprotéine B (ApoB)", and the gloss is + // the app's way of naming a marker two ways at once. + const bare = folded.replace(/\s*\([^)]*\)/gu, "").trim(); + for (const candidate of [folded, bare]) { + if (candidate.length >= 4) names.add(candidate); + } + } + } + const alternation = [...names] + .sort((a, b) => b.length - a.length) + .map((name) => + name + .split(/\s+/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + // Any run of whitespace, so a name broken across a transcribed line + // still reads as one name. + .join(String.raw`\s+`), + ) + .join("|"); + // Bounded by "not a letter or digit" rather than by \b, because several + // names end in punctuation ("Lipoprotéine(a)") where \b cannot match, and + // because the Polish "ł" is not a word character to \b. + return new RegExp( + `(?= 2) return true; + for (const [index, signal] of LAB_SIGNALS.entries()) { + const analyteClass = index === 2; + const hit = + signal.test(folded) || + (analyteClass && DERIVED_ANALYTE_NAMES.test(folded)); + if (hit && ++hits >= 2) return true; } return false; } From 154b6097680cf98cb4ac40f95f4a3faa405819b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 23 Aug 2026 14:44:23 +0200 Subject: [PATCH 5/5] refactor(documents): attach the derived analyte names to their signal class by identity The derived names extended the analyte class by list index. Reordering the signal list would have re-pointed them at the units or the report header with nothing to notice, so the class is a named constant and the check compares against it. --- src/lib/documents/auto-stage-labs.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/lib/documents/auto-stage-labs.ts b/src/lib/documents/auto-stage-labs.ts index b001471d7..52ccaca2c 100644 --- a/src/lib/documents/auto-stage-labs.ts +++ b/src/lib/documents/auto-stage-labs.ts @@ -79,14 +79,23 @@ import { isModuleEnabled } from "@/lib/modules/gate"; * for exactly this reason — the German word "alt" would otherwise make every * German letter a lab report. */ +/** + * The analyte class, hand-written half: word stems and the short acronyms, + * which are too short to derive safely but were vetted one at a time long ago. + * Named rather than inlined so the derived names can be attached to THIS class + * by identity — an index would silently re-point them at another class the + * first time somebody reorders the list. + */ +const ANALYTE_SIGNAL = + /h(?:a|ae|e)moglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i; + const LAB_SIGNALS: readonly RegExp[] = [ // 1 — a reference-range label. Only a report that prints windows says this. /reference range|referenzbereich|normbereich|normal range|ref\.?-?bereich|valeurs? de reference|intervalle de reference|plage de reference|valores? de referencia|rango de referencia|intervalo de referencia|valori di riferimento|intervallo di riferimento|zakres referencyjny|wartosci referencyjne/i, // 2 — a lab unit. Language-independent already; unchanged. /\b(mg\/dl|mmol\/l|nmol\/l|µmol\/l|umol\/l|g\/dl|µg\/l|ug\/l|ng\/ml|pg\/ml|mmol\/mol|u\/l|iu\/l|mg\/l|\/µl|\/ul)\b/i, - // 3 — an analyte name. The hand-written half: word stems and the short - // acronyms, which are too short to derive safely but were vetted long ago. - /h(?:a|ae|e)moglobin|glu[ck]ose|cholesterin|cholesterol|kreatinin|creatinine|hba1c|leuko|erythro|thrombo|ferritin|triglycerid|\btsh\b|\bldl\b|\bhdl\b|\bcrp\b|vitamin\s?d/i, + // 3 — an analyte name. + ANALYTE_SIGNAL, // 4 — a report header. /labor(?:befund|wert)?|laboratory|blutbild|\bbefund\b|laboratoire|bilan sanguin|analyses medicales|hemogramme|laboratorio|analitica|hemograma|analisis de sangre|esami del sangue|emocromo|referto|laboratorium|morfologia krwi|wyniki badan/i, ]; @@ -161,11 +170,10 @@ export function looksLikeLabDocument(text: string): boolean { // pattern needs the same folded form. const folded = stripDiacritics(text.toLowerCase()); let hits = 0; - for (const [index, signal] of LAB_SIGNALS.entries()) { - const analyteClass = index === 2; + for (const signal of LAB_SIGNALS) { const hit = signal.test(folded) || - (analyteClass && DERIVED_ANALYTE_NAMES.test(folded)); + (signal === ANALYTE_SIGNAL && DERIVED_ANALYTE_NAMES.test(folded)); if (hit && ++hits >= 2) return true; } return false;