diff --git a/src/components/labs/lab-form.tsx b/src/components/labs/lab-form.tsx index 6403444d8..f22d1efe8 100644 --- a/src/components/labs/lab-form.tsx +++ b/src/components/labs/lab-form.tsx @@ -22,7 +22,10 @@ import { import { Textarea } from "@/components/ui/textarea"; import { apiGet, apiPost } from "@/lib/api/api-fetch"; import { localizedApiError } from "@/lib/api/localized-error"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; import { formatReferenceRange } from "@/lib/labs/reference-range"; import { formatLabValue } from "@/lib/labs/format-value"; import { EncounterSuggestionField } from "@/components/encounters/encounter-suggestion-field"; @@ -300,7 +303,7 @@ export function LabForm({ // the hint says the text is kept as written rather than staying silent. const parsedSourceRange = parseReferenceRange(sourceRange, selected?.unit); const sourceRangeHint = parsedSourceRange - ? parsedSourceRange.low === null && parsedSourceRange.high === null + ? isUnreadableRange(parsedSourceRange) ? t("labs.form.sourceRangeUnparsed") : `${formatReferenceRange( parsedSourceRange.low, diff --git a/src/lib/documents/commit.ts b/src/lib/documents/commit.ts index b960b39b9..0cecca25a 100644 --- a/src/lib/documents/commit.ts +++ b/src/lib/documents/commit.ts @@ -28,7 +28,11 @@ import { invalidateUserHealthScore } from "@/lib/cache/invalidate"; import { emitDataArrival } from "@/lib/arrivals/emit-shared"; import { decryptFactData } from "@/lib/documents/store"; import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; +import { annotate } from "@/lib/logging/context"; import type { ExtractedFact } from "@/generated/prisma/client"; import type { ConditionFactData, @@ -82,6 +86,19 @@ async function commitObservation( const sourceLow = printed?.low ?? data.referenceLow; const sourceHigh = printed?.high ?? data.referenceHigh; const sourceText = printed?.text ?? null; + // A window the report printed and the parser could not read is the case that + // used to pass without a trace: the reading is then judged against the + // catalog band, or against nothing, and no surface says the lab had stated + // something else. Count it here so an unreadable notation — a language the + // prose table does not carry, a layout nobody anticipated — shows up as a + // number on a dashboard instead of as silence. The string itself is the + // user's document and stays out of the event. + if (isUnreadableRange(printed)) { + annotate({ + action: { name: "labs.referenceRange.unreadable" }, + meta: { surface: "document.commit", length: printed?.text.length ?? 0 }, + }); + } const biomarker = await resolveOrMintBiomarker(userId, { analyte: data.label, diff --git a/src/lib/i18n/fold-for-match.ts b/src/lib/i18n/fold-for-match.ts new file mode 100644 index 000000000..db0815e91 --- /dev/null +++ b/src/lib/i18n/fold-for-match.ts @@ -0,0 +1,59 @@ +/** + * Character folds used when app text has to be matched against text a PERSON + * or a LAB typed, rather than compared byte-for-byte. + * + * Every matcher that reads a shipped translation and looks for it in stored + * or transcribed text needs the same three tolerances — case, accents, and the + * apostrophe a word processor curled — and every matcher that grows its own + * copy of them drifts from the others. One home, so "Perte d'appétit" typed + * with a straight quote meets the bundle's "Perte d’appétit", and a French + * report's "jusqu'à" meets a table written "jusqu'a". + * + * The folds are one-directional and deliberately NOT transliteration: German + * "ue" does not fold onto "ü", "ß" is left alone (it has no canonical + * decomposition), and the Polish "ł" is a single codepoint NFD does not touch. + * That is fine — what has to agree is a shipped string with itself as a person + * retyped it, not two spellings of the same word. + * + * Client-safe: no message bundles are imported here, so a client component may + * use these. Building an index OVER the bundles is a server concern and lives + * in `./shared-resolve`. + */ + +/** Curly, modifier and backtick apostrophes a keyboard or word processor emits. */ +const APOSTROPHES = /[\u2018\u2019\u02bc\u00b4`']/gu; + +/** Combining marks left behind by an NFD decomposition. */ +const COMBINING_MARKS = /[\u0300-\u036f]/gu; + +/** + * Drop accents by decomposing and removing the combining marks: "é" → "e", + * "ż" → "z", "Ü" → "U". Punctuation, case and spacing survive untouched, so + * this is the fold to use where the surrounding characters carry meaning — a + * printed range's "3,5-5,0", a unit's "mg/dL". + */ +export function stripDiacritics(raw: string): string { + return raw.normalize("NFD").replace(COMBINING_MARKS, ""); +} + +/** Collapse every apostrophe variant onto the straight ASCII one. */ +export function normaliseApostrophes(raw: string): string { + return raw.replace(APOSTROPHES, "'"); +} + +/** + * Fold a label to its comparison form: lower case, no accents, and every + * apostrophe / hyphen / underscore / slash reduced to a single space. + * + * This is the fold for whole LABELS — a tag, an analyte name, a qualitative + * result term — where internal punctuation is noise. Apply it identically to + * both sides of a comparison; a fold applied to only one side is an + * exact-string match wearing a disguise. + */ +export function foldForMatch(raw: string): string { + return stripDiacritics(raw.toLowerCase()) + .replace(APOSTROPHES, " ") + .replace(/[-_/]/gu, " ") + .replace(/\s+/gu, " ") + .trim(); +} diff --git a/src/lib/i18n/shared-resolve.ts b/src/lib/i18n/shared-resolve.ts index f08790d05..fba5d019f 100644 --- a/src/lib/i18n/shared-resolve.ts +++ b/src/lib/i18n/shared-resolve.ts @@ -20,9 +20,10 @@ import frMessages from "../../../messages/fr.json"; import esMessages from "../../../messages/es.json"; import itMessages from "../../../messages/it.json"; import plMessages from "../../../messages/pl.json"; -import { type Locale } from "./config"; +import { locales, type Locale } from "./config"; +import { resolveKey } from "./resolve-key"; -export { resolveKey } from "./resolve-key"; +export { resolveKey }; /** * Single source of truth for the per-locale message bundle on the @@ -36,3 +37,29 @@ export const allMessages: Record> = { it: itMessages, pl: plMessages, }; + +/** + * Every shipped translation of one key, in `locales` order, deduplicated and + * with missing entries dropped. + * + * This is the primitive a DERIVED match index is built from. A matcher that + * has to recognise an app-owned label in text a person typed must not + * transcribe the label into its own source: transcribing fixes the locales + * shipped on the day it was written and reopens the same hole when a new + * bundle lands. Reading the bundles means a new locale is matchable the day + * `messages/.json` arrives, with no edit in the matcher. + * + * Known limit, worth stating where it bites: the values returned are what the + * bundles carry TODAY, while a matcher applies them to text written in the + * past. Editing a shipped translation therefore orphans whatever was written + * under the old wording — the fix for that is a legacy alias at the call site, + * not a data migration. + */ +export function localisedValues(key: string): string[] { + const seen = new Set(); + for (const locale of locales) { + const value = resolveKey(allMessages[locale], key); + if (value && value.trim()) seen.add(value.trim()); + } + return [...seen]; +} diff --git a/src/lib/labs/__tests__/parse-reference-range-locales.test.ts b/src/lib/labs/__tests__/parse-reference-range-locales.test.ts new file mode 100644 index 000000000..933bbce75 --- /dev/null +++ b/src/lib/labs/__tests__/parse-reference-range-locales.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from "vitest"; + +import { locales } from "@/lib/i18n/config"; +import { + isUnreadableRange, + parseReferenceRange, + RANGE_PROSE, +} from "@/lib/labs/parse-reference-range"; + +/** + * The notation a lab prints is the LAB's prose, not the app's, so this parser + * cannot derive its vocabulary from `messages/.json` the way a matcher + * over app-owned labels does. The table is hand-written, and these are the + * strings it has to read: real reference-range notation as French, Spanish, + * Italian and Polish reports print it, decimal comma included. + * + * Before the table existed, every case below returned `low: null, high: null` + * — the string was stored and the reading was then judged against a band the + * lab never printed, or against no band at all, with nothing anywhere saying + * so. + */ +describe("parseReferenceRange — French notation", () => { + it.each([ + ["jusqu'à 5,0", null, 5], + ["jusqu’à 5,0", null, 5], + ["JUSQU'À 5,0", null, 5], + ["inférieur à 5,0", null, 5], + ["moins de 5,0", null, 5], + ["à partir de 3,5", 3.5, null], + ["supérieur à 3,5", 3.5, null], + ["au moins 3,5", 3.5, null], + ["3,5 à 5,0", 3.5, 5], + ["de 3,5 à 5,0", 3.5, 5], + ["entre 3,5 et 5,0", 3.5, 5], + ["valeurs de référence 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — Spanish notation", () => { + it.each([ + ["hasta 5,0", null, 5], + ["menor de 5,0", null, 5], + ["inferior a 5,0", null, 5], + ["como máximo 5,0", null, 5], + ["desde 3,5", 3.5, null], + ["mayor que 3,5", 3.5, null], + ["superior a 3,5", 3.5, null], + ["más de 3,5", 3.5, null], + ["3,5 a 5,0", 3.5, 5], + ["de 3,5 a 5,0", 3.5, 5], + ["entre 3,5 y 5,0", 3.5, 5], + ["valores de referencia 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — Italian notation", () => { + it.each([ + ["fino a 5,0", null, 5], + ["inferiore a 5,0", null, 5], + ["meno di 5,0", null, 5], + ["oltre 3,5", 3.5, null], + ["superiore a 3,5", 3.5, null], + ["più di 3,5", 3.5, null], + ["almeno 3,5", 3.5, null], + ["3,5 a 5,0", 3.5, 5], + ["da 3,5 a 5,0", 3.5, 5], + ["tra 3,5 e 5,0", 3.5, 5], + ["valori di riferimento 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); + + it("reads a bare Italian floor rather than swallowing its introducing word", () => { + // "da" both introduces a two-sided window ("da 3,5 a 5,0") and states a + // floor on its own. Stripping introducing words before matching — which is + // what the parser used to do unconditionally — turns the second form into + // a bare number with no bound at all. + expect(parseReferenceRange("da 3,5")).toEqual({ + low: 3.5, + high: null, + text: "da 3,5", + }); + }); +}); + +describe("parseReferenceRange — Polish notation", () => { + it.each([ + ["do 5,0", null, 5], + ["poniżej 5,0", null, 5], + ["mniej niż 5,0", null, 5], + ["maks. 5,0", null, 5], + ["od 3,5", 3.5, null], + ["powyżej 3,5", 3.5, null], + ["co najmniej 3,5", 3.5, null], + ["3,5 do 5,0", 3.5, 5], + ["od 3,5 do 5,0", 3.5, 5], + ["zakres referencyjny 3,5 - 5,0", 3.5, 5], + ])("reads %s as %s..%s", (raw, low, high) => { + expect(parseReferenceRange(raw)).toEqual({ low, high, text: raw }); + }); +}); + +describe("parseReferenceRange — the new prose does not loosen the old rules", () => { + it("still refuses a bare number in any language", () => { + expect(parseReferenceRange("5,0")).toEqual({ + low: null, + high: null, + text: "5,0", + }); + }); + + it("does not read scientific notation as a two-sided window", () => { + // "e" is an Italian separator ("tra 3,5 e 5,0") and "y" a Spanish one. + // Both are one letter, so they are only accepted with whitespace on both + // sides — otherwise "3.5e5" would parse as a 3.5–5 window. + expect(parseReferenceRange("3.5e5")).toEqual({ + low: null, + high: null, + text: "3.5e5", + }); + }); + + it("still refuses a transposed window printed in French", () => { + expect(parseReferenceRange("de 5,0 à 3,5")).toEqual({ + low: null, + high: null, + text: "de 5,0 à 3,5", + }); + }); + + it("still refuses bounds when the printed unit contradicts the reading", () => { + expect(parseReferenceRange("jusqu'à 5,0 mmol/l", "mg/dL")).toEqual({ + low: null, + high: null, + text: "jusqu'à 5,0 mmol/l", + }); + }); +}); + +describe("parseReferenceRange — an unreadable window is named, not silent", () => { + it("separates 'nothing printed' from 'printed and not read'", () => { + expect(parseReferenceRange(null)).toBeNull(); + expect(isUnreadableRange(parseReferenceRange(null))).toBe(false); + + const prose = parseReferenceRange("voir le compte rendu"); + expect(prose).toEqual({ + low: null, + high: null, + text: "voir le compte rendu", + }); + expect(isUnreadableRange(prose)).toBe(true); + }); + + it("reports a window it DID read as readable", () => { + expect(isUnreadableRange(parseReferenceRange("jusqu'à 5,0"))).toBe(false); + }); +}); + +describe("RANGE_PROSE coverage", () => { + it("carries an entry for every shipped locale", () => { + // Keyed by the `Locale` union on purpose: a seventh locale cannot be added + // to `locales` without someone answering how that language prints a range. + // This asserts the runtime side of the same promise. + expect(Object.keys(RANGE_PROSE).sort()).toEqual([...locales].sort()); + }); + + it("states at least one word for each notation in each locale", () => { + for (const locale of locales) { + const prose = RANGE_PROSE[locale]; + expect(prose.twoSided.length, `${locale}.twoSided`).toBeGreaterThan(0); + expect(prose.upper.length, `${locale}.upper`).toBeGreaterThan(0); + expect(prose.lower.length, `${locale}.lower`).toBeGreaterThan(0); + } + }); + + it("carries the table already folded, so both sides of a match agree", () => { + // A row written with an accent would never match: the input is + // diacritic-folded before it reaches the alternation, so an accented row + // is a row that can only ever miss. + for (const locale of locales) { + const prose = RANGE_PROSE[locale]; + for (const word of [ + ...prose.twoSided, + ...prose.upper, + ...prose.lower, + ...prose.filler, + ]) { + expect(word.normalize("NFD"), `${locale}: ${word}`).not.toMatch(/[̀-ͯ]/u); + expect(word, `${locale}: ${word}`).toBe(word.toLowerCase()); + } + } + }); +}); diff --git a/src/lib/labs/ocr-extract.ts b/src/lib/labs/ocr-extract.ts index fcdb01563..4329211b8 100644 --- a/src/lib/labs/ocr-extract.ts +++ b/src/lib/labs/ocr-extract.ts @@ -24,7 +24,10 @@ import { } from "@/lib/ai/types"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { prisma } from "@/lib/db"; -import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { + isUnreadableRange, + parseReferenceRange, +} from "@/lib/labs/parse-reference-range"; import { annotate } from "@/lib/logging/context"; import { extractedLabsSchema, @@ -270,7 +273,16 @@ export async function runOcrExtraction( // Cap defensively (the schema already bounds it) and annotate each row. const rows: OcrExtractedRowDto[] = []; + // How many rows printed a window the parser could not read. Those rows fall + // back to whatever bounds the model itself reported, or to none — either way + // the notation went unread, and a count of it is the difference between + // noticing that a report layout or a language is unsupported and never + // hearing about it. A count only: the string is the user's document. + let unreadableRanges = 0; for (const row of envelope.rows.slice(0, OCR_MAX_ROWS)) { + if (isUnreadableRange(parseReferenceRange(row.referenceText, row.unit))) { + unreadableRanges += 1; + } rows.push(await annotateRow(args.userId, row, reportDate)); } @@ -282,6 +294,7 @@ export async function runOcrExtraction( rows: rows.length, duplicates: rows.filter((r) => r.duplicateOf !== null).length, newBiomarkers: rows.filter((r) => r.biomarkerMatch === "new").length, + unreadableRanges, }, }); diff --git a/src/lib/labs/parse-reference-range.ts b/src/lib/labs/parse-reference-range.ts index 24e413f37..f16765ff6 100644 --- a/src/lib/labs/parse-reference-range.ts +++ b/src/lib/labs/parse-reference-range.ts @@ -19,14 +19,31 @@ * plausible bound out of a string it did not understand, because a fake * window silently reclassifies a reading as normal or abnormal. * - * Notations read (German and English, comma- or dot-decimal): + * Notations read (comma- or dot-decimal): * * two-sided 3.5-5.0 · 3,5 – 5,0 · 3,5 bis 5,0 · 3.5 to 5.0 · 3,5 ... 5,0 - * von 3,5 bis 5,0 · [3,5 - 5,0] · (3.5-5.0) + * von 3,5 bis 5,0 · 3,5 à 5,0 · de 3,5 a 5,0 · da 3,5 a 5,0 · + * od 3,5 do 5,0 · [3,5 - 5,0] · (3.5-5.0) * upper only < 5 · ≤ 5 · <= 5 · bis 5,0 · bis zu 5,0 · max. 5,0 · - * kleiner 5 · unter 5 · up to 5.0 · less than 5 + * kleiner 5 · unter 5 · up to 5.0 · less than 5 · jusqu'à 5,0 · + * hasta 5,0 · fino a 5,0 · do 5,0 · poniżej 5,0 * lower only > 3.5 · ≥ 3,5 · >= 3.5 · ab 3,5 · min. 3,5 · mindestens 3,5 · - * größer 3,5 · über 3,5 · at least 3.5 + * größer 3,5 · über 3,5 · at least 3.5 · à partir de 3,5 · + * desde 3,5 · da 3,5 · od 3,5 · powyżej 3,5 + * + * WHY THE VOCABULARY IS TABULATED AND NOT DERIVED. Elsewhere in the app, a + * matcher that has to recognise a word list in six languages builds its index + * out of `messages/.json`, so a seventh locale works the day its + * bundle lands. That does not apply here. "jusqu'à", "hasta", "fino a" and + * "do" are prose a LAB printed on a sheet of paper; the app owns no string + * they could be derived from, and the language of the report has nothing to do + * with the language of the UI — a French self-hoster running the app in + * English still receives French reports. So the table below is hand-written, + * every locale's prose is tried against every input at once, and the honest + * cost is written down: a report printed in a language absent from the table + * yields text-only. `RANGE_PROSE` is keyed by the shipped `Locale` union + * precisely so that adding a locale to `locales` fails to compile until + * someone has answered "and how does that language print a range?". * * Units are read, never converted. A window may carry the unit the report * printed it in; when the caller states the reading's own unit and the two @@ -34,6 +51,11 @@ * mmol/L into mg/dL needs a per-analyte factor, so doing it here would be the * parser deciding what the report meant. */ +import { type Locale } from "@/lib/i18n/config"; +import { + normaliseApostrophes, + stripDiacritics, +} from "@/lib/i18n/fold-for-match"; /** What a printed range yielded. `text` is set whenever the input was non-empty. */ export interface ParsedReferenceRange { @@ -58,14 +80,224 @@ const MAX_ABS_BOUND = 1e12; /** One number token: optional sign, digits, any number of `.`/`,` groups. */ const NUM = String.raw`[+-]?\d+(?:[.,]\d+)*`; -/** Infix separators for a two-sided window. */ -const TWO_SIDED_SEP = String.raw`(?:-|–|—|‒|…|\.{2,3}|bis\s+zu|bis|to|until)`; +/** How one language prints a range. Every field is a list of literal words. */ +export interface RangeProse { + /** Infix words joining two bounds: "3,5 5,0". */ + twoSided: string[]; + /** Words that introduce a ceiling: " 5,0". */ + upper: string[]; + /** Words that introduce a floor: " 3,5". */ + lower: string[]; + /** + * Words that only announce that a window follows and carry no bound of + * their own ("von 3,5 bis 5,0"). Stripped and the input retried — never on + * the first pass, because several of them ("da", "od", "de") are also a + * floor prefix in their own language, and stripping first would read + * "da 3,5" as a bare number. + */ + filler: string[]; +} + +/** + * The prose, per shipped locale. Written ALREADY FOLDED — lower case and + * accent-free — because the input is diacritic-folded before matching, so + * "jusqu'à", "jusqu'a" and "JUSQU'À" are one row. The German "ß" survives the + * fold (it has no canonical decomposition), which is why "größer" appears here + * as "großer"; its "oe"/"ss" twins are listed beside it, exactly as the + * hand-written alternation carried them before. + * + * The `de` and `en` lists are the pre-existing alternation, split apart word + * for word and nothing more — this change is about the four locales that had + * no entry at all, and a German or English range must read the same afterwards + * as it did before. + * + * Every list is applied to every input regardless of the reader's locale — see + * the module comment for why. Multi-word entries match across any run of + * whitespace, so a report that printed a non-breaking space still reads. + */ +export const RANGE_PROSE: Record = { + de: { + twoSided: ["bis zu", "bis"], + upper: [ + "bis zu", + "bis", + "max.", + "maximal", + "max", + "kleiner als", + "kleiner", + "unter", + "hochstens", + "hoechstens", + ], + lower: [ + "ab", + "mind.", + "mindestens", + "minimal", + "min.", + "min", + "großer als", + "großer", + "groesser als", + "groesser", + "uber", + "ueber", + ], + filler: ["von", "referenz", "ref.", "ref"], + }, + en: { + twoSided: ["to", "until"], + upper: ["up to", "less than", "below"], + lower: ["at least", "greater than", "above"], + filler: ["from", "reference", "ref.", "ref"], + }, + fr: { + twoSided: ["a", "et"], + upper: [ + "jusqu'a", + "jusque", + "inferieur a", + "inferieure a", + "moins de", + "au plus", + "au maximum", + ], + lower: [ + "a partir de", + "superieur a", + "superieure a", + "plus de", + "au moins", + "au minimum", + ], + filler: [ + "de", + "entre", + "valeurs de reference", + "valeur de reference", + "intervalle de reference", + "plage de reference", + ], + }, + es: { + twoSided: ["a", "y"], + upper: [ + "hasta", + "menor que", + "menor de", + "menor a", + "inferior a", + "menos de", + "como maximo", + "maximo", + ], + lower: [ + "desde", + "a partir de", + "mayor que", + "mayor de", + "mayor a", + "superior a", + "mas de", + "al menos", + "como minimo", + "minimo", + ], + filler: [ + "de", + "entre", + "valores de referencia", + "valor de referencia", + "rango de referencia", + "intervalo de referencia", + ], + }, + it: { + twoSided: ["a", "e"], + upper: [ + "fino a", + "inferiore a", + "minore di", + "meno di", + "al massimo", + "massimo", + ], + lower: [ + "da", + "oltre", + "superiore a", + "maggiore di", + "piu di", + "almeno", + "al minimo", + "minimo", + ], + filler: [ + "da", + "tra", + "fra", + "valori di riferimento", + "valore di riferimento", + "intervallo di riferimento", + ], + }, + pl: { + twoSided: ["do"], + upper: ["do", "ponizej", "mniej niz", "nie wiecej niz", "maks.", "maks"], + lower: [ + "od", + "powyzej", + "wiecej niz", + "co najmniej", + "nie mniej niz", + "minimum", + ], + filler: [ + "od", + "zakres referencyjny", + "wartosci referencyjne", + "normy laboratoryjne", + ], + }, +}; + +/** Escape a literal for regex use and let any run of whitespace join its words. */ +function literalToPattern(word: string): string { + return word + .split(/\s+/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join(String.raw`\s+`); +} + +/** + * Alternation over every locale's list for one field, longest literal first so + * the reader sees the specific form ahead of the prefix it contains. + */ +function proseAlternation(field: keyof RangeProse): string { + const words = new Set(); + for (const prose of Object.values(RANGE_PROSE)) { + for (const word of prose[field]) words.add(word); + } + return [...words] + .sort((a, b) => b.length - a.length) + .map(literalToPattern) + .join("|"); +} + +/** + * Symbolic infix separators. Kept apart from the word separators because they + * need no surrounding whitespace, while a word separator must have it: "a", + * "e" and "y" are one letter long, and a bare `e` that could sit between two + * digits would read "3.5e5" as a window. + */ +const SYMBOL_SEP = String.raw`-|–|—|‒|…|\.{2,3}`; /** Prefixes that state a ceiling only. */ -const UPPER_PREFIX = String.raw`(?:<=|<|≤|bis\s+zu|bis|max\.|maximal|max|kleiner\s+als|kleiner|unter|höchstens|hoechstens|up\s+to|less\s+than|below)`; +const UPPER_PREFIX = `(?:<=|<|≤|${proseAlternation("upper")})`; /** Prefixes that state a floor only. */ -const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|min|größer\s+als|größer|groesser\s+als|groesser|über|ueber|at\s+least|greater\s+than|above)`; +const LOWER_PREFIX = `(?:>=|>|≥|${proseAlternation("lower")})`; /** * Trailing unit tail. Deliberately whitespace-free and comma-free: a real unit @@ -76,14 +308,19 @@ const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|m const UNIT_TAIL = String.raw`(?:\s*([^\s,]{1,24}))?`; const TWO_SIDED_RE = new RegExp( - `^(${NUM})\\s*${TWO_SIDED_SEP}\\s*(${NUM})${UNIT_TAIL}$`, + `^(${NUM})(?:\\s*(?:${SYMBOL_SEP})\\s*|\\s+(?:${proseAlternation( + "twoSided", + )})\\s+)(${NUM})${UNIT_TAIL}$`, "iu", ); const UPPER_RE = new RegExp(`^${UPPER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu"); const LOWER_RE = new RegExp(`^${LOWER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu"); /** Leading words that only introduce the window ("von 3,5 bis 5,0"). */ -const LEADING_FILLER_RE = /^(?:von|from|ref\.?|referenz|reference)\s+/iu; +const LEADING_FILLER_RE = new RegExp( + `^(?:${proseAlternation("filler")})\\s+`, + "iu", +); /** * Read one number token into a JS number, or null when its separators are @@ -148,33 +385,18 @@ function unitsAgree(printed: string | undefined, reading: string | undefined) { } /** - * Parse the reference range a lab report printed for one reading. - * - * @param raw the printed string, exactly as transcribed. - * @param unit the reading's own unit, when known. A printed window in a - * different unit yields text-only — never a converted bound. - * @returns null when there was nothing printed at all. + * Try the three notations against one candidate string. Null means none of + * them matched; a result with both bounds null means one matched but the + * numbers or the unit refused it, which is a final answer and not a reason to + * retry a differently-trimmed candidate. */ -export function parseReferenceRange( - raw: string | null | undefined, - unit?: string | null, +function matchNotation( + candidate: string, + text: string, + unit: string | null | undefined, ): ParsedReferenceRange | null { - if (raw === null || raw === undefined) return null; - const text = raw - .replace(/\s+/gu, " ") - .trim() - .slice(0, SOURCE_REFERENCE_TEXT_MAX); - if (text === "") return null; - const noBounds: ParsedReferenceRange = { low: null, high: null, text }; - // Strip a bracket wrapper and an introducing word for MATCHING only — the - // stored text stays exactly what the report printed. - let candidate = text; - const bracketed = /^[[(](.*)[\])]$/u.exec(candidate); - if (bracketed) candidate = bracketed[1].trim(); - candidate = candidate.replace(LEADING_FILLER_RE, "").trim(); - const two = TWO_SIDED_RE.exec(candidate); if (two) { if (!unitsAgree(two[3], unit ?? undefined)) return noBounds; @@ -201,7 +423,76 @@ export function parseReferenceRange( return low === null ? noBounds : { low, high: null, text }; } + return null; +} + +/** + * Parse the reference range a lab report printed for one reading. + * + * @param raw the printed string, exactly as transcribed. + * @param unit the reading's own unit, when known. A printed window in a + * different unit yields text-only — never a converted bound. + * @returns null when there was nothing printed at all. A non-null result with + * both bounds null means the report DID print something the parser + * could not read — see `isUnreadableRange`, which is the difference + * between "no window" and "a window nobody checked". + */ +export function parseReferenceRange( + raw: string | null | undefined, + unit?: string | null, +): ParsedReferenceRange | null { + if (raw === null || raw === undefined) return null; + const text = raw + .replace(/\s+/gu, " ") + .trim() + .slice(0, SOURCE_REFERENCE_TEXT_MAX); + if (text === "") return null; + + // Fold for MATCHING only — the stored text stays exactly what the report + // printed. Accents go so a table written "jusqu'a" meets a report's + // "jusqu'à"; the curled apostrophe a word processor emits becomes the + // straight one the table carries. Everything else, notably the digits and + // their separators, is left alone. + let candidate = normaliseApostrophes(stripDiacritics(text)); + const bracketed = /^[[(](.*)[\])]$/u.exec(candidate); + if (bracketed) candidate = bracketed[1].trim(); + + // Untrimmed first. Several introducing words are also a bound prefix in + // their own language — Italian "da", Polish "od", French "de" — so stripping + // before matching would read "da 3,5" as a bare number with no floor. + const direct = matchNotation(candidate, text, unit); + if (direct) return direct; + + const trimmed = candidate.replace(LEADING_FILLER_RE, "").trim(); + if (trimmed !== candidate) { + const viaFiller = matchNotation(trimmed, text, unit); + if (viaFiller) return viaFiller; + } + // Anything else — "negativ", "siehe Befund", a bare number, prose — keeps its // text and states no window. - return noBounds; + return { low: null, high: null, text }; +} + +/** + * True when the report printed a range and the parser could not derive a + * single bound from it. + * + * This is the case worth naming. `null` from the parser means the report + * stated nothing, and the reading falls back to the biomarker's own band as it + * always did. A parsed result with two null bounds means something else + * entirely: the lab DID state a window, on this reading, and the app is about + * to judge the value against a different band or against none at all. Left + * unnamed it reads identically to "no range" at every call site, which is how + * a French account's `jusqu'à 5,0` became a reading nobody ever flagged. + * + * Callers are expected to SAY so rather than to guess a bound — the manual + * form shows the reader that the string was kept but not read, and the ingest + * paths annotate it so an unreadable notation shows up as a number in the wide + * event instead of as silence. + */ +export function isUnreadableRange( + parsed: ParsedReferenceRange | null | undefined, +): boolean { + return !!parsed && parsed.low === null && parsed.high === null; }