diff --git a/src/lib/documents/__tests__/search-synonyms.test.ts b/src/lib/documents/__tests__/search-synonyms.test.ts new file mode 100644 index 000000000..c99efb1d8 --- /dev/null +++ b/src/lib/documents/__tests__/search-synonyms.test.ts @@ -0,0 +1,89 @@ +/** + * v1.37.20 — the query-time synonym expansion behind document search. + * + * Watched red: the cross-register assertion below ("blutdruck" query hash + * set overlaps a "blood pressure" document's token set) fails against the + * pre-expansion `hashQueryTokens`, which hashed only the typed tokens. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db", () => ({ prisma: {} })); +vi.mock("@/lib/crypto", () => ({ + // Deterministic fake subkey — the HMAC itself is irrelevant here; only + // that query and index hashing agree. + deriveSubkey: vi.fn(() => Buffer.alloc(32, 7)), +})); + +import { + expandQueryTokens, + MAX_EXPANDED_QUERY_TOKENS, + SEARCH_SYNONYM_GROUPS, +} from "../search-synonyms"; +import { hashQueryTokens, tokenise, tokeniseAndHash } from "../content-index"; + +describe("the curated groups", () => { + it("are all in tokeniser normal form (lowercase, no diacritics, length-gated)", () => { + for (const group of SEARCH_SYNONYM_GROUPS) { + for (const token of group) { + // Round-tripping through the real tokeniser must return the token + // unchanged — otherwise it could never match an indexed tag. + expect(tokenise(token), token).toEqual([token]); + } + } + }); + + it("hold at least two members each (a group of one expands nothing)", () => { + for (const group of SEARCH_SYNONYM_GROUPS) { + expect(group.length).toBeGreaterThanOrEqual(2); + } + }); +}); + +describe("expandQueryTokens", () => { + it("keeps the typed tokens first and appends the group members", () => { + const out = expandQueryTokens(["blutdruck"]); + expect(out[0]).toBe("blutdruck"); + expect(out).toContain("blood"); + expect(out).toContain("pressure"); + expect(out).toContain("hypertension"); + }); + + it("passes an unknown token through untouched", () => { + expect(expandQueryTokens(["zeruwatvz"])).toEqual(["zeruwatvz"]); + }); + + it("caps the expanded set without ever trimming typed tokens", () => { + const typed = SEARCH_SYNONYM_GROUPS.slice(0, 24).map((g) => g[0]); + const out = expandQueryTokens(typed); + expect(out.length).toBeLessThanOrEqual(MAX_EXPANDED_QUERY_TOKENS); + for (const token of typed) expect(out).toContain(token); + }); +}); + +describe("hashQueryTokens with expansion", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("lets a German query match an English document (cross-register recall)", () => { + const documentTags = tokeniseAndHash( + "Blood pressure readings were stable throughout the stay", + ); + const queryTags = hashQueryTokens("Blutdruck"); + expect(queryTags.some((tag) => documentTags.includes(tag))).toBe(true); + }); + + it("lets an English query match a German letter", () => { + const documentTags = tokeniseAndHash( + "Der Blutdruck war im Verlauf unauffaellig", + ); + const queryTags = hashQueryTokens("hypertension"); + expect(queryTags.some((tag) => documentTags.includes(tag))).toBe(true); + }); + + it("still matches the typed word itself, expansion or not", () => { + const documentTags = tokeniseAndHash("Koloskopie ohne Befund"); + const queryTags = hashQueryTokens("Koloskopie"); + expect(queryTags.some((tag) => documentTags.includes(tag))).toBe(true); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index b71ca769b..4b88616f3 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -27,6 +27,7 @@ import { createHmac } from "node:crypto"; import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { deriveSubkey } from "@/lib/crypto"; import { prisma } from "@/lib/db"; +import { expandQueryTokens } from "@/lib/documents/search-synonyms"; /** * Tokeniser algorithm version. Bump when the normalisation / tokenisation rules @@ -249,7 +250,12 @@ export function tokeniseAndHash(text: string): string[] { */ export function hashQueryTokens(query: string): string[] { const subkey = indexSubkey(); - const tokens = tokenise(query).slice(0, MAX_QUERY_TOKENS); + // v1.37.20 — expand through the curated German/English medical synonym + // groups (`search-synonyms.ts`) BEFORE hashing, so "Blutdruck" also + // matches a letter that says "blood pressure". Query-time only: the + // stored index and the tokenizer version are untouched, and the expansion + // happens before the one-way hash, so nothing readable moves anywhere new. + const tokens = expandQueryTokens(tokenise(query).slice(0, MAX_QUERY_TOKENS)); const hashes = new Set(); for (const token of tokens) hashes.add(hashToken(token, subkey)); return [...hashes]; diff --git a/src/lib/documents/search-synonyms.ts b/src/lib/documents/search-synonyms.ts new file mode 100644 index 000000000..a9b142e3b --- /dev/null +++ b/src/lib/documents/search-synonyms.ts @@ -0,0 +1,118 @@ +/** + * v1.37.20 — German/English medical synonym groups for document search. + * + * The blind token index (see `content-index.ts`) does whole-token equality + * only, so a search for "Blutdruck" never found the letter that says "blood + * pressure" and a search for "Zucker" missed "Glukose". This module closes + * that gap at QUERY time: the query's tokens are expanded with their group + * members before hashing, so the index itself never changes — no re-index, + * no tokenizer-version bump, no schema movement. Recall widens; the match + * still runs over the same opaque hashes. + * + * Deliberately a static curated list. Each group is a set of interchangeable + * search terms, stored in the SAME normal form the tokeniser produces: + * lowercase, diacritics stripped ("röntgen" → "rontgen"), single tokens of + * three characters or more (shorter ones never reach the index). A + * multi-word clinical phrase appears as its individual content-bearing + * tokens; array-overlap matching means a partial hit still surfaces the + * document, which is the right trade for a search box. + * + * Curation rule: common terms a German- or English-speaking self-hoster + * would type against a letter written in the other register (lay term ↔ + * clinical term, German ↔ English). No vendor names, no drug brand names. + */ + +/** Interchangeable search-term groups, in tokeniser normal form. */ +export const SEARCH_SYNONYM_GROUPS: ReadonlyArray> = [ + // Vitals + ["blutdruck", "blood", "pressure", "hypertonie", "hypertension"], + ["puls", "pulse", "herzfrequenz", "heart", "rate"], + ["gewicht", "weight", "korpergewicht", "body"], + ["blutzucker", "zucker", "glukose", "glucose", "sugar", "diabetes"], + ["sauerstoffsattigung", "sauerstoff", "oxygen", "saturation", "spo2"], + ["temperatur", "temperature", "fieber", "fever"], + // Common labs + ["blutbild", "hamogramm", "hemogram", "cbc", "blutwerte", "laborwerte"], + ["cholesterin", "cholesterol", "ldl", "hdl", "blutfette", "lipide", "lipids"], + ["hba1c", "langzeitzucker", "glykohamoglobin"], + ["schilddruse", "thyroid", "tsh"], + ["leber", "liver", "leberwerte", "got", "gpt", "transaminasen"], + ["niere", "kidney", "renal", "kreatinin", "creatinine", "gfr"], + ["eisen", "iron", "ferritin", "hamoglobin", "hemoglobin", "anamie", "anemia"], + ["vitamin", "vitamine"], + ["entzundung", "inflammation", "crp"], + ["harnsaure", "urat", "urate", "gicht", "gout"], + // Imaging / procedures + ["rontgen", "xray", "radiograph"], + ["ultraschall", "sonographie", "sonografie", "ultrasound"], + ["mrt", "mri", "kernspintomographie", "kernspin"], + ["computertomographie", "tomographie"], + ["ekg", "ecg", "elektrokardiogramm", "electrocardiogram"], + ["darmspiegelung", "koloskopie", "colonoscopy"], + ["magenspiegelung", "gastroskopie", "gastroscopy"], + // Specialties / encounter words + ["impfung", "impfpass", "vaccination", "vaccine", "immunisierung"], + ["hausarzt", "allgemeinmedizin", "practitioner"], + ["kardiologie", "kardiologe", "cardiology", "herz", "cardiac"], + ["dermatologie", "hautarzt", "dermatology", "haut", "skin"], + ["orthopadie", "orthopade", "orthopedics", "orthopaedics"], + ["augenarzt", "ophthalmologie", "ophthalmology", "auge", "eye"], + ["zahnarzt", "dental", "dentist", "zahn", "tooth"], + ["gynakologie", "frauenarzt", "gynecology", "gynaecology"], + ["urologie", "urologe", "urology"], + // Documents / admin + ["arztbrief", "befund", "bericht", "letter", "report", "findings"], + ["uberweisung", "referral"], + ["rezept", "verordnung", "prescription"], + ["entlassung", "entlassungsbrief", "discharge"], + ["operation", "surgery", "eingriff"], + ["allergie", "allergy", "unvertraglichkeit", "intolerance"], + ["medikament", "medikation", "medication", "arznei"], + ["schwangerschaft", "pregnancy"], + ["schlaf", "sleep", "apnoe", "apnea"], +]; + +/** Cap on the expanded token set — bounds the hash array the query matches. */ +export const MAX_EXPANDED_QUERY_TOKENS = 64; + +// Token → its group members, built once at module load. +const SYNONYMS_BY_TOKEN: ReadonlyMap> = (() => { + const map = new Map(); + for (const group of SEARCH_SYNONYM_GROUPS) { + for (const token of group) { + const existing = map.get(token); + if (existing) { + // A token may sit in more than one group ("blood" rides both the + // pressure and the count groups); merge, keep unique. + for (const member of group) { + if (!existing.includes(member)) existing.push(member); + } + } else { + map.set(token, [...group]); + } + } + } + return map; +})(); + +/** + * Expand normalised query tokens with their synonym-group members. The + * original tokens always survive (and lead the list, so the cap trims + * expansions before it ever trims what the user actually typed). + */ +export function expandQueryTokens(tokens: readonly string[]): string[] { + const out: string[] = []; + const seen = new Set(); + const push = (token: string) => { + if (seen.has(token) || out.length >= MAX_EXPANDED_QUERY_TOKENS) return; + seen.add(token); + out.push(token); + }; + for (const token of tokens) push(token); + for (const token of tokens) { + const members = SYNONYMS_BY_TOKEN.get(token); + if (!members) continue; + for (const member of members) push(member); + } + return out; +}