mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 00:12:18 +00:00
test(search): cover the matcher, the scan bounds and the route
This commit is contained in:
1 parent
1766ffa3d2
commit
9b86ec2e10
4 files changed
+471
-10
No files matched your search
@@ -0,0 +1,141 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
foldForMatch,
|
||||
isHit,
|
||||
markMatches,
|
||||
scoreForQuery,
|
||||
queryWords,
|
||||
scoreCandidate,
|
||||
snippetOf,
|
||||
WEIGHT,
|
||||
type SearchCandidate,
|
||||
} from "../match";
|
||||
|
||||
function candidate(
|
||||
fields: SearchCandidate["fields"],
|
||||
): Pick<SearchCandidate, "fields"> {
|
||||
return { fields };
|
||||
}
|
||||
|
||||
const hit = (text: string, query: string, weight: number = WEIGHT.primary) =>
|
||||
isHit(scoreForQuery(candidate([{ text, weight }]), query));
|
||||
|
||||
/** The marked substrings, for readable assertions. */
|
||||
const marks = (text: string, query: string) =>
|
||||
markMatches(text, query).map(([s, e]) => text.slice(s, e));
|
||||
|
||||
describe("folding", () => {
|
||||
it("ignores case, accents and the German keyboard spellings", () => {
|
||||
expect(foldForMatch("Schulter")).toBe("schulter");
|
||||
expect(foldForMatch("Hüfte")).toBe(foldForMatch("Huefte"));
|
||||
expect(foldForMatch("Fuß")).toBe(foldForMatch("Fuss"));
|
||||
expect(foldForMatch("Müdigkeit")).toBe("mudigkeit");
|
||||
expect(foldForMatch(" FSME Impfung ")).toBe("fsme impfung");
|
||||
});
|
||||
|
||||
it("matches a term whatever way either side spells the umlaut", () => {
|
||||
expect(hit("Schulter links", "schulter")).toBe(true);
|
||||
expect(hit("schulter links", "SCHULTER")).toBe(true);
|
||||
expect(hit("Hüftschmerzen", "hueft")).toBe(true);
|
||||
expect(hit("Hueftschmerzen", "hüft")).toBe(true);
|
||||
expect(hit("Fußgelenk", "fuss")).toBe(true);
|
||||
expect(hit("Schläfrig", "schlaefrig")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match what is not there", () => {
|
||||
expect(hit("Knie rechts", "schulter")).toBe(false);
|
||||
// A short word must start a word: "arm" is not inside "Darm".
|
||||
expect(hit("Darmspiegelung", "arm")).toBe(false);
|
||||
expect(hit("Armbruch", "arm")).toBe(true);
|
||||
});
|
||||
|
||||
it("finds a body part across languages", () => {
|
||||
expect(hit("Schulter", "shoulder")).toBe(true);
|
||||
expect(hit("left shoulder", "Schulter")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("queryWords", () => {
|
||||
it("drops filler and keeps the tokeniser's form", () => {
|
||||
expect(queryWords("meine letzte Schulter OP")).toEqual(["schulter"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("marks", () => {
|
||||
it("marks the original characters, not the folded ones", () => {
|
||||
expect(marks("Hüftgelenk und Hüfte", "hueft")).toEqual(["Hüft", "Hüft"]);
|
||||
expect(marks("Fußgelenk", "fuss")).toEqual(["Fuß"]);
|
||||
expect(marks("Huefte", "hüfte")).toEqual(["Huefte"]);
|
||||
expect(marks("SCHULTER links", "schulter")).toEqual(["SCHULTER"]);
|
||||
});
|
||||
|
||||
it("marks a short word only at a word start", () => {
|
||||
expect(marks("Arm und Darm", "arm")).toEqual(["Arm"]);
|
||||
});
|
||||
|
||||
it("marks a query too short for the tokeniser by itself", () => {
|
||||
expect(marks("Hb und HbA1c", "hb")).toEqual(["Hb", "Hb"]);
|
||||
});
|
||||
|
||||
it("merges overlapping marks", () => {
|
||||
expect(markMatches("Schulterschmerz", "schulter schulterschmerz")).toEqual([
|
||||
[0, 15],
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps ranges inside the text for multi-unit characters", () => {
|
||||
const text = "😀 Schulter";
|
||||
const [[start, end]] = markMatches(text, "schulter");
|
||||
expect(text.slice(start, end)).toBe("Schulter");
|
||||
});
|
||||
});
|
||||
|
||||
describe("snippetOf", () => {
|
||||
it("returns the whole short text with its marks", () => {
|
||||
const s = snippetOf("Schmerzen in der Schulter", "schulter");
|
||||
expect(s?.text).toBe("Schmerzen in der Schulter");
|
||||
expect(s?.marks).toEqual([[17, 25]]);
|
||||
});
|
||||
|
||||
it("cuts a long text around the first match at word boundaries", () => {
|
||||
const filler = "Lorem ipsum dolor sit amet ".repeat(20);
|
||||
const s = snippetOf(`${filler}die Schulter tut weh ${filler}`, "schulter");
|
||||
expect(s).not.toBeNull();
|
||||
expect(s!.text.startsWith("…")).toBe(true);
|
||||
expect(s!.text.endsWith("…")).toBe(true);
|
||||
expect(s!.text.length).toBeLessThanOrEqual(142);
|
||||
const [[start, end]] = s!.marks;
|
||||
expect(s!.text.slice(start, end)).toBe("Schulter");
|
||||
});
|
||||
|
||||
it("is null when the text does not contain the query", () => {
|
||||
expect(snippetOf("Knie", "schulter")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("ranking inputs", () => {
|
||||
it("a title hit scores above a note hit", () => {
|
||||
const title = scoreForQuery(
|
||||
candidate([{ text: "Schulter", weight: WEIGHT.primary }]),
|
||||
"schulter",
|
||||
);
|
||||
const note = scoreForQuery(
|
||||
candidate([{ text: "Schulter", weight: WEIGHT.tertiary, body: true }]),
|
||||
"schulter",
|
||||
);
|
||||
expect(title.score).toBeGreaterThan(note.score);
|
||||
});
|
||||
|
||||
it("counts each matched word once", () => {
|
||||
const s = scoreCandidate(
|
||||
candidate([
|
||||
{ text: "Schulter links", weight: WEIGHT.primary },
|
||||
{ text: "links", weight: WEIGHT.secondary },
|
||||
]),
|
||||
["schulter", "links"],
|
||||
"schulter links",
|
||||
);
|
||||
expect(s.matched).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* The bounds of the record search and the masks that pick its sources. The
|
||||
* reads themselves are exercised against Postgres in
|
||||
* `tests/integration/record-search.test.ts`.
|
||||
*
|
||||
* Mutation checks (each run, each seen red):
|
||||
* - drop the deadline check in `scanNewest` → "stops when the budget has
|
||||
* run out" goes red;
|
||||
* - drop the `scanned >= maxRows` stop → "never reads more than the cap"
|
||||
* goes red;
|
||||
* - drop the domain test from `readableSources` → "drops a source the grant
|
||||
* does not cover" goes red.
|
||||
*/
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/db", () => ({ prisma: {} }));
|
||||
|
||||
import type { SearchCandidate } from "../match";
|
||||
import {
|
||||
readableSources,
|
||||
scanNewest,
|
||||
type ScanBudget,
|
||||
} from "../search-records";
|
||||
|
||||
function budget(deadlineIn = 10_000, clock = { t: 0 }): ScanBudget {
|
||||
return {
|
||||
deadline: clock.t + deadlineIn,
|
||||
now: () => clock.t,
|
||||
truncated: false,
|
||||
rows: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** A source of `total` rows, newest first, every `hitEvery`-th one a hit. */
|
||||
function source(total: number, hitEvery = 1) {
|
||||
const rows = Array.from({ length: total }, (_, i) => ({ id: `r${i}`, i }));
|
||||
const reads: number[] = [];
|
||||
return {
|
||||
reads,
|
||||
read: async (cursor: string | null, take: number) => {
|
||||
const start = cursor ? rows.findIndex((r) => r.id === cursor) + 1 : 0;
|
||||
reads.push(take);
|
||||
return rows.slice(start, start + take);
|
||||
},
|
||||
match: (row: { id: string; i: number }) =>
|
||||
row.i % hitEvery === 0
|
||||
? ({ id: row.id } as unknown as SearchCandidate)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("scanNewest", () => {
|
||||
it("reads a small source to its end and is not truncated", async () => {
|
||||
const b = budget();
|
||||
const s = source(30, 10);
|
||||
const out = await scanNewest({ budget: b, wanted: 5, batch: 10, ...s });
|
||||
expect(out.map((c) => c.id)).toEqual(["r0", "r10", "r20"]);
|
||||
expect(b.truncated).toBe(false);
|
||||
expect(b.rows).toBe(30);
|
||||
});
|
||||
|
||||
it("stops once it holds more hits than wanted", async () => {
|
||||
const b = budget();
|
||||
const s = source(10_000, 1);
|
||||
const out = await scanNewest({ budget: b, wanted: 3, batch: 10, ...s });
|
||||
expect(out.length).toBe(10);
|
||||
expect(s.reads).toEqual([10]);
|
||||
expect(b.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("never reads more than the cap, and says so", async () => {
|
||||
const b = budget();
|
||||
const s = source(10_000, 1_000_000);
|
||||
await scanNewest({ budget: b, wanted: 3, batch: 100, maxRows: 450, ...s });
|
||||
expect(b.rows).toBe(450);
|
||||
expect(s.reads).toEqual([100, 100, 100, 100, 50]);
|
||||
expect(b.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("stops when the budget has run out, and says so", async () => {
|
||||
const clock = { t: 0 };
|
||||
const b = budget(100, clock);
|
||||
const s = source(10_000, 1_000_000);
|
||||
const read = async (cursor: string | null, take: number) => {
|
||||
clock.t += 60;
|
||||
return s.read(cursor, take);
|
||||
};
|
||||
await scanNewest({ budget: b, wanted: 3, batch: 10, read, match: s.match });
|
||||
expect(s.reads.length).toBe(2);
|
||||
expect(b.truncated).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("readableSources", () => {
|
||||
const all = () => true;
|
||||
|
||||
it("reads everything on an owner's record with every module on", () => {
|
||||
expect(readableSources({}, all)).toHaveLength(13);
|
||||
});
|
||||
|
||||
it("drops the sources of a switched-off module, never a core one", () => {
|
||||
const sources = readableSources(
|
||||
{ illness: false, mood: false, inboundDocuments: false },
|
||||
all,
|
||||
);
|
||||
expect(sources).not.toContain("condition");
|
||||
expect(sources).not.toContain("symptom");
|
||||
expect(sources).not.toContain("symptomNote");
|
||||
expect(sources).not.toContain("illnessNote");
|
||||
expect(sources).not.toContain("moodNote");
|
||||
expect(sources).not.toContain("document");
|
||||
expect(sources).toContain("visit");
|
||||
expect(sources).toContain("measurementNote");
|
||||
});
|
||||
|
||||
it("drops a source the grant does not cover", () => {
|
||||
const sources = readableSources({}, (domain) => domain !== "mind");
|
||||
expect(sources).not.toContain("moodNote");
|
||||
expect(sources).toContain("visit");
|
||||
});
|
||||
});
|
||||
+33
-10
@@ -285,6 +285,18 @@ export function scoreCandidate(
|
||||
return { matched, score, phrase };
|
||||
}
|
||||
|
||||
/** `scoreCandidate` for a raw query. */
|
||||
export function scoreForQuery(
|
||||
candidate: Pick<SearchCandidate, "fields" | "contentWords">,
|
||||
query: string,
|
||||
): CandidateScore {
|
||||
return scoreCandidate(
|
||||
candidate,
|
||||
queryWords(query),
|
||||
normalizeSearchText(query),
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether a scored candidate stays in the result at all. */
|
||||
export function isHit(score: CandidateScore): boolean {
|
||||
return score.matched > 0 || score.phrase;
|
||||
@@ -355,7 +367,9 @@ function foldWithMap(text: string): {
|
||||
from: number[];
|
||||
to: number[];
|
||||
} {
|
||||
const chars: string[] = [];
|
||||
// One entry per UTF-16 unit of the folded string, so an index `indexOf`
|
||||
// returns can be looked up directly, astral characters included.
|
||||
const units: string[] = [];
|
||||
const from: number[] = [];
|
||||
const to: number[] = [];
|
||||
// Whether the vowel at that position has already taken in its "e".
|
||||
@@ -367,27 +381,29 @@ function foldWithMap(text: string): {
|
||||
if (/\s/u.test(ch)) piece = " ";
|
||||
piece = piece.replace(/ß/g, "ss");
|
||||
for (const out of piece) {
|
||||
const prev = chars.length - 1;
|
||||
const prev = units.length - 1;
|
||||
// "ae", "oe", "ue" fold onto the vowel: the "e" widens the vowel's
|
||||
// source range instead of adding a character.
|
||||
if (
|
||||
out === "e" &&
|
||||
prev >= 0 &&
|
||||
/[aou]/.test(chars[prev]) &&
|
||||
/[aou]/.test(units[prev]) &&
|
||||
!absorbed[prev]
|
||||
) {
|
||||
to[prev] = end;
|
||||
absorbed[prev] = true;
|
||||
continue;
|
||||
}
|
||||
chars.push(out);
|
||||
from.push(index);
|
||||
to.push(end);
|
||||
absorbed.push(false);
|
||||
for (let k = 0; k < out.length; k++) {
|
||||
units.push(out[k]);
|
||||
from.push(index);
|
||||
to.push(end);
|
||||
absorbed.push(false);
|
||||
}
|
||||
}
|
||||
index = end;
|
||||
}
|
||||
return { folded: chars.join(""), from, to };
|
||||
return { folded: units.join(""), from, to };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -397,9 +413,16 @@ function foldWithMap(text: string): {
|
||||
* text holds a different word), so a result can match with nothing marked.
|
||||
*/
|
||||
export function markMatches(text: string, query: string): MarkRange[] {
|
||||
// The matcher's words; a query too short for the tokeniser ("Hb") is
|
||||
// found by the phrase rule, so it is marked by its folded self.
|
||||
const tokens = queryWords(query);
|
||||
const words = [
|
||||
...new Set(queryWords(query).map((w) => foldGerman(w))),
|
||||
].filter(Boolean);
|
||||
...new Set(
|
||||
(tokens.length > 0 ? tokens : foldForMatch(query).split(" ")).map((w) =>
|
||||
foldGerman(w),
|
||||
),
|
||||
),
|
||||
].filter((w) => w.length >= 2);
|
||||
if (words.length === 0 || text.length === 0) return [];
|
||||
const { folded, from, to } = foldWithMap(text);
|
||||
const ranges: MarkRange[] = [];
|
||||
|
||||
Reference in new issue
Block a user