mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 06:06:10 +00:00
fix(ai): let the accented-word safety and grounding patterns match
JavaScript's \b is an ASCII word boundary, so next to an accented letter it never matches. That left patterns, or alternatives of them, dead: the French "en bonne santé", "à cause de", "causé" and "continuez à", "probabilité" and "probabilità" in the French and Italian risk banks, the Dutch "één procent", the German "Ärzte", and the Polish "przedział", "średnia" and the verb endings in "-ą". A \w* stem cannot run through "ł" either, so "doprowadziło do" missed its causal pattern, and the Polish adult pattern was spelled "dorosli" without the "ś". Those boundaries now use lookarounds that count the Latin letters with diacritics as word letters (WORD_START / WORD_END in the shared screen module, the same class inline in the regex literals), the Polish stems take accented letters, and "dorośli" is spelled both ways. Two guards keep the class out: every pattern of every locale's screen and population-norm bank has to match at least one realistic sentence, so a dead pattern fails the suite, and no pattern may put \b next to an accented letter. Both fail on the old patterns.
This commit is contained in:
1 parent
b89cb6a29f
commit
abb79ee6ae
11 files changed
+701
-40
No files matched your search
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Find `\b` word boundaries that sit next to an accented letter in a regular
|
||||
* expression's source.
|
||||
*
|
||||
* JavaScript defines `\b` over ASCII `\w`, so `santé\b` never matches before a
|
||||
* space and `\bépisode` never matches after one; an alternative that starts or
|
||||
* ends that way is dead. The check looks at what the source puts on each side
|
||||
* of the boundary: a literal letter, a character class, or a group whose
|
||||
* alternatives start (or end) with one. It is a text heuristic over the
|
||||
* pattern source, not a regex engine: a boundary next to a backreference or a
|
||||
* lookaround it does not unwrap is not judged.
|
||||
*/
|
||||
function isAccentedLetter(char: string | undefined): boolean {
|
||||
return char !== undefined && char.charCodeAt(0) > 127 && /\p{L}/u.test(char);
|
||||
}
|
||||
|
||||
function classEnd(source: string, open: number): number {
|
||||
let i = open + 1;
|
||||
while (i < source.length && !(source[i] === "]" && source[i - 1] !== "\\")) {
|
||||
i += 1;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
function classHasAccent(source: string, open: number): boolean {
|
||||
return [...source.slice(open, classEnd(source, open))].some(isAccentedLetter);
|
||||
}
|
||||
|
||||
/** Top-level alternatives of the group opening at `open`, and its close. */
|
||||
function alternatives(source: string, open: number): string[] {
|
||||
const prefix = /^\(\?(?:<[=!]|[:=!])/.exec(source.slice(open))?.[0] ?? "(";
|
||||
const alts: string[] = [];
|
||||
let depth = 0;
|
||||
let start = open + prefix.length;
|
||||
for (let i = open; i < source.length; i += 1) {
|
||||
const c = source[i];
|
||||
if (c === "\\") {
|
||||
i += 1;
|
||||
} else if (c === "[") {
|
||||
i = classEnd(source, i);
|
||||
} else if (c === "(") {
|
||||
depth += 1;
|
||||
} else if (c === ")") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
alts.push(source.slice(start, i));
|
||||
return alts;
|
||||
}
|
||||
} else if (c === "|" && depth === 1) {
|
||||
alts.push(source.slice(start, i));
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
return alts;
|
||||
}
|
||||
|
||||
function opensWithAccent(text: string): boolean {
|
||||
if (text.startsWith("[")) return classHasAccent(text, 0);
|
||||
if (text.startsWith("(")) return alternatives(text, 0).some(opensWithAccent);
|
||||
return isAccentedLetter(text[0]);
|
||||
}
|
||||
|
||||
function closesWithAccent(text: string): boolean {
|
||||
const trimmed = text.replace(/(?:[?*+]|\{\d+(?:,\d*)?\})+\??$/, "");
|
||||
const last = trimmed[trimmed.length - 1];
|
||||
if (last === "]") {
|
||||
let open = trimmed.length - 2;
|
||||
while (
|
||||
open >= 0 &&
|
||||
!(trimmed[open] === "[" && trimmed[open - 1] !== "\\")
|
||||
) {
|
||||
open -= 1;
|
||||
}
|
||||
return classHasAccent(trimmed, open);
|
||||
}
|
||||
if (last === ")") {
|
||||
let depth = 0;
|
||||
for (let i = trimmed.length - 1; i >= 0; i -= 1) {
|
||||
if (trimmed[i - 1] === "\\") continue;
|
||||
if (trimmed[i] === ")") depth += 1;
|
||||
else if (trimmed[i] === "(" && --depth === 0) {
|
||||
return alternatives(trimmed, i).some(closesWithAccent);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return isAccentedLetter(last);
|
||||
}
|
||||
|
||||
/** Offsets of every `\b` in `source` that touches an accented letter. */
|
||||
export function accentedBoundaries(source: string): number[] {
|
||||
const hits: number[] = [];
|
||||
for (
|
||||
let i = source.indexOf("\\b");
|
||||
i >= 0;
|
||||
i = source.indexOf("\\b", i + 2)
|
||||
) {
|
||||
if (source[i - 1] === "\\") continue;
|
||||
if (
|
||||
opensWithAccent(source.slice(i + 2)) ||
|
||||
closesWithAccent(source.slice(0, i))
|
||||
) {
|
||||
hits.push(i);
|
||||
}
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
Reference in new issue
Block a user