Files
HealthLog/src/lib/insights/eval/tone-rules.ts
T
Marc-André Bombeck 8b13b46ac7 feat(insights): write the no-key fallbacks in every shipped locale
The deterministic fallbacks nine metric surfaces render when no provider is
reachable carried German and English bodies only. `getLocalizedText` returned
`locale === "de" ? de : en`, so French, Spanish, Italian and Polish readers got
English — on the no-key path, an outage, a spent budget or a timeout, which is
precisely when this text is the whole assessment.

Every string now has a body per locale: the verdict leads and value sentences
of the signal-grounded composer, the per-metric labels and pointers, and the
eight no-signal floors. Selection reads the reader's locale; English stays as
the structural floor for a locale that is ever genuinely missing a body.

The bodies are written per language, not translated word-for-word. fr/es/it
replace the comparative adjective ("higher than your usual average") with a
preposition — au-dessus de / por encima de / sopra — because a comparative
would have to agree in gender with the metric noun, and the labels do not share
one. Polish restructures further: the sentence takes `wynosi`, which is
invariant for the subject's gender, and the comparison uses powyżej / poniżej
with the genitive; the labels span all three genders, so nothing that inflects
can sit in a shared template.

The tone harness grades the four locales by the same rules rather than passing
them by default: false cheer, clinical-verdict and imperative-opener vocabulary
extended to all six, the no-assessment sentinel keyed per locale instead of a
de/en binary, and a byte-identity guard that fails if any locale silently falls
back to the English text again. The value-led-opener rule was rewritten as
well — it assumed the copula sits against the figure, which under-matched its
own second locale (German "liegt aktuell bei 24") and would have missed all
four new ones.
2026-07-19 13:42:09 +02:00

397 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Deterministic tone rules for the assessment surfaces.
*
* The live-judge eval under `src/lib/ai/coach/eval/` grades MODEL OUTPUT and
* needs a provider key, so it can only run conditionally. This module grades
* the two things that are available with no provider at all:
*
* 1. the assembled PROMPT text every assessment surface sends, and
* 2. the DETERMINISTIC FALLBACK text those surfaces render when no provider
* is configured or the call stalls.
*
* That is enough to catch the regression classes that actually happened. Every
* tone regression this repo has shipped was visible in the prompt before it was
* ever visible in the prose: a surface built its prompt inline and skipped the
* shared opening contract; a locale silently composed the German body; an
* instruction told the model to state the value first. Those are string facts,
* so they are checkable in the normal suite in milliseconds and can block a
* merge — which is the whole point, since a judge run that needs a key cannot.
*
* The rules are deliberately narrow. Each one names a regression that has a
* concrete failure story, and each returns the offending excerpt so a red run
* reads as a diff rather than as "tone check failed".
*/
/** One rule violation, with enough context to fix it without re-deriving. */
export interface ToneViolation {
/** Stable rule id — the test reports it, so keep it greppable. */
rule: string;
/** What went wrong, in one line. */
detail: string;
/** The offending excerpt, when the rule found one. */
excerpt?: string;
}
/**
* Words that flip an instruction's polarity.
*
* The shared prompts talk ABOUT the value-first failure mode in order to ban
* it — "do NOT always open with the number", "never lead with the value",
* "führe nie mit dem Wert". A naive substring scan for the banned instruction
* would fire on the ban itself, so every candidate match is tested for a
* negator immediately before it.
*/
const NEGATORS =
/\b(not|never|no|non|rather than|instead of|nicht|nie|niemals|kein(e|en|em|er)?|statt|anstatt)\b/i;
/** How far back a negator still counts as governing the match. */
const NEGATION_WINDOW = 48;
/**
* True when `pattern` matches somewhere it is NOT governed by a preceding
* negator. Exported because the mutation-check and the fixture tests both
* assert on the negation handling itself — it is the load-bearing part of
* every banned-instruction rule.
*/
export function hasUnnegatedMatch(text: string, pattern: RegExp): boolean {
return findUnnegatedMatch(text, pattern) !== null;
}
export function findUnnegatedMatch(
text: string,
pattern: RegExp,
): string | null {
const scan = new RegExp(
pattern.source,
pattern.flags.replace(/g/g, "") + "g",
);
for (const m of text.matchAll(scan)) {
const start = m.index ?? 0;
const before = text.slice(Math.max(0, start - NEGATION_WINDOW), start);
if (NEGATORS.test(before)) continue;
return text.slice(Math.max(0, start - 24), start + m[0].length + 24);
}
return null;
}
/**
* Instructions that put the NUMBER first.
*
* This is the exact class the biomarker card shipped for eight releases
* ("state the current value and how it sits against the reference range"),
* while every sibling card had already been moved to meaning-first. A prompt
* edit that reintroduces any of these — in either instruction body — is the
* regression this list exists to stop.
*/
export const VALUE_FIRST_INSTRUCTIONS: readonly {
id: string;
pattern: RegExp;
}[] = [
{ id: "en/state-the-current-value", pattern: /state the current value/i },
{
id: "en/name-the-value-first",
pattern: /name the (?:current )?value first/i,
},
{
id: "en/open-with-the-number",
pattern:
/(?:open|lead|start|begin)(?:s|ing)? with the (?:number|value|reading|figure)/i,
},
{ id: "de/nenne-den-aktuellen-wert", pattern: /nenne den aktuellen Wert/i },
{
id: "de/mit-der-zahl-eroeffnen",
pattern:
/(?:eröffne|führe|beginne|starte)(?:st)? mit (?:der Zahl|dem Wert|dem Messwert)/i,
},
];
/**
* The meaning-first opening every assessment USER prompt has to carry.
*
* Two halves, both required, because either alone is satisfiable by accident:
* the surface must tell the model to open on the read in plain words, AND it
* must say the number is not what opens. The family phrases both consistently
* ("Open with what it MEANS in plain words — the read, not the number"), so
* matching the shared shape also keeps a new surface from inventing its own
* dialect for the same instruction.
*/
const MEANING_FIRST_OPENER = {
en: /\bOpen with\b[^.]{0,200}\bin plain words\b/,
de: /\bBeginne mit\b[^.]{0,200}\bin klaren Worten\b/,
} as const;
const NUMBER_DEFERRED = {
en: /\bnot (?:the|a) number\b/i,
de: /\bnicht (?:der Zahl|mit einer Zahl|dem Wert)\b/i,
} as const;
/** German-only vocabulary — a leak here means a locale collapsed to the DE body. */
export const GERMAN_BODY_SENTINEL =
/AUSGABEFORMAT|Antworte ausschließlich|Schreibe eine kurze Einschätzung|DEINE DATENGRUNDLAGE|SO BAUST DU|TONALITÄT|ERÖFFNUNG —/;
/**
* Grade one assembled surface.
*
* `instructionBody` is which reviewed body the locale composes (de for German
* readers, en for everyone else) — the meaning-first assertions key off that,
* not off the reader's locale, because a French prompt carries the ENGLISH
* instruction text plus a French output directive.
*/
export function checkPromptToneContract(input: {
systemPrompt: string;
userPrompt: string;
instructionBody: "en" | "de";
/** The shared opening-shape fragment the system prompt must carry. */
openingShapeFragment: string;
}): ToneViolation[] {
const violations: ToneViolation[] = [];
const { systemPrompt, userPrompt, instructionBody } = input;
const both = `${systemPrompt}\n${userPrompt}`;
// 1. The shared opening-shape contract has to be present verbatim. A surface
// that builds its system prompt inline instead of composing the shared base
// loses it silently — which is exactly how the biomarker card drifted.
if (!systemPrompt.includes(input.openingShapeFragment)) {
violations.push({
rule: "opening-shape-contract-present",
detail:
"system prompt does not carry the shared openingShape contract — it is not composing the shared base body",
});
}
// 2. The user prompt has to instruct a meaning-first opening.
if (!MEANING_FIRST_OPENER[instructionBody].test(userPrompt)) {
violations.push({
rule: "user-prompt-leads-with-meaning",
detail: `user prompt has no meaning-first opener instruction (expected ${MEANING_FIRST_OPENER[instructionBody]})`,
});
}
if (!NUMBER_DEFERRED[instructionBody].test(userPrompt)) {
violations.push({
rule: "user-prompt-defers-the-number",
detail: `user prompt does not say the number is not what opens (expected ${NUMBER_DEFERRED[instructionBody]})`,
});
}
// 3. No value-first instruction anywhere in the assembled pair.
for (const banned of VALUE_FIRST_INSTRUCTIONS) {
const hit = findUnnegatedMatch(both, banned.pattern);
if (hit) {
violations.push({
rule: `no-value-first-instruction:${banned.id}`,
detail: "an un-negated value-first instruction is back in the prompt",
excerpt: hit,
});
}
}
return violations;
}
/**
* Locale integrity for one system prompt.
*
* The failure this pins: for four releases the assessment path composed
* `locale === "en" ? "en" : "de"`, so a French, Spanish, Italian or Polish
* reader received the GERMAN instruction body — whose output clause asks for
* German prose — with nothing anywhere telling the model to reply in the
* reader's language. It is invisible to every English and German test.
*/
export function checkLocaleIntegrity(input: {
systemPrompt: string;
/** Non-empty for the four locales that ride the English body. */
directive: string;
/** English name of the reader's language, e.g. "French". */
languageName: string;
}): ToneViolation[] {
const violations: ToneViolation[] = [];
if (input.directive.length === 0) {
violations.push({
rule: "locale-directive-resolves",
detail: "no output-language directive resolved for a rider locale",
});
} else if (!input.systemPrompt.includes(input.directive)) {
violations.push({
rule: "locale-directive-present",
detail:
"system prompt does not carry the reader's output-language directive",
});
}
if (!input.systemPrompt.includes(input.languageName)) {
violations.push({
rule: "locale-language-named",
detail: `system prompt never names the reply language (${input.languageName}) — the output clause collapsed`,
});
}
const german = GERMAN_BODY_SENTINEL.exec(input.systemPrompt);
if (german) {
violations.push({
rule: "locale-no-german-body-leak",
detail:
"a rider locale composed the GERMAN instruction body — the six-locale collapse is back",
excerpt: german[0],
});
}
return violations;
}
// ── deterministic fallback tone ─────────────────────────────────────────────
/**
* Praise that the numbers did not earn.
*
* Six locales, because the deterministic fallbacks now carry a written body in
* each: a rule that only knows English and German praise would grade four
* locales as clean no matter what they said. Vocabulary only — the rule shape
* is language-independent.
*/
const FALSE_CHEER =
/\b(great job|well done|amazing|fantastic|excellent|awesome|congratulations|keep it up|don't worry)\b|\b(super gemacht|toll gemacht|großartig|fantastisch|hervorragend|weiter so|keine sorge)\b|\b(bravo|félicitations|continue comme ça|génial|formidable|ne t['’]inquiète pas)\b|\b(enhorabuena|felicidades|sigue así|genial|estupendo|no te preocupes)\b|\b(complimenti|continua così|ottimo lavoro|fantastico|eccellente|non preoccuparti)\b|\b(gratulacje|brawo|tak trzymaj|świetnie|wspaniale|nie martw się)\b/i;
/**
* Verdict vocabulary a non-diagnostic surface may never use.
*
* Note what is deliberately NOT here: the plain noun for "verdict" in each
* language (verdict / veredicto / verdetto / wyrok / Urteil). Several floors
* use it in the negated form that carries the whole non-diagnostic stance — "a
* reference range is a coarse anchor, not a verdict" — so banning the noun
* would flag the sentence that states the rule. The ban targets diagnosis,
* pathology, attribution of a condition to the reader, and treatment
* direction, which is what a non-diagnostic surface actually may not do.
*/
const CLINICAL_VERDICT =
/\b(diagnos\w*|abnormal|patholog\w*|you (?:have|suffer from)|risk score|clinically significant)\b|\b(Diagnose|krankhaft|pathologisch|du (?:hast|leidest)|behandlungsbedürftig)\b|\b(diagnostiqu\w*|pathologique|tu souffres de|nécessite un traitement)\b|\b(diagnóstic\w*|patológic\w*|sufres de|requiere (?:un )?tratamiento)\b|\b(diagnosi|diagnostic\w*|patologic\w*|soffri di|richiede (?:un )?trattamento)\b|\b(diagnoz\w*|patologiczn\w*|cierpisz na|wymaga leczenia)\b/i;
/**
* The value-led opener §10 bans: "Your <metric> is <number>…".
*
* Rewritten from the de/en-only shape, which assumed the copula sits directly
* against the figure. That is true for "Your BMI is 24" and false for almost
* everything else the surfaces actually render — German "liegt aktuell bei
* 24", French "est à 24", Spanish "está en 24", Polish "wynosi obecnie 24".
* The old pattern therefore under-matched German, its own second locale, and
* would have missed all four new ones. The shape is now: possessive determiner,
* up to three words of metric name, the copula, then up to two positional
* fillers before the figure.
*/
const VALUE_LED_OPENER =
/^(?:Your|Dein(?:e)?|Ihr(?:e)?|Ta|Ton|Tes|Tu|Tus|La tua|Le tue|Il tuo|I tuoi|Twój|Twoja|Twoje)\b(?:\s+\S+){0,3}\s+(?:is|ist|liegt|est|está|è|wynosi)\b(?:\s+(?:at|bei|à|a|en|aktuell|obecnie|right now)){0,2}\s+\d/i;
/**
* The OTHER way a deterministic line stops leading with meaning: it opens on a
* bare order.
*
* This is what the no-signal floors were — "Measure blood pressure at rest and
* under comparable conditions", "Bewerte Gewicht vor allem im Verlauf". Not a
* value-led opener, so the number rules never saw it, but it is the same
* failure: the surface tells the reader what to do before it says anything
* about what it can and cannot see. A pointer is fine LATER in the line; it
* may not be the opening move.
*
* Extended to the four locales whose bodies used to be English. Romance and
* Polish imperatives are a closed enough set here because the pointers these
* floors could regress into are drawn from the same small verb family the
* de/en list already names — measure, weigh, track, compare, read.
*/
const IMPERATIVE_OPENER =
/^(Measure|Track|Evaluate|Monitor|Check|Watch|Log|React|Use|Keep|Pay|Consider|Make sure)\b|^(Miss|Bewerte|Beobachte|Achte|Nutze|Halte|Reagiere|Prüfe|Vergleiche|Erfasse)\b|^(Mesure|Suis|Évalue|Surveille|Vérifie|Note|Utilise|Garde|Compare|Pèse|Prends|Lis)\b|^(Mide|Evalúa|Vigila|Comprueba|Registra|Usa|Mantén|Compara|Pésate|Toma|Lee)\b|^(Misura|Valuta|Monitora|Controlla|Registra|Usa|Mantieni|Confronta|Pesati|Prendi|Leggi)\b|^(Mierz|Oceń|Obserwuj|Sprawdź|Zapisuj|Używaj|Utrzymuj|Porównaj|Waż|Czytaj)\b/;
function firstSentence(text: string): string {
const m = /^[^.!?]*[.!?]/.exec(text.trim());
return (m ? m[0] : text.trim()).trim();
}
/**
* Grade one rendered deterministic fallback.
*
* `expectMeaningFirst` is false for exactly one honest case: a signal with no
* baseline and no usable delta supports NO confident read, so the line opens
* on the value rather than manufacturing a verdict. Pinning that asymmetry is
* the point — "always lead with meaning" would push this surface into claiming
* something the data does not support, which is the failure mode on the other
* side of the same rule.
*/
export function checkFallbackTone(input: {
text: string;
expectMeaningFirst: boolean;
}): ToneViolation[] {
const violations: ToneViolation[] = [];
const text = input.text.trim();
const opener = firstSentence(text);
if (text.length === 0) {
return [{ rule: "fallback-non-empty", detail: "fallback rendered empty" }];
}
if (input.expectMeaningFirst) {
if (/\d/.test(opener)) {
violations.push({
rule: "fallback-leads-with-meaning",
detail:
"the opening sentence carries a figure — the deterministic line opens on a value instead of the read",
excerpt: opener,
});
}
if (IMPERATIVE_OPENER.test(opener)) {
violations.push({
rule: "fallback-no-imperative-opener",
detail:
"the line opens on an order rather than on what the surface can honestly say",
excerpt: opener,
});
}
if (VALUE_LED_OPENER.test(opener)) {
violations.push({
rule: "fallback-no-value-led-opener",
detail: "opening matches the banned value-led opener shape",
excerpt: opener,
});
}
}
const cheer = FALSE_CHEER.exec(text);
if (cheer) {
violations.push({
rule: "fallback-no-false-cheer",
detail: "praise the numbers did not earn",
excerpt: cheer[0],
});
}
const verdict = CLINICAL_VERDICT.exec(text);
if (verdict) {
violations.push({
rule: "fallback-no-clinical-verdict",
detail: "a non-diagnostic surface used verdict vocabulary",
excerpt: verdict[0],
});
}
if (/!/.test(text)) {
violations.push({
rule: "fallback-no-exclamation",
detail: "exclamation mark — the assessment voice never exclaims",
});
}
return violations;
}
/** Render violations as a readable multi-line failure message. */
export function formatViolations(
label: string,
violations: readonly ToneViolation[],
): string {
return [
`${label} — ${violations.length} tone violation(s):`,
...violations.map(
(v) =>
` · [${v.rule}] ${v.detail}${v.excerpt ? `\n …${v.excerpt}…` : ""}`,
),
].join("\n");
}