mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 05:12:14 +00:00
Wire citation coverage into the live path and cover the screen in all six locales
The coverage check hung off generateInsight, whose only caller runWithFallback has no production caller left -- every live surface routes through runRawCompletionWithFallback instead. The admin quality dashboard was reading a metric nothing emitted. Widen the input to the structural shape both payloads share and call it from the comprehensive post-parse step, so the annotation reflects real generations. Writing the locale tests found three gaps the English-only banks hid: Polish verb suffixes fell outside the ASCII word class, a trailing word boundary after an accented character never closed, and German subordinate clauses put the verb last so the natural phrasing of a dose instruction never matched. The payload collector also skipped legacy bare-string recommendations entirely.
This commit is contained in:
1 parent
0becc2a56d
commit
544ae90365
8 files changed
+619
-19
No files matched your search
@@ -1,4 +1,37 @@
|
||||
import type { AIInsightResponse } from "./schema";
|
||||
/**
|
||||
* Reachability note. This module was written for `generateInsight()`, whose
|
||||
* only caller is `runWithFallback()` — and `runWithFallback()` has no
|
||||
* production caller left (every live surface routes through
|
||||
* `runRawCompletionWithFallback` instead). The coverage annotation therefore
|
||||
* never executed in production: an audit surface that implied enforcement it
|
||||
* did not perform. Rather than delete working, tested logic, the input type is
|
||||
* widened to the structural shape both payloads share and the comprehensive
|
||||
* post-parse step now calls it, so the admin quality dashboard sees real data.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The minimum recommendation shape the coverage check needs. Both the strict
|
||||
* `AIInsightResponse` and the live `InsightResult` satisfy it; the live union
|
||||
* also admits a bare string, which carries neither an id nor a citation and is
|
||||
* therefore counted as uncited when it makes a normative claim.
|
||||
*/
|
||||
export interface RecommendationForCoverage {
|
||||
id?: string;
|
||||
text: string;
|
||||
referenceId?: string;
|
||||
/**
|
||||
* Callers pass their own richer recommendation objects (severity, rationale,
|
||||
* metricSource, …). Only the three fields above are read; the index
|
||||
* signature keeps a wider literal assignable without a cast at every site.
|
||||
*/
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface PayloadForCoverage {
|
||||
recommendations: ReadonlyArray<RecommendationForCoverage | string>;
|
||||
/** Callers pass the whole payload; only `recommendations` is read. */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.4.16 phase B5a — citation-coverage post-validation.
|
||||
@@ -68,20 +101,24 @@ export interface CitationCoverage {
|
||||
* the successful parse and forwards the result via `annotate()`.
|
||||
*/
|
||||
export function computeCitationCoverage(
|
||||
parsed: AIInsightResponse,
|
||||
parsed: PayloadForCoverage,
|
||||
): CitationCoverage {
|
||||
const total = parsed.recommendations.length;
|
||||
let normative = 0;
|
||||
let cited = 0;
|
||||
const uncitedIds: string[] = [];
|
||||
|
||||
for (const rec of parsed.recommendations) {
|
||||
for (const [index, raw] of parsed.recommendations.entries()) {
|
||||
const rec: RecommendationForCoverage =
|
||||
typeof raw === "string" ? { text: raw } : raw;
|
||||
if (!detectsNormativeClaim(rec.text)) continue;
|
||||
normative += 1;
|
||||
if (rec.referenceId) {
|
||||
cited += 1;
|
||||
} else {
|
||||
uncitedIds.push(rec.id);
|
||||
// A legacy string rec has no id of its own; index it so the dashboard
|
||||
// can still point at which entry was uncited.
|
||||
uncitedIds.push(rec.id ?? `index:${index}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
collectInsightProse,
|
||||
screenInsightPayloadProse,
|
||||
} from "@/lib/ai/safety/insight-payload-screen";
|
||||
import { locales } from "@/lib/i18n/config";
|
||||
|
||||
/**
|
||||
* The briefing's number-grounding gate covers `paragraph`, `signalsOfDay[]`
|
||||
* and `keyFindings[]`, and it grades NUMBERS only — so a digit-free risk
|
||||
* assertion passed it, and `recommendations[]` / `summary` were not graded at
|
||||
* all. These tests pin that the safety screen reaches every prose field.
|
||||
*/
|
||||
|
||||
const CLEAN_PAYLOAD = {
|
||||
summary: "Your tracked metrics held steady over the last 30 days.",
|
||||
recommendations: [
|
||||
{ id: "r1", text: "Keep logging your morning readings at the same time." },
|
||||
],
|
||||
dailyBriefing: {
|
||||
paragraph: "Weight and sleep both moved with your usual weekday rhythm.",
|
||||
signalsOfDay: [{ headline: "Steady week", nudge: "Keep it going." }],
|
||||
keyFindings: [{ headline: "Sleep", detail: "In line with your baseline." }],
|
||||
},
|
||||
};
|
||||
|
||||
describe("collectInsightProse", () => {
|
||||
it("reaches summary, recommendations and every briefing sub-field", () => {
|
||||
const prose = collectInsightProse(CLEAN_PAYLOAD);
|
||||
expect(prose).toContain(CLEAN_PAYLOAD.summary);
|
||||
expect(prose).toContain(CLEAN_PAYLOAD.recommendations[0].text);
|
||||
expect(prose).toContain(CLEAN_PAYLOAD.dailyBriefing.paragraph);
|
||||
expect(prose).toContain("Steady week");
|
||||
expect(prose).toContain("In line with your baseline.");
|
||||
});
|
||||
|
||||
it("handles the legacy bare-string recommendation shape", () => {
|
||||
const prose = collectInsightProse({
|
||||
recommendations: ["Drink water with each dose."],
|
||||
});
|
||||
expect(prose).toContain("Drink water with each dose.");
|
||||
});
|
||||
|
||||
it("returns nothing for a non-object payload", () => {
|
||||
expect(collectInsightProse(null)).toEqual([]);
|
||||
expect(collectInsightProse("nope")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenInsightPayloadProse — clean payload", () => {
|
||||
for (const locale of locales) {
|
||||
it(`passes a grounded payload for a ${locale} reader`, () => {
|
||||
expect(screenInsightPayloadProse(CLEAN_PAYLOAD, locale)).toBeNull();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("screenInsightPayloadProse — fields the number gate never graded", () => {
|
||||
it("catches a digit-free risk assertion in the briefing paragraph", () => {
|
||||
// The exact shape the number-only gate lets through: no digit anywhere the
|
||||
// grounding check could grade. It trips the RISK contract rather than the
|
||||
// dose one -- "stepping your dose up" carries no number+unit, so the dose
|
||||
// bank (deliberately tight, to protect factual restatements) does not fire.
|
||||
const reason = screenInsightPayloadProse(
|
||||
{
|
||||
recommendations: [],
|
||||
dailyBriefing: {
|
||||
paragraph:
|
||||
"Your 10-year cardiovascular risk is elevated — consider stepping your dose up.",
|
||||
},
|
||||
},
|
||||
"en",
|
||||
);
|
||||
expect(reason).toBe("risk_score");
|
||||
});
|
||||
|
||||
it("catches a dose imperative in recommendations[]", () => {
|
||||
const reason = screenInsightPayloadProse(
|
||||
{
|
||||
recommendations: [
|
||||
{ id: "r1", text: "Increase to 10 mg starting next week." },
|
||||
],
|
||||
},
|
||||
"en",
|
||||
);
|
||||
expect(reason).toBe("dose_prescription");
|
||||
});
|
||||
|
||||
it("catches a causal claim in summary", () => {
|
||||
const reason = screenInsightPayloadProse(
|
||||
{
|
||||
summary: "Your weight fell because you slept more.",
|
||||
recommendations: [],
|
||||
},
|
||||
"en",
|
||||
);
|
||||
expect(reason).toBe("causal_claim");
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenInsightPayloadProse — non-EN/DE locales", () => {
|
||||
const VIOLATIONS: Record<string, string> = {
|
||||
fr: "Augmentez votre dose à 2,4 mg la semaine prochaine.",
|
||||
es: "Aumente su dosis a 2,4 mg la próxima semana.",
|
||||
it: "Aumenti la sua dose a 2,4 mg la prossima settimana.",
|
||||
pl: "Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.",
|
||||
};
|
||||
|
||||
for (const [locale, text] of Object.entries(VIOLATIONS)) {
|
||||
it(`catches a ${locale} dose imperative in recommendations[]`, () => {
|
||||
const reason = screenInsightPayloadProse(
|
||||
{ recommendations: [{ id: "r1", text }] },
|
||||
locale as (typeof locales)[number],
|
||||
);
|
||||
expect(reason).toBe("dose_prescription");
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,211 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
screenModelOutput,
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
INSIGHTS_CONTRACTS,
|
||||
} from "@/lib/ai/safety/outbound-screen";
|
||||
import { locales, type Locale } from "@/lib/i18n/config";
|
||||
|
||||
/**
|
||||
* The whole point of threading locale is that fr / es / it / pl are enforced,
|
||||
* not just en / de. Every contract below is asserted in all six languages: a
|
||||
* violating sentence written in the reader's language must be caught, and an
|
||||
* ordinary sentence in that same language must pass untouched.
|
||||
*/
|
||||
|
||||
// ── dose-prescription ────────────────────────────────────────────────────
|
||||
|
||||
const DOSE_VIOLATIONS: Record<Locale, string> = {
|
||||
en: "You should step up to 2.4 mg next week.",
|
||||
de: "Erhöhe auf 7,5 mg, sobald du dich daran gewöhnt hast.",
|
||||
fr: "Augmentez votre dose à 2,4 mg la semaine prochaine.",
|
||||
es: "Aumente su dosis a 2,4 mg la próxima semana.",
|
||||
it: "Aumenti la sua dose a 2,4 mg la prossima settimana.",
|
||||
pl: "Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.",
|
||||
};
|
||||
|
||||
/**
|
||||
* Factual restatements of a CURRENT dose. The contracts explicitly permit
|
||||
* these, so a screen that flagged them would make the Coach useless on exactly
|
||||
* the topic users bring it.
|
||||
*/
|
||||
const DOSE_CLEAN: Record<Locale, string> = {
|
||||
en: "You're in week 3 on 7.5 mg, and your weight is tracking down.",
|
||||
de: "Du bist in Woche 3 bei 7,5 mg, und dein Gewicht geht zurück.",
|
||||
fr: "Vous êtes en semaine 3 à 7,5 mg, et votre poids diminue.",
|
||||
es: "Está en la semana 3 con 7,5 mg, y su peso está bajando.",
|
||||
it: "È alla settimana 3 con 7,5 mg, e il suo peso sta calando.",
|
||||
pl: "Jest Pan w 3. tygodniu na 7,5 mg, a waga spada.",
|
||||
};
|
||||
|
||||
describe("screenModelOutput — dose-prescription, all six locales", () => {
|
||||
for (const locale of locales) {
|
||||
it(`blocks a dose-change imperative in ${locale}`, () => {
|
||||
const d = screenModelOutput(
|
||||
DOSE_VIOLATIONS[locale],
|
||||
locale,
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(true);
|
||||
expect(d.reason).toBe("dose_prescription");
|
||||
});
|
||||
|
||||
it(`passes a factual dose restatement in ${locale}`, () => {
|
||||
const d = screenModelOutput(
|
||||
DOSE_CLEAN[locale],
|
||||
locale,
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(false);
|
||||
expect(d.reason).toBeNull();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ── risk-score fabrication ───────────────────────────────────────────────
|
||||
|
||||
const RISK_VIOLATIONS: Record<Locale, string> = {
|
||||
en: "Your 10-year cardiovascular risk is about 12%.",
|
||||
de: "Dein Risiko liegt bei etwa 14%.",
|
||||
fr: "Votre risque est d'environ 12%.",
|
||||
es: "Su riesgo es del 12%.",
|
||||
it: "Il suo rischio è del 12%.",
|
||||
pl: "Pana ryzyko wynosi 12%.",
|
||||
};
|
||||
|
||||
/** A plain percentage that is NOT a clinical risk claim must pass. */
|
||||
const RISK_CLEAN: Record<Locale, string> = {
|
||||
en: "Your adherence was 92% over the last 30 days.",
|
||||
de: "Deine Einnahmetreue lag bei 92% in den letzten 30 Tagen.",
|
||||
fr: "Votre observance était de 92% sur les 30 derniers jours.",
|
||||
es: "Su adherencia fue del 92% en los últimos 30 días.",
|
||||
it: "La sua aderenza è stata del 92% negli ultimi 30 giorni.",
|
||||
pl: "Przestrzeganie zaleceń wyniosło 92% w ostatnich 30 dniach.",
|
||||
};
|
||||
|
||||
describe("screenModelOutput — risk-score, all six locales", () => {
|
||||
for (const locale of locales) {
|
||||
it(`blocks a fabricated clinical risk figure in ${locale}`, () => {
|
||||
const d = screenModelOutput(
|
||||
RISK_VIOLATIONS[locale],
|
||||
locale,
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(true);
|
||||
expect(d.reason).toBe("risk_score");
|
||||
});
|
||||
|
||||
it(`passes a non-risk percentage in ${locale}`, () => {
|
||||
const d = screenModelOutput(
|
||||
RISK_CLEAN[locale],
|
||||
locale,
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(false);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ── causal claims (GROUND RULE 12) ───────────────────────────────────────
|
||||
|
||||
const CAUSAL_VIOLATIONS: Record<Locale, string> = {
|
||||
en: "Your weight fell because you slept more.",
|
||||
de: "Dein Gewicht sank wegen des Schlafs.",
|
||||
fr: "Votre poids a baissé parce que vous avez mieux dormi.",
|
||||
es: "Su peso bajó porque durmió más.",
|
||||
it: "Il suo peso è calato perché ha dormito di più.",
|
||||
pl: "Waga spadła, ponieważ spał Pan więcej.",
|
||||
};
|
||||
|
||||
/** Descriptive association framing is what the contract REQUIRES. */
|
||||
const CAUSAL_CLEAN: Record<Locale, string> = {
|
||||
en: "Your weight moved with your sleep this week.",
|
||||
de: "Dein Gewicht bewegte sich diese Woche mit deinem Schlaf.",
|
||||
fr: "Votre poids a évolué avec votre sommeil cette semaine.",
|
||||
es: "Su peso evolucionó junto con su sueño esta semana.",
|
||||
it: "Il suo peso si è mosso insieme al suo sonno questa settimana.",
|
||||
pl: "Waga zmieniała się razem ze snem w tym tygodniu.",
|
||||
};
|
||||
|
||||
describe("screenModelOutput — causal claims, all six locales", () => {
|
||||
for (const locale of locales) {
|
||||
it(`blocks asserted causation in ${locale}`, () => {
|
||||
const d = screenModelOutput(
|
||||
CAUSAL_VIOLATIONS[locale],
|
||||
locale,
|
||||
INSIGHTS_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(true);
|
||||
expect(d.reason).toBe("causal_claim");
|
||||
});
|
||||
|
||||
it(`passes descriptive association framing in ${locale}`, () => {
|
||||
const d = screenModelOutput(
|
||||
CAUSAL_CLEAN[locale],
|
||||
locale,
|
||||
INSIGHTS_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(false);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ── contract selection ───────────────────────────────────────────────────
|
||||
|
||||
describe("screenModelOutput — contract selection", () => {
|
||||
it("does NOT flag causal wording under the conversational contracts", () => {
|
||||
// The Coach must keep "because" — GROUND RULE 12's surface is `insights`,
|
||||
// and blocking it conversationally would gut ordinary explanation.
|
||||
const d = screenModelOutput(
|
||||
CAUSAL_VIOLATIONS.en,
|
||||
"en",
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(false);
|
||||
});
|
||||
|
||||
it("flags the same sentence under the insights contracts", () => {
|
||||
const d = screenModelOutput(CAUSAL_VIOLATIONS.en, "en", INSIGHTS_CONTRACTS);
|
||||
expect(d.block).toBe(true);
|
||||
});
|
||||
|
||||
it("reports dose before risk when a reply trips both", () => {
|
||||
const d = screenModelOutput(
|
||||
"Your risk is 12% — step up to 2.4 mg.",
|
||||
"en",
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.reason).toBe("dose_prescription");
|
||||
});
|
||||
});
|
||||
|
||||
// ── cross-language coverage ──────────────────────────────────────────────
|
||||
|
||||
describe("screenModelOutput — English violations reach non-English readers", () => {
|
||||
/**
|
||||
* A provider routinely answers in English regardless of the locale
|
||||
* directive. If the screen consulted only the reader's bank, the proven
|
||||
* English violation shapes would pass on five of six locales.
|
||||
*/
|
||||
for (const locale of locales) {
|
||||
it(`blocks an English dose imperative for a ${locale} reader`, () => {
|
||||
const d = screenModelOutput(
|
||||
DOSE_VIOLATIONS.en,
|
||||
locale,
|
||||
CONVERSATIONAL_CONTRACTS,
|
||||
);
|
||||
expect(d.block).toBe(true);
|
||||
expect(d.reason).toBe("dose_prescription");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("screenModelOutput — empty input", () => {
|
||||
it("passes empty and whitespace-only text (handled upstream)", () => {
|
||||
expect(screenModelOutput("", "en", INSIGHTS_CONTRACTS).block).toBe(false);
|
||||
expect(screenModelOutput(" ", "fr", INSIGHTS_CONTRACTS).block).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -34,7 +34,12 @@ export function collectInsightProse(parsed: unknown): string[] {
|
||||
for (const rec of Array.isArray(root.recommendations)
|
||||
? root.recommendations
|
||||
: []) {
|
||||
if (rec && typeof rec === "object") {
|
||||
// The live schema admits BOTH a structured object and a legacy bare
|
||||
// string. Reading only `.text` silently skipped every string rec, which is
|
||||
// exactly the shape older cached payloads use.
|
||||
if (typeof rec === "string") {
|
||||
push(rec);
|
||||
} else if (rec && typeof rec === "object") {
|
||||
push((rec as Record<string, unknown>).text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +110,13 @@ const DOSE_PATTERNS: Record<Locale, readonly RegExp[]> = {
|
||||
`\\bn[äa]chste\\s+(?:stufe|dosis)\\b[^.?!]{0,30}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
"i",
|
||||
),
|
||||
// German puts the verb last in a subordinate clause ("auf 7,5 mg zu
|
||||
// erhöhen"), so the verb-first patterns above miss the most natural
|
||||
// phrasing of the instruction. Match the inverted order too.
|
||||
new RegExp(
|
||||
`\\b(?:auf|um)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b[^.?!]{0,30}\\b(?:erhöh\\S*|steiger\\S*|reduzier\\S*|senk\\S*|verringer\\S*|hochsetz\\S*)`,
|
||||
"i",
|
||||
),
|
||||
new RegExp(
|
||||
`\\b(?:erwäg\\w*|probier\\w*|du\\s+solltest|ich\\s+empfehle|ich\\s+schlage\\s+vor)\\b[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
"i",
|
||||
@@ -179,16 +186,16 @@ const DOSE_PATTERNS: Record<Locale, readonly RegExp[]> = {
|
||||
pl: [
|
||||
// zwiększ / podnieś / przejdź do|o N unit
|
||||
new RegExp(
|
||||
`\\b(?:zwi[ęe]ksz\\w*|podnie[śs]\\w*|przejd[źz]\\w*|podwy[żz]sz\\w*)\\s+(?:dawk[ęe]\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
`\\b(?:zwi[ęe]ksz\\S*|podnie[śs]\\S*|przejd[źz]\\S*|podwy[żz]sz\\S*)\\s+(?:dawk\\S*\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
"i",
|
||||
),
|
||||
// zmniejsz / obniż / zredukuj do|o N unit
|
||||
new RegExp(
|
||||
`\\b(?:zmniejsz\\w*|obni[żz]\\w*|zreduk\\w*|reduk\\w*)\\s+(?:dawk[ęe]\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
`\\b(?:zmniejsz\\S*|obni[żz]\\S*|zreduk\\S*|reduk\\S*)\\s+(?:dawk\\S*\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
"i",
|
||||
),
|
||||
new RegExp(
|
||||
`\\b(?:rozwa[żz]\\w*|spr[óo]buj\\w*|powinien|powinna|zalecam|sugeruj[ęe])\\b[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
`\\b(?:rozwa[żz]\\S*|spr[óo]buj\\S*|powinien|powinna|zalecam|sugeruj[ęe])[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
|
||||
"i",
|
||||
),
|
||||
new RegExp(
|
||||
@@ -217,23 +224,23 @@ const RISK_PATTERNS: Record<Locale, readonly RegExp[]> = {
|
||||
/\b(?:10[- ]jahres|zehn[- ]jahres|lebenszeit)[- ]?(?:risiko)\b/i,
|
||||
],
|
||||
fr: [
|
||||
/\brisque\s+(?:de|d'|est\s+de|d'environ)\s+(?:environ\s+|~)?\d{1,3}\s*%/i,
|
||||
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:risque|probabilit[ée]|chance)\b/i,
|
||||
/\brisque\s+(?:est\s+)?(?:de\s+|d['’])?(?:environ\s+|~)?\d{1,3}\s*%/i,
|
||||
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:risque|probabilit[ée]|chance)/i,
|
||||
/\brisque\s+(?:cardiovasculaire|cardiaque|d'avc|de\s+mortalit[ée])\s+[àa]\s+(?:10|dix)\s+ans\b/i,
|
||||
],
|
||||
es: [
|
||||
/\briesgo\s+(?:del?|es\s+del?|de\s+aproximadamente)\s+(?:aproximadamente\s+|~)?\d{1,3}\s*%/i,
|
||||
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:riesgo|probabilidad)\b/i,
|
||||
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:riesgo|probabilidad)/i,
|
||||
/\briesgo\s+(?:cardiovascular|card[íi]aco|de\s+ictus|de\s+mortalidad)\s+a\s+(?:10|diez)\s+a[ñn]os\b/i,
|
||||
],
|
||||
it: [
|
||||
/\brischio\s+(?:del?|dell'|[èe]\s+del?|di\s+circa)\s+(?:circa\s+|~)?\d{1,3}\s*%/i,
|
||||
/\b\d{1,3}\s*%\s+(?:di\s+)?(?:rischio|probabilit[àa])\b/i,
|
||||
/\b\d{1,3}\s*%\s+(?:di\s+)?(?:rischio|probabilit[àa])/i,
|
||||
/\brischio\s+(?:cardiovascolare|cardiaco|di\s+ictus|di\s+mortalit[àa])\s+a\s+(?:10|dieci)\s+anni\b/i,
|
||||
],
|
||||
pl: [
|
||||
/\bryzyko\s+(?:wynosi\s+|około\s+|~)?\d{1,3}\s*%/i,
|
||||
/\b\d{1,3}\s*%\s+(?:ryzyka|prawdopodobie[ńn]stwa)\b/i,
|
||||
/\b\d{1,3}\s*%\s+(?:ryzyka|prawdopodobie[ńn]stwa)/i,
|
||||
/\bryzyk\w*\s+(?:sercowo[- ]naczyniow\w*|zawału|udaru|zgonu)\s+w\s+(?:ci[ąa]gu\s+)?(?:10|dziesi[ęe]ciu)\s+lat\b/i,
|
||||
],
|
||||
};
|
||||
@@ -276,17 +283,17 @@ const CAUSAL_PATTERNS: Record<Locale, readonly RegExp[]> = {
|
||||
/\bcaus(?:e|es|ent|é|ée)\b/i,
|
||||
/\ben\s+raison\s+de\b/i,
|
||||
/\bentra[îi]n\w*/i,
|
||||
/\ba\s+conduit\s+[àa]\b/i,
|
||||
/\bd[ûu]\s+[àa]\b/i,
|
||||
/\ba\s+conduit\s+[àa]/i,
|
||||
/\bd[ûu]\s+[àa]/i,
|
||||
/\bresponsable\s+de\b/i,
|
||||
/\bgr[âa]ce\s+[àa]\b/i,
|
||||
/\bgr[âa]ce\s+[àa]/i,
|
||||
/\bprovoqu\w*/i,
|
||||
],
|
||||
es: [
|
||||
/\bporque\b/i,
|
||||
/\bdebido\s+a\b/i,
|
||||
/\ba\s+causa\s+de\b/i,
|
||||
/\bcaus(?:a|an|ó|ado)\b/i,
|
||||
/\bcaus(?:a|an|ó|ado)/i,
|
||||
/\bprovoc\w*/i,
|
||||
/\bllev[óo]\s+a\b/i,
|
||||
/\bresponsable\s+de\b/i,
|
||||
@@ -294,7 +301,7 @@ const CAUSAL_PATTERNS: Record<Locale, readonly RegExp[]> = {
|
||||
/\bimpuls\w*\s+(?:por|el|la)\b/i,
|
||||
],
|
||||
it: [
|
||||
/\bperch[ée]\b/i,
|
||||
/\bperch[ée]/i,
|
||||
/\ba\s+causa\s+di\b/i,
|
||||
/\bcaus(?:a|ano|ato|ata)\b/i,
|
||||
/\bprovoc\w*/i,
|
||||
@@ -305,7 +312,7 @@ const CAUSAL_PATTERNS: Record<Locale, readonly RegExp[]> = {
|
||||
/\bguid\w*\s+da\b/i,
|
||||
],
|
||||
pl: [
|
||||
/\bponiewa[żz]\b/i,
|
||||
/\bponiewa[żz]/i,
|
||||
/\bz\s+powodu\b/i,
|
||||
/\bpowoduj\w*/i,
|
||||
/\bspowodowa\w*/i,
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
|
||||
import {
|
||||
runDocumentSummary,
|
||||
transcribeDocument,
|
||||
documentSummaryBlockedCopy,
|
||||
} from "@/lib/documents/describe";
|
||||
import type { AIProvider } from "@/lib/ai/types";
|
||||
import { locales } from "@/lib/i18n/config";
|
||||
|
||||
function providerReturning(content: string): AIProvider {
|
||||
return {
|
||||
type: "local",
|
||||
generateCompletion: vi.fn().mockResolvedValue({ content }),
|
||||
} as unknown as AIProvider;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("runDocumentSummary — screened prose", () => {
|
||||
const VIOLATIONS: Record<string, string> = {
|
||||
en: "This report suggests you increase to 10 mg next week.",
|
||||
de: "Der Bericht empfiehlt, auf 7,5 mg zu erhöhen in der nächsten Woche.",
|
||||
fr: "Ce rapport indique d'augmenter votre dose à 2,4 mg la semaine prochaine.",
|
||||
es: "Este informe sugiere aumentar su dosis a 2,4 mg la próxima semana.",
|
||||
it: "Questo referto suggerisce di aumentare la sua dose a 2,4 mg la prossima settimana.",
|
||||
pl: "Ten raport sugeruje, aby zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.",
|
||||
};
|
||||
|
||||
for (const locale of locales) {
|
||||
it(`reports the violation instead of the prose in ${locale}`, async () => {
|
||||
const out = await runDocumentSummary({
|
||||
provider: providerReturning(VIOLATIONS[locale]),
|
||||
providerType: "local",
|
||||
ocrText: "irrelevant",
|
||||
locale,
|
||||
});
|
||||
expect(out.blocked).toBe("dose_prescription");
|
||||
expect(out.summary).toBe("");
|
||||
});
|
||||
}
|
||||
|
||||
it("passes a genuinely descriptive summary through untouched", async () => {
|
||||
const clean =
|
||||
"A lab report from a clinic, dated last month, listing routine blood values.";
|
||||
const out = await runDocumentSummary({
|
||||
provider: providerReturning(clean),
|
||||
providerType: "local",
|
||||
ocrText: "irrelevant",
|
||||
locale: "en",
|
||||
});
|
||||
expect(out.blocked).toBeNull();
|
||||
expect(out.summary).toBe(clean);
|
||||
});
|
||||
});
|
||||
|
||||
describe("documentSummaryBlockedCopy", () => {
|
||||
it("carries a native German body and rides English elsewhere", () => {
|
||||
expect(documentSummaryBlockedCopy("de")).not.toBe(
|
||||
documentSummaryBlockedCopy("en"),
|
||||
);
|
||||
expect(documentSummaryBlockedCopy("fr")).toBe(
|
||||
documentSummaryBlockedCopy("en"),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcribeDocument — deliberately NOT screened", () => {
|
||||
/**
|
||||
* Transcription reproduces the user's OWN document. A discharge letter that
|
||||
* says "increase to 10 mg" is the prescriber's instruction; screening it
|
||||
* would delete the user's record from their own view. The screen exists to
|
||||
* stop the MODEL originating such a claim, not to censor the source.
|
||||
*/
|
||||
it("returns a dose instruction from the source document verbatim", async () => {
|
||||
const letter =
|
||||
"Plan: increase to 10 mg daily from 1 August. Review in six weeks.";
|
||||
const out = await transcribeDocument({
|
||||
provider: providerReturning(letter),
|
||||
providerType: "local",
|
||||
});
|
||||
expect(out.text).toBe(letter);
|
||||
});
|
||||
|
||||
it("echoes posted OCR text verbatim without a provider round-trip", async () => {
|
||||
const letter = "Risque cardiovasculaire à 10 ans : 12%.";
|
||||
const provider = providerReturning("should not be called");
|
||||
const out = await transcribeDocument({
|
||||
provider,
|
||||
providerType: "local-ocr",
|
||||
ocrText: letter,
|
||||
});
|
||||
expect(out.text).toBe(letter);
|
||||
expect(provider.generateCompletion).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import { finalizeStatusSummary } from "@/lib/insights/status-shared";
|
||||
import { locales } from "@/lib/i18n/config";
|
||||
|
||||
/**
|
||||
* The status-card chokepoint. Before this existed the only transform between a
|
||||
* provider and a persisted assessment was whitespace normalisation, so a dose
|
||||
* imperative on the Weight card persisted as the day's cached assessment while
|
||||
* the identical sentence in the Coach was caught and replaced.
|
||||
*
|
||||
* The WITHHOLD policy itself is asserted by the caller's `ok: false` branch —
|
||||
* here we pin that the chokepoint reports the violation instead of returning
|
||||
* text, and that clean prose survives byte-for-byte.
|
||||
*/
|
||||
|
||||
const DOSE_VIOLATION: Record<string, string> = {
|
||||
en: '{"summary":"Consider increasing to 10 mg next week."}',
|
||||
de: '{"summary":"Erhöhe auf 7,5 mg in der nächsten Woche."}',
|
||||
fr: '{"summary":"Augmentez votre dose à 2,4 mg la semaine prochaine."}',
|
||||
es: '{"summary":"Aumente su dosis a 2,4 mg la próxima semana."}',
|
||||
it: '{"summary":"Aumenti la sua dose a 2,4 mg la prossima settimana."}',
|
||||
pl: '{"summary":"Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu."}',
|
||||
};
|
||||
|
||||
const CAUSAL_VIOLATION: Record<string, string> = {
|
||||
en: '{"summary":"Your weight fell because you slept more."}',
|
||||
de: '{"summary":"Dein Gewicht sank wegen des Schlafs."}',
|
||||
fr: '{"summary":"Votre poids a baissé parce que vous avez mieux dormi."}',
|
||||
es: '{"summary":"Su peso bajó porque durmió más."}',
|
||||
it: '{"summary":"Il suo peso è calato perché ha dormito di più."}',
|
||||
pl: '{"summary":"Waga spadła, ponieważ spał Pan więcej."}',
|
||||
};
|
||||
|
||||
const CLEAN: Record<string, string> = {
|
||||
en: '{"summary":"Your weight is averaging 82 kg this week, steady against last month."}',
|
||||
de: '{"summary":"Dein Gewicht liegt diese Woche im Schnitt bei 82 kg, stabil gegenüber dem Vormonat."}',
|
||||
fr: '{"summary":"Votre poids est en moyenne de 82 kg cette semaine, stable par rapport au mois dernier."}',
|
||||
es: '{"summary":"Su peso promedia 82 kg esta semana, estable frente al mes pasado."}',
|
||||
it: '{"summary":"Il suo peso è in media di 82 kg questa settimana, stabile rispetto al mese scorso."}',
|
||||
pl: '{"summary":"Waga wynosi średnio 82 kg w tym tygodniu, stabilnie wobec zeszłego miesiąca."}',
|
||||
};
|
||||
|
||||
describe("finalizeStatusSummary — dose prescriptions are withheld", () => {
|
||||
for (const locale of locales) {
|
||||
it(`reports a violation instead of text in ${locale}`, () => {
|
||||
const out = finalizeStatusSummary(DOSE_VIOLATION[locale], locale);
|
||||
expect(out.ok).toBe(false);
|
||||
if (!out.ok) expect(out.reason).toBe("dose_prescription");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("finalizeStatusSummary — causal claims are withheld", () => {
|
||||
/**
|
||||
* Status cards are the `insights` surface GROUND RULE 12 declares, so unlike
|
||||
* the Coach they DO enforce the no-causation contract.
|
||||
*/
|
||||
for (const locale of locales) {
|
||||
it(`reports a causal claim in ${locale}`, () => {
|
||||
const out = finalizeStatusSummary(CAUSAL_VIOLATION[locale], locale);
|
||||
expect(out.ok).toBe(false);
|
||||
if (!out.ok) expect(out.reason).toBe("causal_claim");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("finalizeStatusSummary — clean assessments pass untouched", () => {
|
||||
for (const locale of locales) {
|
||||
it(`returns the parsed summary verbatim in ${locale}`, () => {
|
||||
const out = finalizeStatusSummary(CLEAN[locale], locale);
|
||||
expect(out.ok).toBe(true);
|
||||
if (out.ok) {
|
||||
expect(out.text).toBe(
|
||||
JSON.parse(CLEAN[locale]).summary as unknown as string,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it("still parses a bare-prose reply that ignores the JSON envelope", () => {
|
||||
const out = finalizeStatusSummary(
|
||||
"Your resting pulse is averaging 58 bpm, in line with your baseline.",
|
||||
"en",
|
||||
);
|
||||
expect(out.ok).toBe(true);
|
||||
if (out.ok) expect(out.text).toContain("58 bpm");
|
||||
});
|
||||
|
||||
it("treats an empty completion as empty text, not a violation", () => {
|
||||
const out = finalizeStatusSummary("", "en");
|
||||
expect(out.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
buildBriefingGroundingCorrection,
|
||||
} from "@/lib/ai/briefing-grounding";
|
||||
import { screenInsightPayloadProse } from "@/lib/ai/safety/insight-payload-screen";
|
||||
import { computeCitationCoverage } from "@/lib/ai/citation-coverage";
|
||||
import { applyInsightsExcludeFilter } from "@/lib/insights/exclude-filter";
|
||||
import { getCachedFeatures } from "@/lib/insights/feature-cache";
|
||||
import {
|
||||
@@ -1039,6 +1040,32 @@ export async function generateComprehensiveInsight(
|
||||
}
|
||||
}
|
||||
|
||||
// Citation coverage. This annotation is what the admin AI-quality dashboard
|
||||
// slices to track how many normative recommendations carry a curated
|
||||
// medical-reference id. It used to hang off `generateInsight()`, which has no
|
||||
// production caller, so the dashboard was reading a metric nothing emitted.
|
||||
// Observational only — an uncited normative rec is a logged warning, never a
|
||||
// reason to fail the generation.
|
||||
if (insights && typeof insights === "object") {
|
||||
const payload = insights as Record<string, unknown>;
|
||||
if (Array.isArray(payload.recommendations)) {
|
||||
const coverage = computeCitationCoverage({
|
||||
recommendations: payload.recommendations,
|
||||
});
|
||||
annotate({
|
||||
action: { name: "insights.generate.citation_coverage" },
|
||||
meta: {
|
||||
locale,
|
||||
totalRecommendations: coverage.totalRecommendations,
|
||||
normativeRecommendations: coverage.normativeRecommendations,
|
||||
citedNormativeRecommendations: coverage.citedNormativeRecommendations,
|
||||
uncitedNormativeCount:
|
||||
coverage.uncitedNormativeRecommendationIds.length,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Outbound safety screen over the WHOLE prose surface.
|
||||
//
|
||||
// SURFACE POLICY — WITHHOLD. This payload is generated in the background and
|
||||
|
||||
Reference in new issue
Block a user