Wire citation coverage into the live path and cover the screen in all six locales

The coverage check hung off generateInsight, whose only caller runWithFallback
has no production caller left -- every live surface routes through
runRawCompletionWithFallback instead. The admin quality dashboard was reading a
metric nothing emitted. Widen the input to the structural shape both payloads
share and call it from the comprehensive post-parse step, so the annotation
reflects real generations.

Writing the locale tests found three gaps the English-only banks hid: Polish
verb suffixes fell outside the ASCII word class, a trailing word boundary after
an accented character never closed, and German subordinate clauses put the verb
last so the natural phrasing of a dose instruction never matched. The payload
collector also skipped legacy bare-string recommendations entirely.
This commit is contained in:
Marc-André Bombeck committed 2026-07-19 03:54:33 +02:00
1 parent 0becc2a56d
commit 544ae90365
8 files changed
+619 -19

No files matched your search

+41 -4
View File
@@ -1,4 +1,37 @@
import type { AIInsightResponse } from "./schema";
/**
* Reachability note. This module was written for `generateInsight()`, whose
* only caller is `runWithFallback()` — and `runWithFallback()` has no
* production caller left (every live surface routes through
* `runRawCompletionWithFallback` instead). The coverage annotation therefore
* never executed in production: an audit surface that implied enforcement it
* did not perform. Rather than delete working, tested logic, the input type is
* widened to the structural shape both payloads share and the comprehensive
* post-parse step now calls it, so the admin quality dashboard sees real data.
*/
/**
* The minimum recommendation shape the coverage check needs. Both the strict
* `AIInsightResponse` and the live `InsightResult` satisfy it; the live union
* also admits a bare string, which carries neither an id nor a citation and is
* therefore counted as uncited when it makes a normative claim.
*/
export interface RecommendationForCoverage {
id?: string;
text: string;
referenceId?: string;
/**
* Callers pass their own richer recommendation objects (severity, rationale,
* metricSource, …). Only the three fields above are read; the index
* signature keeps a wider literal assignable without a cast at every site.
*/
[key: string]: unknown;
}
export interface PayloadForCoverage {
recommendations: ReadonlyArray<RecommendationForCoverage | string>;
/** Callers pass the whole payload; only `recommendations` is read. */
[key: string]: unknown;
}
/**
* v1.4.16 phase B5a — citation-coverage post-validation.
@@ -68,20 +101,24 @@ export interface CitationCoverage {
* the successful parse and forwards the result via `annotate()`.
*/
export function computeCitationCoverage(
parsed: AIInsightResponse,
parsed: PayloadForCoverage,
): CitationCoverage {
const total = parsed.recommendations.length;
let normative = 0;
let cited = 0;
const uncitedIds: string[] = [];
for (const rec of parsed.recommendations) {
for (const [index, raw] of parsed.recommendations.entries()) {
const rec: RecommendationForCoverage =
typeof raw === "string" ? { text: raw } : raw;
if (!detectsNormativeClaim(rec.text)) continue;
normative += 1;
if (rec.referenceId) {
cited += 1;
} else {
uncitedIds.push(rec.id);
// A legacy string rec has no id of its own; index it so the dashboard
// can still point at which entry was uncited.
uncitedIds.push(rec.id ?? `index:${index}`);
}
}
@@ -0,0 +1,119 @@
import { describe, it, expect } from "vitest";
import {
collectInsightProse,
screenInsightPayloadProse,
} from "@/lib/ai/safety/insight-payload-screen";
import { locales } from "@/lib/i18n/config";
/**
* The briefing's number-grounding gate covers `paragraph`, `signalsOfDay[]`
* and `keyFindings[]`, and it grades NUMBERS only — so a digit-free risk
* assertion passed it, and `recommendations[]` / `summary` were not graded at
* all. These tests pin that the safety screen reaches every prose field.
*/
const CLEAN_PAYLOAD = {
summary: "Your tracked metrics held steady over the last 30 days.",
recommendations: [
{ id: "r1", text: "Keep logging your morning readings at the same time." },
],
dailyBriefing: {
paragraph: "Weight and sleep both moved with your usual weekday rhythm.",
signalsOfDay: [{ headline: "Steady week", nudge: "Keep it going." }],
keyFindings: [{ headline: "Sleep", detail: "In line with your baseline." }],
},
};
describe("collectInsightProse", () => {
it("reaches summary, recommendations and every briefing sub-field", () => {
const prose = collectInsightProse(CLEAN_PAYLOAD);
expect(prose).toContain(CLEAN_PAYLOAD.summary);
expect(prose).toContain(CLEAN_PAYLOAD.recommendations[0].text);
expect(prose).toContain(CLEAN_PAYLOAD.dailyBriefing.paragraph);
expect(prose).toContain("Steady week");
expect(prose).toContain("In line with your baseline.");
});
it("handles the legacy bare-string recommendation shape", () => {
const prose = collectInsightProse({
recommendations: ["Drink water with each dose."],
});
expect(prose).toContain("Drink water with each dose.");
});
it("returns nothing for a non-object payload", () => {
expect(collectInsightProse(null)).toEqual([]);
expect(collectInsightProse("nope")).toEqual([]);
});
});
describe("screenInsightPayloadProse — clean payload", () => {
for (const locale of locales) {
it(`passes a grounded payload for a ${locale} reader`, () => {
expect(screenInsightPayloadProse(CLEAN_PAYLOAD, locale)).toBeNull();
});
}
});
describe("screenInsightPayloadProse — fields the number gate never graded", () => {
it("catches a digit-free risk assertion in the briefing paragraph", () => {
// The exact shape the number-only gate lets through: no digit anywhere the
// grounding check could grade. It trips the RISK contract rather than the
// dose one -- "stepping your dose up" carries no number+unit, so the dose
// bank (deliberately tight, to protect factual restatements) does not fire.
const reason = screenInsightPayloadProse(
{
recommendations: [],
dailyBriefing: {
paragraph:
"Your 10-year cardiovascular risk is elevated — consider stepping your dose up.",
},
},
"en",
);
expect(reason).toBe("risk_score");
});
it("catches a dose imperative in recommendations[]", () => {
const reason = screenInsightPayloadProse(
{
recommendations: [
{ id: "r1", text: "Increase to 10 mg starting next week." },
],
},
"en",
);
expect(reason).toBe("dose_prescription");
});
it("catches a causal claim in summary", () => {
const reason = screenInsightPayloadProse(
{
summary: "Your weight fell because you slept more.",
recommendations: [],
},
"en",
);
expect(reason).toBe("causal_claim");
});
});
describe("screenInsightPayloadProse — non-EN/DE locales", () => {
const VIOLATIONS: Record<string, string> = {
fr: "Augmentez votre dose à 2,4 mg la semaine prochaine.",
es: "Aumente su dosis a 2,4 mg la próxima semana.",
it: "Aumenti la sua dose a 2,4 mg la prossima settimana.",
pl: "Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.",
};
for (const [locale, text] of Object.entries(VIOLATIONS)) {
it(`catches a ${locale} dose imperative in recommendations[]`, () => {
const reason = screenInsightPayloadProse(
{ recommendations: [{ id: "r1", text }] },
locale as (typeof locales)[number],
);
expect(reason).toBe("dose_prescription");
});
}
});
@@ -0,0 +1,211 @@
import { describe, it, expect } from "vitest";
import {
screenModelOutput,
CONVERSATIONAL_CONTRACTS,
INSIGHTS_CONTRACTS,
} from "@/lib/ai/safety/outbound-screen";
import { locales, type Locale } from "@/lib/i18n/config";
/**
* The whole point of threading locale is that fr / es / it / pl are enforced,
* not just en / de. Every contract below is asserted in all six languages: a
* violating sentence written in the reader's language must be caught, and an
* ordinary sentence in that same language must pass untouched.
*/
// ── dose-prescription ────────────────────────────────────────────────────
const DOSE_VIOLATIONS: Record<Locale, string> = {
en: "You should step up to 2.4 mg next week.",
de: "Erhöhe auf 7,5 mg, sobald du dich daran gewöhnt hast.",
fr: "Augmentez votre dose à 2,4 mg la semaine prochaine.",
es: "Aumente su dosis a 2,4 mg la próxima semana.",
it: "Aumenti la sua dose a 2,4 mg la prossima settimana.",
pl: "Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.",
};
/**
* Factual restatements of a CURRENT dose. The contracts explicitly permit
* these, so a screen that flagged them would make the Coach useless on exactly
* the topic users bring it.
*/
const DOSE_CLEAN: Record<Locale, string> = {
en: "You're in week 3 on 7.5 mg, and your weight is tracking down.",
de: "Du bist in Woche 3 bei 7,5 mg, und dein Gewicht geht zurück.",
fr: "Vous êtes en semaine 3 à 7,5 mg, et votre poids diminue.",
es: "Está en la semana 3 con 7,5 mg, y su peso está bajando.",
it: "È alla settimana 3 con 7,5 mg, e il suo peso sta calando.",
pl: "Jest Pan w 3. tygodniu na 7,5 mg, a waga spada.",
};
describe("screenModelOutput — dose-prescription, all six locales", () => {
for (const locale of locales) {
it(`blocks a dose-change imperative in ${locale}`, () => {
const d = screenModelOutput(
DOSE_VIOLATIONS[locale],
locale,
CONVERSATIONAL_CONTRACTS,
);
expect(d.block).toBe(true);
expect(d.reason).toBe("dose_prescription");
});
it(`passes a factual dose restatement in ${locale}`, () => {
const d = screenModelOutput(
DOSE_CLEAN[locale],
locale,
CONVERSATIONAL_CONTRACTS,
);
expect(d.block).toBe(false);
expect(d.reason).toBeNull();
});
}
});
// ── risk-score fabrication ───────────────────────────────────────────────
const RISK_VIOLATIONS: Record<Locale, string> = {
en: "Your 10-year cardiovascular risk is about 12%.",
de: "Dein Risiko liegt bei etwa 14%.",
fr: "Votre risque est d'environ 12%.",
es: "Su riesgo es del 12%.",
it: "Il suo rischio è del 12%.",
pl: "Pana ryzyko wynosi 12%.",
};
/** A plain percentage that is NOT a clinical risk claim must pass. */
const RISK_CLEAN: Record<Locale, string> = {
en: "Your adherence was 92% over the last 30 days.",
de: "Deine Einnahmetreue lag bei 92% in den letzten 30 Tagen.",
fr: "Votre observance était de 92% sur les 30 derniers jours.",
es: "Su adherencia fue del 92% en los últimos 30 días.",
it: "La sua aderenza è stata del 92% negli ultimi 30 giorni.",
pl: "Przestrzeganie zaleceń wyniosło 92% w ostatnich 30 dniach.",
};
describe("screenModelOutput — risk-score, all six locales", () => {
for (const locale of locales) {
it(`blocks a fabricated clinical risk figure in ${locale}`, () => {
const d = screenModelOutput(
RISK_VIOLATIONS[locale],
locale,
CONVERSATIONAL_CONTRACTS,
);
expect(d.block).toBe(true);
expect(d.reason).toBe("risk_score");
});
it(`passes a non-risk percentage in ${locale}`, () => {
const d = screenModelOutput(
RISK_CLEAN[locale],
locale,
CONVERSATIONAL_CONTRACTS,
);
expect(d.block).toBe(false);
});
}
});
// ── causal claims (GROUND RULE 12) ───────────────────────────────────────
const CAUSAL_VIOLATIONS: Record<Locale, string> = {
en: "Your weight fell because you slept more.",
de: "Dein Gewicht sank wegen des Schlafs.",
fr: "Votre poids a baissé parce que vous avez mieux dormi.",
es: "Su peso bajó porque durmió más.",
it: "Il suo peso è calato perché ha dormito di più.",
pl: "Waga spadła, ponieważ spał Pan więcej.",
};
/** Descriptive association framing is what the contract REQUIRES. */
const CAUSAL_CLEAN: Record<Locale, string> = {
en: "Your weight moved with your sleep this week.",
de: "Dein Gewicht bewegte sich diese Woche mit deinem Schlaf.",
fr: "Votre poids a évolué avec votre sommeil cette semaine.",
es: "Su peso evolucionó junto con su sueño esta semana.",
it: "Il suo peso si è mosso insieme al suo sonno questa settimana.",
pl: "Waga zmieniała się razem ze snem w tym tygodniu.",
};
describe("screenModelOutput — causal claims, all six locales", () => {
for (const locale of locales) {
it(`blocks asserted causation in ${locale}`, () => {
const d = screenModelOutput(
CAUSAL_VIOLATIONS[locale],
locale,
INSIGHTS_CONTRACTS,
);
expect(d.block).toBe(true);
expect(d.reason).toBe("causal_claim");
});
it(`passes descriptive association framing in ${locale}`, () => {
const d = screenModelOutput(
CAUSAL_CLEAN[locale],
locale,
INSIGHTS_CONTRACTS,
);
expect(d.block).toBe(false);
});
}
});
// ── contract selection ───────────────────────────────────────────────────
describe("screenModelOutput — contract selection", () => {
it("does NOT flag causal wording under the conversational contracts", () => {
// The Coach must keep "because" — GROUND RULE 12's surface is `insights`,
// and blocking it conversationally would gut ordinary explanation.
const d = screenModelOutput(
CAUSAL_VIOLATIONS.en,
"en",
CONVERSATIONAL_CONTRACTS,
);
expect(d.block).toBe(false);
});
it("flags the same sentence under the insights contracts", () => {
const d = screenModelOutput(CAUSAL_VIOLATIONS.en, "en", INSIGHTS_CONTRACTS);
expect(d.block).toBe(true);
});
it("reports dose before risk when a reply trips both", () => {
const d = screenModelOutput(
"Your risk is 12% — step up to 2.4 mg.",
"en",
CONVERSATIONAL_CONTRACTS,
);
expect(d.reason).toBe("dose_prescription");
});
});
// ── cross-language coverage ──────────────────────────────────────────────
describe("screenModelOutput — English violations reach non-English readers", () => {
/**
* A provider routinely answers in English regardless of the locale
* directive. If the screen consulted only the reader's bank, the proven
* English violation shapes would pass on five of six locales.
*/
for (const locale of locales) {
it(`blocks an English dose imperative for a ${locale} reader`, () => {
const d = screenModelOutput(
DOSE_VIOLATIONS.en,
locale,
CONVERSATIONAL_CONTRACTS,
);
expect(d.block).toBe(true);
expect(d.reason).toBe("dose_prescription");
});
}
});
describe("screenModelOutput — empty input", () => {
it("passes empty and whitespace-only text (handled upstream)", () => {
expect(screenModelOutput("", "en", INSIGHTS_CONTRACTS).block).toBe(false);
expect(screenModelOutput(" ", "fr", INSIGHTS_CONTRACTS).block).toBe(
false,
);
});
});
+6 -1
View File
@@ -34,7 +34,12 @@ export function collectInsightProse(parsed: unknown): string[] {
for (const rec of Array.isArray(root.recommendations)
? root.recommendations
: []) {
if (rec && typeof rec === "object") {
// The live schema admits BOTH a structured object and a legacy bare
// string. Reading only `.text` silently skipped every string rec, which is
// exactly the shape older cached payloads use.
if (typeof rec === "string") {
push(rec);
} else if (rec && typeof rec === "object") {
push((rec as Record<string, unknown>).text);
}
}
+21 -14
View File
@@ -110,6 +110,13 @@ const DOSE_PATTERNS: Record<Locale, readonly RegExp[]> = {
`\\bn[äa]chste\\s+(?:stufe|dosis)\\b[^.?!]{0,30}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
"i",
),
// German puts the verb last in a subordinate clause ("auf 7,5 mg zu
// erhöhen"), so the verb-first patterns above miss the most natural
// phrasing of the instruction. Match the inverted order too.
new RegExp(
`\\b(?:auf|um)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b[^.?!]{0,30}\\b(?:erhöh\\S*|steiger\\S*|reduzier\\S*|senk\\S*|verringer\\S*|hochsetz\\S*)`,
"i",
),
new RegExp(
`\\b(?:erwäg\\w*|probier\\w*|du\\s+solltest|ich\\s+empfehle|ich\\s+schlage\\s+vor)\\b[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
"i",
@@ -179,16 +186,16 @@ const DOSE_PATTERNS: Record<Locale, readonly RegExp[]> = {
pl: [
// zwiększ / podnieś / przejdź do|o N unit
new RegExp(
`\\b(?:zwi[ęe]ksz\\w*|podnie[śs]\\w*|przejd[źz]\\w*|podwy[żz]sz\\w*)\\s+(?:dawk[ęe]\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
`\\b(?:zwi[ęe]ksz\\S*|podnie[śs]\\S*|przejd[źz]\\S*|podwy[żz]sz\\S*)\\s+(?:dawk\\S*\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
"i",
),
// zmniejsz / obniż / zredukuj do|o N unit
new RegExp(
`\\b(?:zmniejsz\\w*|obni[żz]\\w*|zreduk\\w*|reduk\\w*)\\s+(?:dawk[ęe]\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
`\\b(?:zmniejsz\\S*|obni[żz]\\S*|zreduk\\S*|reduk\\S*)\\s+(?:dawk\\S*\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`,
"i",
),
new RegExp(
`\\b(?:rozwa[żz]\\w*|spr[óo]buj\\w*|powinien|powinna|zalecam|sugeruj[ęe])\\b[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
`\\b(?:rozwa[żz]\\S*|spr[óo]buj\\S*|powinien|powinna|zalecam|sugeruj[ęe])[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`,
"i",
),
new RegExp(
@@ -217,23 +224,23 @@ const RISK_PATTERNS: Record<Locale, readonly RegExp[]> = {
/\b(?:10[- ]jahres|zehn[- ]jahres|lebenszeit)[- ]?(?:risiko)\b/i,
],
fr: [
/\brisque\s+(?:de|d'|est\s+de|d'environ)\s+(?:environ\s+|~)?\d{1,3}\s*%/i,
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:risque|probabilit[ée]|chance)\b/i,
/\brisque\s+(?:est\s+)?(?:de\s+|d['’])?(?:environ\s+|~)?\d{1,3}\s*%/i,
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:risque|probabilit[ée]|chance)/i,
/\brisque\s+(?:cardiovasculaire|cardiaque|d'avc|de\s+mortalit[ée])\s+[àa]\s+(?:10|dix)\s+ans\b/i,
],
es: [
/\briesgo\s+(?:del?|es\s+del?|de\s+aproximadamente)\s+(?:aproximadamente\s+|~)?\d{1,3}\s*%/i,
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:riesgo|probabilidad)\b/i,
/\b\d{1,3}\s*%\s+(?:de\s+)?(?:riesgo|probabilidad)/i,
/\briesgo\s+(?:cardiovascular|card[íi]aco|de\s+ictus|de\s+mortalidad)\s+a\s+(?:10|diez)\s+a[ñn]os\b/i,
],
it: [
/\brischio\s+(?:del?|dell'|[èe]\s+del?|di\s+circa)\s+(?:circa\s+|~)?\d{1,3}\s*%/i,
/\b\d{1,3}\s*%\s+(?:di\s+)?(?:rischio|probabilit[àa])\b/i,
/\b\d{1,3}\s*%\s+(?:di\s+)?(?:rischio|probabilit[àa])/i,
/\brischio\s+(?:cardiovascolare|cardiaco|di\s+ictus|di\s+mortalit[àa])\s+a\s+(?:10|dieci)\s+anni\b/i,
],
pl: [
/\bryzyko\s+(?:wynosi\s+|około\s+|~)?\d{1,3}\s*%/i,
/\b\d{1,3}\s*%\s+(?:ryzyka|prawdopodobie[ńn]stwa)\b/i,
/\b\d{1,3}\s*%\s+(?:ryzyka|prawdopodobie[ńn]stwa)/i,
/\bryzyk\w*\s+(?:sercowo[- ]naczyniow\w*|zawału|udaru|zgonu)\s+w\s+(?:ci[ąa]gu\s+)?(?:10|dziesi[ęe]ciu)\s+lat\b/i,
],
};
@@ -276,17 +283,17 @@ const CAUSAL_PATTERNS: Record<Locale, readonly RegExp[]> = {
/\bcaus(?:e|es|ent|é|ée)\b/i,
/\ben\s+raison\s+de\b/i,
/\bentra[îi]n\w*/i,
/\ba\s+conduit\s+[àa]\b/i,
/\bd[ûu]\s+[àa]\b/i,
/\ba\s+conduit\s+[àa]/i,
/\bd[ûu]\s+[àa]/i,
/\bresponsable\s+de\b/i,
/\bgr[âa]ce\s+[àa]\b/i,
/\bgr[âa]ce\s+[àa]/i,
/\bprovoqu\w*/i,
],
es: [
/\bporque\b/i,
/\bdebido\s+a\b/i,
/\ba\s+causa\s+de\b/i,
/\bcaus(?:a|an|ó|ado)\b/i,
/\bcaus(?:a|an|ó|ado)/i,
/\bprovoc\w*/i,
/\bllev[óo]\s+a\b/i,
/\bresponsable\s+de\b/i,
@@ -294,7 +301,7 @@ const CAUSAL_PATTERNS: Record<Locale, readonly RegExp[]> = {
/\bimpuls\w*\s+(?:por|el|la)\b/i,
],
it: [
/\bperch[ée]\b/i,
/\bperch[ée]/i,
/\ba\s+causa\s+di\b/i,
/\bcaus(?:a|ano|ato|ata)\b/i,
/\bprovoc\w*/i,
@@ -305,7 +312,7 @@ const CAUSAL_PATTERNS: Record<Locale, readonly RegExp[]> = {
/\bguid\w*\s+da\b/i,
],
pl: [
/\bponiewa[żz]\b/i,
/\bponiewa[żz]/i,
/\bz\s+powodu\b/i,
/\bpowoduj\w*/i,
/\bspowodowa\w*/i,
@@ -0,0 +1,100 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
import {
runDocumentSummary,
transcribeDocument,
documentSummaryBlockedCopy,
} from "@/lib/documents/describe";
import type { AIProvider } from "@/lib/ai/types";
import { locales } from "@/lib/i18n/config";
function providerReturning(content: string): AIProvider {
return {
type: "local",
generateCompletion: vi.fn().mockResolvedValue({ content }),
} as unknown as AIProvider;
}
beforeEach(() => {
vi.clearAllMocks();
});
describe("runDocumentSummary — screened prose", () => {
const VIOLATIONS: Record<string, string> = {
en: "This report suggests you increase to 10 mg next week.",
de: "Der Bericht empfiehlt, auf 7,5 mg zu erhöhen in der nächsten Woche.",
fr: "Ce rapport indique d'augmenter votre dose à 2,4 mg la semaine prochaine.",
es: "Este informe sugiere aumentar su dosis a 2,4 mg la próxima semana.",
it: "Questo referto suggerisce di aumentare la sua dose a 2,4 mg la prossima settimana.",
pl: "Ten raport sugeruje, aby zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.",
};
for (const locale of locales) {
it(`reports the violation instead of the prose in ${locale}`, async () => {
const out = await runDocumentSummary({
provider: providerReturning(VIOLATIONS[locale]),
providerType: "local",
ocrText: "irrelevant",
locale,
});
expect(out.blocked).toBe("dose_prescription");
expect(out.summary).toBe("");
});
}
it("passes a genuinely descriptive summary through untouched", async () => {
const clean =
"A lab report from a clinic, dated last month, listing routine blood values.";
const out = await runDocumentSummary({
provider: providerReturning(clean),
providerType: "local",
ocrText: "irrelevant",
locale: "en",
});
expect(out.blocked).toBeNull();
expect(out.summary).toBe(clean);
});
});
describe("documentSummaryBlockedCopy", () => {
it("carries a native German body and rides English elsewhere", () => {
expect(documentSummaryBlockedCopy("de")).not.toBe(
documentSummaryBlockedCopy("en"),
);
expect(documentSummaryBlockedCopy("fr")).toBe(
documentSummaryBlockedCopy("en"),
);
});
});
describe("transcribeDocument — deliberately NOT screened", () => {
/**
* Transcription reproduces the user's OWN document. A discharge letter that
* says "increase to 10 mg" is the prescriber's instruction; screening it
* would delete the user's record from their own view. The screen exists to
* stop the MODEL originating such a claim, not to censor the source.
*/
it("returns a dose instruction from the source document verbatim", async () => {
const letter =
"Plan: increase to 10 mg daily from 1 August. Review in six weeks.";
const out = await transcribeDocument({
provider: providerReturning(letter),
providerType: "local",
});
expect(out.text).toBe(letter);
});
it("echoes posted OCR text verbatim without a provider round-trip", async () => {
const letter = "Risque cardiovasculaire à 10 ans : 12%.";
const provider = providerReturning("should not be called");
const out = await transcribeDocument({
provider,
providerType: "local-ocr",
ocrText: letter,
});
expect(out.text).toBe(letter);
expect(provider.generateCompletion).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,94 @@
import { describe, it, expect } from "vitest";
import { finalizeStatusSummary } from "@/lib/insights/status-shared";
import { locales } from "@/lib/i18n/config";
/**
* The status-card chokepoint. Before this existed the only transform between a
* provider and a persisted assessment was whitespace normalisation, so a dose
* imperative on the Weight card persisted as the day's cached assessment while
* the identical sentence in the Coach was caught and replaced.
*
* The WITHHOLD policy itself is asserted by the caller's `ok: false` branch —
* here we pin that the chokepoint reports the violation instead of returning
* text, and that clean prose survives byte-for-byte.
*/
const DOSE_VIOLATION: Record<string, string> = {
en: '{"summary":"Consider increasing to 10 mg next week."}',
de: '{"summary":"Erhöhe auf 7,5 mg in der nächsten Woche."}',
fr: '{"summary":"Augmentez votre dose à 2,4 mg la semaine prochaine."}',
es: '{"summary":"Aumente su dosis a 2,4 mg la próxima semana."}',
it: '{"summary":"Aumenti la sua dose a 2,4 mg la prossima settimana."}',
pl: '{"summary":"Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu."}',
};
const CAUSAL_VIOLATION: Record<string, string> = {
en: '{"summary":"Your weight fell because you slept more."}',
de: '{"summary":"Dein Gewicht sank wegen des Schlafs."}',
fr: '{"summary":"Votre poids a baissé parce que vous avez mieux dormi."}',
es: '{"summary":"Su peso bajó porque durmió más."}',
it: '{"summary":"Il suo peso è calato perché ha dormito di più."}',
pl: '{"summary":"Waga spadła, ponieważ spał Pan więcej."}',
};
const CLEAN: Record<string, string> = {
en: '{"summary":"Your weight is averaging 82 kg this week, steady against last month."}',
de: '{"summary":"Dein Gewicht liegt diese Woche im Schnitt bei 82 kg, stabil gegenüber dem Vormonat."}',
fr: '{"summary":"Votre poids est en moyenne de 82 kg cette semaine, stable par rapport au mois dernier."}',
es: '{"summary":"Su peso promedia 82 kg esta semana, estable frente al mes pasado."}',
it: '{"summary":"Il suo peso è in media di 82 kg questa settimana, stabile rispetto al mese scorso."}',
pl: '{"summary":"Waga wynosi średnio 82 kg w tym tygodniu, stabilnie wobec zeszłego miesiąca."}',
};
describe("finalizeStatusSummary — dose prescriptions are withheld", () => {
for (const locale of locales) {
it(`reports a violation instead of text in ${locale}`, () => {
const out = finalizeStatusSummary(DOSE_VIOLATION[locale], locale);
expect(out.ok).toBe(false);
if (!out.ok) expect(out.reason).toBe("dose_prescription");
});
}
});
describe("finalizeStatusSummary — causal claims are withheld", () => {
/**
* Status cards are the `insights` surface GROUND RULE 12 declares, so unlike
* the Coach they DO enforce the no-causation contract.
*/
for (const locale of locales) {
it(`reports a causal claim in ${locale}`, () => {
const out = finalizeStatusSummary(CAUSAL_VIOLATION[locale], locale);
expect(out.ok).toBe(false);
if (!out.ok) expect(out.reason).toBe("causal_claim");
});
}
});
describe("finalizeStatusSummary — clean assessments pass untouched", () => {
for (const locale of locales) {
it(`returns the parsed summary verbatim in ${locale}`, () => {
const out = finalizeStatusSummary(CLEAN[locale], locale);
expect(out.ok).toBe(true);
if (out.ok) {
expect(out.text).toBe(
JSON.parse(CLEAN[locale]).summary as unknown as string,
);
}
});
}
it("still parses a bare-prose reply that ignores the JSON envelope", () => {
const out = finalizeStatusSummary(
"Your resting pulse is averaging 58 bpm, in line with your baseline.",
"en",
);
expect(out.ok).toBe(true);
if (out.ok) expect(out.text).toContain("58 bpm");
});
it("treats an empty completion as empty text, not a violation", () => {
const out = finalizeStatusSummary("", "en");
expect(out.ok).toBe(true);
});
});
@@ -33,6 +33,7 @@ import {
buildBriefingGroundingCorrection,
} from "@/lib/ai/briefing-grounding";
import { screenInsightPayloadProse } from "@/lib/ai/safety/insight-payload-screen";
import { computeCitationCoverage } from "@/lib/ai/citation-coverage";
import { applyInsightsExcludeFilter } from "@/lib/insights/exclude-filter";
import { getCachedFeatures } from "@/lib/insights/feature-cache";
import {
@@ -1039,6 +1040,32 @@ export async function generateComprehensiveInsight(
}
}
// Citation coverage. This annotation is what the admin AI-quality dashboard
// slices to track how many normative recommendations carry a curated
// medical-reference id. It used to hang off `generateInsight()`, which has no
// production caller, so the dashboard was reading a metric nothing emitted.
// Observational only — an uncited normative rec is a logged warning, never a
// reason to fail the generation.
if (insights && typeof insights === "object") {
const payload = insights as Record<string, unknown>;
if (Array.isArray(payload.recommendations)) {
const coverage = computeCitationCoverage({
recommendations: payload.recommendations,
});
annotate({
action: { name: "insights.generate.citation_coverage" },
meta: {
locale,
totalRecommendations: coverage.totalRecommendations,
normativeRecommendations: coverage.normativeRecommendations,
citedNormativeRecommendations: coverage.citedNormativeRecommendations,
uncitedNormativeCount:
coverage.uncitedNormativeRecommendationIds.length,
},
});
}
}
// Outbound safety screen over the WHOLE prose surface.
//
// SURFACE POLICY — WITHHOLD. This payload is generated in the background and