From 544ae903653f8d9cbeed453f1d9bd08b7d7e76d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 19 Jul 2026 03:54:33 +0200 Subject: [PATCH] Wire citation coverage into the live path and cover the screen in all six locales The coverage check hung off generateInsight, whose only caller runWithFallback has no production caller left -- every live surface routes through runRawCompletionWithFallback instead. The admin quality dashboard was reading a metric nothing emitted. Widen the input to the structural shape both payloads share and call it from the comprehensive post-parse step, so the annotation reflects real generations. Writing the locale tests found three gaps the English-only banks hid: Polish verb suffixes fell outside the ASCII word class, a trailing word boundary after an accented character never closed, and German subordinate clauses put the verb last so the natural phrasing of a dose instruction never matched. The payload collector also skipped legacy bare-string recommendations entirely. --- src/lib/ai/citation-coverage.ts | 45 +++- .../__tests__/insight-payload-screen.test.ts | 119 ++++++++++ .../safety/__tests__/outbound-screen.test.ts | 211 ++++++++++++++++++ src/lib/ai/safety/insight-payload-screen.ts | 7 +- src/lib/ai/safety/outbound-screen.ts | 35 +-- .../__tests__/describe-screen.test.ts | 100 +++++++++ .../__tests__/status-summary-screen.test.ts | 94 ++++++++ src/lib/insights/comprehensive-generate.ts | 27 +++ 8 files changed, 619 insertions(+), 19 deletions(-) create mode 100644 src/lib/ai/safety/__tests__/insight-payload-screen.test.ts create mode 100644 src/lib/ai/safety/__tests__/outbound-screen.test.ts create mode 100644 src/lib/documents/__tests__/describe-screen.test.ts create mode 100644 src/lib/insights/__tests__/status-summary-screen.test.ts diff --git a/src/lib/ai/citation-coverage.ts b/src/lib/ai/citation-coverage.ts index bee38e069..c6c249bdc 100644 --- a/src/lib/ai/citation-coverage.ts +++ b/src/lib/ai/citation-coverage.ts @@ -1,4 +1,37 @@ -import type { AIInsightResponse } from "./schema"; +/** + * Reachability note. This module was written for `generateInsight()`, whose + * only caller is `runWithFallback()` — and `runWithFallback()` has no + * production caller left (every live surface routes through + * `runRawCompletionWithFallback` instead). The coverage annotation therefore + * never executed in production: an audit surface that implied enforcement it + * did not perform. Rather than delete working, tested logic, the input type is + * widened to the structural shape both payloads share and the comprehensive + * post-parse step now calls it, so the admin quality dashboard sees real data. + */ + +/** + * The minimum recommendation shape the coverage check needs. Both the strict + * `AIInsightResponse` and the live `InsightResult` satisfy it; the live union + * also admits a bare string, which carries neither an id nor a citation and is + * therefore counted as uncited when it makes a normative claim. + */ +export interface RecommendationForCoverage { + id?: string; + text: string; + referenceId?: string; + /** + * Callers pass their own richer recommendation objects (severity, rationale, + * metricSource, …). Only the three fields above are read; the index + * signature keeps a wider literal assignable without a cast at every site. + */ + [key: string]: unknown; +} + +export interface PayloadForCoverage { + recommendations: ReadonlyArray; + /** Callers pass the whole payload; only `recommendations` is read. */ + [key: string]: unknown; +} /** * v1.4.16 phase B5a — citation-coverage post-validation. @@ -68,20 +101,24 @@ export interface CitationCoverage { * the successful parse and forwards the result via `annotate()`. */ export function computeCitationCoverage( - parsed: AIInsightResponse, + parsed: PayloadForCoverage, ): CitationCoverage { const total = parsed.recommendations.length; let normative = 0; let cited = 0; const uncitedIds: string[] = []; - for (const rec of parsed.recommendations) { + for (const [index, raw] of parsed.recommendations.entries()) { + const rec: RecommendationForCoverage = + typeof raw === "string" ? { text: raw } : raw; if (!detectsNormativeClaim(rec.text)) continue; normative += 1; if (rec.referenceId) { cited += 1; } else { - uncitedIds.push(rec.id); + // A legacy string rec has no id of its own; index it so the dashboard + // can still point at which entry was uncited. + uncitedIds.push(rec.id ?? `index:${index}`); } } diff --git a/src/lib/ai/safety/__tests__/insight-payload-screen.test.ts b/src/lib/ai/safety/__tests__/insight-payload-screen.test.ts new file mode 100644 index 000000000..8dedacf17 --- /dev/null +++ b/src/lib/ai/safety/__tests__/insight-payload-screen.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect } from "vitest"; + +import { + collectInsightProse, + screenInsightPayloadProse, +} from "@/lib/ai/safety/insight-payload-screen"; +import { locales } from "@/lib/i18n/config"; + +/** + * The briefing's number-grounding gate covers `paragraph`, `signalsOfDay[]` + * and `keyFindings[]`, and it grades NUMBERS only — so a digit-free risk + * assertion passed it, and `recommendations[]` / `summary` were not graded at + * all. These tests pin that the safety screen reaches every prose field. + */ + +const CLEAN_PAYLOAD = { + summary: "Your tracked metrics held steady over the last 30 days.", + recommendations: [ + { id: "r1", text: "Keep logging your morning readings at the same time." }, + ], + dailyBriefing: { + paragraph: "Weight and sleep both moved with your usual weekday rhythm.", + signalsOfDay: [{ headline: "Steady week", nudge: "Keep it going." }], + keyFindings: [{ headline: "Sleep", detail: "In line with your baseline." }], + }, +}; + +describe("collectInsightProse", () => { + it("reaches summary, recommendations and every briefing sub-field", () => { + const prose = collectInsightProse(CLEAN_PAYLOAD); + expect(prose).toContain(CLEAN_PAYLOAD.summary); + expect(prose).toContain(CLEAN_PAYLOAD.recommendations[0].text); + expect(prose).toContain(CLEAN_PAYLOAD.dailyBriefing.paragraph); + expect(prose).toContain("Steady week"); + expect(prose).toContain("In line with your baseline."); + }); + + it("handles the legacy bare-string recommendation shape", () => { + const prose = collectInsightProse({ + recommendations: ["Drink water with each dose."], + }); + expect(prose).toContain("Drink water with each dose."); + }); + + it("returns nothing for a non-object payload", () => { + expect(collectInsightProse(null)).toEqual([]); + expect(collectInsightProse("nope")).toEqual([]); + }); +}); + +describe("screenInsightPayloadProse — clean payload", () => { + for (const locale of locales) { + it(`passes a grounded payload for a ${locale} reader`, () => { + expect(screenInsightPayloadProse(CLEAN_PAYLOAD, locale)).toBeNull(); + }); + } +}); + +describe("screenInsightPayloadProse — fields the number gate never graded", () => { + it("catches a digit-free risk assertion in the briefing paragraph", () => { + // The exact shape the number-only gate lets through: no digit anywhere the + // grounding check could grade. It trips the RISK contract rather than the + // dose one -- "stepping your dose up" carries no number+unit, so the dose + // bank (deliberately tight, to protect factual restatements) does not fire. + const reason = screenInsightPayloadProse( + { + recommendations: [], + dailyBriefing: { + paragraph: + "Your 10-year cardiovascular risk is elevated — consider stepping your dose up.", + }, + }, + "en", + ); + expect(reason).toBe("risk_score"); + }); + + it("catches a dose imperative in recommendations[]", () => { + const reason = screenInsightPayloadProse( + { + recommendations: [ + { id: "r1", text: "Increase to 10 mg starting next week." }, + ], + }, + "en", + ); + expect(reason).toBe("dose_prescription"); + }); + + it("catches a causal claim in summary", () => { + const reason = screenInsightPayloadProse( + { + summary: "Your weight fell because you slept more.", + recommendations: [], + }, + "en", + ); + expect(reason).toBe("causal_claim"); + }); +}); + +describe("screenInsightPayloadProse — non-EN/DE locales", () => { + const VIOLATIONS: Record = { + fr: "Augmentez votre dose à 2,4 mg la semaine prochaine.", + es: "Aumente su dosis a 2,4 mg la próxima semana.", + it: "Aumenti la sua dose a 2,4 mg la prossima settimana.", + pl: "Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.", + }; + + for (const [locale, text] of Object.entries(VIOLATIONS)) { + it(`catches a ${locale} dose imperative in recommendations[]`, () => { + const reason = screenInsightPayloadProse( + { recommendations: [{ id: "r1", text }] }, + locale as (typeof locales)[number], + ); + expect(reason).toBe("dose_prescription"); + }); + } +}); diff --git a/src/lib/ai/safety/__tests__/outbound-screen.test.ts b/src/lib/ai/safety/__tests__/outbound-screen.test.ts new file mode 100644 index 000000000..27f66b697 --- /dev/null +++ b/src/lib/ai/safety/__tests__/outbound-screen.test.ts @@ -0,0 +1,211 @@ +import { describe, it, expect } from "vitest"; + +import { + screenModelOutput, + CONVERSATIONAL_CONTRACTS, + INSIGHTS_CONTRACTS, +} from "@/lib/ai/safety/outbound-screen"; +import { locales, type Locale } from "@/lib/i18n/config"; + +/** + * The whole point of threading locale is that fr / es / it / pl are enforced, + * not just en / de. Every contract below is asserted in all six languages: a + * violating sentence written in the reader's language must be caught, and an + * ordinary sentence in that same language must pass untouched. + */ + +// ── dose-prescription ──────────────────────────────────────────────────── + +const DOSE_VIOLATIONS: Record = { + en: "You should step up to 2.4 mg next week.", + de: "Erhöhe auf 7,5 mg, sobald du dich daran gewöhnt hast.", + fr: "Augmentez votre dose à 2,4 mg la semaine prochaine.", + es: "Aumente su dosis a 2,4 mg la próxima semana.", + it: "Aumenti la sua dose a 2,4 mg la prossima settimana.", + pl: "Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.", +}; + +/** + * Factual restatements of a CURRENT dose. The contracts explicitly permit + * these, so a screen that flagged them would make the Coach useless on exactly + * the topic users bring it. + */ +const DOSE_CLEAN: Record = { + en: "You're in week 3 on 7.5 mg, and your weight is tracking down.", + de: "Du bist in Woche 3 bei 7,5 mg, und dein Gewicht geht zurück.", + fr: "Vous êtes en semaine 3 à 7,5 mg, et votre poids diminue.", + es: "Está en la semana 3 con 7,5 mg, y su peso está bajando.", + it: "È alla settimana 3 con 7,5 mg, e il suo peso sta calando.", + pl: "Jest Pan w 3. tygodniu na 7,5 mg, a waga spada.", +}; + +describe("screenModelOutput — dose-prescription, all six locales", () => { + for (const locale of locales) { + it(`blocks a dose-change imperative in ${locale}`, () => { + const d = screenModelOutput( + DOSE_VIOLATIONS[locale], + locale, + CONVERSATIONAL_CONTRACTS, + ); + expect(d.block).toBe(true); + expect(d.reason).toBe("dose_prescription"); + }); + + it(`passes a factual dose restatement in ${locale}`, () => { + const d = screenModelOutput( + DOSE_CLEAN[locale], + locale, + CONVERSATIONAL_CONTRACTS, + ); + expect(d.block).toBe(false); + expect(d.reason).toBeNull(); + }); + } +}); + +// ── risk-score fabrication ─────────────────────────────────────────────── + +const RISK_VIOLATIONS: Record = { + en: "Your 10-year cardiovascular risk is about 12%.", + de: "Dein Risiko liegt bei etwa 14%.", + fr: "Votre risque est d'environ 12%.", + es: "Su riesgo es del 12%.", + it: "Il suo rischio è del 12%.", + pl: "Pana ryzyko wynosi 12%.", +}; + +/** A plain percentage that is NOT a clinical risk claim must pass. */ +const RISK_CLEAN: Record = { + en: "Your adherence was 92% over the last 30 days.", + de: "Deine Einnahmetreue lag bei 92% in den letzten 30 Tagen.", + fr: "Votre observance était de 92% sur les 30 derniers jours.", + es: "Su adherencia fue del 92% en los últimos 30 días.", + it: "La sua aderenza è stata del 92% negli ultimi 30 giorni.", + pl: "Przestrzeganie zaleceń wyniosło 92% w ostatnich 30 dniach.", +}; + +describe("screenModelOutput — risk-score, all six locales", () => { + for (const locale of locales) { + it(`blocks a fabricated clinical risk figure in ${locale}`, () => { + const d = screenModelOutput( + RISK_VIOLATIONS[locale], + locale, + CONVERSATIONAL_CONTRACTS, + ); + expect(d.block).toBe(true); + expect(d.reason).toBe("risk_score"); + }); + + it(`passes a non-risk percentage in ${locale}`, () => { + const d = screenModelOutput( + RISK_CLEAN[locale], + locale, + CONVERSATIONAL_CONTRACTS, + ); + expect(d.block).toBe(false); + }); + } +}); + +// ── causal claims (GROUND RULE 12) ─────────────────────────────────────── + +const CAUSAL_VIOLATIONS: Record = { + en: "Your weight fell because you slept more.", + de: "Dein Gewicht sank wegen des Schlafs.", + fr: "Votre poids a baissé parce que vous avez mieux dormi.", + es: "Su peso bajó porque durmió más.", + it: "Il suo peso è calato perché ha dormito di più.", + pl: "Waga spadła, ponieważ spał Pan więcej.", +}; + +/** Descriptive association framing is what the contract REQUIRES. */ +const CAUSAL_CLEAN: Record = { + en: "Your weight moved with your sleep this week.", + de: "Dein Gewicht bewegte sich diese Woche mit deinem Schlaf.", + fr: "Votre poids a évolué avec votre sommeil cette semaine.", + es: "Su peso evolucionó junto con su sueño esta semana.", + it: "Il suo peso si è mosso insieme al suo sonno questa settimana.", + pl: "Waga zmieniała się razem ze snem w tym tygodniu.", +}; + +describe("screenModelOutput — causal claims, all six locales", () => { + for (const locale of locales) { + it(`blocks asserted causation in ${locale}`, () => { + const d = screenModelOutput( + CAUSAL_VIOLATIONS[locale], + locale, + INSIGHTS_CONTRACTS, + ); + expect(d.block).toBe(true); + expect(d.reason).toBe("causal_claim"); + }); + + it(`passes descriptive association framing in ${locale}`, () => { + const d = screenModelOutput( + CAUSAL_CLEAN[locale], + locale, + INSIGHTS_CONTRACTS, + ); + expect(d.block).toBe(false); + }); + } +}); + +// ── contract selection ─────────────────────────────────────────────────── + +describe("screenModelOutput — contract selection", () => { + it("does NOT flag causal wording under the conversational contracts", () => { + // The Coach must keep "because" — GROUND RULE 12's surface is `insights`, + // and blocking it conversationally would gut ordinary explanation. + const d = screenModelOutput( + CAUSAL_VIOLATIONS.en, + "en", + CONVERSATIONAL_CONTRACTS, + ); + expect(d.block).toBe(false); + }); + + it("flags the same sentence under the insights contracts", () => { + const d = screenModelOutput(CAUSAL_VIOLATIONS.en, "en", INSIGHTS_CONTRACTS); + expect(d.block).toBe(true); + }); + + it("reports dose before risk when a reply trips both", () => { + const d = screenModelOutput( + "Your risk is 12% — step up to 2.4 mg.", + "en", + CONVERSATIONAL_CONTRACTS, + ); + expect(d.reason).toBe("dose_prescription"); + }); +}); + +// ── cross-language coverage ────────────────────────────────────────────── + +describe("screenModelOutput — English violations reach non-English readers", () => { + /** + * A provider routinely answers in English regardless of the locale + * directive. If the screen consulted only the reader's bank, the proven + * English violation shapes would pass on five of six locales. + */ + for (const locale of locales) { + it(`blocks an English dose imperative for a ${locale} reader`, () => { + const d = screenModelOutput( + DOSE_VIOLATIONS.en, + locale, + CONVERSATIONAL_CONTRACTS, + ); + expect(d.block).toBe(true); + expect(d.reason).toBe("dose_prescription"); + }); + } +}); + +describe("screenModelOutput — empty input", () => { + it("passes empty and whitespace-only text (handled upstream)", () => { + expect(screenModelOutput("", "en", INSIGHTS_CONTRACTS).block).toBe(false); + expect(screenModelOutput(" ", "fr", INSIGHTS_CONTRACTS).block).toBe( + false, + ); + }); +}); diff --git a/src/lib/ai/safety/insight-payload-screen.ts b/src/lib/ai/safety/insight-payload-screen.ts index 9d8f659e9..3ee7e9adf 100644 --- a/src/lib/ai/safety/insight-payload-screen.ts +++ b/src/lib/ai/safety/insight-payload-screen.ts @@ -34,7 +34,12 @@ export function collectInsightProse(parsed: unknown): string[] { for (const rec of Array.isArray(root.recommendations) ? root.recommendations : []) { - if (rec && typeof rec === "object") { + // The live schema admits BOTH a structured object and a legacy bare + // string. Reading only `.text` silently skipped every string rec, which is + // exactly the shape older cached payloads use. + if (typeof rec === "string") { + push(rec); + } else if (rec && typeof rec === "object") { push((rec as Record).text); } } diff --git a/src/lib/ai/safety/outbound-screen.ts b/src/lib/ai/safety/outbound-screen.ts index 681b909f0..899f5dd78 100644 --- a/src/lib/ai/safety/outbound-screen.ts +++ b/src/lib/ai/safety/outbound-screen.ts @@ -110,6 +110,13 @@ const DOSE_PATTERNS: Record = { `\\bn[äa]chste\\s+(?:stufe|dosis)\\b[^.?!]{0,30}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`, "i", ), + // German puts the verb last in a subordinate clause ("auf 7,5 mg zu + // erhöhen"), so the verb-first patterns above miss the most natural + // phrasing of the instruction. Match the inverted order too. + new RegExp( + `\\b(?:auf|um)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b[^.?!]{0,30}\\b(?:erhöh\\S*|steiger\\S*|reduzier\\S*|senk\\S*|verringer\\S*|hochsetz\\S*)`, + "i", + ), new RegExp( `\\b(?:erwäg\\w*|probier\\w*|du\\s+solltest|ich\\s+empfehle|ich\\s+schlage\\s+vor)\\b[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`, "i", @@ -179,16 +186,16 @@ const DOSE_PATTERNS: Record = { pl: [ // zwiększ / podnieś / przejdź do|o N unit new RegExp( - `\\b(?:zwi[ęe]ksz\\w*|podnie[śs]\\w*|przejd[źz]\\w*|podwy[żz]sz\\w*)\\s+(?:dawk[ęe]\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`, + `\\b(?:zwi[ęe]ksz\\S*|podnie[śs]\\S*|przejd[źz]\\S*|podwy[żz]sz\\S*)\\s+(?:dawk\\S*\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`, "i", ), // zmniejsz / obniż / zredukuj do|o N unit new RegExp( - `\\b(?:zmniejsz\\w*|obni[żz]\\w*|zreduk\\w*|reduk\\w*)\\s+(?:dawk[ęe]\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`, + `\\b(?:zmniejsz\\S*|obni[żz]\\S*|zreduk\\S*|reduk\\S*)\\s+(?:dawk\\S*\\s+)?(?:do|o)\\s+[\\d.,]+\\s*${DOSE_UNIT}\\b`, "i", ), new RegExp( - `\\b(?:rozwa[żz]\\w*|spr[óo]buj\\w*|powinien|powinna|zalecam|sugeruj[ęe])\\b[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`, + `\\b(?:rozwa[żz]\\S*|spr[óo]buj\\S*|powinien|powinna|zalecam|sugeruj[ęe])[^.?!]{0,40}\\b[\\d.,]+\\s*${DOSE_UNIT}\\b`, "i", ), new RegExp( @@ -217,23 +224,23 @@ const RISK_PATTERNS: Record = { /\b(?:10[- ]jahres|zehn[- ]jahres|lebenszeit)[- ]?(?:risiko)\b/i, ], fr: [ - /\brisque\s+(?:de|d'|est\s+de|d'environ)\s+(?:environ\s+|~)?\d{1,3}\s*%/i, - /\b\d{1,3}\s*%\s+(?:de\s+)?(?:risque|probabilit[ée]|chance)\b/i, + /\brisque\s+(?:est\s+)?(?:de\s+|d['’])?(?:environ\s+|~)?\d{1,3}\s*%/i, + /\b\d{1,3}\s*%\s+(?:de\s+)?(?:risque|probabilit[ée]|chance)/i, /\brisque\s+(?:cardiovasculaire|cardiaque|d'avc|de\s+mortalit[ée])\s+[àa]\s+(?:10|dix)\s+ans\b/i, ], es: [ /\briesgo\s+(?:del?|es\s+del?|de\s+aproximadamente)\s+(?:aproximadamente\s+|~)?\d{1,3}\s*%/i, - /\b\d{1,3}\s*%\s+(?:de\s+)?(?:riesgo|probabilidad)\b/i, + /\b\d{1,3}\s*%\s+(?:de\s+)?(?:riesgo|probabilidad)/i, /\briesgo\s+(?:cardiovascular|card[íi]aco|de\s+ictus|de\s+mortalidad)\s+a\s+(?:10|diez)\s+a[ñn]os\b/i, ], it: [ /\brischio\s+(?:del?|dell'|[èe]\s+del?|di\s+circa)\s+(?:circa\s+|~)?\d{1,3}\s*%/i, - /\b\d{1,3}\s*%\s+(?:di\s+)?(?:rischio|probabilit[àa])\b/i, + /\b\d{1,3}\s*%\s+(?:di\s+)?(?:rischio|probabilit[àa])/i, /\brischio\s+(?:cardiovascolare|cardiaco|di\s+ictus|di\s+mortalit[àa])\s+a\s+(?:10|dieci)\s+anni\b/i, ], pl: [ /\bryzyko\s+(?:wynosi\s+|około\s+|~)?\d{1,3}\s*%/i, - /\b\d{1,3}\s*%\s+(?:ryzyka|prawdopodobie[ńn]stwa)\b/i, + /\b\d{1,3}\s*%\s+(?:ryzyka|prawdopodobie[ńn]stwa)/i, /\bryzyk\w*\s+(?:sercowo[- ]naczyniow\w*|zawału|udaru|zgonu)\s+w\s+(?:ci[ąa]gu\s+)?(?:10|dziesi[ęe]ciu)\s+lat\b/i, ], }; @@ -276,17 +283,17 @@ const CAUSAL_PATTERNS: Record = { /\bcaus(?:e|es|ent|é|ée)\b/i, /\ben\s+raison\s+de\b/i, /\bentra[îi]n\w*/i, - /\ba\s+conduit\s+[àa]\b/i, - /\bd[ûu]\s+[àa]\b/i, + /\ba\s+conduit\s+[àa]/i, + /\bd[ûu]\s+[àa]/i, /\bresponsable\s+de\b/i, - /\bgr[âa]ce\s+[àa]\b/i, + /\bgr[âa]ce\s+[àa]/i, /\bprovoqu\w*/i, ], es: [ /\bporque\b/i, /\bdebido\s+a\b/i, /\ba\s+causa\s+de\b/i, - /\bcaus(?:a|an|ó|ado)\b/i, + /\bcaus(?:a|an|ó|ado)/i, /\bprovoc\w*/i, /\bllev[óo]\s+a\b/i, /\bresponsable\s+de\b/i, @@ -294,7 +301,7 @@ const CAUSAL_PATTERNS: Record = { /\bimpuls\w*\s+(?:por|el|la)\b/i, ], it: [ - /\bperch[ée]\b/i, + /\bperch[ée]/i, /\ba\s+causa\s+di\b/i, /\bcaus(?:a|ano|ato|ata)\b/i, /\bprovoc\w*/i, @@ -305,7 +312,7 @@ const CAUSAL_PATTERNS: Record = { /\bguid\w*\s+da\b/i, ], pl: [ - /\bponiewa[żz]\b/i, + /\bponiewa[żz]/i, /\bz\s+powodu\b/i, /\bpowoduj\w*/i, /\bspowodowa\w*/i, diff --git a/src/lib/documents/__tests__/describe-screen.test.ts b/src/lib/documents/__tests__/describe-screen.test.ts new file mode 100644 index 000000000..8e9cba390 --- /dev/null +++ b/src/lib/documents/__tests__/describe-screen.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { + runDocumentSummary, + transcribeDocument, + documentSummaryBlockedCopy, +} from "@/lib/documents/describe"; +import type { AIProvider } from "@/lib/ai/types"; +import { locales } from "@/lib/i18n/config"; + +function providerReturning(content: string): AIProvider { + return { + type: "local", + generateCompletion: vi.fn().mockResolvedValue({ content }), + } as unknown as AIProvider; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("runDocumentSummary — screened prose", () => { + const VIOLATIONS: Record = { + en: "This report suggests you increase to 10 mg next week.", + de: "Der Bericht empfiehlt, auf 7,5 mg zu erhöhen in der nächsten Woche.", + fr: "Ce rapport indique d'augmenter votre dose à 2,4 mg la semaine prochaine.", + es: "Este informe sugiere aumentar su dosis a 2,4 mg la próxima semana.", + it: "Questo referto suggerisce di aumentare la sua dose a 2,4 mg la prossima settimana.", + pl: "Ten raport sugeruje, aby zwiększyć dawkę do 2,4 mg w przyszłym tygodniu.", + }; + + for (const locale of locales) { + it(`reports the violation instead of the prose in ${locale}`, async () => { + const out = await runDocumentSummary({ + provider: providerReturning(VIOLATIONS[locale]), + providerType: "local", + ocrText: "irrelevant", + locale, + }); + expect(out.blocked).toBe("dose_prescription"); + expect(out.summary).toBe(""); + }); + } + + it("passes a genuinely descriptive summary through untouched", async () => { + const clean = + "A lab report from a clinic, dated last month, listing routine blood values."; + const out = await runDocumentSummary({ + provider: providerReturning(clean), + providerType: "local", + ocrText: "irrelevant", + locale: "en", + }); + expect(out.blocked).toBeNull(); + expect(out.summary).toBe(clean); + }); +}); + +describe("documentSummaryBlockedCopy", () => { + it("carries a native German body and rides English elsewhere", () => { + expect(documentSummaryBlockedCopy("de")).not.toBe( + documentSummaryBlockedCopy("en"), + ); + expect(documentSummaryBlockedCopy("fr")).toBe( + documentSummaryBlockedCopy("en"), + ); + }); +}); + +describe("transcribeDocument — deliberately NOT screened", () => { + /** + * Transcription reproduces the user's OWN document. A discharge letter that + * says "increase to 10 mg" is the prescriber's instruction; screening it + * would delete the user's record from their own view. The screen exists to + * stop the MODEL originating such a claim, not to censor the source. + */ + it("returns a dose instruction from the source document verbatim", async () => { + const letter = + "Plan: increase to 10 mg daily from 1 August. Review in six weeks."; + const out = await transcribeDocument({ + provider: providerReturning(letter), + providerType: "local", + }); + expect(out.text).toBe(letter); + }); + + it("echoes posted OCR text verbatim without a provider round-trip", async () => { + const letter = "Risque cardiovasculaire à 10 ans : 12%."; + const provider = providerReturning("should not be called"); + const out = await transcribeDocument({ + provider, + providerType: "local-ocr", + ocrText: letter, + }); + expect(out.text).toBe(letter); + expect(provider.generateCompletion).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/insights/__tests__/status-summary-screen.test.ts b/src/lib/insights/__tests__/status-summary-screen.test.ts new file mode 100644 index 000000000..d466bf259 --- /dev/null +++ b/src/lib/insights/__tests__/status-summary-screen.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from "vitest"; + +import { finalizeStatusSummary } from "@/lib/insights/status-shared"; +import { locales } from "@/lib/i18n/config"; + +/** + * The status-card chokepoint. Before this existed the only transform between a + * provider and a persisted assessment was whitespace normalisation, so a dose + * imperative on the Weight card persisted as the day's cached assessment while + * the identical sentence in the Coach was caught and replaced. + * + * The WITHHOLD policy itself is asserted by the caller's `ok: false` branch — + * here we pin that the chokepoint reports the violation instead of returning + * text, and that clean prose survives byte-for-byte. + */ + +const DOSE_VIOLATION: Record = { + en: '{"summary":"Consider increasing to 10 mg next week."}', + de: '{"summary":"Erhöhe auf 7,5 mg in der nächsten Woche."}', + fr: '{"summary":"Augmentez votre dose à 2,4 mg la semaine prochaine."}', + es: '{"summary":"Aumente su dosis a 2,4 mg la próxima semana."}', + it: '{"summary":"Aumenti la sua dose a 2,4 mg la prossima settimana."}', + pl: '{"summary":"Proszę zwiększyć dawkę do 2,4 mg w przyszłym tygodniu."}', +}; + +const CAUSAL_VIOLATION: Record = { + en: '{"summary":"Your weight fell because you slept more."}', + de: '{"summary":"Dein Gewicht sank wegen des Schlafs."}', + fr: '{"summary":"Votre poids a baissé parce que vous avez mieux dormi."}', + es: '{"summary":"Su peso bajó porque durmió más."}', + it: '{"summary":"Il suo peso è calato perché ha dormito di più."}', + pl: '{"summary":"Waga spadła, ponieważ spał Pan więcej."}', +}; + +const CLEAN: Record = { + en: '{"summary":"Your weight is averaging 82 kg this week, steady against last month."}', + de: '{"summary":"Dein Gewicht liegt diese Woche im Schnitt bei 82 kg, stabil gegenüber dem Vormonat."}', + fr: '{"summary":"Votre poids est en moyenne de 82 kg cette semaine, stable par rapport au mois dernier."}', + es: '{"summary":"Su peso promedia 82 kg esta semana, estable frente al mes pasado."}', + it: '{"summary":"Il suo peso è in media di 82 kg questa settimana, stabile rispetto al mese scorso."}', + pl: '{"summary":"Waga wynosi średnio 82 kg w tym tygodniu, stabilnie wobec zeszłego miesiąca."}', +}; + +describe("finalizeStatusSummary — dose prescriptions are withheld", () => { + for (const locale of locales) { + it(`reports a violation instead of text in ${locale}`, () => { + const out = finalizeStatusSummary(DOSE_VIOLATION[locale], locale); + expect(out.ok).toBe(false); + if (!out.ok) expect(out.reason).toBe("dose_prescription"); + }); + } +}); + +describe("finalizeStatusSummary — causal claims are withheld", () => { + /** + * Status cards are the `insights` surface GROUND RULE 12 declares, so unlike + * the Coach they DO enforce the no-causation contract. + */ + for (const locale of locales) { + it(`reports a causal claim in ${locale}`, () => { + const out = finalizeStatusSummary(CAUSAL_VIOLATION[locale], locale); + expect(out.ok).toBe(false); + if (!out.ok) expect(out.reason).toBe("causal_claim"); + }); + } +}); + +describe("finalizeStatusSummary — clean assessments pass untouched", () => { + for (const locale of locales) { + it(`returns the parsed summary verbatim in ${locale}`, () => { + const out = finalizeStatusSummary(CLEAN[locale], locale); + expect(out.ok).toBe(true); + if (out.ok) { + expect(out.text).toBe( + JSON.parse(CLEAN[locale]).summary as unknown as string, + ); + } + }); + } + + it("still parses a bare-prose reply that ignores the JSON envelope", () => { + const out = finalizeStatusSummary( + "Your resting pulse is averaging 58 bpm, in line with your baseline.", + "en", + ); + expect(out.ok).toBe(true); + if (out.ok) expect(out.text).toContain("58 bpm"); + }); + + it("treats an empty completion as empty text, not a violation", () => { + const out = finalizeStatusSummary("", "en"); + expect(out.ok).toBe(true); + }); +}); diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index 563f78b67..ecbb94c6c 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -33,6 +33,7 @@ import { buildBriefingGroundingCorrection, } from "@/lib/ai/briefing-grounding"; import { screenInsightPayloadProse } from "@/lib/ai/safety/insight-payload-screen"; +import { computeCitationCoverage } from "@/lib/ai/citation-coverage"; import { applyInsightsExcludeFilter } from "@/lib/insights/exclude-filter"; import { getCachedFeatures } from "@/lib/insights/feature-cache"; import { @@ -1039,6 +1040,32 @@ export async function generateComprehensiveInsight( } } + // Citation coverage. This annotation is what the admin AI-quality dashboard + // slices to track how many normative recommendations carry a curated + // medical-reference id. It used to hang off `generateInsight()`, which has no + // production caller, so the dashboard was reading a metric nothing emitted. + // Observational only — an uncited normative rec is a logged warning, never a + // reason to fail the generation. + if (insights && typeof insights === "object") { + const payload = insights as Record; + if (Array.isArray(payload.recommendations)) { + const coverage = computeCitationCoverage({ + recommendations: payload.recommendations, + }); + annotate({ + action: { name: "insights.generate.citation_coverage" }, + meta: { + locale, + totalRecommendations: coverage.totalRecommendations, + normativeRecommendations: coverage.normativeRecommendations, + citedNormativeRecommendations: coverage.citedNormativeRecommendations, + uncitedNormativeCount: + coverage.uncitedNormativeRecommendationIds.length, + }, + }); + } + } + // Outbound safety screen over the WHOLE prose surface. // // SURFACE POLICY — WITHHOLD. This payload is generated in the background and