From 080ce73475f3b9e9a86853f1a94fbd9c65baa94a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 20 Jun 2026 16:58:33 +0200 Subject: [PATCH] fix(insights): add a grounding gate + retry to the period narrative The period narrative was written straight to storage with no post-generation check and no corrective retry, unlike Comprehensive Insights. A populated context could still yield causal phrasing or a number absent from the context block. Add a validator that flags causal language (EN/DE) and any number not in the structured context; on a trip, retry once with a corrective suffix, then write nothing (the last good row stays) rather than persist an ungrounded or causal story. --- .../__tests__/narrative-grounding.test.ts | 117 ++++++++++ .../insights/narrative/narrative-grounding.ts | 210 ++++++++++++++++++ .../narrative/period-narrative-generate.ts | 57 ++++- 3 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 src/lib/insights/narrative/__tests__/narrative-grounding.test.ts create mode 100644 src/lib/insights/narrative/narrative-grounding.ts diff --git a/src/lib/insights/narrative/__tests__/narrative-grounding.test.ts b/src/lib/insights/narrative/__tests__/narrative-grounding.test.ts new file mode 100644 index 000000000..c1528fc4a --- /dev/null +++ b/src/lib/insights/narrative/__tests__/narrative-grounding.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect } from "vitest"; + +import { + validateNarrativeText, + buildNarrativeCorrection, +} from "@/lib/insights/narrative/narrative-grounding"; +import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative"; + +function ctx( + partial: Partial = {}, +): PeriodNarrativeContext { + return { + status: "ready", + period: "week", + metricDeltas: [ + { + type: "WEIGHT" as PeriodNarrativeContext["metricDeltas"][number]["type"], + unit: "kg", + current: 82, + prior: 84, + delta: -2, + deltaPercent: -2.4, + currentDays: 6, + priorDays: 7, + }, + ], + bandTransitions: [], + drivers: [], + coincidentFlags: [], + pairsTested: 5, + fdrQ: 0.1, + provenance: { + metrics: ["weight"], + window: { from: "2026-06-01", to: "2026-06-07" }, + computedAt: "2026-06-07T00:00:00.000Z", + }, + ...partial, + }; +} + +describe("validateNarrativeText — causal language", () => { + it("flags 'because'", () => { + const out = validateNarrativeText( + "Your weight fell because you slept more.", + ctx(), + ); + expect(out.some((f) => f.reason === "causal_language")).toBe(true); + }); + it("flags German 'wegen'", () => { + const out = validateNarrativeText( + "Dein Gewicht sank wegen des Schlafs.", + ctx(), + ); + expect(out.some((f) => f.reason === "causal_language")).toBe(true); + }); + it("flags 'culprit'", () => { + const out = validateNarrativeText( + "Stress is the likely culprit here.", + ctx(), + ); + expect(out.some((f) => f.reason === "causal_language")).toBe(true); + }); + it("allows descriptive 'associated with' / 'moved with'", () => { + const out = validateNarrativeText( + "Your weight of 82 kg moved with your sleep this week, associated with a 2 kg drop.", + ctx(), + ); + expect(out).toEqual([]); + }); +}); + +describe("validateNarrativeText — number grounding", () => { + it("passes numbers present in the context", () => { + const out = validateNarrativeText( + "Your weight is 82 kg, down 2 kg (about -2.4%) from the prior week.", + ctx(), + ); + expect(out).toEqual([]); + }); + it("flags a number not in the context", () => { + const out = validateNarrativeText( + "Your weight dropped 9 kg this week.", + ctx(), + ); + expect( + out.some((f) => f.reason === "ungrounded_number" && f.source === "9"), + ).toBe(true); + }); + it("does not flag structural integers (7-day window)", () => { + const out = validateNarrativeText( + "Over the last 7 days your weight held at 82 kg.", + ctx(), + ); + expect(out).toEqual([]); + }); +}); + +describe("buildNarrativeCorrection", () => { + it("names causal + ungrounded findings (en)", () => { + const msg = buildNarrativeCorrection( + [ + { reason: "causal_language", source: "because" }, + { reason: "ungrounded_number", source: "9" }, + ], + "en", + ); + expect(msg.toLowerCase()).toContain("causal"); + expect(msg).toContain("9"); + }); + it("produces German copy", () => { + const msg = buildNarrativeCorrection( + [{ reason: "causal_language", source: "weil" }], + "de", + ); + expect(msg).toContain("URSÄCHLICHE"); + }); +}); diff --git a/src/lib/insights/narrative/narrative-grounding.ts b/src/lib/insights/narrative/narrative-grounding.ts new file mode 100644 index 000000000..0422c0ef3 --- /dev/null +++ b/src/lib/insights/narrative/narrative-grounding.ts @@ -0,0 +1,210 @@ +/** + * v1.18.10 (HIGH-3) — period-narrative grounding validator. + * + * The period narrative was plain text written straight to storage after the + * provider call, with NO post-generation check and NO corrective retry — + * unlike Comprehensive Insights, which rejects + retries + 422s on a grounding + * miss. The system prompt bans causal language and forbids stating a number + * not in the context, but those are instructions, not enforcement: a populated + * (non-`insufficient`) context could still yield "stress is the likely + * culprit" or a figure absent from the context block, and nothing rejected it. + * + * This module is the missing gate. Given the generated text and the structured + * `PeriodNarrativeContext`, it flags: + * 1. causal phrasing ("because", "caused", "due to", "weil", "wegen", …); + * 2. any number the narrative states that does not trace to a context figure + * within a rounding tolerance. + * The generator runs it after the completion, retries ONCE with a corrective + * suffix on a trip, and falls back to NO narrative (writes nothing) rather than + * persisting an ungrounded story. + * + * Pure + side-effect-free, so it unit-tests in isolation. + */ +import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative"; + +/** Absolute + relative number-match tolerance (rounding slack). */ +const ABS_TOLERANCE = 0.15; +const REL_TOLERANCE = 0.02; + +/** + * Causal phrases that violate the descriptive-never-causal contract. Word- + * boundary anchored, case-insensitive. The list is the EN + DE causal + * vocabulary; "associated with" / "moved with" / "assoziiert" stay PERMITTED + * (they are the descriptive framing the contract requires). + */ +const CAUSAL_PATTERNS: readonly RegExp[] = [ + /\bbecause\s+of\b/i, + /\bbecause\b/i, + /\bcaused?\s+by\b/i, + /\bcaus(?:e|es|ed|ing)\b/i, + /\bdue\s+to\b/i, + /\bled\s+to\b/i, + /\bresulted\s+in\b/i, + /\bresult\s+of\b/i, + /\bculprit\b/i, + /\bdriven\s+by\b/i, + /\bthanks\s+to\b/i, + /\bowing\s+to\b/i, + // DE + /\bweil\b/i, + /\bwegen\b/i, + /\bverursach\w*/i, + /\baufgrund\b/i, + /\bführt[e]?\s+zu\b/i, + /\bdurch\s+\w+\s+(?:verursacht|ausgelöst)\b/i, + /\bschuld\b/i, + /\bauslöser\b/i, +]; + +export type NarrativeGroundingReason = "causal_language" | "ungrounded_number"; + +export interface NarrativeGroundingFinding { + reason: NarrativeGroundingReason; + /** The matched token (truncated). */ + source: string; +} + +/** Parse every standalone number out of free text (`.`/`,` decimals). */ +function extractNumbers(text: string): Array<{ value: number; raw: string }> { + const out: Array<{ value: number; raw: string }> = []; + const re = /[+-]?\d+(?:[.,]\d+)?/g; + let match: RegExpExecArray | null; + while ((match = re.exec(text)) !== null) { + const raw = match[0]; + const value = Number.parseFloat(raw.replace(",", ".")); + if (Number.isFinite(value)) out.push({ value, raw }); + } + return out; +} + +/** Every authoritative magnitude the context exposes (signed + absolute). */ +function authoritativeValues(context: PeriodNarrativeContext): number[] { + const values: number[] = []; + const push = (n: number | null | undefined) => { + if (typeof n === "number" && Number.isFinite(n)) { + values.push(n); + values.push(Math.abs(n)); + } + }; + for (const d of context.metricDeltas) { + push(d.current); + push(d.prior); + push(d.delta); + push(d.deltaPercent); + push(d.currentDays); + push(d.priorDays); + } + for (const b of context.bandTransitions) { + push(b.center); + push(b.bandLow); + push(b.bandHigh); + push(b.baselineDays); + } + for (const dr of context.drivers) { + push(dr.r); + push(dr.qValue); + push(dr.n); + } + push(context.pairsTested); + push(context.fdrQ); + return values; +} + +function isGrounded(value: number, authoritative: readonly number[]): boolean { + for (const a of authoritative) { + const tol = Math.max(ABS_TOLERANCE, Math.abs(a) * REL_TOLERANCE); + if (Math.abs(value - a) <= tol) return true; + } + return false; +} + +/** + * Structural integers the narrative may state without a context match — small + * ordinals and the literal period lengths the prompt references (7 / 30 days). + */ +const STRUCTURAL_INTEGERS = new Set([7, 30]); + +function isStructural(value: number, raw: string): boolean { + if (raw.includes(".") || raw.includes(",")) return false; + if (!Number.isInteger(value)) return false; + const abs = Math.abs(value); + if (abs <= 3) return true; + return STRUCTURAL_INTEGERS.has(abs); +} + +/** + * Validate a generated narrative against its context. Returns every grounding + * violation found; an empty array means the narrative is grounded and + * causally-clean. + */ +export function validateNarrativeText( + text: string, + context: PeriodNarrativeContext, +): NarrativeGroundingFinding[] { + const findings: NarrativeGroundingFinding[] = []; + if (typeof text !== "string" || text.trim().length === 0) return findings; + + for (const pattern of CAUSAL_PATTERNS) { + const m = pattern.exec(text); + if (m) { + findings.push({ reason: "causal_language", source: m[0].slice(0, 32) }); + break; // one causal flag is enough to trigger the corrective retry + } + } + + const authoritative = authoritativeValues(context); + for (const { value, raw } of extractNumbers(text)) { + if (isStructural(value, raw)) continue; + if (isGrounded(value, authoritative)) continue; + findings.push({ reason: "ungrounded_number", source: raw.slice(0, 32) }); + } + + return findings; +} + +/** + * Corrective suffix appended to the user prompt on the single retry, naming the + * violations so the model can self-correct — mirroring the comprehensive + * insight's `buildRetryCorrectionMessage` posture. + */ +export function buildNarrativeCorrection( + findings: readonly NarrativeGroundingFinding[], + locale: "de" | "en", +): string { + const causal = findings.some((f) => f.reason === "causal_language"); + const numbers = findings + .filter((f) => f.reason === "ungrounded_number") + .slice(0, 6) + .map((f) => f.source) + .join(", "); + if (locale === "de") { + const parts = [ + "Deine vorige Zusammenfassung hat die festen Regeln verletzt.", + ]; + if (causal) { + parts.push( + "Sie enthielt URSÄCHLICHE Formulierungen. Schreibe ausschließlich BESCHREIBEND ('X bewegte sich mit Y', 'X war mit Y assoziiert') — nie 'weil', 'wegen', 'verursacht', 'führte zu'.", + ); + } + if (numbers) { + parts.push( + `Sie nannte Zahlen, die NICHT im Kontext stehen: ${numbers}. Nenne nur Zahlen, die wörtlich im KONTEXT vorkommen, oder lass die Zahl weg.`, + ); + } + parts.push("Schreibe die Zusammenfassung erneut (2–4 kurze Sätze)."); + return `\n\n${parts.join(" ")}`; + } + const parts = ["Your previous summary broke the hard rules."]; + if (causal) { + parts.push( + "It used CAUSAL wording. Write strictly DESCRIPTIVELY ('X moved with Y', 'X was associated with Y') — never 'because', 'due to', 'caused', 'led to'.", + ); + } + if (numbers) { + parts.push( + `It stated numbers NOT in the context: ${numbers}. State only numbers that appear verbatim in the CONTEXT, or drop the number.`, + ); + } + parts.push("Re-write the summary (2-4 short sentences)."); + return `\n\n${parts.join(" ")}`; +} diff --git a/src/lib/insights/narrative/period-narrative-generate.ts b/src/lib/insights/narrative/period-narrative-generate.ts index 9e1efaa8e..b597a5511 100644 --- a/src/lib/insights/narrative/period-narrative-generate.ts +++ b/src/lib/insights/narrative/period-narrative-generate.ts @@ -41,6 +41,10 @@ import { DETERMINISTIC_PROVIDER_TYPE, } from "@/lib/insights/narrative/period-narrative-deterministic"; import { composeSharedContracts } from "@/lib/ai/prompts/shared-contracts"; +import { + validateNarrativeText, + buildNarrativeCorrection, +} from "@/lib/insights/narrative/narrative-grounding"; /** * Stable identifier for the narrative prompt revision. Bumped whenever the @@ -350,10 +354,61 @@ export async function generatePeriodNarrative( return { status: "generated", providerType: DETERMINISTIC_PROVIDER_TYPE }; } - const text = completion.content.trim(); + let text = completion.content.trim(); if (text.length === 0) { return { status: "failed", reason: "empty" }; } + + // v1.18.10 (HIGH-3) — grounding gate on the AI prose, matching the rejecting + // gate Comprehensive Insights already has. Flag causal phrasing and any + // number absent from the context; on a trip, retry ONCE with a corrective + // suffix, then fall back to writing NOTHING (last good row stays) rather than + // persisting an ungrounded or causal story. + let findings = validateNarrativeText(text, context); + if (findings.length > 0) { + annotate({ + action: { name: "insights.narrative.grounding_retry" }, + meta: { + period, + locale, + causal: findings.some((f) => f.reason === "causal_language"), + ungroundedNumbers: findings.filter( + (f) => f.reason === "ungrounded_number", + ).length, + }, + }); + const retry = await runCompletion({ + userId, + cacheAction: `insights.narrative.${period}.${locale}`, + consentSurface: "insights", + systemPrompt: locale === "de" ? SYSTEM_PROMPT_DE : SYSTEM_PROMPT_EN, + userPrompt: + buildNarrativeUserPrompt(context, locale) + + buildNarrativeCorrection(findings, locale), + temperature: AI_BUDGETS.narrative.temperature, + maxTokens: AI_BUDGETS.narrative.maxTokens, + seed: REFERENCE_AI_SEED, + responseFormat: "text", + }); + if (retry.kind === "ok") { + const retryText = retry.content.trim(); + const retryFindings = validateNarrativeText(retryText, context); + if (retryText.length > 0 && retryFindings.length === 0) { + text = retryText; + findings = []; + } else { + findings = retryFindings; + } + } + if (findings.length > 0) { + annotate({ + action: { name: "insights.narrative.grounding_rejected" }, + meta: { period, locale }, + }); + return { status: "failed", reason: "ungrounded" }; + } + } + await persist(text, completion.providerType); return { status: "generated", providerType: completion.providerType }; }