fix(insights): add a grounding gate + retry to the period narrative

The period narrative was written straight to storage with no
post-generation check and no corrective retry, unlike Comprehensive
Insights. A populated context could still yield causal phrasing or a
number absent from the context block.

Add a validator that flags causal language (EN/DE) and any number not in
the structured context; on a trip, retry once with a corrective suffix,
then write nothing (the last good row stays) rather than persist an
ungrounded or causal story.
This commit is contained in:
Marc-André Bombeck committed 2026-06-20 16:58:33 +02:00
1 parent 869f1d9c11
commit 080ce73475
3 files changed
+383 -1

No files matched your search

@@ -0,0 +1,117 @@
import { describe, it, expect } from "vitest";
import {
validateNarrativeText,
buildNarrativeCorrection,
} from "@/lib/insights/narrative/narrative-grounding";
import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative";
function ctx(
partial: Partial<PeriodNarrativeContext> = {},
): PeriodNarrativeContext {
return {
status: "ready",
period: "week",
metricDeltas: [
{
type: "WEIGHT" as PeriodNarrativeContext["metricDeltas"][number]["type"],
unit: "kg",
current: 82,
prior: 84,
delta: -2,
deltaPercent: -2.4,
currentDays: 6,
priorDays: 7,
},
],
bandTransitions: [],
drivers: [],
coincidentFlags: [],
pairsTested: 5,
fdrQ: 0.1,
provenance: {
metrics: ["weight"],
window: { from: "2026-06-01", to: "2026-06-07" },
computedAt: "2026-06-07T00:00:00.000Z",
},
...partial,
};
}
describe("validateNarrativeText — causal language", () => {
it("flags 'because'", () => {
const out = validateNarrativeText(
"Your weight fell because you slept more.",
ctx(),
);
expect(out.some((f) => f.reason === "causal_language")).toBe(true);
});
it("flags German 'wegen'", () => {
const out = validateNarrativeText(
"Dein Gewicht sank wegen des Schlafs.",
ctx(),
);
expect(out.some((f) => f.reason === "causal_language")).toBe(true);
});
it("flags 'culprit'", () => {
const out = validateNarrativeText(
"Stress is the likely culprit here.",
ctx(),
);
expect(out.some((f) => f.reason === "causal_language")).toBe(true);
});
it("allows descriptive 'associated with' / 'moved with'", () => {
const out = validateNarrativeText(
"Your weight of 82 kg moved with your sleep this week, associated with a 2 kg drop.",
ctx(),
);
expect(out).toEqual([]);
});
});
describe("validateNarrativeText — number grounding", () => {
it("passes numbers present in the context", () => {
const out = validateNarrativeText(
"Your weight is 82 kg, down 2 kg (about -2.4%) from the prior week.",
ctx(),
);
expect(out).toEqual([]);
});
it("flags a number not in the context", () => {
const out = validateNarrativeText(
"Your weight dropped 9 kg this week.",
ctx(),
);
expect(
out.some((f) => f.reason === "ungrounded_number" && f.source === "9"),
).toBe(true);
});
it("does not flag structural integers (7-day window)", () => {
const out = validateNarrativeText(
"Over the last 7 days your weight held at 82 kg.",
ctx(),
);
expect(out).toEqual([]);
});
});
describe("buildNarrativeCorrection", () => {
it("names causal + ungrounded findings (en)", () => {
const msg = buildNarrativeCorrection(
[
{ reason: "causal_language", source: "because" },
{ reason: "ungrounded_number", source: "9" },
],
"en",
);
expect(msg.toLowerCase()).toContain("causal");
expect(msg).toContain("9");
});
it("produces German copy", () => {
const msg = buildNarrativeCorrection(
[{ reason: "causal_language", source: "weil" }],
"de",
);
expect(msg).toContain("URSÄCHLICHE");
});
});
@@ -0,0 +1,210 @@
/**
* v1.18.10 (HIGH-3) — period-narrative grounding validator.
*
* The period narrative was plain text written straight to storage after the
* provider call, with NO post-generation check and NO corrective retry —
* unlike Comprehensive Insights, which rejects + retries + 422s on a grounding
* miss. The system prompt bans causal language and forbids stating a number
* not in the context, but those are instructions, not enforcement: a populated
* (non-`insufficient`) context could still yield "stress is the likely
* culprit" or a figure absent from the context block, and nothing rejected it.
*
* This module is the missing gate. Given the generated text and the structured
* `PeriodNarrativeContext`, it flags:
* 1. causal phrasing ("because", "caused", "due to", "weil", "wegen", …);
* 2. any number the narrative states that does not trace to a context figure
* within a rounding tolerance.
* The generator runs it after the completion, retries ONCE with a corrective
* suffix on a trip, and falls back to NO narrative (writes nothing) rather than
* persisting an ungrounded story.
*
* Pure + side-effect-free, so it unit-tests in isolation.
*/
import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative";
/** Absolute + relative number-match tolerance (rounding slack). */
const ABS_TOLERANCE = 0.15;
const REL_TOLERANCE = 0.02;
/**
* Causal phrases that violate the descriptive-never-causal contract. Word-
* boundary anchored, case-insensitive. The list is the EN + DE causal
* vocabulary; "associated with" / "moved with" / "assoziiert" stay PERMITTED
* (they are the descriptive framing the contract requires).
*/
const CAUSAL_PATTERNS: readonly RegExp[] = [
/\bbecause\s+of\b/i,
/\bbecause\b/i,
/\bcaused?\s+by\b/i,
/\bcaus(?:e|es|ed|ing)\b/i,
/\bdue\s+to\b/i,
/\bled\s+to\b/i,
/\bresulted\s+in\b/i,
/\bresult\s+of\b/i,
/\bculprit\b/i,
/\bdriven\s+by\b/i,
/\bthanks\s+to\b/i,
/\bowing\s+to\b/i,
// DE
/\bweil\b/i,
/\bwegen\b/i,
/\bverursach\w*/i,
/\baufgrund\b/i,
/\bführt[e]?\s+zu\b/i,
/\bdurch\s+\w+\s+(?:verursacht|ausgelöst)\b/i,
/\bschuld\b/i,
/\bauslöser\b/i,
];
export type NarrativeGroundingReason = "causal_language" | "ungrounded_number";
export interface NarrativeGroundingFinding {
reason: NarrativeGroundingReason;
/** The matched token (truncated). */
source: string;
}
/** Parse every standalone number out of free text (`.`/`,` decimals). */
function extractNumbers(text: string): Array<{ value: number; raw: string }> {
const out: Array<{ value: number; raw: string }> = [];
const re = /[+-]?\d+(?:[.,]\d+)?/g;
let match: RegExpExecArray | null;
while ((match = re.exec(text)) !== null) {
const raw = match[0];
const value = Number.parseFloat(raw.replace(",", "."));
if (Number.isFinite(value)) out.push({ value, raw });
}
return out;
}
/** Every authoritative magnitude the context exposes (signed + absolute). */
function authoritativeValues(context: PeriodNarrativeContext): number[] {
const values: number[] = [];
const push = (n: number | null | undefined) => {
if (typeof n === "number" && Number.isFinite(n)) {
values.push(n);
values.push(Math.abs(n));
}
};
for (const d of context.metricDeltas) {
push(d.current);
push(d.prior);
push(d.delta);
push(d.deltaPercent);
push(d.currentDays);
push(d.priorDays);
}
for (const b of context.bandTransitions) {
push(b.center);
push(b.bandLow);
push(b.bandHigh);
push(b.baselineDays);
}
for (const dr of context.drivers) {
push(dr.r);
push(dr.qValue);
push(dr.n);
}
push(context.pairsTested);
push(context.fdrQ);
return values;
}
function isGrounded(value: number, authoritative: readonly number[]): boolean {
for (const a of authoritative) {
const tol = Math.max(ABS_TOLERANCE, Math.abs(a) * REL_TOLERANCE);
if (Math.abs(value - a) <= tol) return true;
}
return false;
}
/**
* Structural integers the narrative may state without a context match — small
* ordinals and the literal period lengths the prompt references (7 / 30 days).
*/
const STRUCTURAL_INTEGERS = new Set([7, 30]);
function isStructural(value: number, raw: string): boolean {
if (raw.includes(".") || raw.includes(",")) return false;
if (!Number.isInteger(value)) return false;
const abs = Math.abs(value);
if (abs <= 3) return true;
return STRUCTURAL_INTEGERS.has(abs);
}
/**
* Validate a generated narrative against its context. Returns every grounding
* violation found; an empty array means the narrative is grounded and
* causally-clean.
*/
export function validateNarrativeText(
text: string,
context: PeriodNarrativeContext,
): NarrativeGroundingFinding[] {
const findings: NarrativeGroundingFinding[] = [];
if (typeof text !== "string" || text.trim().length === 0) return findings;
for (const pattern of CAUSAL_PATTERNS) {
const m = pattern.exec(text);
if (m) {
findings.push({ reason: "causal_language", source: m[0].slice(0, 32) });
break; // one causal flag is enough to trigger the corrective retry
}
}
const authoritative = authoritativeValues(context);
for (const { value, raw } of extractNumbers(text)) {
if (isStructural(value, raw)) continue;
if (isGrounded(value, authoritative)) continue;
findings.push({ reason: "ungrounded_number", source: raw.slice(0, 32) });
}
return findings;
}
/**
* Corrective suffix appended to the user prompt on the single retry, naming the
* violations so the model can self-correct — mirroring the comprehensive
* insight's `buildRetryCorrectionMessage` posture.
*/
export function buildNarrativeCorrection(
findings: readonly NarrativeGroundingFinding[],
locale: "de" | "en",
): string {
const causal = findings.some((f) => f.reason === "causal_language");
const numbers = findings
.filter((f) => f.reason === "ungrounded_number")
.slice(0, 6)
.map((f) => f.source)
.join(", ");
if (locale === "de") {
const parts = [
"Deine vorige Zusammenfassung hat die festen Regeln verletzt.",
];
if (causal) {
parts.push(
"Sie enthielt URSÄCHLICHE Formulierungen. Schreibe ausschließlich BESCHREIBEND ('X bewegte sich mit Y', 'X war mit Y assoziiert') — nie 'weil', 'wegen', 'verursacht', 'führte zu'.",
);
}
if (numbers) {
parts.push(
`Sie nannte Zahlen, die NICHT im Kontext stehen: ${numbers}. Nenne nur Zahlen, die wörtlich im KONTEXT vorkommen, oder lass die Zahl weg.`,
);
}
parts.push("Schreibe die Zusammenfassung erneut (2–4 kurze Sätze).");
return `\n\n${parts.join(" ")}`;
}
const parts = ["Your previous summary broke the hard rules."];
if (causal) {
parts.push(
"It used CAUSAL wording. Write strictly DESCRIPTIVELY ('X moved with Y', 'X was associated with Y') — never 'because', 'due to', 'caused', 'led to'.",
);
}
if (numbers) {
parts.push(
`It stated numbers NOT in the context: ${numbers}. State only numbers that appear verbatim in the CONTEXT, or drop the number.`,
);
}
parts.push("Re-write the summary (2-4 short sentences).");
return `\n\n${parts.join(" ")}`;
}
@@ -41,6 +41,10 @@ import {
DETERMINISTIC_PROVIDER_TYPE,
} from "@/lib/insights/narrative/period-narrative-deterministic";
import { composeSharedContracts } from "@/lib/ai/prompts/shared-contracts";
import {
validateNarrativeText,
buildNarrativeCorrection,
} from "@/lib/insights/narrative/narrative-grounding";
/**
* Stable identifier for the narrative prompt revision. Bumped whenever the
@@ -350,10 +354,61 @@ export async function generatePeriodNarrative(
return { status: "generated", providerType: DETERMINISTIC_PROVIDER_TYPE };
}
const text = completion.content.trim();
let text = completion.content.trim();
if (text.length === 0) {
return { status: "failed", reason: "empty" };
}
// v1.18.10 (HIGH-3) — grounding gate on the AI prose, matching the rejecting
// gate Comprehensive Insights already has. Flag causal phrasing and any
// number absent from the context; on a trip, retry ONCE with a corrective
// suffix, then fall back to writing NOTHING (last good row stays) rather than
// persisting an ungrounded or causal story.
let findings = validateNarrativeText(text, context);
if (findings.length > 0) {
annotate({
action: { name: "insights.narrative.grounding_retry" },
meta: {
period,
locale,
causal: findings.some((f) => f.reason === "causal_language"),
ungroundedNumbers: findings.filter(
(f) => f.reason === "ungrounded_number",
).length,
},
});
const retry = await runCompletion({
userId,
cacheAction: `insights.narrative.${period}.${locale}`,
consentSurface: "insights",
systemPrompt: locale === "de" ? SYSTEM_PROMPT_DE : SYSTEM_PROMPT_EN,
userPrompt:
buildNarrativeUserPrompt(context, locale) +
buildNarrativeCorrection(findings, locale),
temperature: AI_BUDGETS.narrative.temperature,
maxTokens: AI_BUDGETS.narrative.maxTokens,
seed: REFERENCE_AI_SEED,
responseFormat: "text",
});
if (retry.kind === "ok") {
const retryText = retry.content.trim();
const retryFindings = validateNarrativeText(retryText, context);
if (retryText.length > 0 && retryFindings.length === 0) {
text = retryText;
findings = [];
} else {
findings = retryFindings;
}
}
if (findings.length > 0) {
annotate({
action: { name: "insights.narrative.grounding_rejected" },
meta: { period, locale },
});
return { status: "failed", reason: "ungrounded" };
}
}
await persist(text, completion.providerType);
return { status: "generated", providerType: completion.providerType };
}