mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 21:32:18 +00:00
fix(insights): add a grounding gate + retry to the period narrative
The period narrative was written straight to storage with no post-generation check and no corrective retry, unlike Comprehensive Insights. A populated context could still yield causal phrasing or a number absent from the context block. Add a validator that flags causal language (EN/DE) and any number not in the structured context; on a trip, retry once with a corrective suffix, then write nothing (the last good row stays) rather than persist an ungrounded or causal story.
This commit is contained in:
1 parent
869f1d9c11
commit
080ce73475
3 files changed
+383
-1
No files matched your search
@@ -0,0 +1,117 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
validateNarrativeText,
|
||||
buildNarrativeCorrection,
|
||||
} from "@/lib/insights/narrative/narrative-grounding";
|
||||
import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative";
|
||||
|
||||
function ctx(
|
||||
partial: Partial<PeriodNarrativeContext> = {},
|
||||
): PeriodNarrativeContext {
|
||||
return {
|
||||
status: "ready",
|
||||
period: "week",
|
||||
metricDeltas: [
|
||||
{
|
||||
type: "WEIGHT" as PeriodNarrativeContext["metricDeltas"][number]["type"],
|
||||
unit: "kg",
|
||||
current: 82,
|
||||
prior: 84,
|
||||
delta: -2,
|
||||
deltaPercent: -2.4,
|
||||
currentDays: 6,
|
||||
priorDays: 7,
|
||||
},
|
||||
],
|
||||
bandTransitions: [],
|
||||
drivers: [],
|
||||
coincidentFlags: [],
|
||||
pairsTested: 5,
|
||||
fdrQ: 0.1,
|
||||
provenance: {
|
||||
metrics: ["weight"],
|
||||
window: { from: "2026-06-01", to: "2026-06-07" },
|
||||
computedAt: "2026-06-07T00:00:00.000Z",
|
||||
},
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe("validateNarrativeText — causal language", () => {
|
||||
it("flags 'because'", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Your weight fell because you slept more.",
|
||||
ctx(),
|
||||
);
|
||||
expect(out.some((f) => f.reason === "causal_language")).toBe(true);
|
||||
});
|
||||
it("flags German 'wegen'", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Dein Gewicht sank wegen des Schlafs.",
|
||||
ctx(),
|
||||
);
|
||||
expect(out.some((f) => f.reason === "causal_language")).toBe(true);
|
||||
});
|
||||
it("flags 'culprit'", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Stress is the likely culprit here.",
|
||||
ctx(),
|
||||
);
|
||||
expect(out.some((f) => f.reason === "causal_language")).toBe(true);
|
||||
});
|
||||
it("allows descriptive 'associated with' / 'moved with'", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Your weight of 82 kg moved with your sleep this week, associated with a 2 kg drop.",
|
||||
ctx(),
|
||||
);
|
||||
expect(out).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateNarrativeText — number grounding", () => {
|
||||
it("passes numbers present in the context", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Your weight is 82 kg, down 2 kg (about -2.4%) from the prior week.",
|
||||
ctx(),
|
||||
);
|
||||
expect(out).toEqual([]);
|
||||
});
|
||||
it("flags a number not in the context", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Your weight dropped 9 kg this week.",
|
||||
ctx(),
|
||||
);
|
||||
expect(
|
||||
out.some((f) => f.reason === "ungrounded_number" && f.source === "9"),
|
||||
).toBe(true);
|
||||
});
|
||||
it("does not flag structural integers (7-day window)", () => {
|
||||
const out = validateNarrativeText(
|
||||
"Over the last 7 days your weight held at 82 kg.",
|
||||
ctx(),
|
||||
);
|
||||
expect(out).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildNarrativeCorrection", () => {
|
||||
it("names causal + ungrounded findings (en)", () => {
|
||||
const msg = buildNarrativeCorrection(
|
||||
[
|
||||
{ reason: "causal_language", source: "because" },
|
||||
{ reason: "ungrounded_number", source: "9" },
|
||||
],
|
||||
"en",
|
||||
);
|
||||
expect(msg.toLowerCase()).toContain("causal");
|
||||
expect(msg).toContain("9");
|
||||
});
|
||||
it("produces German copy", () => {
|
||||
const msg = buildNarrativeCorrection(
|
||||
[{ reason: "causal_language", source: "weil" }],
|
||||
"de",
|
||||
);
|
||||
expect(msg).toContain("URSÄCHLICHE");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* v1.18.10 (HIGH-3) — period-narrative grounding validator.
|
||||
*
|
||||
* The period narrative was plain text written straight to storage after the
|
||||
* provider call, with NO post-generation check and NO corrective retry —
|
||||
* unlike Comprehensive Insights, which rejects + retries + 422s on a grounding
|
||||
* miss. The system prompt bans causal language and forbids stating a number
|
||||
* not in the context, but those are instructions, not enforcement: a populated
|
||||
* (non-`insufficient`) context could still yield "stress is the likely
|
||||
* culprit" or a figure absent from the context block, and nothing rejected it.
|
||||
*
|
||||
* This module is the missing gate. Given the generated text and the structured
|
||||
* `PeriodNarrativeContext`, it flags:
|
||||
* 1. causal phrasing ("because", "caused", "due to", "weil", "wegen", …);
|
||||
* 2. any number the narrative states that does not trace to a context figure
|
||||
* within a rounding tolerance.
|
||||
* The generator runs it after the completion, retries ONCE with a corrective
|
||||
* suffix on a trip, and falls back to NO narrative (writes nothing) rather than
|
||||
* persisting an ungrounded story.
|
||||
*
|
||||
* Pure + side-effect-free, so it unit-tests in isolation.
|
||||
*/
|
||||
import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative";
|
||||
|
||||
/** Absolute + relative number-match tolerance (rounding slack). */
|
||||
const ABS_TOLERANCE = 0.15;
|
||||
const REL_TOLERANCE = 0.02;
|
||||
|
||||
/**
|
||||
* Causal phrases that violate the descriptive-never-causal contract. Word-
|
||||
* boundary anchored, case-insensitive. The list is the EN + DE causal
|
||||
* vocabulary; "associated with" / "moved with" / "assoziiert" stay PERMITTED
|
||||
* (they are the descriptive framing the contract requires).
|
||||
*/
|
||||
const CAUSAL_PATTERNS: readonly RegExp[] = [
|
||||
/\bbecause\s+of\b/i,
|
||||
/\bbecause\b/i,
|
||||
/\bcaused?\s+by\b/i,
|
||||
/\bcaus(?:e|es|ed|ing)\b/i,
|
||||
/\bdue\s+to\b/i,
|
||||
/\bled\s+to\b/i,
|
||||
/\bresulted\s+in\b/i,
|
||||
/\bresult\s+of\b/i,
|
||||
/\bculprit\b/i,
|
||||
/\bdriven\s+by\b/i,
|
||||
/\bthanks\s+to\b/i,
|
||||
/\bowing\s+to\b/i,
|
||||
// DE
|
||||
/\bweil\b/i,
|
||||
/\bwegen\b/i,
|
||||
/\bverursach\w*/i,
|
||||
/\baufgrund\b/i,
|
||||
/\bführt[e]?\s+zu\b/i,
|
||||
/\bdurch\s+\w+\s+(?:verursacht|ausgelöst)\b/i,
|
||||
/\bschuld\b/i,
|
||||
/\bauslöser\b/i,
|
||||
];
|
||||
|
||||
export type NarrativeGroundingReason = "causal_language" | "ungrounded_number";
|
||||
|
||||
export interface NarrativeGroundingFinding {
|
||||
reason: NarrativeGroundingReason;
|
||||
/** The matched token (truncated). */
|
||||
source: string;
|
||||
}
|
||||
|
||||
/** Parse every standalone number out of free text (`.`/`,` decimals). */
|
||||
function extractNumbers(text: string): Array<{ value: number; raw: string }> {
|
||||
const out: Array<{ value: number; raw: string }> = [];
|
||||
const re = /[+-]?\d+(?:[.,]\d+)?/g;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = re.exec(text)) !== null) {
|
||||
const raw = match[0];
|
||||
const value = Number.parseFloat(raw.replace(",", "."));
|
||||
if (Number.isFinite(value)) out.push({ value, raw });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Every authoritative magnitude the context exposes (signed + absolute). */
|
||||
function authoritativeValues(context: PeriodNarrativeContext): number[] {
|
||||
const values: number[] = [];
|
||||
const push = (n: number | null | undefined) => {
|
||||
if (typeof n === "number" && Number.isFinite(n)) {
|
||||
values.push(n);
|
||||
values.push(Math.abs(n));
|
||||
}
|
||||
};
|
||||
for (const d of context.metricDeltas) {
|
||||
push(d.current);
|
||||
push(d.prior);
|
||||
push(d.delta);
|
||||
push(d.deltaPercent);
|
||||
push(d.currentDays);
|
||||
push(d.priorDays);
|
||||
}
|
||||
for (const b of context.bandTransitions) {
|
||||
push(b.center);
|
||||
push(b.bandLow);
|
||||
push(b.bandHigh);
|
||||
push(b.baselineDays);
|
||||
}
|
||||
for (const dr of context.drivers) {
|
||||
push(dr.r);
|
||||
push(dr.qValue);
|
||||
push(dr.n);
|
||||
}
|
||||
push(context.pairsTested);
|
||||
push(context.fdrQ);
|
||||
return values;
|
||||
}
|
||||
|
||||
function isGrounded(value: number, authoritative: readonly number[]): boolean {
|
||||
for (const a of authoritative) {
|
||||
const tol = Math.max(ABS_TOLERANCE, Math.abs(a) * REL_TOLERANCE);
|
||||
if (Math.abs(value - a) <= tol) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural integers the narrative may state without a context match — small
|
||||
* ordinals and the literal period lengths the prompt references (7 / 30 days).
|
||||
*/
|
||||
const STRUCTURAL_INTEGERS = new Set([7, 30]);
|
||||
|
||||
function isStructural(value: number, raw: string): boolean {
|
||||
if (raw.includes(".") || raw.includes(",")) return false;
|
||||
if (!Number.isInteger(value)) return false;
|
||||
const abs = Math.abs(value);
|
||||
if (abs <= 3) return true;
|
||||
return STRUCTURAL_INTEGERS.has(abs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a generated narrative against its context. Returns every grounding
|
||||
* violation found; an empty array means the narrative is grounded and
|
||||
* causally-clean.
|
||||
*/
|
||||
export function validateNarrativeText(
|
||||
text: string,
|
||||
context: PeriodNarrativeContext,
|
||||
): NarrativeGroundingFinding[] {
|
||||
const findings: NarrativeGroundingFinding[] = [];
|
||||
if (typeof text !== "string" || text.trim().length === 0) return findings;
|
||||
|
||||
for (const pattern of CAUSAL_PATTERNS) {
|
||||
const m = pattern.exec(text);
|
||||
if (m) {
|
||||
findings.push({ reason: "causal_language", source: m[0].slice(0, 32) });
|
||||
break; // one causal flag is enough to trigger the corrective retry
|
||||
}
|
||||
}
|
||||
|
||||
const authoritative = authoritativeValues(context);
|
||||
for (const { value, raw } of extractNumbers(text)) {
|
||||
if (isStructural(value, raw)) continue;
|
||||
if (isGrounded(value, authoritative)) continue;
|
||||
findings.push({ reason: "ungrounded_number", source: raw.slice(0, 32) });
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Corrective suffix appended to the user prompt on the single retry, naming the
|
||||
* violations so the model can self-correct — mirroring the comprehensive
|
||||
* insight's `buildRetryCorrectionMessage` posture.
|
||||
*/
|
||||
export function buildNarrativeCorrection(
|
||||
findings: readonly NarrativeGroundingFinding[],
|
||||
locale: "de" | "en",
|
||||
): string {
|
||||
const causal = findings.some((f) => f.reason === "causal_language");
|
||||
const numbers = findings
|
||||
.filter((f) => f.reason === "ungrounded_number")
|
||||
.slice(0, 6)
|
||||
.map((f) => f.source)
|
||||
.join(", ");
|
||||
if (locale === "de") {
|
||||
const parts = [
|
||||
"Deine vorige Zusammenfassung hat die festen Regeln verletzt.",
|
||||
];
|
||||
if (causal) {
|
||||
parts.push(
|
||||
"Sie enthielt URSÄCHLICHE Formulierungen. Schreibe ausschließlich BESCHREIBEND ('X bewegte sich mit Y', 'X war mit Y assoziiert') — nie 'weil', 'wegen', 'verursacht', 'führte zu'.",
|
||||
);
|
||||
}
|
||||
if (numbers) {
|
||||
parts.push(
|
||||
`Sie nannte Zahlen, die NICHT im Kontext stehen: ${numbers}. Nenne nur Zahlen, die wörtlich im KONTEXT vorkommen, oder lass die Zahl weg.`,
|
||||
);
|
||||
}
|
||||
parts.push("Schreibe die Zusammenfassung erneut (2–4 kurze Sätze).");
|
||||
return `\n\n${parts.join(" ")}`;
|
||||
}
|
||||
const parts = ["Your previous summary broke the hard rules."];
|
||||
if (causal) {
|
||||
parts.push(
|
||||
"It used CAUSAL wording. Write strictly DESCRIPTIVELY ('X moved with Y', 'X was associated with Y') — never 'because', 'due to', 'caused', 'led to'.",
|
||||
);
|
||||
}
|
||||
if (numbers) {
|
||||
parts.push(
|
||||
`It stated numbers NOT in the context: ${numbers}. State only numbers that appear verbatim in the CONTEXT, or drop the number.`,
|
||||
);
|
||||
}
|
||||
parts.push("Re-write the summary (2-4 short sentences).");
|
||||
return `\n\n${parts.join(" ")}`;
|
||||
}
|
||||
@@ -41,6 +41,10 @@ import {
|
||||
DETERMINISTIC_PROVIDER_TYPE,
|
||||
} from "@/lib/insights/narrative/period-narrative-deterministic";
|
||||
import { composeSharedContracts } from "@/lib/ai/prompts/shared-contracts";
|
||||
import {
|
||||
validateNarrativeText,
|
||||
buildNarrativeCorrection,
|
||||
} from "@/lib/insights/narrative/narrative-grounding";
|
||||
|
||||
/**
|
||||
* Stable identifier for the narrative prompt revision. Bumped whenever the
|
||||
@@ -350,10 +354,61 @@ export async function generatePeriodNarrative(
|
||||
return { status: "generated", providerType: DETERMINISTIC_PROVIDER_TYPE };
|
||||
}
|
||||
|
||||
const text = completion.content.trim();
|
||||
let text = completion.content.trim();
|
||||
if (text.length === 0) {
|
||||
return { status: "failed", reason: "empty" };
|
||||
}
|
||||
|
||||
// v1.18.10 (HIGH-3) — grounding gate on the AI prose, matching the rejecting
|
||||
// gate Comprehensive Insights already has. Flag causal phrasing and any
|
||||
// number absent from the context; on a trip, retry ONCE with a corrective
|
||||
// suffix, then fall back to writing NOTHING (last good row stays) rather than
|
||||
// persisting an ungrounded or causal story.
|
||||
let findings = validateNarrativeText(text, context);
|
||||
if (findings.length > 0) {
|
||||
annotate({
|
||||
action: { name: "insights.narrative.grounding_retry" },
|
||||
meta: {
|
||||
period,
|
||||
locale,
|
||||
causal: findings.some((f) => f.reason === "causal_language"),
|
||||
ungroundedNumbers: findings.filter(
|
||||
(f) => f.reason === "ungrounded_number",
|
||||
).length,
|
||||
},
|
||||
});
|
||||
const retry = await runCompletion({
|
||||
userId,
|
||||
cacheAction: `insights.narrative.${period}.${locale}`,
|
||||
consentSurface: "insights",
|
||||
systemPrompt: locale === "de" ? SYSTEM_PROMPT_DE : SYSTEM_PROMPT_EN,
|
||||
userPrompt:
|
||||
buildNarrativeUserPrompt(context, locale) +
|
||||
buildNarrativeCorrection(findings, locale),
|
||||
temperature: AI_BUDGETS.narrative.temperature,
|
||||
maxTokens: AI_BUDGETS.narrative.maxTokens,
|
||||
seed: REFERENCE_AI_SEED,
|
||||
responseFormat: "text",
|
||||
});
|
||||
if (retry.kind === "ok") {
|
||||
const retryText = retry.content.trim();
|
||||
const retryFindings = validateNarrativeText(retryText, context);
|
||||
if (retryText.length > 0 && retryFindings.length === 0) {
|
||||
text = retryText;
|
||||
findings = [];
|
||||
} else {
|
||||
findings = retryFindings;
|
||||
}
|
||||
}
|
||||
if (findings.length > 0) {
|
||||
annotate({
|
||||
action: { name: "insights.narrative.grounding_rejected" },
|
||||
meta: { period, locale },
|
||||
});
|
||||
return { status: "failed", reason: "ungrounded" };
|
||||
}
|
||||
}
|
||||
|
||||
await persist(text, completion.providerType);
|
||||
return { status: "generated", providerType: completion.providerType };
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user