diff --git a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts index d8b95d743..3f4a1e17d 100644 --- a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts +++ b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts @@ -154,6 +154,7 @@ const { buildCoachDataInventory, renderDataInventory, runCoachToolLoop } = totalTokens: 80, rounds: 2, toolTrace: [{ name: "get_metric_series", present: true }], + toolResults: [], })), })); vi.mock("@/lib/ai/coach/tools", () => ({ @@ -183,6 +184,9 @@ vi.mock("@/lib/validations/coach-prefs", () => ({ })); const { appendMessage } = await import("@/lib/ai/coach/persistence"); +const { parseKeyValuesSentinel } = await import("@/lib/ai/coach/keyvalues"); +const { parseSuggestReminder } = + await import("@/lib/ai/coach/suggest-reminder"); vi.mock("@/lib/sse/create-stream", () => ({ createSseStream: ( @@ -260,6 +264,50 @@ describe("coach chat — tool-mode routing (F1)", () => { ).toEqual([{ name: "get_metric_series", present: true }]); }); + it("soft-strips a prose number the tools never returned (P6)", async () => { + resolveProviderChain.mockResolvedValue([ + { providerType: "anthropic", instance: {} }, + ]); + // Echo the real prose through the sentinel + suggest parsers so the + // verifier sees the model's actual numbers (the default mocks return a + // fixed string). + const drift = "Your systolic averaged about 138 lately."; + (parseKeyValuesSentinel as ReturnType).mockReturnValue({ + prose: drift, + keyValues: [], + malformed: false, + malformedEntries: [], + }); + (parseSuggestReminder as ReturnType).mockReturnValue({ + prose: drift, + }); + // The tool returned systolic 128; the prose drifts to 138. + (runCoachToolLoop as ReturnType).mockImplementation( + async () => ({ + result: { + content: "Your systolic averaged about 138 lately.", + tokensUsed: 80, + model: "m", + }, + workingProviderType: "anthropic", + totalTokens: 80, + rounds: 2, + toolTrace: [{ name: "get_metric_series", present: true }], + toolResults: [ + { present: true, data: { aggregate: { avgSys30: 128 } } }, + ], + }), + ); + await post(chatReq({ message: "How is my BP?" })); + const calls = (appendMessage as ReturnType).mock.calls; + const assistantCall = calls.find( + (c) => (c[0] as { role: string }).role === "assistant", + ); + const content = (assistantCall?.[0] as { content: string }).content; + expect(content).toContain("[unverified]"); + expect(content).not.toContain("138"); + }); + it("falls back to the snapshot-stuffing path when a provider lacks tools", async () => { resolveProviderChain.mockResolvedValue([ { providerType: "anthropic", instance: {} }, diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index 341476274..9fca7a8d8 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -84,6 +84,10 @@ import { } from "@/lib/ai/coach/tools"; import type { AiMessage } from "@/lib/ai/types"; import { parseKeyValuesSentinel } from "@/lib/ai/coach/keyvalues"; +import { + findUnverifiedCoachNumbers, + stripUnverifiedNumbers, +} from "@/lib/ai/coach/coach-prose-grounding"; import { parseSuggestReminder } from "@/lib/ai/coach/suggest-reminder"; import { gateSuggestion } from "@/lib/ai/coach/suggest-gate"; import { @@ -518,6 +522,9 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`; let result: CompletionResult; let workingProviderType: string; let toolTrace: CoachToolTrace[] = []; + // v1.21.0 (P6) — the present tool-result payloads this turn, for the post-hoc + // prose number-verifier. Empty on the no-tools path. + let toolResultPayloads: unknown[] = []; let totalTokensSpent: number; try { if (toolMode) { @@ -556,6 +563,7 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch result = loop.result; workingProviderType = loop.workingProviderType; toolTrace = loop.toolTrace; + toolResultPayloads = (loop.toolResults ?? []).map((r) => r.data); totalTokensSpent = loop.totalTokens; } else { const fallback = await runRawCompletionWithFallback({ @@ -701,6 +709,38 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch }); } + // v1.21.0 (P6 / C2-5) — post-hoc numeric verifier on the Coach prose. On the + // tool path, cross-check every number the model cited against the figures the + // tools actually returned this turn; an unmatched number (transcription / + // paraphrase drift) is soft-stripped to "[unverified]" and annotated. Cheap, + // non-blocking, and a no-op when no tool returned figures (a qualitative turn + // or the no-tools path) — the prompt-level grounding rule remains the + // backstop there, exactly like the briefing's "no signals → skip". A blocked + // turn already carries canned fallback prose, so skip it. + if (!outbound.block && toolResultPayloads.length > 0) { + const unverified = findUnverifiedCoachNumbers( + replyText, + toolResultPayloads, + ); + if (unverified.length > 0) { + const { prose: corrected, stripped } = stripUnverifiedNumbers( + replyText, + unverified, + ); + replyText = corrected; + annotate({ + action: { name: "coach.prose.number_unverified" }, + meta: { + flagged: unverified.length, + stripped, + // No raw values — just the count + truncated tokens for ops triage. + tokens: unverified.slice(0, 6).map((u) => u.source), + promptVersion: PROMPT_VERSION, + }, + }); + } + } + let surfacedSuggestion: CoachSuggestion | null = null; if (!outbound.block && suggestParse.cadence) { const decision = await gateSuggestion({ diff --git a/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts b/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts new file mode 100644 index 000000000..c6a61a988 --- /dev/null +++ b/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts @@ -0,0 +1,89 @@ +/** + * v1.21.0 (P6 / C2-5) — post-hoc numeric verifier on Coach prose. It must flag + * a number the model cited that no tool returned this turn (transcription + * drift), exempt structural window/ordinal integers, and no-op when no tool + * returned figures. Soft-strip replaces only the flagged tokens. + */ +import { describe, expect, it } from "vitest"; + +import { + collectNumericLeaves, + findUnverifiedCoachNumbers, + stripUnverifiedNumbers, +} from "@/lib/ai/coach/coach-prose-grounding"; + +describe("collectNumericLeaves", () => { + it("walks numbers, numeric strings, arrays and nested objects", () => { + const out = new Set(); + collectNumericLeaves( + { + section: { avgSys30: 128, label: "LDL", value: "120" }, + recent: [{ v: 72 }, { v: "1.2" }], + }, + out, + ); + expect(out.has(128)).toBe(true); + expect(out.has(120)).toBe(true); + expect(out.has(72)).toBe(true); + expect(out.has(1.2)).toBe(true); + }); +}); + +describe("findUnverifiedCoachNumbers", () => { + const payloads = [{ aggregate: { avgSys30: 128, avgDia30: 82 } }]; + + it("returns nothing when every cited number is grounded", () => { + const prose = + "Your systolic averaged 128 and diastolic 82 over the last 30 days."; + expect(findUnverifiedCoachNumbers(prose, payloads)).toEqual([]); + }); + + it("flags a drifted number the tools never returned", () => { + const prose = "Your systolic averaged about 138 recently."; + const findings = findUnverifiedCoachNumbers(prose, payloads); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(138); + }); + + it("allows a rounding within tolerance", () => { + const prose = "Diastolic sat around 82.0 this period."; + expect(findUnverifiedCoachNumbers(prose, payloads)).toEqual([]); + }); + + it("exempts structural window + small-ordinal integers", () => { + const prose = "Over the last 30 days, 2 readings stood out of 7 total."; + // 30 + 7 are window integers; 2 is a small ordinal — none graded. + const findings = findUnverifiedCoachNumbers(prose, [{ x: 999 }]); + expect(findings).toEqual([]); + }); + + it("no-ops when no tool returned any figure", () => { + const prose = "Your systolic averaged 138 recently."; + expect(findUnverifiedCoachNumbers(prose, [])).toEqual([]); + expect( + findUnverifiedCoachNumbers(prose, [{ note: "no numbers here" }]), + ).toEqual([]); + }); + + it("returns empty on empty prose", () => { + expect(findUnverifiedCoachNumbers("", payloads)).toEqual([]); + }); +}); + +describe("stripUnverifiedNumbers", () => { + it("replaces only the flagged tokens, leaving grounded numbers intact", () => { + const prose = "Systolic 128, but it spiked to 138 yesterday."; + const findings = [{ value: 138, source: "138" }]; + const { prose: out, stripped } = stripUnverifiedNumbers(prose, findings); + expect(stripped).toBe(1); + expect(out).toContain("128"); + expect(out).toContain("[unverified]"); + expect(out).not.toContain("138"); + }); + + it("is a no-op when nothing was flagged", () => { + const { prose, stripped } = stripUnverifiedNumbers("all good", []); + expect(prose).toBe("all good"); + expect(stripped).toBe(0); + }); +}); diff --git a/src/lib/ai/coach/__tests__/tools-loop.test.ts b/src/lib/ai/coach/__tests__/tools-loop.test.ts index 7e70c3499..3c4586224 100644 --- a/src/lib/ai/coach/__tests__/tools-loop.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop.test.ts @@ -86,6 +86,9 @@ describe("runCoachToolLoop", () => { expect(out.toolTrace).toEqual([ { name: "get_metric_series", present: true }, ]); + // v1.21.0 (P6) — the present result's payload is retained for the prose + // number-verifier. + expect(out.toolResults).toEqual([{ present: true, data: { x: 1 } }]); // The second call must forbid tools? No — round 2 still offers them // (round <= MAX_ROUNDS). It simply chose to answer. expect(runRawCompletionWithFallback).toHaveBeenCalledTimes(2); diff --git a/src/lib/ai/coach/coach-prose-grounding.ts b/src/lib/ai/coach/coach-prose-grounding.ts new file mode 100644 index 000000000..0772e83b8 --- /dev/null +++ b/src/lib/ai/coach/coach-prose-grounding.ts @@ -0,0 +1,153 @@ +/** + * v1.21.0 (P6 / C2-5) — post-hoc numeric verifier for Coach prose. + * + * The Daily Briefing strips any number absent from the server-computed + * `signalsOfDay` block (`@/lib/ai/briefing-grounding`). The Coach's TOOL path + * had no equivalent: the addendum forbids citing un-fetched figures, but + * nothing deterministically checks the FINAL prose against the figures the + * tools actually returned this turn — so a transcription/paraphrase drift + * (tool says systolic 128, prose says "~138") could ship. + * + * This module closes that gap the same way, scoped to the tool path: + * 1. Collect every numeric leaf from this turn's PRESENT `CoachToolResult` + * data payloads — the authoritative figure set. + * 2. Extract every number the model's prose asserts (reusing the briefing + * verifier's `extractNumbers`). + * 3. Flag any prose number that matches no authoritative figure within a + * rounding tolerance, exempting the structural integers (window lengths, + * small ordinals) the briefing verifier already exempts. + * + * Posture: NON-BLOCKING and cheap. The caller annotates + * `coach.prose.number_unverified` and may SOFT-STRIP the unverified figure + * from the prose (replacing the bare token with a neutral placeholder) — it + * never hard-fails the user's turn. When NO tool returned figures (a + * qualitative answer, or the no-tools path) there is no authoritative set to + * grade against, so the check no-ops and the prompt-level grounding rule + * remains the backstop, exactly like the briefing's "no signals → skip". + */ +import { extractNumbers } from "@/lib/ai/briefing-grounding"; + +/** Absolute + relative tolerance — identical basis to the briefing verifier. */ +const ABS_TOLERANCE = 0.15; +const REL_TOLERANCE = 0.02; + +/** + * Window lengths + small ordinals the prose uses structurally ("last 7 days", + * "2 of your vitals", "3 readings") — never graded, to avoid false positives on + * honest framing. Mirrors the briefing verifier's exemption set. + */ +const STRUCTURAL_INTEGERS = new Set([7, 14, 30, 31, 90, 180, 365]); + +function isStructural(value: number, raw: string): boolean { + if (raw.includes(".") || raw.includes(",")) return false; + if (!Number.isInteger(value)) return false; + const abs = Math.abs(value); + if (abs <= 3) return true; + return STRUCTURAL_INTEGERS.has(abs); +} + +/** One prose number that matched no figure any tool returned this turn. */ +export interface UnverifiedCoachNumber { + /** The numeric value the model wrote, as parsed. */ + value: number; + /** The raw token the value was read from (truncated). */ + source: string; +} + +/** + * Recursively collect every finite numeric leaf from a tool-result payload — + * numbers, and numeric strings ("128", "1.2"). Both forms appear in the + * snapshot sections (a mean may be a number; a lab value a string). The set is + * deliberately broad so a legitimately-cited figure never trips the verifier; + * the verifier's job is only to catch a number the model invented or mis-copied. + */ +export function collectNumericLeaves(value: unknown, out: Set): void { + if (value === null || value === undefined) return; + if (typeof value === "number") { + if (Number.isFinite(value)) out.add(value); + return; + } + if (typeof value === "string") { + // A scalar numeric string ("128", "-1.2", "92%") — pull its magnitudes. + for (const { value: n } of extractNumbers(value)) out.add(n); + return; + } + if (Array.isArray(value)) { + for (const item of value) collectNumericLeaves(item, out); + return; + } + if (typeof value === "object") { + for (const v of Object.values(value as Record)) { + collectNumericLeaves(v, out); + } + } +} + +/** True when `value` matches any authoritative figure within tolerance. */ +function isGrounded( + value: number, + authoritative: ReadonlySet, +): boolean { + for (const a of authoritative) { + const tol = Math.max(ABS_TOLERANCE, Math.abs(a) * REL_TOLERANCE); + if (Math.abs(value - a) <= tol) return true; + } + return false; +} + +/** + * Find every number the Coach prose asserts that does not trace to a figure + * returned by a tool this turn. Returns an empty array when there is no prose, + * no authoritative figure set (no present tool result with numbers), or every + * cited number is grounded. + * + * `toolPayloads` is the `data` payload of each PRESENT tool result this turn. + */ +export function findUnverifiedCoachNumbers( + prose: string, + toolPayloads: ReadonlyArray, +): UnverifiedCoachNumber[] { + if (!prose) return []; + const authoritative = new Set(); + for (const payload of toolPayloads) + collectNumericLeaves(payload, authoritative); + // No authoritative figures (a qualitative turn / no-tools path) — nothing to + // grade against. The prompt-level grounding rule remains the backstop. + if (authoritative.size === 0) return []; + + const findings: UnverifiedCoachNumber[] = []; + for (const { value, raw } of extractNumbers(prose)) { + if (isStructural(value, raw)) continue; + if (isGrounded(value, authoritative)) continue; + findings.push({ value, source: raw.slice(0, 32) }); + } + return findings; +} + +/** + * Soft-correct the prose: replace each unverified numeric token with a neutral + * placeholder so a drifted figure never reaches the user as if authoritative, + * while the surrounding qualitative framing is preserved. Conservative — it + * only rewrites the exact ungrounded tokens the verifier flagged, and only the + * first occurrence of each, leaving every grounded number untouched. + * + * Returns the (possibly unchanged) prose plus the count of tokens stripped. + */ +export function stripUnverifiedNumbers( + prose: string, + findings: ReadonlyArray, +): { prose: string; stripped: number } { + if (findings.length === 0) return { prose, stripped: 0 }; + let out = prose; + let stripped = 0; + for (const f of findings) { + // Replace the first standalone occurrence of the flagged token. The token + // is bounded by non-digit / non-sign edges so we don't clip a larger number. + const token = f.source; + const idx = out.indexOf(token); + if (idx === -1) continue; + out = `${out.slice(0, idx)}[unverified]${out.slice(idx + token.length)}`; + stripped += 1; + } + return { prose: out, stripped }; +} diff --git a/src/lib/ai/coach/tools/loop.ts b/src/lib/ai/coach/tools/loop.ts index 1f0dfb1d6..5e8208e4a 100644 --- a/src/lib/ai/coach/tools/loop.ts +++ b/src/lib/ai/coach/tools/loop.ts @@ -28,7 +28,11 @@ import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; import type { AiMessage, AiToolDef, CompletionResult } from "@/lib/ai/types"; import type { ProviderHealthLedger } from "@/lib/ai/provider-health-ledger"; import type { CoachScopeWindow } from "@/lib/ai/coach/types"; -import { executeCoachTool, type CoachToolTrace } from "./executor"; +import { + executeCoachTool, + type CoachToolResult, + type CoachToolTrace, +} from "./executor"; export const MAX_ROUNDS = 2; /** Absolute ceiling; the final allowed round forces prose (toolChoice none). */ @@ -45,6 +49,13 @@ export interface CoachToolLoopResult { rounds: number; /** Which tools ran + whether each found data (persisted onto provenance). */ toolTrace: CoachToolTrace[]; + /** + * v1.21.0 (P6) — the structured payloads of every PRESENT tool result this + * turn, in call order. The post-hoc prose number-verifier extracts the + * numeric leaves from these to cross-check the figures the model cited. Empty + * on a no-tools answer. + */ + toolResults: CoachToolResult[]; } export async function runCoachToolLoop(args: { @@ -78,6 +89,7 @@ export async function runCoachToolLoop(args: { let rounds = 0; let workingProviderType = ""; const toolTrace: CoachToolTrace[] = []; + const toolResults: CoachToolResult[] = []; // Round budget: rounds 1..HARD_CAP. On the last allowed round we forbid tool // calls so the model must answer. @@ -120,7 +132,14 @@ export async function runCoachToolLoop(args: { forcedFinal: isForcedFinal, }, }); - return { result, workingProviderType, totalTokens, rounds, toolTrace }; + return { + result, + workingProviderType, + totalTokens, + rounds, + toolTrace, + toolResults, + }; } // Append the assistant turn that requested the tools, then execute them in @@ -140,6 +159,10 @@ export async function runCoachToolLoop(args: { fallbackWindow, }); toolTrace.push({ name: call.name, present: toolResult.present }); + // v1.21.0 (P6) — retain the present results' payloads for the post-hoc + // prose number-verifier (the union of numeric leaves grounds the + // figures the model may cite). + if (toolResult.present) toolResults.push(toolResult); return { call, toolResult }; }), );