diff --git a/CHANGELOG.md b/CHANGELOG.md index a15813119..9f4565b19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## [Unreleased] +- **The Coach's numeric verifier no longer flags a correctly-grounded figure.** + A restated delta, average, or percent change computed from two of the + model's own headline numbers, a sample count restated from the always-sent + data inventory, a calendar-date ordinal ("July 21st"), and a numbered-list + marker no longer trip the soft `[unverified]` rewrite. A genuinely + fabricated figure is still caught and stripped. + ## [1.32.0] — 2026-07-22 - **Provider ingestion and daily reactions now preserve the correct identity.** diff --git a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts index b2cd3d496..dde8d8ec4 100644 --- a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts +++ b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts @@ -409,6 +409,76 @@ describe("coach chat — tool-mode routing (F1)", () => { expect(content).not.toContain("138"); }); + it("does not flag a sample count restated from the DATA INVENTORY manifest even when its own tool did not run this turn (v1.32.1)", async () => { + resolveProviderChain.mockResolvedValue([ + { providerType: "anthropic", instance: {} }, + ]); + // The DATA INVENTORY (always sent to the model this turn, regardless of + // which tools actually fire — see `renderDataInventory`) advertises a + // blood-pressure sample count of 42. The model restates it in prose while + // the ONLY tool it actually called this turn was for workouts. + (buildCoachDataInventory as ReturnType).mockResolvedValueOnce( + { + entries: [ + { + tool: "get_metric_series", + metric: "bp", + domain: "blood pressure", + present: true, + count: 42, + }, + { + tool: "get_workouts", + domain: "workouts & training", + present: true, + count: 12, + }, + ], + restMode: false, + cycleEnabled: false, + window: "last30days", + probeScope: { sources: ["bp", "workouts"], window: "last30days" }, + }, + ); + const reply = + "You've logged 42 blood pressure readings this window, and your last run burned 320 kcal."; + (parseKeyValuesSentinel as ReturnType).mockReturnValue({ + prose: reply, + keyValues: [], + malformed: false, + malformedEntries: [], + }); + (parseSuggestReminder as ReturnType).mockReturnValue({ + prose: reply, + }); + (runCoachToolLoop as ReturnType).mockImplementation( + async () => ({ + result: { content: reply, tokensUsed: 80, model: "m" }, + workingProviderType: "anthropic", + totalTokens: 80, + rounds: 2, + toolTrace: [{ name: "get_workouts", present: true }], + // ONLY the workouts tool ran — the BP count "42" was never fetched + // this turn, it came from the always-sent DATA INVENTORY manifest. + toolResults: [ + { + present: true, + data: { workouts: [{ type: "run", calories: 320 }] }, + }, + ], + }), + ); + await postAndDrain({ message: "How is my BP and my training?" }); + const calls = (appendMessage as ReturnType).mock.calls; + const assistantCall = calls.find( + (c) => (c[0] as { role: string }).role === "assistant", + ); + const content = (assistantCall?.[0] as { content: string }).content; + expect(content).not.toContain("[unverified]"); + expect(content).toContain("42"); + expect(content).toContain("320"); + }); + it("falls back to the snapshot-stuffing path when a provider lacks tools", async () => { resolveProviderChain.mockResolvedValue([ { providerType: "anthropic", instance: {} }, diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index ad5fef4fb..fa05230b3 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -653,6 +653,16 @@ async function handleChatRequest(request: NextRequest): Promise { toolTrace = loop.toolTrace; toolResultPayloads = [ ...(workoutEvidence === null ? [] : [workoutEvidence]), + // v1.32.1 (false-positive closure) — the DATA INVENTORY manifest + // (sample counts per domain) rides EVERY tool-mode turn's prompt + // regardless of which tools actually ran this turn + // (`renderDataInventory`) — it is server-computed and handed to the + // model exactly like `workoutEvidence` above, so a plain restatement + // of a domain's sample count ("you've logged 42 BP readings") is + // grounded even when the model did not also call a retrieval tool + // for that domain this turn. Splicing it in here closes that gap + // the same way `workoutEvidence` already does. + inventory.entries, ...(loop.toolResults ?? []).map((r) => r.data), ]; totalTokensSpent = loop.totalTokens; diff --git a/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts b/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts index c6a61a988..126de206b 100644 --- a/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts +++ b/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts @@ -3,6 +3,14 @@ * a number the model cited that no tool returned this turn (transcription * drift), exempt structural window/ordinal integers, and no-op when no tool * returned figures. Soft-strip replaces only the flagged tokens. + * + * v1.32.1 — closes three real-world false-positive shapes reported against + * grounded Coach replies: a correctly-computed derived delta/average/percent + * between two of a payload's own headline figures, a calendar-date ordinal + * ("July 21st") mangled mid-word into "[unverified]st", and a numbered-list + * recommendation line ("1. Cut back on sodium") whose marker got flagged as a + * bare figure. Each new case is paired with an adversarial case proving a + * genuine fabrication in the SAME shape is still caught. */ import { describe, expect, it } from "vitest"; @@ -68,6 +76,137 @@ describe("findUnverifiedCoachNumbers", () => { it("returns empty on empty prose", () => { expect(findUnverifiedCoachNumbers("", payloads)).toEqual([]); }); + + describe("derived arithmetic (delta / average / percent)", () => { + // A realistic weight tool-result payload: the model was given latest, + // avg7 and avg30 — never a pre-computed delta (unlike the Daily + // Briefing's `signalsOfDay`), so a correctly-computed "down 3.5 kg from + // your 7-day average" is a legitimate derivation the checker must not + // flag. + const weightPayload = [ + { + metric: "weight", + section: { aggregate: { latest: 72.4, avg7: 75.9, avg30: 74.1 } }, + }, + ]; + + it("does not flag a correctly-computed delta between two grounded aggregate points", () => { + const prose = + "You're down 3.5 kg from your 7-day average of 75.9 kg, now at 72.4 kg — nice trend."; + expect(findUnverifiedCoachNumbers(prose, weightPayload)).toEqual([]); + }); + + it("does not flag a correctly-computed midpoint average of two grounded aggregate points", () => { + const prose = + "Your latest (72.4 kg) and 30-day average (74.1 kg) put you around 73.25 kg overall."; + expect(findUnverifiedCoachNumbers(prose, weightPayload)).toEqual([]); + }); + + it("does not flag a correctly-computed percent change between two grounded aggregate points", () => { + // (75.9 - 72.4) / 75.9 * 100 ≈ 4.61% + const prose = "That's roughly a 4.6% drop from your 7-day average."; + expect(findUnverifiedCoachNumbers(prose, weightPayload)).toEqual([]); + }); + + it("still flags a fabricated figure that is NOT a derived value of the grounded points", () => { + const prose = + "Your latest reading was 72.4 kg, and I estimate you'll hit 55.0 kg by next month."; + const findings = findUnverifiedCoachNumbers(prose, weightPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(55); + }); + + it("does not let a weight delta ground itself against an unrelated metric's figure (per-payload scoping)", () => { + // Blood pressure's own aggregate lives in a SEPARATE tool-result + // payload; a "difference" here must derive only from ITS OWN points, + // never pool with the unrelated weight payload above. + const bpAndWeight = [ + ...weightPayload, + { + metric: "bp", + section: { aggregate: { avgSys30: 128, avgDia30: 82 } }, + }, + ]; + // 46 is |128 - 82| — a legitimate BP pulse-pressure-shaped delta — + // grounded via the bp payload's own points, not a coincidence. + expect( + findUnverifiedCoachNumbers( + "Your pulse pressure is about 46.", + bpAndWeight, + ), + ).toEqual([]); + }); + }); + + describe("date ordinals and numbered-list markers", () => { + const bpPayload = [ + { + metric: "bp", + section: { aggregate: { latest: 138, avg7: 132, avg30: 128 } }, + }, + ]; + + it("does not flag a day-of-month ordinal even though the bare day number is ungrounded", () => { + const prose = + "Your blood pressure spiked to 138 on July 21st, well above your typical range."; + expect(findUnverifiedCoachNumbers(prose, bpPayload)).toEqual([]); + }); + + it("does not corrupt the ordinal suffix when soft-stripping the rest of the reply", () => { + // Regression guard for the exact mangling reported: "[unverified]st". + const prose = + "Your blood pressure spiked to 138 on July 21st, well above your typical range."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + const { prose: out } = stripUnverifiedNumbers(prose, findings); + expect(out).not.toContain("[unverified]st"); + expect(out).toBe(prose); + }); + + it("does not flag numbered-list recommendation markers", () => { + // Item numbers run past 3 (the pre-existing small-ordinal exemption) so + // this genuinely exercises the line-start list-marker mask, not the + // unrelated small-integer rule. + const prose = + "Your systolic hit 138 today. A few ideas:\n1. Cut back on sodium\n2. Take a short walk after meals\n3. Log your next reading tomorrow\n4. Recheck in the evening\n5. Note any stress today"; + expect(findUnverifiedCoachNumbers(prose, bpPayload)).toEqual([]); + }); + + it("still flags a fabricated figure sitting right next to a date ordinal", () => { + const prose = "On July 21st your systolic hit 199, a new high."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(199); + }); + + it("still flags a genuine leading figure that happens to open a line (not list-shaped without the trailing space)", () => { + // "199kg" has no space after the number at all — never mistaken for a + // "N. " list marker, which requires a literal dot-then-space. + const prose = "199kg would be an alarming reading."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(199); + }); + + it("still flags a decimal figure that opens a line (a digit, not whitespace, follows the dot)", () => { + // "199.5" at line-start is a genuine decimal reading, not a "N. " list + // marker — the masking regex only fires when whitespace follows the + // dot, so this stays fully checked. + const prose = "199.5 would be an alarming systolic reading."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(199.5); + }); + }); + + it("still flags a drifted number when the prose also derives / dates cleanly (adversarial floor)", () => { + // The classic P6 regression case must survive every new exemption. + const findings = findUnverifiedCoachNumbers( + "Your systolic averaged about 138 recently.", + payloads, + ); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(138); + }); }); describe("stripUnverifiedNumbers", () => { diff --git a/src/lib/ai/coach/coach-prose-grounding.ts b/src/lib/ai/coach/coach-prose-grounding.ts index 49efb46f5..e3052cd39 100644 --- a/src/lib/ai/coach/coach-prose-grounding.ts +++ b/src/lib/ai/coach/coach-prose-grounding.ts @@ -95,6 +95,114 @@ function isGrounded( return false; } +/** + * v1.32.1 (false-positive closure) — the small, named "central tendency" + * fields every snapshot block already exposes (`latest`, `avg7`, `avg30`, + * `avgSys30`, `allTimeAvg`, …). A model correctly narrating "down 3.5 kg from + * your 7-day average" is computing a DELTA between two of these — a + * legitimate derivation, not a fabrication — but the raw figure `3.5` is not + * itself a leaf anywhere in the payload, so the plain leaf-matching above + * flags it. Deliberately an ALLOWLIST of field names, not "every numeric + * leaf": a whole month of daily timeline samples would blow the pairwise + * derivation up combinatorially (see `deriveArithmeticValues`) and start + * matching a genuinely fabricated figure by coincidence, which is exactly the + * hole this checker exists to close. A raw sample, a count, or a day-offset + * never matches this pattern. + */ +const AGGREGATE_POINT_KEY = + /^(?:latest|mean|median|avg(?:Sys|Dia)?\d*(?:LastMonth|LastYear)?|allTime(?:Avg|Min|Max)(?:Sys|Dia)?)$/; + +/** + * Defensive cap on how many aggregate points one payload may contribute to + * the pairwise derivation below. The named fields the allowlist targets are a + * handful per metric block (2-6); a payload that somehow surfaces more than + * this is not the small aggregate shape the heuristic targets, so skip + * deriving from it rather than pay an unbounded O(n^2). + */ +const MAX_AGGREGATE_POINTS = 12; + +/** Collect every numeric leaf reachable under an `AGGREGATE_POINT_KEY` field. */ +function collectAggregatePoints( + value: unknown, + out: Set, + keyHint?: string, +): void { + if (value === null || value === undefined) return; + if (typeof value === "number") { + if ( + Number.isFinite(value) && + keyHint !== undefined && + AGGREGATE_POINT_KEY.test(keyHint) + ) { + out.add(value); + } + return; + } + if (Array.isArray(value)) { + for (const item of value) collectAggregatePoints(item, out, keyHint); + return; + } + if (typeof value === "object") { + for (const [k, v] of Object.entries(value as Record)) { + collectAggregatePoints(v, out, k); + } + } +} + +/** + * The delta, midpoint average, and percent-change between every PAIR of a + * SINGLE payload's own aggregate points — the shapes a model legitimately + * narrates ("down 3.5 kg from your 7-day average", "roughly a 5% drop"). + * Scoped per payload (one tool result / the pinned workout block / one + * inventory row), never pooled across the whole turn's authoritative set, so + * a weight delta cannot ground itself against an unrelated blood-pressure + * figure that happens to land nearby. + */ +function deriveArithmeticValues(payload: unknown): number[] { + const points = new Set(); + collectAggregatePoints(payload, points); + const arr = [...points]; + if (arr.length < 2 || arr.length > MAX_AGGREGATE_POINTS) return []; + const derived: number[] = []; + for (let i = 0; i < arr.length; i += 1) { + for (let j = i + 1; j < arr.length; j += 1) { + const a = arr[i]; + const b = arr[j]; + const diff = Math.abs(a - b); + derived.push(diff); + derived.push((a + b) / 2); + if (a !== 0) derived.push((diff / Math.abs(a)) * 100); + if (b !== 0) derived.push((diff / Math.abs(b)) * 100); + } + } + return derived; +} + +/** + * v1.32.1 (false-positive closure) — mask date-ordinal and numbered-list + * markers OUT of a working copy of the prose before it is tokenised for + * numbers, so "on July 21st" or a "1. Reduce sodium" recommendation line never + * reads as a restated health figure. `extractNumbers` has no notion of + * surrounding context, so this pre-processing happens on a throwaway copy — + * the ORIGINAL prose (and hence `stripUnverifiedNumbers`, which edits the real + * text via the flagged token) is untouched. Deliberately high precision: + * - a number immediately followed by an English ordinal suffix (21st, 3rd, + * 142nd) is a day-of-month or a rank, never a restated magnitude; + * - a number at the START of a line followed by ". " (a numbered-list + * marker) is masked, but ONLY at line-start with a space right after the + * dot — "Systolic 128. Blood pressure remained stable." keeps its 128 + * because that number does not open its own line, and "3.5 kg" keeps its + * figure because a digit (not whitespace) follows the dot. + */ +function extractClaimableNumbers( + prose: string, +): Array<{ value: number; raw: string }> { + const masked = prose + .replace(/\b\d{1,3}(?:st|nd|rd|th)\b/gi, "DATE") + .replace(/^(\s*)\d{1,3}\.(\s)/gm, "$1LIST$2"); + return extractNumbers(masked); +} + /** * Find every number the Coach prose asserts that does not trace to a figure * returned by a tool this turn. Returns an empty array when there is no prose, @@ -109,14 +217,19 @@ export function findUnverifiedCoachNumbers( ): UnverifiedCoachNumber[] { if (!prose) return []; const authoritative = new Set(); - for (const payload of toolPayloads) + for (const payload of toolPayloads) { collectNumericLeaves(payload, authoritative); + // Widen the authoritative set with the derived figures a model may + // legitimately compute from this payload's own headline numbers, without + // pooling across unrelated payloads (see `deriveArithmeticValues`). + for (const d of deriveArithmeticValues(payload)) authoritative.add(d); + } // No authoritative figures (a qualitative turn / no-tools path) — nothing to // grade against. The prompt-level grounding rule remains the backstop. if (authoritative.size === 0) return []; const findings: UnverifiedCoachNumber[] = []; - for (const { value, raw } of extractNumbers(prose)) { + for (const { value, raw } of extractClaimableNumbers(prose)) { if (isStructural(value, raw)) continue; if (isGrounded(value, authoritative)) continue; findings.push({ value, source: raw.slice(0, 32) });