From 92243997f6f9770c9b9ebd394966c4d005b603a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 26 Jun 2026 23:52:18 +0200 Subject: [PATCH] fix(coach): run the prose number-verifier on the no-tools path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The post-hoc numeric verifier only graded the tool path, so local / no-tools providers got no grounding net. Grade the no-tools prose against the snapshot the model was shown — its structured sections, which already carry the correlations-snapshot block — and flag any cited number that traces to no snapshot leaf. Non-blocking: the count is annotated under coach.prose.number_unverified and the unverified token soft-stripped, never a stream-killer. Only runs when the full figures were delivered this turn; a cheap follow-up has no fresh set to grade and no-ops. Tool-path behaviour and tolerance unchanged. --- src/app/api/insights/chat/route.ts | 46 +++++++++--- .../coach-prose-grounding-no-tools.test.ts | 71 +++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 src/lib/ai/coach/__tests__/coach-prose-grounding-no-tools.test.ts diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index 27b46004f..873ed71d9 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -535,6 +535,15 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`; // v1.21.0 (P6) — the present tool-result payloads this turn, for the post-hoc // prose number-verifier. Empty on the no-tools path. let toolResultPayloads: unknown[] = []; + // v1.21.2 (A8) — no-tools/local-provider parity for the prose number-verifier. + // The tool path grades prose against the figures the tools returned; the + // no-tools path has no tools, so the authoritative set is the SNAPSHOT the + // model was actually shown this turn — `snapshot.sections`, the structured + // record `snapshotJson` is serialised from, which already carries the + // correlations-snapshot block. Populated only when the full figures were + // delivered this turn (`includeFullSnapshot`); on a cheap follow-up the block + // was not re-sent, so there is no fresh authoritative set to grade against. + let noToolsSnapshotPayloads: unknown[] = []; let totalTokensSpent: number; // v1.21.0 (F3) — cached-input tokens to subtract at reconcile (prompt-cached // input the user did not re-pay for must not be billed to the daily meter). @@ -614,6 +623,14 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch workingProviderType = fallback.workingProvider.providerType; totalTokensSpent = result.tokensUsed ?? 0; cachedTokensSpent = result.cachedInputTokens ?? 0; + // v1.21.2 (A8) — the no-tools path showed the model the full SNAPSHOT only + // when `includeFullSnapshot` was set; otherwise it shipped the + // grounded-elsewhere pointer with no fresh figures, so there is nothing to + // grade. When figures WERE delivered, the authoritative set is the + // structured snapshot record (incl. the correlations block). + if (includeFullSnapshot) { + noToolsSnapshotPayloads = [snapshot.sections]; + } } } catch (err) { // The provider chain failed outright — no tokens were billed, so refund @@ -737,18 +754,27 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch }); } - // v1.21.0 (P6 / C2-5) — post-hoc numeric verifier on the Coach prose. On the - // tool path, cross-check every number the model cited against the figures the - // tools actually returned this turn; an unmatched number (transcription / - // paraphrase drift) is soft-stripped to "[unverified]" and annotated. Cheap, - // non-blocking, and a no-op when no tool returned figures (a qualitative turn - // or the no-tools path) — the prompt-level grounding rule remains the - // backstop there, exactly like the briefing's "no signals → skip". A blocked - // turn already carries canned fallback prose, so skip it. - if (!outbound.block && toolResultPayloads.length > 0) { + // v1.21.0 (P6 / C2-5) — post-hoc numeric verifier on the Coach prose. Cross- + // check every number the model cited against this turn's authoritative figure + // set; an unmatched number (transcription / paraphrase drift) is soft-stripped + // to "[unverified]" and annotated. Cheap, non-blocking, and a no-op when there + // is no authoritative set — the prompt-level grounding rule remains the + // backstop, exactly like the briefing's "no signals → skip". A blocked turn + // already carries canned fallback prose, so skip it. + // + // v1.21.2 (A8) — the authoritative set is the figures the tools returned on the + // tool path, and the SNAPSHOT the model was shown on the no-tools/local path + // (`snapshot.sections`, which already carries the correlations-snapshot block). + // Exactly one is populated per turn; the grading, tolerance, and exemptions are + // identical, so a number the model invents is flagged the same way on both. + const authoritativePayloads = + toolResultPayloads.length > 0 + ? toolResultPayloads + : noToolsSnapshotPayloads; + if (!outbound.block && authoritativePayloads.length > 0) { const unverified = findUnverifiedCoachNumbers( replyText, - toolResultPayloads, + authoritativePayloads, ); if (unverified.length > 0) { const { prose: corrected, stripped } = stripUnverifiedNumbers( diff --git a/src/lib/ai/coach/__tests__/coach-prose-grounding-no-tools.test.ts b/src/lib/ai/coach/__tests__/coach-prose-grounding-no-tools.test.ts new file mode 100644 index 000000000..e3c5ff532 --- /dev/null +++ b/src/lib/ai/coach/__tests__/coach-prose-grounding-no-tools.test.ts @@ -0,0 +1,71 @@ +/** + * v1.21.2 (A8) — no-tools/local-provider parity for the prose number-verifier. + * + * On the no-tools path the Coach has no tools, so the authoritative figure set + * is the SNAPSHOT the model was shown — the structured `snapshot.sections` + * record `snapshotJson` is serialised from, which already carries the + * correlations-snapshot block. This proves the SAME verifier grades prose + * against that record: a number present in the snapshot (including a + * correlations-block leaf) is not flagged; a number absent from it is flagged + * exactly as on the tool path. + */ +import { describe, expect, it } from "vitest"; + +import { findUnverifiedCoachNumbers } from "@/lib/ai/coach/coach-prose-grounding"; + +/** + * A representative `snapshot.sections` record: a derived block plus the + * correlations-snapshot block (a `CorrelationsSnapshotBlock`) the no-tools path + * folds in. The driver's numeric leaves (`r`, `n`, `pairsTested`, `windowDays`) + * are authoritative figures the model may legitimately cite. + */ +const snapshotSections: Record = { + bloodPressure: { + aggregate: { avgSys30: 128, avgDia30: 82 }, + }, + correlations: { + drivers: [ + { + behaviour: "sleep duration", + outcome: "resting heart rate", + direction: "lower", + lagDays: 1, + n: 24, + r: 0.62, + note: "More sleep tracked with a lower next-day resting heart rate.", + }, + ], + pairsTested: 18, + windowDays: 90, + }, +}; + +// The route passes the structured record as a single authoritative payload. +const noToolsPayloads: unknown[] = [snapshotSections]; + +describe("no-tools verifier parity", () => { + it("does not flag a number that traces to a snapshot leaf", () => { + const prose = "Your systolic averaged 128 and diastolic 82 this month."; + expect(findUnverifiedCoachNumbers(prose, noToolsPayloads)).toEqual([]); + }); + + it("does not flag a number that traces to the correlations-snapshot block", () => { + // r = 0.62 is a leaf of the correlations block folded into the snapshot. + const prose = "Sleep and resting heart rate moved together (r = 0.62)."; + expect(findUnverifiedCoachNumbers(prose, noToolsPayloads)).toEqual([]); + }); + + it("flags a number the snapshot never carried", () => { + const prose = "Your systolic averaged about 138 recently."; + const findings = findUnverifiedCoachNumbers(prose, noToolsPayloads); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(138); + }); + + it("no-ops when the snapshot was not delivered this turn", () => { + // Cheap follow-up: the route leaves the no-tools payload set empty, so even + // an invented figure is not graded — the prompt-level rule is the backstop. + const prose = "Your systolic averaged 138 recently."; + expect(findUnverifiedCoachNumbers(prose, [])).toEqual([]); + }); +});