From d657f798d09ffc641004c3692e223582c8004faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 9 May 2026 21:47:37 +0200 Subject: [PATCH] feat(ai): enforce citation-from-data on every recommendation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase C1 (v1.4.15) — citation-from-data hard requirement. Marc, verbatim 2026-05-09: "es muss sich halt irgendwie stützen auf medizinische Dinge." ("It must ground on medical facts / user data.") Builds on the strict response schema (`src/lib/ai/schema.ts`) and schema-enforced wrapper (`src/lib/ai/generate-insight.ts`) — which were swept into sibling commit 5510ed5 by a parallel-agent index race (documented across A2 / A4 / B1-B6 / C2 / C3 of the v1.4.15 marathon; diff is correct on main, message scope misleading). This commit adds a focused 11-test invariant suite that nails down the contract end-to-end: 1. Schema-level: recommendation without metricSource fails parse; empty metricSource.summary fails; missing type or timeRange fails. 2. Cross-check: findUncitedRecommendations() flags any recommendation whose (type, timeRange) does not appear in citations[]. Partial match (type alone) is rejected — both fields must align. 3. Wrapper end-to-end: orphan-citation triggers retry; second attempt either fixes the citation OR drops the recommendation (both valid per the retry-correction message; model is explicitly told an empty recommendations[] is acceptable). The two-pronged guard (schema + cross-check) ensures the model cannot fabricate a recommendation that points at "nothing" AND cannot cite data it hasn't surfaced — closing the v1.4.16 medical-grounding work's prerequisite. Tests: +11 (988 → 999 unit). All AI tests green (67 / 67). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ai/__tests__/citation-enforcement.test.ts | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 src/lib/ai/__tests__/citation-enforcement.test.ts diff --git a/src/lib/ai/__tests__/citation-enforcement.test.ts b/src/lib/ai/__tests__/citation-enforcement.test.ts new file mode 100644 index 000000000..50f27145c --- /dev/null +++ b/src/lib/ai/__tests__/citation-enforcement.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect } from "vitest"; +import { + aiInsightResponseSchema, + findUncitedRecommendations, + metricSourceSchema, + aiRecommendationSchema, + type AIInsightResponse, +} from "../schema"; +import { generateInsight } from "../generate-insight"; +import { MockAIProvider } from "../mock-client"; + +/** + * Phase C1 — citation-from-data hard requirement. + * + * Marc, verbatim 2026-05-09: "es muss sich halt irgendwie stützen auf + * medizinische Dinge." ("It must ground on medical facts / user data.") + * + * Operational meaning: + * + * 1. Every recommendation MUST carry a `metricSource` with all three + * required fields populated (`type`, `timeRange`, `summary`). + * Empty / missing → schema parse fails. The model cannot + * fabricate a recommendation that points at "nothing". + * + * 2. Every recommendation's `metricSource` MUST be backed by an + * entry in `citations[]` (matching `type` + `timeRange`). + * Cross-check enforced by `findUncitedRecommendations` and run + * by the wrapper after schema parse. + * + * 3. Two separate failure modes (missing metricSource vs. missing + * citation) both surface as zod / wrapper errors so the route + * can return 422 in either case. + */ + +const baseMetricSource = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const baseCitation = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const baseValid: AIInsightResponse = { + summary: "BP runs slightly above your 90-day median.", + recommendations: [ + { + id: "rec-1", + text: "Discuss home BP log with your physician.", + severity: "important", + metricSource: baseMetricSource, + }, + ], + citations: [baseCitation], + warnings: [], +}; + +describe("citation-from-data — schema-level enforcement", () => { + it("recommendation without metricSource fails parse", () => { + const broken = { + ...baseValid, + recommendations: [ + { + id: "rec-1", + text: "Walk more.", + severity: "suggestion", + // metricSource missing entirely + }, + ], + }; + const result = aiRecommendationSchema.safeParse(broken.recommendations[0]); + expect(result.success).toBe(false); + }); + + it("recommendation with empty metricSource.summary fails parse", () => { + const broken = { + ...baseValid, + recommendations: [ + { + id: "rec-1", + text: "Walk more.", + severity: "suggestion", + metricSource: { type: "activity", timeRange: "last7days", summary: "" }, + }, + ], + }; + const result = aiInsightResponseSchema.safeParse(broken); + expect(result.success).toBe(false); + }); + + it("recommendation with missing metricSource.type fails parse", () => { + const broken = { + ...baseValid, + recommendations: [ + { + id: "rec-1", + text: "X", + severity: "info", + metricSource: { timeRange: "last7days", summary: "data" }, + }, + ], + }; + const result = aiInsightResponseSchema.safeParse(broken); + expect(result.success).toBe(false); + }); + + it("recommendation with missing metricSource.timeRange fails parse", () => { + const result = aiInsightResponseSchema.safeParse({ + ...baseValid, + recommendations: [ + { + id: "rec-1", + text: "X", + severity: "info", + metricSource: { type: "bloodPressure", summary: "data" }, + }, + ], + }); + expect(result.success).toBe(false); + }); + + it("metricSource alone (without recommendation wrapper) — empty summary still fails", () => { + const result = metricSourceSchema.safeParse({ + type: "x", + timeRange: "y", + summary: "", + }); + expect(result.success).toBe(false); + }); + + it("well-formed citation gets through", () => { + const result = aiInsightResponseSchema.safeParse(baseValid); + expect(result.success).toBe(true); + }); +}); + +describe("citation-from-data — cross-check enforcement", () => { + it("recommendation citing a (type, timeRange) absent from citations[] is flagged", () => { + const orphan: AIInsightResponse = { + ...baseValid, + recommendations: [ + { + id: "rec-orphan", + text: "Cut sodium.", + severity: "suggestion", + metricSource: { + type: "diet", + timeRange: "last7days", + summary: "no diet logging available", + }, + }, + ], + citations: [baseCitation], // diet citation missing + }; + const missing = findUncitedRecommendations(orphan); + expect(missing).toHaveLength(1); + expect(missing[0].recommendationId).toBe("rec-orphan"); + expect(missing[0].missing).toEqual({ type: "diet", timeRange: "last7days" }); + }); + + it("partial-key match on type alone is NOT enough — timeRange must also match", () => { + const orphan: AIInsightResponse = { + ...baseValid, + recommendations: [ + { + id: "rec-mismatch", + text: "X", + severity: "info", + metricSource: { + type: "bloodPressure", + timeRange: "last30days", // citation only covers last7days + summary: "avg over 30 days", + }, + }, + ], + citations: [baseCitation], + }; + const missing = findUncitedRecommendations(orphan); + expect(missing).toHaveLength(1); + }); + + it("multiple recommendations sharing the same citation pass once that one citation is present", () => { + const ok: AIInsightResponse = { + ...baseValid, + recommendations: [ + baseValid.recommendations[0], + { + id: "rec-2", + text: "Continue daily logging.", + severity: "info", + metricSource: { ...baseMetricSource }, + }, + ], + citations: [baseCitation], + }; + const missing = findUncitedRecommendations(ok); + expect(missing).toEqual([]); + }); +}); + +describe("citation-from-data — wrapper end-to-end", () => { + it("uncited recommendation triggers retry; second try with proper citation passes", async () => { + const orphan = { + ...baseValid, + recommendations: [ + { + id: "rec-orphan", + text: "Cut sodium.", + severity: "suggestion", + metricSource: { + type: "diet", + timeRange: "last7days", + summary: "no diet logging available", + }, + }, + ], + citations: [], + }; + const provider = new MockAIProvider({ + responses: [JSON.stringify(orphan), JSON.stringify(baseValid)], + }); + const outcome = await generateInsight(provider, { + systemPrompt: "system", + userPrompt: "user", + }); + expect(outcome.retried).toBe(true); + expect(outcome.parsed.recommendations[0].id).toBe("rec-1"); + }); + + it("retry-attempt that drops the recommendation rather than fixing the citation also passes", async () => { + const orphan = { + ...baseValid, + recommendations: [ + { + id: "rec-orphan", + text: "Cut sodium.", + severity: "suggestion", + metricSource: { + type: "diet", + timeRange: "last7days", + summary: "no diet logging available", + }, + }, + ], + citations: [], + }; + // Second attempt — model decides to drop the unsupported recommendation + // entirely. This is the explicitly-allowed "OMIT it" path from the + // retry-correction message. + const droppedRecs = { + summary: "Insufficient data for actionable recommendations.", + recommendations: [], + citations: [], + warnings: [], + }; + const provider = new MockAIProvider({ + responses: [JSON.stringify(orphan), JSON.stringify(droppedRecs)], + }); + const outcome = await generateInsight(provider, { + systemPrompt: "system", + userPrompt: "user", + }); + expect(outcome.retried).toBe(true); + expect(outcome.parsed.recommendations).toHaveLength(0); + }); +});