diff --git a/src/lib/ai/__tests__/rationale-wrapper.test.ts b/src/lib/ai/__tests__/rationale-wrapper.test.ts new file mode 100644 index 000000000..fba454dd6 --- /dev/null +++ b/src/lib/ai/__tests__/rationale-wrapper.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { generateInsight } from "../generate-insight"; +import { MockAIProvider } from "../mock-client"; + +const annotateMock = vi.fn(); + +vi.mock("@/lib/logging/context", () => ({ + annotate: (fields: { meta?: Record }) => + annotateMock(fields), + getEvent: () => null, +})); + +/** + * v1.4.16 phase B5c — wrapper-level rationale handling. + * + * 1. The corrective retry message lists the rationale fields so a + * first-attempt response without rationale gets a targeted + * reprompt (instead of the model guessing what's wrong). + * 2. The wrapper's post-validation `annotate()` carries a + * `ai_rationale_coverage_*` breakdown so the admin AI quality + * dashboard can chart per-payload coverage of the new field. + * Mirrors the citation-coverage annotations from B5a. + */ + +const baseRationale = { + dataWindow: "last7days" as const, + comparedTo: "your 90-day median (122/78)", + deviation: "+16/+8 mmHg above baseline over 9 of 9 readings", +}; + +const baseMetricSource = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const baseCitation = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const validResponse = { + summary: "BP runs slightly above your 90-day median.", + recommendations: [ + { + id: "rec-1", + text: "Discuss home BP log with your physician.", + severity: "important", + metricSource: baseMetricSource, + rationale: baseRationale, + }, + ], + citations: [baseCitation], + warnings: [], +}; + +describe("generateInsight() — corrective retry mentions rationale", () => { + beforeEach(() => { + annotateMock.mockReset(); + }); + + it("retries with a corrective message naming dataWindow / comparedTo / deviation when rationale missing", async () => { + const recWithoutRationale = { + id: "rec-1", + text: "Walk more", + severity: "suggestion", + metricSource: baseMetricSource, + // rationale missing + }; + const broken = { + ...validResponse, + recommendations: [recWithoutRationale], + }; + const provider = new MockAIProvider({ + responses: [JSON.stringify(broken), JSON.stringify(validResponse)], + }); + const outcome = await generateInsight(provider, { + systemPrompt: "sys", + userPrompt: "Original user prompt.", + }); + expect(outcome.attempts).toBe(2); + expect(outcome.retried).toBe(true); + const retryUserPrompt = provider.calls[1].userPrompt; + expect(retryUserPrompt).toContain("Original user prompt."); + expect(retryUserPrompt).toContain("rationale"); + expect(retryUserPrompt).toContain("dataWindow"); + expect(retryUserPrompt).toContain("comparedTo"); + expect(retryUserPrompt).toContain("deviation"); + }); + + it("retries when first attempt has empty rationale.deviation", async () => { + const recWithEmptyDeviation = { + id: "rec-1", + text: "Walk more", + severity: "suggestion", + metricSource: baseMetricSource, + rationale: { + dataWindow: "last7days", + comparedTo: "your 90-day median", + deviation: "", // empty — should fail + }, + }; + const broken = { + ...validResponse, + recommendations: [recWithEmptyDeviation], + }; + const provider = new MockAIProvider({ + responses: [JSON.stringify(broken), JSON.stringify(validResponse)], + }); + const outcome = await generateInsight(provider, { + systemPrompt: "sys", + userPrompt: "user", + }); + expect(outcome.attempts).toBe(2); + expect(outcome.retried).toBe(true); + }); +}); + +describe("generateInsight() — rationale-coverage annotation", () => { + beforeEach(() => { + annotateMock.mockReset(); + }); + + it("annotates ai_rationale_coverage_* fields on a successful generation", async () => { + const provider = new MockAIProvider({ + responses: JSON.stringify(validResponse), + }); + await generateInsight(provider, { + systemPrompt: "sys", + userPrompt: "user", + }); + expect(annotateMock).toHaveBeenCalled(); + const meta = annotateMock.mock.calls[0][0]?.meta as Record; + expect(meta).toMatchObject({ + ai_total_recommendations: 1, + ai_rationale_total_recommendations: 1, + ai_rationale_with_rationale: 1, + ai_rationale_missing_recommendation_ids: [], + }); + }); + + it("rationale-coverage = total recommendations when every rec carries one", async () => { + const twoRecs = { + ...validResponse, + recommendations: [ + validResponse.recommendations[0], + { + id: "rec-2", + text: "Continue daily logging.", + severity: "info", + metricSource: baseMetricSource, + rationale: baseRationale, + }, + ], + }; + const provider = new MockAIProvider({ + responses: JSON.stringify(twoRecs), + }); + await generateInsight(provider, { + systemPrompt: "sys", + userPrompt: "user", + }); + const meta = annotateMock.mock.calls[0][0]?.meta as Record; + expect(meta).toMatchObject({ + ai_rationale_total_recommendations: 2, + ai_rationale_with_rationale: 2, + ai_rationale_missing_recommendation_ids: [], + }); + }); +}); diff --git a/src/lib/ai/generate-insight.ts b/src/lib/ai/generate-insight.ts index 8c3abe9b5..c107399f4 100644 --- a/src/lib/ai/generate-insight.ts +++ b/src/lib/ai/generate-insight.ts @@ -2,6 +2,7 @@ import type { z } from "zod/v4"; import type { AIProvider, CompletionParams, CompletionResult } from "./types"; import { aiInsightResponseSchema, + findRecommendationsMissingRationale, findUncitedRecommendations, InsightSchemaError, type AIInsightResponse, @@ -56,15 +57,23 @@ include prose, markdown fences, or commentary outside the JSON. Required top-level fields: - summary: string (2-3 sentences) - recommendations: array of objects with - { id: string, text: string, severity: "info" | "suggestion" | "important" | "urgent", - metricSource: { type: string, timeRange: string, summary: string, n?: number } } + { id: string, + text: string, + severity: "info" | "suggestion" | "important" | "urgent", + metricSource: { type: string, timeRange: string, summary: string, n?: number }, + rationale: { dataWindow: "last7days" | "last30days" | "last90days" | "allTime", + comparedTo: non-empty string, + deviation: non-empty string } } - citations: array of objects with { type: string, timeRange: string, summary: string } - warnings: array of objects with { topic: string, message: string, severity?: same enum } Every recommendation's metricSource MUST also appear in citations[] -(same type + timeRange). If you cannot ground a recommendation in -user data from the snapshot you were given, OMIT it. An empty -recommendations[] is acceptable. +(same type + timeRange). Every recommendation MUST carry a +rationale object — dataWindow, comparedTo, and deviation are all +required and rationale.dataWindow MUST equal metricSource.timeRange. +If you cannot ground a recommendation in user data from the +snapshot you were given, OMIT it. An empty recommendations[] is +acceptable. `; } @@ -136,9 +145,17 @@ async function tryOnce( * Forward the citation-coverage breakdown into the active Wide-Event * via `annotate()` (no-op outside a request context). Keys are * snake_case to match the rest of the AI Wide-Event vocabulary. + * + * v1.4.16 phase B5c — additionally emit `ai_rationale_*` keys so the + * admin AI quality dashboard can chart rationale-coverage per + * payload alongside citation-coverage. Rationale is required by the + * strict schema; the missing-ids list will only ever populate when + * a legacy payload sneaks through `.passthrough()` — useful as an + * observability tripwire for the migration window. */ function annotateCitationCoverage(parsed: AIInsightResponse): void { const coverage = computeCitationCoverage(parsed); + const rationaleMissing = findRecommendationsMissingRationale(parsed); annotate({ meta: { ai_total_recommendations: coverage.totalRecommendations, @@ -147,6 +164,10 @@ function annotateCitationCoverage(parsed: AIInsightResponse): void { coverage.citedNormativeRecommendations, ai_uncited_normative_recommendation_ids: coverage.uncitedNormativeRecommendationIds, + ai_rationale_total_recommendations: coverage.totalRecommendations, + ai_rationale_with_rationale: + coverage.totalRecommendations - rationaleMissing.length, + ai_rationale_missing_recommendation_ids: rationaleMissing, }, }); }