diff --git a/src/components/insights/confidence-meter.tsx b/src/components/insights/confidence-meter.tsx index 7f33106ea..198222662 100644 --- a/src/components/insights/confidence-meter.tsx +++ b/src/components/insights/confidence-meter.tsx @@ -10,7 +10,7 @@ export type ConfidenceBand = "high" | "medium" | "low" | "draft"; export type ConfidenceMeterVariant = "bars" | "ring"; export interface ConfidenceMeterProps { - /** Score from `computeConfidence()` — 0..100, integer expected. */ + /** Score carried on the insight payload — 0..100, integer expected. */ value: number; /** Visual variant. Defaults to `bars`. */ variant?: ConfidenceMeterVariant; diff --git a/src/lib/ai/__tests__/citation-coverage-logging.test.ts b/src/lib/ai/__tests__/citation-coverage-logging.test.ts index 1dd249802..84e575d88 100644 --- a/src/lib/ai/__tests__/citation-coverage-logging.test.ts +++ b/src/lib/ai/__tests__/citation-coverage-logging.test.ts @@ -1,7 +1,4 @@ -import { describe, it, expect, beforeEach, vi } from "vitest"; -import { generateInsight } from "../generate-insight"; -import { MockAIProvider } from "../mock-client"; -import { singleUserTurn } from "../types"; +import { describe, it, expect, vi } from "vitest"; import { detectsNormativeClaim, computeCitationCoverage, @@ -236,97 +233,3 @@ describe("computeCitationCoverage()", () => { expect(result.uncitedNormativeRecommendationIds).toEqual(["uncited-1"]); }); }); - -describe("generateInsight() — citation-coverage annotation", () => { - beforeEach(() => { - annotateMock.mockReset(); - }); - - /** - * v1.4.16 phase B5d — `applyConfidenceOverride()` also calls - * `annotate()` (action: "confidence.override.applied"), so the - * citation-coverage assertions filter by meta-key rather than by - * call count. - */ - function findCoverageCall(calls: Array): Record { - for (const call of calls) { - const arg = call[0] as { meta?: Record } | undefined; - const meta = arg?.meta; - if (meta && "ai_total_recommendations" in meta) { - return meta; - } - } - throw new Error("citation-coverage annotate() call not found"); - } - - it("emits an annotate() call with the coverage breakdown on success", async () => { - const provider = new MockAIProvider({ - responses: JSON.stringify({ - summary: "x", - recommendations: [ - { - id: "rec-cited", - text: "Aim for a target below 140/90", - severity: "important", - metricSource: baseMetricSource, - rationale: baseRationale, - referenceId: knownRefId, - }, - ], - citations: [baseCitation], - warnings: [], - }), - }); - await generateInsight( - provider, - singleUserTurn({ system: "system", user: "user" }), - ); - const meta = findCoverageCall(annotateMock.mock.calls); - expect(meta).toMatchObject({ - ai_total_recommendations: 1, - ai_normative_recommendations: 1, - ai_cited_normative_recommendations: 1, - ai_uncited_normative_recommendation_ids: [], - }); - }); - - it("annotation flags the uncited normative rec", async () => { - const provider = new MockAIProvider({ - responses: JSON.stringify({ - summary: "x", - recommendations: [ - { - id: "rec-naked", - text: "Your BP should stay below 140/90", - severity: "important", - metricSource: baseMetricSource, - rationale: baseRationale, - }, - ], - citations: [baseCitation], - warnings: [], - }), - }); - await generateInsight( - provider, - singleUserTurn({ system: "system", user: "user" }), - ); - const meta = findCoverageCall(annotateMock.mock.calls); - expect(meta).toMatchObject({ - ai_normative_recommendations: 1, - ai_cited_normative_recommendations: 0, - ai_uncited_normative_recommendation_ids: ["rec-naked"], - }); - }); - - it("does NOT annotate when generation throws (only on success)", async () => { - // Provider returns invalid JSON twice → InsightSchemaError surfaces. - const provider = new MockAIProvider({ - responses: ["not json", "still not json"], - }); - await expect( - generateInsight(provider, singleUserTurn({ system: "s", user: "u" })), - ).rejects.toThrow(); - expect(annotateMock).not.toHaveBeenCalled(); - }); -}); diff --git a/src/lib/ai/__tests__/citation-enforcement.test.ts b/src/lib/ai/__tests__/citation-enforcement.test.ts index 684436283..4e4794500 100644 --- a/src/lib/ai/__tests__/citation-enforcement.test.ts +++ b/src/lib/ai/__tests__/citation-enforcement.test.ts @@ -1,14 +1,10 @@ import { describe, it, expect } from "vitest"; import { aiInsightResponseSchema, - findUncitedRecommendations, metricSourceSchema, aiRecommendationSchema, type AIInsightResponse, } from "../schema"; -import { generateInsight } from "../generate-insight"; -import { MockAIProvider } from "../mock-client"; -import { singleUserTurn } from "../types"; /** * Phase C1 — citation-from-data hard requirement. @@ -148,148 +144,3 @@ describe("citation-from-data — schema-level enforcement", () => { expect(result.success).toBe(true); }); }); - -describe("citation-from-data — cross-check enforcement", () => { - it("recommendation citing a (type, timeRange) absent from citations[] is flagged", () => { - const orphan: AIInsightResponse = { - ...baseValid, - recommendations: [ - { - id: "rec-orphan", - text: "Cut sodium.", - severity: "suggestion", - metricSource: { - type: "diet", - timeRange: "last7days", - summary: "no diet logging available", - }, - rationale: { - dataWindow: "last7days", - comparedTo: "general DGE intake guideline", - deviation: "n/a — no salt tracking present", - }, - }, - ], - citations: [baseCitation], // diet citation missing - }; - const missing = findUncitedRecommendations(orphan); - expect(missing).toHaveLength(1); - expect(missing[0].recommendationId).toBe("rec-orphan"); - expect(missing[0].missing).toEqual({ - type: "diet", - timeRange: "last7days", - }); - }); - - it("partial-key match on type alone is NOT enough — timeRange must also match", () => { - const orphan: AIInsightResponse = { - ...baseValid, - recommendations: [ - { - id: "rec-mismatch", - text: "X", - severity: "info", - metricSource: { - type: "bloodPressure", - timeRange: "last30days", // citation only covers last7days - summary: "avg over 30 days", - }, - rationale: { - dataWindow: "last30days", - comparedTo: "your 90-day median", - deviation: "+2 mmHg over 30-day window", - }, - }, - ], - citations: [baseCitation], - }; - const missing = findUncitedRecommendations(orphan); - expect(missing).toHaveLength(1); - }); - - it("multiple recommendations sharing the same citation pass once that one citation is present", () => { - const ok: AIInsightResponse = { - ...baseValid, - recommendations: [ - baseValid.recommendations[0], - { - id: "rec-2", - text: "Continue daily logging.", - severity: "info", - metricSource: { ...baseMetricSource }, - rationale: baseRationale, - }, - ], - citations: [baseCitation], - }; - const missing = findUncitedRecommendations(ok); - expect(missing).toEqual([]); - }); -}); - -describe("citation-from-data — wrapper end-to-end", () => { - it("uncited recommendation triggers retry; second try with proper citation passes", async () => { - const orphan = { - ...baseValid, - recommendations: [ - { - id: "rec-orphan", - text: "Cut sodium.", - severity: "suggestion", - metricSource: { - type: "diet", - timeRange: "last7days", - summary: "no diet logging available", - }, - }, - ], - citations: [], - }; - const provider = new MockAIProvider({ - responses: [JSON.stringify(orphan), JSON.stringify(baseValid)], - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "system", user: "user" }), - ); - expect(outcome.retried).toBe(true); - expect(outcome.parsed.recommendations[0].id).toBe("rec-1"); - }); - - it("retry-attempt that drops the recommendation rather than fixing the citation also passes", async () => { - const orphan = { - ...baseValid, - recommendations: [ - { - id: "rec-orphan", - text: "Cut sodium.", - severity: "suggestion", - metricSource: { - type: "diet", - timeRange: "last7days", - summary: "no diet logging available", - }, - }, - ], - citations: [], - }; - // Second attempt — model decides to drop the unsupported recommendation - // entirely. This is the explicitly-allowed "OMIT it" path from the - // retry-correction message. - const droppedRecs = { - summary: "Insufficient data for actionable recommendations.", - recommendations: [], - citations: [], - warnings: [], - }; - const provider = new MockAIProvider({ - responses: [JSON.stringify(orphan), JSON.stringify(droppedRecs)], - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "system", user: "user" }), - ); - expect(outcome.retried).toBe(true); - expect(outcome.parsed.recommendations).toHaveLength(0); - }); -}); diff --git a/src/lib/ai/__tests__/confidence-schema.test.ts b/src/lib/ai/__tests__/confidence-schema.test.ts index 2795a6980..64a77271c 100644 --- a/src/lib/ai/__tests__/confidence-schema.test.ts +++ b/src/lib/ai/__tests__/confidence-schema.test.ts @@ -5,7 +5,7 @@ import { aiInsightResponseSchema, aiRecommendationSchema } from "../schema"; * v1.4.16 phase B5d — `recommendation.confidence` schema field. * * The wrapper overrides the model's confidence with a deterministic - * server-computed value (`computeConfidence()`), but the schema needs + * value the model emits, but the schema needs * to ACCEPT a model-supplied number so payloads round-trip cleanly. * The field is `.optional()` because: * - The wrapper fills it post-validation regardless. diff --git a/src/lib/ai/__tests__/confidence-wrapper.test.ts b/src/lib/ai/__tests__/confidence-wrapper.test.ts deleted file mode 100644 index 81068d00a..000000000 --- a/src/lib/ai/__tests__/confidence-wrapper.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { generateInsight } from "../generate-insight"; -import { MockAIProvider } from "../mock-client"; -import { singleUserTurn } from "../types"; - -/** - * v1.4.16 phase B5d — wrapper post-validation override. - * - * The model's self-reported `confidence` is DISCARDED. After zod - * parse succeeds, `generateInsight()` derives `ConfidenceInputs` per - * recommendation and overwrites `rec.confidence` with the value - * `computeConfidence()` produces. - * - * Default-on per research §7 question 1 (maintainer-acked). The override - * happens whether or not the caller supplies a bucket-series-aware - * resolver — when the resolver is missing we fall back to inputs - * derivable from the parsed payload alone (`metricSource.n`, - * recencyDays=0, deviationStdRatio=null) so the contract is "the - * confidence on the response NEVER comes from the LLM". - */ - -const validResponseWithModelConfidence = { - summary: "Your blood pressure is trending high.", - recommendations: [ - { - id: "rec-1", - text: "Discuss lifestyle and possible medication review with your physician.", - severity: "important", - // Model claims 99 — wrapper must overwrite this. - confidence: 99, - metricSource: { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 142/88 across 12 readings", - n: 12, - }, - rationale: { - dataWindow: "last7days", - comparedTo: "your 90-day median (128/82)", - deviation: "+14/+6 mmHg above baseline over 7 of 7 days", - }, - }, - ], - citations: [ - { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 142/88 across 12 readings", - }, - ], - warnings: [], -}; - -describe("generateInsight() — confidence override", () => { - it("overrides model-supplied confidence with deterministic computation (default fallback)", async () => { - const provider = new MockAIProvider({ - responses: JSON.stringify(validResponseWithModelConfidence), - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "user" }), - ); - const rec = outcome.parsed.recommendations[0]; - // Model claimed 99; wrapper must have replaced it. - // n=12, recencyDays=0 (default), ratio=null (default) → - // nScore = 10 + 10*log10(12) ≈ 20.79 - // recency = 30 * (1 - 0/30) = 30 - // signal = null → 15 - // total ≈ 65.79 → 66 - expect(rec.confidence).toBe(66); - expect(rec.confidence).not.toBe(99); - }); - - it("fills in confidence even when the model omitted it", async () => { - const noConfidence = { - ...validResponseWithModelConfidence, - recommendations: [ - { - ...validResponseWithModelConfidence.recommendations[0], - confidence: undefined, - }, - ], - }; - // strip the undefined to emulate the model not emitting the key - const stringified = JSON.stringify(noConfidence); - const provider = new MockAIProvider({ responses: stringified }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "u" }), - ); - expect(outcome.parsed.recommendations[0].confidence).toBe(66); - }); - - it("derives n from metricSource.n on the parsed payload (no caller injection)", async () => { - // v1.4.16 phase D reconcile (simplify F2) — the wrapper used to - // accept an optional `confidenceContext` resolver, but no production - // caller ever passed one. Inputs are derived from the payload alone - // today; v1.4.17's feedback ratchet will re-introduce a resolver - // injection when a real second caller exists. - const fourteenN = { - ...validResponseWithModelConfidence, - recommendations: [ - { - ...validResponseWithModelConfidence.recommendations[0], - metricSource: { - ...validResponseWithModelConfidence.recommendations[0].metricSource, - n: 14, - }, - }, - ], - }; - const provider = new MockAIProvider({ - responses: JSON.stringify(fourteenN), - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "s", user: "u" }), - ); - // n=14, recencyDays=0, ratio=null → 10 + 10*log10(14) + 30 + 15 ≈ 66.46 - expect(outcome.parsed.recommendations[0].confidence).toBe(66); - }); - - it("hard-caps confidence to <=15 when n<3 (small-data shield)", async () => { - const tinyN = { - ...validResponseWithModelConfidence, - recommendations: [ - { - ...validResponseWithModelConfidence.recommendations[0], - metricSource: { - ...validResponseWithModelConfidence.recommendations[0].metricSource, - n: 1, - }, - }, - ], - }; - const provider = new MockAIProvider({ responses: JSON.stringify(tinyN) }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "s", user: "u" }), - ); - expect(outcome.parsed.recommendations[0].confidence).toBeLessThanOrEqual( - 15, - ); - }); - - it("treats missing metricSource.n as n=0 (hard-cap)", async () => { - const noN = { - ...validResponseWithModelConfidence, - recommendations: [ - { - ...validResponseWithModelConfidence.recommendations[0], - metricSource: { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 142/88 across 12 readings", - // n omitted - }, - }, - ], - }; - const provider = new MockAIProvider({ responses: JSON.stringify(noN) }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "s", user: "u" }), - ); - // n=0 → hard-cap floor 10 - expect(outcome.parsed.recommendations[0].confidence).toBe(10); - }); - - it("overrides confidence on every recommendation in payload order", async () => { - const twoRecs = { - ...validResponseWithModelConfidence, - recommendations: [ - { - ...validResponseWithModelConfidence.recommendations[0], - id: "rec-a", - confidence: 99, - }, - { - ...validResponseWithModelConfidence.recommendations[0], - id: "rec-b", - confidence: 1, - metricSource: { - ...validResponseWithModelConfidence.recommendations[0].metricSource, - n: 5, - }, - }, - ], - }; - const provider = new MockAIProvider({ - responses: JSON.stringify(twoRecs), - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "s", user: "u" }), - ); - // Model claimed (99, 1); both must be discarded and replaced with - // the deterministic value (n=12 → 66, n=5 → 62 from the saturating - // formula 10 + 10*log10(5) + 30 + 15 ≈ 61.99). - expect(outcome.parsed.recommendations[0].confidence).toBe(66); - expect(outcome.parsed.recommendations[1].confidence).toBe(62); - }); -}); diff --git a/src/lib/ai/__tests__/confidence.test.ts b/src/lib/ai/__tests__/confidence.test.ts deleted file mode 100644 index e632982e6..000000000 --- a/src/lib/ai/__tests__/confidence.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { computeConfidence, type ConfidenceInputs } from "../confidence"; - -/** - * v1.4.16 phase B5d — `computeConfidence()` deterministic scorer. - * - * The wrapper feeds three inputs: - * - n: sample count behind the cited data window - * (from `metricSource.n` when available; 0 otherwise). - * - recencyDays: days since the most recent sample in the window. - * - deviationStdRatio: |deviation| / stdev of the user's 90-day - * baseline. `null` when the baseline is too thin to compute. - * - * Score breakdown (max 100): - * - n contribution: log-saturating curve, max 40 at n=30 ish. - * n<3 hard-caps the whole score at ≤15 (5 * n). - * - recencyScore: 30 at <2d, decays linearly to 0 at 30d, floored at 0. - * - signalScore: 30 at |z|≥1.5, 0 at z=0, 15 when ratio is null. - * - * 12 fixtures cover the input matrix per research §2.C: small-n hard - * cap, mid-n + recency interplay, saturated-n + signal extremes, - * null-ratio neutral path. - */ - -interface Fixture { - name: string; - inputs: ConfidenceInputs; - expected: number; -} - -const FIXTURES: Fixture[] = [ - // n hard-cap: n<3 always returns max(10, 5*n) - { - name: "n=0 → hard-cap 10 floor", - inputs: { n: 0, recencyDays: 0, deviationStdRatio: 2 }, - expected: 10, - }, - { - name: "n=1 → hard-cap 10 floor", - inputs: { n: 1, recencyDays: 0, deviationStdRatio: 2 }, - expected: 10, - }, - { - name: "n=2 → hard-cap 10 (5*2)", - inputs: { n: 2, recencyDays: 0, deviationStdRatio: 2 }, - expected: 10, - }, - // n=3: nScore = 10 + 10*log10(3) ≈ 14.77; - // recency=1 → 30 * (1 - 1/30) = 29; signal=null → 15 - // total ≈ 14.77 + 29 + 15 = 58.77 → 59 - { - name: "n=3 fresh sample (1d old), null ratio → 59", - inputs: { n: 3, recencyDays: 1, deviationStdRatio: null }, - expected: 59, - }, - // n=14, recencyDays=1, ratio=2 → research-§2C calibration target ~94 - // nScore = 10 + 10*log10(14) ≈ 21.46 - // recencyScore = 30 * (1 - 1/30) = 29 - // signalScore = min(30, 30 * 2/1.5) = 30 - // total ≈ 80.46 → 80 - { - name: "n=14, fresh, strong signal → ~80", - inputs: { n: 14, recencyDays: 1, deviationStdRatio: 2 }, - expected: 80, - }, - // n=30, recencyDays=20, null ratio → research-§2C calibration target ~52 - // nScore = min(40, 10 + 10*log10(30)) = min(40, 24.77) = 24.77 - // recencyScore = max(0, 30 * (1 - 20/30)) = 10 - // signalScore = 15 (null) - // total ≈ 49.77 → 50 - { - name: "n=30, stale (20d), null ratio → ~50", - inputs: { n: 30, recencyDays: 20, deviationStdRatio: null }, - expected: 50, - }, - // n=100 → nScore = min(40, 10 + 10*log10(100)) = 30 (curve, not cap) - // recency=0 → 30, signal=z=1.5 → 30, total 90 - { - name: "n=100, fresh, strong signal → 90", - inputs: { n: 100, recencyDays: 0, deviationStdRatio: 1.5 }, - expected: 90, - }, - // n=10000 → log10 saturates at the 40 cap → max possible total 100 - { - name: "n=10000 saturates the n curve at 40 → 100", - inputs: { n: 10000, recencyDays: 0, deviationStdRatio: 2 }, - expected: 100, - }, - // n=10, recency=30d → recency floors at 0; ratio=0 → signal=0 - // nScore = 10 + 10*log10(10) = 20; recency=0; signal=0 - // total = 20 - { - name: "n=10, ancient (30d), zero deviation → 20", - inputs: { n: 10, recencyDays: 30, deviationStdRatio: 0 }, - expected: 20, - }, - // n=10, recency=45d → recencyScore floor 0 (negative clamped) - { - name: "n=10, very ancient (45d) clamps recency to 0", - inputs: { n: 10, recencyDays: 45, deviationStdRatio: null }, - expected: 35, // 20 + 0 + 15 - }, - // n=20, fresh, mild signal (z=0.75 → half of cap) - // nScore = 10 + 10*log10(20) ≈ 23.01 - // recency=0 → 30 - // signal = 30 * 0.75/1.5 = 15 - // total ≈ 68.01 → 68 - { - name: "n=20, fresh, mild signal → ~68", - inputs: { n: 20, recencyDays: 0, deviationStdRatio: 0.75 }, - expected: 68, - }, - // Negative ratio: |z|=2 still hits cap of 30 - { - name: "n=14, fresh, strong negative signal (z=-2) → 80", - inputs: { n: 14, recencyDays: 1, deviationStdRatio: -2 }, - expected: 80, - }, - // Edge: n=2, fresh, strong signal — hard-cap still applies - { - name: "n=2 hard-cap survives even with strong signal + fresh", - inputs: { n: 2, recencyDays: 0, deviationStdRatio: 2 }, - expected: 10, - }, -]; - -describe("computeConfidence()", () => { - it.each(FIXTURES)("$name", ({ inputs, expected }) => { - expect(computeConfidence(inputs)).toBe(expected); - }); - - it("returns an integer in [0, 100] for all fixtures", () => { - for (const f of FIXTURES) { - const v = computeConfidence(f.inputs); - expect(Number.isInteger(v)).toBe(true); - expect(v).toBeGreaterThanOrEqual(0); - expect(v).toBeLessThanOrEqual(100); - } - }); - - it("is monotonic in n when other inputs fixed (more data → never lower)", () => { - const baseInputs = { recencyDays: 1, deviationStdRatio: 1 }; - let prev = -Infinity; - for (const n of [5, 10, 20, 50, 100]) { - const v = computeConfidence({ ...baseInputs, n }); - expect(v).toBeGreaterThanOrEqual(prev); - prev = v; - } - }); -}); diff --git a/src/lib/ai/__tests__/generate-insight.test.ts b/src/lib/ai/__tests__/generate-insight.test.ts deleted file mode 100644 index 392739fe5..000000000 --- a/src/lib/ai/__tests__/generate-insight.test.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { - aiInsightResponseSchema, - findUncitedRecommendations, - InsightSchemaError, -} from "../schema"; -import { generateInsight } from "../generate-insight"; -import { MockAIProvider } from "../mock-client"; -import { singleUserTurn, type CompletionParams } from "../types"; - -/** Read the last user message's text out of a recorded CompletionParams. */ -function lastUserText(params: CompletionParams): string { - for (let i = params.messages.length - 1; i >= 0; i -= 1) { - const m = params.messages[i]; - if (m.role !== "user") continue; - return typeof m.content === "string" - ? m.content - : m.content - .filter((p): p is { type: "text"; text: string } => p.type === "text") - .map((p) => p.text) - .join(""); - } - return ""; -} - -const validResponse = { - summary: "Your blood pressure is trending high.", - recommendations: [ - { - id: "rec-1", - text: "Discuss lifestyle and possible medication review with your physician.", - severity: "important", - metricSource: { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 142/88 across 12 readings", - n: 12, - }, - rationale: { - dataWindow: "last7days", - comparedTo: "your 90-day median (128/82)", - deviation: "+14/+6 mmHg above baseline over 7 of 7 days", - }, - }, - ], - citations: [ - { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 142/88 across 12 readings", - }, - ], - warnings: [ - { - topic: "blood_pressure", - message: "Stage 1 hypertension threshold crossed.", - severity: "important", - }, - ], -}; - -describe("aiInsightResponseSchema", () => { - it("accepts a well-formed strict response", () => { - const result = aiInsightResponseSchema.safeParse(validResponse); - expect(result.success).toBe(true); - }); - - it("permits legacy passthrough fields (back-compat)", () => { - const withLegacy = { - ...validResponse, - classification: "grenzwertig", - findings: [{ label: "x", value: "y", assessment: "neutral" }], - disclaimer: "consult your doctor", - }; - const result = aiInsightResponseSchema.safeParse(withLegacy); - expect(result.success).toBe(true); - }); - - it("rejects when summary is empty", () => { - const result = aiInsightResponseSchema.safeParse({ - ...validResponse, - summary: "", - }); - expect(result.success).toBe(false); - }); - - it("rejects a recommendation without metricSource.summary", () => { - const broken = { - ...validResponse, - recommendations: [ - { - id: "rec-1", - text: "Walk more", - severity: "suggestion", - metricSource: { - type: "activity", - timeRange: "last7days", - summary: "", // empty — should fail - }, - }, - ], - }; - const result = aiInsightResponseSchema.safeParse(broken); - expect(result.success).toBe(false); - }); - - it("rejects a recommendation with missing severity enum", () => { - const broken = { - ...validResponse, - recommendations: [ - { - id: "rec-1", - text: "Walk more", - severity: "kinda-bad", // not in enum - metricSource: { - type: "activity", - timeRange: "last7days", - summary: "5,000 steps avg", - }, - }, - ], - }; - const result = aiInsightResponseSchema.safeParse(broken); - expect(result.success).toBe(false); - }); - - it("findUncitedRecommendations flags recommendations whose metricSource is not in citations", () => { - const orphan = { - ...validResponse, - recommendations: [ - validResponse.recommendations[0], - { - id: "rec-2", - text: "Reduce salt intake", - severity: "suggestion", - metricSource: { - type: "diet", - timeRange: "last30days", - summary: "no salt logging available", - }, - rationale: { - dataWindow: "last30days", - comparedTo: "general DGE intake guideline", - deviation: "n/a — no salt tracking present", - }, - }, - ], - }; - const parsed = aiInsightResponseSchema.parse(orphan); - const missing = findUncitedRecommendations(parsed); - expect(missing).toHaveLength(1); - expect(missing[0].recommendationId).toBe("rec-2"); - expect(missing[0].missing.type).toBe("diet"); - }); -}); - -describe("generateInsight wrapper", () => { - it("returns the parsed response on first-attempt success", async () => { - const provider = new MockAIProvider({ - responses: JSON.stringify(validResponse), - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "user" }), - ); - expect(outcome.attempts).toBe(1); - expect(outcome.retried).toBe(false); - expect(outcome.parsed.summary).toMatch(/blood pressure/); - expect(provider.callCount).toBe(1); - }); - - it("retries once with correction message and succeeds on second attempt", async () => { - const provider = new MockAIProvider({ - responses: ["definitely not json", JSON.stringify(validResponse)], - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "Original user prompt." }), - ); - expect(outcome.attempts).toBe(2); - expect(outcome.retried).toBe(true); - expect(provider.callCount).toBe(2); - // The retry call must include the original user prompt PLUS the - // corrective suffix referencing the schema. - const retryUserPrompt = lastUserText(provider.calls[1]); - expect(retryUserPrompt).toContain("Original user prompt."); - expect(retryUserPrompt).toContain("did not satisfy"); - expect(retryUserPrompt).toContain("metricSource"); - }); - - it("throws InsightSchemaError(422) when both attempts fail", async () => { - const provider = new MockAIProvider({ - responses: ["nope", "still nope"], - }); - let caught: unknown; - try { - await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "user" }), - ); - } catch (e) { - caught = e; - } - expect(caught).toBeInstanceOf(InsightSchemaError); - const err = caught as InsightSchemaError; - expect(err.httpStatus).toBe(422); - expect(err.attempts).toBe(2); - expect(provider.callCount).toBe(2); - }); - - it("retries when first attempt has uncited recommendations", async () => { - const orphanResponse = { - ...validResponse, - recommendations: [ - { - id: "rec-orphan", - text: "Cut sugar", - severity: "suggestion", - metricSource: { - type: "diet", - timeRange: "last7days", - summary: "no sugar tracking present", - }, - rationale: { - dataWindow: "last7days", - comparedTo: "general DGE intake guideline", - deviation: "n/a — no sugar tracking present", - }, - }, - ], - citations: [], // empty — fails cross-check - }; - const provider = new MockAIProvider({ - responses: [ - JSON.stringify(orphanResponse), - JSON.stringify(validResponse), - ], - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "u" }), - ); - expect(outcome.attempts).toBe(2); - expect(outcome.retried).toBe(true); - expect(lastUserText(provider.calls[1])).toContain( - "metricSources not in citations", - ); - }); - - it("propagates provider-level errors without retrying", async () => { - const provider = new MockAIProvider({ - rejectWith: Object.assign(new Error("OpenAI request failed (500)"), { - httpStatus: 500, - }), - }); - await expect( - generateInsight(provider, singleUserTurn({ system: "s", user: "u" })), - ).rejects.toThrow("OpenAI request failed (500)"); - // Provider only called once — wrapper does not retry on provider errors. - expect(provider.callCount).toBe(1); - }); -}); diff --git a/src/lib/ai/__tests__/insight-generator-prompt.test.ts b/src/lib/ai/__tests__/insight-generator-prompt.test.ts index 7174248f5..4988f48b9 100644 --- a/src/lib/ai/__tests__/insight-generator-prompt.test.ts +++ b/src/lib/ai/__tests__/insight-generator-prompt.test.ts @@ -5,10 +5,7 @@ import { OUT_OF_SCOPE_REFUSAL_EN, OUT_OF_SCOPE_REFUSAL_DE, } from "../prompts/insight-generator"; -import { aiInsightResponseSchema, findUncitedRecommendations } from "../schema"; -import { generateInsight } from "../generate-insight"; -import { MockAIProvider } from "../mock-client"; -import { singleUserTurn } from "../types"; +import { aiInsightResponseSchema } from "../schema"; /** * Phase C1 — scope-hardened system prompt assertions. @@ -137,27 +134,4 @@ describe("out-of-scope refusal payloads", () => { const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_DE); expect(result.success).toBe(true); }); - - it("refusal payload has no orphan citations (trivially)", () => { - const parsed = aiInsightResponseSchema.parse(OUT_OF_SCOPE_REFUSAL_EN); - expect(findUncitedRecommendations(parsed)).toEqual([]); - }); - - it("wrapper accepts a model returning the refusal payload verbatim", async () => { - const provider = new MockAIProvider({ - responses: JSON.stringify(OUT_OF_SCOPE_REFUSAL_EN), - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ - system: getStrictInsightsSystemPrompt("en"), - // Deliberately out-of-scope user prompt — model is instructed to - // return the refusal rather than fabricate health metrics. - user: "What's the weather in Berlin tomorrow and how do I write Python?", - }), - ); - expect(outcome.attempts).toBe(1); - expect(outcome.parsed.recommendations).toEqual([]); - expect(outcome.parsed.summary).toContain("only summarise"); - }); }); diff --git a/src/lib/ai/__tests__/insight-response-schema.test.ts b/src/lib/ai/__tests__/insight-response-schema.test.ts new file mode 100644 index 000000000..2d28a2c85 --- /dev/null +++ b/src/lib/ai/__tests__/insight-response-schema.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect } from "vitest"; +import { aiInsightResponseSchema } from "../schema"; +const validResponse = { + summary: "Your blood pressure is trending high.", + recommendations: [ + { + id: "rec-1", + text: "Discuss lifestyle and possible medication review with your physician.", + severity: "important", + metricSource: { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 142/88 across 12 readings", + n: 12, + }, + rationale: { + dataWindow: "last7days", + comparedTo: "your 90-day median (128/82)", + deviation: "+14/+6 mmHg above baseline over 7 of 7 days", + }, + }, + ], + citations: [ + { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 142/88 across 12 readings", + }, + ], + warnings: [ + { + topic: "blood_pressure", + message: "Stage 1 hypertension threshold crossed.", + severity: "important", + }, + ], +}; + +describe("aiInsightResponseSchema", () => { + it("accepts a well-formed strict response", () => { + const result = aiInsightResponseSchema.safeParse(validResponse); + expect(result.success).toBe(true); + }); + + it("permits legacy passthrough fields (back-compat)", () => { + const withLegacy = { + ...validResponse, + classification: "grenzwertig", + findings: [{ label: "x", value: "y", assessment: "neutral" }], + disclaimer: "consult your doctor", + }; + const result = aiInsightResponseSchema.safeParse(withLegacy); + expect(result.success).toBe(true); + }); + + it("rejects when summary is empty", () => { + const result = aiInsightResponseSchema.safeParse({ + ...validResponse, + summary: "", + }); + expect(result.success).toBe(false); + }); + + it("rejects a recommendation without metricSource.summary", () => { + const broken = { + ...validResponse, + recommendations: [ + { + id: "rec-1", + text: "Walk more", + severity: "suggestion", + metricSource: { + type: "activity", + timeRange: "last7days", + summary: "", // empty — should fail + }, + }, + ], + }; + const result = aiInsightResponseSchema.safeParse(broken); + expect(result.success).toBe(false); + }); + + it("rejects a recommendation with missing severity enum", () => { + const broken = { + ...validResponse, + recommendations: [ + { + id: "rec-1", + text: "Walk more", + severity: "kinda-bad", // not in enum + metricSource: { + type: "activity", + timeRange: "last7days", + summary: "5,000 steps avg", + }, + }, + ], + }; + const result = aiInsightResponseSchema.safeParse(broken); + expect(result.success).toBe(false); + }); +}); diff --git a/src/lib/ai/__tests__/provider-runner.test.ts b/src/lib/ai/__tests__/provider-runner.test.ts index 46a13fc93..b34a5fd1e 100644 --- a/src/lib/ai/__tests__/provider-runner.test.ts +++ b/src/lib/ai/__tests__/provider-runner.test.ts @@ -33,7 +33,6 @@ import { isHardProviderFailure, runRawCompletionWithFallback, runStreamingRawCompletionWithFallback, - runWithFallback, } from "../provider-runner"; import { AUTH_FAILURE_COOLDOWN_MS, @@ -129,15 +128,6 @@ describe("isHardProviderFailure", () => { expect(isHardProviderFailure(err(404))).toBe(true); }); - it("does NOT flag an InsightSchemaError as hard — the strict JSON 422 surface still bubbles", async () => { - const { InsightSchemaError } = await import("../schema"); - expect( - isHardProviderFailure( - new InsightSchemaError("bad json", { attempts: 2 }), - ), - ).toBe(false); - }); - it("flags network errors (no httpStatus) as hard", () => { expect(isHardProviderFailure(new Error("ECONNRESET"))).toBe(true); }); @@ -151,140 +141,7 @@ describe("isHardProviderFailure", () => { }); }); -describe("runWithFallback — happy path", () => { - it("returns the parsed response from the first working provider", async () => { - const codex = new ScriptedProvider({ - type: "codex", - script: [{ ok: true }], - }); - const result = await runWithFallback({ - userId: "u1", - providers: [{ providerType: "codex", instance: codex }], - params: singleUserTurn({ system: "s", user: "u" }), - }); - expect(result.parsed.summary).toBe("ok"); - expect(result.raw.providerType).toBe("codex"); - expect(result.fallbackHops).toEqual([]); - expect(codex.callCount).toBe(1); - }); - - it("attaches the provider type to the outcome for B5e feedback attribution", async () => { - const openai = new ScriptedProvider({ - type: "admin-key", - script: [{ ok: true }], - }); - const result = await runWithFallback({ - userId: "u1", - providers: [{ providerType: "openai", instance: openai }], - params: singleUserTurn({ system: "s", user: "u" }), - }); - expect(result.workingProvider.providerType).toBe("openai"); - expect(result.raw.providerType).toBe("admin-key"); - }); -}); - -describe("runWithFallback — primary fails 401, secondary succeeds", () => { - it("falls through to the secondary and surfaces the failure reason in the hop log", async () => { - const codex = new ScriptedProvider({ - type: "codex", - script: [{ ok: false, error: err(401, "OAuth expired") }], - }); - const openai = new ScriptedProvider({ - type: "admin-key", - script: [{ ok: true }], - }); - const result = await runWithFallback({ - userId: "u1", - providers: [ - { providerType: "codex", instance: codex }, - { providerType: "openai", instance: openai }, - ], - params: singleUserTurn({ system: "s", user: "u" }), - }); - expect(result.parsed.summary).toBe("ok"); - expect(result.workingProvider.providerType).toBe("openai"); - expect(result.fallbackHops).toHaveLength(1); - expect(result.fallbackHops[0]).toMatchObject({ - providerType: "codex", - attempt: 1, - }); - expect(result.fallbackHops[0].failureReason).toMatch(/401/); - }); -}); - -describe("runWithFallback — all-fail cascade", () => { - it("throws AllProvidersFailedError when every chain entry fails hard", async () => { - const codex = new ScriptedProvider({ - type: "codex", - script: [{ ok: false, error: err(0, "ECONNRESET") }], - }); - const openai = new ScriptedProvider({ - type: "admin-key", - script: [{ ok: false, error: err(503, "upstream down") }], - }); - let caught: unknown; - try { - await runWithFallback({ - userId: "u1", - providers: [ - { providerType: "codex", instance: codex }, - { providerType: "openai", instance: openai }, - ], - params: singleUserTurn({ system: "s", user: "u" }), - }); - } catch (e) { - caught = e; - } - expect(caught).toBeInstanceOf(AllProvidersFailedError); - const e = caught as AllProvidersFailedError; - expect(e.httpStatus).toBe(503); - expect(e.attempts).toHaveLength(2); - expect(e.attempts.map((a) => a.providerType)).toEqual(["codex", "openai"]); - }); -}); - -describe("runWithFallback — non-hard errors bubble unchanged", () => { - it("re-throws a 422 schema error from the first provider rather than walking", async () => { - // schema-mismatch is the wrapper's job, not the runner's. The runner - // only walks on transport / auth / 5xx — bad JSON from the first - // provider must still propagate as the wrapper's - // InsightSchemaError so the user gets the existing 422 surface. - const breaking = new ScriptedProvider({ - type: "codex", - // Both wrapper attempts return invalid JSON → wrapper raises - // InsightSchemaError(422). The runner does not walk to provider - // 2 because that error is not a "hard provider failure". - script: [ - { ok: true, content: "not-json" }, - { ok: true, content: "still-not-json" }, - ], - }); - const openai = new ScriptedProvider({ - type: "admin-key", - script: [{ ok: true }], - }); - let caught: unknown; - try { - await runWithFallback({ - userId: "u1", - providers: [ - { providerType: "codex", instance: breaking }, - { providerType: "openai", instance: openai }, - ], - params: singleUserTurn({ system: "s", user: "u" }), - }); - } catch (e) { - caught = e; - } - // Wrapper threw — fallback runner should NOT have escalated to - // openai because schema-mismatch is not a hard provider failure. - expect(caught).toBeDefined(); - expect((caught as Error).name).toBe("InsightSchemaError"); - expect(openai.callCount).toBe(0); - }); -}); - -describe("runWithFallback — last-working cache", () => { +describe("runRawCompletionWithFallback — last-working cache", () => { it("caches the working provider so the next call starts there", async () => { const codex = new ScriptedProvider({ type: "codex", @@ -296,7 +153,7 @@ describe("runWithFallback — last-working cache", () => { }); // First call: codex fails, openai succeeds. - await runWithFallback({ + await runRawCompletionWithFallback({ userId: "u-cache", providers: [ { providerType: "codex", instance: codex }, @@ -309,7 +166,7 @@ describe("runWithFallback — last-working cache", () => { // Second call: cache reorders so openai is tried first; codex is // never re-invoked even though it appears first in the input chain. - await runWithFallback({ + await runRawCompletionWithFallback({ userId: "u-cache", providers: [ { providerType: "codex", instance: codex }, @@ -339,7 +196,7 @@ describe("runWithFallback — last-working cache", () => { }); // First call: codex fails, openai succeeds, cache = openai. - await runWithFallback({ + await runRawCompletionWithFallback({ userId: "u-ttl", providers: [ { providerType: "codex", instance: codex }, @@ -355,7 +212,7 @@ describe("runWithFallback — last-working cache", () => { // Next call walks the original priority order again (codex first, // and this time codex's second scripted entry succeeds). - await runWithFallback({ + await runRawCompletionWithFallback({ userId: "u-ttl", providers: [ { providerType: "codex", instance: codex }, @@ -371,7 +228,7 @@ describe("runWithFallback — last-working cache", () => { type: "admin-key", script: [{ ok: true }], }); - await runWithFallback({ + await runRawCompletionWithFallback({ userId: "u-clear", providers: [{ providerType: "openai", instance: ok }], params: singleUserTurn({ system: "s", user: "u" }), @@ -502,11 +359,11 @@ describe("runRawCompletionWithFallback — legacy route shim", () => { }); }); -describe("runWithFallback — empty input", () => { +describe("runRawCompletionWithFallback — empty input", () => { it("throws AllProvidersFailedError with httpStatus 422 when zero providers configured", async () => { let caught: unknown; try { - await runWithFallback({ + await runRawCompletionWithFallback({ userId: "u-empty", providers: [], params: singleUserTurn({ system: "s", user: "u" }), diff --git a/src/lib/ai/__tests__/rationale-schema.test.ts b/src/lib/ai/__tests__/rationale-schema.test.ts index 0708d6d1a..9bd1eeac0 100644 --- a/src/lib/ai/__tests__/rationale-schema.test.ts +++ b/src/lib/ai/__tests__/rationale-schema.test.ts @@ -3,7 +3,6 @@ import { aiInsightResponseSchema, aiRecommendationRationaleSchema, aiRecommendationSchema, - findRecommendationsMissingRationale, type AIInsightResponse, } from "../schema"; @@ -195,77 +194,3 @@ describe("aiInsightResponseSchema — rationale integration", () => { expect(parsed.recommendations[0].rationale).toEqual(baseRationale); }); }); - -describe("findRecommendationsMissingRationale()", () => { - it("returns an empty array for a payload where every rec has rationale", () => { - expect(findRecommendationsMissingRationale(baseValid)).toEqual([]); - }); - - it("flags a legacy payload whose recs predate B5c (rationale absent)", () => { - // Legacy payloads from before B5c had no rationale field. Once - // the strict parser flips to mandatory rationale, those payloads - // would fail parse — but we keep `.passthrough()` for one - // milestone and use `findRecommendationsMissingRationale()` to - // *detect* legacy shape so the UI can prompt regeneration. - // - // Cast through unknown because the strict input type already - // requires rationale; this helper exists to handle the legacy - // mismatch defensively at runtime. - const legacy = { - summary: "Older insight from v1.4.15.", - recommendations: [ - { - id: "rec-old-1", - text: "Walk more", - severity: "suggestion", - metricSource: { - type: "activity", - timeRange: "last7days", - summary: "5,000 steps avg", - }, - // no rationale - }, - { - id: "rec-old-2", - text: "Hydrate", - severity: "info", - metricSource: { - type: "fluid", - timeRange: "last7days", - summary: "low intake logged", - }, - // no rationale - }, - ], - citations: [], - warnings: [], - } as unknown as AIInsightResponse; - - const missing = findRecommendationsMissingRationale(legacy); - expect(missing).toEqual(["rec-old-1", "rec-old-2"]); - }); - - it("flags only the recs that are missing, not the well-formed ones", () => { - const partial = { - summary: "Mixed payload", - recommendations: [ - baseRec, - { - id: "rec-bad", - text: "Hydrate", - severity: "info", - metricSource: { - type: "fluid", - timeRange: "last7days", - summary: "low intake logged", - }, - // no rationale - }, - ], - citations: [baseCitation], - warnings: [], - } as unknown as AIInsightResponse; - - expect(findRecommendationsMissingRationale(partial)).toEqual(["rec-bad"]); - }); -}); diff --git a/src/lib/ai/__tests__/rationale-wrapper.test.ts b/src/lib/ai/__tests__/rationale-wrapper.test.ts deleted file mode 100644 index ecf9d78fd..000000000 --- a/src/lib/ai/__tests__/rationale-wrapper.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { describe, it, expect, beforeEach, vi } from "vitest"; -import { generateInsight } from "../generate-insight"; -import { MockAIProvider } from "../mock-client"; -import { singleUserTurn } from "../types"; - -const annotateMock = vi.fn(); - -vi.mock("@/lib/logging/context", () => ({ - annotate: (fields: { meta?: Record }) => - annotateMock(fields), - getEvent: () => null, -})); - -/** - * v1.4.16 phase B5c — wrapper-level rationale handling. - * - * 1. The corrective retry message lists the rationale fields so a - * first-attempt response without rationale gets a targeted - * reprompt (instead of the model guessing what's wrong). - * 2. The wrapper's post-validation `annotate()` carries a - * `ai_rationale_coverage_*` breakdown so the admin AI quality - * dashboard can chart per-payload coverage of the new field. - * Mirrors the citation-coverage annotations from B5a. - */ - -const baseRationale = { - dataWindow: "last7days" as const, - comparedTo: "your 90-day median (122/78)", - deviation: "+16/+8 mmHg above baseline over 9 of 9 readings", -}; - -const baseMetricSource = { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 138/86 across 9 readings", -}; - -const baseCitation = { - type: "bloodPressure", - timeRange: "last7days", - summary: "avg 138/86 across 9 readings", -}; - -const validResponse = { - summary: "BP runs slightly above your 90-day median.", - recommendations: [ - { - id: "rec-1", - text: "Discuss home BP log with your physician.", - severity: "important", - metricSource: baseMetricSource, - rationale: baseRationale, - }, - ], - citations: [baseCitation], - warnings: [], -}; - -describe("generateInsight() — corrective retry mentions rationale", () => { - beforeEach(() => { - annotateMock.mockReset(); - }); - - it("retries with a corrective message naming dataWindow / comparedTo / deviation when rationale missing", async () => { - const recWithoutRationale = { - id: "rec-1", - text: "Walk more", - severity: "suggestion", - metricSource: baseMetricSource, - // rationale missing - }; - const broken = { - ...validResponse, - recommendations: [recWithoutRationale], - }; - const provider = new MockAIProvider({ - responses: [JSON.stringify(broken), JSON.stringify(validResponse)], - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "Original user prompt." }), - ); - expect(outcome.attempts).toBe(2); - expect(outcome.retried).toBe(true); - const retryUserPrompt = provider.calls[1].messages[0].content as string; - expect(retryUserPrompt).toContain("Original user prompt."); - expect(retryUserPrompt).toContain("rationale"); - expect(retryUserPrompt).toContain("dataWindow"); - expect(retryUserPrompt).toContain("comparedTo"); - expect(retryUserPrompt).toContain("deviation"); - }); - - it("retries when first attempt has empty rationale.deviation", async () => { - const recWithEmptyDeviation = { - id: "rec-1", - text: "Walk more", - severity: "suggestion", - metricSource: baseMetricSource, - rationale: { - dataWindow: "last7days", - comparedTo: "your 90-day median", - deviation: "", // empty — should fail - }, - }; - const broken = { - ...validResponse, - recommendations: [recWithEmptyDeviation], - }; - const provider = new MockAIProvider({ - responses: [JSON.stringify(broken), JSON.stringify(validResponse)], - }); - const outcome = await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "user" }), - ); - expect(outcome.attempts).toBe(2); - expect(outcome.retried).toBe(true); - }); -}); - -describe("generateInsight() — rationale-coverage annotation", () => { - beforeEach(() => { - annotateMock.mockReset(); - }); - - it("annotates ai_rationale_coverage_* fields on a successful generation", async () => { - const provider = new MockAIProvider({ - responses: JSON.stringify(validResponse), - }); - await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "user" }), - ); - expect(annotateMock).toHaveBeenCalled(); - const meta = annotateMock.mock.calls[0][0]?.meta as Record; - expect(meta).toMatchObject({ - ai_total_recommendations: 1, - ai_rationale_total_recommendations: 1, - ai_rationale_with_rationale: 1, - ai_rationale_missing_recommendation_ids: [], - }); - }); - - it("rationale-coverage = total recommendations when every rec carries one", async () => { - const twoRecs = { - ...validResponse, - recommendations: [ - validResponse.recommendations[0], - { - id: "rec-2", - text: "Continue daily logging.", - severity: "info", - metricSource: baseMetricSource, - rationale: baseRationale, - }, - ], - }; - const provider = new MockAIProvider({ - responses: JSON.stringify(twoRecs), - }); - await generateInsight( - provider, - singleUserTurn({ system: "sys", user: "user" }), - ); - const meta = annotateMock.mock.calls[0][0]?.meta as Record; - expect(meta).toMatchObject({ - ai_rationale_total_recommendations: 2, - ai_rationale_with_rationale: 2, - ai_rationale_missing_recommendation_ids: [], - }); - }); -}); diff --git a/src/lib/ai/citation-coverage.ts b/src/lib/ai/citation-coverage.ts index c6c249bdc..06dea7174 100644 --- a/src/lib/ai/citation-coverage.ts +++ b/src/lib/ai/citation-coverage.ts @@ -1,12 +1,20 @@ /** * Reachability note. This module was written for `generateInsight()`, whose - * only caller is `runWithFallback()` — and `runWithFallback()` has no - * production caller left (every live surface routes through - * `runRawCompletionWithFallback` instead). The coverage annotation therefore - * never executed in production: an audit surface that implied enforcement it - * did not perform. Rather than delete working, tested logic, the input type is - * widened to the structural shape both payloads share and the comprehensive - * post-parse step now calls it, so the admin quality dashboard sees real data. + * only caller was `runWithFallback()` — and that had no production caller, + * because every live surface routes through `runRawCompletionWithFallback`. + * The coverage annotation therefore never executed: an audit surface that + * implied enforcement it did not perform. + * + * v1.30.24 kept the logic and widened the input type to the structural shape + * both payloads share, so the comprehensive post-parse step could call it and + * the admin quality dashboard could see real data. The dead wrapper and chain + * runner around it have since been removed outright, so this module now has + * exactly one caller and it is a live one. + * + * Observational by design: an uncited normative recommendation is an + * annotation, never a parse failure. It is a METRIC, not a gate — the citation + * ENFORCEMENT that once sat beside it is gone rather than dormant, so nothing + * here should be read as withholding output. */ /** diff --git a/src/lib/ai/confidence.ts b/src/lib/ai/confidence.ts deleted file mode 100644 index d69915fa2..000000000 --- a/src/lib/ai/confidence.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * v1.4.16 phase B5d — deterministic confidence-score computation per - * recommendation. - * - * The maintainer's mandate: the model's self-reported confidence is unreliable - * (calibrated probabilities are not a small-LLM strength), so the - * wrapper OVERRIDES with a server-computed value derived from three - * inputs that the cited data window actually exposes: - * - * - n sample count behind the cited data window - * (`metricSource.n`). - * - recencyDays days since the most recent sample in the window; - * a 7-day rec built off data 25 days old should - * not feel as confident as the same rec built off - * data from this morning. - * - deviationStdRatio |deviation| / stdev of the user's 90-day - * baseline. `null` when the baseline is too thin - * to compute a stdev — we fall back to a neutral - * mid-band score rather than zero. - * - * Score components (sum, max 100): - * - * - n contribution: log-saturating curve, capped at 40. - * n<3 hard-caps the whole score at ≤15 (5 * n) — three samples is - * the hard floor for any quantitative claim per research §2.A - * (Apple's "be quiet when unsure" stance). - * - * - recency contribution: 30 at <2d old, decays linearly to 0 at - * 30d, floored at 0. The shape matches the Oura "Pay Attention" - * band — old data is still data, just stale. - * - * - signal contribution: 30 at |z|≥1.5 (clearly out-of-band), - * scaled linearly down to 0 at z=0. When ratio is null (no - * baseline yet), we award 15 — neither punishing the user for - * thin history nor over-claiming a strong deviation we can't - * measure. - * - * The formula is pre-feedback. v1.4.17 will fit weights to thumbs-up - * rates (B5e feedback ratchet); for v1.4.16 the formula is fixed and - * deterministic so admin observability + the "draft pill below 25" - * UI affordance are reproducible across runs. - */ - -export interface ConfidenceInputs { - /** Sample count behind the cited data window (from metricSource.n). */ - n: number; - /** Days since the most recent sample in the window. */ - recencyDays: number; - /** |deviation| / stdev of the user's 90-day baseline, when computable. */ - deviationStdRatio: number | null; -} - -const HARD_CAP_FLOOR = 10; -const N_CONTRIBUTION_MAX = 40; -const RECENCY_CONTRIBUTION_MAX = 30; -const RECENCY_DECAY_DAYS = 30; -const SIGNAL_CONTRIBUTION_MAX = 30; -const SIGNAL_SATURATION_RATIO = 1.5; -const SIGNAL_NULL_NEUTRAL = 15; - -/** - * Compute the deterministic confidence score (0..100) for a single - * recommendation. Pure function — no I/O, no clock; safe to call from - * test fixtures and from `generateInsight()` post-validation alike. - */ -export function computeConfidence(inputs: ConfidenceInputs): number { - const { n, recencyDays, deviationStdRatio } = inputs; - - // n hard cap: <3 samples means we cannot make a quantitative claim, - // so the whole score is bounded at 5*n (max 15 at n=2). Floor at - // 10 so n=0/n=1 still produce a non-zero "draft" signal — the UI - // shows the rec but tags it as low-confidence rather than hiding it. - if (n < 3) { - return Math.max(HARD_CAP_FLOOR, 5 * n); - } - - // n contribution: log10 saturates so a 100-sample window doesn't - // outweigh a 30-sample window by 3×; both are "enough data" for the - // user-level signal we care about. - const nScore = Math.min(N_CONTRIBUTION_MAX, 10 + 10 * Math.log10(n)); - - // Recency contribution: <2d ≈ full score (30 * (1 - 0/30) = 30, - // 30 * (1 - 1/30) = 29; both round into the meter's top band). - // 30d+ floors to 0 so an ancient window cannot inflate the score. - const recencyScore = Math.max( - 0, - RECENCY_CONTRIBUTION_MAX * (1 - recencyDays / RECENCY_DECAY_DAYS), - ); - - // Signal contribution: null ratio (no baseline) → neutral 15. - // Otherwise scale |z| linearly to the SIGNAL_SATURATION_RATIO cap. - const signalScore = - deviationStdRatio === null - ? SIGNAL_NULL_NEUTRAL - : Math.min( - SIGNAL_CONTRIBUTION_MAX, - SIGNAL_CONTRIBUTION_MAX * - (Math.abs(deviationStdRatio) / SIGNAL_SATURATION_RATIO), - ); - - return Math.round(nScore + recencyScore + signalScore); -} diff --git a/src/lib/ai/generate-insight.ts b/src/lib/ai/generate-insight.ts deleted file mode 100644 index c861b790d..000000000 --- a/src/lib/ai/generate-insight.ts +++ /dev/null @@ -1,279 +0,0 @@ -import type { z } from "zod/v4"; -import { - appendToLastUserMessage, - type AIProvider, - type CompletionParams, - type CompletionResult, -} from "./types"; -import { - aiInsightResponseSchema, - findRecommendationsMissingRationale, - findUncitedRecommendations, - InsightSchemaError, - type AIInsightResponse, -} from "./schema"; -import { computeCitationCoverage } from "./citation-coverage"; -import { computeConfidence } from "./confidence"; -import { annotate } from "@/lib/logging/context"; - -/** - * Schema-enforced wrapper around `AIProvider.generateCompletion()`. - * - * Phase C1 (v1.4.15): the maintainer's mandate ("zero hallucinations, must - * ground on user data") means a free-form LLM response is no longer - * acceptable. This wrapper: - * - * 1. Calls the provider once. - * 2. Parses the response as JSON. - * 3. Validates it against `aiInsightResponseSchema` AND cross-checks - * that every recommendation's `metricSource` appears in - * `citations[]`. - * 4. On failure: retries ONCE, prepending a corrective system - * message that includes the violated zod issues. - * 5. On second failure: throws `InsightSchemaError` (httpStatus 422). - * - * The wrapper does not concern itself with provider-level retries - * (HTTP 5xx, 401 token-refresh, etc.) — each provider client handles - * those internally. This wrapper retries only on schema-violation. - */ - -export interface GenerateInsightOutcome { - parsed: AIInsightResponse; - raw: CompletionResult; - /** Number of provider calls made (1 on success, 2 on retry). */ - attempts: number; - /** True iff the retry attempt produced the parsed response. */ - retried: boolean; -} - -/** - * Build the corrective user-prompt suffix that explains to the model - * what went wrong on the first attempt. Includes the violated zod - * issues verbatim (truncated to 1KB) so the model can self-correct. - */ -export function buildRetryCorrectionMessage( - reason: string, - details: string, -): string { - return ` -Your previous response did not satisfy the required JSON schema. -Reason: ${reason} -Details: ${details.slice(0, 1024)} - -You MUST return a JSON object matching this schema exactly. Do NOT -include prose, markdown fences, or commentary outside the JSON. - -Required top-level fields: - - summary: string (2-3 sentences) - - recommendations: array of objects with - { id: string, - text: string, - severity: "info" | "suggestion" | "important" | "urgent", - metricSource: { type: string, timeRange: string, summary: string, n?: number }, - rationale: { dataWindow: "last7days" | "last30days" | "last90days" | "allTime", - comparedTo: non-empty string, - deviation: non-empty string } } - - citations: array of objects with { type: string, timeRange: string, summary: string } - - warnings: array of objects with { topic: string, message: string, severity?: same enum } - -Every recommendation's metricSource MUST also appear in citations[] -(same type + timeRange). Every recommendation MUST carry a -rationale object — dataWindow, comparedTo, and deviation are all -required and rationale.dataWindow MUST equal metricSource.timeRange. -If you cannot ground a recommendation in user data from the -snapshot you were given, OMIT it. An empty recommendations[] is -acceptable. -`; -} - -interface TryOutcome { - ok: boolean; - parsed?: AIInsightResponse; - raw: CompletionResult; - reason?: string; - details?: string; - zodIssues?: z.ZodIssue[]; -} - -async function tryOnce( - provider: AIProvider, - params: CompletionParams, -): Promise { - const raw = await provider.generateCompletion(params); - - // Parse JSON. - let json: unknown; - try { - json = JSON.parse(raw.content); - } catch (e) { - return { - ok: false, - raw, - reason: "Response was not valid JSON", - details: (e as Error).message, - }; - } - - // Schema validate. - const result = aiInsightResponseSchema.safeParse(json); - if (!result.success) { - return { - ok: false, - raw, - reason: "Response did not match the required JSON schema", - details: result.error.issues - .slice(0, 8) - .map((i) => `${i.path.join(".")}: ${i.message}`) - .join("; "), - zodIssues: result.error.issues, - }; - } - - // Cross-check citations. - const uncited = findUncitedRecommendations(result.data); - if (uncited.length > 0) { - const list = uncited - .slice(0, 4) - .map( - (u) => - `${u.recommendationId} → ${u.missing.type}/${u.missing.timeRange}`, - ) - .join(", "); - return { - ok: false, - raw, - reason: "Recommendations referenced metricSources not in citations[]", - details: `Missing citations: ${list}`, - }; - } - - return { ok: true, parsed: result.data, raw }; -} - -/** - * Forward the citation-coverage breakdown into the active Wide-Event - * via `annotate()` (no-op outside a request context). Keys are - * snake_case to match the rest of the AI Wide-Event vocabulary. - * - * v1.4.16 phase B5c — additionally emit `ai_rationale_*` keys so the - * admin AI quality dashboard can chart rationale-coverage per - * payload alongside citation-coverage. Rationale is required by the - * strict schema; the missing-ids list will only ever populate when - * a legacy payload sneaks through `.passthrough()` — useful as an - * observability tripwire for the migration window. - */ -function annotateCitationCoverage(parsed: AIInsightResponse): void { - const coverage = computeCitationCoverage(parsed); - const rationaleMissing = findRecommendationsMissingRationale(parsed); - annotate({ - meta: { - ai_total_recommendations: coverage.totalRecommendations, - ai_normative_recommendations: coverage.normativeRecommendations, - ai_cited_normative_recommendations: - coverage.citedNormativeRecommendations, - ai_uncited_normative_recommendation_ids: - coverage.uncitedNormativeRecommendationIds, - ai_rationale_total_recommendations: coverage.totalRecommendations, - ai_rationale_with_rationale: - coverage.totalRecommendations - rationaleMissing.length, - ai_rationale_missing_recommendation_ids: rationaleMissing, - }, - }); -} - -/** - * Apply the deterministic confidence override to every recommendation. - * The model's value (if any) is REPLACED. Inputs are derived from the - * parsed payload alone (n from `metricSource.n`, recencyDays=0, - * deviationStdRatio=null) — the route layer doesn't pass bucket-series - * data through here today. v1.4.17's feedback ratchet will need a - * resolver injection; re-introduce one when the second caller exists. - * - * Emits a Wide-Event with per-rec model-vs-computed pairs so admin - * observability can chart drift between what the LLM thinks and what - * the formula produces. - */ -function applyConfidenceOverride(parsed: AIInsightResponse): void { - const overrides: Array<{ - id: string; - model: number | null; - computed: number; - }> = []; - for (const rec of parsed.recommendations) { - const computed = computeConfidence({ - n: rec.metricSource.n ?? 0, - recencyDays: 0, - deviationStdRatio: null, - }); - const modelValue = - typeof rec.confidence === "number" ? rec.confidence : null; - rec.confidence = computed; - overrides.push({ id: rec.id, model: modelValue, computed }); - } - if (overrides.length > 0) { - annotate({ - meta: { - ai_confidence_override_event: "confidence.override.applied", - ai_confidence_override_count: overrides.length, - ai_confidence_override_pairs: overrides, - }, - }); - } -} - -/** - * Run a schema-enforced insight generation. Up to 2 provider calls. - * Throws `InsightSchemaError` (httpStatus 422) if both attempts fail - * the schema; provider-level errors bubble untouched. - * - * v1.4.16 phase B5d — after a successful parse, every recommendation's - * `confidence` field is overwritten with the deterministic - * `computeConfidence()` value. The model's number (if any) is - * discarded; admin observability sees both via the - * `confidence.override.applied` Wide-Event annotation. - */ -export async function generateInsight( - provider: AIProvider, - params: CompletionParams, -): Promise { - const first = await tryOnce(provider, params); - if (first.ok && first.parsed) { - annotateCitationCoverage(first.parsed); - applyConfidenceOverride(first.parsed); - return { - parsed: first.parsed, - raw: first.raw, - attempts: 1, - retried: false, - }; - } - - // Retry-once with corrective context appended to the user prompt. - const correction = buildRetryCorrectionMessage( - first.reason ?? "schema mismatch", - first.details ?? "", - ); - const retryParams: CompletionParams = appendToLastUserMessage( - params, - `\n\n${correction}`, - ); - const second = await tryOnce(provider, retryParams); - if (second.ok && second.parsed) { - annotateCitationCoverage(second.parsed); - applyConfidenceOverride(second.parsed); - return { - parsed: second.parsed, - raw: second.raw, - attempts: 2, - retried: true, - }; - } - - throw new InsightSchemaError( - second.reason ?? first.reason ?? "Schema validation failed twice", - { - issues: second.zodIssues ?? first.zodIssues, - attempts: 2, - }, - ); -} diff --git a/src/lib/ai/prompts/insight-generator.ts b/src/lib/ai/prompts/insight-generator.ts index 184367af3..bc96be96b 100644 --- a/src/lib/ai/prompts/insight-generator.ts +++ b/src/lib/ai/prompts/insight-generator.ts @@ -986,7 +986,7 @@ const INSIGHTS_LOCALE_REPLY_FOOTER_FALLBACK: Record< /** * Returns the active scope-hardened system prompt for a given locale. - * The route consumes this via `generateInsight()`. + * Consumed by the comprehensive generation path. */ export function getStrictInsightsSystemPrompt(locale: Locale): string { if (locale === "de") return SYSTEM_PROMPT_DE; diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index d10faed99..cc62c83da 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -1,9 +1,4 @@ import type { AIProvider, CompletionParams, CompletionResult } from "./types"; -import { - generateInsight, - type GenerateInsightOutcome, -} from "./generate-insight"; -import { InsightSchemaError } from "./schema"; import type { ProviderChainType } from "./provider-chain"; import { postgresProviderHealthLedger, @@ -25,19 +20,17 @@ const NEVER_SKIP: ReadonlySet = new Set(["local"]); /** * v1.4.16 phase B5b — multi-provider redundancy runner. * - * The chain runner wraps `generateInsight()` with a hard-failure + * The chain runner wraps a provider completion with a hard-failure * walkthrough across an ordered list of providers. The motivation is * the maintainer's "ein Provider-Ausfall darf die Insights nicht zerschiessen" * mandate: a 401 from Codex (OAuth expired), a 503 from OpenAI * (upstream brown-out), or a network reset MUST cascade to the next * configured provider rather than 422-ing the whole request. * - * Hard failures (`isHardProviderFailure`) cascade. Schema failures - * (the existing 422 surface) do NOT cascade — they bubble through - * `InsightSchemaError` so the user still sees the helpful "the model - * returned malformed JSON" diagnostic. Walking on schema errors would - * mask provider-specific prompt-following issues that the v1.4.15 - * citation-enforcement guard exists to surface. + * Hard failures (`isHardProviderFailure`) cascade. A malformed-JSON + * reply is NOT a hard failure — it is a prompt-following issue, so it + * bubbles to the caller's own validator rather than masking itself as + * provider unavailability and walking the chain. * * Cross-feature coupling: * - B5a's `MEDICAL_REFERENCES` is shared via the system prompt; each @@ -120,9 +113,9 @@ function rememberWorkingProvider( * - Network errors (no `httpStatus`) — DNS, ECONNRESET, timeout. * * No: - * - `InsightSchemaError` — surfaces the wrapper's existing 422 so the - * user sees "model JSON malformed" rather than getting silent - * provider drift. This is the ONLY non-hard class. + * - Anything with a non-error status. Callers that parse a structured + * reply raise their own validation error, which carries no + * `httpStatus` from a provider wire and is theirs to surface. * * v1.21.3 — a 4xx that the PROVIDER WIRE returned (tagged `upstream`, * e.g. a Codex `400` rejecting the tool-call / structured request shape) @@ -132,12 +125,9 @@ function rememberWorkingProvider( * provider error rethrew as an HTTP 500. Treating every upstream 4xx as * hard means it cascades to the next provider and, on exhaustion, is * wrapped in `AllProvidersFailedError` so the route surfaces a graceful - * `coach.provider.*` frame instead of a 500. `InsightSchemaError` - * (no `upstream` tag) is still excluded, so the strict JSON surface keeps - * its 422 "model JSON malformed" diagnostic. + * `coach.provider.*` frame instead of a 500. */ export function isHardProviderFailure(error: unknown): boolean { - if (error instanceof InsightSchemaError) return false; const err = error as { httpStatus?: number; name?: string } | null; const status = err?.httpStatus; if (typeof status !== "number" || status <= 0) { @@ -169,28 +159,6 @@ export interface FallbackHop { httpStatus: number | null; } -export interface RunWithFallbackParams { - userId: string; - providers: ProviderChainResolved[]; - params: CompletionParams; - /** - * v1.11.0 W1 — durable provider-health ledger. Defaults to the - * Postgres-backed implementation in production; unit tests of the pure - * chain pass an in-memory / no-op ledger to avoid standing up a DB. - * The ledger is read-through (skip-hint reorder) + write-through - * (record outcome); it never gates generation — a ledger error always - * fails open to today's behaviour. - */ - ledger?: ProviderHealthLedger; -} - -export interface RunWithFallbackResult extends GenerateInsightOutcome { - /** Provider that actually produced the parsed response. */ - workingProvider: ProviderChainResolved; - /** Hops that failed before the working provider succeeded. */ - fallbackHops: FallbackHop[]; -} - /** * Hard-fail wrapper around `AllProvidersFailedError.attempts` so the * route layer can pick a single representative status (the worst of @@ -392,78 +360,6 @@ export function classifyErrorBody(body: unknown): string | null { return `unclassified:${body.length}chars`; } -/** - * Run an insight generation across an ordered chain of providers, - * walking past hard provider failures. Schema-class failures (422) - * bubble unchanged from the underlying wrapper. - */ -export async function runWithFallback( - args: RunWithFallbackParams, -): Promise { - const { userId, providers, params } = args; - const ledger = args.ledger ?? postgresProviderHealthLedger; - - if (providers.length === 0) { - throw new AllProvidersFailedError([]); - } - - const ordered = await resolveChainOrder(userId, providers, ledger); - const hops: FallbackHop[] = []; - - for (let i = 0; i < ordered.length; i += 1) { - const candidate = ordered[i]; - try { - const outcome = await generateInsight(candidate.instance, params); - rememberWorkingProvider(userId, candidate.providerType); - void ledger.recordSuccess(userId, candidate.providerType); - annotate({ - meta: { - ai_chain_working_provider: candidate.providerType, - ai_chain_fallback_count: hops.length, - }, - }); - return { - ...outcome, - workingProvider: candidate, - fallbackHops: hops, - }; - } catch (error) { - if (!isHardProviderFailure(error)) { - // Schema/validation error — bubble immediately. NOT recorded in - // the health ledger: a malformed-JSON reply is a prompt-following - // issue, not a provider-availability failure. - throw error; - } - const summary = summariseError(error); - void ledger.recordFailure(userId, candidate.providerType, summary.status); - const hop: FallbackHop = { - providerType: candidate.providerType, - attempt: i + 1, - failureReason: summary.reason, - httpStatus: summary.status, - }; - hops.push(hop); - annotate({ - meta: { - [`ai_chain_hop_${i + 1}_provider`]: candidate.providerType, - [`ai_chain_hop_${i + 1}_status`]: summary.status, - [`ai_chain_hop_${i + 1}_reason`]: summary.reason.slice(0, 240), - [`ai_chain_hop_${i + 1}_body`]: summary.bodyExcerpt, - }, - }); - } - } - - // Every chain entry hard-failed. - annotate({ - meta: { - ai_chain_outcome: "all-failed", - ai_chain_fallback_count: hops.length, - }, - }); - throw new AllProvidersFailedError(hops); -} - /** * Shared chain walker for the two RAW runners (streaming + buffered). Owns the * one set of chain / fallback / health-ledger semantics — empty-chain guard, @@ -593,14 +489,15 @@ export interface RunRawWithFallbackResult { } /** - * Sibling of `runWithFallback()` that calls `provider.generateCompletion` - * directly (no strict-schema wrapper). Used by the legacy route at - * `/api/insights/generate` which still consumes the rich legacy shape. + * Buffered runner: calls `provider.generateCompletion` directly and hands the + * raw reply back for the caller to validate. This is the ONLY buffered chain + * runner — a strict-schema sibling (`runWithFallback`) once sat beside it but + * had no production caller and was removed. * - * Fallback policy is identical: hard provider failures cascade, - * everything else (e.g. a 4xx-but-non-auth from a custom provider) - * bubbles to the caller. The cache + structured logging are shared - * with the strict variant. + * Fallback policy: hard provider failures cascade, everything else (e.g. a + * 4xx-but-non-auth from a custom provider) bubbles to the caller. The + * last-working cache + structured logging are shared with the streaming + * sibling via `runRawChain`. */ export async function runRawCompletionWithFallback(args: { userId: string; diff --git a/src/lib/ai/retry-correction.ts b/src/lib/ai/retry-correction.ts new file mode 100644 index 000000000..26c912b49 --- /dev/null +++ b/src/lib/ai/retry-correction.ts @@ -0,0 +1,60 @@ +/** + * The corrective retry prompt for a structured-output generation that came + * back malformed. + * + * History worth keeping, because the shape of this module is a consequence of + * it: this file used to hold `generateInsight()`, a schema-enforced wrapper + * that parsed a reply against the strict `aiInsightResponseSchema`, + * cross-checked that every recommendation's `metricSource` appeared in + * `citations[]`, retried once with the message below, and threw + * `InsightSchemaError` on a second failure. That wrapper's only caller was + * `runWithFallback()`, which itself had no production caller — every live + * surface runs `runRawCompletionWithFallback` and validates with its own + * schema. So the enforcement never executed. It was removed rather than wired + * in: the live `insightResultSchema` has no `citations[]` field at all and + * treats `metricSource` as optional, so applying the strict cross-check to a + * live payload would have rejected essentially every generation. + * + * What survived is the part that was always reachable — the corrective + * message, which `comprehensive-generate.ts` sends on its own JSON-retry leg. + */ + +/** + * Build the corrective user-prompt suffix that explains to the model + * what went wrong on the first attempt. Includes the violated zod + * issues verbatim (truncated to 1KB) so the model can self-correct. + */ +export function buildRetryCorrectionMessage( + reason: string, + details: string, +): string { + return ` +Your previous response did not satisfy the required JSON schema. +Reason: ${reason} +Details: ${details.slice(0, 1024)} + +You MUST return a JSON object matching this schema exactly. Do NOT +include prose, markdown fences, or commentary outside the JSON. + +Required top-level fields: + - summary: string (2-3 sentences) + - recommendations: array of objects with + { id: string, + text: string, + severity: "info" | "suggestion" | "important" | "urgent", + metricSource: { type: string, timeRange: string, summary: string, n?: number }, + rationale: { dataWindow: "last7days" | "last30days" | "last90days" | "allTime", + comparedTo: non-empty string, + deviation: non-empty string } } + - citations: array of objects with { type: string, timeRange: string, summary: string } + - warnings: array of objects with { topic: string, message: string, severity?: same enum } + +Every recommendation's metricSource MUST also appear in citations[] +(same type + timeRange). Every recommendation MUST carry a +rationale object — dataWindow, comparedTo, and deviation are all +required and rationale.dataWindow MUST equal metricSource.timeRange. +If you cannot ground a recommendation in user data from the +snapshot you were given, OMIT it. An empty recommendations[] is +acceptable. +`; +} diff --git a/src/lib/ai/schema.ts b/src/lib/ai/schema.ts index 7a107c407..994b53fb4 100644 --- a/src/lib/ai/schema.ts +++ b/src/lib/ai/schema.ts @@ -18,7 +18,7 @@ import { MEDICAL_REFERENCE_IDS } from "./medical-references"; * - A `citations[]` array referencing the metrics the response drew * on. May be empty when `recommendations[]` is empty (no data → no * citation), but every recommendation's `metricSource` MUST also - * appear in `citations[]` (cross-check enforced by `validateInsightResponse`). + * appear in `citations[]`. * - A `warnings[]` array for guideline-flagged values (high BP, * critical pulse, etc.). Free-form for now; v1.4.16 adds severity * enum + medical-reference grounding. @@ -137,9 +137,8 @@ export const aiRecommendationSchema = z /** * v1.4.16 phase B5c — per-recommendation explainability rationale. * Required on every rec so the UI's expand-card always has WHY + - * WINDOW + COMPARED-TO content to render. Legacy payloads (pre- - * B5c) ride through `findRecommendationsMissingRationale()` for - * the regenerate-CTA migration; the strict parser rejects them. + * WINDOW + COMPARED-TO content to render. The strict parser + * rejects a legacy (pre-B5c) payload that omits it. */ rationale: aiRecommendationRationaleSchema, /** @@ -159,11 +158,9 @@ export const aiRecommendationSchema = z * * Optional at parse-time: * - The LLM may emit a value (we accept it so the payload round- - * trips cleanly), but `generateInsight()` OVERRIDES with the - * server-computed `computeConfidence()` post-validation. The - * model's number is discarded — calibrated probabilities are - * not a small-LLM strength and the deterministic path keeps - * the v1.4.17 feedback ratchet reproducible. + * trips cleanly). A deterministic server-side override once + * replaced it post-validation, but that path never ran in + * production and was removed with the rest of the wrapper. * - Legacy cached payloads from before B5d landed have no * confidence field; the meter falls back to a "draft" pill. * @@ -507,87 +504,19 @@ export const aiInsightResponseSchema = z export type AIInsightResponse = z.infer; /** - * Tagged error thrown by the schema-enforcement wrapper when even the - * retry attempt fails to produce a valid payload. The route catches - * this and surfaces a 422 to the client. - */ -export class InsightSchemaError extends Error { - readonly httpStatus = 422; - readonly issues: z.ZodIssue[] | null; - readonly attempts: number; - - constructor( - message: string, - options: { issues?: z.ZodIssue[]; attempts: number }, - ) { - super(message); - this.name = "InsightSchemaError"; - this.issues = options.issues ?? null; - this.attempts = options.attempts; - } -} - -/** - * v1.4.16 phase B5c — legacy-payload detector. + * The strict-contract ENFORCEMENT that used to live here is gone: an + * `InsightSchemaError` class no caller threw, a `findUncitedRecommendations` + * cross-check (every recommendation's `metricSource` must appear in + * `citations[]`) and a `findRecommendationsMissingRationale` legacy-payload + * detector. All three existed only for `generateInsight()`, whose sole caller + * `runWithFallback()` had no production caller — so none of them ever ran. * - * The strict schema now requires `rationale` on every recommendation. - * Cached payloads from v1.4.14/v1.4.15 predate the field — they would - * fail `aiInsightResponseSchema.safeParse()` outright. Rather than - * auto-regenerating on every read (expensive + surprising), we let - * the route call this helper against the legacy-shape JSON so the - * UI can show a "Insights updated — regenerate for new explainability - * features" CTA. User-initiated regeneration stays the trigger. - * - * Input is intentionally typed loosely (the runtime shape is the - * canonical AIInsightResponse, but legacy payloads omit `rationale` - * which the static type now requires). Returns the ids of - * recommendations missing rationale; an empty array means the - * payload is well-shaped under B5c. + * The schema above stays because it is not a guard: it is the shape the + * corrective retry prompt (`retry-correction.ts`) describes to the model, and + * it is the reference definition for the strict payload. NOTE the live + * comprehensive path validates against the looser `insightResultSchema` in + * `types.ts`, which has no `citations[]` and treats `metricSource` as + * optional — so the retry prompt asks for a stricter shape than the live + * validator requires. That mismatch predates this cleanup and is left alone + * deliberately; changing the retry prompt changes model output. */ -export function findRecommendationsMissingRationale( - parsed: AIInsightResponse, -): string[] { - const missing: string[] = []; - for (const rec of parsed.recommendations) { - // Defensive runtime check — the static type asserts rationale is - // present, but the helper exists precisely for legacy payloads - // where it isn't. - const r = (rec as { rationale?: unknown }).rationale; - if (r === undefined || r === null) { - missing.push(rec.id); - } - } - return missing; -} - -/** - * Cross-validation beyond zod: every recommendation's `metricSource` - * must also appear in `citations[]` (same `type` + `timeRange`). - * Returns the list of missing-citation issues; an empty list means - * the response is internally consistent. - */ -export function findUncitedRecommendations(parsed: AIInsightResponse): Array<{ - recommendationId: string; - missing: { type: string; timeRange: string }; -}> { - const cited = new Set( - parsed.citations.map((c) => `${c.type}::${c.timeRange}`), - ); - const missing: Array<{ - recommendationId: string; - missing: { type: string; timeRange: string }; - }> = []; - for (const rec of parsed.recommendations) { - const key = `${rec.metricSource.type}::${rec.metricSource.timeRange}`; - if (!cited.has(key)) { - missing.push({ - recommendationId: rec.id, - missing: { - type: rec.metricSource.type, - timeRange: rec.metricSource.timeRange, - }, - }); - } - } - return missing; -} diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index ecbb94c6c..943b4e17a 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -53,7 +53,7 @@ import { buildSystemPromptWithReferences, buildBriefingPersonalisationBlock, } from "@/lib/ai/prompts/insight-generator"; -import { buildRetryCorrectionMessage } from "@/lib/ai/generate-insight"; +import { buildRetryCorrectionMessage } from "@/lib/ai/retry-correction"; import { singleUserTurn } from "@/lib/ai/types"; import { buildAboutMeInsightBlock, @@ -1042,8 +1042,8 @@ export async function generateComprehensiveInsight( // Citation coverage. This annotation is what the admin AI-quality dashboard // slices to track how many normative recommendations carry a curated - // medical-reference id. It used to hang off `generateInsight()`, which has no - // production caller, so the dashboard was reading a metric nothing emitted. + // medical-reference id. It used to hang off a wrapper that had no production + // caller, so the dashboard was reading a metric nothing emitted. // Observational only — an uncited normative rec is a logged warning, never a // reason to fail the generation. if (insights && typeof insights === "object") {