From 53aade981a2f2c6d6d149934ddddc1e8c30788b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 9 May 2026 23:51:59 +0200 Subject: [PATCH] feat(ai): post-validation logs citation-coverage rate per insight generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the schema parse and citation cross-check pass, the wrapper counts how many recommendations make a normative claim ("target", "should", "normal range", "above", "below") and how many of those carry a `referenceId` pointing into the curated medical-reference bundle. The result lands in the active Wide-Event meta annotation under `ai_total_recommendations`, `ai_normative_recommendations`, `ai_cited_normative_recommendations` and `ai_uncited_normative_recommendation_ids` so the admin AI quality dashboard can chart citation-coverage over time. Observational only in v1.4.16 — a normative rec without a referenceId is logged, never raised as a parse error. Phase B5c will flip the check to required for severity >= "important". 15 unit tests pin the keyword heuristic (EN + DE), the coverage counts across cited / uncited / observational mixes, and the `generateInsight()` integration that emits the annotation only on success (skipped on schema-failure throw). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../citation-coverage-logging.test.ts | 301 ++++++++++++++++++ src/lib/ai/citation-coverage.ts | 94 ++++++ src/lib/ai/generate-insight.ts | 23 ++ 3 files changed, 418 insertions(+) create mode 100644 src/lib/ai/__tests__/citation-coverage-logging.test.ts create mode 100644 src/lib/ai/citation-coverage.ts diff --git a/src/lib/ai/__tests__/citation-coverage-logging.test.ts b/src/lib/ai/__tests__/citation-coverage-logging.test.ts new file mode 100644 index 000000000..a3abf526c --- /dev/null +++ b/src/lib/ai/__tests__/citation-coverage-logging.test.ts @@ -0,0 +1,301 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { generateInsight } from "../generate-insight"; +import { MockAIProvider } from "../mock-client"; +import { + detectsNormativeClaim, + computeCitationCoverage, +} from "../citation-coverage"; +import { MEDICAL_REFERENCES } from "../medical-references"; + +const annotateMock = vi.fn(); + +vi.mock("@/lib/logging/context", () => ({ + annotate: (fields: { meta?: Record }) => + annotateMock(fields), + getEvent: () => null, +})); + +const knownRefId = MEDICAL_REFERENCES[0].id; + +const baseMetricSource = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const baseCitation = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +/** + * v1.4.16 phase B5a — citation-coverage post-validation logging. + * + * After the schema parse + cross-citation check pass, the wrapper + * counts how many recommendations make a normative claim and how + * many of those carry a referenceId. The result lands as a Wide-Event + * meta annotation so the admin AI quality dashboard (planned) can + * track citation-coverage over time. + * + * The check is observational only in v1.4.16 — a rec that should cite + * but doesn't gets logged, never raises a parse error. v1.4.16 phase + * B5c flips it to required for severity >= "important". + */ + +describe("detectsNormativeClaim()", () => { + it("detects 'target' in the rec text", () => { + expect(detectsNormativeClaim("Aim for a BP target below 130/80")).toBe( + true, + ); + }); + + it("detects 'should' in the rec text", () => { + expect(detectsNormativeClaim("Your BP should stay below 140/90")).toBe( + true, + ); + }); + + it("detects 'normal range' in the rec text", () => { + expect(detectsNormativeClaim("Pulse is within the normal range")).toBe( + true, + ); + }); + + it("detects 'above' in the rec text", () => { + expect( + detectsNormativeClaim("Reading is above the recommended ceiling"), + ).toBe(true); + }); + + it("detects 'below' in the rec text", () => { + expect(detectsNormativeClaim("Reading is below the lower threshold")).toBe( + true, + ); + }); + + it("returns false for purely observational text", () => { + expect( + detectsNormativeClaim( + "Your avg7 (78 bpm) is 5 bpm higher than your 90-day median (73 bpm)", + ), + ).toBe(false); + }); + + it("is case-insensitive", () => { + expect(detectsNormativeClaim("TARGET range exceeded")).toBe(true); + expect(detectsNormativeClaim("Target range exceeded")).toBe(true); + expect(detectsNormativeClaim("target range exceeded")).toBe(true); + }); +}); + +describe("computeCitationCoverage()", () => { + it("counts an empty recommendations[] as zero / zero", () => { + expect( + computeCitationCoverage({ + summary: "x", + recommendations: [], + citations: [], + warnings: [], + }), + ).toEqual({ + totalRecommendations: 0, + normativeRecommendations: 0, + citedNormativeRecommendations: 0, + uncitedNormativeRecommendationIds: [], + }); + }); + + it("counts a normative rec with referenceId as cited", () => { + const result = computeCitationCoverage({ + summary: "x", + recommendations: [ + { + id: "rec-1", + text: "Aim for a target below 140/90", + severity: "important", + metricSource: baseMetricSource, + referenceId: knownRefId, + }, + ], + citations: [baseCitation], + warnings: [], + }); + expect(result.totalRecommendations).toBe(1); + expect(result.normativeRecommendations).toBe(1); + expect(result.citedNormativeRecommendations).toBe(1); + expect(result.uncitedNormativeRecommendationIds).toEqual([]); + }); + + it("flags a normative rec without referenceId as uncited", () => { + const result = computeCitationCoverage({ + summary: "x", + recommendations: [ + { + id: "rec-naked-target", + text: "Aim for a target below 140/90", + severity: "important", + metricSource: baseMetricSource, + }, + ], + citations: [baseCitation], + warnings: [], + }); + expect(result.normativeRecommendations).toBe(1); + expect(result.citedNormativeRecommendations).toBe(0); + expect(result.uncitedNormativeRecommendationIds).toEqual([ + "rec-naked-target", + ]); + }); + + it("does NOT flag observational recs as uncited", () => { + const result = computeCitationCoverage({ + summary: "x", + recommendations: [ + { + id: "rec-observational", + text: "Your avg7 (78 bpm) is 5 bpm higher than your 90-day median", + severity: "info", + metricSource: baseMetricSource, + }, + ], + citations: [baseCitation], + warnings: [], + }); + expect(result.normativeRecommendations).toBe(0); + expect(result.citedNormativeRecommendations).toBe(0); + expect(result.uncitedNormativeRecommendationIds).toEqual([]); + }); + + it("flags 'above' / 'below' as normative — they imply a threshold", () => { + // The heuristic intentionally treats "5 mmHg above your 90-day + // median" as a normative claim because the comparison invokes a + // baseline threshold; the rec should cite the source of that + // baseline (or be reworded to drop the threshold language). + const result = computeCitationCoverage({ + summary: "x", + recommendations: [ + { + id: "rec-threshold", + text: "Your avg7 is 5 mmHg above your 90-day median", + severity: "info", + metricSource: baseMetricSource, + }, + ], + citations: [baseCitation], + warnings: [], + }); + expect(result.normativeRecommendations).toBe(1); + }); + + it("mixes cited / uncited / observational correctly", () => { + const result = computeCitationCoverage({ + summary: "x", + recommendations: [ + { + id: "cited-1", + text: "Aim for target range 130/80", + severity: "important", + metricSource: baseMetricSource, + referenceId: knownRefId, + }, + { + id: "uncited-1", + text: "Your weight should drop by 2 kg", + severity: "suggestion", + metricSource: baseMetricSource, + }, + { + id: "observational-1", + // No normative keyword — pure within-user comparison. + text: "Your avg7 (78 bpm) is 5 bpm higher than your 90-day median", + severity: "info", + metricSource: baseMetricSource, + }, + ], + citations: [baseCitation], + warnings: [], + }); + expect(result.totalRecommendations).toBe(3); + expect(result.normativeRecommendations).toBe(2); + expect(result.citedNormativeRecommendations).toBe(1); + expect(result.uncitedNormativeRecommendationIds).toEqual(["uncited-1"]); + }); +}); + +describe("generateInsight() — citation-coverage annotation", () => { + beforeEach(() => { + annotateMock.mockReset(); + }); + + it("emits an annotate() call with the coverage breakdown on success", async () => { + const provider = new MockAIProvider({ + responses: JSON.stringify({ + summary: "x", + recommendations: [ + { + id: "rec-cited", + text: "Aim for a target below 140/90", + severity: "important", + metricSource: baseMetricSource, + referenceId: knownRefId, + }, + ], + citations: [baseCitation], + warnings: [], + }), + }); + await generateInsight(provider, { + systemPrompt: "system", + userPrompt: "user", + }); + expect(annotateMock).toHaveBeenCalledTimes(1); + const meta = annotateMock.mock.calls[0][0]?.meta as Record; + expect(meta).toMatchObject({ + ai_total_recommendations: 1, + ai_normative_recommendations: 1, + ai_cited_normative_recommendations: 1, + ai_uncited_normative_recommendation_ids: [], + }); + }); + + it("annotation flags the uncited normative rec", async () => { + const provider = new MockAIProvider({ + responses: JSON.stringify({ + summary: "x", + recommendations: [ + { + id: "rec-naked", + text: "Your BP should stay below 140/90", + severity: "important", + metricSource: baseMetricSource, + }, + ], + citations: [baseCitation], + warnings: [], + }), + }); + await generateInsight(provider, { + systemPrompt: "system", + userPrompt: "user", + }); + expect(annotateMock).toHaveBeenCalledTimes(1); + const meta = annotateMock.mock.calls[0][0]?.meta as Record; + expect(meta).toMatchObject({ + ai_normative_recommendations: 1, + ai_cited_normative_recommendations: 0, + ai_uncited_normative_recommendation_ids: ["rec-naked"], + }); + }); + + it("does NOT annotate when generation throws (only on success)", async () => { + // Provider returns invalid JSON twice → InsightSchemaError surfaces. + const provider = new MockAIProvider({ + responses: ["not json", "still not json"], + }); + await expect( + generateInsight(provider, { systemPrompt: "s", userPrompt: "u" }), + ).rejects.toThrow(); + expect(annotateMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/ai/citation-coverage.ts b/src/lib/ai/citation-coverage.ts new file mode 100644 index 000000000..bee38e069 --- /dev/null +++ b/src/lib/ai/citation-coverage.ts @@ -0,0 +1,94 @@ +import type { AIInsightResponse } from "./schema"; + +/** + * v1.4.16 phase B5a — citation-coverage post-validation. + * + * After the schema parse + cross-citation check pass, the wrapper + * counts how many recommendations make a normative claim + * ("target", "should", "normal range", "above", "below") and how + * many of those carry a `referenceId` pointing into the curated + * medical-reference bundle. The result lands as a Wide-Event meta + * annotation so the admin AI quality dashboard can track coverage + * over time. + * + * The check is observational only — a normative rec without a + * referenceId is logged as a warning, never raised as a parse + * failure. v1.4.16 phase B5c flips it to required for severity + * >= "important". + * + * Heuristic policy: substring match against a small fixed keyword + * list. Locale-agnostic for v1.4.16; phase B5c may swap in a tagged + * model output ("rec.kind: normative | observational"). + */ + +const NORMATIVE_KEYWORDS_EN = [ + "target", + "should", + "normal range", + "above", + "below", +]; + +const NORMATIVE_KEYWORDS_DE = [ + "ziel", + "sollte", + "normalbereich", + "über", + "unter", +]; + +const NORMATIVE_KEYWORDS = [...NORMATIVE_KEYWORDS_EN, ...NORMATIVE_KEYWORDS_DE]; + +/** + * Returns true when the rec text contains a normative-claim keyword. + * Case-insensitive substring match; no word-boundary check because + * the keywords are common-English/-German fragments and the LLM + * sometimes inflects them ("targets", "should be" all hit). + */ +export function detectsNormativeClaim(text: string): boolean { + const lower = text.toLowerCase(); + return NORMATIVE_KEYWORDS.some((kw) => lower.includes(kw)); +} + +export interface CitationCoverage { + /** Total `recommendations[]` length. */ + totalRecommendations: number; + /** Subset of recs that make a normative claim. */ + normativeRecommendations: number; + /** Subset of normative recs that carry a `referenceId`. */ + citedNormativeRecommendations: number; + /** Recommendation ids that are normative but lack a referenceId. */ + uncitedNormativeRecommendationIds: string[]; +} + +/** + * Compute the citation-coverage breakdown for a parsed insight + * response. Pure function — no side effects, safe to unit test in + * isolation. The wrapper at `generate-insight.ts` calls this once on + * the successful parse and forwards the result via `annotate()`. + */ +export function computeCitationCoverage( + parsed: AIInsightResponse, +): CitationCoverage { + const total = parsed.recommendations.length; + let normative = 0; + let cited = 0; + const uncitedIds: string[] = []; + + for (const rec of parsed.recommendations) { + if (!detectsNormativeClaim(rec.text)) continue; + normative += 1; + if (rec.referenceId) { + cited += 1; + } else { + uncitedIds.push(rec.id); + } + } + + return { + totalRecommendations: total, + normativeRecommendations: normative, + citedNormativeRecommendations: cited, + uncitedNormativeRecommendationIds: uncitedIds, + }; +} diff --git a/src/lib/ai/generate-insight.ts b/src/lib/ai/generate-insight.ts index 401780083..8c3abe9b5 100644 --- a/src/lib/ai/generate-insight.ts +++ b/src/lib/ai/generate-insight.ts @@ -6,6 +6,8 @@ import { InsightSchemaError, type AIInsightResponse, } from "./schema"; +import { computeCitationCoverage } from "./citation-coverage"; +import { annotate } from "@/lib/logging/context"; /** * Schema-enforced wrapper around `AIProvider.generateCompletion()`. @@ -130,6 +132,25 @@ async function tryOnce( return { ok: true, parsed: result.data, raw }; } +/** + * Forward the citation-coverage breakdown into the active Wide-Event + * via `annotate()` (no-op outside a request context). Keys are + * snake_case to match the rest of the AI Wide-Event vocabulary. + */ +function annotateCitationCoverage(parsed: AIInsightResponse): void { + const coverage = computeCitationCoverage(parsed); + annotate({ + meta: { + ai_total_recommendations: coverage.totalRecommendations, + ai_normative_recommendations: coverage.normativeRecommendations, + ai_cited_normative_recommendations: + coverage.citedNormativeRecommendations, + ai_uncited_normative_recommendation_ids: + coverage.uncitedNormativeRecommendationIds, + }, + }); +} + /** * Run a schema-enforced insight generation. Up to 2 provider calls. * Throws `InsightSchemaError` (httpStatus 422) if both attempts fail @@ -141,6 +162,7 @@ export async function generateInsight( ): Promise { const first = await tryOnce(provider, params); if (first.ok && first.parsed) { + annotateCitationCoverage(first.parsed); return { parsed: first.parsed, raw: first.raw, @@ -160,6 +182,7 @@ export async function generateInsight( }; const second = await tryOnce(provider, retryParams); if (second.ok && second.parsed) { + annotateCitationCoverage(second.parsed); return { parsed: second.parsed, raw: second.raw,