diff --git a/src/lib/ai/__tests__/reference-id-schema.test.ts b/src/lib/ai/__tests__/reference-id-schema.test.ts new file mode 100644 index 000000000..e3689516d --- /dev/null +++ b/src/lib/ai/__tests__/reference-id-schema.test.ts @@ -0,0 +1,138 @@ +import { describe, it, expect } from "vitest"; +import { + aiInsightResponseSchema, + aiRecommendationSchema, + type AIInsightResponse, +} from "../schema"; +import { MEDICAL_REFERENCES } from "../medical-references"; + +/** + * v1.4.16 phase B5a — `recommendation.referenceId` validation. + * + * The schema accepts optional `referenceId` strings, but only when + * they point into the curated `MEDICAL_REFERENCES` bundle. A + * fabricated id ("invented-guideline") MUST fail parse so the model + * cannot synthesise a citation for a non-existent guideline. + * + * Optional-when-absent (the v1.4.16 baseline). Phase B5c will flip + * the requirement to mandatory for severity >= "important". + */ + +const knownRefId = MEDICAL_REFERENCES[0].id; + +const baseMetricSource = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const baseCitation = { + type: "bloodPressure", + timeRange: "last7days", + summary: "avg 138/86 across 9 readings", +}; + +const baseRec = { + id: "rec-1", + text: "Discuss home BP log with your physician.", + severity: "important" as const, + metricSource: baseMetricSource, +}; + +const baseValid: AIInsightResponse = { + summary: "BP runs slightly above your 90-day median.", + recommendations: [baseRec], + citations: [baseCitation], + warnings: [], +}; + +describe("aiRecommendationSchema — referenceId", () => { + it("accepts an absent referenceId", () => { + const result = aiRecommendationSchema.safeParse(baseRec); + expect(result.success).toBe(true); + }); + + it("accepts a referenceId that exists in the bundle", () => { + const result = aiRecommendationSchema.safeParse({ + ...baseRec, + referenceId: knownRefId, + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.referenceId).toBe(knownRefId); + } + }); + + it("rejects a referenceId that is NOT in the bundle", () => { + const result = aiRecommendationSchema.safeParse({ + ...baseRec, + referenceId: "fabricated-guideline-2099", + }); + expect(result.success).toBe(false); + if (!result.success) { + const issue = result.error.issues.find( + (i) => i.path[i.path.length - 1] === "referenceId", + ); + expect(issue).toBeDefined(); + expect(issue?.message).toMatch(/not in MEDICAL_REFERENCES/); + } + }); + + it("rejects a referenceId that is the empty string", () => { + const result = aiRecommendationSchema.safeParse({ + ...baseRec, + referenceId: "", + }); + expect(result.success).toBe(false); + }); +}); + +describe("aiInsightResponseSchema — referenceId integration", () => { + it("accepts a payload where every rec carries a known referenceId", () => { + const result = aiInsightResponseSchema.safeParse({ + ...baseValid, + recommendations: [{ ...baseRec, referenceId: knownRefId }], + }); + expect(result.success).toBe(true); + }); + + it("rejects a payload with one fabricated referenceId, even when the rest is valid", () => { + const result = aiInsightResponseSchema.safeParse({ + ...baseValid, + recommendations: [ + { ...baseRec, referenceId: knownRefId }, + { + ...baseRec, + id: "rec-2", + referenceId: "esh-9999-fake", + }, + ], + }); + expect(result.success).toBe(false); + if (!result.success) { + const refIssue = result.error.issues.find((i) => + i.path.includes("referenceId"), + ); + expect(refIssue).toBeDefined(); + } + }); + + it("accepts a payload with a mix of cited and uncited recs", () => { + const result = aiInsightResponseSchema.safeParse({ + ...baseValid, + recommendations: [ + { ...baseRec, referenceId: knownRefId }, + { ...baseRec, id: "rec-2" }, // no referenceId — observational + ], + }); + expect(result.success).toBe(true); + }); + + it("preserves the referenceId on round-trip parse", () => { + const parsed = aiInsightResponseSchema.parse({ + ...baseValid, + recommendations: [{ ...baseRec, referenceId: knownRefId }], + }); + expect(parsed.recommendations[0].referenceId).toBe(knownRefId); + }); +}); diff --git a/src/lib/ai/prompts/insight-generator.ts b/src/lib/ai/prompts/insight-generator.ts index 47f4f89e6..4a9691e72 100644 --- a/src/lib/ai/prompts/insight-generator.ts +++ b/src/lib/ai/prompts/insight-generator.ts @@ -23,7 +23,7 @@ import type { Locale } from "@/lib/i18n/config"; /** Stable identifier for the active system prompt revision. */ -export const PROMPT_VERSION = "4.15.0" as const; +export const PROMPT_VERSION = "4.16.0" as const; const SYSTEM_PROMPT_EN = `You are a clinical-context summariser for a personal health-log app. Prompt version: ${PROMPT_VERSION}. diff --git a/src/lib/ai/schema.ts b/src/lib/ai/schema.ts index 0498e069e..dff9eb225 100644 --- a/src/lib/ai/schema.ts +++ b/src/lib/ai/schema.ts @@ -1,4 +1,5 @@ import { z } from "zod/v4"; +import { MEDICAL_REFERENCE_IDS } from "./medical-references"; /** * Strict response schema for v1.4.15 AI hardening (Phase C1). @@ -68,24 +69,58 @@ export const metricSourceSchema = z.object({ export type MetricSource = z.infer; -export const aiRecommendationSchema = z.object({ - /** - * Stable identifier for the recommendation within this response. - * Used by the future user-feedback loop ("was this helpful?", - * v1.4.16 roadmap). Format: short slug (`rec-1`, `bp-elevated`). - */ - id: z.string().min(1, "recommendation.id required"), - /** Human-readable recommendation text. */ - text: z.string().min(1, "recommendation.text required"), - /** - * Mandatory citation — the data point that justified the - * recommendation. Closes the "ungrounded boilerplate" risk Marc - * called out: every recommendation must trace back to user data. - */ - metricSource: metricSourceSchema, - /** Severity — generic for v1.4.15, clinically-keyed in v1.4.16. */ - severity: recommendationSeveritySchema, -}); +/** + * Set of valid `referenceId` values, derived from + * `MEDICAL_REFERENCES` so the two cannot drift. Validated lazily + * inside `superRefine` instead of `z.enum(...)` because the bundle is + * a `string[]` constant — `z.enum` requires a tuple type. + */ +const MEDICAL_REFERENCE_ID_SET: ReadonlySet = new Set( + MEDICAL_REFERENCE_IDS, +); + +export const aiRecommendationSchema = z + .object({ + /** + * Stable identifier for the recommendation within this response. + * Used by the future user-feedback loop ("was this helpful?", + * v1.4.16 roadmap). Format: short slug (`rec-1`, `bp-elevated`). + */ + id: z.string().min(1, "recommendation.id required"), + /** Human-readable recommendation text. */ + text: z.string().min(1, "recommendation.text required"), + /** + * Mandatory citation — the data point that justified the + * recommendation. Closes the "ungrounded boilerplate" risk Marc + * called out: every recommendation must trace back to user data. + */ + metricSource: metricSourceSchema, + /** Severity — generic for v1.4.15, clinically-keyed in v1.4.16. */ + severity: recommendationSeveritySchema, + /** + * Optional pointer into the curated medical-reference bundle + * (`src/lib/ai/medical-references.ts`). When set, the value MUST + * be a known id — fabricated ids are rejected. + * + * Optional in v1.4.16 (B5a): the prompt asks the model to cite a + * matching reference for normative claims, but a missing + * referenceId is logged as a citation-coverage warning rather + * than a parse failure. v1.4.16 phase B5c flips it to required + * when `severity >= "important"`. + */ + referenceId: z.string().optional(), + }) + .superRefine((rec, ctx) => { + if (rec.referenceId !== undefined) { + if (!MEDICAL_REFERENCE_ID_SET.has(rec.referenceId)) { + ctx.addIssue({ + code: "custom", + message: `referenceId "${rec.referenceId}" is not in MEDICAL_REFERENCES`, + path: ["referenceId"], + }); + } + } + }); export type AIRecommendation = z.infer;