feat(ai): schema accepts validated referenceId pointing into the medical-reference bundle

Recommendations now carry an optional `referenceId` that the parser
checks against the curated bundle from the previous commit. A
fabricated id ("invented-guideline-2099") fails parse so the model
cannot synthesise a citation for a guideline that does not exist —
defence in depth on top of the prompt-side instruction.

Optional in v1.4.16 so existing payloads still validate; v1.4.16
phase B5c ratchets it to required when severity >= "important". Also
bumps PROMPT_VERSION to 4.16.0 for the system-prompt update landing
in the next commit.

8 unit tests pin the accept-known / reject-fabricated / mixed-payload
behaviour and the round-trip preservation of referenceId through the
strict response schema.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Marc-André BombeckandClaude Opus 4.7 committed 2026-05-09 23:47:48 +02:00
1 parent 5d1ece1fd5
commit 466b8b5604
3 files changed
+192 -19

No files matched your search

@@ -0,0 +1,138 @@
import { describe, it, expect } from "vitest";
import {
aiInsightResponseSchema,
aiRecommendationSchema,
type AIInsightResponse,
} from "../schema";
import { MEDICAL_REFERENCES } from "../medical-references";
/**
* v1.4.16 phase B5a — `recommendation.referenceId` validation.
*
* The schema accepts optional `referenceId` strings, but only when
* they point into the curated `MEDICAL_REFERENCES` bundle. A
* fabricated id ("invented-guideline") MUST fail parse so the model
* cannot synthesise a citation for a non-existent guideline.
*
* Optional-when-absent (the v1.4.16 baseline). Phase B5c will flip
* the requirement to mandatory for severity >= "important".
*/
const knownRefId = MEDICAL_REFERENCES[0].id;
const baseMetricSource = {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 138/86 across 9 readings",
};
const baseCitation = {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 138/86 across 9 readings",
};
const baseRec = {
id: "rec-1",
text: "Discuss home BP log with your physician.",
severity: "important" as const,
metricSource: baseMetricSource,
};
const baseValid: AIInsightResponse = {
summary: "BP runs slightly above your 90-day median.",
recommendations: [baseRec],
citations: [baseCitation],
warnings: [],
};
describe("aiRecommendationSchema — referenceId", () => {
it("accepts an absent referenceId", () => {
const result = aiRecommendationSchema.safeParse(baseRec);
expect(result.success).toBe(true);
});
it("accepts a referenceId that exists in the bundle", () => {
const result = aiRecommendationSchema.safeParse({
...baseRec,
referenceId: knownRefId,
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.referenceId).toBe(knownRefId);
}
});
it("rejects a referenceId that is NOT in the bundle", () => {
const result = aiRecommendationSchema.safeParse({
...baseRec,
referenceId: "fabricated-guideline-2099",
});
expect(result.success).toBe(false);
if (!result.success) {
const issue = result.error.issues.find(
(i) => i.path[i.path.length - 1] === "referenceId",
);
expect(issue).toBeDefined();
expect(issue?.message).toMatch(/not in MEDICAL_REFERENCES/);
}
});
it("rejects a referenceId that is the empty string", () => {
const result = aiRecommendationSchema.safeParse({
...baseRec,
referenceId: "",
});
expect(result.success).toBe(false);
});
});
describe("aiInsightResponseSchema — referenceId integration", () => {
it("accepts a payload where every rec carries a known referenceId", () => {
const result = aiInsightResponseSchema.safeParse({
...baseValid,
recommendations: [{ ...baseRec, referenceId: knownRefId }],
});
expect(result.success).toBe(true);
});
it("rejects a payload with one fabricated referenceId, even when the rest is valid", () => {
const result = aiInsightResponseSchema.safeParse({
...baseValid,
recommendations: [
{ ...baseRec, referenceId: knownRefId },
{
...baseRec,
id: "rec-2",
referenceId: "esh-9999-fake",
},
],
});
expect(result.success).toBe(false);
if (!result.success) {
const refIssue = result.error.issues.find((i) =>
i.path.includes("referenceId"),
);
expect(refIssue).toBeDefined();
}
});
it("accepts a payload with a mix of cited and uncited recs", () => {
const result = aiInsightResponseSchema.safeParse({
...baseValid,
recommendations: [
{ ...baseRec, referenceId: knownRefId },
{ ...baseRec, id: "rec-2" }, // no referenceId — observational
],
});
expect(result.success).toBe(true);
});
it("preserves the referenceId on round-trip parse", () => {
const parsed = aiInsightResponseSchema.parse({
...baseValid,
recommendations: [{ ...baseRec, referenceId: knownRefId }],
});
expect(parsed.recommendations[0].referenceId).toBe(knownRefId);
});
});
+1 -1
View File
@@ -23,7 +23,7 @@
import type { Locale } from "@/lib/i18n/config";
/** Stable identifier for the active system prompt revision. */
export const PROMPT_VERSION = "4.15.0" as const;
export const PROMPT_VERSION = "4.16.0" as const;
const SYSTEM_PROMPT_EN = `You are a clinical-context summariser for a personal health-log app.
Prompt version: ${PROMPT_VERSION}.
+53 -18
View File
@@ -1,4 +1,5 @@
import { z } from "zod/v4";
import { MEDICAL_REFERENCE_IDS } from "./medical-references";
/**
* Strict response schema for v1.4.15 AI hardening (Phase C1).
@@ -68,24 +69,58 @@ export const metricSourceSchema = z.object({
export type MetricSource = z.infer<typeof metricSourceSchema>;
export const aiRecommendationSchema = z.object({
/**
* Stable identifier for the recommendation within this response.
* Used by the future user-feedback loop ("was this helpful?",
* v1.4.16 roadmap). Format: short slug (`rec-1`, `bp-elevated`).
*/
id: z.string().min(1, "recommendation.id required"),
/** Human-readable recommendation text. */
text: z.string().min(1, "recommendation.text required"),
/**
* Mandatory citation — the data point that justified the
* recommendation. Closes the "ungrounded boilerplate" risk Marc
* called out: every recommendation must trace back to user data.
*/
metricSource: metricSourceSchema,
/** Severity — generic for v1.4.15, clinically-keyed in v1.4.16. */
severity: recommendationSeveritySchema,
});
/**
* Set of valid `referenceId` values, derived from
* `MEDICAL_REFERENCES` so the two cannot drift. Validated lazily
* inside `superRefine` instead of `z.enum(...)` because the bundle is
* a `string[]` constant — `z.enum` requires a tuple type.
*/
const MEDICAL_REFERENCE_ID_SET: ReadonlySet<string> = new Set(
MEDICAL_REFERENCE_IDS,
);
export const aiRecommendationSchema = z
.object({
/**
* Stable identifier for the recommendation within this response.
* Used by the future user-feedback loop ("was this helpful?",
* v1.4.16 roadmap). Format: short slug (`rec-1`, `bp-elevated`).
*/
id: z.string().min(1, "recommendation.id required"),
/** Human-readable recommendation text. */
text: z.string().min(1, "recommendation.text required"),
/**
* Mandatory citation — the data point that justified the
* recommendation. Closes the "ungrounded boilerplate" risk Marc
* called out: every recommendation must trace back to user data.
*/
metricSource: metricSourceSchema,
/** Severity — generic for v1.4.15, clinically-keyed in v1.4.16. */
severity: recommendationSeveritySchema,
/**
* Optional pointer into the curated medical-reference bundle
* (`src/lib/ai/medical-references.ts`). When set, the value MUST
* be a known id — fabricated ids are rejected.
*
* Optional in v1.4.16 (B5a): the prompt asks the model to cite a
* matching reference for normative claims, but a missing
* referenceId is logged as a citation-coverage warning rather
* than a parse failure. v1.4.16 phase B5c flips it to required
* when `severity >= "important"`.
*/
referenceId: z.string().optional(),
})
.superRefine((rec, ctx) => {
if (rec.referenceId !== undefined) {
if (!MEDICAL_REFERENCE_ID_SET.has(rec.referenceId)) {
ctx.addIssue({
code: "custom",
message: `referenceId "${rec.referenceId}" is not in MEDICAL_REFERENCES`,
path: ["referenceId"],
});
}
}
});
export type AIRecommendation = z.infer<typeof aiRecommendationSchema>;