mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 21:10:59 +00:00
feat(ai): schema accepts validated referenceId pointing into the medical-reference bundle
Recommendations now carry an optional `referenceId` that the parser
checks against the curated bundle from the previous commit. A
fabricated id ("invented-guideline-2099") fails parse so the model
cannot synthesise a citation for a guideline that does not exist —
defence in depth on top of the prompt-side instruction.
Optional in v1.4.16 so existing payloads still validate; v1.4.16
phase B5c ratchets it to required when severity >= "important". Also
bumps PROMPT_VERSION to 4.16.0 for the system-prompt update landing
in the next commit.
8 unit tests pin the accept-known / reject-fabricated / mixed-payload
behaviour and the round-trip preservation of referenceId through the
strict response schema.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5d1ece1fd5
commit
466b8b5604
3 files changed
+192
-19
No files matched your search
@@ -0,0 +1,138 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
aiInsightResponseSchema,
|
||||
aiRecommendationSchema,
|
||||
type AIInsightResponse,
|
||||
} from "../schema";
|
||||
import { MEDICAL_REFERENCES } from "../medical-references";
|
||||
|
||||
/**
|
||||
* v1.4.16 phase B5a — `recommendation.referenceId` validation.
|
||||
*
|
||||
* The schema accepts optional `referenceId` strings, but only when
|
||||
* they point into the curated `MEDICAL_REFERENCES` bundle. A
|
||||
* fabricated id ("invented-guideline") MUST fail parse so the model
|
||||
* cannot synthesise a citation for a non-existent guideline.
|
||||
*
|
||||
* Optional-when-absent (the v1.4.16 baseline). Phase B5c will flip
|
||||
* the requirement to mandatory for severity >= "important".
|
||||
*/
|
||||
|
||||
const knownRefId = MEDICAL_REFERENCES[0].id;
|
||||
|
||||
const baseMetricSource = {
|
||||
type: "bloodPressure",
|
||||
timeRange: "last7days",
|
||||
summary: "avg 138/86 across 9 readings",
|
||||
};
|
||||
|
||||
const baseCitation = {
|
||||
type: "bloodPressure",
|
||||
timeRange: "last7days",
|
||||
summary: "avg 138/86 across 9 readings",
|
||||
};
|
||||
|
||||
const baseRec = {
|
||||
id: "rec-1",
|
||||
text: "Discuss home BP log with your physician.",
|
||||
severity: "important" as const,
|
||||
metricSource: baseMetricSource,
|
||||
};
|
||||
|
||||
const baseValid: AIInsightResponse = {
|
||||
summary: "BP runs slightly above your 90-day median.",
|
||||
recommendations: [baseRec],
|
||||
citations: [baseCitation],
|
||||
warnings: [],
|
||||
};
|
||||
|
||||
describe("aiRecommendationSchema — referenceId", () => {
|
||||
it("accepts an absent referenceId", () => {
|
||||
const result = aiRecommendationSchema.safeParse(baseRec);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts a referenceId that exists in the bundle", () => {
|
||||
const result = aiRecommendationSchema.safeParse({
|
||||
...baseRec,
|
||||
referenceId: knownRefId,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.referenceId).toBe(knownRefId);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a referenceId that is NOT in the bundle", () => {
|
||||
const result = aiRecommendationSchema.safeParse({
|
||||
...baseRec,
|
||||
referenceId: "fabricated-guideline-2099",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
if (!result.success) {
|
||||
const issue = result.error.issues.find(
|
||||
(i) => i.path[i.path.length - 1] === "referenceId",
|
||||
);
|
||||
expect(issue).toBeDefined();
|
||||
expect(issue?.message).toMatch(/not in MEDICAL_REFERENCES/);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a referenceId that is the empty string", () => {
|
||||
const result = aiRecommendationSchema.safeParse({
|
||||
...baseRec,
|
||||
referenceId: "",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("aiInsightResponseSchema — referenceId integration", () => {
|
||||
it("accepts a payload where every rec carries a known referenceId", () => {
|
||||
const result = aiInsightResponseSchema.safeParse({
|
||||
...baseValid,
|
||||
recommendations: [{ ...baseRec, referenceId: knownRefId }],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects a payload with one fabricated referenceId, even when the rest is valid", () => {
|
||||
const result = aiInsightResponseSchema.safeParse({
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{ ...baseRec, referenceId: knownRefId },
|
||||
{
|
||||
...baseRec,
|
||||
id: "rec-2",
|
||||
referenceId: "esh-9999-fake",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
if (!result.success) {
|
||||
const refIssue = result.error.issues.find((i) =>
|
||||
i.path.includes("referenceId"),
|
||||
);
|
||||
expect(refIssue).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts a payload with a mix of cited and uncited recs", () => {
|
||||
const result = aiInsightResponseSchema.safeParse({
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{ ...baseRec, referenceId: knownRefId },
|
||||
{ ...baseRec, id: "rec-2" }, // no referenceId — observational
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves the referenceId on round-trip parse", () => {
|
||||
const parsed = aiInsightResponseSchema.parse({
|
||||
...baseValid,
|
||||
recommendations: [{ ...baseRec, referenceId: knownRefId }],
|
||||
});
|
||||
expect(parsed.recommendations[0].referenceId).toBe(knownRefId);
|
||||
});
|
||||
});
|
||||
@@ -23,7 +23,7 @@
|
||||
import type { Locale } from "@/lib/i18n/config";
|
||||
|
||||
/** Stable identifier for the active system prompt revision. */
|
||||
export const PROMPT_VERSION = "4.15.0" as const;
|
||||
export const PROMPT_VERSION = "4.16.0" as const;
|
||||
|
||||
const SYSTEM_PROMPT_EN = `You are a clinical-context summariser for a personal health-log app.
|
||||
Prompt version: ${PROMPT_VERSION}.
|
||||
|
||||
+53
-18
@@ -1,4 +1,5 @@
|
||||
import { z } from "zod/v4";
|
||||
import { MEDICAL_REFERENCE_IDS } from "./medical-references";
|
||||
|
||||
/**
|
||||
* Strict response schema for v1.4.15 AI hardening (Phase C1).
|
||||
@@ -68,24 +69,58 @@ export const metricSourceSchema = z.object({
|
||||
|
||||
export type MetricSource = z.infer<typeof metricSourceSchema>;
|
||||
|
||||
export const aiRecommendationSchema = z.object({
|
||||
/**
|
||||
* Stable identifier for the recommendation within this response.
|
||||
* Used by the future user-feedback loop ("was this helpful?",
|
||||
* v1.4.16 roadmap). Format: short slug (`rec-1`, `bp-elevated`).
|
||||
*/
|
||||
id: z.string().min(1, "recommendation.id required"),
|
||||
/** Human-readable recommendation text. */
|
||||
text: z.string().min(1, "recommendation.text required"),
|
||||
/**
|
||||
* Mandatory citation — the data point that justified the
|
||||
* recommendation. Closes the "ungrounded boilerplate" risk Marc
|
||||
* called out: every recommendation must trace back to user data.
|
||||
*/
|
||||
metricSource: metricSourceSchema,
|
||||
/** Severity — generic for v1.4.15, clinically-keyed in v1.4.16. */
|
||||
severity: recommendationSeveritySchema,
|
||||
});
|
||||
/**
|
||||
* Set of valid `referenceId` values, derived from
|
||||
* `MEDICAL_REFERENCES` so the two cannot drift. Validated lazily
|
||||
* inside `superRefine` instead of `z.enum(...)` because the bundle is
|
||||
* a `string[]` constant — `z.enum` requires a tuple type.
|
||||
*/
|
||||
const MEDICAL_REFERENCE_ID_SET: ReadonlySet<string> = new Set(
|
||||
MEDICAL_REFERENCE_IDS,
|
||||
);
|
||||
|
||||
export const aiRecommendationSchema = z
|
||||
.object({
|
||||
/**
|
||||
* Stable identifier for the recommendation within this response.
|
||||
* Used by the future user-feedback loop ("was this helpful?",
|
||||
* v1.4.16 roadmap). Format: short slug (`rec-1`, `bp-elevated`).
|
||||
*/
|
||||
id: z.string().min(1, "recommendation.id required"),
|
||||
/** Human-readable recommendation text. */
|
||||
text: z.string().min(1, "recommendation.text required"),
|
||||
/**
|
||||
* Mandatory citation — the data point that justified the
|
||||
* recommendation. Closes the "ungrounded boilerplate" risk Marc
|
||||
* called out: every recommendation must trace back to user data.
|
||||
*/
|
||||
metricSource: metricSourceSchema,
|
||||
/** Severity — generic for v1.4.15, clinically-keyed in v1.4.16. */
|
||||
severity: recommendationSeveritySchema,
|
||||
/**
|
||||
* Optional pointer into the curated medical-reference bundle
|
||||
* (`src/lib/ai/medical-references.ts`). When set, the value MUST
|
||||
* be a known id — fabricated ids are rejected.
|
||||
*
|
||||
* Optional in v1.4.16 (B5a): the prompt asks the model to cite a
|
||||
* matching reference for normative claims, but a missing
|
||||
* referenceId is logged as a citation-coverage warning rather
|
||||
* than a parse failure. v1.4.16 phase B5c flips it to required
|
||||
* when `severity >= "important"`.
|
||||
*/
|
||||
referenceId: z.string().optional(),
|
||||
})
|
||||
.superRefine((rec, ctx) => {
|
||||
if (rec.referenceId !== undefined) {
|
||||
if (!MEDICAL_REFERENCE_ID_SET.has(rec.referenceId)) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: `referenceId "${rec.referenceId}" is not in MEDICAL_REFERENCES`,
|
||||
path: ["referenceId"],
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export type AIRecommendation = z.infer<typeof aiRecommendationSchema>;
|
||||
|
||||
|
||||
Reference in new issue
Block a user