mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 12:22:03 +00:00
feat(ai): enforce citation-from-data on every recommendation
Phase C1 (v1.4.15) — citation-from-data hard requirement.
Marc, verbatim 2026-05-09: "es muss sich halt irgendwie stützen auf
medizinische Dinge." ("It must ground on medical facts / user data.")
Builds on the strict response schema (`src/lib/ai/schema.ts`) and
schema-enforced wrapper (`src/lib/ai/generate-insight.ts`) — which were
swept into sibling commit 5510ed5 by a parallel-agent index race
(documented across A2 / A4 / B1-B6 / C2 / C3 of the v1.4.15 marathon;
diff is correct on main, message scope misleading).
This commit adds a focused 11-test invariant suite that nails down the
contract end-to-end:
1. Schema-level: recommendation without metricSource fails parse;
empty metricSource.summary fails; missing type or timeRange fails.
2. Cross-check: findUncitedRecommendations() flags any recommendation
whose (type, timeRange) does not appear in citations[]. Partial
match (type alone) is rejected — both fields must align.
3. Wrapper end-to-end: orphan-citation triggers retry; second attempt
either fixes the citation OR drops the recommendation (both
valid per the retry-correction message; model is explicitly
told an empty recommendations[] is acceptable).
The two-pronged guard (schema + cross-check) ensures the model cannot
fabricate a recommendation that points at "nothing" AND cannot cite
data it hasn't surfaced — closing the v1.4.16 medical-grounding work's
prerequisite.
Tests: +11 (988 → 999 unit). All AI tests green (67 / 67).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Marc-André BombeckandClaude Opus 4.7 committed
1 parent
5510ed529b
commit
d657f798d0
1 file changed
+269
@@ -0,0 +1,269 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
aiInsightResponseSchema,
|
||||
findUncitedRecommendations,
|
||||
metricSourceSchema,
|
||||
aiRecommendationSchema,
|
||||
type AIInsightResponse,
|
||||
} from "../schema";
|
||||
import { generateInsight } from "../generate-insight";
|
||||
import { MockAIProvider } from "../mock-client";
|
||||
|
||||
/**
|
||||
* Phase C1 — citation-from-data hard requirement.
|
||||
*
|
||||
* Marc, verbatim 2026-05-09: "es muss sich halt irgendwie stützen auf
|
||||
* medizinische Dinge." ("It must ground on medical facts / user data.")
|
||||
*
|
||||
* Operational meaning:
|
||||
*
|
||||
* 1. Every recommendation MUST carry a `metricSource` with all three
|
||||
* required fields populated (`type`, `timeRange`, `summary`).
|
||||
* Empty / missing → schema parse fails. The model cannot
|
||||
* fabricate a recommendation that points at "nothing".
|
||||
*
|
||||
* 2. Every recommendation's `metricSource` MUST be backed by an
|
||||
* entry in `citations[]` (matching `type` + `timeRange`).
|
||||
* Cross-check enforced by `findUncitedRecommendations` and run
|
||||
* by the wrapper after schema parse.
|
||||
*
|
||||
* 3. Two separate failure modes (missing metricSource vs. missing
|
||||
* citation) both surface as zod / wrapper errors so the route
|
||||
* can return 422 in either case.
|
||||
*/
|
||||
|
||||
const baseMetricSource = {
|
||||
type: "bloodPressure",
|
||||
timeRange: "last7days",
|
||||
summary: "avg 138/86 across 9 readings",
|
||||
};
|
||||
|
||||
const baseCitation = {
|
||||
type: "bloodPressure",
|
||||
timeRange: "last7days",
|
||||
summary: "avg 138/86 across 9 readings",
|
||||
};
|
||||
|
||||
const baseValid: AIInsightResponse = {
|
||||
summary: "BP runs slightly above your 90-day median.",
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-1",
|
||||
text: "Discuss home BP log with your physician.",
|
||||
severity: "important",
|
||||
metricSource: baseMetricSource,
|
||||
},
|
||||
],
|
||||
citations: [baseCitation],
|
||||
warnings: [],
|
||||
};
|
||||
|
||||
describe("citation-from-data — schema-level enforcement", () => {
|
||||
it("recommendation without metricSource fails parse", () => {
|
||||
const broken = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-1",
|
||||
text: "Walk more.",
|
||||
severity: "suggestion",
|
||||
// metricSource missing entirely
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = aiRecommendationSchema.safeParse(broken.recommendations[0]);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("recommendation with empty metricSource.summary fails parse", () => {
|
||||
const broken = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-1",
|
||||
text: "Walk more.",
|
||||
severity: "suggestion",
|
||||
metricSource: { type: "activity", timeRange: "last7days", summary: "" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = aiInsightResponseSchema.safeParse(broken);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("recommendation with missing metricSource.type fails parse", () => {
|
||||
const broken = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-1",
|
||||
text: "X",
|
||||
severity: "info",
|
||||
metricSource: { timeRange: "last7days", summary: "data" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = aiInsightResponseSchema.safeParse(broken);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("recommendation with missing metricSource.timeRange fails parse", () => {
|
||||
const result = aiInsightResponseSchema.safeParse({
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-1",
|
||||
text: "X",
|
||||
severity: "info",
|
||||
metricSource: { type: "bloodPressure", summary: "data" },
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("metricSource alone (without recommendation wrapper) — empty summary still fails", () => {
|
||||
const result = metricSourceSchema.safeParse({
|
||||
type: "x",
|
||||
timeRange: "y",
|
||||
summary: "",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("well-formed citation gets through", () => {
|
||||
const result = aiInsightResponseSchema.safeParse(baseValid);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("citation-from-data — cross-check enforcement", () => {
|
||||
it("recommendation citing a (type, timeRange) absent from citations[] is flagged", () => {
|
||||
const orphan: AIInsightResponse = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-orphan",
|
||||
text: "Cut sodium.",
|
||||
severity: "suggestion",
|
||||
metricSource: {
|
||||
type: "diet",
|
||||
timeRange: "last7days",
|
||||
summary: "no diet logging available",
|
||||
},
|
||||
},
|
||||
],
|
||||
citations: [baseCitation], // diet citation missing
|
||||
};
|
||||
const missing = findUncitedRecommendations(orphan);
|
||||
expect(missing).toHaveLength(1);
|
||||
expect(missing[0].recommendationId).toBe("rec-orphan");
|
||||
expect(missing[0].missing).toEqual({ type: "diet", timeRange: "last7days" });
|
||||
});
|
||||
|
||||
it("partial-key match on type alone is NOT enough — timeRange must also match", () => {
|
||||
const orphan: AIInsightResponse = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-mismatch",
|
||||
text: "X",
|
||||
severity: "info",
|
||||
metricSource: {
|
||||
type: "bloodPressure",
|
||||
timeRange: "last30days", // citation only covers last7days
|
||||
summary: "avg over 30 days",
|
||||
},
|
||||
},
|
||||
],
|
||||
citations: [baseCitation],
|
||||
};
|
||||
const missing = findUncitedRecommendations(orphan);
|
||||
expect(missing).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("multiple recommendations sharing the same citation pass once that one citation is present", () => {
|
||||
const ok: AIInsightResponse = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
baseValid.recommendations[0],
|
||||
{
|
||||
id: "rec-2",
|
||||
text: "Continue daily logging.",
|
||||
severity: "info",
|
||||
metricSource: { ...baseMetricSource },
|
||||
},
|
||||
],
|
||||
citations: [baseCitation],
|
||||
};
|
||||
const missing = findUncitedRecommendations(ok);
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("citation-from-data — wrapper end-to-end", () => {
|
||||
it("uncited recommendation triggers retry; second try with proper citation passes", async () => {
|
||||
const orphan = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-orphan",
|
||||
text: "Cut sodium.",
|
||||
severity: "suggestion",
|
||||
metricSource: {
|
||||
type: "diet",
|
||||
timeRange: "last7days",
|
||||
summary: "no diet logging available",
|
||||
},
|
||||
},
|
||||
],
|
||||
citations: [],
|
||||
};
|
||||
const provider = new MockAIProvider({
|
||||
responses: [JSON.stringify(orphan), JSON.stringify(baseValid)],
|
||||
});
|
||||
const outcome = await generateInsight(provider, {
|
||||
systemPrompt: "system",
|
||||
userPrompt: "user",
|
||||
});
|
||||
expect(outcome.retried).toBe(true);
|
||||
expect(outcome.parsed.recommendations[0].id).toBe("rec-1");
|
||||
});
|
||||
|
||||
it("retry-attempt that drops the recommendation rather than fixing the citation also passes", async () => {
|
||||
const orphan = {
|
||||
...baseValid,
|
||||
recommendations: [
|
||||
{
|
||||
id: "rec-orphan",
|
||||
text: "Cut sodium.",
|
||||
severity: "suggestion",
|
||||
metricSource: {
|
||||
type: "diet",
|
||||
timeRange: "last7days",
|
||||
summary: "no diet logging available",
|
||||
},
|
||||
},
|
||||
],
|
||||
citations: [],
|
||||
};
|
||||
// Second attempt — model decides to drop the unsupported recommendation
|
||||
// entirely. This is the explicitly-allowed "OMIT it" path from the
|
||||
// retry-correction message.
|
||||
const droppedRecs = {
|
||||
summary: "Insufficient data for actionable recommendations.",
|
||||
recommendations: [],
|
||||
citations: [],
|
||||
warnings: [],
|
||||
};
|
||||
const provider = new MockAIProvider({
|
||||
responses: [JSON.stringify(orphan), JSON.stringify(droppedRecs)],
|
||||
});
|
||||
const outcome = await generateInsight(provider, {
|
||||
systemPrompt: "system",
|
||||
userPrompt: "user",
|
||||
});
|
||||
expect(outcome.retried).toBe(true);
|
||||
expect(outcome.parsed.recommendations).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user