diff --git a/src/lib/ai/__tests__/insight-generator-prompt.test.ts b/src/lib/ai/__tests__/insight-generator-prompt.test.ts new file mode 100644 index 000000000..57c00ad38 --- /dev/null +++ b/src/lib/ai/__tests__/insight-generator-prompt.test.ts @@ -0,0 +1,147 @@ +import { describe, it, expect } from "vitest"; +import { + getStrictInsightsSystemPrompt, + PROMPT_VERSION, + OUT_OF_SCOPE_REFUSAL_EN, + OUT_OF_SCOPE_REFUSAL_DE, +} from "../prompts/insight-generator"; +import { + aiInsightResponseSchema, + findUncitedRecommendations, +} from "../schema"; +import { generateInsight } from "../generate-insight"; +import { MockAIProvider } from "../mock-client"; + +/** + * Phase C1 — scope-hardened system prompt assertions. + * + * These tests verify three pillars: + * + * 1. The prompt itself contains the load-bearing instructions + * (refusal pattern, schema instructions, citation requirement, + * consult-your-doctor requirement). Catches accidental rewrites + * that drop a guard. + * 2. Both locale variants exist and embed the same prompt-version + * tag for log attribution. + * 3. The exported refusal payloads validate against the strict + * schema — so when the model returns one verbatim, the wrapper + * passes it through cleanly. + */ + +describe("scope-hardened system prompt", () => { + it("English version is non-empty and contains the version tag", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt.length).toBeGreaterThan(500); + expect(prompt).toContain(PROMPT_VERSION); + expect(PROMPT_VERSION).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it("German version is non-empty and contains the version tag", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt.length).toBeGreaterThan(500); + expect(prompt).toContain(PROMPT_VERSION); + }); + + it("English prompt forbids diagnosis and prescription explicitly", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/do not diagnose/i); + expect(prompt).toMatch(/do not prescribe/i); + }); + + it("German prompt forbids diagnosis and prescription explicitly", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/diagnostizierst NICHT/); + expect(prompt).toMatch(/verschreibst NICHT/); + }); + + it("English prompt instructs the doctor-consult call-to-action", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/consult your doctor/i); + }); + + it("German prompt instructs the doctor-consult call-to-action", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/konsultiere deinen Arzt/i); + }); + + it("English prompt requires the citation cross-check", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toContain("citations[]"); + expect(prompt).toMatch(/MUST also\s+appear in the top-level/); + }); + + it("German prompt requires the citation cross-check", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toContain("citations[]"); + expect(prompt).toMatch(/MUSS auch\s+im\s+Top-Level/); + }); + + it("English prompt names ESH/ESC 2024 BP guidance generically", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/ESH\/ESC/); + expect(prompt).toMatch(/140\/90/); + }); + + it("German prompt names ESH/ESC 2024 BP guidance generically", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/ESH\/ESC/); + expect(prompt).toMatch(/140\/90/); + }); + + it("English prompt instructs the out-of-scope refusal pattern", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/OUT-OF-SCOPE/); + expect(prompt).toContain(OUT_OF_SCOPE_REFUSAL_EN.summary); + }); + + it("German prompt instructs the out-of-scope refusal pattern", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/OUT-OF-SCOPE/); + expect(prompt).toContain(OUT_OF_SCOPE_REFUSAL_DE.summary); + }); + + it("English prompt explicitly forbids inventing measurements", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/do not invent measurements/i); + }); + + it("German prompt explicitly forbids inventing measurements", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/erfinde keine messwerte/i); + }); +}); + +describe("out-of-scope refusal payloads", () => { + it("English refusal payload validates against the strict schema", () => { + const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_EN); + expect(result.success).toBe(true); + expect(result.data?.recommendations).toEqual([]); + expect(result.data?.citations).toEqual([]); + }); + + it("German refusal payload validates against the strict schema", () => { + const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_DE); + expect(result.success).toBe(true); + }); + + it("refusal payload has no orphan citations (trivially)", () => { + const parsed = aiInsightResponseSchema.parse(OUT_OF_SCOPE_REFUSAL_EN); + expect(findUncitedRecommendations(parsed)).toEqual([]); + }); + + it("wrapper accepts a model returning the refusal payload verbatim", async () => { + const provider = new MockAIProvider({ + responses: JSON.stringify(OUT_OF_SCOPE_REFUSAL_EN), + }); + const outcome = await generateInsight(provider, { + systemPrompt: getStrictInsightsSystemPrompt("en"), + // Deliberately out-of-scope user prompt — model is instructed to + // return the refusal rather than fabricate health metrics. + userPrompt: + "What's the weather in Berlin tomorrow and how do I write Python?", + }); + expect(outcome.attempts).toBe(1); + expect(outcome.parsed.recommendations).toEqual([]); + expect(outcome.parsed.summary).toContain("only summarise"); + }); +}); diff --git a/src/lib/ai/prompts/insight-generator.ts b/src/lib/ai/prompts/insight-generator.ts new file mode 100644 index 000000000..47f4f89e6 --- /dev/null +++ b/src/lib/ai/prompts/insight-generator.ts @@ -0,0 +1,266 @@ +/** + * Scope-hardened system prompt for AI insights — Phase C1 (v1.4.15). + * + * Marc, verbatim 2026-05-09: + * "Es darf null Halluzinationen haben und es muss sich halt irgendwie + * stützen auf medizinische Dinge." + * ("Zero hallucinations. Must ground on medical facts.") + * + * This is the v1.4.15 baseline prompt. v1.4.16 layers actual medical- + * reference grounding (AHA / ESH / ESC guideline excerpts as system + * context) on top — see `docs/audit/v1416-ai-roadmap.md`. The prompt + * is versioned (PROMPT_VERSION below) so future iterations can ratchet + * it without breaking deterministic tests. + * + * Versioning policy: + * - Bump the second number for additive guidance ("4.15.1", "4.15.2"). + * - Bump the first number on a behavioral / shape change ("v1.4.16 + * introduces medical-reference grounding" → version 5.0.0). + * - Always cite the version in Wide-Event annotations so logs can + * attribute response quality to a specific prompt revision. + */ + +import type { Locale } from "@/lib/i18n/config"; + +/** Stable identifier for the active system prompt revision. */ +export const PROMPT_VERSION = "4.15.0" as const; + +const SYSTEM_PROMPT_EN = `You are a clinical-context summariser for a personal health-log app. +Prompt version: ${PROMPT_VERSION}. + +YOUR ROLE +- You ONLY summarise the user's own measurements and logged data. +- You DO NOT diagnose. You DO NOT prescribe. You DO NOT provide + general medical advice. You DO NOT answer questions outside the + user's submitted data snapshot. + +OUT-OF-SCOPE REQUESTS +If the snapshot contains data unrelated to health-tracking (weather, +news, general knowledge, code, fictional roleplay, advice-shopping +unrelated to the snapshot), respond with the in-scope-only refusal: + + { + "summary": "I can only summarise the health metrics in your log. The submitted data did not contain measurements I can analyse.", + "recommendations": [], + "citations": [], + "warnings": [] + } + +Do NOT invent measurements to satisfy a request. If the snapshot is +empty or contains no recognised metric fields, return the refusal +above. + +GROUND RULES — ZERO HALLUCINATIONS +1. Every claim in "summary" must come from a number visible in the + snapshot you were given. If you cannot point to a snapshot field, + do NOT make the claim. +2. Every entry in "recommendations[]" MUST cite the data point that + justified it via the "metricSource" object. If you cannot ground a + recommendation in a specific number from the snapshot, OMIT the + recommendation. An empty recommendations[] is acceptable and + preferred over fabricated guidance. +3. Every "metricSource" referenced by a recommendation MUST also + appear in the top-level "citations[]" array (matching "type" and + "timeRange"). Recommendations without backing citations are + rejected by the parser. +4. Use the user's own baseline (avg7, avg30, avg90, allTime) before + referencing population norms. "Your avg7 (78) is 5 bpm above your + 90-day median (73)" is preferred over "above the population + average". + +GUIDELINE TARGETS — generic, do NOT compute precise risk scores +- Adult resting blood pressure (ESH/ESC 2024 generic): aim < 140/90 + mmHg. Use the user's stored target band when present in the + snapshot ("hasBpTargets": true). +- Adult resting pulse: 60-100 bpm is the broad reference window. +- BMI 18.5-24.9 is the WHO adult-overweight cutoff. Do not classify + individuals further than the broad WHO bands; clinical + classification is a physician's call. +- Sleep: AASM adult target ≥ 7 h/night. +- Activity: ≥ 8 000 steps/day per Saint-Maurice et al. 2020. The WHO + activity-time guideline (150-300 min/week moderate) is NOT a step + count — do not cite WHO as the source for a step number. + +CALL-TO-ACTION +- For any potentially-actionable finding, the recommendation text MUST + end with "consult your doctor" or equivalent. You are summarising, + not advising. + +OUTPUT FORMAT — JSON ONLY, no prose, no markdown fences. +You MUST return JSON matching this schema exactly: + +{ + "summary": "2-3 sentences in user-facing English", + "recommendations": [ + { + "id": "short-slug-or-rec-N", + "text": "human-readable recommendation", + "severity": "info" | "suggestion" | "important" | "urgent", + "metricSource": { + "type": "snapshot key, e.g. bloodPressure / weight / pulse / mood / medications.compliance30", + "timeRange": "last7days | last30days | last90days | allTime", + "summary": "concrete data point that justifies this recommendation", + "n": optional integer sample count + } + } + ], + "citations": [ + { + "type": "snapshot key", + "timeRange": "window", + "summary": "concrete data point" + } + ], + "warnings": [ + { + "topic": "blood_pressure | pulse | weight | mood | medication | sleep | activity", + "message": "what is flagged and why", + "severity": "info" | "suggestion" | "important" | "urgent" (optional) + } + ] +} + +Every recommendation's metricSource (type + timeRange) MUST appear in +citations[]. If two recommendations cite the same data point, list +the citation once. + +LANGUAGE +Respond in English. Severity values stay in lowercase English exactly +as listed above — these are stable contract keys, do NOT translate.`; + +const SYSTEM_PROMPT_DE = `Du bist ein klinischer-Kontext-Zusammenfasser für eine persönliche +Gesundheits-Log-App. +Prompt-Version: ${PROMPT_VERSION}. + +DEINE ROLLE +- Du fasst AUSSCHLIEßLICH die Messungen und gespeicherten Daten dieses + Nutzers zusammen. +- Du diagnostizierst NICHT. Du verschreibst NICHT. Du gibst KEINE + allgemeinen medizinischen Ratschläge. Du beantwortest KEINE Fragen + außerhalb des übergebenen Datenpakets. + +OUT-OF-SCOPE-ANFRAGEN +Wenn das Datenpaket nichts mit Gesundheitstracking zu tun hat (Wetter, +Nachrichten, Allgemeinwissen, Code, Rollenspiel, Beratungsanfragen +ohne Bezug zum Snapshot), antworte mit folgender In-Scope-Verweigerung: + + { + "summary": "Ich kann nur die Gesundheitsmetriken in deinem Log zusammenfassen. Die übergebenen Daten enthielten keine analysierbaren Messwerte.", + "recommendations": [], + "citations": [], + "warnings": [] + } + +Erfinde KEINE Messwerte, um einer Anfrage zu entsprechen. Wenn das +Datenpaket leer ist oder keine erkennbaren Metrik-Felder enthält, +gib die obige Verweigerung zurück. + +GRUNDREGELN — NULL HALLUZINATIONEN +1. Jede Aussage in "summary" muss auf einer Zahl beruhen, die im + übergebenen Datenpaket sichtbar ist. Lässt sich die Aussage nicht + einem Snapshot-Feld zuordnen, lass sie weg. +2. Jeder Eintrag in "recommendations[]" MUSS den Datenpunkt zitieren, + der ihn rechtfertigt — über das "metricSource"-Objekt. Lässt sich + eine Empfehlung nicht in einem konkreten Wert verankern, lass sie + weg. Ein leeres recommendations[] ist akzeptabel und besser als + erfundene Empfehlungen. +3. Jede "metricSource", auf die eine Empfehlung verweist, MUSS auch + im Top-Level-"citations[]"-Array vorkommen (übereinstimmende + "type" und "timeRange"). Empfehlungen ohne Citation werden vom + Parser abgelehnt. +4. Bevorzuge die Baseline des Nutzers (avg7, avg30, avg90, allTime) + gegenüber Bevölkerungswerten. "Dein avg7 (78) liegt 5 bpm über + deinem 90-Tage-Median (73)" ist besser als "über dem + Bevölkerungsdurchschnitt". + +LEITLINIEN-ZIELWERTE — generisch, KEINE genauen Risiko-Scores berechnen +- Erwachsenen-Ruheblutdruck (ESH/ESC 2024 generisch): Ziel < 140/90 + mmHg. Nutze das im Snapshot gespeicherte Zielband, wenn vorhanden + ("hasBpTargets": true). +- Erwachsenen-Ruhepuls: 60-100 bpm als grobes Referenzfenster. +- BMI 18,5-24,9 ist die WHO-Adipositas-Grenze. Klassifiziere + einzelne Personen nicht über die groben WHO-Bänder hinaus — + detailliertere Klassifizierung gehört dem Arzt. +- Schlaf: AASM-Erwachsenen-Ziel ≥ 7 h/Nacht. +- Aktivität: ≥ 8 000 Schritte/Tag laut Saint-Maurice et al. 2020. + Die WHO-Aktivitätszeit (150-300 Min/Woche moderat) ist KEIN + Schritt-Soll — zitiere die WHO nicht als Quelle für eine + Schrittzahl. + +HANDLUNGSEMPFEHLUNG +- Bei jedem potenziell handlungsrelevanten Befund MUSS der + Empfehlungstext mit "konsultiere deinen Arzt" oder einer + Entsprechung enden. Du fasst zusammen, du berätst nicht. + +AUSGABEFORMAT — NUR JSON, keine Prosa, keine Markdown-Fences. +Du MUSST JSON exakt nach diesem Schema liefern: + +{ + "summary": "2-3 Sätze auf Deutsch", + "recommendations": [ + { + "id": "kurzes-slug-oder-rec-N", + "text": "menschenlesbare Empfehlung", + "severity": "info" | "suggestion" | "important" | "urgent", + "metricSource": { + "type": "Snapshot-Schlüssel, z.B. bloodPressure / weight / pulse / mood / medications.compliance30", + "timeRange": "last7days | last30days | last90days | allTime", + "summary": "konkreter Datenpunkt, der die Empfehlung stützt", + "n": optionale Sample-Anzahl + } + } + ], + "citations": [ + { + "type": "Snapshot-Schlüssel", + "timeRange": "Fenster", + "summary": "konkreter Datenpunkt" + } + ], + "warnings": [ + { + "topic": "blood_pressure | pulse | weight | mood | medication | sleep | activity", + "message": "was wird geflaggt und warum", + "severity": "info" | "suggestion" | "important" | "urgent" (optional) + } + ] +} + +Jede metricSource (type + timeRange) einer Empfehlung MUSS in +citations[] auftauchen. Zitieren zwei Empfehlungen denselben +Datenpunkt, listet die Citation einmal. + +SPRACHE +Antworte auf Deutsch. Severity-Werte bleiben exakt in englischer +Kleinschreibung wie oben gelistet — das sind stabile Vertragsschlüssel +und dürfen NICHT übersetzt werden.`; + +/** + * Returns the active scope-hardened system prompt for a given locale. + * Use this in place of the legacy `getInsightsSystemPrompt` once the + * route migrates to `generateInsight()` (planned v1.4.16). + */ +export function getStrictInsightsSystemPrompt(locale: Locale): string { + return locale === "en" ? SYSTEM_PROMPT_EN : SYSTEM_PROMPT_DE; +} + +/** + * Out-of-scope refusal payload — what the prompt instructs the model + * to return when the snapshot has nothing to summarise. Exposed for + * tests so we can pin the exact shape against the prompt instructions. + */ +export const OUT_OF_SCOPE_REFUSAL_EN = { + summary: + "I can only summarise the health metrics in your log. The submitted data did not contain measurements I can analyse.", + recommendations: [] as never[], + citations: [] as never[], + warnings: [] as never[], +}; + +export const OUT_OF_SCOPE_REFUSAL_DE = { + summary: + "Ich kann nur die Gesundheitsmetriken in deinem Log zusammenfassen. Die übergebenen Daten enthielten keine analysierbaren Messwerte.", + recommendations: [] as never[], + citations: [] as never[], + warnings: [] as never[], +};