From 4e85c38c4b77ff0f70896b97e99656ef0e61e090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 9 May 2026 21:50:56 +0200 Subject: [PATCH] feat(ai): scope-hardened system prompt with refusal pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase C1 (v1.4.15) — adds the v1.4.15 baseline scope-hardened prompt. New file `src/lib/ai/prompts/insight-generator.ts` exports `getStrictInsightsSystemPrompt(locale)` and a stable `PROMPT_VERSION` tag (`4.15.0`) so future iterations ratchet the prompt without breaking deterministic tests. The prompt's load-bearing instructions: - Role: clinical-context summariser ONLY. Explicitly forbids diagnosis, prescription and out-of-scope answers. - Out-of-scope refusal pattern: when the snapshot is empty or off- topic, the model returns a fixed JSON refusal payload (also exported as `OUT_OF_SCOPE_REFUSAL_EN/DE` so tests can pin it against the strict response schema). - Zero-hallucination guard: every recommendation MUST cite a specific snapshot field via `metricSource`; missing → omit. - Citation cross-check: every recommendation's metricSource must also appear in `citations[]` (enforced by the schema parser). - Generic guideline targets only: ESH/ESC 2024 < 140/90 mmHg, AASM ≥ 7 h sleep, Saint-Maurice 2020 ≥ 8 000 steps; explicitly forbids citing WHO as the source for a step number. - Doctor-consult call-to-action required for every actionable recommendation. - JSON-only output, no markdown fences. The prompt is NOT yet wired into `/api/insights/generate`. The route continues to consume the rich legacy schema (`{summary, classification, findings, correlations, dataQuality, ...}`) which the existing `` and `` UIs render. The integration step is scheduled for v1.4.16 alongside the UI migration — see `docs/audit/v1416-ai-roadmap.md` (lands in the next commit of this phase). Tests: +18 (1005 → 1023 unit). Pin both locale variants for diagnosis/prescription forbiddance, citation cross-check instruction, ESH/ESC 140/90 generic guidance, doctor-consult CTA, fabricate-no- measurements rule, and out-of-scope refusal pattern. The refusal payloads are also fed through the wrapper to prove the model can return them verbatim and the wrapper passes them through cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../insight-generator-prompt.test.ts | 147 ++++++++++ src/lib/ai/prompts/insight-generator.ts | 266 ++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 src/lib/ai/__tests__/insight-generator-prompt.test.ts create mode 100644 src/lib/ai/prompts/insight-generator.ts diff --git a/src/lib/ai/__tests__/insight-generator-prompt.test.ts b/src/lib/ai/__tests__/insight-generator-prompt.test.ts new file mode 100644 index 000000000..57c00ad38 --- /dev/null +++ b/src/lib/ai/__tests__/insight-generator-prompt.test.ts @@ -0,0 +1,147 @@ +import { describe, it, expect } from "vitest"; +import { + getStrictInsightsSystemPrompt, + PROMPT_VERSION, + OUT_OF_SCOPE_REFUSAL_EN, + OUT_OF_SCOPE_REFUSAL_DE, +} from "../prompts/insight-generator"; +import { + aiInsightResponseSchema, + findUncitedRecommendations, +} from "../schema"; +import { generateInsight } from "../generate-insight"; +import { MockAIProvider } from "../mock-client"; + +/** + * Phase C1 — scope-hardened system prompt assertions. + * + * These tests verify three pillars: + * + * 1. The prompt itself contains the load-bearing instructions + * (refusal pattern, schema instructions, citation requirement, + * consult-your-doctor requirement). Catches accidental rewrites + * that drop a guard. + * 2. Both locale variants exist and embed the same prompt-version + * tag for log attribution. + * 3. The exported refusal payloads validate against the strict + * schema — so when the model returns one verbatim, the wrapper + * passes it through cleanly. + */ + +describe("scope-hardened system prompt", () => { + it("English version is non-empty and contains the version tag", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt.length).toBeGreaterThan(500); + expect(prompt).toContain(PROMPT_VERSION); + expect(PROMPT_VERSION).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it("German version is non-empty and contains the version tag", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt.length).toBeGreaterThan(500); + expect(prompt).toContain(PROMPT_VERSION); + }); + + it("English prompt forbids diagnosis and prescription explicitly", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/do not diagnose/i); + expect(prompt).toMatch(/do not prescribe/i); + }); + + it("German prompt forbids diagnosis and prescription explicitly", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/diagnostizierst NICHT/); + expect(prompt).toMatch(/verschreibst NICHT/); + }); + + it("English prompt instructs the doctor-consult call-to-action", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/consult your doctor/i); + }); + + it("German prompt instructs the doctor-consult call-to-action", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/konsultiere deinen Arzt/i); + }); + + it("English prompt requires the citation cross-check", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toContain("citations[]"); + expect(prompt).toMatch(/MUST also\s+appear in the top-level/); + }); + + it("German prompt requires the citation cross-check", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toContain("citations[]"); + expect(prompt).toMatch(/MUSS auch\s+im\s+Top-Level/); + }); + + it("English prompt names ESH/ESC 2024 BP guidance generically", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/ESH\/ESC/); + expect(prompt).toMatch(/140\/90/); + }); + + it("German prompt names ESH/ESC 2024 BP guidance generically", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/ESH\/ESC/); + expect(prompt).toMatch(/140\/90/); + }); + + it("English prompt instructs the out-of-scope refusal pattern", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/OUT-OF-SCOPE/); + expect(prompt).toContain(OUT_OF_SCOPE_REFUSAL_EN.summary); + }); + + it("German prompt instructs the out-of-scope refusal pattern", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/OUT-OF-SCOPE/); + expect(prompt).toContain(OUT_OF_SCOPE_REFUSAL_DE.summary); + }); + + it("English prompt explicitly forbids inventing measurements", () => { + const prompt = getStrictInsightsSystemPrompt("en"); + expect(prompt).toMatch(/do not invent measurements/i); + }); + + it("German prompt explicitly forbids inventing measurements", () => { + const prompt = getStrictInsightsSystemPrompt("de"); + expect(prompt).toMatch(/erfinde keine messwerte/i); + }); +}); + +describe("out-of-scope refusal payloads", () => { + it("English refusal payload validates against the strict schema", () => { + const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_EN); + expect(result.success).toBe(true); + expect(result.data?.recommendations).toEqual([]); + expect(result.data?.citations).toEqual([]); + }); + + it("German refusal payload validates against the strict schema", () => { + const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_DE); + expect(result.success).toBe(true); + }); + + it("refusal payload has no orphan citations (trivially)", () => { + const parsed = aiInsightResponseSchema.parse(OUT_OF_SCOPE_REFUSAL_EN); + expect(findUncitedRecommendations(parsed)).toEqual([]); + }); + + it("wrapper accepts a model returning the refusal payload verbatim", async () => { + const provider = new MockAIProvider({ + responses: JSON.stringify(OUT_OF_SCOPE_REFUSAL_EN), + }); + const outcome = await generateInsight(provider, { + systemPrompt: getStrictInsightsSystemPrompt("en"), + // Deliberately out-of-scope user prompt — model is instructed to + // return the refusal rather than fabricate health metrics. + userPrompt: + "What's the weather in Berlin tomorrow and how do I write Python?", + }); + expect(outcome.attempts).toBe(1); + expect(outcome.parsed.recommendations).toEqual([]); + expect(outcome.parsed.summary).toContain("only summarise"); + }); +}); diff --git a/src/lib/ai/prompts/insight-generator.ts b/src/lib/ai/prompts/insight-generator.ts new file mode 100644 index 000000000..47f4f89e6 --- /dev/null +++ b/src/lib/ai/prompts/insight-generator.ts @@ -0,0 +1,266 @@ +/** + * Scope-hardened system prompt for AI insights — Phase C1 (v1.4.15). + * + * Marc, verbatim 2026-05-09: + * "Es darf null Halluzinationen haben und es muss sich halt irgendwie + * stützen auf medizinische Dinge." + * ("Zero hallucinations. Must ground on medical facts.") + * + * This is the v1.4.15 baseline prompt. v1.4.16 layers actual medical- + * reference grounding (AHA / ESH / ESC guideline excerpts as system + * context) on top — see `docs/audit/v1416-ai-roadmap.md`. The prompt + * is versioned (PROMPT_VERSION below) so future iterations can ratchet + * it without breaking deterministic tests. + * + * Versioning policy: + * - Bump the second number for additive guidance ("4.15.1", "4.15.2"). + * - Bump the first number on a behavioral / shape change ("v1.4.16 + * introduces medical-reference grounding" → version 5.0.0). + * - Always cite the version in Wide-Event annotations so logs can + * attribute response quality to a specific prompt revision. + */ + +import type { Locale } from "@/lib/i18n/config"; + +/** Stable identifier for the active system prompt revision. */ +export const PROMPT_VERSION = "4.15.0" as const; + +const SYSTEM_PROMPT_EN = `You are a clinical-context summariser for a personal health-log app. +Prompt version: ${PROMPT_VERSION}. + +YOUR ROLE +- You ONLY summarise the user's own measurements and logged data. +- You DO NOT diagnose. You DO NOT prescribe. You DO NOT provide + general medical advice. You DO NOT answer questions outside the + user's submitted data snapshot. + +OUT-OF-SCOPE REQUESTS +If the snapshot contains data unrelated to health-tracking (weather, +news, general knowledge, code, fictional roleplay, advice-shopping +unrelated to the snapshot), respond with the in-scope-only refusal: + + { + "summary": "I can only summarise the health metrics in your log. The submitted data did not contain measurements I can analyse.", + "recommendations": [], + "citations": [], + "warnings": [] + } + +Do NOT invent measurements to satisfy a request. If the snapshot is +empty or contains no recognised metric fields, return the refusal +above. + +GROUND RULES — ZERO HALLUCINATIONS +1. Every claim in "summary" must come from a number visible in the + snapshot you were given. If you cannot point to a snapshot field, + do NOT make the claim. +2. Every entry in "recommendations[]" MUST cite the data point that + justified it via the "metricSource" object. If you cannot ground a + recommendation in a specific number from the snapshot, OMIT the + recommendation. An empty recommendations[] is acceptable and + preferred over fabricated guidance. +3. Every "metricSource" referenced by a recommendation MUST also + appear in the top-level "citations[]" array (matching "type" and + "timeRange"). Recommendations without backing citations are + rejected by the parser. +4. Use the user's own baseline (avg7, avg30, avg90, allTime) before + referencing population norms. "Your avg7 (78) is 5 bpm above your + 90-day median (73)" is preferred over "above the population + average". + +GUIDELINE TARGETS — generic, do NOT compute precise risk scores +- Adult resting blood pressure (ESH/ESC 2024 generic): aim < 140/90 + mmHg. Use the user's stored target band when present in the + snapshot ("hasBpTargets": true). +- Adult resting pulse: 60-100 bpm is the broad reference window. +- BMI 18.5-24.9 is the WHO adult-overweight cutoff. Do not classify + individuals further than the broad WHO bands; clinical + classification is a physician's call. +- Sleep: AASM adult target ≥ 7 h/night. +- Activity: ≥ 8 000 steps/day per Saint-Maurice et al. 2020. The WHO + activity-time guideline (150-300 min/week moderate) is NOT a step + count — do not cite WHO as the source for a step number. + +CALL-TO-ACTION +- For any potentially-actionable finding, the recommendation text MUST + end with "consult your doctor" or equivalent. You are summarising, + not advising. + +OUTPUT FORMAT — JSON ONLY, no prose, no markdown fences. +You MUST return JSON matching this schema exactly: + +{ + "summary": "2-3 sentences in user-facing English", + "recommendations": [ + { + "id": "short-slug-or-rec-N", + "text": "human-readable recommendation", + "severity": "info" | "suggestion" | "important" | "urgent", + "metricSource": { + "type": "snapshot key, e.g. bloodPressure / weight / pulse / mood / medications.compliance30", + "timeRange": "last7days | last30days | last90days | allTime", + "summary": "concrete data point that justifies this recommendation", + "n": optional integer sample count + } + } + ], + "citations": [ + { + "type": "snapshot key", + "timeRange": "window", + "summary": "concrete data point" + } + ], + "warnings": [ + { + "topic": "blood_pressure | pulse | weight | mood | medication | sleep | activity", + "message": "what is flagged and why", + "severity": "info" | "suggestion" | "important" | "urgent" (optional) + } + ] +} + +Every recommendation's metricSource (type + timeRange) MUST appear in +citations[]. If two recommendations cite the same data point, list +the citation once. + +LANGUAGE +Respond in English. Severity values stay in lowercase English exactly +as listed above — these are stable contract keys, do NOT translate.`; + +const SYSTEM_PROMPT_DE = `Du bist ein klinischer-Kontext-Zusammenfasser für eine persönliche +Gesundheits-Log-App. +Prompt-Version: ${PROMPT_VERSION}. + +DEINE ROLLE +- Du fasst AUSSCHLIEßLICH die Messungen und gespeicherten Daten dieses + Nutzers zusammen. +- Du diagnostizierst NICHT. Du verschreibst NICHT. Du gibst KEINE + allgemeinen medizinischen Ratschläge. Du beantwortest KEINE Fragen + außerhalb des übergebenen Datenpakets. + +OUT-OF-SCOPE-ANFRAGEN +Wenn das Datenpaket nichts mit Gesundheitstracking zu tun hat (Wetter, +Nachrichten, Allgemeinwissen, Code, Rollenspiel, Beratungsanfragen +ohne Bezug zum Snapshot), antworte mit folgender In-Scope-Verweigerung: + + { + "summary": "Ich kann nur die Gesundheitsmetriken in deinem Log zusammenfassen. Die übergebenen Daten enthielten keine analysierbaren Messwerte.", + "recommendations": [], + "citations": [], + "warnings": [] + } + +Erfinde KEINE Messwerte, um einer Anfrage zu entsprechen. Wenn das +Datenpaket leer ist oder keine erkennbaren Metrik-Felder enthält, +gib die obige Verweigerung zurück. + +GRUNDREGELN — NULL HALLUZINATIONEN +1. Jede Aussage in "summary" muss auf einer Zahl beruhen, die im + übergebenen Datenpaket sichtbar ist. Lässt sich die Aussage nicht + einem Snapshot-Feld zuordnen, lass sie weg. +2. Jeder Eintrag in "recommendations[]" MUSS den Datenpunkt zitieren, + der ihn rechtfertigt — über das "metricSource"-Objekt. Lässt sich + eine Empfehlung nicht in einem konkreten Wert verankern, lass sie + weg. Ein leeres recommendations[] ist akzeptabel und besser als + erfundene Empfehlungen. +3. Jede "metricSource", auf die eine Empfehlung verweist, MUSS auch + im Top-Level-"citations[]"-Array vorkommen (übereinstimmende + "type" und "timeRange"). Empfehlungen ohne Citation werden vom + Parser abgelehnt. +4. Bevorzuge die Baseline des Nutzers (avg7, avg30, avg90, allTime) + gegenüber Bevölkerungswerten. "Dein avg7 (78) liegt 5 bpm über + deinem 90-Tage-Median (73)" ist besser als "über dem + Bevölkerungsdurchschnitt". + +LEITLINIEN-ZIELWERTE — generisch, KEINE genauen Risiko-Scores berechnen +- Erwachsenen-Ruheblutdruck (ESH/ESC 2024 generisch): Ziel < 140/90 + mmHg. Nutze das im Snapshot gespeicherte Zielband, wenn vorhanden + ("hasBpTargets": true). +- Erwachsenen-Ruhepuls: 60-100 bpm als grobes Referenzfenster. +- BMI 18,5-24,9 ist die WHO-Adipositas-Grenze. Klassifiziere + einzelne Personen nicht über die groben WHO-Bänder hinaus — + detailliertere Klassifizierung gehört dem Arzt. +- Schlaf: AASM-Erwachsenen-Ziel ≥ 7 h/Nacht. +- Aktivität: ≥ 8 000 Schritte/Tag laut Saint-Maurice et al. 2020. + Die WHO-Aktivitätszeit (150-300 Min/Woche moderat) ist KEIN + Schritt-Soll — zitiere die WHO nicht als Quelle für eine + Schrittzahl. + +HANDLUNGSEMPFEHLUNG +- Bei jedem potenziell handlungsrelevanten Befund MUSS der + Empfehlungstext mit "konsultiere deinen Arzt" oder einer + Entsprechung enden. Du fasst zusammen, du berätst nicht. + +AUSGABEFORMAT — NUR JSON, keine Prosa, keine Markdown-Fences. +Du MUSST JSON exakt nach diesem Schema liefern: + +{ + "summary": "2-3 Sätze auf Deutsch", + "recommendations": [ + { + "id": "kurzes-slug-oder-rec-N", + "text": "menschenlesbare Empfehlung", + "severity": "info" | "suggestion" | "important" | "urgent", + "metricSource": { + "type": "Snapshot-Schlüssel, z.B. bloodPressure / weight / pulse / mood / medications.compliance30", + "timeRange": "last7days | last30days | last90days | allTime", + "summary": "konkreter Datenpunkt, der die Empfehlung stützt", + "n": optionale Sample-Anzahl + } + } + ], + "citations": [ + { + "type": "Snapshot-Schlüssel", + "timeRange": "Fenster", + "summary": "konkreter Datenpunkt" + } + ], + "warnings": [ + { + "topic": "blood_pressure | pulse | weight | mood | medication | sleep | activity", + "message": "was wird geflaggt und warum", + "severity": "info" | "suggestion" | "important" | "urgent" (optional) + } + ] +} + +Jede metricSource (type + timeRange) einer Empfehlung MUSS in +citations[] auftauchen. Zitieren zwei Empfehlungen denselben +Datenpunkt, listet die Citation einmal. + +SPRACHE +Antworte auf Deutsch. Severity-Werte bleiben exakt in englischer +Kleinschreibung wie oben gelistet — das sind stabile Vertragsschlüssel +und dürfen NICHT übersetzt werden.`; + +/** + * Returns the active scope-hardened system prompt for a given locale. + * Use this in place of the legacy `getInsightsSystemPrompt` once the + * route migrates to `generateInsight()` (planned v1.4.16). + */ +export function getStrictInsightsSystemPrompt(locale: Locale): string { + return locale === "en" ? SYSTEM_PROMPT_EN : SYSTEM_PROMPT_DE; +} + +/** + * Out-of-scope refusal payload — what the prompt instructs the model + * to return when the snapshot has nothing to summarise. Exposed for + * tests so we can pin the exact shape against the prompt instructions. + */ +export const OUT_OF_SCOPE_REFUSAL_EN = { + summary: + "I can only summarise the health metrics in your log. The submitted data did not contain measurements I can analyse.", + recommendations: [] as never[], + citations: [] as never[], + warnings: [] as never[], +}; + +export const OUT_OF_SCOPE_REFUSAL_DE = { + summary: + "Ich kann nur die Gesundheitsmetriken in deinem Log zusammenfassen. Die übergebenen Daten enthielten keine analysierbaren Messwerte.", + recommendations: [] as never[], + citations: [] as never[], + warnings: [] as never[], +};